Merge master.kernel.org:/pub/scm/linux/kernel/git/davem/sparc-2.6

13 files changed, 285 insertions, 604 deletions

diff --git a/arch/sparc/kernel/sparc_ksyms.c b/arch/sparc/kernel/sparc_ksyms.c
index 8faa8dc4de43..5d974a2b735a 100644
--- a/arch/sparc/kernel/sparc_ksyms.c
+++ b/arch/sparc/kernel/sparc_ksyms.c
@@ -175,7 +175,6 @@ EXPORT_SYMBOL(set_auxio);
 EXPORT_SYMBOL(get_auxio);
 #endif
 EXPORT_SYMBOL(request_fast_irq);
-EXPORT_SYMBOL(io_remap_page_range);
 EXPORT_SYMBOL(io_remap_pfn_range);
   /* P3: iounit_xxx may be needed, sun4d users */
 /* EXPORT_SYMBOL(iounit_map_dma_init); */
diff --git a/arch/sparc/mm/generic.c b/arch/sparc/mm/generic.c
index db27eee3bda1..20ccb957fb77 100644
--- a/arch/sparc/mm/generic.c
+++ b/arch/sparc/mm/generic.c
@@ -16,31 +16,6 @@
 #include <asm/cacheflush.h>
 #include <asm/tlbflush.h>
 
-static inline void forget_pte(pte_t page)
-{
-#if 0 /* old 2.4 code */
-	if (pte_none(page))
-		return;
-	if (pte_present(page)) {
-		unsigned long pfn = pte_pfn(page);
-		struct page *ptpage;
-		if (!pfn_valid(pfn))
-			return;
-		ptpage = pfn_to_page(pfn);
-		if (PageReserved(ptpage))
-			return;
-		page_cache_release(ptpage);
-		return;
-	}
-	swap_free(pte_to_swp_entry(page));
-#else
-	if (!pte_none(page)) {
-		printk("forget_pte: old mapping existed!\n");
-		BUG();
-	}
-#endif
-}
-
 /* Remap IO memory, the same way as remap_pfn_range(), but use
  * the obio memory space.
  *
@@ -60,7 +35,6 @@ static inline void io_remap_pte_range(struct mm_struct *mm, pte_t * pte, unsigne
 		pte_t oldpage = *pte;
 		pte_clear(mm, address, pte);
 		set_pte(pte, mk_pte_io(offset, prot, space));
-		forget_pte(oldpage);
 		address += PAGE_SIZE;
 		offset += PAGE_SIZE;
 		pte++;
@@ -88,37 +62,6 @@ static inline int io_remap_pmd_range(struct mm_struct *mm, pmd_t * pmd, unsigned
 	return 0;
 }
 
-int io_remap_page_range(struct vm_area_struct *vma, unsigned long from, unsigned long offset, unsigned long size, pgprot_t prot, int space)
-{
-	int error = 0;
-	pgd_t * dir;
-	unsigned long beg = from;
-	unsigned long end = from + size;
-	struct mm_struct *mm = vma->vm_mm;
-
-	prot = __pgprot(pg_iobits);
-	offset -= from;
-	dir = pgd_offset(mm, from);
-	flush_cache_range(vma, beg, end);
-
-	spin_lock(&mm->page_table_lock);
-	while (from < end) {
-		pmd_t *pmd = pmd_alloc(current->mm, dir, from);
-		error = -ENOMEM;
-		if (!pmd)
-			break;
-		error = io_remap_pmd_range(mm, pmd, from, end - from, offset + from, prot, space);
-		if (error)
-			break;
-		from = (from + PGDIR_SIZE) & PGDIR_MASK;
-		dir++;
-	}
-	spin_unlock(&mm->page_table_lock);
-
-	flush_tlb_range(vma, beg, end);
-	return error;
-}
-
 int io_remap_pfn_range(struct vm_area_struct *vma, unsigned long from,
 			unsigned long pfn, unsigned long size, pgprot_t prot)
 {
diff --git a/arch/sparc64/Kconfig b/arch/sparc64/Kconfig
index 9afd28e2c4d5..17846f4ba9b6 100644
--- a/arch/sparc64/Kconfig
+++ b/arch/sparc64/Kconfig
@@ -5,6 +5,16 @@
 
 mainmenu "Linux/UltraSPARC Kernel Configuration"
 
+config SPARC64
+	bool
+	default y
+	help
+	  SPARC is a family of RISC microprocessors designed and marketed by
+	  Sun Microsystems, incorporated.  This port covers the newer 64-bit
+	  UltraSPARC.  The UltraLinux project maintains both the SPARC32 and
+	  SPARC64 ports; its web page is available at
+	  <http://www.ultralinux.org/>.
+
 config 64BIT
 	def_bool y
 
@@ -71,75 +81,6 @@ config SYSVIPC_COMPAT
 
 menu "General machine setup"
 
-config BBC_I2C
-	tristate "UltraSPARC-III bootbus i2c controller driver"
-	depends on PCI
-	help
-	  The BBC devices on the UltraSPARC III have two I2C controllers.  The
-	  first I2C controller connects mainly to configuration PROMs (NVRAM,
-	  CPU configuration, DIMM types, etc.).  The second I2C controller
-	  connects to environmental control devices such as fans and
-	  temperature sensors.  The second controller also connects to the
-	  smartcard reader, if present.  Say Y to enable support for these.
-
-config VT
-	bool "Virtual terminal" if EMBEDDED
-	select INPUT
-	default y
-	---help---
-	  If you say Y here, you will get support for terminal devices with
-	  display and keyboard devices. These are called "virtual" because you
-	  can run several virtual terminals (also called virtual consoles) on
-	  one physical terminal. This is rather useful, for example one
-	  virtual terminal can collect system messages and warnings, another
-	  one can be used for a text-mode user session, and a third could run
-	  an X session, all in parallel. Switching between virtual terminals
-	  is done with certain key combinations, usually Alt-<function key>.
-
-	  The setterm command ("man setterm") can be used to change the
-	  properties (such as colors or beeping) of a virtual terminal. The
-	  man page console_codes(4) ("man console_codes") contains the special
-	  character sequences that can be used to change those properties
-	  directly. The fonts used on virtual terminals can be changed with
-	  the setfont ("man setfont") command and the key bindings are defined
-	  with the loadkeys ("man loadkeys") command.
-
-	  You need at least one virtual terminal device in order to make use
-	  of your keyboard and monitor. Therefore, only people configuring an
-	  embedded system would want to say N here in order to save some
-	  memory; the only way to log into such a system is then via a serial
-	  or network connection.
-
-	  If unsure, say Y, or else you won't be able to do much with your new
-	  shiny Linux system :-)
-
-config VT_CONSOLE
-	bool "Support for console on virtual terminal" if EMBEDDED
-	depends on VT
-	default y
-	---help---
-	  The system console is the device which receives all kernel messages
-	  and warnings and which allows logins in single user mode. If you
-	  answer Y here, a virtual terminal (the device used to interact with
-	  a physical terminal) can be used as system console. This is the most
-	  common mode of operations, so you should say Y here unless you want
-	  the kernel messages be output only to a serial port (in which case
-	  you should say Y to "Console on serial port", below).
-
-	  If you do say Y here, by default the currently visible virtual
-	  terminal (/dev/tty0) will be used as system console. You can change
-	  that with a kernel command line option such as "console=tty3" which
-	  would use the third virtual terminal as system console. (Try "man
-	  bootparam" or see the documentation of your boot loader (lilo or
-	  loadlin) about how to pass options to the kernel at boot time.)
-
-	  If unsure, say Y.
-
-config HW_CONSOLE
-	bool
-	depends on VT
-	default y
-
 config SMP
 	bool "Symmetric multi-processing support"
 	---help---
@@ -205,17 +146,6 @@ config US2E_FREQ
 
 	  If in doubt, say N.
 
-# Identify this as a Sparc64 build
-config SPARC64
-	bool
-	default y
-	help
-	  SPARC is a family of RISC microprocessors designed and marketed by
-	  Sun Microsystems, incorporated.  This port covers the newer 64-bit
-	  UltraSPARC.  The UltraLinux project maintains both the SPARC32 and
-	  SPARC64 ports; its web page is available at
-	  <http://www.ultralinux.org/>.
-
 # Global things across all Sun machines.
 config RWSEM_GENERIC_SPINLOCK
 	bool
@@ -246,6 +176,12 @@ config HUGETLB_PAGE_SIZE_64K
 
 endchoice
 
+endmenu
+
+source "drivers/firmware/Kconfig"
+
+source "mm/Kconfig"
+
 config GENERIC_ISA_DMA
 	bool
 	default y
@@ -344,33 +280,6 @@ config PCI_DOMAINS
 	bool
 	default PCI
 
-config RTC
-	tristate
-	depends on PCI
-	default y
-	---help---
-	  If you say Y here and create a character special file /dev/rtc with
-	  major number 10 and minor number 135 using mknod ("man mknod"), you
-	  will get access to the real time clock (or hardware clock) built
-	  into your computer.
-
-	  Every PC has such a clock built in. It can be used to generate
-	  signals from as low as 1Hz up to 8192Hz, and can also be used
-	  as a 24 hour alarm. It reports status information via the file
-	  /proc/driver/rtc and its behaviour is set by various ioctls on
-	  /dev/rtc.
-
-	  If you run Linux on a multiprocessor machine and said Y to
-	  "Symmetric Multi Processing" above, you should say Y here to read
-	  and set the RTC in an SMP compatible fashion.
-
-	  If you think you have a use for such a device (such as periodic data
-	  sampling), then say Y here, and read <file:Documentation/rtc.txt>
-	  for details.
-
-	  To compile this driver as a module, choose M here: the
-	  module will be called rtc.
-
 source "drivers/pci/Kconfig"
 
 config SUN_OPENPROMFS
@@ -414,6 +323,8 @@ config BINFMT_AOUT32
 	  If you want to run SunOS binaries (see SunOS binary emulation below)
 	  or other a.out binaries, say Y. If unsure, say N.
 
+menu "Executable file formats"
+
 source "fs/Kconfig.binfmt"
 
 config SUNOS_EMUL
@@ -436,74 +347,7 @@ config SOLARIS_EMUL
 	  To compile this code as a module, choose M here: the
 	  module will be called solaris.
 
-source "drivers/parport/Kconfig"
-
-config PRINTER
-	tristate "Parallel printer support"
-	depends on PARPORT
-	---help---
-	  If you intend to attach a printer to the parallel port of your Linux
-	  box (as opposed to using a serial printer; if the connector at the
-	  printer has 9 or 25 holes ["female"], then it's serial), say Y.
-	  Also read the Printing-HOWTO, available from
-	  <http://www.tldp.org/docs.html#howto>.
-
-	  It is possible to share one parallel port among several devices
-	  (e.g. printer and ZIP drive) and it is safe to compile the
-	  corresponding drivers into the kernel.
-	  To compile this driver as a module, choose M here and read
-	  <file:Documentation/parport.txt>.  The module will be called lp.
-
-	  If you have several parallel ports, you can specify which ports to
-	  use with the "lp" kernel command line option.  (Try "man bootparam"
-	  or see the documentation of your boot loader (lilo or loadlin) about
-	  how to pass options to the kernel at boot time.)  The syntax of the
-	  "lp" command line option can be found in <file:drivers/char/lp.c>.
-
-	  If you have more than 8 printers, you need to increase the LP_NO
-	  macro in lp.c and the PARPORT_MAX macro in parport.h.
-
-config PPDEV
-	tristate "Support for user-space parallel port device drivers"
-	depends on PARPORT
-	---help---
-	  Saying Y to this adds support for /dev/parport device nodes.  This
-	  is needed for programs that want portable access to the parallel
-	  port, for instance deviceid (which displays Plug-and-Play device
-	  IDs).
-
-	  This is the parallel port equivalent of SCSI generic support (sg).
-	  It is safe to say N to this -- it is not needed for normal printing
-	  or parallel port CD-ROM/disk support.
-
-	  To compile this driver as a module, choose M here: the
-	  module will be called ppdev.
-
-	  If unsure, say N.
-
-config ENVCTRL
-	tristate "SUNW, envctrl support"
-	depends on PCI
-	help
-	  Kernel support for temperature and fan monitoring on Sun SME
-	  machines.
-
-	  To compile this driver as a module, choose M here: the
-	  module will be called envctrl.
-
-config DISPLAY7SEG
-	tristate "7-Segment Display support"
-	depends on PCI
-	---help---
-	  This is the driver for the 7-segment display and LED present on
-	  Sun Microsystems CompactPCI models CP1400 and CP1500.
-
-	  To compile this driver as a module, choose M here: the
-	  module will be called display7seg.
-
-	  If you do not have a CompactPCI model CP1400 or CP1500, or
-	  another UltraSPARC-IIi-cEngine boardset with a 7-segment display,
-	  you should say N to this option.
+endmenu
 
 config CMDLINE_BOOL
 	bool "Default bootloader kernel arguments"
@@ -521,148 +365,16 @@ config CMDLINE
 
 	  NOTE: This option WILL override the PROM bootargs setting!
 
-source "mm/Kconfig"
-
-endmenu
-
 source "net/Kconfig"
 
-source "drivers/base/Kconfig"
-
-source "drivers/video/Kconfig"
-
-source "drivers/serial/Kconfig"
+source "drivers/Kconfig"
 
 source "drivers/sbus/char/Kconfig"
 
-source "drivers/mtd/Kconfig"
-
-source "drivers/block/Kconfig"
-
-source "drivers/ide/Kconfig"
-
-source "drivers/scsi/Kconfig"
-
 source "drivers/fc4/Kconfig"
 
-source "drivers/md/Kconfig"
-
-if PCI
-source "drivers/message/fusion/Kconfig"
-endif
-
-source "drivers/ieee1394/Kconfig"
-
-source "drivers/net/Kconfig"
-
-source "drivers/isdn/Kconfig"
-
-source "drivers/telephony/Kconfig"
-
-# This one must be before the filesystem configs. -DaveM
-
-menu "Unix98 PTY support"
-
-config UNIX98_PTYS
-	bool "Unix98 PTY support"
-	---help---
-	  A pseudo terminal (PTY) is a software device consisting of two
-	  halves: a master and a slave. The slave device behaves identical to
-	  a physical terminal; the master device is used by a process to
-	  read data from and write data to the slave, thereby emulating a
-	  terminal. Typical programs for the master side are telnet servers
-	  and xterms.
-
-	  Linux has traditionally used the BSD-like names /dev/ptyxx for
-	  masters and /dev/ttyxx for slaves of pseudo terminals. This scheme
-	  has a number of problems. The GNU C library glibc 2.1 and later,
-	  however, supports the Unix98 naming standard: in order to acquire a
-	  pseudo terminal, a process opens /dev/ptmx; the number of the pseudo
-	  terminal is then made available to the process and the pseudo
-	  terminal slave can be accessed as /dev/pts/<number>. What was
-	  traditionally /dev/ttyp2 will then be /dev/pts/2, for example.
-
-	  The entries in /dev/pts/ are created on the fly by a virtual
-	  file system; therefore, if you say Y here you should say Y to
-	  "/dev/pts file system for Unix98 PTYs" as well.
-
-	  If you want to say Y here, you need to have the C library glibc 2.1
-	  or later (equal to libc-6.1, check with "ls -l /lib/libc.so.*").
-	  Read the instructions in <file:Documentation/Changes> pertaining to
-	  pseudo terminals. It's safe to say N.
-
-config UNIX98_PTY_COUNT
-	int "Maximum number of Unix98 PTYs in use (0-2048)"
-	depends on UNIX98_PTYS
-	default "256"
-	help
-	  The maximum number of Unix98 PTYs that can be used at any one time.
-	  The default is 256, and should be enough for desktop systems. Server
-	  machines which support incoming telnet/rlogin/ssh connections and/or
-	  serve several X terminals may want to increase this: every incoming
-	  connection and every xterm uses up one PTY.
-
-	  When not in use, each additional set of 256 PTYs occupy
-	  approximately 8 KB of kernel memory on 32-bit architectures.
-
-endmenu
-
-menu "XFree86 DRI support"
-
-config DRM
-	bool "Direct Rendering Manager (XFree86 DRI support)"
-	help
-	  Kernel-level support for the Direct Rendering Infrastructure (DRI)
-	  introduced in XFree86 4.0. If you say Y here, you need to select
-	  the module that's right for your graphics card from the list below.
-	  These modules provide support for synchronization, security, and
-	  DMA transfers. Please see <http://dri.sourceforge.net/> for more
-	  details.  You should also select and configure AGP
-	  (/dev/agpgart) support.
-
-config DRM_FFB
-	tristate "Creator/Creator3D"
-	depends on DRM && BROKEN
-	help
-	  Choose this option if you have one of Sun's Creator3D-based graphics
-	  and frame buffer cards.  Product page at
-	  <http://www.sun.com/desktop/products/Graphics/creator3d.html>.
-
-config DRM_TDFX
-	tristate "3dfx Banshee/Voodoo3+"
-	depends on DRM
-	help
-	  Choose this option if you have a 3dfx Banshee or Voodoo3 (or later),
-	  graphics card.  If M is selected, the module will be called tdfx.
-
-config DRM_R128
-	tristate "ATI Rage 128"
-	depends on DRM
-	help
-	  Choose this option if you have an ATI Rage 128 graphics card.  If M
-	  is selected, the module will be called r128.  AGP support for
-	  this card is strongly suggested (unless you have a PCI version).
-
-endmenu
-
-source "drivers/input/Kconfig"
-
-source "drivers/i2c/Kconfig"
-
-source "drivers/hwmon/Kconfig"
-
 source "fs/Kconfig"
 
-source "drivers/media/Kconfig"
-
-source "sound/Kconfig"
-
-source "drivers/usb/Kconfig"
-
-source "drivers/infiniband/Kconfig"
-
-source "drivers/char/watchdog/Kconfig"
-
 source "arch/sparc64/oprofile/Kconfig"
 
 source "arch/sparc64/Kconfig.debug"
diff --git a/arch/sparc64/kernel/entry.S b/arch/sparc64/kernel/entry.S
index cecdc0a7521f..3e0badb820c5 100644
--- a/arch/sparc64/kernel/entry.S
+++ b/arch/sparc64/kernel/entry.S
@@ -927,139 +927,6 @@ __spitfire_insn_access_exception:
 	ba,pt		%xcc, rtrap
 	 clr		%l6
 
-	/* Capture I/D/E-cache state into per-cpu error scoreboard.
-	 *
-	 * %g1:		(TL>=0) ? 1 : 0
-	 * %g2:		scratch
-	 * %g3:		scratch
-	 * %g4:		AFSR
-	 * %g5:		AFAR
-	 * %g6:		current thread ptr
-	 * %g7:		scratch
-	 */
-#define CHEETAH_LOG_ERROR						\
-	/* Put "TL1" software bit into AFSR. */				\
-	and		%g1, 0x1, %g1;					\
-	sllx		%g1, 63, %g2;					\
-	or		%g4, %g2, %g4;					\
-	/* Get log entry pointer for this cpu at this trap level. */	\
-	BRANCH_IF_JALAPENO(g2,g3,50f)					\
-	ldxa		[%g0] ASI_SAFARI_CONFIG, %g2;			\
-	srlx		%g2, 17, %g2;					\
-	ba,pt		%xcc, 60f; 					\
-	 and		%g2, 0x3ff, %g2;				\
-50:	ldxa		[%g0] ASI_JBUS_CONFIG, %g2;			\
-	srlx		%g2, 17, %g2;					\
-	and		%g2, 0x1f, %g2;					\
-60:	sllx		%g2, 9, %g2;					\
-	sethi		%hi(cheetah_error_log), %g3;			\
-	ldx		[%g3 + %lo(cheetah_error_log)], %g3;		\
-	brz,pn		%g3, 80f;					\
-	 nop;								\
-	add		%g3, %g2, %g3;					\
-	sllx		%g1, 8, %g1;					\
-	add		%g3, %g1, %g1;					\
-	/* %g1 holds pointer to the top of the logging scoreboard */	\
-	ldx		[%g1 + 0x0], %g7;				\
-	cmp		%g7, -1;					\
-	bne,pn		%xcc, 80f;					\
-	 nop;								\
-	stx		%g4, [%g1 + 0x0];				\
-	stx		%g5, [%g1 + 0x8];				\
-	add		%g1, 0x10, %g1;					\
-	/* %g1 now points to D-cache logging area */			\
-	set		0x3ff8, %g2;	/* DC_addr mask		*/	\
-	and		%g5, %g2, %g2;	/* DC_addr bits of AFAR	*/	\
-	srlx		%g5, 12, %g3;					\
-	or		%g3, 1, %g3;	/* PHYS tag + valid	*/	\
-10:	ldxa		[%g2] ASI_DCACHE_TAG, %g7;			\
-	cmp		%g3, %g7;	/* TAG match?		*/	\
-	bne,pt		%xcc, 13f;					\
-	 nop;								\
-	/* Yep, what we want, capture state. */				\
-	stx		%g2, [%g1 + 0x20];				\
-	stx		%g7, [%g1 + 0x28];				\
-	/* A membar Sync is required before and after utag access. */	\
-	membar		#Sync;						\
-	ldxa		[%g2] ASI_DCACHE_UTAG, %g7;			\
-	membar		#Sync;						\
-	stx		%g7, [%g1 + 0x30];				\
-	ldxa		[%g2] ASI_DCACHE_SNOOP_TAG, %g7;		\
-	stx		%g7, [%g1 + 0x38];				\
-	clr		%g3;						\
-12:	ldxa		[%g2 + %g3] ASI_DCACHE_DATA, %g7;		\
-	stx		%g7, [%g1];					\
-	add		%g3, (1 << 5), %g3;				\
-	cmp		%g3, (4 << 5);					\
-	bl,pt		%xcc, 12b;					\
-	 add		%g1, 0x8, %g1;					\
-	ba,pt		%xcc, 20f;					\
-	 add		%g1, 0x20, %g1;					\
-13:	sethi		%hi(1 << 14), %g7;				\
-	add		%g2, %g7, %g2;					\
-	srlx		%g2, 14, %g7;					\
-	cmp		%g7, 4;						\
-	bl,pt		%xcc, 10b;					\
-	 nop;								\
-	add		%g1, 0x40, %g1;					\
-20:	/* %g1 now points to I-cache logging area */			\
-	set		0x1fe0, %g2;	/* IC_addr mask		*/	\
-	and		%g5, %g2, %g2;	/* IC_addr bits of AFAR	*/	\
-	sllx		%g2, 1, %g2;	/* IC_addr[13:6]==VA[12:5] */	\
-	srlx		%g5, (13 - 8), %g3; /* Make PTAG */		\
-	andn		%g3, 0xff, %g3;	/* Mask off undefined bits */	\
-21:	ldxa		[%g2] ASI_IC_TAG, %g7;				\
-	andn		%g7, 0xff, %g7;					\
-	cmp		%g3, %g7;					\
-	bne,pt		%xcc, 23f;					\
-	 nop;								\
-	/* Yep, what we want, capture state. */				\
-	stx		%g2, [%g1 + 0x40];				\
-	stx		%g7, [%g1 + 0x48];				\
-	add		%g2, (1 << 3), %g2;				\
-	ldxa		[%g2] ASI_IC_TAG, %g7;				\
-	add		%g2, (1 << 3), %g2;				\
-	stx		%g7, [%g1 + 0x50];				\
-	ldxa		[%g2] ASI_IC_TAG, %g7;				\
-	add		%g2, (1 << 3), %g2;				\
-	stx		%g7, [%g1 + 0x60];				\
-	ldxa		[%g2] ASI_IC_TAG, %g7;				\
-	stx		%g7, [%g1 + 0x68];				\
-	sub		%g2, (3 << 3), %g2;				\
-	ldxa		[%g2] ASI_IC_STAG, %g7;				\
-	stx		%g7, [%g1 + 0x58];				\
-	clr		%g3;						\
-	srlx		%g2, 2, %g2;					\
-22:	ldxa		[%g2 + %g3] ASI_IC_INSTR, %g7;			\
-	stx		%g7, [%g1];					\
-	add		%g3, (1 << 3), %g3;				\
-	cmp		%g3, (8 << 3);					\
-	bl,pt		%xcc, 22b;					\
-	 add		%g1, 0x8, %g1;					\
-	ba,pt		%xcc, 30f;					\
-	 add		%g1, 0x30, %g1;					\
-23:	sethi		%hi(1 << 14), %g7;				\
-	add		%g2, %g7, %g2;					\
-	srlx		%g2, 14, %g7;					\
-	cmp		%g7, 4;						\
-	bl,pt		%xcc, 21b;					\
-	 nop;								\
-	add		%g1, 0x70, %g1;					\
-30:	/* %g1 now points to E-cache logging area */			\
-	andn		%g5, (32 - 1), %g2;	/* E-cache subblock */	\
-	stx		%g2, [%g1 + 0x20];				\
-	ldxa		[%g2] ASI_EC_TAG_DATA, %g7;			\
-	stx		%g7, [%g1 + 0x28];				\
-	ldxa		[%g2] ASI_EC_R, %g0;				\
-	clr		%g3;						\
-31:	ldxa		[%g3] ASI_EC_DATA, %g7;				\
-	stx		%g7, [%g1 + %g3];				\
-	add		%g3, 0x8, %g3;					\
-	cmp		%g3, 0x20;					\
-	bl,pt		%xcc, 31b;					\
-	 nop;								\
-80:	/* DONE */
-
 	/* These get patched into the trap table at boot time
 	 * once we know we have a cheetah processor.
 	 */
@@ -1296,6 +1163,170 @@ dcpe_icpe_tl1_common:
 	membar		#Sync
 	retry
 
+	/* Capture I/D/E-cache state into per-cpu error scoreboard.
+	 *
+	 * %g1:		(TL>=0) ? 1 : 0
+	 * %g2:		scratch
+	 * %g3:		scratch
+	 * %g4:		AFSR
+	 * %g5:		AFAR
+	 * %g6:		current thread ptr
+	 * %g7:		scratch
+	 */
+__cheetah_log_error:
+	/* Put "TL1" software bit into AFSR. */
+	and		%g1, 0x1, %g1
+	sllx		%g1, 63, %g2
+	or		%g4, %g2, %g4
+
+	/* Get log entry pointer for this cpu at this trap level. */
+	BRANCH_IF_JALAPENO(g2,g3,50f)
+	ldxa		[%g0] ASI_SAFARI_CONFIG, %g2
+	srlx		%g2, 17, %g2
+	ba,pt		%xcc, 60f
+	 and		%g2, 0x3ff, %g2
+
+50:	ldxa		[%g0] ASI_JBUS_CONFIG, %g2
+	srlx		%g2, 17, %g2
+	and		%g2, 0x1f, %g2
+
+60:	sllx		%g2, 9, %g2
+	sethi		%hi(cheetah_error_log), %g3
+	ldx		[%g3 + %lo(cheetah_error_log)], %g3
+	brz,pn		%g3, 80f
+	 nop
+
+	add		%g3, %g2, %g3
+	sllx		%g1, 8, %g1
+	add		%g3, %g1, %g1
+
+	/* %g1 holds pointer to the top of the logging scoreboard */
+	ldx		[%g1 + 0x0], %g7
+	cmp		%g7, -1
+	bne,pn		%xcc, 80f
+	 nop
+
+	stx		%g4, [%g1 + 0x0]
+	stx		%g5, [%g1 + 0x8]
+	add		%g1, 0x10, %g1
+
+	/* %g1 now points to D-cache logging area */
+	set		0x3ff8, %g2	/* DC_addr mask		*/
+	and		%g5, %g2, %g2	/* DC_addr bits of AFAR	*/
+	srlx		%g5, 12, %g3
+	or		%g3, 1, %g3	/* PHYS tag + valid	*/
+
+10:	ldxa		[%g2] ASI_DCACHE_TAG, %g7
+	cmp		%g3, %g7	/* TAG match?		*/
+	bne,pt		%xcc, 13f
+	 nop
+
+	/* Yep, what we want, capture state. */
+	stx		%g2, [%g1 + 0x20]
+	stx		%g7, [%g1 + 0x28]
+
+	/* A membar Sync is required before and after utag access. */
+	membar		#Sync
+	ldxa		[%g2] ASI_DCACHE_UTAG, %g7
+	membar		#Sync
+	stx		%g7, [%g1 + 0x30]
+	ldxa		[%g2] ASI_DCACHE_SNOOP_TAG, %g7
+	stx		%g7, [%g1 + 0x38]
+	clr		%g3
+
+12:	ldxa		[%g2 + %g3] ASI_DCACHE_DATA, %g7
+	stx		%g7, [%g1]
+	add		%g3, (1 << 5), %g3
+	cmp		%g3, (4 << 5)
+	bl,pt		%xcc, 12b
+	 add		%g1, 0x8, %g1
+
+	ba,pt		%xcc, 20f
+	 add		%g1, 0x20, %g1
+
+13:	sethi		%hi(1 << 14), %g7
+	add		%g2, %g7, %g2
+	srlx		%g2, 14, %g7
+	cmp		%g7, 4
+	bl,pt		%xcc, 10b
+	 nop
+
+	add		%g1, 0x40, %g1
+
+	/* %g1 now points to I-cache logging area */
+20:	set		0x1fe0, %g2	/* IC_addr mask		*/
+	and		%g5, %g2, %g2	/* IC_addr bits of AFAR	*/
+	sllx		%g2, 1, %g2	/* IC_addr[13:6]==VA[12:5] */
+	srlx		%g5, (13 - 8), %g3 /* Make PTAG */
+	andn		%g3, 0xff, %g3	/* Mask off undefined bits */
+
+21:	ldxa		[%g2] ASI_IC_TAG, %g7
+	andn		%g7, 0xff, %g7
+	cmp		%g3, %g7
+	bne,pt		%xcc, 23f
+	 nop
+
+	/* Yep, what we want, capture state. */
+	stx		%g2, [%g1 + 0x40]
+	stx		%g7, [%g1 + 0x48]
+	add		%g2, (1 << 3), %g2
+	ldxa		[%g2] ASI_IC_TAG, %g7
+	add		%g2, (1 << 3), %g2
+	stx		%g7, [%g1 + 0x50]
+	ldxa		[%g2] ASI_IC_TAG, %g7
+	add		%g2, (1 << 3), %g2
+	stx		%g7, [%g1 + 0x60]
+	ldxa		[%g2] ASI_IC_TAG, %g7
+	stx		%g7, [%g1 + 0x68]
+	sub		%g2, (3 << 3), %g2
+	ldxa		[%g2] ASI_IC_STAG, %g7
+	stx		%g7, [%g1 + 0x58]
+	clr		%g3
+	srlx		%g2, 2, %g2
+
+22:	ldxa		[%g2 + %g3] ASI_IC_INSTR, %g7
+	stx		%g7, [%g1]
+	add		%g3, (1 << 3), %g3
+	cmp		%g3, (8 << 3)
+	bl,pt		%xcc, 22b
+	 add		%g1, 0x8, %g1
+
+	ba,pt		%xcc, 30f
+	 add		%g1, 0x30, %g1
+
+23:	sethi		%hi(1 << 14), %g7
+	add		%g2, %g7, %g2
+	srlx		%g2, 14, %g7
+	cmp		%g7, 4
+	bl,pt		%xcc, 21b
+	 nop
+
+	add		%g1, 0x70, %g1
+
+	/* %g1 now points to E-cache logging area */
+30:	andn		%g5, (32 - 1), %g2
+	stx		%g2, [%g1 + 0x20]
+	ldxa		[%g2] ASI_EC_TAG_DATA, %g7
+	stx		%g7, [%g1 + 0x28]
+	ldxa		[%g2] ASI_EC_R, %g0
+	clr		%g3
+
+31:	ldxa		[%g3] ASI_EC_DATA, %g7
+	stx		%g7, [%g1 + %g3]
+	add		%g3, 0x8, %g3
+	cmp		%g3, 0x20
+
+	bl,pt		%xcc, 31b
+	 nop
+80:
+	rdpr		%tt, %g2
+	cmp		%g2, 0x70
+	be		c_fast_ecc
+	 cmp		%g2, 0x63
+	be		c_cee
+	 nop
+	ba,pt		%xcc, c_deferred
+
 	/* Cheetah FECC trap handling, we get here from tl{0,1}_fecc
 	 * in the trap table.  That code has done a memory barrier
 	 * and has disabled both the I-cache and D-cache in the DCU
@@ -1321,8 +1352,10 @@ cheetah_fast_ecc:
 	stxa		%g4, [%g0] ASI_AFSR
 	membar		#Sync
 
-	CHEETAH_LOG_ERROR
+	ba,pt		%xcc, __cheetah_log_error
+	 nop
 
+c_fast_ecc:
 	rdpr		%pil, %g2
 	wrpr		%g0, 15, %pil
 	ba,pt		%xcc, etrap_irq
@@ -1347,8 +1380,10 @@ cheetah_cee:
 	stxa		%g4, [%g0] ASI_AFSR
 	membar		#Sync
 
-	CHEETAH_LOG_ERROR
+	ba,pt		%xcc, __cheetah_log_error
+	 nop
 
+c_cee:
 	rdpr		%pil, %g2
 	wrpr		%g0, 15, %pil
 	ba,pt		%xcc, etrap_irq
@@ -1373,8 +1408,10 @@ cheetah_deferred_trap:
 	stxa		%g4, [%g0] ASI_AFSR
 	membar		#Sync
 
-	CHEETAH_LOG_ERROR
+	ba,pt		%xcc, __cheetah_log_error
+	 nop
 
+c_deferred:
 	rdpr		%pil, %g2
 	wrpr		%g0, 15, %pil
 	ba,pt		%xcc, etrap_irq
diff --git a/arch/sparc64/kernel/head.S b/arch/sparc64/kernel/head.S
index 8104a56ca2d8..1fa06c4e3bdb 100644
--- a/arch/sparc64/kernel/head.S
+++ b/arch/sparc64/kernel/head.S
@@ -538,11 +538,12 @@ cheetah_tlb_fixup:
 	 nop
 	call	cheetah_plus_patch_winfixup
 	 nop
-	
 
 2:	/* Patch copy/page operations to cheetah optimized versions. */
 	call	cheetah_patch_copyops
 	 nop
+	call	cheetah_patch_copy_page
+	 nop
 	call	cheetah_patch_cachetlbops
 	 nop
 
diff --git a/arch/sparc64/kernel/pci.c b/arch/sparc64/kernel/pci.c
index f21c993f8856..ec8bf4012c0c 100644
--- a/arch/sparc64/kernel/pci.c
+++ b/arch/sparc64/kernel/pci.c
@@ -736,8 +736,7 @@ static void __pci_mmap_set_flags(struct pci_dev *dev, struct vm_area_struct *vma
 static void __pci_mmap_set_pgprot(struct pci_dev *dev, struct vm_area_struct *vma,
 					     enum pci_mmap_state mmap_state)
 {
-	/* Our io_remap_page_range/io_remap_pfn_range takes care of this,
-	   do nothing. */
+	/* Our io_remap_pfn_range takes care of this, do nothing.  */
 }
 
 /* Perform the actual remap of the pages for a PCI device mapping, as appropriate
diff --git a/arch/sparc64/kernel/rtrap.S b/arch/sparc64/kernel/rtrap.S
index 0696ed4b9d64..fafd227735fa 100644
--- a/arch/sparc64/kernel/rtrap.S
+++ b/arch/sparc64/kernel/rtrap.S
@@ -153,11 +153,14 @@ __handle_signal:
 rtrap_irq:
 rtrap_clr_l6:	clr			%l6
 rtrap:
-		ldub			[%g6 + TI_CPU], %l0
-		sethi			%hi(irq_stat), %l2	! &softirq_active
-		or			%l2, %lo(irq_stat), %l2	! &softirq_active
-irqsz_patchme:	sllx			%l0, 0, %l0
-		lduw			[%l2 + %l0], %l1	! softirq_pending
+#ifndef CONFIG_SMP
+		sethi			%hi(per_cpu____cpu_data), %l0
+		lduw			[%l0 + %lo(per_cpu____cpu_data)], %l1
+#else
+		sethi			%hi(per_cpu____cpu_data), %l0
+		or			%l0, %lo(per_cpu____cpu_data), %l0
+		lduw			[%l0 + %g5], %l1
+#endif
 		cmp			%l1, 0
 
 		/* mm/ultra.S:xcall_report_regs KNOWS about this load. */
diff --git a/arch/sparc64/kernel/setup.c b/arch/sparc64/kernel/setup.c
index fbdfed3798d8..ddbed3341a23 100644
--- a/arch/sparc64/kernel/setup.c
+++ b/arch/sparc64/kernel/setup.c
@@ -511,18 +511,6 @@ void __init setup_arch(char **cmdline_p)
 	conswitchp = &prom_con;
 #endif
 
-#ifdef CONFIG_SMP
-	i = (unsigned long)&irq_stat[1] - (unsigned long)&irq_stat[0];
-	if ((i == SMP_CACHE_BYTES) || (i == (2 * SMP_CACHE_BYTES))) {
-		extern unsigned int irqsz_patchme[1];
-		irqsz_patchme[0] |= ((i == SMP_CACHE_BYTES) ? SMP_CACHE_BYTES_SHIFT : \
-							SMP_CACHE_BYTES_SHIFT + 1);
-		flushi((long)&irqsz_patchme[0]);
-	} else {
-		prom_printf("Unexpected size of irq_stat[] elements\n");
-		prom_halt();
-	}
-#endif
 	/* Work out if we are starfire early on */
 	check_if_starfire();
 
diff --git a/arch/sparc64/kernel/sparc64_ksyms.c b/arch/sparc64/kernel/sparc64_ksyms.c
index a3ea697f1adb..d89fc24808d3 100644
--- a/arch/sparc64/kernel/sparc64_ksyms.c
+++ b/arch/sparc64/kernel/sparc64_ksyms.c
@@ -88,8 +88,6 @@ extern int svr4_setcontext(svr4_ucontext_t *uc, struct pt_regs *regs);
 extern int compat_sys_ioctl(unsigned int fd, unsigned int cmd, u32 arg);
 extern int (*handle_mathemu)(struct pt_regs *, struct fpustate *);
 extern long sparc32_open(const char __user * filename, int flags, int mode);
-extern int io_remap_page_range(struct vm_area_struct *vma, unsigned long from,
-	unsigned long offset, unsigned long size, pgprot_t prot, int space);
 extern int io_remap_pfn_range(struct vm_area_struct *vma, unsigned long from,
 	unsigned long pfn, unsigned long size, pgprot_t prot);
 extern void (*prom_palette)(int);
@@ -245,7 +243,6 @@ EXPORT_SYMBOL(pci_dma_supported);
 #endif
 
 /* I/O device mmaping on Sparc64. */
-EXPORT_SYMBOL(io_remap_page_range);
 EXPORT_SYMBOL(io_remap_pfn_range);
 
 /* Solaris/SunOS binary compatibility */
diff --git a/arch/sparc64/lib/PeeCeeI.c b/arch/sparc64/lib/PeeCeeI.c
index 3008d536e8c2..3c6cfbb20360 100644
--- a/arch/sparc64/lib/PeeCeeI.c
+++ b/arch/sparc64/lib/PeeCeeI.c
@@ -7,28 +7,31 @@
 #include <asm/io.h>
 #include <asm/byteorder.h>
 
-void outsb(void __iomem *addr, const void *src, unsigned long count)
+void outsb(unsigned long __addr, const void *src, unsigned long count)
 {
+	void __iomem *addr = (void __iomem *) __addr;
 	const u8 *p = src;
 
-	while(count--)
+	while (count--)
 		outb(*p++, addr);
 }
 
-void outsw(void __iomem *addr, const void *src, unsigned long count)
+void outsw(unsigned long __addr, const void *src, unsigned long count)
 {
-	if(count) {
+	void __iomem *addr = (void __iomem *) __addr;
+
+	if (count) {
 		u16 *ps = (u16 *)src;
 		u32 *pi;
 
-		if(((u64)src) & 0x2) {
+		if (((u64)src) & 0x2) {
 			u16 val = le16_to_cpup(ps);
 			outw(val, addr);
 			ps++;
 			count--;
 		}
 		pi = (u32 *)ps;
-		while(count >= 2) {
+		while (count >= 2) {
 			u32 w = le32_to_cpup(pi);
 
 			pi++;
@@ -37,19 +40,21 @@ void outsw(void __iomem *addr, const void *src, unsigned long count)
 			count -= 2;
 		}
 		ps = (u16 *)pi;
-		if(count) {
+		if (count) {
 			u16 val = le16_to_cpup(ps);
 			outw(val, addr);
 		}
 	}
 }
 
-void outsl(void __iomem *addr, const void *src, unsigned long count)
+void outsl(unsigned long __addr, const void *src, unsigned long count)
 {
-	if(count) {
-		if((((u64)src) & 0x3) == 0) {
+	void __iomem *addr = (void __iomem *) __addr;
+
+	if (count) {
+		if ((((u64)src) & 0x3) == 0) {
 			u32 *p = (u32 *)src;
-			while(count--) {
+			while (count--) {
 				u32 val = cpu_to_le32p(p);
 				outl(val, addr);
 				p++;
@@ -60,13 +65,13 @@ void outsl(void __iomem *addr, const void *src, unsigned long count)
 			u32 l = 0, l2;
 			u32 *pi;
 
-			switch(((u64)src) & 0x3) {
+			switch (((u64)src) & 0x3) {
 			case 0x2:
 				count -= 1;
 				l = cpu_to_le16p(ps) << 16;
 				ps++;
 				pi = (u32 *)ps;
-				while(count--) {
+				while (count--) {
 					l2 = cpu_to_le32p(pi);
 					pi++;
 					outl(((l >> 16) | (l2 << 16)), addr);
@@ -86,7 +91,7 @@ void outsl(void __iomem *addr, const void *src, unsigned long count)
 				ps++;
 				l |= (l2 << 16);
 				pi = (u32 *)ps;
-				while(count--) {
+				while (count--) {
 					l2 = cpu_to_le32p(pi);
 					pi++;
 					outl(((l >> 8) | (l2 << 24)), addr);
@@ -101,7 +106,7 @@ void outsl(void __iomem *addr, const void *src, unsigned long count)
 				pb = (u8 *)src;
 				l = (*pb++ << 24);
 				pi = (u32 *)pb;
-				while(count--) {
+				while (count--) {
 					l2 = cpu_to_le32p(pi);
 					pi++;
 					outl(((l >> 24) | (l2 << 8)), addr);
@@ -119,16 +124,18 @@ void outsl(void __iomem *addr, const void *src, unsigned long count)
 	}
 }
 
-void insb(void __iomem *addr, void *dst, unsigned long count)
+void insb(unsigned long __addr, void *dst, unsigned long count)
 {
-	if(count) {
+	void __iomem *addr = (void __iomem *) __addr;
+
+	if (count) {
 		u32 *pi;
 		u8 *pb = dst;
 
-		while((((unsigned long)pb) & 0x3) && count--)
+		while ((((unsigned long)pb) & 0x3) && count--)
 			*pb++ = inb(addr);
 		pi = (u32 *)pb;
-		while(count >= 4) {
+		while (count >= 4) {
 			u32 w;
 
 			w  = (inb(addr) << 24);
@@ -139,23 +146,25 @@ void insb(void __iomem *addr, void *dst, unsigned long count)
 			count -= 4;
 		}
 		pb = (u8 *)pi;
-		while(count--)
+		while (count--)
 			*pb++ = inb(addr);
 	}
 }
 
-void insw(void __iomem *addr, void *dst, unsigned long count)
+void insw(unsigned long __addr, void *dst, unsigned long count)
 {
-	if(count) {
+	void __iomem *addr = (void __iomem *) __addr;
+
+	if (count) {
 		u16 *ps = dst;
 		u32 *pi;
 
-		if(((unsigned long)ps) & 0x2) {
+		if (((unsigned long)ps) & 0x2) {
 			*ps++ = le16_to_cpu(inw(addr));
 			count--;
 		}
 		pi = (u32 *)ps;
-		while(count >= 2) {
+		while (count >= 2) {
 			u32 w;
 
 			w  = (le16_to_cpu(inw(addr)) << 16);
@@ -164,31 +173,33 @@ void insw(void __iomem *addr, void *dst, unsigned long count)
 			count -= 2;
 		}
 		ps = (u16 *)pi;
-		if(count)
+		if (count)
 			*ps = le16_to_cpu(inw(addr));
 	}
 }
 
-void insl(void __iomem *addr, void *dst, unsigned long count)
+void insl(unsigned long __addr, void *dst, unsigned long count)
 {
-	if(count) {
-		if((((unsigned long)dst) & 0x3) == 0) {
+	void __iomem *addr = (void __iomem *) __addr;
+
+	if (count) {
+		if ((((unsigned long)dst) & 0x3) == 0) {
 			u32 *pi = dst;
-			while(count--)
+			while (count--)
 				*pi++ = le32_to_cpu(inl(addr));
 		} else {
 			u32 l = 0, l2, *pi;
 			u16 *ps;
 			u8 *pb;
 
-			switch(((unsigned long)dst) & 3) {
+			switch (((unsigned long)dst) & 3) {
 			case 0x2:
 				ps = dst;
 				count -= 1;
 				l = le32_to_cpu(inl(addr));
 				*ps++ = l;
 				pi = (u32 *)ps;
-				while(count--) {
+				while (count--) {
 					l2 = le32_to_cpu(inl(addr));
 					*pi++ = (l << 16) | (l2 >> 16);
 					l = l2;
@@ -205,7 +216,7 @@ void insl(void __iomem *addr, void *dst, unsigned long count)
 				ps = (u16 *)pb;
 				*ps++ = ((l >> 8) & 0xffff);
 				pi = (u32 *)ps;
-				while(count--) {
+				while (count--) {
 					l2 = le32_to_cpu(inl(addr));
 					*pi++ = (l << 24) | (l2 >> 8);
 					l = l2;
@@ -220,7 +231,7 @@ void insl(void __iomem *addr, void *dst, unsigned long count)
 				l = le32_to_cpu(inl(addr));
 				*pb++ = l >> 24;
 				pi = (u32 *)pb;
-				while(count--) {
+				while (count--) {
 					l2 = le32_to_cpu(inl(addr));
 					*pi++ = (l << 8) | (l2 >> 24);
 					l = l2;
diff --git a/arch/sparc64/lib/copy_page.S b/arch/sparc64/lib/copy_page.S
index 23ebf2c970b7..feebb14fd27a 100644
--- a/arch/sparc64/lib/copy_page.S
+++ b/arch/sparc64/lib/copy_page.S
@@ -87,7 +87,7 @@ copy_user_page:		/* %o0=dest, %o1=src, %o2=vaddr */
 	membar		#Sync
 	wrpr		%o2, 0x0, %pstate
 
-	BRANCH_IF_ANY_CHEETAH(g3,o2,1f)
+cheetah_copy_page_insn:
 	ba,pt		%xcc, 9f
 	 nop
 
@@ -240,3 +240,14 @@ copy_user_page:		/* %o0=dest, %o1=src, %o2=vaddr */
 	 stw		%o4, [%g6 + TI_PRE_COUNT]
 
 	.size		copy_user_page, .-copy_user_page
+
+	.globl		cheetah_patch_copy_page
+cheetah_patch_copy_page:
+	sethi		%hi(0x01000000), %o1	! NOP
+	sethi		%hi(cheetah_copy_page_insn), %o0
+	or		%o0, %lo(cheetah_copy_page_insn), %o0
+	stw		%o1, [%o0]
+	membar		#StoreStore
+	flush		%o0
+	retl
+	 nop
diff --git a/arch/sparc64/mm/generic.c b/arch/sparc64/mm/generic.c
index 6b31f6117a95..c954d91f01d0 100644
--- a/arch/sparc64/mm/generic.c
+++ b/arch/sparc64/mm/generic.c
@@ -116,37 +116,6 @@ static inline int io_remap_pud_range(struct mm_struct *mm, pud_t * pud, unsigned
 	return 0;
 }
 
-int io_remap_page_range(struct vm_area_struct *vma, unsigned long from, unsigned long offset, unsigned long size, pgprot_t prot, int space)
-{
-	int error = 0;
-	pgd_t * dir;
-	unsigned long beg = from;
-	unsigned long end = from + size;
-	struct mm_struct *mm = vma->vm_mm;
-
-	prot = __pgprot(pg_iobits);
-	offset -= from;
-	dir = pgd_offset(mm, from);
-	flush_cache_range(vma, beg, end);
-
-	spin_lock(&mm->page_table_lock);
-	while (from < end) {
-		pud_t *pud = pud_alloc(mm, dir, from);
-		error = -ENOMEM;
-		if (!pud)
-			break;
-		error = io_remap_pud_range(mm, pud, from, end - from, offset + from, prot, space);
-		if (error)
-			break;
-		from = (from + PGDIR_SIZE) & PGDIR_MASK;
-		dir++;
-	}
-	flush_tlb_range(vma, beg, end);
-	spin_unlock(&mm->page_table_lock);
-
-	return error;
-}
-
 int io_remap_pfn_range(struct vm_area_struct *vma, unsigned long from,
 		unsigned long pfn, unsigned long size, pgprot_t prot)
 {
diff --git a/arch/sparc64/mm/ultra.S b/arch/sparc64/mm/ultra.S
index 363770893797..8dfa825eca51 100644
--- a/arch/sparc64/mm/ultra.S
+++ b/arch/sparc64/mm/ultra.S
@@ -10,6 +10,7 @@
 #include <asm/page.h>
 #include <asm/spitfire.h>
 #include <asm/mmu_context.h>
+#include <asm/mmu.h>
 #include <asm/pil.h>
 #include <asm/head.h>
 #include <asm/thread_info.h>
@@ -45,6 +46,8 @@ __flush_tlb_mm: /* %o0=(ctx & TAG_CONTEXT_BITS), %o1=SECONDARY_CONTEXT */
 	nop
 	nop
 	nop
+	nop
+	nop
 
 	.align		32
 	.globl		__flush_tlb_pending
@@ -73,6 +76,9 @@ __flush_tlb_pending:
 	retl
 	 wrpr		%g7, 0x0, %pstate
 	nop
+	nop
+	nop
+	nop
 
 	.align		32
 	.globl		__flush_tlb_kernel_range
@@ -224,16 +230,8 @@ __update_mmu_cache:	/* %o0=hw_context, %o1=address, %o2=pte, %o3=fault_code */
 	 or		%o5, %o0, %o5
 	ba,a,pt		%xcc, __prefill_itlb
 
-	/* Cheetah specific versions, patched at boot time.
-	 *
-	 * This writes of the PRIMARY_CONTEXT register in this file are
-	 * safe even on Cheetah+ and later wrt. the page size fields.
-	 * The nucleus page size fields do not matter because we make
-	 * no data references, and these instructions execute out of a
-	 * locked I-TLB entry sitting in the fully assosciative I-TLB.
-	 * This sequence should also never trap.
-	 */
-__cheetah_flush_tlb_mm: /* 15 insns */
+	/* Cheetah specific versions, patched at boot time. */
+__cheetah_flush_tlb_mm: /* 18 insns */
 	rdpr		%pstate, %g7
 	andn		%g7, PSTATE_IE, %g2
 	wrpr		%g2, 0x0, %pstate
@@ -241,6 +239,9 @@ __cheetah_flush_tlb_mm: /* 15 insns */
 	mov		PRIMARY_CONTEXT, %o2
 	mov		0x40, %g3
 	ldxa		[%o2] ASI_DMMU, %g2
+	srlx		%g2, CTX_PGSZ1_NUC_SHIFT, %o1
+	sllx		%o1, CTX_PGSZ1_NUC_SHIFT, %o1
+	or		%o0, %o1, %o0	/* Preserve nucleus page size fields */
 	stxa		%o0, [%o2] ASI_DMMU
 	stxa		%g0, [%g3] ASI_DMMU_DEMAP
 	stxa		%g0, [%g3] ASI_IMMU_DEMAP
@@ -250,7 +251,7 @@ __cheetah_flush_tlb_mm: /* 15 insns */
 	retl
 	 wrpr		%g7, 0x0, %pstate
 
-__cheetah_flush_tlb_pending:	/* 23 insns */
+__cheetah_flush_tlb_pending:	/* 26 insns */
 	/* %o0 = context, %o1 = nr, %o2 = vaddrs[] */
 	rdpr		%pstate, %g7
 	sllx		%o1, 3, %o1
@@ -259,6 +260,9 @@ __cheetah_flush_tlb_pending:	/* 23 insns */
 	wrpr		%g0, 1, %tl
 	mov		PRIMARY_CONTEXT, %o4
 	ldxa		[%o4] ASI_DMMU, %g2
+	srlx		%g2, CTX_PGSZ1_NUC_SHIFT, %o3
+	sllx		%o3, CTX_PGSZ1_NUC_SHIFT, %o3
+	or		%o0, %o3, %o0	/* Preserve nucleus page size fields */
 	stxa		%o0, [%o4] ASI_DMMU
 1:	sub		%o1, (1 << 3), %o1
 	ldx		[%o2 + %o1], %o3
@@ -311,14 +315,14 @@ cheetah_patch_cachetlbops:
 	sethi		%hi(__cheetah_flush_tlb_mm), %o1
 	or		%o1, %lo(__cheetah_flush_tlb_mm), %o1
 	call		cheetah_patch_one
-	 mov		15, %o2
+	 mov		18, %o2
 
 	sethi		%hi(__flush_tlb_pending), %o0
 	or		%o0, %lo(__flush_tlb_pending), %o0
 	sethi		%hi(__cheetah_flush_tlb_pending), %o1
 	or		%o1, %lo(__cheetah_flush_tlb_pending), %o1
 	call		cheetah_patch_one
-	 mov		23, %o2
+	 mov		26, %o2
 
 #ifdef DCACHE_ALIASING_POSSIBLE
 	sethi		%hi(__flush_dcache_page), %o0
@@ -352,9 +356,12 @@ cheetah_patch_cachetlbops:
 	.globl		xcall_flush_tlb_mm
 xcall_flush_tlb_mm:
 	mov		PRIMARY_CONTEXT, %g2
-	mov		0x40, %g4
 	ldxa		[%g2] ASI_DMMU, %g3
+	srlx		%g3, CTX_PGSZ1_NUC_SHIFT, %g4
+	sllx		%g4, CTX_PGSZ1_NUC_SHIFT, %g4
+	or		%g5, %g4, %g5	/* Preserve nucleus page size fields */
 	stxa		%g5, [%g2] ASI_DMMU
+	mov		0x40, %g4
 	stxa		%g0, [%g4] ASI_DMMU_DEMAP
 	stxa		%g0, [%g4] ASI_IMMU_DEMAP
 	stxa		%g3, [%g2] ASI_DMMU
@@ -366,6 +373,10 @@ xcall_flush_tlb_pending:
 	sllx		%g1, 3, %g1
 	mov		PRIMARY_CONTEXT, %g4
 	ldxa		[%g4] ASI_DMMU, %g2
+	srlx		%g2, CTX_PGSZ1_NUC_SHIFT, %g4
+	sllx		%g4, CTX_PGSZ1_NUC_SHIFT, %g4
+	or		%g5, %g4, %g5
+	mov		PRIMARY_CONTEXT, %g4
 	stxa		%g5, [%g4] ASI_DMMU
 1:	sub		%g1, (1 << 3), %g1
 	ldx		[%g7 + %g1], %g5