[SPARC64]: Dynamically grow TSB in response to RSS growth.

As the RSS grows, grow the TSB in order to reduce the likelyhood
of hash collisions and thus poor hit rates in the TSB.

This definitely needs some serious tuning.

Signed-off-by: David S. Miller <davem@davemloft.net>

5 files changed, 184 insertions, 10 deletions

diff --git a/arch/sparc64/kernel/smp.c b/arch/sparc64/kernel/smp.c
index 8c245859d212..3c14b549cf91 100644
--- a/arch/sparc64/kernel/smp.c
+++ b/arch/sparc64/kernel/smp.c
@@ -581,11 +581,11 @@ extern unsigned long xcall_call_function;
  * You must not call this function with disabled interrupts or from a
  * hardware interrupt handler or from a bottom half handler.
  */
-int smp_call_function(void (*func)(void *info), void *info,
-		      int nonatomic, int wait)
+static int smp_call_function_mask(void (*func)(void *info), void *info,
+				  int nonatomic, int wait, cpumask_t mask)
 {
 	struct call_data_struct data;
-	int cpus = num_online_cpus() - 1;
+	int cpus = cpus_weight(mask) - 1;
 	long timeout;
 
 	if (!cpus)
@@ -603,7 +603,7 @@ int smp_call_function(void (*func)(void *info), void *info,
 
 	call_data = &data;
 
-	smp_cross_call(&xcall_call_function, 0, 0, 0);
+	smp_cross_call_masked(&xcall_call_function, 0, 0, 0, mask);
 
 	/* 
 	 * Wait for other cpus to complete function or at
@@ -629,6 +629,13 @@ out_timeout:
 	return 0;
 }
 
+int smp_call_function(void (*func)(void *info), void *info,
+		      int nonatomic, int wait)
+{
+	return smp_call_function_mask(func, info, nonatomic, wait,
+				      cpu_online_map);
+}
+
 void smp_call_function_client(int irq, struct pt_regs *regs)
 {
 	void (*func) (void *info) = call_data->func;
@@ -646,6 +653,19 @@ void smp_call_function_client(int irq, struct pt_regs *regs)
 	}
 }
 
+static void tsb_sync(void *info)
+{
+	struct mm_struct *mm = info;
+
+	if (current->active_mm == mm)
+		tsb_context_switch(mm);
+}
+
+void smp_tsb_sync(struct mm_struct *mm)
+{
+	smp_call_function_mask(tsb_sync, mm, 0, 1, mm->cpu_vm_mask);
+}
+
 extern unsigned long xcall_flush_tlb_mm;
 extern unsigned long xcall_flush_tlb_pending;
 extern unsigned long xcall_flush_tlb_kernel_range;
diff --git a/arch/sparc64/mm/init.c b/arch/sparc64/mm/init.c
index 7c456afaa9a5..a8119cb4fa32 100644
--- a/arch/sparc64/mm/init.c
+++ b/arch/sparc64/mm/init.c
@@ -246,9 +246,11 @@ static __inline__ void clear_dcache_dirty_cpu(struct page *page, unsigned long c
 
 void update_mmu_cache(struct vm_area_struct *vma, unsigned long address, pte_t pte)
 {
+	struct mm_struct *mm;
 	struct page *page;
 	unsigned long pfn;
 	unsigned long pg_flags;
+	unsigned long mm_rss;
 
 	pfn = pte_pfn(pte);
 	if (pfn_valid(pfn) &&
@@ -270,6 +272,11 @@ void update_mmu_cache(struct vm_area_struct *vma, unsigned long address, pte_t p
 
 		put_cpu();
 	}
+
+	mm = vma->vm_mm;
+	mm_rss = get_mm_rss(mm);
+	if (mm_rss >= mm->context.tsb_rss_limit)
+		tsb_grow(mm, mm_rss, GFP_ATOMIC);
 }
 
 void flush_dcache_page(struct page *page)
diff --git a/arch/sparc64/mm/tsb.c b/arch/sparc64/mm/tsb.c
index dfe7144fcdf6..707af4b84a0e 100644
--- a/arch/sparc64/mm/tsb.c
+++ b/arch/sparc64/mm/tsb.c
@@ -10,6 +10,7 @@
 #include <asm/tlb.h>
 #include <asm/mmu_context.h>
 #include <asm/pgtable.h>
+#include <asm/tsb.h>
 
 /* We use an 8K TSB for the whole kernel, this allows to
  * handle about 4MB of modules and vmalloc mappings without
@@ -146,6 +147,9 @@ static void setup_tsb_params(struct mm_struct *mm, unsigned long tsb_bytes)
 		tte |= _PAGE_SZ4MB;
 		page_sz = 4 * 1024 * 1024;
 		break;
+
+	default:
+		BUG();
 	};
 
 	tsb_reg |= base;
@@ -157,23 +161,158 @@ static void setup_tsb_params(struct mm_struct *mm, unsigned long tsb_bytes)
 	mm->context.tsb_map_pte = tte;
 }
 
+/* The page tables are locked against modifications while this
+ * runs.
+ *
+ * XXX do some prefetching...
+ */
+static void copy_tsb(struct tsb *old_tsb, unsigned long old_size,
+		     struct tsb *new_tsb, unsigned long new_size)
+{
+	unsigned long old_nentries = old_size / sizeof(struct tsb);
+	unsigned long new_nentries = new_size / sizeof(struct tsb);
+	unsigned long i;
+
+	for (i = 0; i < old_nentries; i++) {
+		register unsigned long tag asm("o4");
+		register unsigned long pte asm("o5");
+		unsigned long v;
+		unsigned int hash;
+
+		__asm__ __volatile__(
+			"ldda [%2] %3, %0"
+			: "=r" (tag), "=r" (pte)
+			: "r" (&old_tsb[i]), "i" (ASI_NUCLEUS_QUAD_LDD));
+
+		if (!tag || (tag & TSB_TAG_LOCK))
+			continue;
+
+		/* We only put base page size PTEs into the TSB,
+		 * but that might change in the future.  This code
+		 * would need to be changed if we start putting larger
+		 * page size PTEs into there.
+		 */
+		WARN_ON((pte & _PAGE_ALL_SZ_BITS) != _PAGE_SZBITS);
+
+		/* The tag holds bits 22 to 63 of the virtual address
+		 * and the context.  Clear out the context, and shift
+		 * up to make a virtual address.
+		 */
+		v = (tag & ((1UL << 42UL) - 1UL)) << 22UL;
+
+		/* The implied bits of the tag (bits 13 to 21) are
+		 * determined by the TSB entry index, so fill that in.
+		 */
+		v |= (i & (512UL - 1UL)) << 13UL;
+
+		hash = tsb_hash(v, new_nentries);
+		new_tsb[hash].tag = tag;
+		new_tsb[hash].pte = pte;
+	}
+}
+
+/* When the RSS of an address space exceeds mm->context.tsb_rss_limit,
+ * update_mmu_cache() invokes this routine to try and grow the TSB.
+ * When we reach the maximum TSB size supported, we stick ~0UL into
+ * mm->context.tsb_rss_limit so the grow checks in update_mmu_cache()
+ * will not trigger any longer.
+ *
+ * The TSB can be anywhere from 8K to 1MB in size, in increasing powers
+ * of two.  The TSB must be aligned to it's size, so f.e. a 512K TSB
+ * must be 512K aligned.
+ *
+ * The idea here is to grow the TSB when the RSS of the process approaches
+ * the number of entries that the current TSB can hold at once.  Currently,
+ * we trigger when the RSS hits 3/4 of the TSB capacity.
+ */
+void tsb_grow(struct mm_struct *mm, unsigned long rss, gfp_t gfp_flags)
+{
+	unsigned long max_tsb_size = 1 * 1024 * 1024;
+	unsigned long size, old_size;
+	struct page *page;
+	struct tsb *old_tsb;
+
+	if (max_tsb_size > (PAGE_SIZE << MAX_ORDER))
+		max_tsb_size = (PAGE_SIZE << MAX_ORDER);
+
+	for (size = PAGE_SIZE; size < max_tsb_size; size <<= 1UL) {
+		unsigned long n_entries = size / sizeof(struct tsb);
+
+		n_entries = (n_entries * 3) / 4;
+		if (n_entries > rss)
+			break;
+	}
+
+	page = alloc_pages(gfp_flags | __GFP_ZERO, get_order(size));
+	if (unlikely(!page))
+		return;
+
+	if (size == max_tsb_size)
+		mm->context.tsb_rss_limit = ~0UL;
+	else
+		mm->context.tsb_rss_limit =
+			((size / sizeof(struct tsb)) * 3) / 4;
+
+	old_tsb = mm->context.tsb;
+	old_size = mm->context.tsb_nentries * sizeof(struct tsb);
+
+	if (old_tsb)
+		copy_tsb(old_tsb, old_size, page_address(page), size);
+
+	mm->context.tsb = page_address(page);
+	setup_tsb_params(mm, size);
+
+	/* If old_tsb is NULL, we're being invoked for the first time
+	 * from init_new_context().
+	 */
+	if (old_tsb) {
+		/* Now force all other processors to reload the new
+		 * TSB state.
+		 */
+		smp_tsb_sync(mm);
+
+		/* Finally reload it on the local cpu.  No further
+		 * references will remain to the old TSB and we can
+		 * thus free it up.
+		 */
+		tsb_context_switch(mm);
+
+		free_pages((unsigned long) old_tsb, get_order(old_size));
+	}
+}
+
 int init_new_context(struct task_struct *tsk, struct mm_struct *mm)
 {
-	unsigned long page = get_zeroed_page(GFP_KERNEL);
+	unsigned long initial_rss;
 
 	mm->context.sparc64_ctx_val = 0UL;
-	if (unlikely(!page))
-		return -ENOMEM;
 
-	mm->context.tsb = (struct tsb *) page;
-	setup_tsb_params(mm, PAGE_SIZE);
+	/* copy_mm() copies over the parent's mm_struct before calling
+	 * us, so we need to zero out the TSB pointer or else tsb_grow()
+	 * will be confused and think there is an older TSB to free up.
+	 */
+	mm->context.tsb = NULL;
+
+	/* If this is fork, inherit the parent's TSB size.  We would
+	 * grow it to that size on the first page fault anyways.
+	 */
+	initial_rss = mm->context.tsb_nentries;
+	if (initial_rss)
+		initial_rss -= 1;
+
+	tsb_grow(mm, initial_rss, GFP_KERNEL);
+
+	if (unlikely(!mm->context.tsb))
+		return -ENOMEM;
 
 	return 0;
 }
 
 void destroy_context(struct mm_struct *mm)
 {
-	free_page((unsigned long) mm->context.tsb);
+	unsigned long size = mm->context.tsb_nentries * sizeof(struct tsb);
+
+	free_pages((unsigned long) mm->context.tsb, get_order(size));
 
 	/* We can remove these later, but for now it's useful
 	 * to catch any bogus post-destroy_context() references
diff --git a/include/asm-sparc64/mmu.h b/include/asm-sparc64/mmu.h
index 2effeba2476c..76008ff6a90b 100644
--- a/include/asm-sparc64/mmu.h
+++ b/include/asm-sparc64/mmu.h
@@ -100,6 +100,7 @@ struct tsb {
 typedef struct {
 	unsigned long	sparc64_ctx_val;
 	struct tsb	*tsb;
+	unsigned long	tsb_rss_limit;
 	unsigned long	tsb_nentries;
 	unsigned long	tsb_reg_val;
 	unsigned long	tsb_map_vaddr;
diff --git a/include/asm-sparc64/mmu_context.h b/include/asm-sparc64/mmu_context.h
index 0a950f151d2b..1d232678821d 100644
--- a/include/asm-sparc64/mmu_context.h
+++ b/include/asm-sparc64/mmu_context.h
@@ -32,6 +32,13 @@ static inline void tsb_context_switch(struct mm_struct *mm)
 			     mm->context.tsb_map_pte);
 }
 
+extern void tsb_grow(struct mm_struct *mm, unsigned long mm_rss, gfp_t gfp_flags);
+#ifdef CONFIG_SMP
+extern void smp_tsb_sync(struct mm_struct *mm);
+#else
+#define smp_tsb_sync(__mm) do { } while (0)
+#endif
+
 /* Set MMU context in the actual hardware. */
 #define load_secondary_context(__mm) \
 	__asm__ __volatile__("stxa	%0, [%1] %2\n\t" \