From 44b6cc8130e80e673ba8b3baf8e41891fe484786 Mon Sep 17 00:00:00 2001 From: Martin Schwidefsky Date: Mon, 13 Jun 2016 10:36:00 +0200 Subject: s390/mm,kvm: flush gmap address space with IDTE The __tlb_flush_mm() helper uses a global flush if the mm struct has a gmap structure attached to it. Replace the global flush with two individual flushes by means of the IDTE instruction if only a single gmap is attached the the mm. Signed-off-by: Martin Schwidefsky --- arch/s390/include/asm/mmu.h | 1 + arch/s390/include/asm/mmu_context.h | 1 + arch/s390/include/asm/tlbflush.h | 40 ++++++++++++++++--------------------- arch/s390/mm/gmap.c | 15 ++++++++++++++ 4 files changed, 34 insertions(+), 23 deletions(-) diff --git a/arch/s390/include/asm/mmu.h b/arch/s390/include/asm/mmu.h index 6d39329c894b..bea785d7f853 100644 --- a/arch/s390/include/asm/mmu.h +++ b/arch/s390/include/asm/mmu.h @@ -12,6 +12,7 @@ typedef struct { struct list_head pgtable_list; spinlock_t gmap_lock; struct list_head gmap_list; + unsigned long gmap_asce; unsigned long asce; unsigned long asce_limit; unsigned long vdso_base; diff --git a/arch/s390/include/asm/mmu_context.h b/arch/s390/include/asm/mmu_context.h index c6a088c91aee..515fea5a3fc4 100644 --- a/arch/s390/include/asm/mmu_context.h +++ b/arch/s390/include/asm/mmu_context.h @@ -21,6 +21,7 @@ static inline int init_new_context(struct task_struct *tsk, INIT_LIST_HEAD(&mm->context.gmap_list); cpumask_clear(&mm->context.cpu_attach_mask); atomic_set(&mm->context.flush_count, 0); + mm->context.gmap_asce = 0; mm->context.flush_mm = 0; #ifdef CONFIG_PGSTE mm->context.alloc_pgste = page_table_allocate_pgste; diff --git a/arch/s390/include/asm/tlbflush.h b/arch/s390/include/asm/tlbflush.h index 490014c48b13..39846100682a 100644 --- a/arch/s390/include/asm/tlbflush.h +++ b/arch/s390/include/asm/tlbflush.h @@ -60,18 +60,25 @@ static inline void __tlb_flush_full(struct mm_struct *mm) preempt_enable(); } -/* - * Flush TLB entries for a specific ASCE on all CPUs. Should never be used - * when more than one asce (e.g. gmap) ran on this mm. - */ -static inline void __tlb_flush_asce(struct mm_struct *mm, unsigned long asce) +static inline void __tlb_flush_mm(struct mm_struct *mm) { + unsigned long gmap_asce; + + /* + * If the machine has IDTE we prefer to do a per mm flush + * on all cpus instead of doing a local flush if the mm + * only ran on the local cpu. + */ preempt_disable(); atomic_inc(&mm->context.flush_count); - if (MACHINE_HAS_IDTE) - __tlb_flush_idte(asce); - else - __tlb_flush_global(); + gmap_asce = READ_ONCE(mm->context.gmap_asce); + if (MACHINE_HAS_IDTE && gmap_asce != -1UL) { + if (gmap_asce) + __tlb_flush_idte(gmap_asce); + __tlb_flush_idte(mm->context.asce); + } else { + __tlb_flush_full(mm); + } /* Reset TLB flush mask */ cpumask_copy(mm_cpumask(mm), &mm->context.cpu_attach_mask); atomic_dec(&mm->context.flush_count); @@ -92,7 +99,7 @@ static inline void __tlb_flush_kernel(void) /* * Flush TLB entries for a specific ASCE on all CPUs. */ -static inline void __tlb_flush_asce(struct mm_struct *mm, unsigned long asce) +static inline void __tlb_flush_mm(struct mm_struct *mm) { __tlb_flush_local(); } @@ -103,19 +110,6 @@ static inline void __tlb_flush_kernel(void) } #endif -static inline void __tlb_flush_mm(struct mm_struct * mm) -{ - /* - * If the machine has IDTE we prefer to do a per mm flush - * on all cpus instead of doing a local flush if the mm - * only ran on the local cpu. - */ - if (MACHINE_HAS_IDTE && list_empty(&mm->context.gmap_list)) - __tlb_flush_asce(mm, mm->context.asce); - else - __tlb_flush_full(mm); -} - static inline void __tlb_flush_mm_lazy(struct mm_struct * mm) { if (mm->context.flush_mm) { diff --git a/arch/s390/mm/gmap.c b/arch/s390/mm/gmap.c index 2ce6bb3bab32..3ba622702ce4 100644 --- a/arch/s390/mm/gmap.c +++ b/arch/s390/mm/gmap.c @@ -94,6 +94,7 @@ out: struct gmap *gmap_create(struct mm_struct *mm, unsigned long limit) { struct gmap *gmap; + unsigned long gmap_asce; gmap = gmap_alloc(limit); if (!gmap) @@ -101,6 +102,11 @@ struct gmap *gmap_create(struct mm_struct *mm, unsigned long limit) gmap->mm = mm; spin_lock(&mm->context.gmap_lock); list_add_rcu(&gmap->list, &mm->context.gmap_list); + if (list_is_singular(&mm->context.gmap_list)) + gmap_asce = gmap->asce; + else + gmap_asce = -1UL; + WRITE_ONCE(mm->context.gmap_asce, gmap_asce); spin_unlock(&mm->context.gmap_lock); return gmap; } @@ -230,6 +236,7 @@ EXPORT_SYMBOL_GPL(gmap_put); void gmap_remove(struct gmap *gmap) { struct gmap *sg, *next; + unsigned long gmap_asce; /* Remove all shadow gmaps linked to this gmap */ if (!list_empty(&gmap->children)) { @@ -243,6 +250,14 @@ void gmap_remove(struct gmap *gmap) /* Remove gmap from the pre-mm list */ spin_lock(&gmap->mm->context.gmap_lock); list_del_rcu(&gmap->list); + if (list_empty(&gmap->mm->context.gmap_list)) + gmap_asce = 0; + else if (list_is_singular(&gmap->mm->context.gmap_list)) + gmap_asce = list_first_entry(&gmap->mm->context.gmap_list, + struct gmap, list)->asce; + else + gmap_asce = -1UL; + WRITE_ONCE(gmap->mm->context.gmap_asce, gmap_asce); spin_unlock(&gmap->mm->context.gmap_lock); synchronize_rcu(); /* Put reference */ -- cgit v1.2.3