From e7cc3dd48c7b0a4b2135a5a2145ad43a8192fa6a Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Thu, 13 Nov 2014 13:46:09 +1100 Subject: iommu/amd: use new invalidate_range mmu-notifier MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Make use of the new invalidate_range mmu_notifier call-back and remove the old logic of assigning an empty page-table between invalidate_range_start and invalidate_range_end. Signed-off-by: Joerg Roedel Tested-by: Oded Gabbay Reviewed-by: Andrea Arcangeli Reviewed-by: Jérôme Glisse Cc: Peter Zijlstra Cc: Rik van Riel Cc: Hugh Dickins Cc: Mel Gorman Cc: Johannes Weiner Cc: Jay Cornwall Cc: Suravee Suthikulpanit Cc: Jesse Barnes Cc: David Woodhouse Signed-off-by: Andrew Morton Signed-off-by: Oded Gabbay --- drivers/iommu/amd_iommu_v2.c | 61 +++++++------------------------------------- 1 file changed, 9 insertions(+), 52 deletions(-) (limited to 'drivers/iommu/amd_iommu_v2.c') diff --git a/drivers/iommu/amd_iommu_v2.c b/drivers/iommu/amd_iommu_v2.c index 90d734bbf467..f724bdc50d25 100644 --- a/drivers/iommu/amd_iommu_v2.c +++ b/drivers/iommu/amd_iommu_v2.c @@ -92,13 +92,6 @@ static spinlock_t state_lock; static struct workqueue_struct *iommu_wq; -/* - * Empty page table - Used between - * mmu_notifier_invalidate_range_start and - * mmu_notifier_invalidate_range_end - */ -static u64 *empty_page_table; - static void free_pasid_states(struct device_state *dev_state); static u16 device_id(struct pci_dev *pdev) @@ -418,46 +411,21 @@ static void mn_invalidate_page(struct mmu_notifier *mn, __mn_flush_page(mn, address); } -static void mn_invalidate_range_start(struct mmu_notifier *mn, - struct mm_struct *mm, - unsigned long start, unsigned long end) -{ - struct pasid_state *pasid_state; - struct device_state *dev_state; - unsigned long flags; - - pasid_state = mn_to_state(mn); - dev_state = pasid_state->device_state; - - spin_lock_irqsave(&pasid_state->lock, flags); - if (pasid_state->mmu_notifier_count == 0) { - amd_iommu_domain_set_gcr3(dev_state->domain, - pasid_state->pasid, - __pa(empty_page_table)); - } - pasid_state->mmu_notifier_count += 1; - spin_unlock_irqrestore(&pasid_state->lock, flags); -} - -static void mn_invalidate_range_end(struct mmu_notifier *mn, - struct mm_struct *mm, - unsigned long start, unsigned long end) +static void mn_invalidate_range(struct mmu_notifier *mn, + struct mm_struct *mm, + unsigned long start, unsigned long end) { struct pasid_state *pasid_state; struct device_state *dev_state; - unsigned long flags; pasid_state = mn_to_state(mn); dev_state = pasid_state->device_state; - spin_lock_irqsave(&pasid_state->lock, flags); - pasid_state->mmu_notifier_count -= 1; - if (pasid_state->mmu_notifier_count == 0) { - amd_iommu_domain_set_gcr3(dev_state->domain, - pasid_state->pasid, - __pa(pasid_state->mm->pgd)); - } - spin_unlock_irqrestore(&pasid_state->lock, flags); + if ((start ^ (end - 1)) < PAGE_SIZE) + amd_iommu_flush_page(dev_state->domain, pasid_state->pasid, + start); + else + amd_iommu_flush_tlb(dev_state->domain, pasid_state->pasid); } static void mn_release(struct mmu_notifier *mn, struct mm_struct *mm) @@ -482,8 +450,7 @@ static struct mmu_notifier_ops iommu_mn = { .release = mn_release, .clear_flush_young = mn_clear_flush_young, .invalidate_page = mn_invalidate_page, - .invalidate_range_start = mn_invalidate_range_start, - .invalidate_range_end = mn_invalidate_range_end, + .invalidate_range = mn_invalidate_range, }; static void set_pri_tag_status(struct pasid_state *pasid_state, @@ -954,18 +921,10 @@ static int __init amd_iommu_v2_init(void) if (iommu_wq == NULL) goto out; - ret = -ENOMEM; - empty_page_table = (u64 *)get_zeroed_page(GFP_KERNEL); - if (empty_page_table == NULL) - goto out_destroy_wq; - amd_iommu_register_ppr_notifier(&ppr_nb); return 0; -out_destroy_wq: - destroy_workqueue(iommu_wq); - out: return ret; } @@ -999,8 +958,6 @@ static void __exit amd_iommu_v2_exit(void) } destroy_workqueue(iommu_wq); - - free_page((unsigned long)empty_page_table); } module_init(amd_iommu_v2_init); -- cgit v1.2.3 From a015c1e92639cd65ebb49350abdf5ad15bce4448 Mon Sep 17 00:00:00 2001 From: Oded Gabbay Date: Mon, 10 Nov 2014 10:57:36 +0200 Subject: iommu/amd: fix accounting of device_state This patch fixes a bug in the accounting of the device_state. In the current code, the device_state was put (decremented) too many times, which sometimes lead to the driver getting stuck permanently in put_device_state_wait(). That happen because the device_state->count would go below zero, which is never supposed to happen. The root cause is that the device_state was decremented in put_pasid_state() and put_pasid_state_wait() but also in all the functions that call those functions. Therefore, the device_state was decremented twice in each of these code paths. The fix is to decouple the device_state accounting from the pasid_state accounting - remove the call to put_device_state() from the put_pasid_state() and the put_pasid_state_wait()) Signed-off-by: Oded Gabbay --- drivers/iommu/amd_iommu_v2.c | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) (limited to 'drivers/iommu/amd_iommu_v2.c') diff --git a/drivers/iommu/amd_iommu_v2.c b/drivers/iommu/amd_iommu_v2.c index f724bdc50d25..1e6360e7ae44 100644 --- a/drivers/iommu/amd_iommu_v2.c +++ b/drivers/iommu/amd_iommu_v2.c @@ -272,10 +272,8 @@ static void free_pasid_state(struct pasid_state *pasid_state) static void put_pasid_state(struct pasid_state *pasid_state) { - if (atomic_dec_and_test(&pasid_state->count)) { - put_device_state(pasid_state->device_state); + if (atomic_dec_and_test(&pasid_state->count)) wake_up(&pasid_state->wq); - } } static void put_pasid_state_wait(struct pasid_state *pasid_state) @@ -284,9 +282,7 @@ static void put_pasid_state_wait(struct pasid_state *pasid_state) prepare_to_wait(&pasid_state->wq, &wait, TASK_UNINTERRUPTIBLE); - if (atomic_dec_and_test(&pasid_state->count)) - put_device_state(pasid_state->device_state); - else + if (!atomic_dec_and_test(&pasid_state->count)) schedule(); finish_wait(&pasid_state->wq, &wait); -- cgit v1.2.3