diff options
Diffstat (limited to 'drivers/iommu')
| -rw-r--r-- | drivers/iommu/Kconfig | 2 | ||||
| -rw-r--r-- | drivers/iommu/amd_iommu.c | 499 | ||||
| -rw-r--r-- | drivers/iommu/amd_iommu_init.c | 184 | ||||
| -rw-r--r-- | drivers/iommu/amd_iommu_proto.h | 7 | ||||
| -rw-r--r-- | drivers/iommu/amd_iommu_types.h | 149 | ||||
| -rw-r--r-- | drivers/iommu/arm-smmu-v3.c | 568 | ||||
| -rw-r--r-- | drivers/iommu/arm-smmu.c | 1027 | ||||
| -rw-r--r-- | drivers/iommu/dma-iommu.c | 161 | ||||
| -rw-r--r-- | drivers/iommu/exynos-iommu.c | 4 | ||||
| -rw-r--r-- | drivers/iommu/intel-iommu.c | 103 | ||||
| -rw-r--r-- | drivers/iommu/io-pgtable-arm-v7s.c | 8 | ||||
| -rw-r--r-- | drivers/iommu/iommu.c | 58 | ||||
| -rw-r--r-- | drivers/iommu/ipmmu-vmsa.c | 2 | ||||
| -rw-r--r-- | drivers/iommu/of_iommu.c | 52 |
14 files changed, 1884 insertions, 940 deletions
diff --git a/drivers/iommu/Kconfig b/drivers/iommu/Kconfig index d432ca828472..8ee54d71c7eb 100644 --- a/drivers/iommu/Kconfig +++ b/drivers/iommu/Kconfig @@ -309,7 +309,7 @@ config ARM_SMMU config ARM_SMMU_V3 bool "ARM Ltd. System MMU Version 3 (SMMUv3) Support" - depends on ARM64 && PCI + depends on ARM64 select IOMMU_API select IOMMU_IO_PGTABLE_LPAE select GENERIC_MSI_IRQ_DOMAIN diff --git a/drivers/iommu/amd_iommu.c b/drivers/iommu/amd_iommu.c index 96de97a46079..a2479d0df292 100644 --- a/drivers/iommu/amd_iommu.c +++ b/drivers/iommu/amd_iommu.c @@ -103,7 +103,7 @@ struct flush_queue { struct flush_queue_entry *entries; }; -DEFINE_PER_CPU(struct flush_queue, flush_queue); +static DEFINE_PER_CPU(struct flush_queue, flush_queue); static atomic_t queue_timer_on; static struct timer_list queue_timer; @@ -137,6 +137,7 @@ struct iommu_dev_data { bool pri_tlp; /* PASID TLB required for PPR completions */ u32 errata; /* Bitmap for errata to apply */ + bool use_vapic; /* Enable device to use vapic mode */ }; /* @@ -707,14 +708,74 @@ static void iommu_poll_ppr_log(struct amd_iommu *iommu) } } +#ifdef CONFIG_IRQ_REMAP +static int (*iommu_ga_log_notifier)(u32); + +int amd_iommu_register_ga_log_notifier(int (*notifier)(u32)) +{ + iommu_ga_log_notifier = notifier; + + return 0; +} +EXPORT_SYMBOL(amd_iommu_register_ga_log_notifier); + +static void iommu_poll_ga_log(struct amd_iommu *iommu) +{ + u32 head, tail, cnt = 0; + + if (iommu->ga_log == NULL) + return; + + head = readl(iommu->mmio_base + MMIO_GA_HEAD_OFFSET); + tail = readl(iommu->mmio_base + MMIO_GA_TAIL_OFFSET); + + while (head != tail) { + volatile u64 *raw; + u64 log_entry; + + raw = (u64 *)(iommu->ga_log + head); + cnt++; + + /* Avoid memcpy function-call overhead */ + log_entry = *raw; + + /* Update head pointer of hardware ring-buffer */ + head = (head + GA_ENTRY_SIZE) % GA_LOG_SIZE; + writel(head, iommu->mmio_base + MMIO_GA_HEAD_OFFSET); + + /* Handle GA entry */ + switch (GA_REQ_TYPE(log_entry)) { + case GA_GUEST_NR: + if (!iommu_ga_log_notifier) + break; + + pr_debug("AMD-Vi: %s: devid=%#x, ga_tag=%#x\n", + __func__, GA_DEVID(log_entry), + GA_TAG(log_entry)); + + if (iommu_ga_log_notifier(GA_TAG(log_entry)) != 0) + pr_err("AMD-Vi: GA log notifier failed.\n"); + break; + default: + break; + } + } +} +#endif /* CONFIG_IRQ_REMAP */ + +#define AMD_IOMMU_INT_MASK \ + (MMIO_STATUS_EVT_INT_MASK | \ + MMIO_STATUS_PPR_INT_MASK | \ + MMIO_STATUS_GALOG_INT_MASK) + irqreturn_t amd_iommu_int_thread(int irq, void *data) { struct amd_iommu *iommu = (struct amd_iommu *) data; u32 status = readl(iommu->mmio_base + MMIO_STATUS_OFFSET); - while (status & (MMIO_STATUS_EVT_INT_MASK | MMIO_STATUS_PPR_INT_MASK)) { - /* Enable EVT and PPR interrupts again */ - writel((MMIO_STATUS_EVT_INT_MASK | MMIO_STATUS_PPR_INT_MASK), + while (status & AMD_IOMMU_INT_MASK) { + /* Enable EVT and PPR and GA interrupts again */ + writel(AMD_IOMMU_INT_MASK, iommu->mmio_base + MMIO_STATUS_OFFSET); if (status & MMIO_STATUS_EVT_INT_MASK) { @@ -727,6 +788,13 @@ irqreturn_t amd_iommu_int_thread(int irq, void *data) iommu_poll_ppr_log(iommu); } +#ifdef CONFIG_IRQ_REMAP + if (status & MMIO_STATUS_GALOG_INT_MASK) { + pr_devel("AMD-Vi: Processing IOMMU GA Log\n"); + iommu_poll_ga_log(iommu); + } +#endif + /* * Hardware bug: ERBT1312 * When re-enabling interrupt (by writing 1 @@ -1274,7 +1342,8 @@ static u64 *alloc_pte(struct protection_domain *domain, __npte = PM_LEVEL_PDE(level, virt_to_phys(page)); - if (cmpxchg64(pte, __pte, __npte)) { + /* pte could have been changed somewhere. */ + if (cmpxchg64(pte, __pte, __npte) != __pte) { free_page((unsigned long)page); continue; } @@ -1654,6 +1723,9 @@ static void dma_ops_domain_free(struct dma_ops_domain *dom) free_pagetable(&dom->domain); + if (dom->domain.id) + domain_id_free(dom->domain.id); + kfree(dom); } @@ -2948,6 +3020,12 @@ static void amd_iommu_detach_device(struct iommu_domain *dom, if (!iommu) return; +#ifdef CONFIG_IRQ_REMAP + if (AMD_IOMMU_GUEST_IR_VAPIC(amd_iommu_guest_ir) && + (dom->type == IOMMU_DOMAIN_UNMANAGED)) + dev_data->use_vapic = 0; +#endif + iommu_completion_wait(iommu); } @@ -2973,6 +3051,15 @@ static int amd_iommu_attach_device(struct iommu_domain *dom, ret = attach_device(dev, domain); +#ifdef CONFIG_IRQ_REMAP + if (AMD_IOMMU_GUEST_IR_VAPIC(amd_iommu_guest_ir)) { + if (dom->type == IOMMU_DOMAIN_UNMANAGED) + dev_data->use_vapic = 1; + else + dev_data->use_vapic = 0; + } +#endif + iommu_completion_wait(iommu); return ret; @@ -3511,34 +3598,6 @@ EXPORT_SYMBOL(amd_iommu_device_info); * *****************************************************************************/ -union irte { - u32 val; - struct { - u32 valid : 1, - no_fault : 1, - int_type : 3, - rq_eoi : 1, - dm : 1, - rsvd_1 : 1, - destination : 8, - vector : 8, - rsvd_2 : 8; - } fields; -}; - -struct irq_2_irte { - u16 devid; /* Device ID for IRTE table */ - u16 index; /* Index into IRTE table*/ -}; - -struct amd_ir_data { - struct irq_2_irte irq_2_irte; - union irte irte_entry; - union { - struct msi_msg msi_entry; - }; -}; - static struct irq_chip amd_ir_chip; #define DTE_IRQ_PHYS_ADDR_MASK (((1ULL << 45)-1) << 6) @@ -3560,8 +3619,6 @@ static void set_dte_irq_entry(u16 devid, struct irq_remap_table *table) amd_iommu_dev_table[devid].data[2] = dte; } -#define IRTE_ALLOCATED (~1U) - static struct irq_remap_table *get_irq_table(u16 devid, bool ioapic) { struct irq_remap_table *table = NULL; @@ -3577,7 +3634,7 @@ static struct irq_remap_table *get_irq_table(u16 devid, bool ioapic) table = irq_lookup_table[devid]; if (table) - goto out; + goto out_unlock; alias = amd_iommu_alias_table[devid]; table = irq_lookup_table[alias]; @@ -3591,7 +3648,7 @@ static struct irq_remap_table *get_irq_table(u16 devid, bool ioapic) /* Nothing there yet, allocate new irq remapping table */ table = kzalloc(sizeof(*table), GFP_ATOMIC); if (!table) - goto out; + goto out_unlock; /* Initialize table spin-lock */ spin_lock_init(&table->lock); @@ -3604,16 +3661,21 @@ static struct irq_remap_table *get_irq_table(u16 devid, bool ioapic) if (!table->table) { kfree(table); table = NULL; - goto out; + goto out_unlock; } - memset(table->table, 0, MAX_IRQS_PER_TABLE * sizeof(u32)); + if (!AMD_IOMMU_GUEST_IR_GA(amd_iommu_guest_ir)) + memset(table->table, 0, + MAX_IRQS_PER_TABLE * sizeof(u32)); + else + memset(table->table, 0, + (MAX_IRQS_PER_TABLE * (sizeof(u64) * 2))); if (ioapic) { int i; for (i = 0; i < 32; ++i) - table->table[i] = IRTE_ALLOCATED; + iommu->irte_ops->set_allocated(table, i); } irq_lookup_table[devid] = table; @@ -3639,6 +3701,10 @@ static int alloc_irq_index(u16 devid, int count) struct irq_remap_table *table; unsigned long flags; int index, c; + struct amd_iommu *iommu = amd_iommu_rlookup_table[devid]; + + if (!iommu) + return -ENODEV; table = get_irq_table(devid, false); if (!table) @@ -3650,14 +3716,14 @@ static int alloc_irq_index(u16 devid, int count) for (c = 0, index = table->min_index; index < MAX_IRQS_PER_TABLE; ++index) { - if (table->table[index] == 0) + if (!iommu->irte_ops->is_allocated(table, index)) c += 1; else c = 0; if (c == count) { for (; c != 0; --c) - table->table[index - c + 1] = IRTE_ALLOCATED; + iommu->irte_ops->set_allocated(table, index - c + 1); index -= count - 1; goto out; @@ -3672,7 +3738,42 @@ out: return index; } -static int modify_irte(u16 devid, int index, union irte irte) +static int modify_irte_ga(u16 devid, int index, struct irte_ga *irte, + struct amd_ir_data *data) +{ + struct irq_remap_table *table; + struct amd_iommu *iommu; + unsigned long flags; + struct irte_ga *entry; + + iommu = amd_iommu_rlookup_table[devid]; + if (iommu == NULL) + return -EINVAL; + + table = get_irq_table(devid, false); + if (!table) + return -ENOMEM; + + spin_lock_irqsave(&table->lock, flags); + + entry = (struct irte_ga *)table->table; + entry = &entry[index]; + entry->lo.fields_remap.valid = 0; + entry->hi.val = irte->hi.val; + entry->lo.val = irte->lo.val; + entry->lo.fields_remap.valid = 1; + if (data) + data->ref = entry; + + spin_unlock_irqrestore(&table->lock, flags); + + iommu_flush_irt(iommu, devid); + iommu_completion_wait(iommu); + + return 0; +} + +static int modify_irte(u16 devid, int index, union irte *irte) { struct irq_remap_table *table; struct amd_iommu *iommu; @@ -3687,7 +3788,7 @@ static int modify_irte(u16 devid, int index, union irte irte) return -ENOMEM; spin_lock_irqsave(&table->lock, flags); - table->table[index] = irte.val; + table->table[index] = irte->val; spin_unlock_irqrestore(&table->lock, flags); iommu_flush_irt(iommu, devid); @@ -3711,13 +3812,146 @@ static void free_irte(u16 devid, int index) return; spin_lock_irqsave(&table->lock, flags); - table->table[index] = 0; + iommu->irte_ops->clear_allocated(table, index); spin_unlock_irqrestore(&table->lock, flags); iommu_flush_irt(iommu, devid); iommu_completion_wait(iommu); } +static void irte_prepare(void *entry, + u32 delivery_mode, u32 dest_mode, + u8 vector, u32 dest_apicid, int devid) +{ + union irte *irte = (union irte *) entry; + + irte->val = 0; + irte->fields.vector = vector; + irte->fields.int_type = delivery_mode; + irte->fields.destination = dest_apicid; + irte->fields.dm = dest_mode; + irte->fields.valid = 1; +} + +static void irte_ga_prepare(void *entry, + u32 delivery_mode, u32 dest_mode, + u8 vector, u32 dest_apicid, int devid) +{ + struct irte_ga *irte = (struct irte_ga *) entry; + struct iommu_dev_data *dev_data = search_dev_data(devid); + + irte->lo.val = 0; + irte->hi.val = 0; + irte->lo.fields_remap.guest_mode = dev_data ? dev_data->use_vapic : 0; + irte->lo.fields_remap.int_type = delivery_mode; + irte->lo.fields_remap.dm = dest_mode; + irte->hi.fields.vector = vector; + irte->lo.fields_remap.destination = dest_apicid; + irte->lo.fields_remap.valid = 1; +} + +static void irte_activate(void *entry, u16 devid, u16 index) +{ + union irte *irte = (union irte *) entry; + + irte->fields.valid = 1; + modify_irte(devid, index, irte); +} + +static void irte_ga_activate(void *entry, u16 devid, u16 index) +{ + struct irte_ga *irte = (struct irte_ga *) entry; + + irte->lo.fields_remap.valid = 1; + modify_irte_ga(devid, index, irte, NULL); +} + +static void irte_deactivate(void *entry, u16 devid, u16 index) +{ + union irte *irte = (union irte *) entry; + + irte->fields.valid = 0; + modify_irte(devid, index, irte); +} + +static void irte_ga_deactivate(void *entry, u16 devid, u16 index) +{ + struct irte_ga *irte = (struct irte_ga *) entry; + + irte->lo.fields_remap.valid = 0; + modify_irte_ga(devid, index, irte, NULL); +} + +static void irte_set_affinity(void *entry, u16 devid, u16 index, + u8 vector, u32 dest_apicid) +{ + union irte *irte = (union irte *) entry; + + irte->fields.vector = vector; + irte->fields.destination = dest_apicid; + modify_irte(devid, index, irte); +} + +static void irte_ga_set_affinity(void *entry, u16 devid, u16 index, + u8 vector, u32 dest_apicid) +{ + struct irte_ga *irte = (struct irte_ga *) entry; + struct iommu_dev_data *dev_data = search_dev_data(devid); + + if (!dev_data || !dev_data->use_vapic) { + irte->hi.fields.vector = vector; + irte->lo.fields_remap.destination = dest_apicid; + irte->lo.fields_remap.guest_mode = 0; + modify_irte_ga(devid, index, irte, NULL); + } +} + +#define IRTE_ALLOCATED (~1U) +static void irte_set_allocated(struct irq_remap_table *table, int index) +{ + table->table[index] = IRTE_ALLOCATED; +} + +static void irte_ga_set_allocated(struct irq_remap_table *table, int index) +{ + struct irte_ga *ptr = (struct irte_ga *)table->table; + struct irte_ga *irte = &ptr[index]; + + memset(&irte->lo.val, 0, sizeof(u64)); + memset(&irte->hi.val, 0, sizeof(u64)); + irte->hi.fields.vector = 0xff; +} + +static bool irte_is_allocated(struct irq_remap_table *table, int index) +{ + union irte *ptr = (union irte *)table->table; + union irte *irte = &ptr[index]; + + return irte->val != 0; +} + +static bool irte_ga_is_allocated(struct irq_remap_table *table, int index) +{ + struct irte_ga *ptr = (struct irte_ga *)table->table; + struct irte_ga *irte = &ptr[index]; + + return irte->hi.fields.vector != 0; +} + +static void irte_clear_allocated(struct irq_remap_table *table, int index) +{ + table->table[index] = 0; +} + +static void irte_ga_clear_allocated(struct irq_remap_table *table, int index) +{ + struct irte_ga *ptr = (struct irte_ga *)table->table; + struct irte_ga *irte = &ptr[index]; + + memset(&irte->lo.val, 0, sizeof(u64)); + memset(&irte->hi.val, 0, sizeof(u64)); +} + static int get_devid(struct irq_alloc_info *info) { int devid = -1; @@ -3802,19 +4036,17 @@ static void irq_remapping_prepare_irte(struct amd_ir_data *data, { struct irq_2_irte *irte_info = &data->irq_2_irte; struct msi_msg *msg = &data->msi_entry; - union irte *irte = &data->irte_entry; struct IO_APIC_route_entry *entry; + struct amd_iommu *iommu = amd_iommu_rlookup_table[devid]; + + if (!iommu) + return; data->irq_2_irte.devid = devid; data->irq_2_irte.index = index + sub_handle; - - /* Setup IRTE for IOMMU */ - irte->val = 0; - irte->fields.vector = irq_cfg->vector; - irte->fields.int_type = apic->irq_delivery_mode; - irte->fields.destination = irq_cfg->dest_apicid; - irte->fields.dm = apic->irq_dest_mode; - irte->fields.valid = 1; + iommu->irte_ops->prepare(data->entry, apic->irq_delivery_mode, + apic->irq_dest_mode, irq_cfg->vector, + irq_cfg->dest_apicid, devid); switch (info->type) { case X86_IRQ_ALLOC_TYPE_IOAPIC: @@ -3845,12 +4077,32 @@ static void irq_remapping_prepare_irte(struct amd_ir_data *data, } } +struct amd_irte_ops irte_32_ops = { + .prepare = irte_prepare, + .activate = irte_activate, + .deactivate = irte_deactivate, + .set_affinity = irte_set_affinity, + .set_allocated = irte_set_allocated, + .is_allocated = irte_is_allocated, + .clear_allocated = irte_clear_allocated, +}; + +struct amd_irte_ops irte_128_ops = { + .prepare = irte_ga_prepare, + .activate = irte_ga_activate, + .deactivate = irte_ga_deactivate, + .set_affinity = irte_ga_set_affinity, + .set_allocated = irte_ga_set_allocated, + .is_allocated = irte_ga_is_allocated, + .clear_allocated = irte_ga_clear_allocated, +}; + static int irq_remapping_alloc(struct irq_domain *domain, unsigned int virq, unsigned int nr_irqs, void *arg) { struct irq_alloc_info *info = arg; struct irq_data *irq_data; - struct amd_ir_data *data; + struct amd_ir_data *data = NULL; struct irq_cfg *cfg; int i, ret, devid; int index = -1; @@ -3886,6 +4138,7 @@ static int irq_remapping_alloc(struct irq_domain *domain, unsigned int virq, } if (index < 0) { pr_warn("Failed to allocate IRTE\n"); + ret = index; goto out_free_parent; } @@ -3902,6 +4155,16 @@ static int irq_remapping_alloc(struct irq_domain *domain, unsigned int virq, if (!data) goto out_free_data; + if (!AMD_IOMMU_GUEST_IR_GA(amd_iommu_guest_ir)) + data->entry = kzalloc(sizeof(union irte), GFP_KERNEL); + else + data->entry = kzalloc(sizeof(struct irte_ga), + GFP_KERNEL); + if (!data->entry) { + kfree(data); + goto out_free_data; + } + irq_data->hwirq = (devid << 16) + i; irq_data->chip_data = data; irq_data->chip = &amd_ir_chip; @@ -3938,6 +4201,7 @@ static void irq_remapping_free(struct irq_domain *domain, unsigned int virq, data = irq_data->chip_data; irte_info = &data->irq_2_irte; free_irte(irte_info->devid, irte_info->index); + kfree(data->entry); kfree(data); } } @@ -3949,8 +4213,11 @@ static void irq_remapping_activate(struct irq_domain *domain, { struct amd_ir_data *data = irq_data->chip_data; struct irq_2_irte *irte_info = &data->irq_2_irte; + struct amd_iommu *iommu = amd_iommu_rlookup_table[irte_info->devid]; - modify_irte(irte_info->devid, irte_info->index, data->irte_entry); + if (iommu) + iommu->irte_ops->activate(data->entry, irte_info->devid, + irte_info->index); } static void irq_remapping_deactivate(struct irq_domain *domain, @@ -3958,10 +4225,11 @@ static void irq_remapping_deactivate(struct irq_domain *domain, { struct amd_ir_data *data = irq_data->chip_data; struct irq_2_irte *irte_info = &data->irq_2_irte; - union irte entry; + struct amd_iommu *iommu = amd_iommu_rlookup_table[irte_info->devid]; - entry.val = 0; - modify_irte(irte_info->devid, irte_info->index, data->irte_entry); + if (iommu) + iommu->irte_ops->deactivate(data->entry, irte_info->devid, + irte_info->index); } static struct irq_domain_ops amd_ir_domain_ops = { @@ -3971,6 +4239,70 @@ static struct irq_domain_ops amd_ir_domain_ops = { .deactivate = irq_remapping_deactivate, }; +static int amd_ir_set_vcpu_affinity(struct irq_data *data, void *vcpu_info) +{ + struct amd_iommu *iommu; + struct amd_iommu_pi_data *pi_data = vcpu_info; + struct vcpu_data *vcpu_pi_info = pi_data->vcpu_data; + struct amd_ir_data *ir_data = data->chip_data; + struct irte_ga *irte = (struct irte_ga *) ir_data->entry; + struct irq_2_irte *irte_info = &ir_data->irq_2_irte; + struct iommu_dev_data *dev_data = search_dev_data(irte_info->devid); + + /* Note: + * This device has never been set up for guest mode. + * we should not modify the IRTE + */ + if (!dev_data || !dev_data->use_vapic) + return 0; + + pi_data->ir_data = ir_data; + + /* Note: + * SVM tries to set up for VAPIC mode, but we are in + * legacy mode. So, we force legacy mode instead. + */ + if (!AMD_IOMMU_GUEST_IR_VAPIC(amd_iommu_guest_ir)) { + pr_debug("AMD-Vi: %s: Fall back to using intr legacy remap\n", + __func__); + pi_data->is_guest_mode = false; + } + + iommu = amd_iommu_rlookup_table[irte_info->devid]; + if (iommu == NULL) + return -EINVAL; + + pi_data->prev_ga_tag = ir_data->cached_ga_tag; + if (pi_data->is_guest_mode) { + /* Setting */ + irte->hi.fields.ga_root_ptr = (pi_data->base >> 12); + irte->hi.fields.vector = vcpu_pi_info->vector; + irte->lo.fields_vapic.guest_mode = 1; + irte->lo.fields_vapic.ga_tag = pi_data->ga_tag; + + ir_data->cached_ga_tag = pi_data->ga_tag; + } else { + /* Un-Setting */ + struct irq_cfg *cfg = irqd_cfg(data); + + irte->hi.val = 0; + irte->lo.val = 0; + irte->hi.fields.vector = cfg->vector; + irte->lo.fields_remap.guest_mode = 0; + irte->lo.fields_remap.destination = cfg->dest_apicid; + irte->lo.fields_remap.int_type = apic->irq_delivery_mode; + irte->lo.fields_remap.dm = apic->irq_dest_mode; + + /* + * This communicates the ga_tag back to the caller + * so that it can do all the necessary clean up. + */ + ir_data->cached_ga_tag = 0; + } + + return modify_irte_ga(irte_info->devid, irte_info->index, irte, ir_data); +} + static int amd_ir_set_affinity(struct irq_data *data, const struct cpumask *mask, bool force) { @@ -3978,8 +4310,12 @@ static int amd_ir_set_affinity(struct irq_data *data, struct irq_2_irte *irte_info = &ir_data->irq_2_irte; struct irq_cfg *cfg = irqd_cfg(data); struct irq_data *parent = data->parent_data; + struct amd_iommu *iommu = amd_iommu_rlookup_table[irte_info->devid]; int ret; + if (!iommu) + return -ENODEV; + ret = parent->chip->irq_set_affinity(parent, mask, force); if (ret < 0 || ret == IRQ_SET_MASK_OK_DONE) return ret; @@ -3988,9 +4324,8 @@ static int amd_ir_set_affinity(struct irq_data *data, * Atomically updates the IRTE with the new destination, vector * and flushes the interrupt entry cache. */ - ir_data->irte_entry.fields.vector = cfg->vector; - ir_data->irte_entry.fields.destination = cfg->dest_apicid; - modify_irte(irte_info->devid, irte_info->index, ir_data->irte_entry); + iommu->irte_ops->set_affinity(ir_data->entry, irte_info->devid, + irte_info->index, cfg->vector, cfg->dest_apicid); /* * After this point, all the interrupts will start arriving @@ -4012,6 +4347,7 @@ static void ir_compose_msi_msg(struct irq_data *irq_data, struct msi_msg *msg) static struct irq_chip amd_ir_chip = { .irq_ack = ir_ack_apic_edge, .irq_set_affinity = amd_ir_set_affinity, + .irq_set_vcpu_affinity = amd_ir_set_vcpu_affinity, .irq_compose_msi_msg = ir_compose_msi_msg, }; @@ -4026,4 +4362,43 @@ int amd_iommu_create_irq_domain(struct amd_iommu *iommu) return 0; } + +int amd_iommu_update_ga(int cpu, bool is_run, void *data) +{ + unsigned long flags; + struct amd_iommu *iommu; + struct irq_remap_table *irt; + struct amd_ir_data *ir_data = (struct amd_ir_data *)data; + int devid = ir_data->irq_2_irte.devid; + struct irte_ga *entry = (struct irte_ga *) ir_data->entry; + struct irte_ga *ref = (struct irte_ga *) ir_data->ref; + + if (!AMD_IOMMU_GUEST_IR_VAPIC(amd_iommu_guest_ir) || + !ref || !entry || !entry->lo.fields_vapic.guest_mode) + return 0; + + iommu = amd_iommu_rlookup_table[devid]; + if (!iommu) + return -ENODEV; + + irt = get_irq_table(devid, false); + if (!irt) + return -ENODEV; + + spin_lock_irqsave(&irt->lock, flags); + + if (ref->lo.fields_vapic.guest_mode) { + if (cpu >= 0) + ref->lo.fields_vapic.destination = cpu; + ref->lo.fields_vapic.is_run = is_run; + barrier(); + } + + spin_unlock_irqrestore(&irt->lock, flags); + + iommu_flush_irt(iommu, devid); + iommu_completion_wait(iommu); + return 0; +} +EXPORT_SYMBOL(amd_iommu_update_ga); #endif diff --git a/drivers/iommu/amd_iommu_init.c b/drivers/iommu/amd_iommu_init.c index 59741ead7e15..157e93421fb8 100644 --- a/drivers/iommu/amd_iommu_init.c +++ b/drivers/iommu/amd_iommu_init.c @@ -20,6 +20,7 @@ #include <linux/pci.h> #include <linux/acpi.h> #include <linux/list.h> +#include <linux/bitmap.h> #include <linux/slab.h> #include <linux/syscore_ops.h> #include <linux/interrupt.h> @@ -84,6 +85,7 @@ #define ACPI_DEVFLAG_LINT1 0x80 #define ACPI_DEVFLAG_ATSDIS 0x10000000 +#define LOOP_TIMEOUT 100000 /* * ACPI table definitions * @@ -145,6 +147,8 @@ struct ivmd_header { bool amd_iommu_dump; bool amd_iommu_irq_remap __read_mostly; +int amd_iommu_guest_ir = AMD_IOMMU_GUEST_IR_VAPIC; + static bool amd_iommu_detected; static bool __initdata amd_iommu_disabled; static int amd_iommu_target_ivhd_type; @@ -386,6 +390,10 @@ static void iommu_disable(struct amd_iommu *iommu) iommu_feature_disable(iommu, CONTROL_EVT_INT_EN); iommu_feature_disable(iommu, CONTROL_EVT_LOG_EN); + /* Disable IOMMU GA_LOG */ + iommu_feature_disable(iommu, CONTROL_GALOG_EN); + iommu_feature_disable(iommu, CONTROL_GAINT_EN); + /* Disable IOMMU hardware itself */ iommu_feature_disable(iommu, CONTROL_IOMMU_EN); } @@ -671,6 +679,99 @@ static void __init free_ppr_log(struct amd_iommu *iommu) free_pages((unsigned long)iommu->ppr_log, get_order(PPR_LOG_SIZE)); } +static void free_ga_log(struct amd_iommu *iommu) +{ +#ifdef CONFIG_IRQ_REMAP + if (iommu->ga_log) + free_pages((unsigned long)iommu->ga_log, + get_order(GA_LOG_SIZE)); + if (iommu->ga_log_tail) + free_pages((unsigned long)iommu->ga_log_tail, + get_order(8)); +#endif +} + +static int iommu_ga_log_enable(struct amd_iommu *iommu) +{ +#ifdef CONFIG_IRQ_REMAP + u32 status, i; + + if (!iommu->ga_log) + return -EINVAL; + + status = readl(iommu->mmio_base + MMIO_STATUS_OFFSET); + + /* Check if already running */ + if (status & (MMIO_STATUS_GALOG_RUN_MASK)) + return 0; + + iommu_feature_enable(iommu, CONTROL_GAINT_EN); + iommu_feature_enable(iommu, CONTROL_GALOG_EN); + + for (i = 0; i < LOOP_TIMEOUT; ++i) { + status = readl(iommu->mmio_base + MMIO_STATUS_OFFSET); + if (status & (MMIO_STATUS_GALOG_RUN_MASK)) + break; + } + + if (i >= LOOP_TIMEOUT) + return -EINVAL; +#endif /* CONFIG_IRQ_REMAP */ + return 0; +} + +#ifdef CONFIG_IRQ_REMAP +static int iommu_init_ga_log(struct amd_iommu *iommu) +{ + u64 entry; + + if (!AMD_IOMMU_GUEST_IR_VAPIC(amd_iommu_guest_ir)) + return 0; + + iommu->ga_log = (u8 *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, + get_order(GA_LOG_SIZE)); + if (!iommu->ga_log) + goto err_out; + + iommu->ga_log_tail = (u8 *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, + get_order(8)); + if (!iommu->ga_log_tail) + goto err_out; + + entry = (u64)virt_to_phys(iommu->ga_log) | GA_LOG_SIZE_512; + memcpy_toio(iommu->mmio_base + MMIO_GA_LOG_BASE_OFFSET, + &entry, sizeof(entry)); + entry = ((u64)virt_to_phys(iommu->ga_log) & 0xFFFFFFFFFFFFFULL) & ~7ULL; + memcpy_toio(iommu->mmio_base + MMIO_GA_LOG_TAIL_OFFSET, + &entry, sizeof(entry)); + writel(0x00, iommu->mmio_base + MMIO_GA_HEAD_OFFSET); + writel(0x00, iommu->mmio_base + MMIO_GA_TAIL_OFFSET); + + return 0; +err_out: + free_ga_log(iommu); + return -EINVAL; +} +#endif /* CONFIG_IRQ_REMAP */ + +static int iommu_init_ga(struct amd_iommu *iommu) +{ + int ret = 0; + +#ifdef CONFIG_IRQ_REMAP + /* Note: We have already checked GASup from IVRS table. + * Now, we need to make sure that GAMSup is set. + */ + if (AMD_IOMMU_GUEST_IR_VAPIC(amd_iommu_guest_ir) && + !iommu_feature(iommu, FEATURE_GAM_VAPIC)) + amd_iommu_guest_ir = AMD_IOMMU_GUEST_IR_LEGACY_GA; + + ret = iommu_init_ga_log(iommu); +#endif /* CONFIG_IRQ_REMAP */ + + return ret; +} + static void iommu_enable_gt(struct amd_iommu *iommu) { if (!iommu_feature(iommu, FEATURE_GT)) @@ -1144,6 +1245,7 @@ static void __init free_iommu_one(struct amd_iommu *iommu) free_command_buffer(iommu); free_event_buffer(iommu); free_ppr_log(iommu); + free_ga_log(iommu); iommu_unmap_mmio_space(iommu); } @@ -1258,6 +1360,8 @@ static int __init init_iommu_one(struct amd_iommu *iommu, struct ivhd_header *h) iommu->mmio_phys_end = MMIO_REG_END_OFFSET; else iommu->mmio_phys_end = MMIO_CNTR_CONF_OFFSET; + if (((h->efr_attr & (0x1 << IOMMU_FEAT_GASUP_SHIFT)) == 0)) + amd_iommu_guest_ir = AMD_IOMMU_GUEST_IR_LEGACY; break; case 0x11: case 0x40: @@ -1265,6 +1369,8 @@ static int __init init_iommu_one(struct amd_iommu *iommu, struct ivhd_header *h) iommu->mmio_phys_end = MMIO_REG_END_OFFSET; else iommu->mmio_phys_end = MMIO_CNTR_CONF_OFFSET; + if (((h->efr_reg & (0x1 << IOMMU_EFR_GASUP_SHIFT)) == 0)) + amd_iommu_guest_ir = AMD_IOMMU_GUEST_IR_LEGACY; break; default: return -EINVAL; @@ -1432,6 +1538,7 @@ static int iommu_init_pci(struct amd_iommu *iommu) { int cap_ptr = iommu->cap_ptr; u32 range, misc, low, high; + int ret; iommu->dev = pci_get_bus_and_slot(PCI_BUS_NUM(iommu->devid), iommu->devid & 0xff); @@ -1488,6 +1595,10 @@ static int iommu_init_pci(struct amd_iommu *iommu) if (iommu_feature(iommu, FEATURE_PPR) && alloc_ppr_log(iommu)) return -ENOMEM; + ret = iommu_init_ga(iommu); + if (ret) + return ret; + if (iommu->cap & (1UL << IOMMU_CAP_NPCACHE)) amd_iommu_np_cache = true; @@ -1545,16 +1656,24 @@ static void print_iommu_info(void) dev_name(&iommu->dev->dev), iommu->cap_ptr); if (iommu->cap & (1 << IOMMU_CAP_EFR)) { - pr_info("AMD-Vi: Extended features: "); + pr_info("AMD-Vi: Extended features (%#llx):\n", + iommu->features); for (i = 0; i < ARRAY_SIZE(feat_str); ++i) { if (iommu_feature(iommu, (1ULL << i))) pr_cont(" %s", feat_str[i]); } + + if (iommu->features & FEATURE_GAM_VAPIC) + pr_cont(" GA_vAPIC"); + pr_cont("\n"); } } - if (irq_remapping_enabled) + if (irq_remapping_enabled) { pr_info("AMD-Vi: Interrupt remapping enabled\n"); + if (AMD_IOMMU_GUEST_IR_VAPIC(amd_iommu_guest_ir)) + pr_info("AMD-Vi: virtual APIC enabled\n"); + } } static int __init amd_iommu_init_pci(void) @@ -1645,6 +1764,8 @@ enable_faults: if (iommu->ppr_log != NULL) iommu_feature_enable(iommu, CONTROL_PPFINT_EN); + iommu_ga_log_enable(iommu); + return 0; } @@ -1862,6 +1983,24 @@ static void iommu_apply_resume_quirks(struct amd_iommu *iommu) iommu->stored_addr_lo | 1); } +static void iommu_enable_ga(struct amd_iommu *iommu) +{ +#ifdef CONFIG_IRQ_REMAP + switch (amd_iommu_guest_ir) { + case AMD_IOMMU_GUEST_IR_VAPIC: + iommu_feature_enable(iommu, CONTROL_GAM_EN); + /* Fall through */ + case AMD_IOMMU_GUEST_IR_LEGACY_GA: + iommu_feature_enable(iommu, CONTROL_GA_EN); + iommu->irte_ops = &irte_128_ops; + break; + default: + iommu->irte_ops = &irte_32_ops; + break; + } +#endif +} + /* * This function finally enables all IOMMUs found in the system after * they have been initialized @@ -1877,9 +2016,15 @@ static void early_enable_iommus(void) iommu_enable_command_buffer(iommu); iommu_enable_event_buffer(iommu); iommu_set_exclusion_range(iommu); + iommu_enable_ga(iommu); iommu_enable(iommu); iommu_flush_all_caches(iommu); } + +#ifdef CONFIG_IRQ_REMAP + if (AMD_IOMMU_GUEST_IR_VAPIC(amd_iommu_guest_ir)) + amd_iommu_irq_ops.capability |= (1 << IRQ_POSTING_CAP); +#endif } static void enable_iommus_v2(void) @@ -1905,6 +2050,11 @@ static void disable_iommus(void) for_each_iommu(iommu) iommu_disable(iommu); + +#ifdef CONFIG_IRQ_REMAP + if (AMD_IOMMU_GUEST_IR_VAPIC(amd_iommu_guest_ir)) + amd_iommu_irq_ops.capability &= ~(1 << IRQ_POSTING_CAP); +#endif } /* @@ -2059,7 +2209,7 @@ static int __init early_amd_iommu_init(void) struct acpi_table_header *ivrs_base; acpi_size ivrs_size; acpi_status status; - int i, ret = 0; + int i, remap_cache_sz, ret = 0; if (!amd_iommu_detected) return -ENODEV; @@ -2136,7 +2286,7 @@ static int __init early_amd_iommu_init(void) * never allocate domain 0 because its used as the non-allocated and * error value placeholder */ - amd_iommu_pd_alloc_bitmap[0] = 1; + __set_bit(0, amd_iommu_pd_alloc_bitmap); spin_lock_init(&amd_iommu_pd_lock); @@ -2157,10 +2307,14 @@ static int __init early_amd_iommu_init(void) * remapping tables. */ ret = -ENOMEM; + if (!AMD_IOMMU_GUEST_IR_GA(amd_iommu_guest_ir)) + remap_cache_sz = MAX_IRQS_PER_TABLE * sizeof(u32); + else + remap_cache_sz = MAX_IRQS_PER_TABLE * (sizeof(u64) * 2); amd_iommu_irq_cache = kmem_cache_create("irq_remap_cache", - MAX_IRQS_PER_TABLE * sizeof(u32), - IRQ_TABLE_ALIGNMENT, - 0, NULL); + remap_cache_sz, + IRQ_TABLE_ALIGNMENT, + 0, NULL); if (!amd_iommu_irq_cache) goto out; @@ -2413,6 +2567,21 @@ static int __init parse_amd_iommu_dump(char *str) return 1; } +static int __init parse_amd_iommu_intr(char *str) +{ + for (; *str; ++str) { + if (strncmp(str, "legacy", 6) == 0) { + amd_iommu_guest_ir = AMD_IOMMU_GUEST_IR_LEGACY; + break; + } + if (strncmp(str, "vapic", 5) == 0) { + amd_iommu_guest_ir = AMD_IOMMU_GUEST_IR_VAPIC; + break; + } + } + return 1; +} + static int __init parse_amd_iommu_options(char *str) { for (; *str; ++str) { @@ -2521,6 +2690,7 @@ static int __init parse_ivrs_acpihid(char *str) __setup("amd_iommu_dump", parse_amd_iommu_dump); __setup("amd_iommu=", parse_amd_iommu_options); +__setup("amd_iommu_intr=", parse_amd_iommu_intr); __setup("ivrs_ioapic", parse_ivrs_ioapic); __setup("ivrs_hpet", parse_ivrs_hpet); __setup("ivrs_acpihid", parse_ivrs_acpihid); diff --git a/drivers/iommu/amd_iommu_proto.h b/drivers/iommu/amd_iommu_proto.h index 0bd9eb374462..7eb60c15c582 100644 --- a/drivers/iommu/amd_iommu_proto.h +++ b/drivers/iommu/amd_iommu_proto.h @@ -38,6 +38,7 @@ extern int amd_iommu_enable(void); extern void amd_iommu_disable(void); extern int amd_iommu_reenable(int); extern int amd_iommu_enable_faulting(void); +extern int amd_iommu_guest_ir; /* IOMMUv2 specific functions */ struct iommu_domain; @@ -78,12 +79,6 @@ static inline int amd_iommu_create_irq_domain(struct amd_iommu *iommu) extern int amd_iommu_complete_ppr(struct pci_dev *pdev, int pasid, int status, int tag); -#ifndef CONFIG_AMD_IOMMU_STATS - -static inline void amd_iommu_stats_init(void) { } - -#endif /* !CONFIG_AMD_IOMMU_STATS */ - static inline bool is_rd890_iommu(struct pci_dev *pdev) { return (pdev->vendor == PCI_VENDOR_ID_ATI) && diff --git a/drivers/iommu/amd_iommu_types.h b/drivers/iommu/amd_iommu_types.h index caf5e3822715..fa766eefd590 100644 --- a/drivers/iommu/amd_iommu_types.h +++ b/drivers/iommu/amd_iommu_types.h @@ -22,6 +22,7 @@ #include <linux/types.h> #include <linux/mutex.h> +#include <linux/msi.h> #include <linux/list.h> #include <linux/spinlock.h> #include <linux/pci.h> @@ -69,6 +70,8 @@ #define MMIO_EXCL_LIMIT_OFFSET 0x0028 #define MMIO_EXT_FEATURES 0x0030 #define MMIO_PPR_LOG_OFFSET 0x0038 +#define MMIO_GA_LOG_BASE_OFFSET 0x00e0 +#define MMIO_GA_LOG_TAIL_OFFSET 0x00e8 #define MMIO_CMD_HEAD_OFFSET 0x2000 #define MMIO_CMD_TAIL_OFFSET 0x2008 #define MMIO_EVT_HEAD_OFFSET 0x2010 @@ -76,6 +79,8 @@ #define MMIO_STATUS_OFFSET 0x2020 #define MMIO_PPR_HEAD_OFFSET 0x2030 #define MMIO_PPR_TAIL_OFFSET 0x2038 +#define MMIO_GA_HEAD_OFFSET 0x2040 +#define MMIO_GA_TAIL_OFFSET 0x2048 #define MMIO_CNTR_CONF_OFFSET 0x4000 #define MMIO_CNTR_REG_OFFSET 0x40000 #define MMIO_REG_END_OFFSET 0x80000 @@ -92,6 +97,7 @@ #define FEATURE_GA (1ULL<<7) #define FEATURE_HE (1ULL<<8) #define FEATURE_PC (1ULL<<9) +#define FEATURE_GAM_VAPIC (1ULL<<21) #define FEATURE_PASID_SHIFT 32 #define FEATURE_PASID_MASK (0x1fULL << FEATURE_PASID_SHIFT) @@ -110,6 +116,9 @@ #define MMIO_STATUS_EVT_INT_MASK (1 << 1) #define MMIO_STATUS_COM_WAIT_INT_MASK (1 << 2) #define MMIO_STATUS_PPR_INT_MASK (1 << 6) +#define MMIO_STATUS_GALOG_RUN_MASK (1 << 8) +#define MMIO_STATUS_GALOG_OVERFLOW_MASK (1 << 9) +#define MMIO_STATUS_GALOG_INT_MASK (1 << 10) /* event logging constants */ #define EVENT_ENTRY_SIZE 0x10 @@ -146,6 +155,10 @@ #define CONTROL_PPFINT_EN 0x0eULL #define CONTROL_PPR_EN 0x0fULL #define CONTROL_GT_EN 0x10ULL +#define CONTROL_GA_EN 0x11ULL +#define CONTROL_GAM_EN 0x19ULL +#define CONTROL_GALOG_EN 0x1CULL +#define CONTROL_GAINT_EN 0x1DULL #define CTRL_INV_TO_MASK (7 << CONTROL_INV_TIMEOUT) #define CTRL_INV_TO_NONE 0 @@ -224,6 +237,19 @@ #define PPR_REQ_FAULT 0x01 +/* Constants for GA Log handling */ +#define GA_LOG_ENTRIES 512 +#define GA_LOG_SIZE_SHIFT 56 +#define GA_LOG_SIZE_512 (0x8ULL << GA_LOG_SIZE_SHIFT) +#define GA_ENTRY_SIZE 8 +#define GA_LOG_SIZE (GA_ENTRY_SIZE * GA_LOG_ENTRIES) + +#define GA_TAG(x) (u32)(x & 0xffffffffULL) +#define GA_DEVID(x) (u16)(((x) >> 32) & 0xffffULL) +#define GA_REQ_TYPE(x) (((x) >> 60) & 0xfULL) + +#define GA_GUEST_NR 0x1 + #define PAGE_MODE_NONE 0x00 #define PAGE_MODE_1_LEVEL 0x01 #define PAGE_MODE_2_LEVEL 0x02 @@ -329,6 +355,12 @@ #define IOMMU_CAP_NPCACHE 26 #define IOMMU_CAP_EFR 27 +/* IOMMU Feature Reporting Field (for IVHD type 10h */ +#define IOMMU_FEAT_GASUP_SHIFT 6 + +/* IOMMU Extended Feature Register (EFR) */ +#define IOMMU_EFR_GASUP_SHIFT 7 + #define MAX_DOMAIN_ID 65536 /* Protection domain flags */ @@ -400,6 +432,7 @@ struct amd_iommu_fault { struct iommu_domain; struct irq_domain; +struct amd_irte_ops; /* * This structure contains generic data for IOMMU protection domains @@ -490,6 +523,12 @@ struct amd_iommu { /* Base of the PPR log, if present */ u8 *ppr_log; + /* Base of the GA log, if present */ + u8 *ga_log; + + /* Tail of the GA log, if present */ + u8 *ga_log_tail; + /* true if interrupts for this IOMMU are already enabled */ bool int_enabled; @@ -523,6 +562,8 @@ struct amd_iommu { #ifdef CONFIG_IRQ_REMAP struct irq_domain *ir_domain; struct irq_domain *msi_domain; + + struct amd_irte_ops *irte_ops; #endif }; @@ -681,4 +722,112 @@ static inline int get_hpet_devid(int id) return -EINVAL; } +enum amd_iommu_intr_mode_type { + AMD_IOMMU_GUEST_IR_LEGACY, + + /* This mode is not visible to users. It is used when + * we cannot fully enable vAPIC and fallback to only support + * legacy interrupt remapping via 128-bit IRTE. + */ + AMD_IOMMU_GUEST_IR_LEGACY_GA, + AMD_IOMMU_GUEST_IR_VAPIC, +}; + +#define AMD_IOMMU_GUEST_IR_GA(x) (x == AMD_IOMMU_GUEST_IR_VAPIC || \ + x == AMD_IOMMU_GUEST_IR_LEGACY_GA) + +#define AMD_IOMMU_GUEST_IR_VAPIC(x) (x == AMD_IOMMU_GUEST_IR_VAPIC) + +union irte { + u32 val; + struct { + u32 valid : 1, + no_fault : 1, + int_type : 3, + rq_eoi : 1, + dm : 1, + rsvd_1 : 1, + destination : 8, + vector : 8, + rsvd_2 : 8; + } fields; +}; + +union irte_ga_lo { + u64 val; + + /* For int remapping */ + struct { + u64 valid : 1, + no_fault : 1, + /* ------ */ + int_type : 3, + rq_eoi : 1, + dm : 1, + /* ------ */ + guest_mode : 1, + destination : 8, + rsvd : 48; + } fields_remap; + + /* For guest vAPIC */ + struct { + u64 valid : 1, + no_fault : 1, + /* ------ */ + ga_log_intr : 1, + rsvd1 : 3, + is_run : 1, + /* ------ */ + guest_mode : 1, + destination : 8, + rsvd2 : 16, + ga_tag : 32; + } fields_vapic; +}; + +union irte_ga_hi { + u64 val; + struct { + u64 vector : 8, + rsvd_1 : 4, + ga_root_ptr : 40, + rsvd_2 : 12; + } fields; +}; + +struct irte_ga { + union irte_ga_lo lo; + union irte_ga_hi hi; +}; + +struct irq_2_irte { + u16 devid; /* Device ID for IRTE table */ + u16 index; /* Index into IRTE table*/ +}; + +struct amd_ir_data { + u32 cached_ga_tag; + struct irq_2_irte irq_2_irte; + struct msi_msg msi_entry; + void *entry; /* Pointer to union irte or struct irte_ga */ + void *ref; /* Pointer to the actual irte */ +}; + +struct amd_irte_ops { + void (*prepare)(void *, u32, u32, u8, u32, int); + void (*activate)(void *, u16, u16); + void (*deactivate)(void *, u16, u16); + void (*set_affinity)(void *, u16, u16, u8, u32); + void *(*get)(struct irq_remap_table *, int); + void (*set_allocated)(struct irq_remap_table *, int); + bool (*is_allocated)(struct irq_remap_table *, int); + void (*clear_allocated)(struct irq_remap_table *, int); +}; + +#ifdef CONFIG_IRQ_REMAP +extern struct amd_irte_ops irte_32_ops; +extern struct amd_irte_ops irte_128_ops; +#endif + #endif /* _ASM_X86_AMD_IOMMU_TYPES_H */ diff --git a/drivers/iommu/arm-smmu-v3.c b/drivers/iommu/arm-smmu-v3.c index ce801170d5f2..15c01c3cd540 100644 --- a/drivers/iommu/arm-smmu-v3.c +++ b/drivers/iommu/arm-smmu-v3.c @@ -30,10 +30,13 @@ #include <linux/msi.h> #include <linux/of.h> #include <linux/of_address.h> +#include <linux/of_iommu.h> #include <linux/of_platform.h> #include <linux/pci.h> #include <linux/platform_device.h> +#include <linux/amba/bus.h> + #include "io-pgtable.h" /* MMIO registers */ @@ -123,6 +126,10 @@ #define CR2_RECINVSID (1 << 1) #define CR2_E2H (1 << 0) +#define ARM_SMMU_GBPA 0x44 +#define GBPA_ABORT (1 << 20) +#define GBPA_UPDATE (1 << 31) + #define ARM_SMMU_IRQ_CTRL 0x50 #define IRQ_CTRL_EVTQ_IRQEN (1 << 2) #define IRQ_CTRL_PRIQ_IRQEN (1 << 1) @@ -260,6 +267,9 @@ #define STRTAB_STE_1_SHCFG_INCOMING 1UL #define STRTAB_STE_1_SHCFG_SHIFT 44 +#define STRTAB_STE_1_PRIVCFG_UNPRIV 2UL +#define STRTAB_STE_1_PRIVCFG_SHIFT 48 + #define STRTAB_STE_2_S2VMID_SHIFT 0 #define STRTAB_STE_2_S2VMID_MASK 0xffffUL #define STRTAB_STE_2_VTCR_SHIFT 32 @@ -606,12 +616,9 @@ struct arm_smmu_device { struct arm_smmu_strtab_cfg strtab_cfg; }; -/* SMMU private data for an IOMMU group */ -struct arm_smmu_group { +/* SMMU private data for each master */ +struct arm_smmu_master_data { struct arm_smmu_device *smmu; - struct arm_smmu_domain *domain; - int num_sids; - u32 *sids; struct arm_smmu_strtab_ent ste; }; @@ -713,19 +720,15 @@ static void queue_inc_prod(struct arm_smmu_queue *q) writel(q->prod, q->prod_reg); } -static bool __queue_cons_before(struct arm_smmu_queue *q, u32 until) -{ - if (Q_WRP(q, q->cons) == Q_WRP(q, until)) - return Q_IDX(q, q->cons) < Q_IDX(q, until); - - return Q_IDX(q, q->cons) >= Q_IDX(q, until); -} - -static int queue_poll_cons(struct arm_smmu_queue *q, u32 until, bool wfe) +/* + * Wait for the SMMU to consume items. If drain is true, wait until the queue + * is empty. Otherwise, wait until there is at least one free slot. + */ +static int queue_poll_cons(struct arm_smmu_queue *q, bool drain, bool wfe) { ktime_t timeout = ktime_add_us(ktime_get(), ARM_SMMU_POLL_TIMEOUT_US); - while (queue_sync_cons(q), __queue_cons_before(q, until)) { + while (queue_sync_cons(q), (drain ? !queue_empty(q) : queue_full(q))) { if (ktime_compare(ktime_get(), timeout) > 0) return -ETIMEDOUT; @@ -879,7 +882,7 @@ static void arm_smmu_cmdq_skip_err(struct arm_smmu_device *smmu) * We may have concurrent producers, so we need to be careful * not to touch any of the shadow cmdq state. */ - queue_read(cmd, Q_ENT(q, idx), q->ent_dwords); + queue_read(cmd, Q_ENT(q, cons), q->ent_dwords); dev_err(smmu->dev, "skipping command in error state:\n"); for (i = 0; i < ARRAY_SIZE(cmd); ++i) dev_err(smmu->dev, "\t0x%016llx\n", (unsigned long long)cmd[i]); @@ -890,14 +893,14 @@ static void arm_smmu_cmdq_skip_err(struct arm_smmu_device *smmu) return; } - queue_write(cmd, Q_ENT(q, idx), q->ent_dwords); + queue_write(Q_ENT(q, cons), cmd, q->ent_dwords); } static void arm_smmu_cmdq_issue_cmd(struct arm_smmu_device *smmu, struct arm_smmu_cmdq_ent *ent) { - u32 until; u64 cmd[CMDQ_ENT_DWORDS]; + unsigned long flags; bool wfe = !!(smmu->features & ARM_SMMU_FEAT_SEV); struct arm_smmu_queue *q = &smmu->cmdq.q; @@ -907,20 +910,15 @@ static void arm_smmu_cmdq_issue_cmd(struct arm_smmu_device *smmu, return; } - spin_lock(&smmu->cmdq.lock); - while (until = q->prod + 1, queue_insert_raw(q, cmd) == -ENOSPC) { - /* - * Keep the queue locked, otherwise the producer could wrap - * twice and we could see a future consumer pointer that looks - * like it's behind us. - */ - if (queue_poll_cons(q, until, wfe)) + spin_lock_irqsave(&smmu->cmdq.lock, flags); + while (queue_insert_raw(q, cmd) == -ENOSPC) { + if (queue_poll_cons(q, false, wfe)) dev_err_ratelimited(smmu->dev, "CMDQ timeout\n"); } - if (ent->opcode == CMDQ_OP_CMD_SYNC && queue_poll_cons(q, until, wfe)) + if (ent->opcode == CMDQ_OP_CMD_SYNC && queue_poll_cons(q, true, wfe)) dev_err_ratelimited(smmu->dev, "CMD_SYNC timeout\n"); - spin_unlock(&smmu->cmdq.lock); + spin_unlock_irqrestore(&smmu->cmdq.lock, flags); } /* Context descriptor manipulation functions */ @@ -1034,6 +1032,9 @@ static void arm_smmu_write_strtab_ent(struct arm_smmu_device *smmu, u32 sid, case STRTAB_STE_0_CFG_S2_TRANS: ste_live = true; break; + case STRTAB_STE_0_CFG_ABORT: + if (disable_bypass) + break; default: BUG(); /* STE corruption */ } @@ -1070,7 +1071,9 @@ static void arm_smmu_write_strtab_ent(struct arm_smmu_device *smmu, u32 sid, #ifdef CONFIG_PCI_ATS STRTAB_STE_1_EATS_TRANS << STRTAB_STE_1_EATS_SHIFT | #endif - STRTAB_STE_1_STRW_NSEL1 << STRTAB_STE_1_STRW_SHIFT); + STRTAB_STE_1_STRW_NSEL1 << STRTAB_STE_1_STRW_SHIFT | + STRTAB_STE_1_PRIVCFG_UNPRIV << + STRTAB_STE_1_PRIVCFG_SHIFT); if (smmu->features & ARM_SMMU_FEAT_STALLS) dst[1] |= cpu_to_le64(STRTAB_STE_1_S1STALLD); @@ -1158,36 +1161,66 @@ static irqreturn_t arm_smmu_evtq_thread(int irq, void *dev) struct arm_smmu_queue *q = &smmu->evtq.q; u64 evt[EVTQ_ENT_DWORDS]; - while (!queue_remove_raw(q, evt)) { - u8 id = evt[0] >> EVTQ_0_ID_SHIFT & EVTQ_0_ID_MASK; + do { + while (!queue_remove_raw(q, evt)) { + u8 id = evt[0] >> EVTQ_0_ID_SHIFT & EVTQ_0_ID_MASK; - dev_info(smmu->dev, "event 0x%02x received:\n", id); - for (i = 0; i < ARRAY_SIZE(evt); ++i) - dev_info(smmu->dev, "\t0x%016llx\n", - (unsigned long long)evt[i]); - } + dev_info(smmu->dev, "event 0x%02x received:\n", id); + for (i = 0; i < ARRAY_SIZE(evt); ++i) + dev_info(smmu->dev, "\t0x%016llx\n", + (unsigned long long)evt[i]); + + } + + /* + * Not much we can do on overflow, so scream and pretend we're + * trying harder. + */ + if (queue_sync_prod(q) == -EOVERFLOW) + dev_err(smmu->dev, "EVTQ overflow detected -- events lost\n"); + } while (!queue_empty(q)); /* Sync our overflow flag, as we believe we're up to speed */ q->cons = Q_OVF(q, q->prod) | Q_WRP(q, q->cons) | Q_IDX(q, q->cons); return IRQ_HANDLED; } -static irqreturn_t arm_smmu_evtq_handler(int irq, void *dev) -{ - irqreturn_t ret = IRQ_WAKE_THREAD; - struct arm_smmu_device *smmu = dev; - struct arm_smmu_queue *q = &smmu->evtq.q; +static void arm_smmu_handle_ppr(struct arm_smmu_device *smmu, u64 *evt) +{ + u32 sid, ssid; + u16 grpid; + bool ssv, last; + + sid = evt[0] >> PRIQ_0_SID_SHIFT & PRIQ_0_SID_MASK; + ssv = evt[0] & PRIQ_0_SSID_V; + ssid = ssv ? evt[0] >> PRIQ_0_SSID_SHIFT & PRIQ_0_SSID_MASK : 0; + last = evt[0] & PRIQ_0_PRG_LAST; + grpid = evt[1] >> PRIQ_1_PRG_IDX_SHIFT & PRIQ_1_PRG_IDX_MASK; + + dev_info(smmu->dev, "unexpected PRI request received:\n"); + dev_info(smmu->dev, + "\tsid 0x%08x.0x%05x: [%u%s] %sprivileged %s%s%s access at iova 0x%016llx\n", + sid, ssid, grpid, last ? "L" : "", + evt[0] & PRIQ_0_PERM_PRIV ? "" : "un", + evt[0] & PRIQ_0_PERM_READ ? "R" : "", + evt[0] & PRIQ_0_PERM_WRITE ? "W" : "", + evt[0] & PRIQ_0_PERM_EXEC ? "X" : "", + evt[1] & PRIQ_1_ADDR_MASK << PRIQ_1_ADDR_SHIFT); + + if (last) { + struct arm_smmu_cmdq_ent cmd = { + .opcode = CMDQ_OP_PRI_RESP, + .substream_valid = ssv, + .pri = { + .sid = sid, + .ssid = ssid, + .grpid = grpid, + .resp = PRI_RESP_DENY, + }, + }; - /* - * Not much we can do on overflow, so scream and pretend we're - * trying harder. - */ - if (queue_sync_prod(q) == -EOVERFLOW) - dev_err(smmu->dev, "EVTQ overflow detected -- events lost\n"); - else if (queue_empty(q)) - ret = IRQ_NONE; - - return ret; + arm_smmu_cmdq_issue_cmd(smmu, &cmd); + } } static irqreturn_t arm_smmu_priq_thread(int irq, void *dev) @@ -1196,63 +1229,19 @@ static irqreturn_t arm_smmu_priq_thread(int irq, void *dev) struct arm_smmu_queue *q = &smmu->priq.q; u64 evt[PRIQ_ENT_DWORDS]; - while (!queue_remove_raw(q, evt)) { - u32 sid, ssid; - u16 grpid; - bool ssv, last; + do { + while (!queue_remove_raw(q, evt)) + arm_smmu_handle_ppr(smmu, evt); - sid = evt[0] >> PRIQ_0_SID_SHIFT & PRIQ_0_SID_MASK; - ssv = evt[0] & PRIQ_0_SSID_V; - ssid = ssv ? evt[0] >> PRIQ_0_SSID_SHIFT & PRIQ_0_SSID_MASK : 0; - last = evt[0] & PRIQ_0_PRG_LAST; - grpid = evt[1] >> PRIQ_1_PRG_IDX_SHIFT & PRIQ_1_PRG_IDX_MASK; - - dev_info(smmu->dev, "unexpected PRI request received:\n"); - dev_info(smmu->dev, - "\tsid 0x%08x.0x%05x: [%u%s] %sprivileged %s%s%s access at iova 0x%016llx\n", - sid, ssid, grpid, last ? "L" : "", - evt[0] & PRIQ_0_PERM_PRIV ? "" : "un", - evt[0] & PRIQ_0_PERM_READ ? "R" : "", - evt[0] & PRIQ_0_PERM_WRITE ? "W" : "", - evt[0] & PRIQ_0_PERM_EXEC ? "X" : "", - evt[1] & PRIQ_1_ADDR_MASK << PRIQ_1_ADDR_SHIFT); - - if (last) { - struct arm_smmu_cmdq_ent cmd = { - .opcode = CMDQ_OP_PRI_RESP, - .substream_valid = ssv, - .pri = { - .sid = sid, - .ssid = ssid, - .grpid = grpid, - .resp = PRI_RESP_DENY, - }, - }; - - arm_smmu_cmdq_issue_cmd(smmu, &cmd); - } - } + if (queue_sync_prod(q) == -EOVERFLOW) + dev_err(smmu->dev, "PRIQ overflow detected -- requests lost\n"); + } while (!queue_empty(q)); /* Sync our overflow flag, as we believe we're up to speed */ q->cons = Q_OVF(q, q->prod) | Q_WRP(q, q->cons) | Q_IDX(q, q->cons); return IRQ_HANDLED; } -static irqreturn_t arm_smmu_priq_handler(int irq, void *dev) -{ - irqreturn_t ret = IRQ_WAKE_THREAD; - struct arm_smmu_device *smmu = dev; - struct arm_smmu_queue *q = &smmu->priq.q; - - /* PRIQ overflow indicates a programming error */ - if (queue_sync_prod(q) == -EOVERFLOW) - dev_err(smmu->dev, "PRIQ overflow detected -- requests lost\n"); - else if (queue_empty(q)) - ret = IRQ_NONE; - - return ret; -} - static irqreturn_t arm_smmu_cmdq_sync_handler(int irq, void *dev) { /* We don't actually use CMD_SYNC interrupts for anything */ @@ -1285,15 +1274,11 @@ static irqreturn_t arm_smmu_gerror_handler(int irq, void *dev) if (active & GERROR_MSI_GERROR_ABT_ERR) dev_warn(smmu->dev, "GERROR MSI write aborted\n"); - if (active & GERROR_MSI_PRIQ_ABT_ERR) { + if (active & GERROR_MSI_PRIQ_ABT_ERR) dev_warn(smmu->dev, "PRIQ MSI write aborted\n"); - arm_smmu_priq_handler(irq, smmu->dev); - } - if (active & GERROR_MSI_EVTQ_ABT_ERR) { + if (active & GERROR_MSI_EVTQ_ABT_ERR) dev_warn(smmu->dev, "EVTQ MSI write aborted\n"); - arm_smmu_evtq_handler(irq, smmu->dev); - } if (active & GERROR_MSI_CMDQ_ABT_ERR) { dev_warn(smmu->dev, "CMDQ MSI write aborted\n"); @@ -1566,6 +1551,8 @@ static int arm_smmu_domain_finalise(struct iommu_domain *domain) return -ENOMEM; domain->pgsize_bitmap = pgtbl_cfg.pgsize_bitmap; + domain->geometry.aperture_end = (1UL << ias) - 1; + domain->geometry.force_aperture = true; smmu_domain->pgtbl_ops = pgtbl_ops; ret = finalise_stage_fn(smmu_domain, &pgtbl_cfg); @@ -1575,20 +1562,6 @@ static int arm_smmu_domain_finalise(struct iommu_domain *domain) return ret; } -static struct arm_smmu_group *arm_smmu_group_get(struct device *dev) -{ - struct iommu_group *group; - struct arm_smmu_group *smmu_group; - - group = iommu_group_get(dev); - if (!group) - return NULL; - - smmu_group = iommu_group_get_iommudata(group); - iommu_group_put(group); - return smmu_group; -} - static __le64 *arm_smmu_get_step_for_sid(struct arm_smmu_device *smmu, u32 sid) { __le64 *step; @@ -1611,27 +1584,17 @@ static __le64 *arm_smmu_get_step_for_sid(struct arm_smmu_device *smmu, u32 sid) return step; } -static int arm_smmu_install_ste_for_group(struct arm_smmu_group *smmu_group) +static int arm_smmu_install_ste_for_dev(struct iommu_fwspec *fwspec) { int i; - struct arm_smmu_domain *smmu_domain = smmu_group->domain; - struct arm_smmu_strtab_ent *ste = &smmu_group->ste; - struct arm_smmu_device *smmu = smmu_group->smmu; + struct arm_smmu_master_data *master = fwspec->iommu_priv; + struct arm_smmu_device *smmu = master->smmu; - if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) { - ste->s1_cfg = &smmu_domain->s1_cfg; - ste->s2_cfg = NULL; - arm_smmu_write_ctx_desc(smmu, ste->s1_cfg); - } else { - ste->s1_cfg = NULL; - ste->s2_cfg = &smmu_domain->s2_cfg; - } - - for (i = 0; i < smmu_group->num_sids; ++i) { - u32 sid = smmu_group->sids[i]; + for (i = 0; i < fwspec->num_ids; ++i) { + u32 sid = fwspec->ids[i]; __le64 *step = arm_smmu_get_step_for_sid(smmu, sid); - arm_smmu_write_strtab_ent(smmu, sid, step, ste); + arm_smmu_write_strtab_ent(smmu, sid, step, &master->ste); } return 0; @@ -1639,13 +1602,11 @@ static int arm_smmu_install_ste_for_group(struct arm_smmu_group *smmu_group) static void arm_smmu_detach_dev(struct device *dev) { - struct arm_smmu_group *smmu_group = arm_smmu_group_get(dev); + struct arm_smmu_master_data *master = dev->iommu_fwspec->iommu_priv; - smmu_group->ste.bypass = true; - if (arm_smmu_install_ste_for_group(smmu_group) < 0) + master->ste.bypass = true; + if (arm_smmu_install_ste_for_dev(dev->iommu_fwspec) < 0) dev_warn(dev, "failed to install bypass STE\n"); - - smmu_group->domain = NULL; } static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev) @@ -1653,16 +1614,20 @@ static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev) int ret = 0; struct arm_smmu_device *smmu; struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); - struct arm_smmu_group *smmu_group = arm_smmu_group_get(dev); + struct arm_smmu_master_data *master; + struct arm_smmu_strtab_ent *ste; - if (!smmu_group) + if (!dev->iommu_fwspec) return -ENOENT; + master = dev->iommu_fwspec->iommu_priv; + smmu = master->smmu; + ste = &master->ste; + /* Already attached to a different domain? */ - if (smmu_group->domain && smmu_group->domain != smmu_domain) + if (!ste->bypass) arm_smmu_detach_dev(dev); - smmu = smmu_group->smmu; mutex_lock(&smmu_domain->init_mutex); if (!smmu_domain->smmu) { @@ -1681,21 +1646,21 @@ static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev) goto out_unlock; } - /* Group already attached to this domain? */ - if (smmu_group->domain) - goto out_unlock; - - smmu_group->domain = smmu_domain; + ste->bypass = false; + ste->valid = true; - /* - * FIXME: This should always be "false" once we have IOMMU-backed - * DMA ops for all devices behind the SMMU. - */ - smmu_group->ste.bypass = domain->type == IOMMU_DOMAIN_DMA; + if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) { + ste->s1_cfg = &smmu_domain->s1_cfg; + ste->s2_cfg = NULL; + arm_smmu_write_ctx_desc(smmu, ste->s1_cfg); + } else { + ste->s1_cfg = NULL; + ste->s2_cfg = &smmu_domain->s2_cfg; + } - ret = arm_smmu_install_ste_for_group(smmu_group); + ret = arm_smmu_install_ste_for_dev(dev->iommu_fwspec); if (ret < 0) - smmu_group->domain = NULL; + ste->valid = false; out_unlock: mutex_unlock(&smmu_domain->init_mutex); @@ -1754,40 +1719,19 @@ arm_smmu_iova_to_phys(struct iommu_domain *domain, dma_addr_t iova) return ret; } -static int __arm_smmu_get_pci_sid(struct pci_dev *pdev, u16 alias, void *sidp) -{ - *(u32 *)sidp = alias; - return 0; /* Continue walking */ -} +static struct platform_driver arm_smmu_driver; -static void __arm_smmu_release_pci_iommudata(void *data) +static int arm_smmu_match_node(struct device *dev, void *data) { - kfree(data); + return dev->of_node == data; } -static struct arm_smmu_device *arm_smmu_get_for_pci_dev(struct pci_dev *pdev) +static struct arm_smmu_device *arm_smmu_get_by_node(struct device_node *np) { - struct device_node *of_node; - struct platform_device *smmu_pdev; - struct arm_smmu_device *smmu = NULL; - struct pci_bus *bus = pdev->bus; - - /* Walk up to the root bus */ - while (!pci_is_root_bus(bus)) - bus = bus->parent; - - /* Follow the "iommus" phandle from the host controller */ - of_node = of_parse_phandle(bus->bridge->parent->of_node, "iommus", 0); - if (!of_node) - return NULL; - - /* See if we can find an SMMU corresponding to the phandle */ - smmu_pdev = of_find_device_by_node(of_node); - if (smmu_pdev) - smmu = platform_get_drvdata(smmu_pdev); - - of_node_put(of_node); - return smmu; + struct device *dev = driver_find_device(&arm_smmu_driver.driver, NULL, + np, arm_smmu_match_node); + put_device(dev); + return dev ? dev_get_drvdata(dev) : NULL; } static bool arm_smmu_sid_in_range(struct arm_smmu_device *smmu, u32 sid) @@ -1800,94 +1744,91 @@ static bool arm_smmu_sid_in_range(struct arm_smmu_device *smmu, u32 sid) return sid < limit; } +static struct iommu_ops arm_smmu_ops; + static int arm_smmu_add_device(struct device *dev) { int i, ret; - u32 sid, *sids; - struct pci_dev *pdev; - struct iommu_group *group; - struct arm_smmu_group *smmu_group; struct arm_smmu_device *smmu; + struct arm_smmu_master_data *master; + struct iommu_fwspec *fwspec = dev->iommu_fwspec; + struct iommu_group *group; - /* We only support PCI, for now */ - if (!dev_is_pci(dev)) + if (!fwspec || fwspec->ops != &arm_smmu_ops) return -ENODEV; - - pdev = to_pci_dev(dev); - group = iommu_group_get_for_dev(dev); - if (IS_ERR(group)) - return PTR_ERR(group); - - smmu_group = iommu_group_get_iommudata(group); - if (!smmu_group) { - smmu = arm_smmu_get_for_pci_dev(pdev); - if (!smmu) { - ret = -ENOENT; - goto out_remove_dev; - } - - smmu_group = kzalloc(sizeof(*smmu_group), GFP_KERNEL); - if (!smmu_group) { - ret = -ENOMEM; - goto out_remove_dev; - } - - smmu_group->ste.valid = true; - smmu_group->smmu = smmu; - iommu_group_set_iommudata(group, smmu_group, - __arm_smmu_release_pci_iommudata); + /* + * We _can_ actually withstand dodgy bus code re-calling add_device() + * without an intervening remove_device()/of_xlate() sequence, but + * we're not going to do so quietly... + */ + if (WARN_ON_ONCE(fwspec->iommu_priv)) { + master = fwspec->iommu_priv; + smmu = master->smmu; } else { - smmu = smmu_group->smmu; - } + smmu = arm_smmu_get_by_node(to_of_node(fwspec->iommu_fwnode)); + if (!smmu) + return -ENODEV; + master = kzalloc(sizeof(*master), GFP_KERNEL); + if (!master) + return -ENOMEM; - /* Assume SID == RID until firmware tells us otherwise */ - pci_for_each_dma_alias(pdev, __arm_smmu_get_pci_sid, &sid); - for (i = 0; i < smmu_group->num_sids; ++i) { - /* If we already know about this SID, then we're done */ - if (smmu_group->sids[i] == sid) - goto out_put_group; + master->smmu = smmu; + fwspec->iommu_priv = master; } - /* Check the SID is in range of the SMMU and our stream table */ - if (!arm_smmu_sid_in_range(smmu, sid)) { - ret = -ERANGE; - goto out_remove_dev; - } + /* Check the SIDs are in range of the SMMU and our stream table */ + for (i = 0; i < fwspec->num_ids; i++) { + u32 sid = fwspec->ids[i]; - /* Ensure l2 strtab is initialised */ - if (smmu->features & ARM_SMMU_FEAT_2_LVL_STRTAB) { - ret = arm_smmu_init_l2_strtab(smmu, sid); - if (ret) - goto out_remove_dev; - } + if (!arm_smmu_sid_in_range(smmu, sid)) + return -ERANGE; - /* Resize the SID array for the group */ - smmu_group->num_sids++; - sids = krealloc(smmu_group->sids, smmu_group->num_sids * sizeof(*sids), - GFP_KERNEL); - if (!sids) { - smmu_group->num_sids--; - ret = -ENOMEM; - goto out_remove_dev; + /* Ensure l2 strtab is initialised */ + if (smmu->features & ARM_SMMU_FEAT_2_LVL_STRTAB) { + ret = arm_smmu_init_l2_strtab(smmu, sid); + if (ret) + return ret; + } } - /* Add the new SID */ - sids[smmu_group->num_sids - 1] = sid; - smmu_group->sids = sids; - -out_put_group: - iommu_group_put(group); - return 0; + group = iommu_group_get_for_dev(dev); + if (!IS_ERR(group)) + iommu_group_put(group); -out_remove_dev: - iommu_group_remove_device(dev); - iommu_group_put(group); - return ret; + return PTR_ERR_OR_ZERO(group); } static void arm_smmu_remove_device(struct device *dev) { + struct iommu_fwspec *fwspec = dev->iommu_fwspec; + struct arm_smmu_master_data *master; + + if (!fwspec || fwspec->ops != &arm_smmu_ops) + return; + + master = fwspec->iommu_priv; + if (master && master->ste.valid) + arm_smmu_detach_dev(dev); iommu_group_remove_device(dev); + kfree(master); + iommu_fwspec_free(dev); +} + +static struct iommu_group *arm_smmu_device_group(struct device *dev) +{ + struct iommu_group *group; + + /* + * We don't support devices sharing stream IDs other than PCI RID + * aliases, since the necessary ID-to-device lookup becomes rather + * impractical given a potential sparse 32-bit stream ID space. + */ + if (dev_is_pci(dev)) + group = pci_device_group(dev); + else + group = generic_device_group(dev); + + return group; } static int arm_smmu_domain_get_attr(struct iommu_domain *domain, @@ -1934,6 +1875,11 @@ out_unlock: return ret; } +static int arm_smmu_of_xlate(struct device *dev, struct of_phandle_args *args) +{ + return iommu_fwspec_add_ids(dev, args->args, 1); +} + static struct iommu_ops arm_smmu_ops = { .capable = arm_smmu_capable, .domain_alloc = arm_smmu_domain_alloc, @@ -1945,9 +1891,10 @@ static struct iommu_ops arm_smmu_ops = { .iova_to_phys = arm_smmu_iova_to_phys, .add_device = arm_smmu_add_device, .remove_device = arm_smmu_remove_device, - .device_group = pci_device_group, + .device_group = arm_smmu_device_group, .domain_get_attr = arm_smmu_domain_get_attr, .domain_set_attr = arm_smmu_domain_set_attr, + .of_xlate = arm_smmu_of_xlate, .pgsize_bitmap = -1UL, /* Restricted during device attach */ }; @@ -2148,6 +2095,24 @@ static int arm_smmu_write_reg_sync(struct arm_smmu_device *smmu, u32 val, 1, ARM_SMMU_POLL_TIMEOUT_US); } +/* GBPA is "special" */ +static int arm_smmu_update_gbpa(struct arm_smmu_device *smmu, u32 set, u32 clr) +{ + int ret; + u32 reg, __iomem *gbpa = smmu->base + ARM_SMMU_GBPA; + + ret = readl_relaxed_poll_timeout(gbpa, reg, !(reg & GBPA_UPDATE), + 1, ARM_SMMU_POLL_TIMEOUT_US); + if (ret) + return ret; + + reg &= ~clr; + reg |= set; + writel_relaxed(reg | GBPA_UPDATE, gbpa); + return readl_relaxed_poll_timeout(gbpa, reg, !(reg & GBPA_UPDATE), + 1, ARM_SMMU_POLL_TIMEOUT_US); +} + static void arm_smmu_free_msis(void *data) { struct device *dev = data; @@ -2232,10 +2197,10 @@ static int arm_smmu_setup_irqs(struct arm_smmu_device *smmu) /* Request interrupt lines */ irq = smmu->evtq.q.irq; if (irq) { - ret = devm_request_threaded_irq(smmu->dev, irq, - arm_smmu_evtq_handler, + ret = devm_request_threaded_irq(smmu->dev, irq, NULL, arm_smmu_evtq_thread, - 0, "arm-smmu-v3-evtq", smmu); + IRQF_ONESHOT, + "arm-smmu-v3-evtq", smmu); if (ret < 0) dev_warn(smmu->dev, "failed to enable evtq irq\n"); } @@ -2260,10 +2225,10 @@ static int arm_smmu_setup_irqs(struct arm_smmu_device *smmu) if (smmu->features & ARM_SMMU_FEAT_PRI) { irq = smmu->priq.q.irq; if (irq) { - ret = devm_request_threaded_irq(smmu->dev, irq, - arm_smmu_priq_handler, + ret = devm_request_threaded_irq(smmu->dev, irq, NULL, arm_smmu_priq_thread, - 0, "arm-smmu-v3-priq", + IRQF_ONESHOT, + "arm-smmu-v3-priq", smmu); if (ret < 0) dev_warn(smmu->dev, @@ -2293,7 +2258,7 @@ static int arm_smmu_device_disable(struct arm_smmu_device *smmu) return ret; } -static int arm_smmu_device_reset(struct arm_smmu_device *smmu) +static int arm_smmu_device_reset(struct arm_smmu_device *smmu, bool bypass) { int ret; u32 reg, enables; @@ -2394,8 +2359,17 @@ static int arm_smmu_device_reset(struct arm_smmu_device *smmu) return ret; } - /* Enable the SMMU interface */ - enables |= CR0_SMMUEN; + + /* Enable the SMMU interface, or ensure bypass */ + if (!bypass || disable_bypass) { + enables |= CR0_SMMUEN; + } else { + ret = arm_smmu_update_gbpa(smmu, 0, GBPA_ABORT); + if (ret) { + dev_err(smmu->dev, "GBPA not responding to update\n"); + return ret; + } + } ret = arm_smmu_write_reg_sync(smmu, enables, ARM_SMMU_CR0, ARM_SMMU_CR0ACK); if (ret) { @@ -2594,6 +2568,15 @@ static int arm_smmu_device_dt_probe(struct platform_device *pdev) struct resource *res; struct arm_smmu_device *smmu; struct device *dev = &pdev->dev; + bool bypass = true; + u32 cells; + + if (of_property_read_u32(dev->of_node, "#iommu-cells", &cells)) + dev_err(dev, "missing #iommu-cells property\n"); + else if (cells != 1) + dev_err(dev, "invalid #iommu-cells value (%d)\n", cells); + else + bypass = false; smmu = devm_kzalloc(dev, sizeof(*smmu), GFP_KERNEL); if (!smmu) { @@ -2646,7 +2629,24 @@ static int arm_smmu_device_dt_probe(struct platform_device *pdev) platform_set_drvdata(pdev, smmu); /* Reset the device */ - return arm_smmu_device_reset(smmu); + ret = arm_smmu_device_reset(smmu, bypass); + if (ret) + return ret; + + /* And we're up. Go go go! */ + of_iommu_set_ops(dev->of_node, &arm_smmu_ops); +#ifdef CONFIG_PCI + pci_request_acs(); + ret = bus_set_iommu(&pci_bus_type, &arm_smmu_ops); + if (ret) + return ret; +#endif +#ifdef CONFIG_ARM_AMBA + ret = bus_set_iommu(&amba_bustype, &arm_smmu_ops); + if (ret) + return ret; +#endif + return bus_set_iommu(&platform_bus_type, &arm_smmu_ops); } static int arm_smmu_device_remove(struct platform_device *pdev) @@ -2674,22 +2674,14 @@ static struct platform_driver arm_smmu_driver = { static int __init arm_smmu_init(void) { - struct device_node *np; - int ret; - - np = of_find_matching_node(NULL, arm_smmu_of_match); - if (!np) - return 0; - - of_node_put(np); - - ret = platform_driver_register(&arm_smmu_driver); - if (ret) - return ret; - - pci_request_acs(); + static bool registered; + int ret = 0; - return bus_set_iommu(&pci_bus_type, &arm_smmu_ops); + if (!registered) { + ret = platform_driver_register(&arm_smmu_driver); + registered = !ret; + } + return ret; } static void __exit arm_smmu_exit(void) @@ -2700,6 +2692,20 @@ static void __exit arm_smmu_exit(void) subsys_initcall(arm_smmu_init); module_exit(arm_smmu_exit); +static int __init arm_smmu_of_init(struct device_node *np) +{ + int ret = arm_smmu_init(); + + if (ret) + return ret; + + if (!of_platform_device_create(np, NULL, platform_bus_type.dev_root)) + return -ENODEV; + + return 0; +} +IOMMU_OF_DECLARE(arm_smmuv3, "arm,smmu-v3", arm_smmu_of_init); + MODULE_DESCRIPTION("IOMMU API for ARM architected SMMUv3 implementations"); MODULE_AUTHOR("Will Deacon <will.deacon@arm.com>"); MODULE_LICENSE("GPL v2"); diff --git a/drivers/iommu/arm-smmu.c b/drivers/iommu/arm-smmu.c index 4f49fe29f202..c841eb7a1a74 100644 --- a/drivers/iommu/arm-smmu.c +++ b/drivers/iommu/arm-smmu.c @@ -28,6 +28,7 @@ #define pr_fmt(fmt) "arm-smmu: " fmt +#include <linux/atomic.h> #include <linux/delay.h> #include <linux/dma-iommu.h> #include <linux/dma-mapping.h> @@ -40,6 +41,8 @@ #include <linux/module.h> #include <linux/of.h> #include <linux/of_address.h> +#include <linux/of_device.h> +#include <linux/of_iommu.h> #include <linux/pci.h> #include <linux/platform_device.h> #include <linux/slab.h> @@ -49,15 +52,9 @@ #include "io-pgtable.h" -/* Maximum number of stream IDs assigned to a single device */ -#define MAX_MASTER_STREAMIDS 128 - /* Maximum number of context banks per SMMU */ #define ARM_SMMU_MAX_CBS 128 -/* Maximum number of mapping groups per SMMU */ -#define ARM_SMMU_MAX_SMRS 128 - /* SMMU global address space */ #define ARM_SMMU_GR0(smmu) ((smmu)->base) #define ARM_SMMU_GR1(smmu) ((smmu)->base + (1 << (smmu)->pgshift)) @@ -165,21 +162,27 @@ #define ARM_SMMU_GR0_SMR(n) (0x800 + ((n) << 2)) #define SMR_VALID (1 << 31) #define SMR_MASK_SHIFT 16 -#define SMR_MASK_MASK 0x7fff #define SMR_ID_SHIFT 0 -#define SMR_ID_MASK 0x7fff #define ARM_SMMU_GR0_S2CR(n) (0xc00 + ((n) << 2)) #define S2CR_CBNDX_SHIFT 0 #define S2CR_CBNDX_MASK 0xff #define S2CR_TYPE_SHIFT 16 #define S2CR_TYPE_MASK 0x3 -#define S2CR_TYPE_TRANS (0 << S2CR_TYPE_SHIFT) -#define S2CR_TYPE_BYPASS (1 << S2CR_TYPE_SHIFT) -#define S2CR_TYPE_FAULT (2 << S2CR_TYPE_SHIFT) +enum arm_smmu_s2cr_type { + S2CR_TYPE_TRANS, + S2CR_TYPE_BYPASS, + S2CR_TYPE_FAULT, +}; #define S2CR_PRIVCFG_SHIFT 24 -#define S2CR_PRIVCFG_UNPRIV (2 << S2CR_PRIVCFG_SHIFT) +#define S2CR_PRIVCFG_MASK 0x3 +enum arm_smmu_s2cr_privcfg { + S2CR_PRIVCFG_DEFAULT, + S2CR_PRIVCFG_DIPAN, + S2CR_PRIVCFG_UNPRIV, + S2CR_PRIVCFG_PRIV, +}; /* Context bank attribute registers */ #define ARM_SMMU_GR1_CBAR(n) (0x0 + ((n) << 2)) @@ -217,6 +220,7 @@ #define ARM_SMMU_CB_TTBR0 0x20 #define ARM_SMMU_CB_TTBR1 0x28 #define ARM_SMMU_CB_TTBCR 0x30 +#define ARM_SMMU_CB_CONTEXTIDR 0x34 #define ARM_SMMU_CB_S1_MAIR0 0x38 #define ARM_SMMU_CB_S1_MAIR1 0x3c #define ARM_SMMU_CB_PAR 0x50 @@ -239,7 +243,6 @@ #define SCTLR_AFE (1 << 2) #define SCTLR_TRE (1 << 1) #define SCTLR_M (1 << 0) -#define SCTLR_EAE_SBOP (SCTLR_AFE | SCTLR_TRE) #define ARM_MMU500_ACTLR_CPRE (1 << 1) @@ -296,23 +299,33 @@ enum arm_smmu_implementation { CAVIUM_SMMUV2, }; +struct arm_smmu_s2cr { + struct iommu_group *group; + int count; + enum arm_smmu_s2cr_type type; + enum arm_smmu_s2cr_privcfg privcfg; + u8 cbndx; +}; + +#define s2cr_init_val (struct arm_smmu_s2cr){ \ + .type = disable_bypass ? S2CR_TYPE_FAULT : S2CR_TYPE_BYPASS, \ +} + struct arm_smmu_smr { - u8 idx; u16 mask; u16 id; + bool valid; }; struct arm_smmu_master_cfg { - int num_streamids; - u16 streamids[MAX_MASTER_STREAMIDS]; - struct arm_smmu_smr *smrs; -}; - -struct arm_smmu_master { - struct device_node *of_node; - struct rb_node node; - struct arm_smmu_master_cfg cfg; + struct arm_smmu_device *smmu; + s16 smendx[]; }; +#define INVALID_SMENDX -1 +#define __fwspec_cfg(fw) ((struct arm_smmu_master_cfg *)fw->iommu_priv) +#define fwspec_smmu(fw) (__fwspec_cfg(fw)->smmu) +#define for_each_cfg_sme(fw, i, idx) \ + for (i = 0; idx = __fwspec_cfg(fw)->smendx[i], i < fw->num_ids; ++i) struct arm_smmu_device { struct device *dev; @@ -346,7 +359,11 @@ struct arm_smmu_device { atomic_t irptndx; u32 num_mapping_groups; - DECLARE_BITMAP(smr_map, ARM_SMMU_MAX_SMRS); + u16 streamid_mask; + u16 smr_mask_mask; + struct arm_smmu_smr *smrs; + struct arm_smmu_s2cr *s2crs; + struct mutex stream_map_mutex; unsigned long va_size; unsigned long ipa_size; @@ -357,9 +374,6 @@ struct arm_smmu_device { u32 num_context_irqs; unsigned int *irqs; - struct list_head list; - struct rb_root masters; - u32 cavium_id_base; /* Specific to Cavium */ }; @@ -397,15 +411,6 @@ struct arm_smmu_domain { struct iommu_domain domain; }; -struct arm_smmu_phandle_args { - struct device_node *np; - int args_count; - uint32_t args[MAX_MASTER_STREAMIDS]; -}; - -static DEFINE_SPINLOCK(arm_smmu_devices_lock); -static LIST_HEAD(arm_smmu_devices); - struct arm_smmu_option_prop { u32 opt; const char *prop; @@ -413,6 +418,8 @@ struct arm_smmu_option_prop { static atomic_t cavium_smmu_context_count = ATOMIC_INIT(0); +static bool using_legacy_binding, using_generic_binding; + static struct arm_smmu_option_prop arm_smmu_options[] = { { ARM_SMMU_OPT_SECURE_CFG_ACCESS, "calxeda,smmu-secure-config-access" }, { 0, NULL}, @@ -444,131 +451,86 @@ static struct device_node *dev_get_dev_node(struct device *dev) while (!pci_is_root_bus(bus)) bus = bus->parent; - return bus->bridge->parent->of_node; + return of_node_get(bus->bridge->parent->of_node); } - return dev->of_node; + return of_node_get(dev->of_node); } -static struct arm_smmu_master *find_smmu_master(struct arm_smmu_device *smmu, - struct device_node *dev_node) +static int __arm_smmu_get_pci_sid(struct pci_dev *pdev, u16 alias, void *data) { - struct rb_node *node = smmu->masters.rb_node; - - while (node) { - struct arm_smmu_master *master; - - master = container_of(node, struct arm_smmu_master, node); - - if (dev_node < master->of_node) - node = node->rb_left; - else if (dev_node > master->of_node) - node = node->rb_right; - else - return master; - } - - return NULL; + *((__be32 *)data) = cpu_to_be32(alias); + return 0; /* Continue walking */ } -static struct arm_smmu_master_cfg * -find_smmu_master_cfg(struct device *dev) +static int __find_legacy_master_phandle(struct device *dev, void *data) { - struct arm_smmu_master_cfg *cfg = NULL; - struct iommu_group *group = iommu_group_get(dev); - - if (group) { - cfg = iommu_group_get_iommudata(group); - iommu_group_put(group); - } - - return cfg; + struct of_phandle_iterator *it = *(void **)data; + struct device_node *np = it->node; + int err; + + of_for_each_phandle(it, err, dev->of_node, "mmu-masters", + "#stream-id-cells", 0) + if (it->node == np) { + *(void **)data = dev; + return 1; + } + it->node = np; + return err == -ENOENT ? 0 : err; } -static int insert_smmu_master(struct arm_smmu_device *smmu, - struct arm_smmu_master *master) +static struct platform_driver arm_smmu_driver; +static struct iommu_ops arm_smmu_ops; + +static int arm_smmu_register_legacy_master(struct device *dev, + struct arm_smmu_device **smmu) { - struct rb_node **new, *parent; - - new = &smmu->masters.rb_node; - parent = NULL; - while (*new) { - struct arm_smmu_master *this - = container_of(*new, struct arm_smmu_master, node); - - parent = *new; - if (master->of_node < this->of_node) - new = &((*new)->rb_left); - else if (master->of_node > this->of_node) - new = &((*new)->rb_right); - else - return -EEXIST; + struct device *smmu_dev; + struct device_node *np; + struct of_phandle_iterator it; + void *data = ⁢ + u32 *sids; + __be32 pci_sid; + int err; + + np = dev_get_dev_node(dev); + if (!np || !of_find_property(np, "#stream-id-cells", NULL)) { + of_node_put(np); + return -ENODEV; } - rb_link_node(&master->node, parent, new); - rb_insert_color(&master->node, &smmu->masters); - return 0; -} - -static int register_smmu_master(struct arm_smmu_device *smmu, - struct device *dev, - struct arm_smmu_phandle_args *masterspec) -{ - int i; - struct arm_smmu_master *master; + it.node = np; + err = driver_for_each_device(&arm_smmu_driver.driver, NULL, &data, + __find_legacy_master_phandle); + smmu_dev = data; + of_node_put(np); + if (err == 0) + return -ENODEV; + if (err < 0) + return err; - master = find_smmu_master(smmu, masterspec->np); - if (master) { - dev_err(dev, - "rejecting multiple registrations for master device %s\n", - masterspec->np->name); - return -EBUSY; + if (dev_is_pci(dev)) { + /* "mmu-masters" assumes Stream ID == Requester ID */ + pci_for_each_dma_alias(to_pci_dev(dev), __arm_smmu_get_pci_sid, + &pci_sid); + it.cur = &pci_sid; + it.cur_count = 1; } - if (masterspec->args_count > MAX_MASTER_STREAMIDS) { - dev_err(dev, - "reached maximum number (%d) of stream IDs for master device %s\n", - MAX_MASTER_STREAMIDS, masterspec->np->name); - return -ENOSPC; - } + err = iommu_fwspec_init(dev, &smmu_dev->of_node->fwnode, + &arm_smmu_ops); + if (err) + return err; - master = devm_kzalloc(dev, sizeof(*master), GFP_KERNEL); - if (!master) + sids = kcalloc(it.cur_count, sizeof(*sids), GFP_KERNEL); + if (!sids) return -ENOMEM; - master->of_node = masterspec->np; - master->cfg.num_streamids = masterspec->args_count; - - for (i = 0; i < master->cfg.num_streamids; ++i) { - u16 streamid = masterspec->args[i]; - - if (!(smmu->features & ARM_SMMU_FEAT_STREAM_MATCH) && - (streamid >= smmu->num_mapping_groups)) { - dev_err(dev, - "stream ID for master device %s greater than maximum allowed (%d)\n", - masterspec->np->name, smmu->num_mapping_groups); - return -ERANGE; - } - master->cfg.streamids[i] = streamid; - } - return insert_smmu_master(smmu, master); -} - -static struct arm_smmu_device *find_smmu_for_device(struct device *dev) -{ - struct arm_smmu_device *smmu; - struct arm_smmu_master *master = NULL; - struct device_node *dev_node = dev_get_dev_node(dev); - - spin_lock(&arm_smmu_devices_lock); - list_for_each_entry(smmu, &arm_smmu_devices, list) { - master = find_smmu_master(smmu, dev_node); - if (master) - break; - } - spin_unlock(&arm_smmu_devices_lock); - - return master ? smmu : NULL; + *smmu = dev_get_drvdata(smmu_dev); + of_phandle_iterator_args(&it, sids, it.cur_count); + err = iommu_fwspec_add_ids(dev, sids, it.cur_count); + kfree(sids); + return err; } static int __arm_smmu_alloc_bitmap(unsigned long *map, int start, int end) @@ -686,8 +648,7 @@ static struct iommu_gather_ops arm_smmu_gather_ops = { static irqreturn_t arm_smmu_context_fault(int irq, void *dev) { - int flags, ret; - u32 fsr, fsynr, resume; + u32 fsr, fsynr; unsigned long iova; struct iommu_domain *domain = dev; struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); @@ -701,34 +662,15 @@ static irqreturn_t arm_smmu_context_fault(int irq, void *dev) if (!(fsr & FSR_FAULT)) return IRQ_NONE; - if (fsr & FSR_IGN) - dev_err_ratelimited(smmu->dev, - "Unexpected context fault (fsr 0x%x)\n", - fsr); - fsynr = readl_relaxed(cb_base + ARM_SMMU_CB_FSYNR0); - flags = fsynr & FSYNR0_WNR ? IOMMU_FAULT_WRITE : IOMMU_FAULT_READ; - iova = readq_relaxed(cb_base + ARM_SMMU_CB_FAR); - if (!report_iommu_fault(domain, smmu->dev, iova, flags)) { - ret = IRQ_HANDLED; - resume = RESUME_RETRY; - } else { - dev_err_ratelimited(smmu->dev, - "Unhandled context fault: iova=0x%08lx, fsynr=0x%x, cb=%d\n", - iova, fsynr, cfg->cbndx); - ret = IRQ_NONE; - resume = RESUME_TERMINATE; - } - - /* Clear the faulting FSR */ - writel(fsr, cb_base + ARM_SMMU_CB_FSR); - /* Retry or terminate any stalled transactions */ - if (fsr & FSR_SS) - writel_relaxed(resume, cb_base + ARM_SMMU_CB_RESUME); + dev_err_ratelimited(smmu->dev, + "Unhandled context fault: fsr=0x%x, iova=0x%08lx, fsynr=0x%x, cb=%d\n", + fsr, iova, fsynr, cfg->cbndx); - return ret; + writel(fsr, cb_base + ARM_SMMU_CB_FSR); + return IRQ_HANDLED; } static irqreturn_t arm_smmu_global_fault(int irq, void *dev) @@ -758,7 +700,7 @@ static irqreturn_t arm_smmu_global_fault(int irq, void *dev) static void arm_smmu_init_context_bank(struct arm_smmu_domain *smmu_domain, struct io_pgtable_cfg *pgtbl_cfg) { - u32 reg; + u32 reg, reg2; u64 reg64; bool stage1; struct arm_smmu_cfg *cfg = &smmu_domain->cfg; @@ -801,14 +743,22 @@ static void arm_smmu_init_context_bank(struct arm_smmu_domain *smmu_domain, /* TTBRs */ if (stage1) { - reg64 = pgtbl_cfg->arm_lpae_s1_cfg.ttbr[0]; - - reg64 |= ((u64)ARM_SMMU_CB_ASID(smmu, cfg)) << TTBRn_ASID_SHIFT; - writeq_relaxed(reg64, cb_base + ARM_SMMU_CB_TTBR0); - - reg64 = pgtbl_cfg->arm_lpae_s1_cfg.ttbr[1]; - reg64 |= ((u64)ARM_SMMU_CB_ASID(smmu, cfg)) << TTBRn_ASID_SHIFT; - writeq_relaxed(reg64, cb_base + ARM_SMMU_CB_TTBR1); + u16 asid = ARM_SMMU_CB_ASID(smmu, cfg); + + if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) { + reg = pgtbl_cfg->arm_v7s_cfg.ttbr[0]; + writel_relaxed(reg, cb_base + ARM_SMMU_CB_TTBR0); + reg = pgtbl_cfg->arm_v7s_cfg.ttbr[1]; + writel_relaxed(reg, cb_base + ARM_SMMU_CB_TTBR1); + writel_relaxed(asid, cb_base + ARM_SMMU_CB_CONTEXTIDR); + } else { + reg64 = pgtbl_cfg->arm_lpae_s1_cfg.ttbr[0]; + reg64 |= (u64)asid << TTBRn_ASID_SHIFT; + writeq_relaxed(reg64, cb_base + ARM_SMMU_CB_TTBR0); + reg64 = pgtbl_cfg->arm_lpae_s1_cfg.ttbr[1]; + reg64 |= (u64)asid << TTBRn_ASID_SHIFT; + writeq_relaxed(reg64, cb_base + ARM_SMMU_CB_TTBR1); + } } else { reg64 = pgtbl_cfg->arm_lpae_s2_cfg.vttbr; writeq_relaxed(reg64, cb_base + ARM_SMMU_CB_TTBR0); @@ -816,28 +766,36 @@ static void arm_smmu_init_context_bank(struct arm_smmu_domain *smmu_domain, /* TTBCR */ if (stage1) { - reg = pgtbl_cfg->arm_lpae_s1_cfg.tcr; - writel_relaxed(reg, cb_base + ARM_SMMU_CB_TTBCR); - if (smmu->version > ARM_SMMU_V1) { - reg = pgtbl_cfg->arm_lpae_s1_cfg.tcr >> 32; - reg |= TTBCR2_SEP_UPSTREAM; - writel_relaxed(reg, cb_base + ARM_SMMU_CB_TTBCR2); + if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) { + reg = pgtbl_cfg->arm_v7s_cfg.tcr; + reg2 = 0; + } else { + reg = pgtbl_cfg->arm_lpae_s1_cfg.tcr; + reg2 = pgtbl_cfg->arm_lpae_s1_cfg.tcr >> 32; + reg2 |= TTBCR2_SEP_UPSTREAM; } + if (smmu->version > ARM_SMMU_V1) + writel_relaxed(reg2, cb_base + ARM_SMMU_CB_TTBCR2); } else { reg = pgtbl_cfg->arm_lpae_s2_cfg.vtcr; - writel_relaxed(reg, cb_base + ARM_SMMU_CB_TTBCR); } + writel_relaxed(reg, cb_base + ARM_SMMU_CB_TTBCR); /* MAIRs (stage-1 only) */ if (stage1) { - reg = pgtbl_cfg->arm_lpae_s1_cfg.mair[0]; + if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) { + reg = pgtbl_cfg->arm_v7s_cfg.prrr; + reg2 = pgtbl_cfg->arm_v7s_cfg.nmrr; + } else { + reg = pgtbl_cfg->arm_lpae_s1_cfg.mair[0]; + reg2 = pgtbl_cfg->arm_lpae_s1_cfg.mair[1]; + } writel_relaxed(reg, cb_base + ARM_SMMU_CB_S1_MAIR0); - reg = pgtbl_cfg->arm_lpae_s1_cfg.mair[1]; - writel_relaxed(reg, cb_base + ARM_SMMU_CB_S1_MAIR1); + writel_relaxed(reg2, cb_base + ARM_SMMU_CB_S1_MAIR1); } /* SCTLR */ - reg = SCTLR_CFCFG | SCTLR_CFIE | SCTLR_CFRE | SCTLR_M | SCTLR_EAE_SBOP; + reg = SCTLR_CFIE | SCTLR_CFRE | SCTLR_AFE | SCTLR_TRE | SCTLR_M; if (stage1) reg |= SCTLR_S1_ASIDPNE; #ifdef __BIG_ENDIAN @@ -861,12 +819,6 @@ static int arm_smmu_init_domain_context(struct iommu_domain *domain, if (smmu_domain->smmu) goto out_unlock; - /* We're bypassing these SIDs, so don't allocate an actual context */ - if (domain->type == IOMMU_DOMAIN_DMA) { - smmu_domain->smmu = smmu; - goto out_unlock; - } - /* * Mapping the requested stage onto what we support is surprisingly * complicated, mainly because the spec allows S1+S2 SMMUs without @@ -900,6 +852,11 @@ static int arm_smmu_init_domain_context(struct iommu_domain *domain, */ if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_L) cfg->fmt = ARM_SMMU_CTX_FMT_AARCH32_L; + if (IS_ENABLED(CONFIG_IOMMU_IO_PGTABLE_ARMV7S) && + !IS_ENABLED(CONFIG_64BIT) && !IS_ENABLED(CONFIG_ARM_LPAE) && + (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_S) && + (smmu_domain->stage == ARM_SMMU_DOMAIN_S1)) + cfg->fmt = ARM_SMMU_CTX_FMT_AARCH32_S; if ((IS_ENABLED(CONFIG_64BIT) || cfg->fmt == ARM_SMMU_CTX_FMT_NONE) && (smmu->features & (ARM_SMMU_FEAT_FMT_AARCH64_64K | ARM_SMMU_FEAT_FMT_AARCH64_16K | @@ -919,10 +876,14 @@ static int arm_smmu_init_domain_context(struct iommu_domain *domain, oas = smmu->ipa_size; if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64) { fmt = ARM_64_LPAE_S1; - } else { + } else if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_L) { fmt = ARM_32_LPAE_S1; ias = min(ias, 32UL); oas = min(oas, 40UL); + } else { + fmt = ARM_V7S; + ias = min(ias, 32UL); + oas = min(oas, 32UL); } break; case ARM_SMMU_DOMAIN_NESTED: @@ -978,6 +939,8 @@ static int arm_smmu_init_domain_context(struct iommu_domain *domain, /* Update the domain's page sizes to reflect the page table format */ domain->pgsize_bitmap = pgtbl_cfg.pgsize_bitmap; + domain->geometry.aperture_end = (1UL << ias) - 1; + domain->geometry.force_aperture = true; /* Initialise the context bank with our page table cfg */ arm_smmu_init_context_bank(smmu_domain, &pgtbl_cfg); @@ -1016,7 +979,7 @@ static void arm_smmu_destroy_domain_context(struct iommu_domain *domain) void __iomem *cb_base; int irq; - if (!smmu || domain->type == IOMMU_DOMAIN_DMA) + if (!smmu) return; /* @@ -1050,8 +1013,8 @@ static struct iommu_domain *arm_smmu_domain_alloc(unsigned type) if (!smmu_domain) return NULL; - if (type == IOMMU_DOMAIN_DMA && - iommu_get_dma_cookie(&smmu_domain->domain)) { + if (type == IOMMU_DOMAIN_DMA && (using_legacy_binding || + iommu_get_dma_cookie(&smmu_domain->domain))) { kfree(smmu_domain); return NULL; } @@ -1075,162 +1038,197 @@ static void arm_smmu_domain_free(struct iommu_domain *domain) kfree(smmu_domain); } -static int arm_smmu_master_configure_smrs(struct arm_smmu_device *smmu, - struct arm_smmu_master_cfg *cfg) +static void arm_smmu_write_smr(struct arm_smmu_device *smmu, int idx) { - int i; - struct arm_smmu_smr *smrs; - void __iomem *gr0_base = ARM_SMMU_GR0(smmu); + struct arm_smmu_smr *smr = smmu->smrs + idx; + u32 reg = smr->id << SMR_ID_SHIFT | smr->mask << SMR_MASK_SHIFT; - if (!(smmu->features & ARM_SMMU_FEAT_STREAM_MATCH)) - return 0; + if (smr->valid) + reg |= SMR_VALID; + writel_relaxed(reg, ARM_SMMU_GR0(smmu) + ARM_SMMU_GR0_SMR(idx)); +} - if (cfg->smrs) - return -EEXIST; +static void arm_smmu_write_s2cr(struct arm_smmu_device *smmu, int idx) +{ + struct arm_smmu_s2cr *s2cr = smmu->s2crs + idx; + u32 reg = (s2cr->type & S2CR_TYPE_MASK) << S2CR_TYPE_SHIFT | + (s2cr->cbndx & S2CR_CBNDX_MASK) << S2CR_CBNDX_SHIFT | + (s2cr->privcfg & S2CR_PRIVCFG_MASK) << S2CR_PRIVCFG_SHIFT; - smrs = kmalloc_array(cfg->num_streamids, sizeof(*smrs), GFP_KERNEL); - if (!smrs) { - dev_err(smmu->dev, "failed to allocate %d SMRs\n", - cfg->num_streamids); - return -ENOMEM; - } + writel_relaxed(reg, ARM_SMMU_GR0(smmu) + ARM_SMMU_GR0_S2CR(idx)); +} - /* Allocate the SMRs on the SMMU */ - for (i = 0; i < cfg->num_streamids; ++i) { - int idx = __arm_smmu_alloc_bitmap(smmu->smr_map, 0, - smmu->num_mapping_groups); - if (idx < 0) { - dev_err(smmu->dev, "failed to allocate free SMR\n"); - goto err_free_smrs; - } +static void arm_smmu_write_sme(struct arm_smmu_device *smmu, int idx) +{ + arm_smmu_write_s2cr(smmu, idx); + if (smmu->smrs) + arm_smmu_write_smr(smmu, idx); +} - smrs[i] = (struct arm_smmu_smr) { - .idx = idx, - .mask = 0, /* We don't currently share SMRs */ - .id = cfg->streamids[i], - }; - } +static int arm_smmu_find_sme(struct arm_smmu_device *smmu, u16 id, u16 mask) +{ + struct arm_smmu_smr *smrs = smmu->smrs; + int i, free_idx = -ENOSPC; - /* It worked! Now, poke the actual hardware */ - for (i = 0; i < cfg->num_streamids; ++i) { - u32 reg = SMR_VALID | smrs[i].id << SMR_ID_SHIFT | - smrs[i].mask << SMR_MASK_SHIFT; - writel_relaxed(reg, gr0_base + ARM_SMMU_GR0_SMR(smrs[i].idx)); - } + /* Stream indexing is blissfully easy */ + if (!smrs) + return id; - cfg->smrs = smrs; - return 0; + /* Validating SMRs is... less so */ + for (i = 0; i < smmu->num_mapping_groups; ++i) { + if (!smrs[i].valid) { + /* + * Note the first free entry we come across, which + * we'll claim in the end if nothing else matches. + */ + if (free_idx < 0) + free_idx = i; + continue; + } + /* + * If the new entry is _entirely_ matched by an existing entry, + * then reuse that, with the guarantee that there also cannot + * be any subsequent conflicting entries. In normal use we'd + * expect simply identical entries for this case, but there's + * no harm in accommodating the generalisation. + */ + if ((mask & smrs[i].mask) == mask && + !((id ^ smrs[i].id) & ~smrs[i].mask)) + return i; + /* + * If the new entry has any other overlap with an existing one, + * though, then there always exists at least one stream ID + * which would cause a conflict, and we can't allow that risk. + */ + if (!((id ^ smrs[i].id) & ~(smrs[i].mask | mask))) + return -EINVAL; + } -err_free_smrs: - while (--i >= 0) - __arm_smmu_free_bitmap(smmu->smr_map, smrs[i].idx); - kfree(smrs); - return -ENOSPC; + return free_idx; } -static void arm_smmu_master_free_smrs(struct arm_smmu_device *smmu, - struct arm_smmu_master_cfg *cfg) +static bool arm_smmu_free_sme(struct arm_smmu_device *smmu, int idx) { - int i; - void __iomem *gr0_base = ARM_SMMU_GR0(smmu); - struct arm_smmu_smr *smrs = cfg->smrs; - - if (!smrs) - return; + if (--smmu->s2crs[idx].count) + return false; - /* Invalidate the SMRs before freeing back to the allocator */ - for (i = 0; i < cfg->num_streamids; ++i) { - u8 idx = smrs[i].idx; + smmu->s2crs[idx] = s2cr_init_val; + if (smmu->smrs) + smmu->smrs[idx].valid = false; - writel_relaxed(~SMR_VALID, gr0_base + ARM_SMMU_GR0_SMR(idx)); - __arm_smmu_free_bitmap(smmu->smr_map, idx); - } - - cfg->smrs = NULL; - kfree(smrs); + return true; } -static int arm_smmu_domain_add_master(struct arm_smmu_domain *smmu_domain, - struct arm_smmu_master_cfg *cfg) +static int arm_smmu_master_alloc_smes(struct device *dev) { - int i, ret; - struct arm_smmu_device *smmu = smmu_domain->smmu; - void __iomem *gr0_base = ARM_SMMU_GR0(smmu); + struct iommu_fwspec *fwspec = dev->iommu_fwspec; + struct arm_smmu_master_cfg *cfg = fwspec->iommu_priv; + struct arm_smmu_device *smmu = cfg->smmu; + struct arm_smmu_smr *smrs = smmu->smrs; + struct iommu_group *group; + int i, idx, ret; - /* - * FIXME: This won't be needed once we have IOMMU-backed DMA ops - * for all devices behind the SMMU. Note that we need to take - * care configuring SMRs for devices both a platform_device and - * and a PCI device (i.e. a PCI host controller) - */ - if (smmu_domain->domain.type == IOMMU_DOMAIN_DMA) - return 0; + mutex_lock(&smmu->stream_map_mutex); + /* Figure out a viable stream map entry allocation */ + for_each_cfg_sme(fwspec, i, idx) { + u16 sid = fwspec->ids[i]; + u16 mask = fwspec->ids[i] >> SMR_MASK_SHIFT; - /* Devices in an IOMMU group may already be configured */ - ret = arm_smmu_master_configure_smrs(smmu, cfg); - if (ret) - return ret == -EEXIST ? 0 : ret; + if (idx != INVALID_SMENDX) { + ret = -EEXIST; + goto out_err; + } + + ret = arm_smmu_find_sme(smmu, sid, mask); + if (ret < 0) + goto out_err; - for (i = 0; i < cfg->num_streamids; ++i) { - u32 idx, s2cr; + idx = ret; + if (smrs && smmu->s2crs[idx].count == 0) { + smrs[idx].id = sid; + smrs[idx].mask = mask; + smrs[idx].valid = true; + } + smmu->s2crs[idx].count++; + cfg->smendx[i] = (s16)idx; + } + + group = iommu_group_get_for_dev(dev); + if (!group) + group = ERR_PTR(-ENOMEM); + if (IS_ERR(group)) { + ret = PTR_ERR(group); + goto out_err; + } + iommu_group_put(group); - idx = cfg->smrs ? cfg->smrs[i].idx : cfg->streamids[i]; - s2cr = S2CR_TYPE_TRANS | S2CR_PRIVCFG_UNPRIV | - (smmu_domain->cfg.cbndx << S2CR_CBNDX_SHIFT); - writel_relaxed(s2cr, gr0_base + ARM_SMMU_GR0_S2CR(idx)); + /* It worked! Now, poke the actual hardware */ + for_each_cfg_sme(fwspec, i, idx) { + arm_smmu_write_sme(smmu, idx); + smmu->s2crs[idx].group = group; } + mutex_unlock(&smmu->stream_map_mutex); return 0; + +out_err: + while (i--) { + arm_smmu_free_sme(smmu, cfg->smendx[i]); + cfg->smendx[i] = INVALID_SMENDX; + } + mutex_unlock(&smmu->stream_map_mutex); + return ret; } -static void arm_smmu_domain_remove_master(struct arm_smmu_domain *smmu_domain, - struct arm_smmu_master_cfg *cfg) +static void arm_smmu_master_free_smes(struct iommu_fwspec *fwspec) { - int i; - struct arm_smmu_device *smmu = smmu_domain->smmu; - void __iomem *gr0_base = ARM_SMMU_GR0(smmu); - - /* An IOMMU group is torn down by the first device to be removed */ - if ((smmu->features & ARM_SMMU_FEAT_STREAM_MATCH) && !cfg->smrs) - return; + struct arm_smmu_device *smmu = fwspec_smmu(fwspec); + struct arm_smmu_master_cfg *cfg = fwspec->iommu_priv; + int i, idx; - /* - * We *must* clear the S2CR first, because freeing the SMR means - * that it can be re-allocated immediately. - */ - for (i = 0; i < cfg->num_streamids; ++i) { - u32 idx = cfg->smrs ? cfg->smrs[i].idx : cfg->streamids[i]; - u32 reg = disable_bypass ? S2CR_TYPE_FAULT : S2CR_TYPE_BYPASS; - - writel_relaxed(reg, gr0_base + ARM_SMMU_GR0_S2CR(idx)); + mutex_lock(&smmu->stream_map_mutex); + for_each_cfg_sme(fwspec, i, idx) { + if (arm_smmu_free_sme(smmu, idx)) + arm_smmu_write_sme(smmu, idx); + cfg->smendx[i] = INVALID_SMENDX; } - - arm_smmu_master_free_smrs(smmu, cfg); + mutex_unlock(&smmu->stream_map_mutex); } -static void arm_smmu_detach_dev(struct device *dev, - struct arm_smmu_master_cfg *cfg) +static int arm_smmu_domain_add_master(struct arm_smmu_domain *smmu_domain, + struct iommu_fwspec *fwspec) { - struct iommu_domain *domain = dev->archdata.iommu; - struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); + struct arm_smmu_device *smmu = smmu_domain->smmu; + struct arm_smmu_s2cr *s2cr = smmu->s2crs; + enum arm_smmu_s2cr_type type = S2CR_TYPE_TRANS; + u8 cbndx = smmu_domain->cfg.cbndx; + int i, idx; + + for_each_cfg_sme(fwspec, i, idx) { + if (type == s2cr[idx].type && cbndx == s2cr[idx].cbndx) + continue; - dev->archdata.iommu = NULL; - arm_smmu_domain_remove_master(smmu_domain, cfg); + s2cr[idx].type = type; + s2cr[idx].privcfg = S2CR_PRIVCFG_UNPRIV; + s2cr[idx].cbndx = cbndx; + arm_smmu_write_s2cr(smmu, idx); + } + return 0; } static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev) { int ret; - struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); + struct iommu_fwspec *fwspec = dev->iommu_fwspec; struct arm_smmu_device *smmu; - struct arm_smmu_master_cfg *cfg; + struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); - smmu = find_smmu_for_device(dev); - if (!smmu) { + if (!fwspec || fwspec->ops != &arm_smmu_ops) { dev_err(dev, "cannot attach to SMMU, is it on the same bus?\n"); return -ENXIO; } + smmu = fwspec_smmu(fwspec); /* Ensure that the domain is finalised */ ret = arm_smmu_init_domain_context(domain, smmu); if (ret < 0) @@ -1248,18 +1246,7 @@ static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev) } /* Looks ok, so add the device to the domain */ - cfg = find_smmu_master_cfg(dev); - if (!cfg) - return -ENODEV; - - /* Detach the dev from its current domain */ - if (dev->archdata.iommu) - arm_smmu_detach_dev(dev, cfg); - - ret = arm_smmu_domain_add_master(smmu_domain, cfg); - if (!ret) - dev->archdata.iommu = domain; - return ret; + return arm_smmu_domain_add_master(smmu_domain, fwspec); } static int arm_smmu_map(struct iommu_domain *domain, unsigned long iova, @@ -1378,110 +1365,113 @@ static bool arm_smmu_capable(enum iommu_cap cap) } } -static int __arm_smmu_get_pci_sid(struct pci_dev *pdev, u16 alias, void *data) +static int arm_smmu_match_node(struct device *dev, void *data) { - *((u16 *)data) = alias; - return 0; /* Continue walking */ + return dev->of_node == data; } -static void __arm_smmu_release_pci_iommudata(void *data) +static struct arm_smmu_device *arm_smmu_get_by_node(struct device_node *np) { - kfree(data); + struct device *dev = driver_find_device(&arm_smmu_driver.driver, NULL, + np, arm_smmu_match_node); + put_device(dev); + return dev ? dev_get_drvdata(dev) : NULL; } -static int arm_smmu_init_pci_device(struct pci_dev *pdev, - struct iommu_group *group) +static int arm_smmu_add_device(struct device *dev) { + struct arm_smmu_device *smmu; struct arm_smmu_master_cfg *cfg; - u16 sid; - int i; - - cfg = iommu_group_get_iommudata(group); - if (!cfg) { - cfg = kzalloc(sizeof(*cfg), GFP_KERNEL); - if (!cfg) - return -ENOMEM; + struct iommu_fwspec *fwspec = dev->iommu_fwspec; + int i, ret; - iommu_group_set_iommudata(group, cfg, - __arm_smmu_release_pci_iommudata); + if (using_legacy_binding) { + ret = arm_smmu_register_legacy_master(dev, &smmu); + fwspec = dev->iommu_fwspec; + if (ret) + goto out_free; + } else if (fwspec) { + smmu = arm_smmu_get_by_node(to_of_node(fwspec->iommu_fwnode)); + } else { + return -ENODEV; } - if (cfg->num_streamids >= MAX_MASTER_STREAMIDS) - return -ENOSPC; - - /* - * Assume Stream ID == Requester ID for now. - * We need a way to describe the ID mappings in FDT. - */ - pci_for_each_dma_alias(pdev, __arm_smmu_get_pci_sid, &sid); - for (i = 0; i < cfg->num_streamids; ++i) - if (cfg->streamids[i] == sid) - break; - - /* Avoid duplicate SIDs, as this can lead to SMR conflicts */ - if (i == cfg->num_streamids) - cfg->streamids[cfg->num_streamids++] = sid; - - return 0; -} + ret = -EINVAL; + for (i = 0; i < fwspec->num_ids; i++) { + u16 sid = fwspec->ids[i]; + u16 mask = fwspec->ids[i] >> SMR_MASK_SHIFT; -static int arm_smmu_init_platform_device(struct device *dev, - struct iommu_group *group) -{ - struct arm_smmu_device *smmu = find_smmu_for_device(dev); - struct arm_smmu_master *master; + if (sid & ~smmu->streamid_mask) { + dev_err(dev, "stream ID 0x%x out of range for SMMU (0x%x)\n", + sid, smmu->streamid_mask); + goto out_free; + } + if (mask & ~smmu->smr_mask_mask) { + dev_err(dev, "SMR mask 0x%x out of range for SMMU (0x%x)\n", + sid, smmu->smr_mask_mask); + goto out_free; + } + } - if (!smmu) - return -ENODEV; + ret = -ENOMEM; + cfg = kzalloc(offsetof(struct arm_smmu_master_cfg, smendx[i]), + GFP_KERNEL); + if (!cfg) + goto out_free; - master = find_smmu_master(smmu, dev->of_node); - if (!master) - return -ENODEV; + cfg->smmu = smmu; + fwspec->iommu_priv = cfg; + while (i--) + cfg->smendx[i] = INVALID_SMENDX; - iommu_group_set_iommudata(group, &master->cfg, NULL); + ret = arm_smmu_master_alloc_smes(dev); + if (ret) + goto out_free; return 0; -} - -static int arm_smmu_add_device(struct device *dev) -{ - struct iommu_group *group; - - group = iommu_group_get_for_dev(dev); - if (IS_ERR(group)) - return PTR_ERR(group); - iommu_group_put(group); - return 0; +out_free: + if (fwspec) + kfree(fwspec->iommu_priv); + iommu_fwspec_free(dev); + return ret; } static void arm_smmu_remove_device(struct device *dev) { + struct iommu_fwspec *fwspec = dev->iommu_fwspec; + + if (!fwspec || fwspec->ops != &arm_smmu_ops) + return; + + arm_smmu_master_free_smes(fwspec); iommu_group_remove_device(dev); + kfree(fwspec->iommu_priv); + iommu_fwspec_free(dev); } static struct iommu_group *arm_smmu_device_group(struct device *dev) { - struct iommu_group *group; - int ret; + struct iommu_fwspec *fwspec = dev->iommu_fwspec; + struct arm_smmu_device *smmu = fwspec_smmu(fwspec); + struct iommu_group *group = NULL; + int i, idx; - if (dev_is_pci(dev)) - group = pci_device_group(dev); - else - group = generic_device_group(dev); + for_each_cfg_sme(fwspec, i, idx) { + if (group && smmu->s2crs[idx].group && + group != smmu->s2crs[idx].group) + return ERR_PTR(-EINVAL); - if (IS_ERR(group)) + group = smmu->s2crs[idx].group; + } + + if (group) return group; if (dev_is_pci(dev)) - ret = arm_smmu_init_pci_device(to_pci_dev(dev), group); + group = pci_device_group(dev); else - ret = arm_smmu_init_platform_device(dev, group); - - if (ret) { - iommu_group_put(group); - group = ERR_PTR(ret); - } + group = generic_device_group(dev); return group; } @@ -1530,6 +1520,19 @@ out_unlock: return ret; } +static int arm_smmu_of_xlate(struct device *dev, struct of_phandle_args *args) +{ + u32 fwid = 0; + + if (args->args_count > 0) + fwid |= (u16)args->args[0]; + + if (args->args_count > 1) + fwid |= (u16)args->args[1] << SMR_MASK_SHIFT; + + return iommu_fwspec_add_ids(dev, &fwid, 1); +} + static struct iommu_ops arm_smmu_ops = { .capable = arm_smmu_capable, .domain_alloc = arm_smmu_domain_alloc, @@ -1544,6 +1547,7 @@ static struct iommu_ops arm_smmu_ops = { .device_group = arm_smmu_device_group, .domain_get_attr = arm_smmu_domain_get_attr, .domain_set_attr = arm_smmu_domain_set_attr, + .of_xlate = arm_smmu_of_xlate, .pgsize_bitmap = -1UL, /* Restricted during device attach */ }; @@ -1551,19 +1555,19 @@ static void arm_smmu_device_reset(struct arm_smmu_device *smmu) { void __iomem *gr0_base = ARM_SMMU_GR0(smmu); void __iomem *cb_base; - int i = 0; + int i; u32 reg, major; /* clear global FSR */ reg = readl_relaxed(ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sGFSR); writel(reg, ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sGFSR); - /* Mark all SMRn as invalid and all S2CRn as bypass unless overridden */ - reg = disable_bypass ? S2CR_TYPE_FAULT : S2CR_TYPE_BYPASS; - for (i = 0; i < smmu->num_mapping_groups; ++i) { - writel_relaxed(0, gr0_base + ARM_SMMU_GR0_SMR(i)); - writel_relaxed(reg, gr0_base + ARM_SMMU_GR0_S2CR(i)); - } + /* + * Reset stream mapping groups: Initial values mark all SMRn as + * invalid and all S2CRn as bypass unless overridden. + */ + for (i = 0; i < smmu->num_mapping_groups; ++i) + arm_smmu_write_sme(smmu, i); /* * Before clearing ARM_MMU500_ACTLR_CPRE, need to @@ -1652,6 +1656,7 @@ static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu) void __iomem *gr0_base = ARM_SMMU_GR0(smmu); u32 id; bool cttw_dt, cttw_reg; + int i; dev_notice(smmu->dev, "probing hardware configuration...\n"); dev_notice(smmu->dev, "SMMUv%d with:\n", @@ -1710,39 +1715,55 @@ static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu) dev_notice(smmu->dev, "\t(IDR0.CTTW overridden by dma-coherent property)\n"); + /* Max. number of entries we have for stream matching/indexing */ + size = 1 << ((id >> ID0_NUMSIDB_SHIFT) & ID0_NUMSIDB_MASK); + smmu->streamid_mask = size - 1; if (id & ID0_SMS) { - u32 smr, sid, mask; + u32 smr; smmu->features |= ARM_SMMU_FEAT_STREAM_MATCH; - smmu->num_mapping_groups = (id >> ID0_NUMSMRG_SHIFT) & - ID0_NUMSMRG_MASK; - if (smmu->num_mapping_groups == 0) { + size = (id >> ID0_NUMSMRG_SHIFT) & ID0_NUMSMRG_MASK; + if (size == 0) { dev_err(smmu->dev, "stream-matching supported, but no SMRs present!\n"); return -ENODEV; } - smr = SMR_MASK_MASK << SMR_MASK_SHIFT; - smr |= (SMR_ID_MASK << SMR_ID_SHIFT); + /* + * SMR.ID bits may not be preserved if the corresponding MASK + * bits are set, so check each one separately. We can reject + * masters later if they try to claim IDs outside these masks. + */ + smr = smmu->streamid_mask << SMR_ID_SHIFT; + writel_relaxed(smr, gr0_base + ARM_SMMU_GR0_SMR(0)); + smr = readl_relaxed(gr0_base + ARM_SMMU_GR0_SMR(0)); + smmu->streamid_mask = smr >> SMR_ID_SHIFT; + + smr = smmu->streamid_mask << SMR_MASK_SHIFT; writel_relaxed(smr, gr0_base + ARM_SMMU_GR0_SMR(0)); smr = readl_relaxed(gr0_base + ARM_SMMU_GR0_SMR(0)); + smmu->smr_mask_mask = smr >> SMR_MASK_SHIFT; - mask = (smr >> SMR_MASK_SHIFT) & SMR_MASK_MASK; - sid = (smr >> SMR_ID_SHIFT) & SMR_ID_MASK; - if ((mask & sid) != sid) { - dev_err(smmu->dev, - "SMR mask bits (0x%x) insufficient for ID field (0x%x)\n", - mask, sid); - return -ENODEV; - } + /* Zero-initialised to mark as invalid */ + smmu->smrs = devm_kcalloc(smmu->dev, size, sizeof(*smmu->smrs), + GFP_KERNEL); + if (!smmu->smrs) + return -ENOMEM; dev_notice(smmu->dev, - "\tstream matching with %u register groups, mask 0x%x", - smmu->num_mapping_groups, mask); - } else { - smmu->num_mapping_groups = (id >> ID0_NUMSIDB_SHIFT) & - ID0_NUMSIDB_MASK; + "\tstream matching with %lu register groups, mask 0x%x", + size, smmu->smr_mask_mask); } + /* s2cr->type == 0 means translation, so initialise explicitly */ + smmu->s2crs = devm_kmalloc_array(smmu->dev, size, sizeof(*smmu->s2crs), + GFP_KERNEL); + if (!smmu->s2crs) + return -ENOMEM; + for (i = 0; i < size; i++) + smmu->s2crs[i] = s2cr_init_val; + + smmu->num_mapping_groups = size; + mutex_init(&smmu->stream_map_mutex); if (smmu->version < ARM_SMMU_V2 || !(id & ID0_PTFS_NO_AARCH32)) { smmu->features |= ARM_SMMU_FEAT_FMT_AARCH32_L; @@ -1875,15 +1896,24 @@ MODULE_DEVICE_TABLE(of, arm_smmu_of_match); static int arm_smmu_device_dt_probe(struct platform_device *pdev) { - const struct of_device_id *of_id; const struct arm_smmu_match_data *data; struct resource *res; struct arm_smmu_device *smmu; struct device *dev = &pdev->dev; - struct rb_node *node; - struct of_phandle_iterator it; - struct arm_smmu_phandle_args *masterspec; int num_irqs, i, err; + bool legacy_binding; + + legacy_binding = of_find_property(dev->of_node, "mmu-masters", NULL); + if (legacy_binding && !using_generic_binding) { + if (!using_legacy_binding) + pr_notice("deprecated \"mmu-masters\" DT property in use; DMA API support unavailable\n"); + using_legacy_binding = true; + } else if (!legacy_binding && !using_legacy_binding) { + using_generic_binding = true; + } else { + dev_err(dev, "not probing due to mismatched DT properties\n"); + return -ENODEV; + } smmu = devm_kzalloc(dev, sizeof(*smmu), GFP_KERNEL); if (!smmu) { @@ -1892,8 +1922,7 @@ static int arm_smmu_device_dt_probe(struct platform_device *pdev) } smmu->dev = dev; - of_id = of_match_node(arm_smmu_of_match, dev->of_node); - data = of_id->data; + data = of_device_get_match_data(dev); smmu->version = data->version; smmu->model = data->model; @@ -1943,37 +1972,6 @@ static int arm_smmu_device_dt_probe(struct platform_device *pdev) if (err) return err; - i = 0; - smmu->masters = RB_ROOT; - - err = -ENOMEM; - /* No need to zero the memory for masterspec */ - masterspec = kmalloc(sizeof(*masterspec), GFP_KERNEL); - if (!masterspec) - goto out_put_masters; - - of_for_each_phandle(&it, err, dev->of_node, - "mmu-masters", "#stream-id-cells", 0) { - int count = of_phandle_iterator_args(&it, masterspec->args, - MAX_MASTER_STREAMIDS); - masterspec->np = of_node_get(it.node); - masterspec->args_count = count; - - err = register_smmu_master(smmu, dev, masterspec); - if (err) { - dev_err(dev, "failed to add master %s\n", - masterspec->np->name); - kfree(masterspec); - goto out_put_masters; - } - - i++; - } - - dev_notice(dev, "registered %d master devices\n", i); - - kfree(masterspec); - parse_driver_options(smmu); if (smmu->version == ARM_SMMU_V2 && @@ -1981,8 +1979,7 @@ static int arm_smmu_device_dt_probe(struct platform_device *pdev) dev_err(dev, "found only %d context interrupt(s) but %d required\n", smmu->num_context_irqs, smmu->num_context_banks); - err = -ENODEV; - goto out_put_masters; + return -ENODEV; } for (i = 0; i < smmu->num_global_irqs; ++i) { @@ -1994,59 +1991,39 @@ static int arm_smmu_device_dt_probe(struct platform_device *pdev) if (err) { dev_err(dev, "failed to request global IRQ %d (%u)\n", i, smmu->irqs[i]); - goto out_put_masters; + return err; } } - INIT_LIST_HEAD(&smmu->list); - spin_lock(&arm_smmu_devices_lock); - list_add(&smmu->list, &arm_smmu_devices); - spin_unlock(&arm_smmu_devices_lock); - + of_iommu_set_ops(dev->of_node, &arm_smmu_ops); + platform_set_drvdata(pdev, smmu); arm_smmu_device_reset(smmu); - return 0; -out_put_masters: - for (node = rb_first(&smmu->masters); node; node = rb_next(node)) { - struct arm_smmu_master *master - = container_of(node, struct arm_smmu_master, node); - of_node_put(master->of_node); + /* Oh, for a proper bus abstraction */ + if (!iommu_present(&platform_bus_type)) + bus_set_iommu(&platform_bus_type, &arm_smmu_ops); +#ifdef CONFIG_ARM_AMBA + if (!iommu_present(&amba_bustype)) + bus_set_iommu(&amba_bustype, &arm_smmu_ops); +#endif +#ifdef CONFIG_PCI + if (!iommu_present(&pci_bus_type)) { + pci_request_acs(); + bus_set_iommu(&pci_bus_type, &arm_smmu_ops); } - - return err; +#endif + return 0; } static int arm_smmu_device_remove(struct platform_device *pdev) { - int i; - struct device *dev = &pdev->dev; - struct arm_smmu_device *curr, *smmu = NULL; - struct rb_node *node; - - spin_lock(&arm_smmu_devices_lock); - list_for_each_entry(curr, &arm_smmu_devices, list) { - if (curr->dev == dev) { - smmu = curr; - list_del(&smmu->list); - break; - } - } - spin_unlock(&arm_smmu_devices_lock); + struct arm_smmu_device *smmu = platform_get_drvdata(pdev); if (!smmu) return -ENODEV; - for (node = rb_first(&smmu->masters); node; node = rb_next(node)) { - struct arm_smmu_master *master - = container_of(node, struct arm_smmu_master, node); - of_node_put(master->of_node); - } - if (!bitmap_empty(smmu->context_map, ARM_SMMU_MAX_CBS)) - dev_err(dev, "removing device with active domains!\n"); - - for (i = 0; i < smmu->num_global_irqs; ++i) - devm_free_irq(smmu->dev, smmu->irqs[i], smmu); + dev_err(&pdev->dev, "removing device with active domains!\n"); /* Turn the thing off */ writel(sCR0_CLIENTPD, ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sCR0); @@ -2064,41 +2041,14 @@ static struct platform_driver arm_smmu_driver = { static int __init arm_smmu_init(void) { - struct device_node *np; - int ret; - - /* - * Play nice with systems that don't have an ARM SMMU by checking that - * an ARM SMMU exists in the system before proceeding with the driver - * and IOMMU bus operation registration. - */ - np = of_find_matching_node(NULL, arm_smmu_of_match); - if (!np) - return 0; - - of_node_put(np); - - ret = platform_driver_register(&arm_smmu_driver); - if (ret) - return ret; - - /* Oh, for a proper bus abstraction */ - if (!iommu_present(&platform_bus_type)) - bus_set_iommu(&platform_bus_type, &arm_smmu_ops); - -#ifdef CONFIG_ARM_AMBA - if (!iommu_present(&amba_bustype)) - bus_set_iommu(&amba_bustype, &arm_smmu_ops); -#endif + static bool registered; + int ret = 0; -#ifdef CONFIG_PCI - if (!iommu_present(&pci_bus_type)) { - pci_request_acs(); - bus_set_iommu(&pci_bus_type, &arm_smmu_ops); + if (!registered) { + ret = platform_driver_register(&arm_smmu_driver); + registered = !ret; } -#endif - - return 0; + return ret; } static void __exit arm_smmu_exit(void) @@ -2109,6 +2059,25 @@ static void __exit arm_smmu_exit(void) subsys_initcall(arm_smmu_init); module_exit(arm_smmu_exit); +static int __init arm_smmu_of_init(struct device_node *np) +{ + int ret = arm_smmu_init(); + + if (ret) + return ret; + + if (!of_platform_device_create(np, NULL, platform_bus_type.dev_root)) + return -ENODEV; + + return 0; +} +IOMMU_OF_DECLARE(arm_smmuv1, "arm,smmu-v1", arm_smmu_of_init); +IOMMU_OF_DECLARE(arm_smmuv2, "arm,smmu-v2", arm_smmu_of_init); +IOMMU_OF_DECLARE(arm_mmu400, "arm,mmu-400", arm_smmu_of_init); +IOMMU_OF_DECLARE(arm_mmu401, "arm,mmu-401", arm_smmu_of_init); +IOMMU_OF_DECLARE(arm_mmu500, "arm,mmu-500", arm_smmu_of_init); +IOMMU_OF_DECLARE(cavium_smmuv2, "cavium,smmu-v2", arm_smmu_of_init); + MODULE_DESCRIPTION("IOMMU API for ARM architected SMMU implementations"); MODULE_AUTHOR("Will Deacon <will.deacon@arm.com>"); MODULE_LICENSE("GPL v2"); diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c index 00c8a08d56e7..c5ab8667e6f2 100644 --- a/drivers/iommu/dma-iommu.c +++ b/drivers/iommu/dma-iommu.c @@ -25,10 +25,29 @@ #include <linux/huge_mm.h> #include <linux/iommu.h> #include <linux/iova.h> +#include <linux/irq.h> #include <linux/mm.h> +#include <linux/pci.h> #include <linux/scatterlist.h> #include <linux/vmalloc.h> +struct iommu_dma_msi_page { + struct list_head list; + dma_addr_t iova; + phys_addr_t phys; +}; + +struct iommu_dma_cookie { + struct iova_domain iovad; + struct list_head msi_page_list; + spinlock_t msi_lock; +}; + +static inline struct iova_domain *cookie_iovad(struct iommu_domain *domain) +{ + return &((struct iommu_dma_cookie *)domain->iova_cookie)->iovad; +} + int iommu_dma_init(void) { return iova_cache_get(); @@ -43,15 +62,19 @@ int iommu_dma_init(void) */ int iommu_get_dma_cookie(struct iommu_domain *domain) { - struct iova_domain *iovad; + struct iommu_dma_cookie *cookie; if (domain->iova_cookie) return -EEXIST; - iovad = kzalloc(sizeof(*iovad), GFP_KERNEL); - domain->iova_cookie = iovad; + cookie = kzalloc(sizeof(*cookie), GFP_KERNEL); + if (!cookie) + return -ENOMEM; - return iovad ? 0 : -ENOMEM; + spin_lock_init(&cookie->msi_lock); + INIT_LIST_HEAD(&cookie->msi_page_list); + domain->iova_cookie = cookie; + return 0; } EXPORT_SYMBOL(iommu_get_dma_cookie); @@ -63,32 +86,58 @@ EXPORT_SYMBOL(iommu_get_dma_cookie); */ void iommu_put_dma_cookie(struct iommu_domain *domain) { - struct iova_domain *iovad = domain->iova_cookie; + struct iommu_dma_cookie *cookie = domain->iova_cookie; + struct iommu_dma_msi_page *msi, *tmp; - if (!iovad) + if (!cookie) return; - if (iovad->granule) - put_iova_domain(iovad); - kfree(iovad); + if (cookie->iovad.granule) + put_iova_domain(&cookie->iovad); + + list_for_each_entry_safe(msi, tmp, &cookie->msi_page_list, list) { + list_del(&msi->list); + kfree(msi); + } + kfree(cookie); domain->iova_cookie = NULL; } EXPORT_SYMBOL(iommu_put_dma_cookie); +static void iova_reserve_pci_windows(struct pci_dev *dev, + struct iova_domain *iovad) +{ + struct pci_host_bridge *bridge = pci_find_host_bridge(dev->bus); + struct resource_entry *window; + unsigned long lo, hi; + + resource_list_for_each_entry(window, &bridge->windows) { + if (resource_type(window->res) != IORESOURCE_MEM && + resource_type(window->res) != IORESOURCE_IO) + continue; + + lo = iova_pfn(iovad, window->res->start - window->offset); + hi = iova_pfn(iovad, window->res->end - window->offset); + reserve_iova(iovad, lo, hi); + } +} + /** * iommu_dma_init_domain - Initialise a DMA mapping domain * @domain: IOMMU domain previously prepared by iommu_get_dma_cookie() * @base: IOVA at which the mappable address space starts * @size: Size of IOVA space + * @dev: Device the domain is being initialised for * * @base and @size should be exact multiples of IOMMU page granularity to * avoid rounding surprises. If necessary, we reserve the page at address 0 * to ensure it is an invalid IOVA. It is safe to reinitialise a domain, but * any change which could make prior IOVAs invalid will fail. */ -int iommu_dma_init_domain(struct iommu_domain *domain, dma_addr_t base, u64 size) +int iommu_dma_init_domain(struct iommu_domain *domain, dma_addr_t base, + u64 size, struct device *dev) { - struct iova_domain *iovad = domain->iova_cookie; + struct iova_domain *iovad = cookie_iovad(domain); unsigned long order, base_pfn, end_pfn; if (!iovad) @@ -124,6 +173,8 @@ int iommu_dma_init_domain(struct iommu_domain *domain, dma_addr_t base, u64 size iovad->dma_32bit_pfn = end_pfn; } else { init_iova_domain(iovad, 1UL << order, base_pfn, end_pfn); + if (dev && dev_is_pci(dev)) + iova_reserve_pci_windows(to_pci_dev(dev), iovad); } return 0; } @@ -155,7 +206,7 @@ int dma_direction_to_prot(enum dma_data_direction dir, bool coherent) static struct iova *__alloc_iova(struct iommu_domain *domain, size_t size, dma_addr_t dma_limit) { - struct iova_domain *iovad = domain->iova_cookie; + struct iova_domain *iovad = cookie_iovad(domain); unsigned long shift = iova_shift(iovad); unsigned long length = iova_align(iovad, size) >> shift; @@ -171,7 +222,7 @@ static struct iova *__alloc_iova(struct iommu_domain *domain, size_t size, /* The IOVA allocator knows what we mapped, so just unmap whatever that was */ static void __iommu_dma_unmap(struct iommu_domain *domain, dma_addr_t dma_addr) { - struct iova_domain *iovad = domain->iova_cookie; + struct iova_domain *iovad = cookie_iovad(domain); unsigned long shift = iova_shift(iovad); unsigned long pfn = dma_addr >> shift; struct iova *iova = find_iova(iovad, pfn); @@ -294,7 +345,7 @@ struct page **iommu_dma_alloc(struct device *dev, size_t size, gfp_t gfp, void (*flush_page)(struct device *, const void *, phys_addr_t)) { struct iommu_domain *domain = iommu_get_domain_for_dev(dev); - struct iova_domain *iovad = domain->iova_cookie; + struct iova_domain *iovad = cookie_iovad(domain); struct iova *iova; struct page **pages; struct sg_table sgt; @@ -386,7 +437,7 @@ dma_addr_t iommu_dma_map_page(struct device *dev, struct page *page, { dma_addr_t dma_addr; struct iommu_domain *domain = iommu_get_domain_for_dev(dev); - struct iova_domain *iovad = domain->iova_cookie; + struct iova_domain *iovad = cookie_iovad(domain); phys_addr_t phys = page_to_phys(page) + offset; size_t iova_off = iova_offset(iovad, phys); size_t len = iova_align(iovad, size + iova_off); @@ -495,7 +546,7 @@ int iommu_dma_map_sg(struct device *dev, struct scatterlist *sg, int nents, int prot) { struct iommu_domain *domain = iommu_get_domain_for_dev(dev); - struct iova_domain *iovad = domain->iova_cookie; + struct iova_domain *iovad = cookie_iovad(domain); struct iova *iova; struct scatterlist *s, *prev = NULL; dma_addr_t dma_addr; @@ -587,3 +638,81 @@ int iommu_dma_mapping_error(struct device *dev, dma_addr_t dma_addr) { return dma_addr == DMA_ERROR_CODE; } + +static struct iommu_dma_msi_page *iommu_dma_get_msi_page(struct device *dev, + phys_addr_t msi_addr, struct iommu_domain *domain) +{ + struct iommu_dma_cookie *cookie = domain->iova_cookie; + struct iommu_dma_msi_page *msi_page; + struct iova_domain *iovad = &cookie->iovad; + struct iova *iova; + int prot = IOMMU_WRITE | IOMMU_NOEXEC | IOMMU_MMIO; + + msi_addr &= ~(phys_addr_t)iova_mask(iovad); + list_for_each_entry(msi_page, &cookie->msi_page_list, list) + if (msi_page->phys == msi_addr) + return msi_page; + + msi_page = kzalloc(sizeof(*msi_page), GFP_ATOMIC); + if (!msi_page) + return NULL; + + iova = __alloc_iova(domain, iovad->granule, dma_get_mask(dev)); + if (!iova) + goto out_free_page; + + msi_page->phys = msi_addr; + msi_page->iova = iova_dma_addr(iovad, iova); + if (iommu_map(domain, msi_page->iova, msi_addr, iovad->granule, prot)) + goto out_free_iova; + + INIT_LIST_HEAD(&msi_page->list); + list_add(&msi_page->list, &cookie->msi_page_list); + return msi_page; + +out_free_iova: + __free_iova(iovad, iova); +out_free_page: + kfree(msi_page); + return NULL; +} + +void iommu_dma_map_msi_msg(int irq, struct msi_msg *msg) +{ + struct device *dev = msi_desc_to_dev(irq_get_msi_desc(irq)); + struct iommu_domain *domain = iommu_get_domain_for_dev(dev); + struct iommu_dma_cookie *cookie; + struct iommu_dma_msi_page *msi_page; + phys_addr_t msi_addr = (u64)msg->address_hi << 32 | msg->address_lo; + unsigned long flags; + + if (!domain || !domain->iova_cookie) + return; + + cookie = domain->iova_cookie; + + /* + * We disable IRQs to rule out a possible inversion against + * irq_desc_lock if, say, someone tries to retarget the affinity + * of an MSI from within an IPI handler. + */ + spin_lock_irqsave(&cookie->msi_lock, flags); + msi_page = iommu_dma_get_msi_page(dev, msi_addr, domain); + spin_unlock_irqrestore(&cookie->msi_lock, flags); + + if (WARN_ON(!msi_page)) { + /* + * We're called from a void callback, so the best we can do is + * 'fail' by filling the message with obviously bogus values. + * Since we got this far due to an IOMMU being present, it's + * not like the existing address would have worked anyway... + */ + msg->address_hi = ~0U; + msg->address_lo = ~0U; + msg->data = ~0U; + } else { + msg->address_hi = upper_32_bits(msi_page->iova); + msg->address_lo &= iova_mask(&cookie->iovad); + msg->address_lo += lower_32_bits(msi_page->iova); + } +} diff --git a/drivers/iommu/exynos-iommu.c b/drivers/iommu/exynos-iommu.c index 33dcc29ec200..30808e91b775 100644 --- a/drivers/iommu/exynos-iommu.c +++ b/drivers/iommu/exynos-iommu.c @@ -1345,8 +1345,8 @@ static int __init exynos_iommu_of_setup(struct device_node *np) exynos_iommu_init(); pdev = of_platform_device_create(np, NULL, platform_bus_type.dev_root); - if (IS_ERR(pdev)) - return PTR_ERR(pdev); + if (!pdev) + return -ENODEV; /* * use the first registered sysmmu device for performing diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c index ebb5bf3ddbd9..a4407eabf0e6 100644 --- a/drivers/iommu/intel-iommu.c +++ b/drivers/iommu/intel-iommu.c @@ -2452,20 +2452,15 @@ static int get_last_alias(struct pci_dev *pdev, u16 alias, void *opaque) return 0; } -/* domain is initialized */ -static struct dmar_domain *get_domain_for_dev(struct device *dev, int gaw) +static struct dmar_domain *find_or_alloc_domain(struct device *dev, int gaw) { struct device_domain_info *info = NULL; - struct dmar_domain *domain, *tmp; + struct dmar_domain *domain = NULL; struct intel_iommu *iommu; u16 req_id, dma_alias; unsigned long flags; u8 bus, devfn; - domain = find_domain(dev); - if (domain) - return domain; - iommu = device_to_iommu(dev, &bus, &devfn); if (!iommu) return NULL; @@ -2487,9 +2482,9 @@ static struct dmar_domain *get_domain_for_dev(struct device *dev, int gaw) } spin_unlock_irqrestore(&device_domain_lock, flags); - /* DMA alias already has a domain, uses it */ + /* DMA alias already has a domain, use it */ if (info) - goto found_domain; + goto out; } /* Allocate and initialize new domain for the device */ @@ -2501,28 +2496,67 @@ static struct dmar_domain *get_domain_for_dev(struct device *dev, int gaw) return NULL; } - /* register PCI DMA alias device */ - if (dev_is_pci(dev) && req_id != dma_alias) { - tmp = dmar_insert_one_dev_info(iommu, PCI_BUS_NUM(dma_alias), - dma_alias & 0xff, NULL, domain); +out: - if (!tmp || tmp != domain) { - domain_exit(domain); - domain = tmp; - } + return domain; +} - if (!domain) - return NULL; +static struct dmar_domain *set_domain_for_dev(struct device *dev, + struct dmar_domain *domain) +{ + struct intel_iommu *iommu; + struct dmar_domain *tmp; + u16 req_id, dma_alias; + u8 bus, devfn; + + iommu = device_to_iommu(dev, &bus, &devfn); + if (!iommu) + return NULL; + + req_id = ((u16)bus << 8) | devfn; + + if (dev_is_pci(dev)) { + struct pci_dev *pdev = to_pci_dev(dev); + + pci_for_each_dma_alias(pdev, get_last_alias, &dma_alias); + + /* register PCI DMA alias device */ + if (req_id != dma_alias) { + tmp = dmar_insert_one_dev_info(iommu, PCI_BUS_NUM(dma_alias), + dma_alias & 0xff, NULL, domain); + + if (!tmp || tmp != domain) + return tmp; + } } -found_domain: tmp = dmar_insert_one_dev_info(iommu, bus, devfn, dev, domain); + if (!tmp || tmp != domain) + return tmp; + + return domain; +} - if (!tmp || tmp != domain) { +static struct dmar_domain *get_domain_for_dev(struct device *dev, int gaw) +{ + struct dmar_domain *domain, *tmp; + + domain = find_domain(dev); + if (domain) + goto out; + + domain = find_or_alloc_domain(dev, gaw); + if (!domain) + goto out; + + tmp = set_domain_for_dev(dev, domain); + if (!tmp || domain != tmp) { domain_exit(domain); domain = tmp; } +out: + return domain; } @@ -3394,17 +3428,18 @@ static unsigned long intel_alloc_iova(struct device *dev, static struct dmar_domain *__get_valid_domain_for_dev(struct device *dev) { + struct dmar_domain *domain, *tmp; struct dmar_rmrr_unit *rmrr; - struct dmar_domain *domain; struct device *i_dev; int i, ret; - domain = get_domain_for_dev(dev, DEFAULT_DOMAIN_ADDRESS_WIDTH); - if (!domain) { - pr_err("Allocating domain for %s failed\n", - dev_name(dev)); - return NULL; - } + domain = find_domain(dev); + if (domain) + goto out; + + domain = find_or_alloc_domain(dev, DEFAULT_DOMAIN_ADDRESS_WIDTH); + if (!domain) + goto out; /* We have a new domain - setup possible RMRRs for the device */ rcu_read_lock(); @@ -3423,6 +3458,18 @@ static struct dmar_domain *__get_valid_domain_for_dev(struct device *dev) } rcu_read_unlock(); + tmp = set_domain_for_dev(dev, domain); + if (!tmp || domain != tmp) { + domain_exit(domain); + domain = tmp; + } + +out: + + if (!domain) + pr_err("Allocating domain for %s failed\n", dev_name(dev)); + + return domain; } diff --git a/drivers/iommu/io-pgtable-arm-v7s.c b/drivers/iommu/io-pgtable-arm-v7s.c index 8c6139986d7d..f50e51c1a9c8 100644 --- a/drivers/iommu/io-pgtable-arm-v7s.c +++ b/drivers/iommu/io-pgtable-arm-v7s.c @@ -286,12 +286,14 @@ static int arm_v7s_pte_to_prot(arm_v7s_iopte pte, int lvl) int prot = IOMMU_READ; arm_v7s_iopte attr = pte >> ARM_V7S_ATTR_SHIFT(lvl); - if (attr & ARM_V7S_PTE_AP_RDONLY) + if (!(attr & ARM_V7S_PTE_AP_RDONLY)) prot |= IOMMU_WRITE; if ((attr & (ARM_V7S_TEX_MASK << ARM_V7S_TEX_SHIFT)) == 0) prot |= IOMMU_MMIO; else if (pte & ARM_V7S_ATTR_C) prot |= IOMMU_CACHE; + if (pte & ARM_V7S_ATTR_XN(lvl)) + prot |= IOMMU_NOEXEC; return prot; } @@ -631,6 +633,10 @@ static struct io_pgtable *arm_v7s_alloc_pgtable(struct io_pgtable_cfg *cfg, { struct arm_v7s_io_pgtable *data; +#ifdef PHYS_OFFSET + if (upper_32_bits(PHYS_OFFSET)) + return NULL; +#endif if (cfg->ias > ARM_V7S_ADDR_BITS || cfg->oas > ARM_V7S_ADDR_BITS) return NULL; diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c index b06d93594436..9a2f1960873b 100644 --- a/drivers/iommu/iommu.c +++ b/drivers/iommu/iommu.c @@ -31,6 +31,7 @@ #include <linux/err.h> #include <linux/pci.h> #include <linux/bitops.h> +#include <linux/property.h> #include <trace/events/iommu.h> static struct kset *iommu_group_kset; @@ -1613,3 +1614,60 @@ out: return ret; } + +int iommu_fwspec_init(struct device *dev, struct fwnode_handle *iommu_fwnode, + const struct iommu_ops *ops) +{ + struct iommu_fwspec *fwspec = dev->iommu_fwspec; + + if (fwspec) + return ops == fwspec->ops ? 0 : -EINVAL; + + fwspec = kzalloc(sizeof(*fwspec), GFP_KERNEL); + if (!fwspec) + return -ENOMEM; + + of_node_get(to_of_node(iommu_fwnode)); + fwspec->iommu_fwnode = iommu_fwnode; + fwspec->ops = ops; + dev->iommu_fwspec = fwspec; + return 0; +} +EXPORT_SYMBOL_GPL(iommu_fwspec_init); + +void iommu_fwspec_free(struct device *dev) +{ + struct iommu_fwspec *fwspec = dev->iommu_fwspec; + + if (fwspec) { + fwnode_handle_put(fwspec->iommu_fwnode); + kfree(fwspec); + dev->iommu_fwspec = NULL; + } +} +EXPORT_SYMBOL_GPL(iommu_fwspec_free); + +int iommu_fwspec_add_ids(struct device *dev, u32 *ids, int num_ids) +{ + struct iommu_fwspec *fwspec = dev->iommu_fwspec; + size_t size; + int i; + + if (!fwspec) + return -EINVAL; + + size = offsetof(struct iommu_fwspec, ids[fwspec->num_ids + num_ids]); + if (size > sizeof(*fwspec)) { + fwspec = krealloc(dev->iommu_fwspec, size, GFP_KERNEL); + if (!fwspec) + return -ENOMEM; + } + + for (i = 0; i < num_ids; i++) + fwspec->ids[fwspec->num_ids + i] = ids[i]; + + fwspec->num_ids += num_ids; + dev->iommu_fwspec = fwspec; + return 0; +} +EXPORT_SYMBOL_GPL(iommu_fwspec_add_ids); diff --git a/drivers/iommu/ipmmu-vmsa.c b/drivers/iommu/ipmmu-vmsa.c index 2fdbac67a77f..ace331da6459 100644 --- a/drivers/iommu/ipmmu-vmsa.c +++ b/drivers/iommu/ipmmu-vmsa.c @@ -636,7 +636,7 @@ static int ipmmu_add_device(struct device *dev) spin_unlock(&ipmmu_devices_lock); if (ret < 0) - return -ENODEV; + goto error; for (i = 0; i < num_utlbs; ++i) { if (utlbs[i] >= mmu->num_utlbs) { diff --git a/drivers/iommu/of_iommu.c b/drivers/iommu/of_iommu.c index 57f23eaaa2f9..5b82862f571f 100644 --- a/drivers/iommu/of_iommu.c +++ b/drivers/iommu/of_iommu.c @@ -22,6 +22,7 @@ #include <linux/limits.h> #include <linux/of.h> #include <linux/of_iommu.h> +#include <linux/of_pci.h> #include <linux/slab.h> static const struct of_device_id __iommu_of_table_sentinel @@ -134,6 +135,47 @@ const struct iommu_ops *of_iommu_get_ops(struct device_node *np) return ops; } +static int __get_pci_rid(struct pci_dev *pdev, u16 alias, void *data) +{ + struct of_phandle_args *iommu_spec = data; + + iommu_spec->args[0] = alias; + return iommu_spec->np == pdev->bus->dev.of_node; +} + +static const struct iommu_ops +*of_pci_iommu_configure(struct pci_dev *pdev, struct device_node *bridge_np) +{ + const struct iommu_ops *ops; + struct of_phandle_args iommu_spec; + + /* + * Start by tracing the RID alias down the PCI topology as + * far as the host bridge whose OF node we have... + * (we're not even attempting to handle multi-alias devices yet) + */ + iommu_spec.args_count = 1; + iommu_spec.np = bridge_np; + pci_for_each_dma_alias(pdev, __get_pci_rid, &iommu_spec); + /* + * ...then find out what that becomes once it escapes the PCI + * bus into the system beyond, and which IOMMU it ends up at. + */ + iommu_spec.np = NULL; + if (of_pci_map_rid(bridge_np, iommu_spec.args[0], "iommu-map", + "iommu-map-mask", &iommu_spec.np, iommu_spec.args)) + return NULL; + + ops = of_iommu_get_ops(iommu_spec.np); + if (!ops || !ops->of_xlate || + iommu_fwspec_init(&pdev->dev, &iommu_spec.np->fwnode, ops) || + ops->of_xlate(&pdev->dev, &iommu_spec)) + ops = NULL; + + of_node_put(iommu_spec.np); + return ops; +} + const struct iommu_ops *of_iommu_configure(struct device *dev, struct device_node *master_np) { @@ -142,12 +184,8 @@ const struct iommu_ops *of_iommu_configure(struct device *dev, const struct iommu_ops *ops = NULL; int idx = 0; - /* - * We can't do much for PCI devices without knowing how - * device IDs are wired up from the PCI bus to the IOMMU. - */ if (dev_is_pci(dev)) - return NULL; + return of_pci_iommu_configure(to_pci_dev(dev), master_np); /* * We don't currently walk up the tree looking for a parent IOMMU. @@ -160,7 +198,9 @@ const struct iommu_ops *of_iommu_configure(struct device *dev, np = iommu_spec.np; ops = of_iommu_get_ops(np); - if (!ops || !ops->of_xlate || ops->of_xlate(dev, &iommu_spec)) + if (!ops || !ops->of_xlate || + iommu_fwspec_init(dev, &np->fwnode, ops) || + ops->of_xlate(dev, &iommu_spec)) goto err_put_node; of_node_put(np); |