diff options
Diffstat (limited to 'drivers/iommu')
| -rw-r--r-- | drivers/iommu/Kconfig | 25 | ||||
| -rw-r--r-- | drivers/iommu/Makefile | 5 | ||||
| -rw-r--r-- | drivers/iommu/dmar.c | 246 | ||||
| -rw-r--r-- | drivers/iommu/intel-iommu.c | 278 | ||||
| -rw-r--r-- | drivers/iommu/intr_remapping.c | 93 |
5 files changed, 351 insertions, 296 deletions
diff --git a/drivers/iommu/Kconfig b/drivers/iommu/Kconfig index ae46776c1b80..5414253b185a 100644 --- a/drivers/iommu/Kconfig +++ b/drivers/iommu/Kconfig @@ -59,10 +59,14 @@ config AMD_IOMMU_STATS If unsure, say N. # Intel IOMMU support -config DMAR - bool "Support for DMA Remapping Devices" +config DMAR_TABLE + bool + +config INTEL_IOMMU + bool "Support for Intel IOMMU using DMA Remapping Devices" depends on PCI_MSI && ACPI && (X86 || IA64_GENERIC) select IOMMU_API + select DMAR_TABLE help DMA remapping (DMAR) devices support enables independent address translations for Direct Memory Access (DMA) from devices. @@ -70,18 +74,18 @@ config DMAR and include PCI device scope covered by these DMA remapping devices. -config DMAR_DEFAULT_ON +config INTEL_IOMMU_DEFAULT_ON def_bool y - prompt "Enable DMA Remapping Devices by default" - depends on DMAR + prompt "Enable Intel DMA Remapping Devices by default" + depends on INTEL_IOMMU help Selecting this option will enable a DMAR device at boot time if one is found. If this option is not selected, DMAR support can be enabled by passing intel_iommu=on to the kernel. -config DMAR_BROKEN_GFX_WA +config INTEL_IOMMU_BROKEN_GFX_WA bool "Workaround broken graphics drivers (going away soon)" - depends on DMAR && BROKEN && X86 + depends on INTEL_IOMMU && BROKEN && X86 ---help--- Current Graphics drivers tend to use physical address for DMA and avoid using DMA APIs. Setting this config @@ -90,18 +94,19 @@ config DMAR_BROKEN_GFX_WA to use physical addresses for DMA, at least until this option is removed in the 2.6.32 kernel. -config DMAR_FLOPPY_WA +config INTEL_IOMMU_FLOPPY_WA def_bool y - depends on DMAR && X86 + depends on INTEL_IOMMU && X86 ---help--- Floppy disk drivers are known to bypass DMA API calls thereby failing to work when IOMMU is enabled. This workaround will setup a 1:1 mapping for the first 16MiB to make floppy (an ISA device) work. -config INTR_REMAP +config IRQ_REMAP bool "Support for Interrupt Remapping (EXPERIMENTAL)" depends on X86_64 && X86_IO_APIC && PCI_MSI && ACPI && EXPERIMENTAL + select DMAR_TABLE ---help--- Supports Interrupt remapping for IO-APIC and MSI devices. To use x2apic mode in the CPU's which support x2APIC enhancements or diff --git a/drivers/iommu/Makefile b/drivers/iommu/Makefile index f798cdd3699e..2f4448794bc7 100644 --- a/drivers/iommu/Makefile +++ b/drivers/iommu/Makefile @@ -1,8 +1,9 @@ obj-$(CONFIG_IOMMU_API) += iommu.o obj-$(CONFIG_MSM_IOMMU) += msm_iommu.o msm_iommu_dev.o obj-$(CONFIG_AMD_IOMMU) += amd_iommu.o amd_iommu_init.o -obj-$(CONFIG_DMAR) += dmar.o iova.o intel-iommu.o -obj-$(CONFIG_INTR_REMAP) += dmar.o intr_remapping.o +obj-$(CONFIG_DMAR_TABLE) += dmar.o +obj-$(CONFIG_INTEL_IOMMU) += iova.o intel-iommu.o +obj-$(CONFIG_IRQ_REMAP) += intr_remapping.o obj-$(CONFIG_OMAP_IOMMU) += omap-iommu.o obj-$(CONFIG_OMAP_IOVMM) += omap-iovmm.o obj-$(CONFIG_OMAP_IOMMU_DEBUG) += omap-iommu-debug.o diff --git a/drivers/iommu/dmar.c b/drivers/iommu/dmar.c index 6dcc7e2d54de..35c1e17fce1d 100644 --- a/drivers/iommu/dmar.c +++ b/drivers/iommu/dmar.c @@ -46,7 +46,7 @@ */ LIST_HEAD(dmar_drhd_units); -static struct acpi_table_header * __initdata dmar_tbl; +struct acpi_table_header * __initdata dmar_tbl; static acpi_size dmar_tbl_size; static void __init dmar_register_drhd_unit(struct dmar_drhd_unit *drhd) @@ -118,8 +118,8 @@ static int __init dmar_parse_one_dev_scope(struct acpi_dmar_device_scope *scope, return 0; } -static int __init dmar_parse_dev_scope(void *start, void *end, int *cnt, - struct pci_dev ***devices, u16 segment) +int __init dmar_parse_dev_scope(void *start, void *end, int *cnt, + struct pci_dev ***devices, u16 segment) { struct acpi_dmar_device_scope *scope; void * tmp = start; @@ -217,133 +217,6 @@ static int __init dmar_parse_dev(struct dmar_drhd_unit *dmaru) return ret; } -#ifdef CONFIG_DMAR -LIST_HEAD(dmar_rmrr_units); - -static void __init dmar_register_rmrr_unit(struct dmar_rmrr_unit *rmrr) -{ - list_add(&rmrr->list, &dmar_rmrr_units); -} - - -static int __init -dmar_parse_one_rmrr(struct acpi_dmar_header *header) -{ - struct acpi_dmar_reserved_memory *rmrr; - struct dmar_rmrr_unit *rmrru; - - rmrru = kzalloc(sizeof(*rmrru), GFP_KERNEL); - if (!rmrru) - return -ENOMEM; - - rmrru->hdr = header; - rmrr = (struct acpi_dmar_reserved_memory *)header; - rmrru->base_address = rmrr->base_address; - rmrru->end_address = rmrr->end_address; - - dmar_register_rmrr_unit(rmrru); - return 0; -} - -static int __init -rmrr_parse_dev(struct dmar_rmrr_unit *rmrru) -{ - struct acpi_dmar_reserved_memory *rmrr; - int ret; - - rmrr = (struct acpi_dmar_reserved_memory *) rmrru->hdr; - ret = dmar_parse_dev_scope((void *)(rmrr + 1), - ((void *)rmrr) + rmrr->header.length, - &rmrru->devices_cnt, &rmrru->devices, rmrr->segment); - - if (ret || (rmrru->devices_cnt == 0)) { - list_del(&rmrru->list); - kfree(rmrru); - } - return ret; -} - -static LIST_HEAD(dmar_atsr_units); - -static int __init dmar_parse_one_atsr(struct acpi_dmar_header *hdr) -{ - struct acpi_dmar_atsr *atsr; - struct dmar_atsr_unit *atsru; - - atsr = container_of(hdr, struct acpi_dmar_atsr, header); - atsru = kzalloc(sizeof(*atsru), GFP_KERNEL); - if (!atsru) - return -ENOMEM; - - atsru->hdr = hdr; - atsru->include_all = atsr->flags & 0x1; - - list_add(&atsru->list, &dmar_atsr_units); - - return 0; -} - -static int __init atsr_parse_dev(struct dmar_atsr_unit *atsru) -{ - int rc; - struct acpi_dmar_atsr *atsr; - - if (atsru->include_all) - return 0; - - atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header); - rc = dmar_parse_dev_scope((void *)(atsr + 1), - (void *)atsr + atsr->header.length, - &atsru->devices_cnt, &atsru->devices, - atsr->segment); - if (rc || !atsru->devices_cnt) { - list_del(&atsru->list); - kfree(atsru); - } - - return rc; -} - -int dmar_find_matched_atsr_unit(struct pci_dev *dev) -{ - int i; - struct pci_bus *bus; - struct acpi_dmar_atsr *atsr; - struct dmar_atsr_unit *atsru; - - dev = pci_physfn(dev); - - list_for_each_entry(atsru, &dmar_atsr_units, list) { - atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header); - if (atsr->segment == pci_domain_nr(dev->bus)) - goto found; - } - - return 0; - -found: - for (bus = dev->bus; bus; bus = bus->parent) { - struct pci_dev *bridge = bus->self; - - if (!bridge || !pci_is_pcie(bridge) || - bridge->pcie_type == PCI_EXP_TYPE_PCI_BRIDGE) - return 0; - - if (bridge->pcie_type == PCI_EXP_TYPE_ROOT_PORT) { - for (i = 0; i < atsru->devices_cnt; i++) - if (atsru->devices[i] == bridge) - return 1; - break; - } - } - - if (atsru->include_all) - return 1; - - return 0; -} -#endif - #ifdef CONFIG_ACPI_NUMA static int __init dmar_parse_one_rhsa(struct acpi_dmar_header *header) @@ -484,14 +357,10 @@ parse_dmar_table(void) ret = dmar_parse_one_drhd(entry_header); break; case ACPI_DMAR_TYPE_RESERVED_MEMORY: -#ifdef CONFIG_DMAR ret = dmar_parse_one_rmrr(entry_header); -#endif break; case ACPI_DMAR_TYPE_ATSR: -#ifdef CONFIG_DMAR ret = dmar_parse_one_atsr(entry_header); -#endif break; case ACPI_DMAR_HARDWARE_AFFINITY: #ifdef CONFIG_ACPI_NUMA @@ -557,34 +426,31 @@ dmar_find_matched_drhd_unit(struct pci_dev *dev) int __init dmar_dev_scope_init(void) { + static int dmar_dev_scope_initialized; struct dmar_drhd_unit *drhd, *drhd_n; int ret = -ENODEV; + if (dmar_dev_scope_initialized) + return dmar_dev_scope_initialized; + + if (list_empty(&dmar_drhd_units)) + goto fail; + list_for_each_entry_safe(drhd, drhd_n, &dmar_drhd_units, list) { ret = dmar_parse_dev(drhd); if (ret) - return ret; + goto fail; } -#ifdef CONFIG_DMAR - { - struct dmar_rmrr_unit *rmrr, *rmrr_n; - struct dmar_atsr_unit *atsr, *atsr_n; - - list_for_each_entry_safe(rmrr, rmrr_n, &dmar_rmrr_units, list) { - ret = rmrr_parse_dev(rmrr); - if (ret) - return ret; - } + ret = dmar_parse_rmrr_atsr_dev(); + if (ret) + goto fail; - list_for_each_entry_safe(atsr, atsr_n, &dmar_atsr_units, list) { - ret = atsr_parse_dev(atsr); - if (ret) - return ret; - } - } -#endif + dmar_dev_scope_initialized = 1; + return 0; +fail: + dmar_dev_scope_initialized = ret; return ret; } @@ -611,14 +477,6 @@ int __init dmar_table_init(void) return -ENODEV; } -#ifdef CONFIG_DMAR - if (list_empty(&dmar_rmrr_units)) - printk(KERN_INFO PREFIX "No RMRR found\n"); - - if (list_empty(&dmar_atsr_units)) - printk(KERN_INFO PREFIX "No ATSR found\n"); -#endif - return 0; } @@ -682,9 +540,6 @@ int __init check_zero_address(void) return 1; failed: -#ifdef CONFIG_DMAR - dmar_disabled = 1; -#endif return 0; } @@ -696,22 +551,21 @@ int __init detect_intel_iommu(void) if (ret) ret = check_zero_address(); { -#ifdef CONFIG_INTR_REMAP struct acpi_table_dmar *dmar; dmar = (struct acpi_table_dmar *) dmar_tbl; - if (ret && cpu_has_x2apic && dmar->flags & 0x1) + + if (ret && intr_remapping_enabled && cpu_has_x2apic && + dmar->flags & 0x1) printk(KERN_INFO - "Queued invalidation will be enabled to support " - "x2apic and Intr-remapping.\n"); -#endif -#ifdef CONFIG_DMAR + "Queued invalidation will be enabled to support x2apic and Intr-remapping.\n"); + if (ret && !no_iommu && !iommu_detected && !dmar_disabled) { iommu_detected = 1; /* Make sure ACS will be enabled */ pci_request_acs(); } -#endif + #ifdef CONFIG_X86 if (ret) x86_init.iommu.iommu_init = intel_iommu_init; @@ -758,7 +612,6 @@ int alloc_iommu(struct dmar_drhd_unit *drhd) goto err_unmap; } -#ifdef CONFIG_DMAR agaw = iommu_calculate_agaw(iommu); if (agaw < 0) { printk(KERN_ERR @@ -773,7 +626,6 @@ int alloc_iommu(struct dmar_drhd_unit *drhd) iommu->seq_id); goto err_unmap; } -#endif iommu->agaw = agaw; iommu->msagaw = msagaw; @@ -800,7 +652,7 @@ int alloc_iommu(struct dmar_drhd_unit *drhd) (unsigned long long)iommu->cap, (unsigned long long)iommu->ecap); - spin_lock_init(&iommu->register_lock); + raw_spin_lock_init(&iommu->register_lock); drhd->iommu = iommu; return 0; @@ -817,9 +669,7 @@ void free_iommu(struct intel_iommu *iommu) if (!iommu) return; -#ifdef CONFIG_DMAR free_dmar_iommu(iommu); -#endif if (iommu->reg) iounmap(iommu->reg); @@ -921,11 +771,11 @@ int qi_submit_sync(struct qi_desc *desc, struct intel_iommu *iommu) restart: rc = 0; - spin_lock_irqsave(&qi->q_lock, flags); + raw_spin_lock_irqsave(&qi->q_lock, flags); while (qi->free_cnt < 3) { - spin_unlock_irqrestore(&qi->q_lock, flags); + raw_spin_unlock_irqrestore(&qi->q_lock, flags); cpu_relax(); - spin_lock_irqsave(&qi->q_lock, flags); + raw_spin_lock_irqsave(&qi->q_lock, flags); } index = qi->free_head; @@ -965,15 +815,15 @@ restart: if (rc) break; - spin_unlock(&qi->q_lock); + raw_spin_unlock(&qi->q_lock); cpu_relax(); - spin_lock(&qi->q_lock); + raw_spin_lock(&qi->q_lock); } qi->desc_status[index] = QI_DONE; reclaim_free_desc(qi); - spin_unlock_irqrestore(&qi->q_lock, flags); + raw_spin_unlock_irqrestore(&qi->q_lock, flags); if (rc == -EAGAIN) goto restart; @@ -1062,7 +912,7 @@ void dmar_disable_qi(struct intel_iommu *iommu) if (!ecap_qis(iommu->ecap)) return; - spin_lock_irqsave(&iommu->register_lock, flags); + raw_spin_lock_irqsave(&iommu->register_lock, flags); sts = dmar_readq(iommu->reg + DMAR_GSTS_REG); if (!(sts & DMA_GSTS_QIES)) @@ -1082,7 +932,7 @@ void dmar_disable_qi(struct intel_iommu *iommu) IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, readl, !(sts & DMA_GSTS_QIES), sts); end: - spin_unlock_irqrestore(&iommu->register_lock, flags); + raw_spin_unlock_irqrestore(&iommu->register_lock, flags); } /* @@ -1097,7 +947,7 @@ static void __dmar_enable_qi(struct intel_iommu *iommu) qi->free_head = qi->free_tail = 0; qi->free_cnt = QI_LENGTH; - spin_lock_irqsave(&iommu->register_lock, flags); + raw_spin_lock_irqsave(&iommu->register_lock, flags); /* write zero to the tail reg */ writel(0, iommu->reg + DMAR_IQT_REG); @@ -1110,7 +960,7 @@ static void __dmar_enable_qi(struct intel_iommu *iommu) /* Make sure hardware complete it */ IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, readl, (sts & DMA_GSTS_QIES), sts); - spin_unlock_irqrestore(&iommu->register_lock, flags); + raw_spin_unlock_irqrestore(&iommu->register_lock, flags); } /* @@ -1159,7 +1009,7 @@ int dmar_enable_qi(struct intel_iommu *iommu) qi->free_head = qi->free_tail = 0; qi->free_cnt = QI_LENGTH; - spin_lock_init(&qi->q_lock); + raw_spin_lock_init(&qi->q_lock); __dmar_enable_qi(iommu); @@ -1225,11 +1075,11 @@ void dmar_msi_unmask(struct irq_data *data) unsigned long flag; /* unmask it */ - spin_lock_irqsave(&iommu->register_lock, flag); + raw_spin_lock_irqsave(&iommu->register_lock, flag); writel(0, iommu->reg + DMAR_FECTL_REG); /* Read a reg to force flush the post write */ readl(iommu->reg + DMAR_FECTL_REG); - spin_unlock_irqrestore(&iommu->register_lock, flag); + raw_spin_unlock_irqrestore(&iommu->register_lock, flag); } void dmar_msi_mask(struct irq_data *data) @@ -1238,11 +1088,11 @@ void dmar_msi_mask(struct irq_data *data) struct intel_iommu *iommu = irq_data_get_irq_handler_data(data); /* mask it */ - spin_lock_irqsave(&iommu->register_lock, flag); + raw_spin_lock_irqsave(&iommu->register_lock, flag); writel(DMA_FECTL_IM, iommu->reg + DMAR_FECTL_REG); /* Read a reg to force flush the post write */ readl(iommu->reg + DMAR_FECTL_REG); - spin_unlock_irqrestore(&iommu->register_lock, flag); + raw_spin_unlock_irqrestore(&iommu->register_lock, flag); } void dmar_msi_write(int irq, struct msi_msg *msg) @@ -1250,11 +1100,11 @@ void dmar_msi_write(int irq, struct msi_msg *msg) struct intel_iommu *iommu = irq_get_handler_data(irq); unsigned long flag; - spin_lock_irqsave(&iommu->register_lock, flag); + raw_spin_lock_irqsave(&iommu->register_lock, flag); writel(msg->data, iommu->reg + DMAR_FEDATA_REG); writel(msg->address_lo, iommu->reg + DMAR_FEADDR_REG); writel(msg->address_hi, iommu->reg + DMAR_FEUADDR_REG); - spin_unlock_irqrestore(&iommu->register_lock, flag); + raw_spin_unlock_irqrestore(&iommu->register_lock, flag); } void dmar_msi_read(int irq, struct msi_msg *msg) @@ -1262,11 +1112,11 @@ void dmar_msi_read(int irq, struct msi_msg *msg) struct intel_iommu *iommu = irq_get_handler_data(irq); unsigned long flag; - spin_lock_irqsave(&iommu->register_lock, flag); + raw_spin_lock_irqsave(&iommu->register_lock, flag); msg->data = readl(iommu->reg + DMAR_FEDATA_REG); msg->address_lo = readl(iommu->reg + DMAR_FEADDR_REG); msg->address_hi = readl(iommu->reg + DMAR_FEUADDR_REG); - spin_unlock_irqrestore(&iommu->register_lock, flag); + raw_spin_unlock_irqrestore(&iommu->register_lock, flag); } static int dmar_fault_do_one(struct intel_iommu *iommu, int type, @@ -1303,7 +1153,7 @@ irqreturn_t dmar_fault(int irq, void *dev_id) u32 fault_status; unsigned long flag; - spin_lock_irqsave(&iommu->register_lock, flag); + raw_spin_lock_irqsave(&iommu->register_lock, flag); fault_status = readl(iommu->reg + DMAR_FSTS_REG); if (fault_status) printk(KERN_ERR "DRHD: handling fault status reg %x\n", @@ -1342,7 +1192,7 @@ irqreturn_t dmar_fault(int irq, void *dev_id) writel(DMA_FRCD_F, iommu->reg + reg + fault_index * PRIMARY_FAULT_REG_LEN + 12); - spin_unlock_irqrestore(&iommu->register_lock, flag); + raw_spin_unlock_irqrestore(&iommu->register_lock, flag); dmar_fault_do_one(iommu, type, fault_reason, source_id, guest_addr); @@ -1350,14 +1200,14 @@ irqreturn_t dmar_fault(int irq, void *dev_id) fault_index++; if (fault_index >= cap_num_fault_regs(iommu->cap)) fault_index = 0; - spin_lock_irqsave(&iommu->register_lock, flag); + raw_spin_lock_irqsave(&iommu->register_lock, flag); } clear_rest: /* clear all the other faults */ fault_status = readl(iommu->reg + DMAR_FSTS_REG); writel(fault_status, iommu->reg + DMAR_FSTS_REG); - spin_unlock_irqrestore(&iommu->register_lock, flag); + raw_spin_unlock_irqrestore(&iommu->register_lock, flag); return IRQ_HANDLED; } diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c index 2d53c3df12dc..bb161d2fa03c 100644 --- a/drivers/iommu/intel-iommu.c +++ b/drivers/iommu/intel-iommu.c @@ -306,6 +306,11 @@ static inline bool dma_pte_present(struct dma_pte *pte) return (pte->val & 3) != 0; } +static inline bool dma_pte_superpage(struct dma_pte *pte) +{ + return (pte->val & (1 << 7)); +} + static inline int first_pte_in_page(struct dma_pte *pte) { return !((unsigned long)pte & ~VTD_PAGE_MASK); @@ -393,17 +398,20 @@ static long list_size; static void domain_remove_dev_info(struct dmar_domain *domain); -#ifdef CONFIG_DMAR_DEFAULT_ON +#ifdef CONFIG_INTEL_IOMMU_DEFAULT_ON int dmar_disabled = 0; #else int dmar_disabled = 1; -#endif /*CONFIG_DMAR_DEFAULT_ON*/ +#endif /*CONFIG_INTEL_IOMMU_DEFAULT_ON*/ static int dmar_map_gfx = 1; static int dmar_forcedac; static int intel_iommu_strict; static int intel_iommu_superpage = 1; +int intel_iommu_gfx_mapped; +EXPORT_SYMBOL_GPL(intel_iommu_gfx_mapped); + #define DUMMY_DEVICE_DOMAIN_INFO ((struct device_domain_info *)(-1)) static DEFINE_SPINLOCK(device_domain_lock); static LIST_HEAD(device_domain_list); @@ -577,17 +585,18 @@ static void domain_update_iommu_snooping(struct dmar_domain *domain) static void domain_update_iommu_superpage(struct dmar_domain *domain) { - int i, mask = 0xf; + struct dmar_drhd_unit *drhd; + struct intel_iommu *iommu = NULL; + int mask = 0xf; if (!intel_iommu_superpage) { domain->iommu_superpage = 0; return; } - domain->iommu_superpage = 4; /* 1TiB */ - - for_each_set_bit(i, &domain->iommu_bmp, g_num_of_iommus) { - mask |= cap_super_page_val(g_iommus[i]->cap); + /* set iommu_superpage to the smallest common denominator */ + for_each_active_iommu(iommu, drhd) { + mask &= cap_super_page_val(iommu->cap); if (!mask) { break; } @@ -730,29 +739,23 @@ out: } static struct dma_pte *pfn_to_dma_pte(struct dmar_domain *domain, - unsigned long pfn, int large_level) + unsigned long pfn, int target_level) { int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT; struct dma_pte *parent, *pte = NULL; int level = agaw_to_level(domain->agaw); - int offset, target_level; + int offset; BUG_ON(!domain->pgd); BUG_ON(addr_width < BITS_PER_LONG && pfn >> addr_width); parent = domain->pgd; - /* Search pte */ - if (!large_level) - target_level = 1; - else - target_level = large_level; - while (level > 0) { void *tmp_page; offset = pfn_level_offset(pfn, level); pte = &parent[offset]; - if (!large_level && (pte->val & DMA_PTE_LARGE_PAGE)) + if (!target_level && (dma_pte_superpage(pte) || !dma_pte_present(pte))) break; if (level == target_level) break; @@ -816,13 +819,14 @@ static struct dma_pte *dma_pfn_level_pte(struct dmar_domain *domain, } /* clear last level pte, a tlb flush should be followed */ -static void dma_pte_clear_range(struct dmar_domain *domain, +static int dma_pte_clear_range(struct dmar_domain *domain, unsigned long start_pfn, unsigned long last_pfn) { int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT; unsigned int large_page = 1; struct dma_pte *first_pte, *pte; + int order; BUG_ON(addr_width < BITS_PER_LONG && start_pfn >> addr_width); BUG_ON(addr_width < BITS_PER_LONG && last_pfn >> addr_width); @@ -846,6 +850,9 @@ static void dma_pte_clear_range(struct dmar_domain *domain, (void *)pte - (void *)first_pte); } while (start_pfn && start_pfn <= last_pfn); + + order = (large_page - 1) * 9; + return order; } /* free page table pages. last level pte should already be cleared */ @@ -932,7 +939,7 @@ static void iommu_set_root_entry(struct intel_iommu *iommu) addr = iommu->root_entry; - spin_lock_irqsave(&iommu->register_lock, flag); + raw_spin_lock_irqsave(&iommu->register_lock, flag); dmar_writeq(iommu->reg + DMAR_RTADDR_REG, virt_to_phys(addr)); writel(iommu->gcmd | DMA_GCMD_SRTP, iommu->reg + DMAR_GCMD_REG); @@ -941,7 +948,7 @@ static void iommu_set_root_entry(struct intel_iommu *iommu) IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, readl, (sts & DMA_GSTS_RTPS), sts); - spin_unlock_irqrestore(&iommu->register_lock, flag); + raw_spin_unlock_irqrestore(&iommu->register_lock, flag); } static void iommu_flush_write_buffer(struct intel_iommu *iommu) @@ -952,14 +959,14 @@ static void iommu_flush_write_buffer(struct intel_iommu *iommu) if (!rwbf_quirk && !cap_rwbf(iommu->cap)) return; - spin_lock_irqsave(&iommu->register_lock, flag); + raw_spin_lock_irqsave(&iommu->register_lock, flag); writel(iommu->gcmd | DMA_GCMD_WBF, iommu->reg + DMAR_GCMD_REG); /* Make sure hardware complete it */ IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, readl, (!(val & DMA_GSTS_WBFS)), val); - spin_unlock_irqrestore(&iommu->register_lock, flag); + raw_spin_unlock_irqrestore(&iommu->register_lock, flag); } /* return value determine if we need a write buffer flush */ @@ -986,14 +993,14 @@ static void __iommu_flush_context(struct intel_iommu *iommu, } val |= DMA_CCMD_ICC; - spin_lock_irqsave(&iommu->register_lock, flag); + raw_spin_lock_irqsave(&iommu->register_lock, flag); dmar_writeq(iommu->reg + DMAR_CCMD_REG, val); /* Make sure hardware complete it */ IOMMU_WAIT_OP(iommu, DMAR_CCMD_REG, dmar_readq, (!(val & DMA_CCMD_ICC)), val); - spin_unlock_irqrestore(&iommu->register_lock, flag); + raw_spin_unlock_irqrestore(&iommu->register_lock, flag); } /* return value determine if we need a write buffer flush */ @@ -1032,7 +1039,7 @@ static void __iommu_flush_iotlb(struct intel_iommu *iommu, u16 did, if (cap_write_drain(iommu->cap)) val |= DMA_TLB_WRITE_DRAIN; - spin_lock_irqsave(&iommu->register_lock, flag); + raw_spin_lock_irqsave(&iommu->register_lock, flag); /* Note: Only uses first TLB reg currently */ if (val_iva) dmar_writeq(iommu->reg + tlb_offset, val_iva); @@ -1042,7 +1049,7 @@ static void __iommu_flush_iotlb(struct intel_iommu *iommu, u16 did, IOMMU_WAIT_OP(iommu, tlb_offset + 8, dmar_readq, (!(val & DMA_TLB_IVT)), val); - spin_unlock_irqrestore(&iommu->register_lock, flag); + raw_spin_unlock_irqrestore(&iommu->register_lock, flag); /* check IOTLB invalidation granularity */ if (DMA_TLB_IAIG(val) == 0) @@ -1158,7 +1165,7 @@ static void iommu_disable_protect_mem_regions(struct intel_iommu *iommu) u32 pmen; unsigned long flags; - spin_lock_irqsave(&iommu->register_lock, flags); + raw_spin_lock_irqsave(&iommu->register_lock, flags); pmen = readl(iommu->reg + DMAR_PMEN_REG); pmen &= ~DMA_PMEN_EPM; writel(pmen, iommu->reg + DMAR_PMEN_REG); @@ -1167,7 +1174,7 @@ static void iommu_disable_protect_mem_regions(struct intel_iommu *iommu) IOMMU_WAIT_OP(iommu, DMAR_PMEN_REG, readl, !(pmen & DMA_PMEN_PRS), pmen); - spin_unlock_irqrestore(&iommu->register_lock, flags); + raw_spin_unlock_irqrestore(&iommu->register_lock, flags); } static int iommu_enable_translation(struct intel_iommu *iommu) @@ -1175,7 +1182,7 @@ static int iommu_enable_translation(struct intel_iommu *iommu) u32 sts; unsigned long flags; - spin_lock_irqsave(&iommu->register_lock, flags); + raw_spin_lock_irqsave(&iommu->register_lock, flags); iommu->gcmd |= DMA_GCMD_TE; writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG); @@ -1183,7 +1190,7 @@ static int iommu_enable_translation(struct intel_iommu *iommu) IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, readl, (sts & DMA_GSTS_TES), sts); - spin_unlock_irqrestore(&iommu->register_lock, flags); + raw_spin_unlock_irqrestore(&iommu->register_lock, flags); return 0; } @@ -1192,7 +1199,7 @@ static int iommu_disable_translation(struct intel_iommu *iommu) u32 sts; unsigned long flag; - spin_lock_irqsave(&iommu->register_lock, flag); + raw_spin_lock_irqsave(&iommu->register_lock, flag); iommu->gcmd &= ~DMA_GCMD_TE; writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG); @@ -1200,7 +1207,7 @@ static int iommu_disable_translation(struct intel_iommu *iommu) IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, readl, (!(sts & DMA_GSTS_TES)), sts); - spin_unlock_irqrestore(&iommu->register_lock, flag); + raw_spin_unlock_irqrestore(&iommu->register_lock, flag); return 0; } @@ -2150,7 +2157,7 @@ static inline int iommu_prepare_rmrr_dev(struct dmar_rmrr_unit *rmrr, rmrr->end_address); } -#ifdef CONFIG_DMAR_FLOPPY_WA +#ifdef CONFIG_INTEL_IOMMU_FLOPPY_WA static inline void iommu_prepare_isa(void) { struct pci_dev *pdev; @@ -2173,7 +2180,7 @@ static inline void iommu_prepare_isa(void) { return; } -#endif /* !CONFIG_DMAR_FLPY_WA */ +#endif /* !CONFIG_INTEL_IOMMU_FLPY_WA */ static int md_domain_init(struct dmar_domain *domain, int guest_width); @@ -2484,7 +2491,7 @@ static int __init init_dmars(void) if (iommu_pass_through) iommu_identity_mapping |= IDENTMAP_ALL; -#ifdef CONFIG_DMAR_BROKEN_GFX_WA +#ifdef CONFIG_INTEL_IOMMU_BROKEN_GFX_WA iommu_identity_mapping |= IDENTMAP_GFX; #endif @@ -3226,9 +3233,6 @@ static void __init init_no_remapping_devices(void) } } - if (dmar_map_gfx) - return; - for_each_drhd_unit(drhd) { int i; if (drhd->ignored || drhd->include_all) @@ -3236,18 +3240,23 @@ static void __init init_no_remapping_devices(void) for (i = 0; i < drhd->devices_cnt; i++) if (drhd->devices[i] && - !IS_GFX_DEVICE(drhd->devices[i])) + !IS_GFX_DEVICE(drhd->devices[i])) break; if (i < drhd->devices_cnt) continue; - /* bypass IOMMU if it is just for gfx devices */ - drhd->ignored = 1; - for (i = 0; i < drhd->devices_cnt; i++) { - if (!drhd->devices[i]) - continue; - drhd->devices[i]->dev.archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO; + /* This IOMMU has *only* gfx devices. Either bypass it or + set the gfx_mapped flag, as appropriate */ + if (dmar_map_gfx) { + intel_iommu_gfx_mapped = 1; + } else { + drhd->ignored = 1; + for (i = 0; i < drhd->devices_cnt; i++) { + if (!drhd->devices[i]) + continue; + drhd->devices[i]->dev.archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO; + } } } } @@ -3320,7 +3329,7 @@ static int iommu_suspend(void) for_each_active_iommu(iommu, drhd) { iommu_disable_translation(iommu); - spin_lock_irqsave(&iommu->register_lock, flag); + raw_spin_lock_irqsave(&iommu->register_lock, flag); iommu->iommu_state[SR_DMAR_FECTL_REG] = readl(iommu->reg + DMAR_FECTL_REG); @@ -3331,7 +3340,7 @@ static int iommu_suspend(void) iommu->iommu_state[SR_DMAR_FEUADDR_REG] = readl(iommu->reg + DMAR_FEUADDR_REG); - spin_unlock_irqrestore(&iommu->register_lock, flag); + raw_spin_unlock_irqrestore(&iommu->register_lock, flag); } return 0; @@ -3358,7 +3367,7 @@ static void iommu_resume(void) for_each_active_iommu(iommu, drhd) { - spin_lock_irqsave(&iommu->register_lock, flag); + raw_spin_lock_irqsave(&iommu->register_lock, flag); writel(iommu->iommu_state[SR_DMAR_FECTL_REG], iommu->reg + DMAR_FECTL_REG); @@ -3369,7 +3378,7 @@ static void iommu_resume(void) writel(iommu->iommu_state[SR_DMAR_FEUADDR_REG], iommu->reg + DMAR_FEUADDR_REG); - spin_unlock_irqrestore(&iommu->register_lock, flag); + raw_spin_unlock_irqrestore(&iommu->register_lock, flag); } for_each_active_iommu(iommu, drhd) @@ -3390,6 +3399,151 @@ static void __init init_iommu_pm_ops(void) static inline void init_iommu_pm_ops(void) {} #endif /* CONFIG_PM */ +LIST_HEAD(dmar_rmrr_units); + +static void __init dmar_register_rmrr_unit(struct dmar_rmrr_unit *rmrr) +{ + list_add(&rmrr->list, &dmar_rmrr_units); +} + + +int __init dmar_parse_one_rmrr(struct acpi_dmar_header *header) +{ + struct acpi_dmar_reserved_memory *rmrr; + struct dmar_rmrr_unit *rmrru; + + rmrru = kzalloc(sizeof(*rmrru), GFP_KERNEL); + if (!rmrru) + return -ENOMEM; + + rmrru->hdr = header; + rmrr = (struct acpi_dmar_reserved_memory *)header; + rmrru->base_address = rmrr->base_address; + rmrru->end_address = rmrr->end_address; + + dmar_register_rmrr_unit(rmrru); + return 0; +} + +static int __init +rmrr_parse_dev(struct dmar_rmrr_unit *rmrru) +{ + struct acpi_dmar_reserved_memory *rmrr; + int ret; + + rmrr = (struct acpi_dmar_reserved_memory *) rmrru->hdr; + ret = dmar_parse_dev_scope((void *)(rmrr + 1), + ((void *)rmrr) + rmrr->header.length, + &rmrru->devices_cnt, &rmrru->devices, rmrr->segment); + + if (ret || (rmrru->devices_cnt == 0)) { + list_del(&rmrru->list); + kfree(rmrru); + } + return ret; +} + +static LIST_HEAD(dmar_atsr_units); + +int __init dmar_parse_one_atsr(struct acpi_dmar_header *hdr) +{ + struct acpi_dmar_atsr *atsr; + struct dmar_atsr_unit *atsru; + + atsr = container_of(hdr, struct acpi_dmar_atsr, header); + atsru = kzalloc(sizeof(*atsru), GFP_KERNEL); + if (!atsru) + return -ENOMEM; + + atsru->hdr = hdr; + atsru->include_all = atsr->flags & 0x1; + + list_add(&atsru->list, &dmar_atsr_units); + + return 0; +} + +static int __init atsr_parse_dev(struct dmar_atsr_unit *atsru) +{ + int rc; + struct acpi_dmar_atsr *atsr; + + if (atsru->include_all) + return 0; + + atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header); + rc = dmar_parse_dev_scope((void *)(atsr + 1), + (void *)atsr + atsr->header.length, + &atsru->devices_cnt, &atsru->devices, + atsr->segment); + if (rc || !atsru->devices_cnt) { + list_del(&atsru->list); + kfree(atsru); + } + + return rc; +} + +int dmar_find_matched_atsr_unit(struct pci_dev *dev) +{ + int i; + struct pci_bus *bus; + struct acpi_dmar_atsr *atsr; + struct dmar_atsr_unit *atsru; + + dev = pci_physfn(dev); + + list_for_each_entry(atsru, &dmar_atsr_units, list) { + atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header); + if (atsr->segment == pci_domain_nr(dev->bus)) + goto found; + } + + return 0; + +found: + for (bus = dev->bus; bus; bus = bus->parent) { + struct pci_dev *bridge = bus->self; + + if (!bridge || !pci_is_pcie(bridge) || + bridge->pcie_type == PCI_EXP_TYPE_PCI_BRIDGE) + return 0; + + if (bridge->pcie_type == PCI_EXP_TYPE_ROOT_PORT) { + for (i = 0; i < atsru->devices_cnt; i++) + if (atsru->devices[i] == bridge) + return 1; + break; + } + } + + if (atsru->include_all) + return 1; + + return 0; +} + +int dmar_parse_rmrr_atsr_dev(void) +{ + struct dmar_rmrr_unit *rmrr, *rmrr_n; + struct dmar_atsr_unit *atsr, *atsr_n; + int ret = 0; + + list_for_each_entry_safe(rmrr, rmrr_n, &dmar_rmrr_units, list) { + ret = rmrr_parse_dev(rmrr); + if (ret) + return ret; + } + + list_for_each_entry_safe(atsr, atsr_n, &dmar_atsr_units, list) { + ret = atsr_parse_dev(atsr); + if (ret) + return ret; + } + + return ret; +} + /* * Here we only respond to action of unbound device from driver. * @@ -3439,16 +3593,12 @@ int __init intel_iommu_init(void) return -ENODEV; } - if (dmar_dev_scope_init()) { + if (dmar_dev_scope_init() < 0) { if (force_on) panic("tboot: Failed to initialize DMAR device scope\n"); return -ENODEV; } - /* - * Check the need for DMA-remapping initialization now. - * Above initialization will also be used by Interrupt-remapping. - */ if (no_iommu || dmar_disabled) return -ENODEV; @@ -3458,6 +3608,12 @@ int __init intel_iommu_init(void) return -ENODEV; } + if (list_empty(&dmar_rmrr_units)) + printk(KERN_INFO "DMAR: No RMRR found\n"); + + if (list_empty(&dmar_atsr_units)) + printk(KERN_INFO "DMAR: No ATSR found\n"); + if (dmar_init_reserved_ranges()) { if (force_on) panic("tboot: Failed to reserve iommu ranges\n"); @@ -3568,6 +3724,8 @@ static void domain_remove_one_dev_info(struct dmar_domain *domain, found = 1; } + spin_unlock_irqrestore(&device_domain_lock, flags); + if (found == 0) { unsigned long tmp_flags; spin_lock_irqsave(&domain->iommu_lock, tmp_flags); @@ -3584,8 +3742,6 @@ static void domain_remove_one_dev_info(struct dmar_domain *domain, spin_unlock_irqrestore(&iommu->lock, tmp_flags); } } - - spin_unlock_irqrestore(&device_domain_lock, flags); } static void vm_domain_remove_all_dev_info(struct dmar_domain *domain) @@ -3739,6 +3895,7 @@ static int intel_iommu_domain_init(struct iommu_domain *domain) vm_domain_exit(dmar_domain); return -ENOMEM; } + domain_update_iommu_cap(dmar_domain); domain->priv = dmar_domain; return 0; @@ -3864,14 +4021,15 @@ static int intel_iommu_unmap(struct iommu_domain *domain, { struct dmar_domain *dmar_domain = domain->priv; size_t size = PAGE_SIZE << gfp_order; + int order; - dma_pte_clear_range(dmar_domain, iova >> VTD_PAGE_SHIFT, + order = dma_pte_clear_range(dmar_domain, iova >> VTD_PAGE_SHIFT, (iova + size - 1) >> VTD_PAGE_SHIFT); if (dmar_domain->max_addr == iova + size) dmar_domain->max_addr = iova; - return gfp_order; + return order; } static phys_addr_t intel_iommu_iova_to_phys(struct iommu_domain *domain, @@ -3950,7 +4108,11 @@ static void __devinit quirk_calpella_no_shadow_gtt(struct pci_dev *dev) if (!(ggc & GGC_MEMORY_VT_ENABLED)) { printk(KERN_INFO "DMAR: BIOS has allocated no shadow GTT; disabling IOMMU for graphics\n"); dmar_map_gfx = 0; - } + } else if (dmar_map_gfx) { + /* we have to ensure the gfx device is idle before we flush */ + printk(KERN_INFO "DMAR: Disabling batched IOTLB flush on Ironlake\n"); + intel_iommu_strict = 1; + } } DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0040, quirk_calpella_no_shadow_gtt); DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0044, quirk_calpella_no_shadow_gtt); diff --git a/drivers/iommu/intr_remapping.c b/drivers/iommu/intr_remapping.c index 1a89d4a2cadf..07c9f189f314 100644 --- a/drivers/iommu/intr_remapping.c +++ b/drivers/iommu/intr_remapping.c @@ -21,6 +21,7 @@ int intr_remapping_enabled; static int disable_intremap; static int disable_sourceid_checking; +static int no_x2apic_optout; static __init int setup_nointremap(char *str) { @@ -34,18 +35,26 @@ static __init int setup_intremap(char *str) if (!str) return -EINVAL; - if (!strncmp(str, "on", 2)) - disable_intremap = 0; - else if (!strncmp(str, "off", 3)) - disable_intremap = 1; - else if (!strncmp(str, "nosid", 5)) - disable_sourceid_checking = 1; + while (*str) { + if (!strncmp(str, "on", 2)) + disable_intremap = 0; + else if (!strncmp(str, "off", 3)) + disable_intremap = 1; + else if (!strncmp(str, "nosid", 5)) + disable_sourceid_checking = 1; + else if (!strncmp(str, "no_x2apic_optout", 16)) + no_x2apic_optout = 1; + + str += strcspn(str, ","); + while (*str == ',') + str++; + } return 0; } early_param("intremap", setup_intremap); -static DEFINE_SPINLOCK(irq_2_ir_lock); +static DEFINE_RAW_SPINLOCK(irq_2_ir_lock); static struct irq_2_iommu *irq_2_iommu(unsigned int irq) { @@ -62,12 +71,12 @@ int get_irte(int irq, struct irte *entry) if (!entry || !irq_iommu) return -1; - spin_lock_irqsave(&irq_2_ir_lock, flags); + raw_spin_lock_irqsave(&irq_2_ir_lock, flags); index = irq_iommu->irte_index + irq_iommu->sub_handle; *entry = *(irq_iommu->iommu->ir_table->base + index); - spin_unlock_irqrestore(&irq_2_ir_lock, flags); + raw_spin_unlock_irqrestore(&irq_2_ir_lock, flags); return 0; } @@ -101,7 +110,7 @@ int alloc_irte(struct intel_iommu *iommu, int irq, u16 count) return -1; } - spin_lock_irqsave(&irq_2_ir_lock, flags); + raw_spin_lock_irqsave(&irq_2_ir_lock, flags); do { for (i = index; i < index + count; i++) if (table->base[i].present) @@ -113,7 +122,7 @@ int alloc_irte(struct intel_iommu *iommu, int irq, u16 count) index = (index + count) % INTR_REMAP_TABLE_ENTRIES; if (index == start_index) { - spin_unlock_irqrestore(&irq_2_ir_lock, flags); + raw_spin_unlock_irqrestore(&irq_2_ir_lock, flags); printk(KERN_ERR "can't allocate an IRTE\n"); return -1; } @@ -127,7 +136,7 @@ int alloc_irte(struct intel_iommu *iommu, int irq, u16 count) irq_iommu->sub_handle = 0; irq_iommu->irte_mask = mask; - spin_unlock_irqrestore(&irq_2_ir_lock, flags); + raw_spin_unlock_irqrestore(&irq_2_ir_lock, flags); return index; } @@ -152,10 +161,10 @@ int map_irq_to_irte_handle(int irq, u16 *sub_handle) if (!irq_iommu) return -1; - spin_lock_irqsave(&irq_2_ir_lock, flags); + raw_spin_lock_irqsave(&irq_2_ir_lock, flags); *sub_handle = irq_iommu->sub_handle; index = irq_iommu->irte_index; - spin_unlock_irqrestore(&irq_2_ir_lock, flags); + raw_spin_unlock_irqrestore(&irq_2_ir_lock, flags); return index; } @@ -167,14 +176,14 @@ int set_irte_irq(int irq, struct intel_iommu *iommu, u16 index, u16 subhandle) if (!irq_iommu) return -1; - spin_lock_irqsave(&irq_2_ir_lock, flags); + raw_spin_lock_irqsave(&irq_2_ir_lock, flags); irq_iommu->iommu = iommu; irq_iommu->irte_index = index; irq_iommu->sub_handle = subhandle; irq_iommu->irte_mask = 0; - spin_unlock_irqrestore(&irq_2_ir_lock, flags); + raw_spin_unlock_irqrestore(&irq_2_ir_lock, flags); return 0; } @@ -190,7 +199,7 @@ int modify_irte(int irq, struct irte *irte_modified) if (!irq_iommu) return -1; - spin_lock_irqsave(&irq_2_ir_lock, flags); + raw_spin_lock_irqsave(&irq_2_ir_lock, flags); iommu = irq_iommu->iommu; @@ -202,7 +211,7 @@ int modify_irte(int irq, struct irte *irte_modified) __iommu_flush_cache(iommu, irte, sizeof(*irte)); rc = qi_flush_iec(iommu, index, 0); - spin_unlock_irqrestore(&irq_2_ir_lock, flags); + raw_spin_unlock_irqrestore(&irq_2_ir_lock, flags); return rc; } @@ -270,7 +279,7 @@ int free_irte(int irq) if (!irq_iommu) return -1; - spin_lock_irqsave(&irq_2_ir_lock, flags); + raw_spin_lock_irqsave(&irq_2_ir_lock, flags); rc = clear_entries(irq_iommu); @@ -279,7 +288,7 @@ int free_irte(int irq) irq_iommu->sub_handle = 0; irq_iommu->irte_mask = 0; - spin_unlock_irqrestore(&irq_2_ir_lock, flags); + raw_spin_unlock_irqrestore(&irq_2_ir_lock, flags); return rc; } @@ -409,7 +418,7 @@ static void iommu_set_intr_remapping(struct intel_iommu *iommu, int mode) addr = virt_to_phys((void *)iommu->ir_table->base); - spin_lock_irqsave(&iommu->register_lock, flags); + raw_spin_lock_irqsave(&iommu->register_lock, flags); dmar_writeq(iommu->reg + DMAR_IRTA_REG, (addr) | IR_X2APIC_MODE(mode) | INTR_REMAP_TABLE_REG_SIZE); @@ -420,7 +429,7 @@ static void iommu_set_intr_remapping(struct intel_iommu *iommu, int mode) IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, readl, (sts & DMA_GSTS_IRTPS), sts); - spin_unlock_irqrestore(&iommu->register_lock, flags); + raw_spin_unlock_irqrestore(&iommu->register_lock, flags); /* * global invalidation of interrupt entry cache before enabling @@ -428,7 +437,7 @@ static void iommu_set_intr_remapping(struct intel_iommu *iommu, int mode) */ qi_global_iec(iommu); - spin_lock_irqsave(&iommu->register_lock, flags); + raw_spin_lock_irqsave(&iommu->register_lock, flags); /* Enable interrupt-remapping */ iommu->gcmd |= DMA_GCMD_IRE; @@ -437,7 +446,7 @@ static void iommu_set_intr_remapping(struct intel_iommu *iommu, int mode) IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, readl, (sts & DMA_GSTS_IRES), sts); - spin_unlock_irqrestore(&iommu->register_lock, flags); + raw_spin_unlock_irqrestore(&iommu->register_lock, flags); } @@ -485,7 +494,7 @@ static void iommu_disable_intr_remapping(struct intel_iommu *iommu) */ qi_global_iec(iommu); - spin_lock_irqsave(&iommu->register_lock, flags); + raw_spin_lock_irqsave(&iommu->register_lock, flags); sts = dmar_readq(iommu->reg + DMAR_GSTS_REG); if (!(sts & DMA_GSTS_IRES)) @@ -498,7 +507,16 @@ static void iommu_disable_intr_remapping(struct intel_iommu *iommu) readl, !(sts & DMA_GSTS_IRES), sts); end: - spin_unlock_irqrestore(&iommu->register_lock, flags); + raw_spin_unlock_irqrestore(&iommu->register_lock, flags); +} + +static int __init dmar_x2apic_optout(void) +{ + struct acpi_table_dmar *dmar; + dmar = (struct acpi_table_dmar *)dmar_tbl; + if (!dmar || no_x2apic_optout) + return 0; + return dmar->flags & DMAR_X2APIC_OPT_OUT; } int __init intr_remapping_supported(void) @@ -521,16 +539,25 @@ int __init intr_remapping_supported(void) return 1; } -int __init enable_intr_remapping(int eim) +int __init enable_intr_remapping(void) { struct dmar_drhd_unit *drhd; int setup = 0; + int eim = 0; if (parse_ioapics_under_ir() != 1) { printk(KERN_INFO "Not enable interrupt remapping\n"); return -1; } + if (x2apic_supported()) { + eim = !dmar_x2apic_optout(); + WARN(!eim, KERN_WARNING + "Your BIOS is broken and requested that x2apic be disabled\n" + "This will leave your machine vulnerable to irq-injection attacks\n" + "Use 'intremap=no_x2apic_optout' to override BIOS request\n"); + } + for_each_drhd_unit(drhd) { struct intel_iommu *iommu = drhd->iommu; @@ -606,8 +633,9 @@ int __init enable_intr_remapping(int eim) goto error; intr_remapping_enabled = 1; + pr_info("Enabled IRQ remapping in %s mode\n", eim ? "x2apic" : "xapic"); - return 0; + return eim ? IRQ_REMAP_X2APIC_MODE : IRQ_REMAP_XAPIC_MODE; error: /* @@ -745,6 +773,15 @@ int __init parse_ioapics_under_ir(void) return ir_supported; } +int ir_dev_scope_init(void) +{ + if (!intr_remapping_enabled) + return 0; + + return dmar_dev_scope_init(); +} +rootfs_initcall(ir_dev_scope_init); + void disable_intr_remapping(void) { struct dmar_drhd_unit *drhd; |