diff options
Diffstat (limited to 'drivers/iommu/dma-iommu.c')
| -rw-r--r-- | drivers/iommu/dma-iommu.c | 471 |
1 files changed, 369 insertions, 102 deletions
diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c index 129c4badf9ae..f802255219d3 100644 --- a/drivers/iommu/dma-iommu.c +++ b/drivers/iommu/dma-iommu.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0-only /* * A fairly generic DMA-API to IOMMU-API glue layer. * @@ -5,23 +6,13 @@ * * based in part on arch/arm/mm/dma-mapping.c: * Copyright (C) 2000-2004 Russell King - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. */ #include <linux/acpi_iort.h> #include <linux/device.h> +#include <linux/dma-contiguous.h> #include <linux/dma-iommu.h> +#include <linux/dma-noncoherent.h> #include <linux/gfp.h> #include <linux/huge_mm.h> #include <linux/iommu.h> @@ -78,11 +69,6 @@ static struct iommu_dma_cookie *cookie_alloc(enum iommu_dma_cookie_type type) return cookie; } -int iommu_dma_init(void) -{ - return iova_cache_get(); -} - /** * iommu_get_dma_cookie - Acquire DMA-API resources for a domain * @domain: IOMMU domain to prepare for DMA-API usage @@ -240,8 +226,8 @@ resv_iova: start = window->res->end - window->offset + 1; /* If window is last entry */ if (window->node.next == &bridge->dma_ranges && - end != ~(dma_addr_t)0) { - end = ~(dma_addr_t)0; + end != ~(phys_addr_t)0) { + end = ~(phys_addr_t)0; goto resv_iova; } } @@ -313,7 +299,7 @@ static void iommu_dma_flush_iotlb_all(struct iova_domain *iovad) * to ensure it is an invalid IOVA. It is safe to reinitialise a domain, but * any change which could make prior IOVAs invalid will fail. */ -int iommu_dma_init_domain(struct iommu_domain *domain, dma_addr_t base, +static int iommu_dma_init_domain(struct iommu_domain *domain, dma_addr_t base, u64 size, struct device *dev) { struct iommu_dma_cookie *cookie = domain->iova_cookie; @@ -364,7 +350,6 @@ int iommu_dma_init_domain(struct iommu_domain *domain, dma_addr_t base, return iova_reserve_iommu_regions(dev, domain); } -EXPORT_SYMBOL(iommu_dma_init_domain); /** * dma_info_to_prot - Translate DMA API directions and attributes to IOMMU API @@ -375,7 +360,7 @@ EXPORT_SYMBOL(iommu_dma_init_domain); * * Return: corresponding IOMMU API page protection flags */ -int dma_info_to_prot(enum dma_data_direction dir, bool coherent, +static int dma_info_to_prot(enum dma_data_direction dir, bool coherent, unsigned long attrs) { int prot = coherent ? IOMMU_CACHE : 0; @@ -452,9 +437,10 @@ static void iommu_dma_free_iova(struct iommu_dma_cookie *cookie, size >> iova_shift(iovad)); } -static void __iommu_dma_unmap(struct iommu_domain *domain, dma_addr_t dma_addr, +static void __iommu_dma_unmap(struct device *dev, dma_addr_t dma_addr, size_t size) { + struct iommu_domain *domain = iommu_get_dma_domain(dev); struct iommu_dma_cookie *cookie = domain->iova_cookie; struct iova_domain *iovad = &cookie->iovad; size_t iova_off = iova_offset(iovad, dma_addr); @@ -468,6 +454,30 @@ static void __iommu_dma_unmap(struct iommu_domain *domain, dma_addr_t dma_addr, iommu_dma_free_iova(cookie, dma_addr, size); } +static dma_addr_t __iommu_dma_map(struct device *dev, phys_addr_t phys, + size_t size, int prot) +{ + struct iommu_domain *domain = iommu_get_dma_domain(dev); + struct iommu_dma_cookie *cookie = domain->iova_cookie; + size_t iova_off = 0; + dma_addr_t iova; + + if (cookie->type == IOMMU_DMA_IOVA_COOKIE) { + iova_off = iova_offset(&cookie->iovad, phys); + size = iova_align(&cookie->iovad, size + iova_off); + } + + iova = iommu_dma_alloc_iova(domain, size, dma_get_mask(dev), dev); + if (!iova) + return DMA_MAPPING_ERROR; + + if (iommu_map(domain, iova, phys - iova_off, size, prot)) { + iommu_dma_free_iova(cookie, iova, size); + return DMA_MAPPING_ERROR; + } + return iova + iova_off; +} + static void __iommu_dma_free_pages(struct page **pages, int count) { while (count--) @@ -533,55 +543,45 @@ static struct page **__iommu_dma_alloc_pages(struct device *dev, return pages; } -/** - * iommu_dma_free - Free a buffer allocated by iommu_dma_alloc() - * @dev: Device which owns this buffer - * @pages: Array of buffer pages as returned by iommu_dma_alloc() - * @size: Size of buffer in bytes - * @handle: DMA address of buffer - * - * Frees both the pages associated with the buffer, and the array - * describing them - */ -void iommu_dma_free(struct device *dev, struct page **pages, size_t size, - dma_addr_t *handle) +static struct page **__iommu_dma_get_pages(void *cpu_addr) { - __iommu_dma_unmap(iommu_get_dma_domain(dev), *handle, size); - __iommu_dma_free_pages(pages, PAGE_ALIGN(size) >> PAGE_SHIFT); - *handle = DMA_MAPPING_ERROR; + struct vm_struct *area = find_vm_area(cpu_addr); + + if (!area || !area->pages) + return NULL; + return area->pages; } /** - * iommu_dma_alloc - Allocate and map a buffer contiguous in IOVA space + * iommu_dma_alloc_remap - Allocate and map a buffer contiguous in IOVA space * @dev: Device to allocate memory for. Must be a real device * attached to an iommu_dma_domain * @size: Size of buffer in bytes + * @dma_handle: Out argument for allocated DMA handle * @gfp: Allocation flags * @attrs: DMA attributes for this allocation - * @prot: IOMMU mapping flags - * @handle: Out argument for allocated DMA handle - * @flush_page: Arch callback which must ensure PAGE_SIZE bytes from the - * given VA/PA are visible to the given non-coherent device. * * If @size is less than PAGE_SIZE, then a full CPU page will be allocated, * but an IOMMU which supports smaller pages might not map the whole thing. * - * Return: Array of struct page pointers describing the buffer, - * or NULL on failure. + * Return: Mapped virtual address, or NULL on failure. */ -struct page **iommu_dma_alloc(struct device *dev, size_t size, gfp_t gfp, - unsigned long attrs, int prot, dma_addr_t *handle, - void (*flush_page)(struct device *, const void *, phys_addr_t)) +static void *iommu_dma_alloc_remap(struct device *dev, size_t size, + dma_addr_t *dma_handle, gfp_t gfp, unsigned long attrs) { struct iommu_domain *domain = iommu_get_dma_domain(dev); struct iommu_dma_cookie *cookie = domain->iova_cookie; struct iova_domain *iovad = &cookie->iovad; + bool coherent = dev_is_dma_coherent(dev); + int ioprot = dma_info_to_prot(DMA_BIDIRECTIONAL, coherent, attrs); + pgprot_t prot = arch_dma_mmap_pgprot(dev, PAGE_KERNEL, attrs); + unsigned int count, min_size, alloc_sizes = domain->pgsize_bitmap; struct page **pages; struct sg_table sgt; dma_addr_t iova; - unsigned int count, min_size, alloc_sizes = domain->pgsize_bitmap; + void *vaddr; - *handle = DMA_MAPPING_ERROR; + *dma_handle = DMA_MAPPING_ERROR; min_size = alloc_sizes & -alloc_sizes; if (min_size < PAGE_SIZE) { @@ -607,26 +607,29 @@ struct page **iommu_dma_alloc(struct device *dev, size_t size, gfp_t gfp, if (sg_alloc_table_from_pages(&sgt, pages, count, 0, size, GFP_KERNEL)) goto out_free_iova; - if (!(prot & IOMMU_CACHE)) { - struct sg_mapping_iter miter; - /* - * The CPU-centric flushing implied by SG_MITER_TO_SG isn't - * sufficient here, so skip it by using the "wrong" direction. - */ - sg_miter_start(&miter, sgt.sgl, sgt.orig_nents, SG_MITER_FROM_SG); - while (sg_miter_next(&miter)) - flush_page(dev, miter.addr, page_to_phys(miter.page)); - sg_miter_stop(&miter); + if (!(ioprot & IOMMU_CACHE)) { + struct scatterlist *sg; + int i; + + for_each_sg(sgt.sgl, sg, sgt.orig_nents, i) + arch_dma_prep_coherent(sg_page(sg), sg->length); } - if (iommu_map_sg(domain, iova, sgt.sgl, sgt.orig_nents, prot) + if (iommu_map_sg(domain, iova, sgt.sgl, sgt.orig_nents, ioprot) < size) goto out_free_sg; - *handle = iova; + vaddr = dma_common_pages_remap(pages, size, VM_USERMAP, prot, + __builtin_return_address(0)); + if (!vaddr) + goto out_unmap; + + *dma_handle = iova; sg_free_table(&sgt); - return pages; + return vaddr; +out_unmap: + __iommu_dma_unmap(dev, iova, size); out_free_sg: sg_free_table(&sgt); out_free_iova: @@ -637,54 +640,94 @@ out_free_pages: } /** - * iommu_dma_mmap - Map a buffer into provided user VMA - * @pages: Array representing buffer from iommu_dma_alloc() + * __iommu_dma_mmap - Map a buffer into provided user VMA + * @pages: Array representing buffer from __iommu_dma_alloc() * @size: Size of buffer in bytes * @vma: VMA describing requested userspace mapping * * Maps the pages of the buffer in @pages into @vma. The caller is responsible * for verifying the correct size and protection of @vma beforehand. */ - -int iommu_dma_mmap(struct page **pages, size_t size, struct vm_area_struct *vma) +static int __iommu_dma_mmap(struct page **pages, size_t size, + struct vm_area_struct *vma) { return vm_map_pages(vma, pages, PAGE_ALIGN(size) >> PAGE_SHIFT); } -static dma_addr_t __iommu_dma_map(struct device *dev, phys_addr_t phys, - size_t size, int prot, struct iommu_domain *domain) +static void iommu_dma_sync_single_for_cpu(struct device *dev, + dma_addr_t dma_handle, size_t size, enum dma_data_direction dir) { - struct iommu_dma_cookie *cookie = domain->iova_cookie; - size_t iova_off = 0; - dma_addr_t iova; + phys_addr_t phys; - if (cookie->type == IOMMU_DMA_IOVA_COOKIE) { - iova_off = iova_offset(&cookie->iovad, phys); - size = iova_align(&cookie->iovad, size + iova_off); - } + if (dev_is_dma_coherent(dev)) + return; - iova = iommu_dma_alloc_iova(domain, size, dma_get_mask(dev), dev); - if (!iova) - return DMA_MAPPING_ERROR; + phys = iommu_iova_to_phys(iommu_get_dma_domain(dev), dma_handle); + arch_sync_dma_for_cpu(dev, phys, size, dir); +} - if (iommu_map(domain, iova, phys - iova_off, size, prot)) { - iommu_dma_free_iova(cookie, iova, size); - return DMA_MAPPING_ERROR; - } - return iova + iova_off; +static void iommu_dma_sync_single_for_device(struct device *dev, + dma_addr_t dma_handle, size_t size, enum dma_data_direction dir) +{ + phys_addr_t phys; + + if (dev_is_dma_coherent(dev)) + return; + + phys = iommu_iova_to_phys(iommu_get_dma_domain(dev), dma_handle); + arch_sync_dma_for_device(dev, phys, size, dir); +} + +static void iommu_dma_sync_sg_for_cpu(struct device *dev, + struct scatterlist *sgl, int nelems, + enum dma_data_direction dir) +{ + struct scatterlist *sg; + int i; + + if (dev_is_dma_coherent(dev)) + return; + + for_each_sg(sgl, sg, nelems, i) + arch_sync_dma_for_cpu(dev, sg_phys(sg), sg->length, dir); +} + +static void iommu_dma_sync_sg_for_device(struct device *dev, + struct scatterlist *sgl, int nelems, + enum dma_data_direction dir) +{ + struct scatterlist *sg; + int i; + + if (dev_is_dma_coherent(dev)) + return; + + for_each_sg(sgl, sg, nelems, i) + arch_sync_dma_for_device(dev, sg_phys(sg), sg->length, dir); } -dma_addr_t iommu_dma_map_page(struct device *dev, struct page *page, - unsigned long offset, size_t size, int prot) +static dma_addr_t iommu_dma_map_page(struct device *dev, struct page *page, + unsigned long offset, size_t size, enum dma_data_direction dir, + unsigned long attrs) { - return __iommu_dma_map(dev, page_to_phys(page) + offset, size, prot, - iommu_get_dma_domain(dev)); + phys_addr_t phys = page_to_phys(page) + offset; + bool coherent = dev_is_dma_coherent(dev); + int prot = dma_info_to_prot(dir, coherent, attrs); + dma_addr_t dma_handle; + + dma_handle =__iommu_dma_map(dev, phys, size, prot); + if (!coherent && !(attrs & DMA_ATTR_SKIP_CPU_SYNC) && + dma_handle != DMA_MAPPING_ERROR) + arch_sync_dma_for_device(dev, phys, size, dir); + return dma_handle; } -void iommu_dma_unmap_page(struct device *dev, dma_addr_t handle, size_t size, - enum dma_data_direction dir, unsigned long attrs) +static void iommu_dma_unmap_page(struct device *dev, dma_addr_t dma_handle, + size_t size, enum dma_data_direction dir, unsigned long attrs) { - __iommu_dma_unmap(iommu_get_dma_domain(dev), handle, size); + if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC)) + iommu_dma_sync_single_for_cpu(dev, dma_handle, size, dir); + __iommu_dma_unmap(dev, dma_handle, size); } /* @@ -769,18 +812,22 @@ static void __invalidate_sg(struct scatterlist *sg, int nents) * impedance-matching, to be able to hand off a suitably-aligned list, * but still preserve the original offsets and sizes for the caller. */ -int iommu_dma_map_sg(struct device *dev, struct scatterlist *sg, - int nents, int prot) +static int iommu_dma_map_sg(struct device *dev, struct scatterlist *sg, + int nents, enum dma_data_direction dir, unsigned long attrs) { struct iommu_domain *domain = iommu_get_dma_domain(dev); struct iommu_dma_cookie *cookie = domain->iova_cookie; struct iova_domain *iovad = &cookie->iovad; struct scatterlist *s, *prev = NULL; + int prot = dma_info_to_prot(dir, dev_is_dma_coherent(dev), attrs); dma_addr_t iova; size_t iova_len = 0; unsigned long mask = dma_get_seg_boundary(dev); int i; + if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC)) + iommu_dma_sync_sg_for_device(dev, sg, nents, dir); + /* * Work out how much IOVA space we need, and align the segments to * IOVA granules for the IOMMU driver to handle. With some clever @@ -840,12 +887,16 @@ out_restore_sg: return 0; } -void iommu_dma_unmap_sg(struct device *dev, struct scatterlist *sg, int nents, - enum dma_data_direction dir, unsigned long attrs) +static void iommu_dma_unmap_sg(struct device *dev, struct scatterlist *sg, + int nents, enum dma_data_direction dir, unsigned long attrs) { dma_addr_t start, end; struct scatterlist *tmp; int i; + + if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC)) + iommu_dma_sync_sg_for_cpu(dev, sg, nents, dir); + /* * The scatterlist segments are mapped into a single * contiguous IOVA allocation, so this is incredibly easy. @@ -857,21 +908,231 @@ void iommu_dma_unmap_sg(struct device *dev, struct scatterlist *sg, int nents, sg = tmp; } end = sg_dma_address(sg) + sg_dma_len(sg); - __iommu_dma_unmap(iommu_get_dma_domain(dev), start, end - start); + __iommu_dma_unmap(dev, start, end - start); } -dma_addr_t iommu_dma_map_resource(struct device *dev, phys_addr_t phys, +static dma_addr_t iommu_dma_map_resource(struct device *dev, phys_addr_t phys, size_t size, enum dma_data_direction dir, unsigned long attrs) { return __iommu_dma_map(dev, phys, size, - dma_info_to_prot(dir, false, attrs) | IOMMU_MMIO, - iommu_get_dma_domain(dev)); + dma_info_to_prot(dir, false, attrs) | IOMMU_MMIO); } -void iommu_dma_unmap_resource(struct device *dev, dma_addr_t handle, +static void iommu_dma_unmap_resource(struct device *dev, dma_addr_t handle, size_t size, enum dma_data_direction dir, unsigned long attrs) { - __iommu_dma_unmap(iommu_get_dma_domain(dev), handle, size); + __iommu_dma_unmap(dev, handle, size); +} + +static void __iommu_dma_free(struct device *dev, size_t size, void *cpu_addr) +{ + size_t alloc_size = PAGE_ALIGN(size); + int count = alloc_size >> PAGE_SHIFT; + struct page *page = NULL, **pages = NULL; + + /* Non-coherent atomic allocation? Easy */ + if (IS_ENABLED(CONFIG_DMA_DIRECT_REMAP) && + dma_free_from_pool(cpu_addr, alloc_size)) + return; + + if (IS_ENABLED(CONFIG_DMA_REMAP) && is_vmalloc_addr(cpu_addr)) { + /* + * If it the address is remapped, then it's either non-coherent + * or highmem CMA, or an iommu_dma_alloc_remap() construction. + */ + pages = __iommu_dma_get_pages(cpu_addr); + if (!pages) + page = vmalloc_to_page(cpu_addr); + dma_common_free_remap(cpu_addr, alloc_size, VM_USERMAP); + } else { + /* Lowmem means a coherent atomic or CMA allocation */ + page = virt_to_page(cpu_addr); + } + + if (pages) + __iommu_dma_free_pages(pages, count); + if (page && !dma_release_from_contiguous(dev, page, count)) + __free_pages(page, get_order(alloc_size)); +} + +static void iommu_dma_free(struct device *dev, size_t size, void *cpu_addr, + dma_addr_t handle, unsigned long attrs) +{ + __iommu_dma_unmap(dev, handle, size); + __iommu_dma_free(dev, size, cpu_addr); +} + +static void *iommu_dma_alloc_pages(struct device *dev, size_t size, + struct page **pagep, gfp_t gfp, unsigned long attrs) +{ + bool coherent = dev_is_dma_coherent(dev); + size_t alloc_size = PAGE_ALIGN(size); + struct page *page = NULL; + void *cpu_addr; + + if (gfpflags_allow_blocking(gfp)) + page = dma_alloc_from_contiguous(dev, alloc_size >> PAGE_SHIFT, + get_order(alloc_size), + gfp & __GFP_NOWARN); + if (!page) + page = alloc_pages(gfp, get_order(alloc_size)); + if (!page) + return NULL; + + if (IS_ENABLED(CONFIG_DMA_REMAP) && (!coherent || PageHighMem(page))) { + pgprot_t prot = arch_dma_mmap_pgprot(dev, PAGE_KERNEL, attrs); + + cpu_addr = dma_common_contiguous_remap(page, alloc_size, + VM_USERMAP, prot, __builtin_return_address(0)); + if (!cpu_addr) + goto out_free_pages; + + if (!coherent) + arch_dma_prep_coherent(page, size); + } else { + cpu_addr = page_address(page); + } + + *pagep = page; + memset(cpu_addr, 0, alloc_size); + return cpu_addr; +out_free_pages: + if (!dma_release_from_contiguous(dev, page, alloc_size >> PAGE_SHIFT)) + __free_pages(page, get_order(alloc_size)); + return NULL; +} + +static void *iommu_dma_alloc(struct device *dev, size_t size, + dma_addr_t *handle, gfp_t gfp, unsigned long attrs) +{ + bool coherent = dev_is_dma_coherent(dev); + int ioprot = dma_info_to_prot(DMA_BIDIRECTIONAL, coherent, attrs); + struct page *page = NULL; + void *cpu_addr; + + gfp |= __GFP_ZERO; + + if (IS_ENABLED(CONFIG_DMA_REMAP) && gfpflags_allow_blocking(gfp) && + !(attrs & DMA_ATTR_FORCE_CONTIGUOUS)) + return iommu_dma_alloc_remap(dev, size, handle, gfp, attrs); + + if (IS_ENABLED(CONFIG_DMA_DIRECT_REMAP) && + !gfpflags_allow_blocking(gfp) && !coherent) + cpu_addr = dma_alloc_from_pool(PAGE_ALIGN(size), &page, gfp); + else + cpu_addr = iommu_dma_alloc_pages(dev, size, &page, gfp, attrs); + if (!cpu_addr) + return NULL; + + *handle = __iommu_dma_map(dev, page_to_phys(page), size, ioprot); + if (*handle == DMA_MAPPING_ERROR) { + __iommu_dma_free(dev, size, cpu_addr); + return NULL; + } + + return cpu_addr; +} + +static int iommu_dma_mmap(struct device *dev, struct vm_area_struct *vma, + void *cpu_addr, dma_addr_t dma_addr, size_t size, + unsigned long attrs) +{ + unsigned long nr_pages = PAGE_ALIGN(size) >> PAGE_SHIFT; + unsigned long pfn, off = vma->vm_pgoff; + int ret; + + vma->vm_page_prot = arch_dma_mmap_pgprot(dev, vma->vm_page_prot, attrs); + + if (dma_mmap_from_dev_coherent(dev, vma, cpu_addr, size, &ret)) + return ret; + + if (off >= nr_pages || vma_pages(vma) > nr_pages - off) + return -ENXIO; + + if (IS_ENABLED(CONFIG_DMA_REMAP) && is_vmalloc_addr(cpu_addr)) { + struct page **pages = __iommu_dma_get_pages(cpu_addr); + + if (pages) + return __iommu_dma_mmap(pages, size, vma); + pfn = vmalloc_to_pfn(cpu_addr); + } else { + pfn = page_to_pfn(virt_to_page(cpu_addr)); + } + + return remap_pfn_range(vma, vma->vm_start, pfn + off, + vma->vm_end - vma->vm_start, + vma->vm_page_prot); +} + +static int iommu_dma_get_sgtable(struct device *dev, struct sg_table *sgt, + void *cpu_addr, dma_addr_t dma_addr, size_t size, + unsigned long attrs) +{ + struct page *page; + int ret; + + if (IS_ENABLED(CONFIG_DMA_REMAP) && is_vmalloc_addr(cpu_addr)) { + struct page **pages = __iommu_dma_get_pages(cpu_addr); + + if (pages) { + return sg_alloc_table_from_pages(sgt, pages, + PAGE_ALIGN(size) >> PAGE_SHIFT, + 0, size, GFP_KERNEL); + } + + page = vmalloc_to_page(cpu_addr); + } else { + page = virt_to_page(cpu_addr); + } + + ret = sg_alloc_table(sgt, 1, GFP_KERNEL); + if (!ret) + sg_set_page(sgt->sgl, page, PAGE_ALIGN(size), 0); + return ret; +} + +static const struct dma_map_ops iommu_dma_ops = { + .alloc = iommu_dma_alloc, + .free = iommu_dma_free, + .mmap = iommu_dma_mmap, + .get_sgtable = iommu_dma_get_sgtable, + .map_page = iommu_dma_map_page, + .unmap_page = iommu_dma_unmap_page, + .map_sg = iommu_dma_map_sg, + .unmap_sg = iommu_dma_unmap_sg, + .sync_single_for_cpu = iommu_dma_sync_single_for_cpu, + .sync_single_for_device = iommu_dma_sync_single_for_device, + .sync_sg_for_cpu = iommu_dma_sync_sg_for_cpu, + .sync_sg_for_device = iommu_dma_sync_sg_for_device, + .map_resource = iommu_dma_map_resource, + .unmap_resource = iommu_dma_unmap_resource, +}; + +/* + * The IOMMU core code allocates the default DMA domain, which the underlying + * IOMMU driver needs to support via the dma-iommu layer. + */ +void iommu_setup_dma_ops(struct device *dev, u64 dma_base, u64 size) +{ + struct iommu_domain *domain = iommu_get_domain_for_dev(dev); + + if (!domain) + goto out_err; + + /* + * The IOMMU core code allocates the default DMA domain, which the + * underlying IOMMU driver needs to support via the dma-iommu layer. + */ + if (domain->type == IOMMU_DOMAIN_DMA) { + if (iommu_dma_init_domain(domain, dma_base, size, dev)) + goto out_err; + dev->dma_ops = &iommu_dma_ops; + } + + return; +out_err: + pr_warn("Failed to set up IOMMU for device %s; retaining platform DMA ops\n", + dev_name(dev)); } static struct iommu_dma_msi_page *iommu_dma_get_msi_page(struct device *dev, @@ -892,7 +1153,7 @@ static struct iommu_dma_msi_page *iommu_dma_get_msi_page(struct device *dev, if (!msi_page) return NULL; - iova = __iommu_dma_map(dev, msi_addr, size, prot, domain); + iova = __iommu_dma_map(dev, msi_addr, size, prot); if (iova == DMA_MAPPING_ERROR) goto out_free_page; @@ -954,3 +1215,9 @@ void iommu_dma_compose_msi_msg(struct msi_desc *desc, msg->address_lo &= cookie_msi_granule(domain->iova_cookie) - 1; msg->address_lo += lower_32_bits(msi_page->iova); } + +static int iommu_dma_init(void) +{ + return iova_cache_get(); +} +arch_initcall(iommu_dma_init); |