From a445e940ea686fc60475564009821010eb213be3 Mon Sep 17 00:00:00 2001 From: Vladimir Murzin Date: Wed, 30 Oct 2019 10:13:13 +0000 Subject: dma-mapping: fix handling of dma-ranges for reserved memory (again) Daniele reported that issue previously fixed in c41f9ea998f3 ("drivers: dma-coherent: Account dma_pfn_offset when used with device tree") reappear shortly after 43fc509c3efb ("dma-coherent: introduce interface for default DMA pool") where fix was accidentally dropped. Lets put fix back in place and respect dma-ranges for reserved memory. Fixes: 43fc509c3efb ("dma-coherent: introduce interface for default DMA pool") Reported-by: Daniele Alessandrelli Tested-by: Daniele Alessandrelli Tested-by: Alexandre Torgue Signed-off-by: Vladimir Murzin Signed-off-by: Christoph Hellwig --- arch/arm/mm/dma-mapping-nommu.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'arch') diff --git a/arch/arm/mm/dma-mapping-nommu.c b/arch/arm/mm/dma-mapping-nommu.c index db9247898300..287ef898a55e 100644 --- a/arch/arm/mm/dma-mapping-nommu.c +++ b/arch/arm/mm/dma-mapping-nommu.c @@ -35,7 +35,7 @@ static void *arm_nommu_dma_alloc(struct device *dev, size_t size, unsigned long attrs) { - void *ret = dma_alloc_from_global_coherent(size, dma_handle); + void *ret = dma_alloc_from_global_coherent(dev, size, dma_handle); /* * dma_alloc_from_global_coherent() may fail because: -- cgit v1.2.3-59-g8ed1b From 34dc0ea6bc960f1f57b2148f01a3f4da23f87013 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Tue, 29 Oct 2019 11:01:37 +0100 Subject: dma-direct: provide mmap and get_sgtable method overrides For dma-direct we know that the DMA address is an encoding of the physical address that we can trivially decode. Use that fact to provide implementations that do not need the arch_dma_coherent_to_pfn architecture hook. Note that we still can only support mmap of non-coherent memory only if the architecture provides a way to set an uncached bit in the page tables. This must be true for architectures that use the generic remap helpers, but other architectures can also manually select it. Signed-off-by: Christoph Hellwig Reviewed-by: Max Filippov --- arch/arc/Kconfig | 1 - arch/arm/Kconfig | 1 - arch/arm/mm/dma-mapping.c | 6 ---- arch/arm64/Kconfig | 1 - arch/ia64/Kconfig | 2 +- arch/ia64/kernel/dma-mapping.c | 6 ---- arch/microblaze/Kconfig | 1 - arch/mips/Kconfig | 4 +-- arch/mips/mm/dma-noncoherent.c | 6 ---- arch/powerpc/platforms/Kconfig.cputype | 1 - include/linux/dma-direct.h | 7 ++++ include/linux/dma-noncoherent.h | 2 -- kernel/dma/Kconfig | 12 +++++-- kernel/dma/direct.c | 59 ++++++++++++++++++++++++++++++++++ kernel/dma/mapping.c | 45 ++++---------------------- kernel/dma/remap.c | 6 ---- 16 files changed, 85 insertions(+), 75 deletions(-) (limited to 'arch') diff --git a/arch/arc/Kconfig b/arch/arc/Kconfig index 8383155c8c82..4d7b671c8ff4 100644 --- a/arch/arc/Kconfig +++ b/arch/arc/Kconfig @@ -6,7 +6,6 @@ config ARC def_bool y select ARC_TIMERS - select ARCH_HAS_DMA_COHERENT_TO_PFN select ARCH_HAS_DMA_PREP_COHERENT select ARCH_HAS_PTE_SPECIAL select ARCH_HAS_SETUP_DMA_OPS diff --git a/arch/arm/Kconfig b/arch/arm/Kconfig index 8a50efb559f3..80e795aacd3a 100644 --- a/arch/arm/Kconfig +++ b/arch/arm/Kconfig @@ -7,7 +7,6 @@ config ARM select ARCH_HAS_BINFMT_FLAT select ARCH_HAS_DEBUG_VIRTUAL if MMU select ARCH_HAS_DEVMEM_IS_ALLOWED - select ARCH_HAS_DMA_COHERENT_TO_PFN if SWIOTLB select ARCH_HAS_DMA_WRITE_COMBINE if !ARM_DMA_MEM_BUFFERABLE select ARCH_HAS_ELF_RANDOMIZE select ARCH_HAS_FORTIFY_SOURCE diff --git a/arch/arm/mm/dma-mapping.c b/arch/arm/mm/dma-mapping.c index 7d042d5c43e3..f3cbeba7f9cb 100644 --- a/arch/arm/mm/dma-mapping.c +++ b/arch/arm/mm/dma-mapping.c @@ -2346,12 +2346,6 @@ void arch_sync_dma_for_cpu(struct device *dev, phys_addr_t paddr, size, dir); } -long arch_dma_coherent_to_pfn(struct device *dev, void *cpu_addr, - dma_addr_t dma_addr) -{ - return dma_to_pfn(dev, dma_addr); -} - void *arch_dma_alloc(struct device *dev, size_t size, dma_addr_t *dma_handle, gfp_t gfp, unsigned long attrs) { diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index 3f047afb982c..57606307fe34 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -12,7 +12,6 @@ config ARM64 select ARCH_CLOCKSOURCE_DATA select ARCH_HAS_DEBUG_VIRTUAL select ARCH_HAS_DEVMEM_IS_ALLOWED - select ARCH_HAS_DMA_COHERENT_TO_PFN select ARCH_HAS_DMA_PREP_COHERENT select ARCH_HAS_ACPI_TABLE_UPGRADE if ACPI select ARCH_HAS_FAST_MULTIPLIER diff --git a/arch/ia64/Kconfig b/arch/ia64/Kconfig index 16714477eef4..bab7cd878464 100644 --- a/arch/ia64/Kconfig +++ b/arch/ia64/Kconfig @@ -33,7 +33,7 @@ config IA64 select HAVE_ARCH_TRACEHOOK select HAVE_MEMBLOCK_NODE_MAP select HAVE_VIRT_CPU_ACCOUNTING - select ARCH_HAS_DMA_COHERENT_TO_PFN + select DMA_NONCOHERENT_MMAP select ARCH_HAS_SYNC_DMA_FOR_CPU select VIRT_TO_BUS select GENERIC_IRQ_PROBE diff --git a/arch/ia64/kernel/dma-mapping.c b/arch/ia64/kernel/dma-mapping.c index 4a3262795890..09ef9ce9988d 100644 --- a/arch/ia64/kernel/dma-mapping.c +++ b/arch/ia64/kernel/dma-mapping.c @@ -19,9 +19,3 @@ void arch_dma_free(struct device *dev, size_t size, void *cpu_addr, { dma_direct_free_pages(dev, size, cpu_addr, dma_addr, attrs); } - -long arch_dma_coherent_to_pfn(struct device *dev, void *cpu_addr, - dma_addr_t dma_addr) -{ - return page_to_pfn(virt_to_page(cpu_addr)); -} diff --git a/arch/microblaze/Kconfig b/arch/microblaze/Kconfig index c9c4be822456..261c26df1c9f 100644 --- a/arch/microblaze/Kconfig +++ b/arch/microblaze/Kconfig @@ -4,7 +4,6 @@ config MICROBLAZE select ARCH_32BIT_OFF_T select ARCH_NO_SWAP select ARCH_HAS_BINFMT_FLAT if !MMU - select ARCH_HAS_DMA_COHERENT_TO_PFN if MMU select ARCH_HAS_DMA_PREP_COHERENT select ARCH_HAS_GCOV_PROFILE_ALL select ARCH_HAS_SYNC_DMA_FOR_CPU diff --git a/arch/mips/Kconfig b/arch/mips/Kconfig index a0bd9bdb5f83..248d39b8a160 100644 --- a/arch/mips/Kconfig +++ b/arch/mips/Kconfig @@ -1134,9 +1134,9 @@ config DMA_NONCOHERENT select ARCH_HAS_DMA_WRITE_COMBINE select ARCH_HAS_SYNC_DMA_FOR_DEVICE select ARCH_HAS_UNCACHED_SEGMENT - select NEED_DMA_MAP_STATE - select ARCH_HAS_DMA_COHERENT_TO_PFN + select DMA_NONCOHERENT_MMAP select DMA_NONCOHERENT_CACHE_SYNC + select NEED_DMA_MAP_STATE config SYS_HAS_EARLY_PRINTK bool diff --git a/arch/mips/mm/dma-noncoherent.c b/arch/mips/mm/dma-noncoherent.c index 1d4d57dd9acf..fcf6d3eaac66 100644 --- a/arch/mips/mm/dma-noncoherent.c +++ b/arch/mips/mm/dma-noncoherent.c @@ -59,12 +59,6 @@ void *cached_kernel_address(void *addr) return __va(addr) - UNCAC_BASE; } -long arch_dma_coherent_to_pfn(struct device *dev, void *cpu_addr, - dma_addr_t dma_addr) -{ - return page_to_pfn(virt_to_page(cached_kernel_address(cpu_addr))); -} - static inline void dma_sync_virt(void *addr, size_t size, enum dma_data_direction dir) { diff --git a/arch/powerpc/platforms/Kconfig.cputype b/arch/powerpc/platforms/Kconfig.cputype index 12543e53fa96..303752f97c19 100644 --- a/arch/powerpc/platforms/Kconfig.cputype +++ b/arch/powerpc/platforms/Kconfig.cputype @@ -459,7 +459,6 @@ config NOT_COHERENT_CACHE bool depends on 4xx || PPC_8xx || E200 || PPC_MPC512x || \ GAMECUBE_COMMON || AMIGAONE - select ARCH_HAS_DMA_COHERENT_TO_PFN select ARCH_HAS_DMA_PREP_COHERENT select ARCH_HAS_SYNC_DMA_FOR_DEVICE select ARCH_HAS_SYNC_DMA_FOR_CPU diff --git a/include/linux/dma-direct.h b/include/linux/dma-direct.h index ff3d5edc44b9..bcd953fb1f5a 100644 --- a/include/linux/dma-direct.h +++ b/include/linux/dma-direct.h @@ -68,5 +68,12 @@ void dma_direct_free_pages(struct device *dev, size_t size, void *cpu_addr, dma_addr_t dma_addr, unsigned long attrs); struct page *__dma_direct_alloc_pages(struct device *dev, size_t size, gfp_t gfp, unsigned long attrs); +int dma_direct_get_sgtable(struct device *dev, struct sg_table *sgt, + void *cpu_addr, dma_addr_t dma_addr, size_t size, + unsigned long attrs); +bool dma_direct_can_mmap(struct device *dev); +int dma_direct_mmap(struct device *dev, struct vm_area_struct *vma, + void *cpu_addr, dma_addr_t dma_addr, size_t size, + unsigned long attrs); int dma_direct_supported(struct device *dev, u64 mask); #endif /* _LINUX_DMA_DIRECT_H */ diff --git a/include/linux/dma-noncoherent.h b/include/linux/dma-noncoherent.h index dd3de6d88fc0..e30fca1f1b12 100644 --- a/include/linux/dma-noncoherent.h +++ b/include/linux/dma-noncoherent.h @@ -41,8 +41,6 @@ void *arch_dma_alloc(struct device *dev, size_t size, dma_addr_t *dma_handle, gfp_t gfp, unsigned long attrs); void arch_dma_free(struct device *dev, size_t size, void *cpu_addr, dma_addr_t dma_addr, unsigned long attrs); -long arch_dma_coherent_to_pfn(struct device *dev, void *cpu_addr, - dma_addr_t dma_addr); #ifdef CONFIG_MMU /* diff --git a/kernel/dma/Kconfig b/kernel/dma/Kconfig index 73c5c2b8e824..4c103a24e380 100644 --- a/kernel/dma/Kconfig +++ b/kernel/dma/Kconfig @@ -51,9 +51,6 @@ config ARCH_HAS_SYNC_DMA_FOR_CPU_ALL config ARCH_HAS_DMA_PREP_COHERENT bool -config ARCH_HAS_DMA_COHERENT_TO_PFN - bool - config ARCH_HAS_FORCE_DMA_UNENCRYPTED bool @@ -68,9 +65,18 @@ config SWIOTLB bool select NEED_DMA_MAP_STATE +# +# Should be selected if we can mmap non-coherent mappings to userspace. +# The only thing that is really required is a way to set an uncached bit +# in the pagetables +# +config DMA_NONCOHERENT_MMAP + bool + config DMA_REMAP depends on MMU select GENERIC_ALLOCATOR + select DMA_NONCOHERENT_MMAP bool config DMA_DIRECT_REMAP diff --git a/kernel/dma/direct.c b/kernel/dma/direct.c index 724c282dd943..58beaa9ddd27 100644 --- a/kernel/dma/direct.c +++ b/kernel/dma/direct.c @@ -43,6 +43,12 @@ static inline dma_addr_t phys_to_dma_direct(struct device *dev, return phys_to_dma(dev, phys); } +static inline struct page *dma_direct_to_page(struct device *dev, + dma_addr_t dma_addr) +{ + return pfn_to_page(PHYS_PFN(dma_to_phys(dev, dma_addr))); +} + u64 dma_direct_get_required_mask(struct device *dev) { u64 max_dma = phys_to_dma_direct(dev, (max_pfn - 1) << PAGE_SHIFT); @@ -379,6 +385,59 @@ dma_addr_t dma_direct_map_resource(struct device *dev, phys_addr_t paddr, } EXPORT_SYMBOL(dma_direct_map_resource); +int dma_direct_get_sgtable(struct device *dev, struct sg_table *sgt, + void *cpu_addr, dma_addr_t dma_addr, size_t size, + unsigned long attrs) +{ + struct page *page = dma_direct_to_page(dev, dma_addr); + int ret; + + ret = sg_alloc_table(sgt, 1, GFP_KERNEL); + if (!ret) + sg_set_page(sgt->sgl, page, PAGE_ALIGN(size), 0); + return ret; +} + +#ifdef CONFIG_MMU +bool dma_direct_can_mmap(struct device *dev) +{ + return dev_is_dma_coherent(dev) || + IS_ENABLED(CONFIG_DMA_NONCOHERENT_MMAP); +} + +int dma_direct_mmap(struct device *dev, struct vm_area_struct *vma, + void *cpu_addr, dma_addr_t dma_addr, size_t size, + unsigned long attrs) +{ + unsigned long user_count = vma_pages(vma); + unsigned long count = PAGE_ALIGN(size) >> PAGE_SHIFT; + unsigned long pfn = PHYS_PFN(dma_to_phys(dev, dma_addr)); + int ret = -ENXIO; + + vma->vm_page_prot = dma_pgprot(dev, vma->vm_page_prot, attrs); + + if (dma_mmap_from_dev_coherent(dev, vma, cpu_addr, size, &ret)) + return ret; + + if (vma->vm_pgoff >= count || user_count > count - vma->vm_pgoff) + return -ENXIO; + return remap_pfn_range(vma, vma->vm_start, pfn + vma->vm_pgoff, + user_count << PAGE_SHIFT, vma->vm_page_prot); +} +#else /* CONFIG_MMU */ +bool dma_direct_can_mmap(struct device *dev) +{ + return false; +} + +int dma_direct_mmap(struct device *dev, struct vm_area_struct *vma, + void *cpu_addr, dma_addr_t dma_addr, size_t size, + unsigned long attrs) +{ + return -ENXIO; +} +#endif /* CONFIG_MMU */ + /* * Because 32-bit DMA masks are so common we expect every architecture to be * able to satisfy them - either by not supporting more physical memory, or by diff --git a/kernel/dma/mapping.c b/kernel/dma/mapping.c index d9334f31a5af..12ff766ec1fa 100644 --- a/kernel/dma/mapping.c +++ b/kernel/dma/mapping.c @@ -112,24 +112,9 @@ int dma_common_get_sgtable(struct device *dev, struct sg_table *sgt, void *cpu_addr, dma_addr_t dma_addr, size_t size, unsigned long attrs) { - struct page *page; + struct page *page = virt_to_page(cpu_addr); int ret; - if (!dev_is_dma_coherent(dev)) { - unsigned long pfn; - - if (!IS_ENABLED(CONFIG_ARCH_HAS_DMA_COHERENT_TO_PFN)) - return -ENXIO; - - /* If the PFN is not valid, we do not have a struct page */ - pfn = arch_dma_coherent_to_pfn(dev, cpu_addr, dma_addr); - if (!pfn_valid(pfn)) - return -ENXIO; - page = pfn_to_page(pfn); - } else { - page = virt_to_page(cpu_addr); - } - ret = sg_alloc_table(sgt, 1, GFP_KERNEL); if (!ret) sg_set_page(sgt->sgl, page, PAGE_ALIGN(size), 0); @@ -154,7 +139,7 @@ int dma_get_sgtable_attrs(struct device *dev, struct sg_table *sgt, const struct dma_map_ops *ops = get_dma_ops(dev); if (dma_is_direct(ops)) - return dma_common_get_sgtable(dev, sgt, cpu_addr, dma_addr, + return dma_direct_get_sgtable(dev, sgt, cpu_addr, dma_addr, size, attrs); if (!ops->get_sgtable) return -ENXIO; @@ -192,7 +177,6 @@ int dma_common_mmap(struct device *dev, struct vm_area_struct *vma, unsigned long user_count = vma_pages(vma); unsigned long count = PAGE_ALIGN(size) >> PAGE_SHIFT; unsigned long off = vma->vm_pgoff; - unsigned long pfn; int ret = -ENXIO; vma->vm_page_prot = dma_pgprot(dev, vma->vm_page_prot, attrs); @@ -203,19 +187,8 @@ int dma_common_mmap(struct device *dev, struct vm_area_struct *vma, if (off >= count || user_count > count - off) return -ENXIO; - if (!dev_is_dma_coherent(dev)) { - if (!IS_ENABLED(CONFIG_ARCH_HAS_DMA_COHERENT_TO_PFN)) - return -ENXIO; - - /* If the PFN is not valid, we do not have a struct page */ - pfn = arch_dma_coherent_to_pfn(dev, cpu_addr, dma_addr); - if (!pfn_valid(pfn)) - return -ENXIO; - } else { - pfn = page_to_pfn(virt_to_page(cpu_addr)); - } - - return remap_pfn_range(vma, vma->vm_start, pfn + vma->vm_pgoff, + return remap_pfn_range(vma, vma->vm_start, + page_to_pfn(virt_to_page(cpu_addr)) + vma->vm_pgoff, user_count << PAGE_SHIFT, vma->vm_page_prot); #else return -ENXIO; @@ -233,12 +206,8 @@ bool dma_can_mmap(struct device *dev) { const struct dma_map_ops *ops = get_dma_ops(dev); - if (dma_is_direct(ops)) { - return IS_ENABLED(CONFIG_MMU) && - (dev_is_dma_coherent(dev) || - IS_ENABLED(CONFIG_ARCH_HAS_DMA_COHERENT_TO_PFN)); - } - + if (dma_is_direct(ops)) + return dma_direct_can_mmap(dev); return ops->mmap != NULL; } EXPORT_SYMBOL_GPL(dma_can_mmap); @@ -263,7 +232,7 @@ int dma_mmap_attrs(struct device *dev, struct vm_area_struct *vma, const struct dma_map_ops *ops = get_dma_ops(dev); if (dma_is_direct(ops)) - return dma_common_mmap(dev, vma, cpu_addr, dma_addr, size, + return dma_direct_mmap(dev, vma, cpu_addr, dma_addr, size, attrs); if (!ops->mmap) return -ENXIO; diff --git a/kernel/dma/remap.c b/kernel/dma/remap.c index 90d5ce77c189..3c49499ee6b0 100644 --- a/kernel/dma/remap.c +++ b/kernel/dma/remap.c @@ -259,10 +259,4 @@ void arch_dma_free(struct device *dev, size_t size, void *vaddr, dma_free_contiguous(dev, page, size); } } - -long arch_dma_coherent_to_pfn(struct device *dev, void *cpu_addr, - dma_addr_t dma_addr) -{ - return __phys_to_pfn(dma_to_phys(dev, dma_addr)); -} #endif /* CONFIG_DMA_DIRECT_REMAP */ -- cgit v1.2.3-59-g8ed1b From 0f665b9e2a6d4cc963e6cd349d40320ed5281f95 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Tue, 29 Oct 2019 10:53:30 +0100 Subject: xtensa: use the generic uncached segment support Switch xtensa over to use the generic uncached support, and thus the generic implementations of dma_alloc_* and dma_alloc_*, which also gains support for mmaping DMA memory. The non-working nommu DMA support has been disabled, but could be re-enabled easily if platforms that actually have an uncached segment show up. Signed-off-by: Christoph Hellwig Reviewed-by: Max Filippov Tested-by: Max Filippov --- arch/xtensa/Kconfig | 6 +- arch/xtensa/include/asm/platform.h | 27 --------- arch/xtensa/kernel/Makefile | 3 +- arch/xtensa/kernel/pci-dma.c | 121 ++++--------------------------------- 4 files changed, 18 insertions(+), 139 deletions(-) (limited to 'arch') diff --git a/arch/xtensa/Kconfig b/arch/xtensa/Kconfig index a8e7beb6b7b5..c95a34702242 100644 --- a/arch/xtensa/Kconfig +++ b/arch/xtensa/Kconfig @@ -3,8 +3,10 @@ config XTENSA def_bool y select ARCH_32BIT_OFF_T select ARCH_HAS_BINFMT_FLAT if !MMU - select ARCH_HAS_SYNC_DMA_FOR_CPU - select ARCH_HAS_SYNC_DMA_FOR_DEVICE + select ARCH_HAS_DMA_PREP_COHERENT if MMU + select ARCH_HAS_SYNC_DMA_FOR_CPU if MMU + select ARCH_HAS_SYNC_DMA_FOR_DEVICE if MMU + select ARCH_HAS_UNCACHED_SEGMENT if MMU select ARCH_USE_QUEUED_RWLOCKS select ARCH_USE_QUEUED_SPINLOCKS select ARCH_WANT_FRAME_POINTERS diff --git a/arch/xtensa/include/asm/platform.h b/arch/xtensa/include/asm/platform.h index 913826dfa838..f2c48522c5a1 100644 --- a/arch/xtensa/include/asm/platform.h +++ b/arch/xtensa/include/asm/platform.h @@ -65,31 +65,4 @@ extern void platform_calibrate_ccount (void); */ void cpu_reset(void) __attribute__((noreturn)); -/* - * Memory caching is platform-dependent in noMMU xtensa configurations. - * The following set of functions should be implemented in platform code - * in order to enable coherent DMA memory operations when CONFIG_MMU is not - * enabled. Default implementations do nothing and issue a warning. - */ - -/* - * Check whether p points to a cached memory. - */ -bool platform_vaddr_cached(const void *p); - -/* - * Check whether p points to an uncached memory. - */ -bool platform_vaddr_uncached(const void *p); - -/* - * Return pointer to an uncached view of the cached sddress p. - */ -void *platform_vaddr_to_uncached(void *p); - -/* - * Return pointer to a cached view of the uncached sddress p. - */ -void *platform_vaddr_to_cached(void *p); - #endif /* _XTENSA_PLATFORM_H */ diff --git a/arch/xtensa/kernel/Makefile b/arch/xtensa/kernel/Makefile index 6f629027ac7d..d4082c6a121b 100644 --- a/arch/xtensa/kernel/Makefile +++ b/arch/xtensa/kernel/Makefile @@ -5,10 +5,11 @@ extra-y := head.o vmlinux.lds -obj-y := align.o coprocessor.o entry.o irq.o pci-dma.o platform.o process.o \ +obj-y := align.o coprocessor.o entry.o irq.o platform.o process.o \ ptrace.o setup.o signal.o stacktrace.o syscall.o time.o traps.o \ vectors.o +obj-$(CONFIG_MMU) += pci-dma.o obj-$(CONFIG_PCI) += pci.o obj-$(CONFIG_MODULES) += xtensa_ksyms.o module.o obj-$(CONFIG_FUNCTION_TRACER) += mcount.o diff --git a/arch/xtensa/kernel/pci-dma.c b/arch/xtensa/kernel/pci-dma.c index 154979d62b73..1c82e21de4f6 100644 --- a/arch/xtensa/kernel/pci-dma.c +++ b/arch/xtensa/kernel/pci-dma.c @@ -81,122 +81,25 @@ void arch_sync_dma_for_device(struct device *dev, phys_addr_t paddr, } } -#ifdef CONFIG_MMU -bool platform_vaddr_cached(const void *p) -{ - unsigned long addr = (unsigned long)p; - - return addr >= XCHAL_KSEG_CACHED_VADDR && - addr - XCHAL_KSEG_CACHED_VADDR < XCHAL_KSEG_SIZE; -} - -bool platform_vaddr_uncached(const void *p) -{ - unsigned long addr = (unsigned long)p; - - return addr >= XCHAL_KSEG_BYPASS_VADDR && - addr - XCHAL_KSEG_BYPASS_VADDR < XCHAL_KSEG_SIZE; -} - -void *platform_vaddr_to_uncached(void *p) -{ - return p + XCHAL_KSEG_BYPASS_VADDR - XCHAL_KSEG_CACHED_VADDR; -} - -void *platform_vaddr_to_cached(void *p) -{ - return p + XCHAL_KSEG_CACHED_VADDR - XCHAL_KSEG_BYPASS_VADDR; -} -#else -bool __attribute__((weak)) platform_vaddr_cached(const void *p) -{ - WARN_ONCE(1, "Default %s implementation is used\n", __func__); - return true; -} - -bool __attribute__((weak)) platform_vaddr_uncached(const void *p) -{ - WARN_ONCE(1, "Default %s implementation is used\n", __func__); - return false; -} - -void __attribute__((weak)) *platform_vaddr_to_uncached(void *p) +void arch_dma_prep_coherent(struct page *page, size_t size) { - WARN_ONCE(1, "Default %s implementation is used\n", __func__); - return p; -} - -void __attribute__((weak)) *platform_vaddr_to_cached(void *p) -{ - WARN_ONCE(1, "Default %s implementation is used\n", __func__); - return p; + __invalidate_dcache_range((unsigned long)page_address(page), size); } -#endif /* - * Note: We assume that the full memory space is always mapped to 'kseg' - * Otherwise we have to use page attributes (not implemented). + * Memory caching is platform-dependent in noMMU xtensa configurations. + * The following two functions should be implemented in platform code + * in order to enable coherent DMA memory operations when CONFIG_MMU is not + * enabled. */ - -void *arch_dma_alloc(struct device *dev, size_t size, dma_addr_t *handle, - gfp_t flag, unsigned long attrs) -{ - unsigned long count = PAGE_ALIGN(size) >> PAGE_SHIFT; - struct page *page = NULL; - - /* ignore region speicifiers */ - - flag &= ~(__GFP_DMA | __GFP_HIGHMEM); - - if (dev == NULL || (dev->coherent_dma_mask < 0xffffffff)) - flag |= GFP_DMA; - - if (gfpflags_allow_blocking(flag)) - page = dma_alloc_from_contiguous(dev, count, get_order(size), - flag & __GFP_NOWARN); - - if (!page) - page = alloc_pages(flag | __GFP_ZERO, get_order(size)); - - if (!page) - return NULL; - - *handle = phys_to_dma(dev, page_to_phys(page)); - #ifdef CONFIG_MMU - if (PageHighMem(page)) { - void *p; - - p = dma_common_contiguous_remap(page, size, - pgprot_noncached(PAGE_KERNEL), - __builtin_return_address(0)); - if (!p) { - if (!dma_release_from_contiguous(dev, page, count)) - __free_pages(page, get_order(size)); - } - return p; - } -#endif - BUG_ON(!platform_vaddr_cached(page_address(page))); - __invalidate_dcache_range((unsigned long)page_address(page), size); - return platform_vaddr_to_uncached(page_address(page)); +void *uncached_kernel_address(void *p) +{ + return p + XCHAL_KSEG_BYPASS_VADDR - XCHAL_KSEG_CACHED_VADDR; } -void arch_dma_free(struct device *dev, size_t size, void *vaddr, - dma_addr_t dma_handle, unsigned long attrs) +void *cached_kernel_address(void *p) { - unsigned long count = PAGE_ALIGN(size) >> PAGE_SHIFT; - struct page *page; - - if (platform_vaddr_uncached(vaddr)) { - page = virt_to_page(platform_vaddr_to_cached(vaddr)); - } else { -#ifdef CONFIG_MMU - dma_common_free_remap(vaddr, size); -#endif - page = pfn_to_page(PHYS_PFN(dma_to_phys(dev, dma_handle))); - } - - if (!dma_release_from_contiguous(dev, page, count)) - __free_pages(page, get_order(size)); + return p + XCHAL_KSEG_CACHED_VADDR - XCHAL_KSEG_BYPASS_VADDR; } +#endif /* CONFIG_MMU */ -- cgit v1.2.3-59-g8ed1b From e380a0394c36a3a878c858418d5dd7f5f195b6fc Mon Sep 17 00:00:00 2001 From: Nicolas Saenz Julienne Date: Thu, 7 Nov 2019 16:06:45 +0100 Subject: x86/PCI: sta2x11: use default DMA address translation The devices found behind this PCIe chip have unusual DMA mapping constraints as there is an AMBA interconnect placed in between them and the different PCI endpoints. The offset between physical memory addresses and AMBA's view is provided by reading a PCI config register, which is saved and used whenever DMA mapping is needed. It turns out that this DMA setup can be represented by properly setting 'dma_pfn_offset', 'dma_bus_mask' and 'dma_mask' during the PCI device enable fixup. And ultimately allows us to get rid of this device's custom DMA functions. Aside from the code deletion and DMA setup, sta2x11_pdev_to_mapping() is moved to avoid warnings whenever CONFIG_PM is not enabled. Signed-off-by: Nicolas Saenz Julienne Signed-off-by: Christoph Hellwig --- arch/x86/Kconfig | 1 - arch/x86/include/asm/device.h | 3 - arch/x86/include/asm/dma-direct.h | 9 --- arch/x86/pci/sta2x11-fixup.c | 135 ++++++++------------------------------ 4 files changed, 26 insertions(+), 122 deletions(-) delete mode 100644 arch/x86/include/asm/dma-direct.h (limited to 'arch') diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index d6e1faa28c58..35dd8f147f99 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig @@ -708,7 +708,6 @@ config X86_SUPPORTS_MEMORY_FAILURE config STA2X11 bool "STA2X11 Companion Chip Support" depends on X86_32_NON_STANDARD && PCI - select ARCH_HAS_PHYS_TO_DMA select SWIOTLB select MFD_STA2X11 select GPIOLIB diff --git a/arch/x86/include/asm/device.h b/arch/x86/include/asm/device.h index a8f6c809d9b1..5e12c63b47aa 100644 --- a/arch/x86/include/asm/device.h +++ b/arch/x86/include/asm/device.h @@ -6,9 +6,6 @@ struct dev_archdata { #if defined(CONFIG_INTEL_IOMMU) || defined(CONFIG_AMD_IOMMU) void *iommu; /* hook for IOMMU specific extension */ #endif -#ifdef CONFIG_STA2X11 - bool is_sta2x11; -#endif }; #if defined(CONFIG_X86_DEV_DMA_OPS) && defined(CONFIG_PCI_DOMAINS) diff --git a/arch/x86/include/asm/dma-direct.h b/arch/x86/include/asm/dma-direct.h deleted file mode 100644 index 1a19251eaac9..000000000000 --- a/arch/x86/include/asm/dma-direct.h +++ /dev/null @@ -1,9 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#ifndef ASM_X86_DMA_DIRECT_H -#define ASM_X86_DMA_DIRECT_H 1 - -bool dma_capable(struct device *dev, dma_addr_t addr, size_t size); -dma_addr_t __phys_to_dma(struct device *dev, phys_addr_t paddr); -phys_addr_t __dma_to_phys(struct device *dev, dma_addr_t daddr); - -#endif /* ASM_X86_DMA_DIRECT_H */ diff --git a/arch/x86/pci/sta2x11-fixup.c b/arch/x86/pci/sta2x11-fixup.c index 6269a175385d..4a631264b809 100644 --- a/arch/x86/pci/sta2x11-fixup.c +++ b/arch/x86/pci/sta2x11-fixup.c @@ -30,7 +30,6 @@ struct sta2x11_ahb_regs { /* saved during suspend */ }; struct sta2x11_mapping { - u32 amba_base; int is_suspended; struct sta2x11_ahb_regs regs[STA2X11_NR_FUNCS]; }; @@ -92,18 +91,6 @@ static int sta2x11_pdev_to_ep(struct pci_dev *pdev) return pdev->bus->number - instance->bus0; } -static struct sta2x11_mapping *sta2x11_pdev_to_mapping(struct pci_dev *pdev) -{ - struct sta2x11_instance *instance; - int ep; - - instance = sta2x11_pdev_to_instance(pdev); - if (!instance) - return NULL; - ep = sta2x11_pdev_to_ep(pdev); - return instance->map + ep; -} - /* This is exported, as some devices need to access the MFD registers */ struct sta2x11_instance *sta2x11_get_instance(struct pci_dev *pdev) { @@ -111,39 +98,6 @@ struct sta2x11_instance *sta2x11_get_instance(struct pci_dev *pdev) } EXPORT_SYMBOL(sta2x11_get_instance); - -/** - * p2a - Translate physical address to STA2x11 AMBA address, - * used for DMA transfers to STA2x11 - * @p: Physical address - * @pdev: PCI device (must be hosted within the connext) - */ -static dma_addr_t p2a(dma_addr_t p, struct pci_dev *pdev) -{ - struct sta2x11_mapping *map; - dma_addr_t a; - - map = sta2x11_pdev_to_mapping(pdev); - a = p + map->amba_base; - return a; -} - -/** - * a2p - Translate STA2x11 AMBA address to physical address - * used for DMA transfers from STA2x11 - * @a: STA2x11 AMBA address - * @pdev: PCI device (must be hosted within the connext) - */ -static dma_addr_t a2p(dma_addr_t a, struct pci_dev *pdev) -{ - struct sta2x11_mapping *map; - dma_addr_t p; - - map = sta2x11_pdev_to_mapping(pdev); - p = a - map->amba_base; - return p; -} - /* At setup time, we use our own ops if the device is a ConneXt one */ static void sta2x11_setup_pdev(struct pci_dev *pdev) { @@ -151,70 +105,12 @@ static void sta2x11_setup_pdev(struct pci_dev *pdev) if (!instance) /* either a sta2x11 bridge or another ST device */ return; - pci_set_consistent_dma_mask(pdev, STA2X11_AMBA_SIZE - 1); - pci_set_dma_mask(pdev, STA2X11_AMBA_SIZE - 1); - pdev->dev.archdata.is_sta2x11 = true; /* We must enable all devices as master, for audio DMA to work */ pci_set_master(pdev); } DECLARE_PCI_FIXUP_ENABLE(PCI_VENDOR_ID_STMICRO, PCI_ANY_ID, sta2x11_setup_pdev); -/* - * The following three functions are exported (used in swiotlb: FIXME) - */ -/** - * dma_capable - Check if device can manage DMA transfers (FIXME: kill it) - * @dev: device for a PCI device - * @addr: DMA address - * @size: DMA size - */ -bool dma_capable(struct device *dev, dma_addr_t addr, size_t size) -{ - struct sta2x11_mapping *map; - - if (!dev->archdata.is_sta2x11) { - if (!dev->dma_mask) - return false; - return addr + size - 1 <= *dev->dma_mask; - } - - map = sta2x11_pdev_to_mapping(to_pci_dev(dev)); - - if (!map || (addr < map->amba_base)) - return false; - if (addr + size >= map->amba_base + STA2X11_AMBA_SIZE) { - return false; - } - - return true; -} - -/** - * __phys_to_dma - Return the DMA AMBA address used for this STA2x11 device - * @dev: device for a PCI device - * @paddr: Physical address - */ -dma_addr_t __phys_to_dma(struct device *dev, phys_addr_t paddr) -{ - if (!dev->archdata.is_sta2x11) - return paddr; - return p2a(paddr, to_pci_dev(dev)); -} - -/** - * dma_to_phys - Return the physical address used for this STA2x11 DMA address - * @dev: device for a PCI device - * @daddr: STA2x11 AMBA DMA address - */ -phys_addr_t __dma_to_phys(struct device *dev, dma_addr_t daddr) -{ - if (!dev->archdata.is_sta2x11) - return daddr; - return a2p(daddr, to_pci_dev(dev)); -} - - /* * At boot we must set up the mappings for the pcie-to-amba bridge. * It involves device access, and the same happens at suspend/resume time @@ -234,12 +130,22 @@ phys_addr_t __dma_to_phys(struct device *dev, dma_addr_t daddr) /* At probe time, enable mapping for each endpoint, using the pdev */ static void sta2x11_map_ep(struct pci_dev *pdev) { - struct sta2x11_mapping *map = sta2x11_pdev_to_mapping(pdev); + struct sta2x11_instance *instance = sta2x11_pdev_to_instance(pdev); + struct device *dev = &pdev->dev; + u32 amba_base, max_amba_addr; int i; - if (!map) + if (!instance) return; - pci_read_config_dword(pdev, AHB_BASE(0), &map->amba_base); + + pci_read_config_dword(pdev, AHB_BASE(0), &amba_base); + max_amba_addr = amba_base + STA2X11_AMBA_SIZE - 1; + + dev->dma_pfn_offset = PFN_DOWN(-amba_base); + + dev->bus_dma_mask = max_amba_addr; + pci_set_consistent_dma_mask(pdev, max_amba_addr); + pci_set_dma_mask(pdev, max_amba_addr); /* Configure AHB mapping */ pci_write_config_dword(pdev, AHB_PEXLBASE(0), 0); @@ -253,13 +159,24 @@ static void sta2x11_map_ep(struct pci_dev *pdev) dev_info(&pdev->dev, "sta2x11: Map EP %i: AMBA address %#8x-%#8x\n", - sta2x11_pdev_to_ep(pdev), map->amba_base, - map->amba_base + STA2X11_AMBA_SIZE - 1); + sta2x11_pdev_to_ep(pdev), amba_base, max_amba_addr); } DECLARE_PCI_FIXUP_ENABLE(PCI_VENDOR_ID_STMICRO, PCI_ANY_ID, sta2x11_map_ep); #ifdef CONFIG_PM /* Some register values must be saved and restored */ +static struct sta2x11_mapping *sta2x11_pdev_to_mapping(struct pci_dev *pdev) +{ + struct sta2x11_instance *instance; + int ep; + + instance = sta2x11_pdev_to_instance(pdev); + if (!instance) + return NULL; + ep = sta2x11_pdev_to_ep(pdev); + return instance->map + ep; +} + static void suspend_mapping(struct pci_dev *pdev) { struct sta2x11_mapping *map = sta2x11_pdev_to_mapping(pdev); -- cgit v1.2.3-59-g8ed1b From 56e35f9c5b87ec1ae93e483284e189c84388de16 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Thu, 7 Nov 2019 18:03:11 +0100 Subject: dma-mapping: drop the dev argument to arch_sync_dma_for_* These are pure cache maintainance routines, so drop the unused struct device argument. Signed-off-by: Christoph Hellwig Suggested-by: Daniel Vetter --- arch/arc/mm/dma.c | 8 ++++---- arch/arm/mm/dma-mapping.c | 8 ++++---- arch/arm/xen/mm.c | 12 ++++++------ arch/arm64/mm/dma-mapping.c | 8 ++++---- arch/c6x/mm/dma-coherent.c | 14 +++++++------- arch/csky/mm/dma-mapping.c | 8 ++++---- arch/hexagon/kernel/dma.c | 4 ++-- arch/ia64/mm/init.c | 4 ++-- arch/m68k/kernel/dma.c | 4 ++-- arch/microblaze/kernel/dma.c | 14 +++++++------- arch/mips/bmips/dma.c | 2 +- arch/mips/jazz/jazzdma.c | 17 ++++++++--------- arch/mips/mm/dma-noncoherent.c | 12 ++++++------ arch/nds32/kernel/dma.c | 8 ++++---- arch/nios2/mm/dma-mapping.c | 8 ++++---- arch/openrisc/kernel/dma.c | 2 +- arch/parisc/kernel/pci-dma.c | 8 ++++---- arch/powerpc/mm/dma-noncoherent.c | 8 ++++---- arch/sh/kernel/dma-coherent.c | 6 +++--- arch/sparc/kernel/ioport.c | 4 ++-- arch/xtensa/kernel/pci-dma.c | 8 ++++---- drivers/iommu/dma-iommu.c | 10 +++++----- drivers/xen/swiotlb-xen.c | 8 ++++---- include/linux/dma-noncoherent.h | 20 ++++++++++---------- include/xen/swiotlb-xen.h | 8 ++++---- kernel/dma/direct.c | 14 +++++++------- 26 files changed, 113 insertions(+), 114 deletions(-) (limited to 'arch') diff --git a/arch/arc/mm/dma.c b/arch/arc/mm/dma.c index 73a7e88a1e92..e947572a521e 100644 --- a/arch/arc/mm/dma.c +++ b/arch/arc/mm/dma.c @@ -48,8 +48,8 @@ void arch_dma_prep_coherent(struct page *page, size_t size) * upper layer functions (in include/linux/dma-mapping.h) */ -void arch_sync_dma_for_device(struct device *dev, phys_addr_t paddr, - size_t size, enum dma_data_direction dir) +void arch_sync_dma_for_device(phys_addr_t paddr, size_t size, + enum dma_data_direction dir) { switch (dir) { case DMA_TO_DEVICE: @@ -69,8 +69,8 @@ void arch_sync_dma_for_device(struct device *dev, phys_addr_t paddr, } } -void arch_sync_dma_for_cpu(struct device *dev, phys_addr_t paddr, - size_t size, enum dma_data_direction dir) +void arch_sync_dma_for_cpu(phys_addr_t paddr, size_t size, + enum dma_data_direction dir) { switch (dir) { case DMA_TO_DEVICE: diff --git a/arch/arm/mm/dma-mapping.c b/arch/arm/mm/dma-mapping.c index f3cbeba7f9cb..da1a32b5e192 100644 --- a/arch/arm/mm/dma-mapping.c +++ b/arch/arm/mm/dma-mapping.c @@ -2332,15 +2332,15 @@ void arch_teardown_dma_ops(struct device *dev) } #ifdef CONFIG_SWIOTLB -void arch_sync_dma_for_device(struct device *dev, phys_addr_t paddr, - size_t size, enum dma_data_direction dir) +void arch_sync_dma_for_device(phys_addr_t paddr, size_t size, + enum dma_data_direction dir) { __dma_page_cpu_to_dev(phys_to_page(paddr), paddr & (PAGE_SIZE - 1), size, dir); } -void arch_sync_dma_for_cpu(struct device *dev, phys_addr_t paddr, - size_t size, enum dma_data_direction dir) +void arch_sync_dma_for_cpu(phys_addr_t paddr, size_t size, + enum dma_data_direction dir) { __dma_page_dev_to_cpu(phys_to_page(paddr), paddr & (PAGE_SIZE - 1), size, dir); diff --git a/arch/arm/xen/mm.c b/arch/arm/xen/mm.c index 38fa917c8585..a6a2514e5fe8 100644 --- a/arch/arm/xen/mm.c +++ b/arch/arm/xen/mm.c @@ -70,20 +70,20 @@ static void dma_cache_maint(dma_addr_t handle, size_t size, u32 op) * pfn_valid returns true the pages is local and we can use the native * dma-direct functions, otherwise we call the Xen specific version. */ -void xen_dma_sync_for_cpu(struct device *dev, dma_addr_t handle, - phys_addr_t paddr, size_t size, enum dma_data_direction dir) +void xen_dma_sync_for_cpu(dma_addr_t handle, phys_addr_t paddr, size_t size, + enum dma_data_direction dir) { if (pfn_valid(PFN_DOWN(handle))) - arch_sync_dma_for_cpu(dev, paddr, size, dir); + arch_sync_dma_for_cpu(paddr, size, dir); else if (dir != DMA_TO_DEVICE) dma_cache_maint(handle, size, GNTTAB_CACHE_INVAL); } -void xen_dma_sync_for_device(struct device *dev, dma_addr_t handle, - phys_addr_t paddr, size_t size, enum dma_data_direction dir) +void xen_dma_sync_for_device(dma_addr_t handle, phys_addr_t paddr, size_t size, + enum dma_data_direction dir) { if (pfn_valid(PFN_DOWN(handle))) - arch_sync_dma_for_device(dev, paddr, size, dir); + arch_sync_dma_for_device(paddr, size, dir); else if (dir == DMA_FROM_DEVICE) dma_cache_maint(handle, size, GNTTAB_CACHE_INVAL); else diff --git a/arch/arm64/mm/dma-mapping.c b/arch/arm64/mm/dma-mapping.c index 9239416e93d4..6c45350e33aa 100644 --- a/arch/arm64/mm/dma-mapping.c +++ b/arch/arm64/mm/dma-mapping.c @@ -13,14 +13,14 @@ #include -void arch_sync_dma_for_device(struct device *dev, phys_addr_t paddr, - size_t size, enum dma_data_direction dir) +void arch_sync_dma_for_device(phys_addr_t paddr, size_t size, + enum dma_data_direction dir) { __dma_map_area(phys_to_virt(paddr), size, dir); } -void arch_sync_dma_for_cpu(struct device *dev, phys_addr_t paddr, - size_t size, enum dma_data_direction dir) +void arch_sync_dma_for_cpu(phys_addr_t paddr, size_t size, + enum dma_data_direction dir) { __dma_unmap_area(phys_to_virt(paddr), size, dir); } diff --git a/arch/c6x/mm/dma-coherent.c b/arch/c6x/mm/dma-coherent.c index b319808e8f6b..a5909091cb14 100644 --- a/arch/c6x/mm/dma-coherent.c +++ b/arch/c6x/mm/dma-coherent.c @@ -140,7 +140,7 @@ void __init coherent_mem_init(phys_addr_t start, u32 size) sizeof(long)); } -static void c6x_dma_sync(struct device *dev, phys_addr_t paddr, size_t size, +static void c6x_dma_sync(phys_addr_t paddr, size_t size, enum dma_data_direction dir) { BUG_ON(!valid_dma_direction(dir)); @@ -160,14 +160,14 @@ static void c6x_dma_sync(struct device *dev, phys_addr_t paddr, size_t size, } } -void arch_sync_dma_for_device(struct device *dev, phys_addr_t paddr, - size_t size, enum dma_data_direction dir) +void arch_sync_dma_for_device(phys_addr_t paddr, size_t size, + enum dma_data_direction dir) { - return c6x_dma_sync(dev, paddr, size, dir); + return c6x_dma_sync(paddr, size, dir); } -void arch_sync_dma_for_cpu(struct device *dev, phys_addr_t paddr, - size_t size, enum dma_data_direction dir) +void arch_sync_dma_for_cpu(phys_addr_t paddr, size_t size, + enum dma_data_direction dir) { - return c6x_dma_sync(dev, paddr, size, dir); + return c6x_dma_sync(paddr, size, dir); } diff --git a/arch/csky/mm/dma-mapping.c b/arch/csky/mm/dma-mapping.c index 06e85b565454..8f6571ae27c8 100644 --- a/arch/csky/mm/dma-mapping.c +++ b/arch/csky/mm/dma-mapping.c @@ -58,8 +58,8 @@ void arch_dma_prep_coherent(struct page *page, size_t size) cache_op(page_to_phys(page), size, dma_wbinv_set_zero_range); } -void arch_sync_dma_for_device(struct device *dev, phys_addr_t paddr, - size_t size, enum dma_data_direction dir) +void arch_sync_dma_for_device(phys_addr_t paddr, size_t size, + enum dma_data_direction dir) { switch (dir) { case DMA_TO_DEVICE: @@ -74,8 +74,8 @@ void arch_sync_dma_for_device(struct device *dev, phys_addr_t paddr, } } -void arch_sync_dma_for_cpu(struct device *dev, phys_addr_t paddr, - size_t size, enum dma_data_direction dir) +void arch_sync_dma_for_cpu(phys_addr_t paddr, size_t size, + enum dma_data_direction dir) { switch (dir) { case DMA_TO_DEVICE: diff --git a/arch/hexagon/kernel/dma.c b/arch/hexagon/kernel/dma.c index f561b127c4b4..25f388d9cfcc 100644 --- a/arch/hexagon/kernel/dma.c +++ b/arch/hexagon/kernel/dma.c @@ -55,8 +55,8 @@ void arch_dma_free(struct device *dev, size_t size, void *vaddr, gen_pool_free(coherent_pool, (unsigned long) vaddr, size); } -void arch_sync_dma_for_device(struct device *dev, phys_addr_t paddr, - size_t size, enum dma_data_direction dir) +void arch_sync_dma_for_device(phys_addr_t paddr, size_t size, + enum dma_data_direction dir) { void *addr = phys_to_virt(paddr); diff --git a/arch/ia64/mm/init.c b/arch/ia64/mm/init.c index bf9df2625bc8..58fd67068bac 100644 --- a/arch/ia64/mm/init.c +++ b/arch/ia64/mm/init.c @@ -73,8 +73,8 @@ __ia64_sync_icache_dcache (pte_t pte) * DMA can be marked as "clean" so that lazy_mmu_prot_update() doesn't have to * flush them when they get mapped into an executable vm-area. */ -void arch_sync_dma_for_cpu(struct device *dev, phys_addr_t paddr, - size_t size, enum dma_data_direction dir) +void arch_sync_dma_for_cpu(phys_addr_t paddr, size_t size, + enum dma_data_direction dir) { unsigned long pfn = PHYS_PFN(paddr); diff --git a/arch/m68k/kernel/dma.c b/arch/m68k/kernel/dma.c index 3fab684cc0db..871a0e11da34 100644 --- a/arch/m68k/kernel/dma.c +++ b/arch/m68k/kernel/dma.c @@ -61,8 +61,8 @@ void arch_dma_free(struct device *dev, size_t size, void *vaddr, #endif /* CONFIG_MMU && !CONFIG_COLDFIRE */ -void arch_sync_dma_for_device(struct device *dev, phys_addr_t handle, - size_t size, enum dma_data_direction dir) +void arch_sync_dma_for_device(phys_addr_t handle, size_t size, + enum dma_data_direction dir) { switch (dir) { case DMA_BIDIRECTIONAL: diff --git a/arch/microblaze/kernel/dma.c b/arch/microblaze/kernel/dma.c index a89c2d4ed5ff..d7bebd04247b 100644 --- a/arch/microblaze/kernel/dma.c +++ b/arch/microblaze/kernel/dma.c @@ -15,7 +15,7 @@ #include #include -static void __dma_sync(struct device *dev, phys_addr_t paddr, size_t size, +static void __dma_sync(phys_addr_t paddr, size_t size, enum dma_data_direction direction) { switch (direction) { @@ -31,14 +31,14 @@ static void __dma_sync(struct device *dev, phys_addr_t paddr, size_t size, } } -void arch_sync_dma_for_device(struct device *dev, phys_addr_t paddr, - size_t size, enum dma_data_direction dir) +void arch_sync_dma_for_device(phys_addr_t paddr, size_t size, + enum dma_data_direction dir) { - __dma_sync(dev, paddr, size, dir); + __dma_sync(paddr, size, dir); } -void arch_sync_dma_for_cpu(struct device *dev, phys_addr_t paddr, - size_t size, enum dma_data_direction dir) +void arch_sync_dma_for_cpu(phys_addr_t paddr, size_t size, + enum dma_data_direction dir) { - __dma_sync(dev, paddr, size, dir); + __dma_sync(paddr, size, dir); } diff --git a/arch/mips/bmips/dma.c b/arch/mips/bmips/dma.c index 3d13c77c125f..df56bf4179e3 100644 --- a/arch/mips/bmips/dma.c +++ b/arch/mips/bmips/dma.c @@ -64,7 +64,7 @@ phys_addr_t __dma_to_phys(struct device *dev, dma_addr_t dma_addr) return dma_addr; } -void arch_sync_dma_for_cpu_all(struct device *dev) +void arch_sync_dma_for_cpu_all(void) { void __iomem *cbr = BMIPS_GET_CBR(); u32 cfg; diff --git a/arch/mips/jazz/jazzdma.c b/arch/mips/jazz/jazzdma.c index a01e14955187..c64a297e82b3 100644 --- a/arch/mips/jazz/jazzdma.c +++ b/arch/mips/jazz/jazzdma.c @@ -592,7 +592,7 @@ static dma_addr_t jazz_dma_map_page(struct device *dev, struct page *page, phys_addr_t phys = page_to_phys(page) + offset; if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC)) - arch_sync_dma_for_device(dev, phys, size, dir); + arch_sync_dma_for_device(phys, size, dir); return vdma_alloc(phys, size); } @@ -600,7 +600,7 @@ static void jazz_dma_unmap_page(struct device *dev, dma_addr_t dma_addr, size_t size, enum dma_data_direction dir, unsigned long attrs) { if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC)) - arch_sync_dma_for_cpu(dev, vdma_log2phys(dma_addr), size, dir); + arch_sync_dma_for_cpu(vdma_log2phys(dma_addr), size, dir); vdma_free(dma_addr); } @@ -612,7 +612,7 @@ static int jazz_dma_map_sg(struct device *dev, struct scatterlist *sglist, for_each_sg(sglist, sg, nents, i) { if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC)) - arch_sync_dma_for_device(dev, sg_phys(sg), sg->length, + arch_sync_dma_for_device(sg_phys(sg), sg->length, dir); sg->dma_address = vdma_alloc(sg_phys(sg), sg->length); if (sg->dma_address == DMA_MAPPING_ERROR) @@ -631,8 +631,7 @@ static void jazz_dma_unmap_sg(struct device *dev, struct scatterlist *sglist, for_each_sg(sglist, sg, nents, i) { if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC)) - arch_sync_dma_for_cpu(dev, sg_phys(sg), sg->length, - dir); + arch_sync_dma_for_cpu(sg_phys(sg), sg->length, dir); vdma_free(sg->dma_address); } } @@ -640,13 +639,13 @@ static void jazz_dma_unmap_sg(struct device *dev, struct scatterlist *sglist, static void jazz_dma_sync_single_for_device(struct device *dev, dma_addr_t addr, size_t size, enum dma_data_direction dir) { - arch_sync_dma_for_device(dev, vdma_log2phys(addr), size, dir); + arch_sync_dma_for_device(vdma_log2phys(addr), size, dir); } static void jazz_dma_sync_single_for_cpu(struct device *dev, dma_addr_t addr, size_t size, enum dma_data_direction dir) { - arch_sync_dma_for_cpu(dev, vdma_log2phys(addr), size, dir); + arch_sync_dma_for_cpu(vdma_log2phys(addr), size, dir); } static void jazz_dma_sync_sg_for_device(struct device *dev, @@ -656,7 +655,7 @@ static void jazz_dma_sync_sg_for_device(struct device *dev, int i; for_each_sg(sgl, sg, nents, i) - arch_sync_dma_for_device(dev, sg_phys(sg), sg->length, dir); + arch_sync_dma_for_device(sg_phys(sg), sg->length, dir); } static void jazz_dma_sync_sg_for_cpu(struct device *dev, @@ -666,7 +665,7 @@ static void jazz_dma_sync_sg_for_cpu(struct device *dev, int i; for_each_sg(sgl, sg, nents, i) - arch_sync_dma_for_cpu(dev, sg_phys(sg), sg->length, dir); + arch_sync_dma_for_cpu(sg_phys(sg), sg->length, dir); } const struct dma_map_ops jazz_dma_ops = { diff --git a/arch/mips/mm/dma-noncoherent.c b/arch/mips/mm/dma-noncoherent.c index fcf6d3eaac66..dc42ffc83825 100644 --- a/arch/mips/mm/dma-noncoherent.c +++ b/arch/mips/mm/dma-noncoherent.c @@ -27,7 +27,7 @@ * R10000 and R12000 are used in such systems, the SGI IP28 Indigo² rsp. * SGI IP32 aka O2. */ -static inline bool cpu_needs_post_dma_flush(struct device *dev) +static inline bool cpu_needs_post_dma_flush(void) { switch (boot_cpu_type()) { case CPU_R10000: @@ -112,17 +112,17 @@ static inline void dma_sync_phys(phys_addr_t paddr, size_t size, } while (left); } -void arch_sync_dma_for_device(struct device *dev, phys_addr_t paddr, - size_t size, enum dma_data_direction dir) +void arch_sync_dma_for_device(phys_addr_t paddr, size_t size, + enum dma_data_direction dir) { dma_sync_phys(paddr, size, dir); } #ifdef CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU -void arch_sync_dma_for_cpu(struct device *dev, phys_addr_t paddr, - size_t size, enum dma_data_direction dir) +void arch_sync_dma_for_cpu(phys_addr_t paddr, size_t size, + enum dma_data_direction dir) { - if (cpu_needs_post_dma_flush(dev)) + if (cpu_needs_post_dma_flush()) dma_sync_phys(paddr, size, dir); } #endif diff --git a/arch/nds32/kernel/dma.c b/arch/nds32/kernel/dma.c index 4206d4b6c8ce..69d762182d49 100644 --- a/arch/nds32/kernel/dma.c +++ b/arch/nds32/kernel/dma.c @@ -46,8 +46,8 @@ static inline void cache_op(phys_addr_t paddr, size_t size, } while (left); } -void arch_sync_dma_for_device(struct device *dev, phys_addr_t paddr, - size_t size, enum dma_data_direction dir) +void arch_sync_dma_for_device(phys_addr_t paddr, size_t size, + enum dma_data_direction dir) { switch (dir) { case DMA_FROM_DEVICE: @@ -61,8 +61,8 @@ void arch_sync_dma_for_device(struct device *dev, phys_addr_t paddr, } } -void arch_sync_dma_for_cpu(struct device *dev, phys_addr_t paddr, - size_t size, enum dma_data_direction dir) +void arch_sync_dma_for_cpu(phys_addr_t paddr, size_t size, + enum dma_data_direction dir) { switch (dir) { case DMA_TO_DEVICE: diff --git a/arch/nios2/mm/dma-mapping.c b/arch/nios2/mm/dma-mapping.c index 9cb238664584..0ed711e37902 100644 --- a/arch/nios2/mm/dma-mapping.c +++ b/arch/nios2/mm/dma-mapping.c @@ -18,8 +18,8 @@ #include #include -void arch_sync_dma_for_device(struct device *dev, phys_addr_t paddr, - size_t size, enum dma_data_direction dir) +void arch_sync_dma_for_device(phys_addr_t paddr, size_t size, + enum dma_data_direction dir) { void *vaddr = phys_to_virt(paddr); @@ -42,8 +42,8 @@ void arch_sync_dma_for_device(struct device *dev, phys_addr_t paddr, } } -void arch_sync_dma_for_cpu(struct device *dev, phys_addr_t paddr, - size_t size, enum dma_data_direction dir) +void arch_sync_dma_for_cpu(phys_addr_t paddr, size_t size, + enum dma_data_direction dir) { void *vaddr = phys_to_virt(paddr); diff --git a/arch/openrisc/kernel/dma.c b/arch/openrisc/kernel/dma.c index 4d5b8bd1d795..adec711ad39d 100644 --- a/arch/openrisc/kernel/dma.c +++ b/arch/openrisc/kernel/dma.c @@ -125,7 +125,7 @@ arch_dma_free(struct device *dev, size_t size, void *vaddr, free_pages_exact(vaddr, size); } -void arch_sync_dma_for_device(struct device *dev, phys_addr_t addr, size_t size, +void arch_sync_dma_for_device(phys_addr_t addr, size_t size, enum dma_data_direction dir) { unsigned long cl; diff --git a/arch/parisc/kernel/pci-dma.c b/arch/parisc/kernel/pci-dma.c index ca35d9a76e50..a60d47fd4d55 100644 --- a/arch/parisc/kernel/pci-dma.c +++ b/arch/parisc/kernel/pci-dma.c @@ -439,14 +439,14 @@ void arch_dma_free(struct device *dev, size_t size, void *vaddr, free_pages((unsigned long)__va(dma_handle), order); } -void arch_sync_dma_for_device(struct device *dev, phys_addr_t paddr, - size_t size, enum dma_data_direction dir) +void arch_sync_dma_for_device(phys_addr_t paddr, size_t size, + enum dma_data_direction dir) { flush_kernel_dcache_range((unsigned long)phys_to_virt(paddr), size); } -void arch_sync_dma_for_cpu(struct device *dev, phys_addr_t paddr, - size_t size, enum dma_data_direction dir) +void arch_sync_dma_for_cpu(phys_addr_t paddr, size_t size, + enum dma_data_direction dir) { flush_kernel_dcache_range((unsigned long)phys_to_virt(paddr), size); } diff --git a/arch/powerpc/mm/dma-noncoherent.c b/arch/powerpc/mm/dma-noncoherent.c index 2a82984356f8..5ab4f868e919 100644 --- a/arch/powerpc/mm/dma-noncoherent.c +++ b/arch/powerpc/mm/dma-noncoherent.c @@ -104,14 +104,14 @@ static void __dma_sync_page(phys_addr_t paddr, size_t size, int dir) #endif } -void arch_sync_dma_for_device(struct device *dev, phys_addr_t paddr, - size_t size, enum dma_data_direction dir) +void arch_sync_dma_for_device(phys_addr_t paddr, size_t size, + enum dma_data_direction dir) { __dma_sync_page(paddr, size, dir); } -void arch_sync_dma_for_cpu(struct device *dev, phys_addr_t paddr, - size_t size, enum dma_data_direction dir) +void arch_sync_dma_for_cpu(phys_addr_t paddr, size_t size, + enum dma_data_direction dir) { __dma_sync_page(paddr, size, dir); } diff --git a/arch/sh/kernel/dma-coherent.c b/arch/sh/kernel/dma-coherent.c index b17514619b7e..eeb25a4fa55f 100644 --- a/arch/sh/kernel/dma-coherent.c +++ b/arch/sh/kernel/dma-coherent.c @@ -25,7 +25,7 @@ void *arch_dma_alloc(struct device *dev, size_t size, dma_addr_t *dma_handle, * Pages from the page allocator may have data present in * cache. So flush the cache before using uncached memory. */ - arch_sync_dma_for_device(dev, virt_to_phys(ret), size, + arch_sync_dma_for_device(virt_to_phys(ret), size, DMA_BIDIRECTIONAL); ret_nocache = (void __force *)ioremap_nocache(virt_to_phys(ret), size); @@ -59,8 +59,8 @@ void arch_dma_free(struct device *dev, size_t size, void *vaddr, iounmap(vaddr); } -void arch_sync_dma_for_device(struct device *dev, phys_addr_t paddr, - size_t size, enum dma_data_direction dir) +void arch_sync_dma_for_device(phys_addr_t paddr, size_t size, + enum dma_data_direction dir) { void *addr = sh_cacheop_vaddr(phys_to_virt(paddr)); diff --git a/arch/sparc/kernel/ioport.c b/arch/sparc/kernel/ioport.c index f89603855f1e..e59461d03b9a 100644 --- a/arch/sparc/kernel/ioport.c +++ b/arch/sparc/kernel/ioport.c @@ -366,8 +366,8 @@ void arch_dma_free(struct device *dev, size_t size, void *cpu_addr, /* IIep is write-through, not flushing on cpu to device transfer. */ -void arch_sync_dma_for_cpu(struct device *dev, phys_addr_t paddr, - size_t size, enum dma_data_direction dir) +void arch_sync_dma_for_cpu(phys_addr_t paddr, size_t size, + enum dma_data_direction dir) { if (dir != PCI_DMA_TODEVICE) dma_make_coherent(paddr, PAGE_ALIGN(size)); diff --git a/arch/xtensa/kernel/pci-dma.c b/arch/xtensa/kernel/pci-dma.c index 1c82e21de4f6..72b6222daa0b 100644 --- a/arch/xtensa/kernel/pci-dma.c +++ b/arch/xtensa/kernel/pci-dma.c @@ -44,8 +44,8 @@ static void do_cache_op(phys_addr_t paddr, size_t size, } } -void arch_sync_dma_for_cpu(struct device *dev, phys_addr_t paddr, - size_t size, enum dma_data_direction dir) +void arch_sync_dma_for_cpu(phys_addr_t paddr, size_t size, + enum dma_data_direction dir) { switch (dir) { case DMA_BIDIRECTIONAL: @@ -62,8 +62,8 @@ void arch_sync_dma_for_cpu(struct device *dev, phys_addr_t paddr, } } -void arch_sync_dma_for_device(struct device *dev, phys_addr_t paddr, - size_t size, enum dma_data_direction dir) +void arch_sync_dma_for_device(phys_addr_t paddr, size_t size, + enum dma_data_direction dir) { switch (dir) { case DMA_BIDIRECTIONAL: diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c index f321279baf9e..0fa8c1d818b7 100644 --- a/drivers/iommu/dma-iommu.c +++ b/drivers/iommu/dma-iommu.c @@ -659,7 +659,7 @@ static void iommu_dma_sync_single_for_cpu(struct device *dev, return; phys = iommu_iova_to_phys(iommu_get_dma_domain(dev), dma_handle); - arch_sync_dma_for_cpu(dev, phys, size, dir); + arch_sync_dma_for_cpu(phys, size, dir); } static void iommu_dma_sync_single_for_device(struct device *dev, @@ -671,7 +671,7 @@ static void iommu_dma_sync_single_for_device(struct device *dev, return; phys = iommu_iova_to_phys(iommu_get_dma_domain(dev), dma_handle); - arch_sync_dma_for_device(dev, phys, size, dir); + arch_sync_dma_for_device(phys, size, dir); } static void iommu_dma_sync_sg_for_cpu(struct device *dev, @@ -685,7 +685,7 @@ static void iommu_dma_sync_sg_for_cpu(struct device *dev, return; for_each_sg(sgl, sg, nelems, i) - arch_sync_dma_for_cpu(dev, sg_phys(sg), sg->length, dir); + arch_sync_dma_for_cpu(sg_phys(sg), sg->length, dir); } static void iommu_dma_sync_sg_for_device(struct device *dev, @@ -699,7 +699,7 @@ static void iommu_dma_sync_sg_for_device(struct device *dev, return; for_each_sg(sgl, sg, nelems, i) - arch_sync_dma_for_device(dev, sg_phys(sg), sg->length, dir); + arch_sync_dma_for_device(sg_phys(sg), sg->length, dir); } static dma_addr_t iommu_dma_map_page(struct device *dev, struct page *page, @@ -714,7 +714,7 @@ static dma_addr_t iommu_dma_map_page(struct device *dev, struct page *page, dma_handle =__iommu_dma_map(dev, phys, size, prot); if (!coherent && !(attrs & DMA_ATTR_SKIP_CPU_SYNC) && dma_handle != DMA_MAPPING_ERROR) - arch_sync_dma_for_device(dev, phys, size, dir); + arch_sync_dma_for_device(phys, size, dir); return dma_handle; } diff --git a/drivers/xen/swiotlb-xen.c b/drivers/xen/swiotlb-xen.c index bd3a10dfac15..3f8b2cdb4acb 100644 --- a/drivers/xen/swiotlb-xen.c +++ b/drivers/xen/swiotlb-xen.c @@ -405,7 +405,7 @@ static dma_addr_t xen_swiotlb_map_page(struct device *dev, struct page *page, done: if (!dev_is_dma_coherent(dev) && !(attrs & DMA_ATTR_SKIP_CPU_SYNC)) - xen_dma_sync_for_device(dev, dev_addr, phys, size, dir); + xen_dma_sync_for_device(dev_addr, phys, size, dir); return dev_addr; } @@ -425,7 +425,7 @@ static void xen_swiotlb_unmap_page(struct device *hwdev, dma_addr_t dev_addr, BUG_ON(dir == DMA_NONE); if (!dev_is_dma_coherent(hwdev) && !(attrs & DMA_ATTR_SKIP_CPU_SYNC)) - xen_dma_sync_for_cpu(hwdev, dev_addr, paddr, size, dir); + xen_dma_sync_for_cpu(dev_addr, paddr, size, dir); /* NOTE: We use dev_addr here, not paddr! */ if (is_xen_swiotlb_buffer(dev_addr)) @@ -439,7 +439,7 @@ xen_swiotlb_sync_single_for_cpu(struct device *dev, dma_addr_t dma_addr, phys_addr_t paddr = xen_bus_to_phys(dma_addr); if (!dev_is_dma_coherent(dev)) - xen_dma_sync_for_cpu(dev, dma_addr, paddr, size, dir); + xen_dma_sync_for_cpu(dma_addr, paddr, size, dir); if (is_xen_swiotlb_buffer(dma_addr)) swiotlb_tbl_sync_single(dev, paddr, size, dir, SYNC_FOR_CPU); @@ -455,7 +455,7 @@ xen_swiotlb_sync_single_for_device(struct device *dev, dma_addr_t dma_addr, swiotlb_tbl_sync_single(dev, paddr, size, dir, SYNC_FOR_DEVICE); if (!dev_is_dma_coherent(dev)) - xen_dma_sync_for_device(dev, dma_addr, paddr, size, dir); + xen_dma_sync_for_device(dma_addr, paddr, size, dir); } /* diff --git a/include/linux/dma-noncoherent.h b/include/linux/dma-noncoherent.h index e30fca1f1b12..ca9b5770caee 100644 --- a/include/linux/dma-noncoherent.h +++ b/include/linux/dma-noncoherent.h @@ -73,29 +73,29 @@ static inline void arch_dma_cache_sync(struct device *dev, void *vaddr, #endif /* CONFIG_DMA_NONCOHERENT_CACHE_SYNC */ #ifdef CONFIG_ARCH_HAS_SYNC_DMA_FOR_DEVICE -void arch_sync_dma_for_device(struct device *dev, phys_addr_t paddr, - size_t size, enum dma_data_direction dir); +void arch_sync_dma_for_device(phys_addr_t paddr, size_t size, + enum dma_data_direction dir); #else -static inline void arch_sync_dma_for_device(struct device *dev, - phys_addr_t paddr, size_t size, enum dma_data_direction dir) +static inline void arch_sync_dma_for_device(phys_addr_t paddr, size_t size, + enum dma_data_direction dir) { } #endif /* ARCH_HAS_SYNC_DMA_FOR_DEVICE */ #ifdef CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU -void arch_sync_dma_for_cpu(struct device *dev, phys_addr_t paddr, - size_t size, enum dma_data_direction dir); +void arch_sync_dma_for_cpu(phys_addr_t paddr, size_t size, + enum dma_data_direction dir); #else -static inline void arch_sync_dma_for_cpu(struct device *dev, - phys_addr_t paddr, size_t size, enum dma_data_direction dir) +static inline void arch_sync_dma_for_cpu(phys_addr_t paddr, size_t size, + enum dma_data_direction dir) { } #endif /* ARCH_HAS_SYNC_DMA_FOR_CPU */ #ifdef CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU_ALL -void arch_sync_dma_for_cpu_all(struct device *dev); +void arch_sync_dma_for_cpu_all(void); #else -static inline void arch_sync_dma_for_cpu_all(struct device *dev) +static inline void arch_sync_dma_for_cpu_all(void) { } #endif /* CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU_ALL */ diff --git a/include/xen/swiotlb-xen.h b/include/xen/swiotlb-xen.h index d71380f6ed0b..ffc0d3902b71 100644 --- a/include/xen/swiotlb-xen.h +++ b/include/xen/swiotlb-xen.h @@ -4,10 +4,10 @@ #include -void xen_dma_sync_for_cpu(struct device *dev, dma_addr_t handle, - phys_addr_t paddr, size_t size, enum dma_data_direction dir); -void xen_dma_sync_for_device(struct device *dev, dma_addr_t handle, - phys_addr_t paddr, size_t size, enum dma_data_direction dir); +void xen_dma_sync_for_cpu(dma_addr_t handle, phys_addr_t paddr, size_t size, + enum dma_data_direction dir); +void xen_dma_sync_for_device(dma_addr_t handle, phys_addr_t paddr, size_t size, + enum dma_data_direction dir); extern int xen_swiotlb_init(int verbose, bool early); extern const struct dma_map_ops xen_swiotlb_dma_ops; diff --git a/kernel/dma/direct.c b/kernel/dma/direct.c index 22a2e0833862..077876ae5c74 100644 --- a/kernel/dma/direct.c +++ b/kernel/dma/direct.c @@ -268,7 +268,7 @@ void dma_direct_sync_single_for_device(struct device *dev, swiotlb_tbl_sync_single(dev, paddr, size, dir, SYNC_FOR_DEVICE); if (!dev_is_dma_coherent(dev)) - arch_sync_dma_for_device(dev, paddr, size, dir); + arch_sync_dma_for_device(paddr, size, dir); } EXPORT_SYMBOL(dma_direct_sync_single_for_device); @@ -286,7 +286,7 @@ void dma_direct_sync_sg_for_device(struct device *dev, dir, SYNC_FOR_DEVICE); if (!dev_is_dma_coherent(dev)) - arch_sync_dma_for_device(dev, paddr, sg->length, + arch_sync_dma_for_device(paddr, sg->length, dir); } } @@ -302,8 +302,8 @@ void dma_direct_sync_single_for_cpu(struct device *dev, phys_addr_t paddr = dma_to_phys(dev, addr); if (!dev_is_dma_coherent(dev)) { - arch_sync_dma_for_cpu(dev, paddr, size, dir); - arch_sync_dma_for_cpu_all(dev); + arch_sync_dma_for_cpu(paddr, size, dir); + arch_sync_dma_for_cpu_all(); } if (unlikely(is_swiotlb_buffer(paddr))) @@ -321,7 +321,7 @@ void dma_direct_sync_sg_for_cpu(struct device *dev, phys_addr_t paddr = dma_to_phys(dev, sg_dma_address(sg)); if (!dev_is_dma_coherent(dev)) - arch_sync_dma_for_cpu(dev, paddr, sg->length, dir); + arch_sync_dma_for_cpu(paddr, sg->length, dir); if (unlikely(is_swiotlb_buffer(paddr))) swiotlb_tbl_sync_single(dev, paddr, sg->length, dir, @@ -329,7 +329,7 @@ void dma_direct_sync_sg_for_cpu(struct device *dev, } if (!dev_is_dma_coherent(dev)) - arch_sync_dma_for_cpu_all(dev); + arch_sync_dma_for_cpu_all(); } EXPORT_SYMBOL(dma_direct_sync_sg_for_cpu); @@ -380,7 +380,7 @@ dma_addr_t dma_direct_map_page(struct device *dev, struct page *page, } if (!dev_is_dma_coherent(dev) && !(attrs & DMA_ATTR_SKIP_CPU_SYNC)) - arch_sync_dma_for_device(dev, phys, size, dir); + arch_sync_dma_for_device(phys, size, dir); return dma_addr; } EXPORT_SYMBOL(dma_direct_map_page); -- cgit v1.2.3-59-g8ed1b From 130c1ccbf55330b55e82612a6e54eebb82c9d746 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Tue, 12 Nov 2019 17:06:04 +0100 Subject: dma-direct: unify the dma_capable definitions Currently each architectures that wants to override dma_to_phys and phys_to_dma also has to provide dma_capable. But there isn't really any good reason for that. powerpc and mips just have copies of the generic one minus the latests fix, and the arm one was the inspiration for said fix, but misses the bus_dma_mask handling. Make all architectures use the generic version instead. Signed-off-by: Christoph Hellwig Acked-by: Michael Ellerman (powerpc) Reviewed-by: Nicolas Saenz Julienne --- arch/arm/include/asm/dma-direct.h | 19 ------------------- arch/mips/include/asm/dma-direct.h | 8 -------- arch/powerpc/include/asm/dma-direct.h | 9 --------- include/linux/dma-direct.h | 2 +- 4 files changed, 1 insertion(+), 37 deletions(-) (limited to 'arch') diff --git a/arch/arm/include/asm/dma-direct.h b/arch/arm/include/asm/dma-direct.h index b67e5fc1fe43..7c3001a6a775 100644 --- a/arch/arm/include/asm/dma-direct.h +++ b/arch/arm/include/asm/dma-direct.h @@ -14,23 +14,4 @@ static inline phys_addr_t __dma_to_phys(struct device *dev, dma_addr_t dev_addr) return __pfn_to_phys(dma_to_pfn(dev, dev_addr)) + offset; } -static inline bool dma_capable(struct device *dev, dma_addr_t addr, size_t size) -{ - u64 limit, mask; - - if (!dev->dma_mask) - return 0; - - mask = *dev->dma_mask; - - limit = (mask + 1) & ~mask; - if (limit && size > limit) - return 0; - - if ((addr | (addr + size - 1)) & ~mask) - return 0; - - return 1; -} - #endif /* ASM_ARM_DMA_DIRECT_H */ diff --git a/arch/mips/include/asm/dma-direct.h b/arch/mips/include/asm/dma-direct.h index b5c240806e1b..14e352651ce9 100644 --- a/arch/mips/include/asm/dma-direct.h +++ b/arch/mips/include/asm/dma-direct.h @@ -2,14 +2,6 @@ #ifndef _MIPS_DMA_DIRECT_H #define _MIPS_DMA_DIRECT_H 1 -static inline bool dma_capable(struct device *dev, dma_addr_t addr, size_t size) -{ - if (!dev->dma_mask) - return false; - - return addr + size - 1 <= *dev->dma_mask; -} - dma_addr_t __phys_to_dma(struct device *dev, phys_addr_t paddr); phys_addr_t __dma_to_phys(struct device *dev, dma_addr_t daddr); diff --git a/arch/powerpc/include/asm/dma-direct.h b/arch/powerpc/include/asm/dma-direct.h index a2912b47102c..e29e8a236b8d 100644 --- a/arch/powerpc/include/asm/dma-direct.h +++ b/arch/powerpc/include/asm/dma-direct.h @@ -2,15 +2,6 @@ #ifndef ASM_POWERPC_DMA_DIRECT_H #define ASM_POWERPC_DMA_DIRECT_H 1 -static inline bool dma_capable(struct device *dev, dma_addr_t addr, size_t size) -{ - if (!dev->dma_mask) - return false; - - return addr + size - 1 <= - min_not_zero(*dev->dma_mask, dev->bus_dma_mask); -} - static inline dma_addr_t __phys_to_dma(struct device *dev, phys_addr_t paddr) { if (!dev) diff --git a/include/linux/dma-direct.h b/include/linux/dma-direct.h index 6db863c3eb93..991f8aa2676e 100644 --- a/include/linux/dma-direct.h +++ b/include/linux/dma-direct.h @@ -24,6 +24,7 @@ static inline phys_addr_t __dma_to_phys(struct device *dev, dma_addr_t dev_addr) return paddr + ((phys_addr_t)dev->dma_pfn_offset << PAGE_SHIFT); } +#endif /* !CONFIG_ARCH_HAS_PHYS_TO_DMA */ static inline bool dma_capable(struct device *dev, dma_addr_t addr, size_t size) { @@ -38,7 +39,6 @@ static inline bool dma_capable(struct device *dev, dma_addr_t addr, size_t size) return end <= min_not_zero(*dev->dma_mask, dev->bus_dma_mask); } -#endif /* !CONFIG_ARCH_HAS_PHYS_TO_DMA */ #ifdef CONFIG_ARCH_HAS_FORCE_DMA_UNENCRYPTED bool force_dma_unencrypted(struct device *dev); -- cgit v1.2.3-59-g8ed1b From cb6f6392dbefb45b6aabd9a6f566440ef642b6f8 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Tue, 12 Nov 2019 17:08:49 +0100 Subject: powerpc: remove support for NULL dev in __phys_to_dma / __dma_to_phys Support for calling the DMA API functions without a valid device pointer was removed a while ago, so remove the stale support for that from the powerpc __phys_to_dma / __dma_to_phys helpers. Signed-off-by: Christoph Hellwig Acked-by: Michael Ellerman Reviewed-by: Nicolas Saenz Julienne --- arch/powerpc/include/asm/dma-direct.h | 4 ---- 1 file changed, 4 deletions(-) (limited to 'arch') diff --git a/arch/powerpc/include/asm/dma-direct.h b/arch/powerpc/include/asm/dma-direct.h index e29e8a236b8d..abc154d784b0 100644 --- a/arch/powerpc/include/asm/dma-direct.h +++ b/arch/powerpc/include/asm/dma-direct.h @@ -4,15 +4,11 @@ static inline dma_addr_t __phys_to_dma(struct device *dev, phys_addr_t paddr) { - if (!dev) - return paddr + PCI_DRAM_OFFSET; return paddr + dev->archdata.dma_offset; } static inline phys_addr_t __dma_to_phys(struct device *dev, dma_addr_t daddr) { - if (!dev) - return daddr - PCI_DRAM_OFFSET; return daddr - dev->archdata.dma_offset; } #endif /* ASM_POWERPC_DMA_DIRECT_H */ -- cgit v1.2.3-59-g8ed1b From 68a33b1794665ba8a1d1ef1d3bfcc7c587d380a6 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Tue, 19 Nov 2019 17:38:58 +0100 Subject: dma-direct: exclude dma_direct_map_resource from the min_low_pfn check The valid memory address check in dma_capable only makes sense when mapping normal memory, not when using dma_map_resource to map a device resource. Add a new boolean argument to dma_capable to exclude that check for the dma_map_resource case. Fixes: b12d66278dd6 ("dma-direct: check for overflows on 32 bit DMA addresses") Reported-by: Marek Szyprowski Signed-off-by: Christoph Hellwig Acked-by: Marek Szyprowski Tested-by: Marek Szyprowski --- arch/x86/kernel/amd_gart_64.c | 4 ++-- drivers/xen/swiotlb-xen.c | 4 ++-- include/linux/dma-direct.h | 5 +++-- kernel/dma/direct.c | 4 ++-- kernel/dma/swiotlb.c | 2 +- 5 files changed, 10 insertions(+), 9 deletions(-) (limited to 'arch') diff --git a/arch/x86/kernel/amd_gart_64.c b/arch/x86/kernel/amd_gart_64.c index a6ac3712db8b..5cfab41e8509 100644 --- a/arch/x86/kernel/amd_gart_64.c +++ b/arch/x86/kernel/amd_gart_64.c @@ -185,13 +185,13 @@ static void iommu_full(struct device *dev, size_t size, int dir) static inline int need_iommu(struct device *dev, unsigned long addr, size_t size) { - return force_iommu || !dma_capable(dev, addr, size); + return force_iommu || !dma_capable(dev, addr, size, true); } static inline int nonforced_iommu(struct device *dev, unsigned long addr, size_t size) { - return !dma_capable(dev, addr, size); + return !dma_capable(dev, addr, size, true); } /* Map a single continuous physical area into the IOMMU. diff --git a/drivers/xen/swiotlb-xen.c b/drivers/xen/swiotlb-xen.c index 3f8b2cdb4acb..b6d27762c6f8 100644 --- a/drivers/xen/swiotlb-xen.c +++ b/drivers/xen/swiotlb-xen.c @@ -375,7 +375,7 @@ static dma_addr_t xen_swiotlb_map_page(struct device *dev, struct page *page, * we can safely return the device addr and not worry about bounce * buffering it. */ - if (dma_capable(dev, dev_addr, size) && + if (dma_capable(dev, dev_addr, size, true) && !range_straddles_page_boundary(phys, size) && !xen_arch_need_swiotlb(dev, phys, dev_addr) && swiotlb_force != SWIOTLB_FORCE) @@ -397,7 +397,7 @@ static dma_addr_t xen_swiotlb_map_page(struct device *dev, struct page *page, /* * Ensure that the address returned is DMA'ble */ - if (unlikely(!dma_capable(dev, dev_addr, size))) { + if (unlikely(!dma_capable(dev, dev_addr, size, true))) { swiotlb_tbl_unmap_single(dev, map, size, size, dir, attrs | DMA_ATTR_SKIP_CPU_SYNC); return DMA_MAPPING_ERROR; diff --git a/include/linux/dma-direct.h b/include/linux/dma-direct.h index f8959f75e496..99b77dd5f79b 100644 --- a/include/linux/dma-direct.h +++ b/include/linux/dma-direct.h @@ -49,14 +49,15 @@ static inline phys_addr_t dma_to_phys(struct device *dev, dma_addr_t daddr) return __sme_clr(__dma_to_phys(dev, daddr)); } -static inline bool dma_capable(struct device *dev, dma_addr_t addr, size_t size) +static inline bool dma_capable(struct device *dev, dma_addr_t addr, size_t size, + bool is_ram) { dma_addr_t end = addr + size - 1; if (!dev->dma_mask) return false; - if (!IS_ENABLED(CONFIG_ARCH_DMA_ADDR_T_64BIT) && + if (is_ram && !IS_ENABLED(CONFIG_ARCH_DMA_ADDR_T_64BIT) && min(addr, end) < phys_to_dma(dev, PFN_PHYS(min_low_pfn))) return false; diff --git a/kernel/dma/direct.c b/kernel/dma/direct.c index a479bd2d1e8b..40f1f0aac4b1 100644 --- a/kernel/dma/direct.c +++ b/kernel/dma/direct.c @@ -363,7 +363,7 @@ static inline bool dma_direct_possible(struct device *dev, dma_addr_t dma_addr, size_t size) { return swiotlb_force != SWIOTLB_FORCE && - dma_capable(dev, dma_addr, size); + dma_capable(dev, dma_addr, size, true); } dma_addr_t dma_direct_map_page(struct device *dev, struct page *page, @@ -412,7 +412,7 @@ dma_addr_t dma_direct_map_resource(struct device *dev, phys_addr_t paddr, { dma_addr_t dma_addr = paddr; - if (unlikely(!dma_capable(dev, dma_addr, size))) { + if (unlikely(!dma_capable(dev, dma_addr, size, false))) { report_addr(dev, dma_addr, size); return DMA_MAPPING_ERROR; } diff --git a/kernel/dma/swiotlb.c b/kernel/dma/swiotlb.c index 673a2cdb2656..9280d6f8271e 100644 --- a/kernel/dma/swiotlb.c +++ b/kernel/dma/swiotlb.c @@ -678,7 +678,7 @@ bool swiotlb_map(struct device *dev, phys_addr_t *phys, dma_addr_t *dma_addr, /* Ensure that the address returned is DMA'ble */ *dma_addr = __phys_to_dma(dev, *phys); - if (unlikely(!dma_capable(dev, *dma_addr, size))) { + if (unlikely(!dma_capable(dev, *dma_addr, size, true))) { swiotlb_tbl_unmap_single(dev, *phys, size, size, dir, attrs | DMA_ATTR_SKIP_CPU_SYNC); return false; -- cgit v1.2.3-59-g8ed1b From a7ba70f1787f977f970cd116076c6fce4b9e01cc Mon Sep 17 00:00:00 2001 From: Nicolas Saenz Julienne Date: Thu, 21 Nov 2019 10:26:44 +0100 Subject: dma-mapping: treat dev->bus_dma_mask as a DMA limit Using a mask to represent bus DMA constraints has a set of limitations. The biggest one being it can only hold a power of two (minus one). The DMA mapping code is already aware of this and treats dev->bus_dma_mask as a limit. This quirk is already used by some architectures although still rare. With the introduction of the Raspberry Pi 4 we've found a new contender for the use of bus DMA limits, as its PCIe bus can only address the lower 3GB of memory (of a total of 4GB). This is impossible to represent with a mask. To make things worse the device-tree code rounds non power of two bus DMA limits to the next power of two, which is unacceptable in this case. In the light of this, rename dev->bus_dma_mask to dev->bus_dma_limit all over the tree and treat it as such. Note that dev->bus_dma_limit should contain the higher accessible DMA address. Signed-off-by: Nicolas Saenz Julienne Reviewed-by: Robin Murphy Signed-off-by: Christoph Hellwig --- arch/mips/pci/fixup-sb1250.c | 16 ++++++++-------- arch/powerpc/sysdev/fsl_pci.c | 6 +++--- arch/x86/kernel/pci-dma.c | 2 +- arch/x86/mm/mem_encrypt.c | 2 +- arch/x86/pci/sta2x11-fixup.c | 2 +- drivers/acpi/arm64/iort.c | 20 +++++++------------- drivers/ata/ahci.c | 2 +- drivers/iommu/dma-iommu.c | 3 +-- drivers/of/device.c | 9 +++++---- include/linux/device.h | 6 +++--- include/linux/dma-direct.h | 2 +- include/linux/dma-mapping.h | 2 +- kernel/dma/direct.c | 27 +++++++++++++-------------- 13 files changed, 46 insertions(+), 53 deletions(-) (limited to 'arch') diff --git a/arch/mips/pci/fixup-sb1250.c b/arch/mips/pci/fixup-sb1250.c index 8a41b359cf90..40efc990cdce 100644 --- a/arch/mips/pci/fixup-sb1250.c +++ b/arch/mips/pci/fixup-sb1250.c @@ -21,22 +21,22 @@ DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_SIBYTE, PCI_DEVICE_ID_BCM1250_PCI, /* * The BCM1250, etc. PCI host bridge does not support DAC on its 32-bit - * bus, so we set the bus's DMA mask accordingly. However the HT link + * bus, so we set the bus's DMA limit accordingly. However the HT link * down the artificial PCI-HT bridge supports 40-bit addressing and the * SP1011 HT-PCI bridge downstream supports both DAC and a 64-bit bus * width, so we record the PCI-HT bridge's secondary and subordinate bus - * numbers and do not set the mask for devices present in the inclusive + * numbers and do not set the limit for devices present in the inclusive * range of those. */ -struct sb1250_bus_dma_mask_exclude { +struct sb1250_bus_dma_limit_exclude { bool set; unsigned char start; unsigned char end; }; -static int sb1250_bus_dma_mask(struct pci_dev *dev, void *data) +static int sb1250_bus_dma_limit(struct pci_dev *dev, void *data) { - struct sb1250_bus_dma_mask_exclude *exclude = data; + struct sb1250_bus_dma_limit_exclude *exclude = data; bool exclude_this; bool ht_bridge; @@ -55,7 +55,7 @@ static int sb1250_bus_dma_mask(struct pci_dev *dev, void *data) exclude->start, exclude->end); } else { dev_dbg(&dev->dev, "disabling DAC for device"); - dev->dev.bus_dma_mask = DMA_BIT_MASK(32); + dev->dev.bus_dma_limit = DMA_BIT_MASK(32); } return 0; @@ -63,9 +63,9 @@ static int sb1250_bus_dma_mask(struct pci_dev *dev, void *data) static void quirk_sb1250_pci_dac(struct pci_dev *dev) { - struct sb1250_bus_dma_mask_exclude exclude = { .set = false }; + struct sb1250_bus_dma_limit_exclude exclude = { .set = false }; - pci_walk_bus(dev->bus, sb1250_bus_dma_mask, &exclude); + pci_walk_bus(dev->bus, sb1250_bus_dma_limit, &exclude); } DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_SIBYTE, PCI_DEVICE_ID_BCM1250_PCI, quirk_sb1250_pci_dac); diff --git a/arch/powerpc/sysdev/fsl_pci.c b/arch/powerpc/sysdev/fsl_pci.c index ff0e2b156cb5..617a443d673d 100644 --- a/arch/powerpc/sysdev/fsl_pci.c +++ b/arch/powerpc/sysdev/fsl_pci.c @@ -115,8 +115,8 @@ static void pci_dma_dev_setup_swiotlb(struct pci_dev *pdev) { struct pci_controller *hose = pci_bus_to_host(pdev->bus); - pdev->dev.bus_dma_mask = - hose->dma_window_base_cur + hose->dma_window_size; + pdev->dev.bus_dma_limit = + hose->dma_window_base_cur + hose->dma_window_size - 1; } static void setup_swiotlb_ops(struct pci_controller *hose) @@ -135,7 +135,7 @@ static void fsl_pci_dma_set_mask(struct device *dev, u64 dma_mask) * mapping that allows addressing any RAM address from across PCI. */ if (dev_is_pci(dev) && dma_mask >= pci64_dma_offset * 2 - 1) { - dev->bus_dma_mask = 0; + dev->bus_dma_limit = 0; dev->archdata.dma_offset = pci64_dma_offset; } } diff --git a/arch/x86/kernel/pci-dma.c b/arch/x86/kernel/pci-dma.c index fa4352dce491..3a75d665d43c 100644 --- a/arch/x86/kernel/pci-dma.c +++ b/arch/x86/kernel/pci-dma.c @@ -146,7 +146,7 @@ rootfs_initcall(pci_iommu_init); static int via_no_dac_cb(struct pci_dev *pdev, void *data) { - pdev->dev.bus_dma_mask = DMA_BIT_MASK(32); + pdev->dev.bus_dma_limit = DMA_BIT_MASK(32); return 0; } diff --git a/arch/x86/mm/mem_encrypt.c b/arch/x86/mm/mem_encrypt.c index 9268c12458c8..a03614bd3e1a 100644 --- a/arch/x86/mm/mem_encrypt.c +++ b/arch/x86/mm/mem_encrypt.c @@ -367,7 +367,7 @@ bool force_dma_unencrypted(struct device *dev) if (sme_active()) { u64 dma_enc_mask = DMA_BIT_MASK(__ffs64(sme_me_mask)); u64 dma_dev_mask = min_not_zero(dev->coherent_dma_mask, - dev->bus_dma_mask); + dev->bus_dma_limit); if (dma_dev_mask <= dma_enc_mask) return true; diff --git a/arch/x86/pci/sta2x11-fixup.c b/arch/x86/pci/sta2x11-fixup.c index 4a631264b809..c313d784efab 100644 --- a/arch/x86/pci/sta2x11-fixup.c +++ b/arch/x86/pci/sta2x11-fixup.c @@ -143,7 +143,7 @@ static void sta2x11_map_ep(struct pci_dev *pdev) dev->dma_pfn_offset = PFN_DOWN(-amba_base); - dev->bus_dma_mask = max_amba_addr; + dev->bus_dma_limit = max_amba_addr; pci_set_consistent_dma_mask(pdev, max_amba_addr); pci_set_dma_mask(pdev, max_amba_addr); diff --git a/drivers/acpi/arm64/iort.c b/drivers/acpi/arm64/iort.c index 5a7551d060f2..33f71983e001 100644 --- a/drivers/acpi/arm64/iort.c +++ b/drivers/acpi/arm64/iort.c @@ -1057,8 +1057,8 @@ static int rc_dma_get_range(struct device *dev, u64 *size) */ void iort_dma_setup(struct device *dev, u64 *dma_addr, u64 *dma_size) { - u64 mask, dmaaddr = 0, size = 0, offset = 0; - int ret, msb; + u64 end, mask, dmaaddr = 0, size = 0, offset = 0; + int ret; /* * If @dev is expected to be DMA-capable then the bus code that created @@ -1085,19 +1085,13 @@ void iort_dma_setup(struct device *dev, u64 *dma_addr, u64 *dma_size) } if (!ret) { - msb = fls64(dmaaddr + size - 1); /* - * Round-up to the power-of-two mask or set - * the mask to the whole 64-bit address space - * in case the DMA region covers the full - * memory window. + * Limit coherent and dma mask based on size retrieved from + * firmware. */ - mask = msb == 64 ? U64_MAX : (1ULL << msb) - 1; - /* - * Limit coherent and dma mask based on size - * retrieved from firmware. - */ - dev->bus_dma_mask = mask; + end = dmaaddr + size - 1; + mask = DMA_BIT_MASK(ilog2(end) + 1); + dev->bus_dma_limit = end; dev->coherent_dma_mask = mask; *dev->dma_mask = mask; } diff --git a/drivers/ata/ahci.c b/drivers/ata/ahci.c index 05c2b32dcc4d..7c6d06ffb586 100644 --- a/drivers/ata/ahci.c +++ b/drivers/ata/ahci.c @@ -897,7 +897,7 @@ static int ahci_configure_dma_masks(struct pci_dev *pdev, int using_dac) * value, don't extend it here. This happens on STA2X11, for example. * * XXX: manipulating the DMA mask from platform code is completely - * bogus, platform code should use dev->bus_dma_mask instead.. + * bogus, platform code should use dev->bus_dma_limit instead.. */ if (pdev->dma_mask && pdev->dma_mask < DMA_BIT_MASK(32)) return 0; diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c index 0fa8c1d818b7..646332fbf3d7 100644 --- a/drivers/iommu/dma-iommu.c +++ b/drivers/iommu/dma-iommu.c @@ -405,8 +405,7 @@ static dma_addr_t iommu_dma_alloc_iova(struct iommu_domain *domain, if (iova_len < (1 << (IOVA_RANGE_CACHE_MAX_SIZE - 1))) iova_len = roundup_pow_of_two(iova_len); - if (dev->bus_dma_mask) - dma_limit &= dev->bus_dma_mask; + dma_limit = min_not_zero(dma_limit, dev->bus_dma_limit); if (domain->geometry.force_aperture) dma_limit = min(dma_limit, domain->geometry.aperture_end); diff --git a/drivers/of/device.c b/drivers/of/device.c index da8158392010..e9127db7b067 100644 --- a/drivers/of/device.c +++ b/drivers/of/device.c @@ -93,7 +93,7 @@ int of_dma_configure(struct device *dev, struct device_node *np, bool force_dma) bool coherent; unsigned long offset; const struct iommu_ops *iommu; - u64 mask; + u64 mask, end; ret = of_dma_get_range(np, &dma_addr, &paddr, &size); if (ret < 0) { @@ -148,12 +148,13 @@ int of_dma_configure(struct device *dev, struct device_node *np, bool force_dma) * Limit coherent and dma mask based on size and default mask * set by the driver. */ - mask = DMA_BIT_MASK(ilog2(dma_addr + size - 1) + 1); + end = dma_addr + size - 1; + mask = DMA_BIT_MASK(ilog2(end) + 1); dev->coherent_dma_mask &= mask; *dev->dma_mask &= mask; - /* ...but only set bus mask if we found valid dma-ranges earlier */ + /* ...but only set bus limit if we found valid dma-ranges earlier */ if (!ret) - dev->bus_dma_mask = mask; + dev->bus_dma_limit = end; coherent = of_dma_is_coherent(np); dev_dbg(dev, "device is%sdma coherent\n", diff --git a/include/linux/device.h b/include/linux/device.h index 297239a08bb7..e396de656f20 100644 --- a/include/linux/device.h +++ b/include/linux/device.h @@ -1186,8 +1186,8 @@ struct dev_links_info { * @coherent_dma_mask: Like dma_mask, but for alloc_coherent mapping as not all * hardware supports 64-bit addresses for consistent allocations * such descriptors. - * @bus_dma_mask: Mask of an upstream bridge or bus which imposes a smaller DMA - * limit than the device itself supports. + * @bus_dma_limit: Limit of an upstream bridge or bus which imposes a smaller + * DMA limit than the device itself supports. * @dma_pfn_offset: offset of DMA memory range relatively of RAM * @dma_parms: A low level driver may set these to teach IOMMU code about * segment limitations. @@ -1270,7 +1270,7 @@ struct device { not all hardware supports 64 bit addresses for consistent allocations such descriptors. */ - u64 bus_dma_mask; /* upstream dma_mask constraint */ + u64 bus_dma_limit; /* upstream dma constraint */ unsigned long dma_pfn_offset; struct device_dma_parameters *dma_parms; diff --git a/include/linux/dma-direct.h b/include/linux/dma-direct.h index 452f5280cde3..24b8684aa21d 100644 --- a/include/linux/dma-direct.h +++ b/include/linux/dma-direct.h @@ -63,7 +63,7 @@ static inline bool dma_capable(struct device *dev, dma_addr_t addr, size_t size, min(addr, end) < phys_to_dma(dev, PFN_PHYS(min_low_pfn))) return false; - return end <= min_not_zero(*dev->dma_mask, dev->bus_dma_mask); + return end <= min_not_zero(*dev->dma_mask, dev->bus_dma_limit); } u64 dma_direct_get_required_mask(struct device *dev); diff --git a/include/linux/dma-mapping.h b/include/linux/dma-mapping.h index 4d450672b7d6..c4d8741264bd 100644 --- a/include/linux/dma-mapping.h +++ b/include/linux/dma-mapping.h @@ -697,7 +697,7 @@ static inline int dma_coerce_mask_and_coherent(struct device *dev, u64 mask) */ static inline bool dma_addressing_limited(struct device *dev) { - return min_not_zero(dma_get_mask(dev), dev->bus_dma_mask) < + return min_not_zero(dma_get_mask(dev), dev->bus_dma_limit) < dma_get_required_mask(dev); } diff --git a/kernel/dma/direct.c b/kernel/dma/direct.c index 267b23a13b69..6af7ae83c4ad 100644 --- a/kernel/dma/direct.c +++ b/kernel/dma/direct.c @@ -27,10 +27,10 @@ static void report_addr(struct device *dev, dma_addr_t dma_addr, size_t size) { if (!dev->dma_mask) { dev_err_once(dev, "DMA map on device without dma_mask\n"); - } else if (*dev->dma_mask >= DMA_BIT_MASK(32) || dev->bus_dma_mask) { + } else if (*dev->dma_mask >= DMA_BIT_MASK(32) || dev->bus_dma_limit) { dev_err_once(dev, - "overflow %pad+%zu of DMA mask %llx bus mask %llx\n", - &dma_addr, size, *dev->dma_mask, dev->bus_dma_mask); + "overflow %pad+%zu of DMA mask %llx bus limit %llx\n", + &dma_addr, size, *dev->dma_mask, dev->bus_dma_limit); } WARN_ON_ONCE(1); } @@ -57,15 +57,14 @@ u64 dma_direct_get_required_mask(struct device *dev) } static gfp_t __dma_direct_optimal_gfp_mask(struct device *dev, u64 dma_mask, - u64 *phys_mask) + u64 *phys_limit) { - if (dev->bus_dma_mask && dev->bus_dma_mask < dma_mask) - dma_mask = dev->bus_dma_mask; + u64 dma_limit = min_not_zero(dma_mask, dev->bus_dma_limit); if (force_dma_unencrypted(dev)) - *phys_mask = __dma_to_phys(dev, dma_mask); + *phys_limit = __dma_to_phys(dev, dma_limit); else - *phys_mask = dma_to_phys(dev, dma_mask); + *phys_limit = dma_to_phys(dev, dma_limit); /* * Optimistically try the zone that the physical address mask falls @@ -75,9 +74,9 @@ static gfp_t __dma_direct_optimal_gfp_mask(struct device *dev, u64 dma_mask, * Note that GFP_DMA32 and GFP_DMA are no ops without the corresponding * zones. */ - if (*phys_mask <= DMA_BIT_MASK(zone_dma_bits)) + if (*phys_limit <= DMA_BIT_MASK(zone_dma_bits)) return GFP_DMA; - if (*phys_mask <= DMA_BIT_MASK(32)) + if (*phys_limit <= DMA_BIT_MASK(32)) return GFP_DMA32; return 0; } @@ -85,7 +84,7 @@ static gfp_t __dma_direct_optimal_gfp_mask(struct device *dev, u64 dma_mask, static bool dma_coherent_ok(struct device *dev, phys_addr_t phys, size_t size) { return phys_to_dma_direct(dev, phys) + size - 1 <= - min_not_zero(dev->coherent_dma_mask, dev->bus_dma_mask); + min_not_zero(dev->coherent_dma_mask, dev->bus_dma_limit); } struct page *__dma_direct_alloc_pages(struct device *dev, size_t size, @@ -94,7 +93,7 @@ struct page *__dma_direct_alloc_pages(struct device *dev, size_t size, size_t alloc_size = PAGE_ALIGN(size); int node = dev_to_node(dev); struct page *page = NULL; - u64 phys_mask; + u64 phys_limit; if (attrs & DMA_ATTR_NO_WARN) gfp |= __GFP_NOWARN; @@ -102,7 +101,7 @@ struct page *__dma_direct_alloc_pages(struct device *dev, size_t size, /* we always manually zero the memory once we are done: */ gfp &= ~__GFP_ZERO; gfp |= __dma_direct_optimal_gfp_mask(dev, dev->coherent_dma_mask, - &phys_mask); + &phys_limit); page = dma_alloc_contiguous(dev, alloc_size, gfp); if (page && !dma_coherent_ok(dev, page_to_phys(page), size)) { dma_free_contiguous(dev, page, alloc_size); @@ -116,7 +115,7 @@ again: page = NULL; if (IS_ENABLED(CONFIG_ZONE_DMA32) && - phys_mask < DMA_BIT_MASK(64) && + phys_limit < DMA_BIT_MASK(64) && !(gfp & (GFP_DMA32 | GFP_DMA))) { gfp |= GFP_DMA32; goto again; -- cgit v1.2.3-59-g8ed1b