From f02716db951c5e01acf48df9fe815007674fecff Mon Sep 17 00:00:00 2001 From: Dan Williams Date: Wed, 15 Jun 2016 14:59:17 -0700 Subject: libnvdimm: use devm_add_action_or_reset() Clean up needless calls to the action routine by letting devm_add_action_or_reset() call it automatically. This does cause the disk to registered and immediately unregistered when a memory allocation fails, but the block layer should be prepared for such an event. Reported-by: Sudip Mukherjee Signed-off-by: Dan Williams --- drivers/nvdimm/blk.c | 11 ++++------- drivers/nvdimm/pmem.c | 14 ++++++-------- 2 files changed, 10 insertions(+), 15 deletions(-) (limited to 'drivers/nvdimm') diff --git a/drivers/nvdimm/blk.c b/drivers/nvdimm/blk.c index 495e06d9f7e7..dfe691cf4d74 100644 --- a/drivers/nvdimm/blk.c +++ b/drivers/nvdimm/blk.c @@ -267,10 +267,8 @@ static int nsblk_attach_disk(struct nd_namespace_blk *nsblk) q = blk_alloc_queue(GFP_KERNEL); if (!q) return -ENOMEM; - if (devm_add_action(dev, nd_blk_release_queue, q)) { - blk_cleanup_queue(q); + if (devm_add_action_or_reset(dev, nd_blk_release_queue, q)) return -ENOMEM; - } blk_queue_make_request(q, nd_blk_make_request); blk_queue_max_hw_sectors(q, UINT_MAX); @@ -282,10 +280,6 @@ static int nsblk_attach_disk(struct nd_namespace_blk *nsblk) disk = alloc_disk(0); if (!disk) return -ENOMEM; - if (devm_add_action(dev, nd_blk_release_disk, disk)) { - put_disk(disk); - return -ENOMEM; - } disk->driverfs_dev = dev; disk->first_minor = 0; @@ -296,6 +290,9 @@ static int nsblk_attach_disk(struct nd_namespace_blk *nsblk) set_capacity(disk, 0); add_disk(disk); + if (devm_add_action_or_reset(dev, nd_blk_release_disk, disk)) + return -ENOMEM; + if (nsblk_meta_size(nsblk)) { int rc = nd_integrity_init(disk, nsblk_meta_size(nsblk)); diff --git a/drivers/nvdimm/pmem.c b/drivers/nvdimm/pmem.c index 608fc4464574..ba253df6233f 100644 --- a/drivers/nvdimm/pmem.c +++ b/drivers/nvdimm/pmem.c @@ -195,7 +195,7 @@ static void pmem_release_queue(void *q) blk_cleanup_queue(q); } -void pmem_release_disk(void *disk) +static void pmem_release_disk(void *disk) { del_gendisk(disk); put_disk(disk); @@ -269,10 +269,8 @@ static int pmem_attach_disk(struct device *dev, * At release time the queue must be dead before * devm_memremap_pages is unwound */ - if (devm_add_action(dev, pmem_release_queue, q)) { - blk_cleanup_queue(q); + if (devm_add_action_or_reset(dev, pmem_release_queue, q)) return -ENOMEM; - } if (IS_ERR(addr)) return PTR_ERR(addr); @@ -288,10 +286,6 @@ static int pmem_attach_disk(struct device *dev, disk = alloc_disk_node(0, nid); if (!disk) return -ENOMEM; - if (devm_add_action(dev, pmem_release_disk, disk)) { - put_disk(disk); - return -ENOMEM; - } disk->fops = &pmem_fops; disk->queue = q; @@ -305,6 +299,10 @@ static int pmem_attach_disk(struct device *dev, nvdimm_badblocks_populate(to_nd_region(dev->parent), &pmem->bb, res); disk->bb = &pmem->bb; add_disk(disk); + + if (devm_add_action_or_reset(dev, pmem_release_disk, disk)) + return -ENOMEM; + revalidate_disk(disk); return 0; -- cgit v1.2.3-59-g8ed1b From 42588958147c7cc5a68726f14579011ad96663bb Mon Sep 17 00:00:00 2001 From: Dan Williams Date: Fri, 27 May 2016 13:28:31 -0700 Subject: libnvdimm: IS_ERR() usage cleanup Prompted by commit 287980e49ffc "remove lots of IS_ERR_VALUE abuses", I ran make coccicheck against drivers/nvdimm/ and found that: if (IS_ERR(x)) return PTR_ERR(x); return 0; ...can be replaced with PTR_ERR_OR_ZERO(). Reported-by: Linus Torvalds Reviewed-by: Johannes Thumshirn Signed-off-by: Dan Williams --- drivers/nvdimm/bus.c | 6 ++---- drivers/nvdimm/claim.c | 5 ++--- 2 files changed, 4 insertions(+), 7 deletions(-) (limited to 'drivers/nvdimm') diff --git a/drivers/nvdimm/bus.c b/drivers/nvdimm/bus.c index f085f8bceae8..e4882e63bece 100644 --- a/drivers/nvdimm/bus.c +++ b/drivers/nvdimm/bus.c @@ -395,12 +395,10 @@ int nvdimm_bus_create_ndctl(struct nvdimm_bus *nvdimm_bus) dev = device_create(nd_class, &nvdimm_bus->dev, devt, nvdimm_bus, "ndctl%d", nvdimm_bus->id); - if (IS_ERR(dev)) { + if (IS_ERR(dev)) dev_dbg(&nvdimm_bus->dev, "failed to register ndctl%d: %ld\n", nvdimm_bus->id, PTR_ERR(dev)); - return PTR_ERR(dev); - } - return 0; + return PTR_ERR_OR_ZERO(dev); } void nvdimm_bus_destroy_ndctl(struct nvdimm_bus *nvdimm_bus) diff --git a/drivers/nvdimm/claim.c b/drivers/nvdimm/claim.c index 8b2e3c4fb0ad..9997ff94a132 100644 --- a/drivers/nvdimm/claim.c +++ b/drivers/nvdimm/claim.c @@ -266,9 +266,8 @@ int devm_nsio_enable(struct device *dev, struct nd_namespace_io *nsio) nsio->addr = devm_memremap(dev, res->start, resource_size(res), ARCH_MEMREMAP_PMEM); - if (IS_ERR(nsio->addr)) - return PTR_ERR(nsio->addr); - return 0; + + return PTR_ERR_OR_ZERO(nsio->addr); } EXPORT_SYMBOL_GPL(devm_nsio_enable); -- cgit v1.2.3-59-g8ed1b From f295e53b60eb93ee53ed5ac610374ed293caa57b Mon Sep 17 00:00:00 2001 From: Dan Williams Date: Fri, 17 Jun 2016 11:08:06 -0700 Subject: libnvdimm, pmem: allow nfit_test to override pmem_direct_access() Currently phys_to_pfn_t() is an exported symbol to allow nfit_test to override it and indicate that nfit_test-pmem is not device-mapped. Now, we want to enable nfit_test to operate without DMA_CMA and the pmem it provides will no longer be physically contiguous, i.e. won't be capable of supporting direct_access requests larger than a page. Make pmem_direct_access() a weak symbol so that it can be replaced by the tools/testing/nvdimm/ version, and move phys_to_pfn_t() to a static inline now that it no longer needs to be overridden. Acked-by: Johannes Thumshirn Signed-off-by: Dan Williams --- drivers/nvdimm/pmem.c | 18 +++------------ drivers/nvdimm/pmem.h | 24 ++++++++++++++++++++ include/linux/pfn_t.h | 5 ++++- kernel/memremap.c | 6 ----- tools/testing/nvdimm/Kbuild | 3 ++- tools/testing/nvdimm/pmem-dax.c | 42 +++++++++++++++++++++++++++++++++++ tools/testing/nvdimm/test/iomap.c | 3 ++- tools/testing/nvdimm/test/nfit_test.h | 2 ++ 8 files changed, 79 insertions(+), 24 deletions(-) create mode 100644 drivers/nvdimm/pmem.h create mode 100644 tools/testing/nvdimm/pmem-dax.c (limited to 'drivers/nvdimm') diff --git a/drivers/nvdimm/pmem.c b/drivers/nvdimm/pmem.c index ba253df6233f..b6fcb97a601c 100644 --- a/drivers/nvdimm/pmem.c +++ b/drivers/nvdimm/pmem.c @@ -29,23 +29,10 @@ #include #include #include +#include "pmem.h" #include "pfn.h" #include "nd.h" -struct pmem_device { - /* One contiguous memory region per device */ - phys_addr_t phys_addr; - /* when non-zero this device is hosting a 'pfn' instance */ - phys_addr_t data_offset; - u64 pfn_flags; - void __pmem *virt_addr; - /* immutable base size of the namespace */ - size_t size; - /* trim size when namespace capacity has been section aligned */ - u32 pfn_pad; - struct badblocks bb; -}; - static void pmem_clear_poison(struct pmem_device *pmem, phys_addr_t offset, unsigned int len) { @@ -163,7 +150,8 @@ static int pmem_rw_page(struct block_device *bdev, sector_t sector, return rc; } -static long pmem_direct_access(struct block_device *bdev, sector_t sector, +/* see "strong" declaration in tools/testing/nvdimm/pmem-dax.c */ +__weak long pmem_direct_access(struct block_device *bdev, sector_t sector, void __pmem **kaddr, pfn_t *pfn, long size) { struct pmem_device *pmem = bdev->bd_queue->queuedata; diff --git a/drivers/nvdimm/pmem.h b/drivers/nvdimm/pmem.h new file mode 100644 index 000000000000..c48d4e3aa346 --- /dev/null +++ b/drivers/nvdimm/pmem.h @@ -0,0 +1,24 @@ +#ifndef __NVDIMM_PMEM_H__ +#define __NVDIMM_PMEM_H__ +#include +#include +#include +#include + +long pmem_direct_access(struct block_device *bdev, sector_t sector, + void __pmem **kaddr, pfn_t *pfn, long size); +/* this definition is in it's own header for tools/testing/nvdimm to consume */ +struct pmem_device { + /* One contiguous memory region per device */ + phys_addr_t phys_addr; + /* when non-zero this device is hosting a 'pfn' instance */ + phys_addr_t data_offset; + u64 pfn_flags; + void __pmem *virt_addr; + /* immutable base size of the namespace */ + size_t size; + /* trim size when namespace capacity has been section aligned */ + u32 pfn_pad; + struct badblocks bb; +}; +#endif /* __NVDIMM_PMEM_H__ */ diff --git a/include/linux/pfn_t.h b/include/linux/pfn_t.h index 94994810c7c0..a3d90b9da18d 100644 --- a/include/linux/pfn_t.h +++ b/include/linux/pfn_t.h @@ -28,7 +28,10 @@ static inline pfn_t pfn_to_pfn_t(unsigned long pfn) return __pfn_to_pfn_t(pfn, 0); } -extern pfn_t phys_to_pfn_t(phys_addr_t addr, u64 flags); +static inline pfn_t phys_to_pfn_t(phys_addr_t addr, u64 flags) +{ + return __pfn_to_pfn_t(addr >> PAGE_SHIFT, flags); +} static inline bool pfn_t_has_page(pfn_t pfn) { diff --git a/kernel/memremap.c b/kernel/memremap.c index 017532193fb1..852e5266124a 100644 --- a/kernel/memremap.c +++ b/kernel/memremap.c @@ -169,12 +169,6 @@ void devm_memunmap(struct device *dev, void *addr) } EXPORT_SYMBOL(devm_memunmap); -pfn_t phys_to_pfn_t(phys_addr_t addr, u64 flags) -{ - return __pfn_to_pfn_t(addr >> PAGE_SHIFT, flags); -} -EXPORT_SYMBOL(phys_to_pfn_t); - #ifdef CONFIG_ZONE_DEVICE static DEFINE_MUTEX(pgmap_lock); static RADIX_TREE(pgmap_radix, GFP_KERNEL); diff --git a/tools/testing/nvdimm/Kbuild b/tools/testing/nvdimm/Kbuild index 785985677159..4adfa9cdf26f 100644 --- a/tools/testing/nvdimm/Kbuild +++ b/tools/testing/nvdimm/Kbuild @@ -11,12 +11,12 @@ ldflags-y += --wrap=__devm_release_region ldflags-y += --wrap=__request_region ldflags-y += --wrap=__release_region ldflags-y += --wrap=devm_memremap_pages -ldflags-y += --wrap=phys_to_pfn_t DRIVERS := ../../../drivers NVDIMM_SRC := $(DRIVERS)/nvdimm ACPI_SRC := $(DRIVERS)/acpi DAX_SRC := $(DRIVERS)/dax +ccflags-y := -I$(src)/$(NVDIMM_SRC)/ obj-$(CONFIG_LIBNVDIMM) += libnvdimm.o obj-$(CONFIG_BLK_DEV_PMEM) += nd_pmem.o @@ -31,6 +31,7 @@ nfit-y := $(ACPI_SRC)/nfit.o nfit-y += config_check.o nd_pmem-y := $(NVDIMM_SRC)/pmem.o +nd_pmem-y += pmem-dax.o nd_pmem-y += config_check.o nd_btt-y := $(NVDIMM_SRC)/btt.o diff --git a/tools/testing/nvdimm/pmem-dax.c b/tools/testing/nvdimm/pmem-dax.c new file mode 100644 index 000000000000..fdba77f2bb06 --- /dev/null +++ b/tools/testing/nvdimm/pmem-dax.c @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2014-2016, Intel Corporation. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + */ +#include "test/nfit_test.h" +#include +#include +#include + +long pmem_direct_access(struct block_device *bdev, sector_t sector, + void __pmem **kaddr, pfn_t *pfn, long size) +{ + struct pmem_device *pmem = bdev->bd_queue->queuedata; + resource_size_t offset = sector * 512 + pmem->data_offset; + + /* disable DAX for nfit_test pmem devices */ + if (get_nfit_res(pmem->phys_addr + offset)) { + dev_info_once(pmem->bb.dev, "dax is disabled for nfit_test\n"); + return -EIO; + } + + if (unlikely(is_bad_pmem(&pmem->bb, sector, size))) + return -EIO; + *kaddr = pmem->virt_addr + offset; + *pfn = phys_to_pfn_t(pmem->phys_addr + offset, pmem->pfn_flags); + + /* + * If badblocks are present, limit known good range to the + * requested range. + */ + if (unlikely(pmem->bb.count)) + return size; + return pmem->size - pmem->pfn_pad - offset; +} diff --git a/tools/testing/nvdimm/test/iomap.c b/tools/testing/nvdimm/test/iomap.c index c842095f2801..9966f093b4ff 100644 --- a/tools/testing/nvdimm/test/iomap.c +++ b/tools/testing/nvdimm/test/iomap.c @@ -52,7 +52,7 @@ static struct nfit_test_resource *__get_nfit_res(resource_size_t resource) return NULL; } -static struct nfit_test_resource *get_nfit_res(resource_size_t resource) +struct nfit_test_resource *get_nfit_res(resource_size_t resource) { struct nfit_test_resource *res; @@ -62,6 +62,7 @@ static struct nfit_test_resource *get_nfit_res(resource_size_t resource) return res; } +EXPORT_SYMBOL(get_nfit_res); void __iomem *__nfit_test_ioremap(resource_size_t offset, unsigned long size, void __iomem *(*fallback_fn)(resource_size_t, unsigned long)) diff --git a/tools/testing/nvdimm/test/nfit_test.h b/tools/testing/nvdimm/test/nfit_test.h index 96c5e16d7db9..9f18e2a4a862 100644 --- a/tools/testing/nvdimm/test/nfit_test.h +++ b/tools/testing/nvdimm/test/nfit_test.h @@ -12,6 +12,7 @@ */ #ifndef __NFIT_TEST_H__ #define __NFIT_TEST_H__ +#include struct nfit_test_resource { struct list_head list; @@ -26,4 +27,5 @@ void __iomem *__wrap_ioremap_nocache(resource_size_t offset, void __wrap_iounmap(volatile void __iomem *addr); void nfit_test_setup(nfit_test_lookup_fn lookup); void nfit_test_teardown(void); +struct nfit_test_resource *get_nfit_res(resource_size_t resource); #endif -- cgit v1.2.3-59-g8ed1b From 8729bdea829ebecfbc9601ba792e35f63331f63e Mon Sep 17 00:00:00 2001 From: Johannes Thumshirn Date: Thu, 23 Jun 2016 11:52:04 +0200 Subject: libnvdimm: initialize struct blk_integrity with 0 Initialize struct blk_integrity with 0 as blk_integrity_register() takes the then unitialized struct blk_integrity::flags and ORs it to the resulting block integrity structure. Signed-off-by: Johannes Thumshirn Signed-off-by: Dan Williams --- drivers/nvdimm/core.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'drivers/nvdimm') diff --git a/drivers/nvdimm/core.c b/drivers/nvdimm/core.c index be89764315c2..32e4fe2f6274 100644 --- a/drivers/nvdimm/core.c +++ b/drivers/nvdimm/core.c @@ -601,7 +601,8 @@ int nd_integrity_init(struct gendisk *disk, unsigned long meta_size) if (meta_size == 0) return 0; - bi.profile = NULL; + memset(&bi, 0, sizeof(bi)); + bi.tuple_size = meta_size; bi.tag_size = meta_size; -- cgit v1.2.3-59-g8ed1b From 29b9aa0aa3837c93ecd804dd3ada39b8cc75607d Mon Sep 17 00:00:00 2001 From: Dan Williams Date: Mon, 6 Jun 2016 17:42:38 -0700 Subject: libnvdimm: introduce devm_nvdimm_memremap(), convert nfit_spa_map() users In preparation for generically mapping flush hint addresses for both the BLK and PMEM use case, provide a generic / reference counted mapping api. Given the fact that a dimm may belong to multiple regions (PMEM and BLK), the flush hint addresses need to be held valid as long as any region associated with the dimm is active. This is similar to the existing BLK-region case where multiple BLK-regions may share an aperture mapping. Up-level this shared / reference-counted mapping capability from the nfit driver to a core nvdimm capability. This eliminates the need for the nd_blk_region.disable() callback. Note that the removal of nfit_spa_map() and related infrastructure is deferred to a later patch. Signed-off-by: Dan Williams --- drivers/acpi/nfit.c | 14 +++--- drivers/nvdimm/Kconfig | 2 +- drivers/nvdimm/core.c | 123 ++++++++++++++++++++++++++++++++++++++++++++++ drivers/nvdimm/nd-core.h | 1 + include/linux/libnvdimm.h | 9 ++++ 5 files changed, 141 insertions(+), 8 deletions(-) (limited to 'drivers/nvdimm') diff --git a/drivers/acpi/nfit.c b/drivers/acpi/nfit.c index f8c1a850effc..b047dbe13bed 100644 --- a/drivers/acpi/nfit.c +++ b/drivers/acpi/nfit.c @@ -1616,7 +1616,8 @@ static void __iomem *__nfit_spa_map(struct acpi_nfit_desc *acpi_desc, * when all region devices referencing the same mapping are disabled / * unbound. */ -static void __iomem *nfit_spa_map(struct acpi_nfit_desc *acpi_desc, +static __maybe_unused void __iomem *nfit_spa_map( + struct acpi_nfit_desc *acpi_desc, struct acpi_nfit_system_address *spa, enum spa_map_type type) { void __iomem *iomem; @@ -1669,7 +1670,6 @@ static int acpi_nfit_blk_region_enable(struct nvdimm_bus *nvdimm_bus, struct device *dev) { struct nvdimm_bus_descriptor *nd_desc = to_nd_desc(nvdimm_bus); - struct acpi_nfit_desc *acpi_desc = to_acpi_desc(nd_desc); struct nd_blk_region *ndbr = to_nd_blk_region(dev); struct nfit_flush *nfit_flush; struct nfit_blk_mmio *mmio; @@ -1697,8 +1697,8 @@ static int acpi_nfit_blk_region_enable(struct nvdimm_bus *nvdimm_bus, /* map block aperture memory */ nfit_blk->bdw_offset = nfit_mem->bdw->offset; mmio = &nfit_blk->mmio[BDW]; - mmio->addr.base = nfit_spa_map(acpi_desc, nfit_mem->spa_bdw, - SPA_MAP_APERTURE); + mmio->addr.base = devm_nvdimm_memremap(dev, nfit_mem->spa_bdw->address, + nfit_mem->spa_bdw->length, ARCH_MEMREMAP_PMEM); if (!mmio->addr.base) { dev_dbg(dev, "%s: %s failed to map bdw\n", __func__, nvdimm_name(nvdimm)); @@ -1720,8 +1720,8 @@ static int acpi_nfit_blk_region_enable(struct nvdimm_bus *nvdimm_bus, nfit_blk->cmd_offset = nfit_mem->dcr->command_offset; nfit_blk->stat_offset = nfit_mem->dcr->status_offset; mmio = &nfit_blk->mmio[DCR]; - mmio->addr.base = nfit_spa_map(acpi_desc, nfit_mem->spa_dcr, - SPA_MAP_CONTROL); + mmio->addr.base = devm_nvdimm_ioremap(dev, nfit_mem->spa_dcr->address, + nfit_mem->spa_dcr->length); if (!mmio->addr.base) { dev_dbg(dev, "%s: %s failed to map dcr\n", __func__, nvdimm_name(nvdimm)); @@ -1748,7 +1748,7 @@ static int acpi_nfit_blk_region_enable(struct nvdimm_bus *nvdimm_bus, nfit_flush = nfit_mem->nfit_flush; if (nfit_flush && nfit_flush->flush->hint_count != 0) { - nfit_blk->nvdimm_flush = devm_ioremap_nocache(dev, + nfit_blk->nvdimm_flush = devm_nvdimm_ioremap(dev, nfit_flush->flush->hint_address[0], 8); if (!nfit_blk->nvdimm_flush) return -ENOMEM; diff --git a/drivers/nvdimm/Kconfig b/drivers/nvdimm/Kconfig index 7c8a3bf07884..124c2432ac9c 100644 --- a/drivers/nvdimm/Kconfig +++ b/drivers/nvdimm/Kconfig @@ -1,6 +1,7 @@ menuconfig LIBNVDIMM tristate "NVDIMM (Non-Volatile Memory Device) Support" depends on PHYS_ADDR_T_64BIT + depends on HAS_IOMEM depends on BLK_DEV help Generic support for non-volatile memory devices including @@ -19,7 +20,6 @@ if LIBNVDIMM config BLK_DEV_PMEM tristate "PMEM: Persistent memory block device support" default LIBNVDIMM - depends on HAS_IOMEM select ND_BTT if BTT select ND_PFN if NVDIMM_PFN help diff --git a/drivers/nvdimm/core.c b/drivers/nvdimm/core.c index 32e4fe2f6274..757e0cf028bf 100644 --- a/drivers/nvdimm/core.c +++ b/drivers/nvdimm/core.c @@ -20,6 +20,7 @@ #include #include #include +#include #include "nd-core.h" #include "nd.h" @@ -57,6 +58,127 @@ bool is_nvdimm_bus_locked(struct device *dev) } EXPORT_SYMBOL(is_nvdimm_bus_locked); +struct nvdimm_map { + struct nvdimm_bus *nvdimm_bus; + struct list_head list; + resource_size_t offset; + unsigned long flags; + size_t size; + union { + void *mem; + void __iomem *iomem; + }; + struct kref kref; +}; + +static struct nvdimm_map *find_nvdimm_map(struct device *dev, + resource_size_t offset) +{ + struct nvdimm_bus *nvdimm_bus = walk_to_nvdimm_bus(dev); + struct nvdimm_map *nvdimm_map; + + list_for_each_entry(nvdimm_map, &nvdimm_bus->mapping_list, list) + if (nvdimm_map->offset == offset) + return nvdimm_map; + return NULL; +} + +static struct nvdimm_map *alloc_nvdimm_map(struct device *dev, + resource_size_t offset, size_t size, unsigned long flags) +{ + struct nvdimm_bus *nvdimm_bus = walk_to_nvdimm_bus(dev); + struct nvdimm_map *nvdimm_map; + + nvdimm_map = kzalloc(sizeof(*nvdimm_map), GFP_KERNEL); + if (!nvdimm_map) + return NULL; + + INIT_LIST_HEAD(&nvdimm_map->list); + nvdimm_map->nvdimm_bus = nvdimm_bus; + nvdimm_map->offset = offset; + nvdimm_map->flags = flags; + nvdimm_map->size = size; + kref_init(&nvdimm_map->kref); + + if (!request_mem_region(offset, size, dev_name(&nvdimm_bus->dev))) + goto err_request_region; + + if (flags) + nvdimm_map->mem = memremap(offset, size, flags); + else + nvdimm_map->iomem = ioremap(offset, size); + + if (!nvdimm_map->mem) + goto err_map; + + dev_WARN_ONCE(dev, !is_nvdimm_bus_locked(dev), "%s: bus unlocked!", + __func__); + list_add(&nvdimm_map->list, &nvdimm_bus->mapping_list); + + return nvdimm_map; + + err_map: + release_mem_region(offset, size); + err_request_region: + kfree(nvdimm_map); + return NULL; +} + +static void nvdimm_map_release(struct kref *kref) +{ + struct nvdimm_bus *nvdimm_bus; + struct nvdimm_map *nvdimm_map; + + nvdimm_map = container_of(kref, struct nvdimm_map, kref); + nvdimm_bus = nvdimm_map->nvdimm_bus; + + dev_dbg(&nvdimm_bus->dev, "%s: %pa\n", __func__, &nvdimm_map->offset); + list_del(&nvdimm_map->list); + if (nvdimm_map->flags) + memunmap(nvdimm_map->mem); + else + iounmap(nvdimm_map->iomem); + release_mem_region(nvdimm_map->offset, nvdimm_map->size); + kfree(nvdimm_map); +} + +static void nvdimm_map_put(void *data) +{ + struct nvdimm_map *nvdimm_map = data; + struct nvdimm_bus *nvdimm_bus = nvdimm_map->nvdimm_bus; + + nvdimm_bus_lock(&nvdimm_bus->dev); + kref_put(&nvdimm_map->kref, nvdimm_map_release); + nvdimm_bus_unlock(&nvdimm_bus->dev); +} + +/** + * devm_nvdimm_memremap - map a resource that is shared across regions + * @dev: device that will own a reference to the shared mapping + * @offset: physical base address of the mapping + * @size: mapping size + * @flags: memremap flags, or, if zero, perform an ioremap instead + */ +void *devm_nvdimm_memremap(struct device *dev, resource_size_t offset, + size_t size, unsigned long flags) +{ + struct nvdimm_map *nvdimm_map; + + nvdimm_bus_lock(dev); + nvdimm_map = find_nvdimm_map(dev, offset); + if (!nvdimm_map) + nvdimm_map = alloc_nvdimm_map(dev, offset, size, flags); + else + kref_get(&nvdimm_map->kref); + nvdimm_bus_unlock(dev); + + if (devm_add_action_or_reset(dev, nvdimm_map_put, nvdimm_map)) + return NULL; + + return nvdimm_map->mem; +} +EXPORT_SYMBOL_GPL(devm_nvdimm_memremap); + u64 nd_fletcher64(void *addr, size_t len, bool le) { u32 *buf = addr; @@ -335,6 +457,7 @@ struct nvdimm_bus *__nvdimm_bus_register(struct device *parent, if (!nvdimm_bus) return NULL; INIT_LIST_HEAD(&nvdimm_bus->list); + INIT_LIST_HEAD(&nvdimm_bus->mapping_list); INIT_LIST_HEAD(&nvdimm_bus->poison_list); init_waitqueue_head(&nvdimm_bus->probe_wait); nvdimm_bus->id = ida_simple_get(&nd_ida, 0, 0, GFP_KERNEL); diff --git a/drivers/nvdimm/nd-core.h b/drivers/nvdimm/nd-core.h index 284cdaa268cf..790b62cc81ed 100644 --- a/drivers/nvdimm/nd-core.h +++ b/drivers/nvdimm/nd-core.h @@ -31,6 +31,7 @@ struct nvdimm_bus { struct device dev; int id, probe_active; struct list_head poison_list; + struct list_head mapping_list; struct mutex reconfig_mutex; }; diff --git a/include/linux/libnvdimm.h b/include/linux/libnvdimm.h index 0c3c30cbbea5..18c3cc48a970 100644 --- a/include/linux/libnvdimm.h +++ b/include/linux/libnvdimm.h @@ -99,6 +99,15 @@ struct nd_region_desc { unsigned long flags; }; +struct device; +void *devm_nvdimm_memremap(struct device *dev, resource_size_t offset, + size_t size, unsigned long flags); +static inline void __iomem *devm_nvdimm_ioremap(struct device *dev, + resource_size_t offset, size_t size) +{ + return (void __iomem *) devm_nvdimm_memremap(dev, offset, size, 0); +} + struct nvdimm_bus; struct module; struct device; -- cgit v1.2.3-59-g8ed1b From a8a6d2e04c4ffda055db70814c50bd106e44730f Mon Sep 17 00:00:00 2001 From: Dan Williams Date: Tue, 7 Jun 2016 16:38:04 -0700 Subject: libnvdimm, nfit: remove nfit_spa_map() infrastructure Now that all shared mappings are handled by devm_nvdimm_memremap() we no longer need nfit_spa_map() nor do we need to trigger a callback to the bus provider at region disable time. Signed-off-by: Dan Williams --- drivers/acpi/nfit.c | 146 ------------------------------------------- drivers/acpi/nfit.h | 21 ------- drivers/nvdimm/nd.h | 1 - drivers/nvdimm/region_devs.c | 3 - include/linux/libnvdimm.h | 1 - 5 files changed, 172 deletions(-) (limited to 'drivers/nvdimm') diff --git a/drivers/acpi/nfit.c b/drivers/acpi/nfit.c index b047dbe13bed..b76c95981547 100644 --- a/drivers/acpi/nfit.c +++ b/drivers/acpi/nfit.c @@ -1509,126 +1509,6 @@ static int acpi_nfit_blk_region_do_io(struct nd_blk_region *ndbr, return rc; } -static void nfit_spa_mapping_release(struct kref *kref) -{ - struct nfit_spa_mapping *spa_map = to_spa_map(kref); - struct acpi_nfit_system_address *spa = spa_map->spa; - struct acpi_nfit_desc *acpi_desc = spa_map->acpi_desc; - - WARN_ON(!mutex_is_locked(&acpi_desc->spa_map_mutex)); - dev_dbg(acpi_desc->dev, "%s: SPA%d\n", __func__, spa->range_index); - if (spa_map->type == SPA_MAP_APERTURE) - memunmap((void __force *)spa_map->addr.aperture); - else - iounmap(spa_map->addr.base); - release_mem_region(spa->address, spa->length); - list_del(&spa_map->list); - kfree(spa_map); -} - -static struct nfit_spa_mapping *find_spa_mapping( - struct acpi_nfit_desc *acpi_desc, - struct acpi_nfit_system_address *spa) -{ - struct nfit_spa_mapping *spa_map; - - WARN_ON(!mutex_is_locked(&acpi_desc->spa_map_mutex)); - list_for_each_entry(spa_map, &acpi_desc->spa_maps, list) - if (spa_map->spa == spa) - return spa_map; - - return NULL; -} - -static void nfit_spa_unmap(struct acpi_nfit_desc *acpi_desc, - struct acpi_nfit_system_address *spa) -{ - struct nfit_spa_mapping *spa_map; - - mutex_lock(&acpi_desc->spa_map_mutex); - spa_map = find_spa_mapping(acpi_desc, spa); - - if (spa_map) - kref_put(&spa_map->kref, nfit_spa_mapping_release); - mutex_unlock(&acpi_desc->spa_map_mutex); -} - -static void __iomem *__nfit_spa_map(struct acpi_nfit_desc *acpi_desc, - struct acpi_nfit_system_address *spa, enum spa_map_type type) -{ - resource_size_t start = spa->address; - resource_size_t n = spa->length; - struct nfit_spa_mapping *spa_map; - struct resource *res; - - WARN_ON(!mutex_is_locked(&acpi_desc->spa_map_mutex)); - - spa_map = find_spa_mapping(acpi_desc, spa); - if (spa_map) { - kref_get(&spa_map->kref); - return spa_map->addr.base; - } - - spa_map = kzalloc(sizeof(*spa_map), GFP_KERNEL); - if (!spa_map) - return NULL; - - INIT_LIST_HEAD(&spa_map->list); - spa_map->spa = spa; - kref_init(&spa_map->kref); - spa_map->acpi_desc = acpi_desc; - - res = request_mem_region(start, n, dev_name(acpi_desc->dev)); - if (!res) - goto err_mem; - - spa_map->type = type; - if (type == SPA_MAP_APERTURE) - spa_map->addr.aperture = (void __pmem *)memremap(start, n, - ARCH_MEMREMAP_PMEM); - else - spa_map->addr.base = ioremap_nocache(start, n); - - - if (!spa_map->addr.base) - goto err_map; - - list_add_tail(&spa_map->list, &acpi_desc->spa_maps); - return spa_map->addr.base; - - err_map: - release_mem_region(start, n); - err_mem: - kfree(spa_map); - return NULL; -} - -/** - * nfit_spa_map - interleave-aware managed-mappings of acpi_nfit_system_address ranges - * @nvdimm_bus: NFIT-bus that provided the spa table entry - * @nfit_spa: spa table to map - * @type: aperture or control region - * - * In the case where block-data-window apertures and - * dimm-control-regions are interleaved they will end up sharing a - * single request_mem_region() + ioremap() for the address range. In - * the style of devm nfit_spa_map() mappings are automatically dropped - * when all region devices referencing the same mapping are disabled / - * unbound. - */ -static __maybe_unused void __iomem *nfit_spa_map( - struct acpi_nfit_desc *acpi_desc, - struct acpi_nfit_system_address *spa, enum spa_map_type type) -{ - void __iomem *iomem; - - mutex_lock(&acpi_desc->spa_map_mutex); - iomem = __nfit_spa_map(acpi_desc, spa, type); - mutex_unlock(&acpi_desc->spa_map_mutex); - - return iomem; -} - static int nfit_blk_init_interleave(struct nfit_blk_mmio *mmio, struct acpi_nfit_interleave *idt, u16 interleave_ways) { @@ -1773,29 +1653,6 @@ static int acpi_nfit_blk_region_enable(struct nvdimm_bus *nvdimm_bus, return 0; } -static void acpi_nfit_blk_region_disable(struct nvdimm_bus *nvdimm_bus, - struct device *dev) -{ - struct nvdimm_bus_descriptor *nd_desc = to_nd_desc(nvdimm_bus); - struct acpi_nfit_desc *acpi_desc = to_acpi_desc(nd_desc); - struct nd_blk_region *ndbr = to_nd_blk_region(dev); - struct nfit_blk *nfit_blk = nd_blk_region_provider_data(ndbr); - int i; - - if (!nfit_blk) - return; /* never enabled */ - - /* auto-free BLK spa mappings */ - for (i = 0; i < 2; i++) { - struct nfit_blk_mmio *mmio = &nfit_blk->mmio[i]; - - if (mmio->addr.base) - nfit_spa_unmap(acpi_desc, mmio->spa); - } - nd_blk_region_set_provider_data(ndbr, NULL); - /* devm will free nfit_blk */ -} - static int ars_get_cap(struct acpi_nfit_desc *acpi_desc, struct nd_cmd_ars_cap *cmd, struct nfit_spa *nfit_spa) { @@ -1969,7 +1826,6 @@ static int acpi_nfit_init_mapping(struct acpi_nfit_desc *acpi_desc, ndr_desc->num_mappings = blk_valid; ndbr_desc = to_blk_region_desc(ndr_desc); ndbr_desc->enable = acpi_nfit_blk_region_enable; - ndbr_desc->disable = acpi_nfit_blk_region_disable; ndbr_desc->do_io = acpi_desc->blk_do_io; nfit_spa->nd_region = nvdimm_blk_region_create(acpi_desc->nvdimm_bus, ndr_desc); @@ -2509,7 +2365,6 @@ void acpi_nfit_desc_init(struct acpi_nfit_desc *acpi_desc, struct device *dev) nd_desc->clear_to_send = acpi_nfit_clear_to_send; nd_desc->attr_groups = acpi_nfit_attribute_groups; - INIT_LIST_HEAD(&acpi_desc->spa_maps); INIT_LIST_HEAD(&acpi_desc->spas); INIT_LIST_HEAD(&acpi_desc->dcrs); INIT_LIST_HEAD(&acpi_desc->bdws); @@ -2517,7 +2372,6 @@ void acpi_nfit_desc_init(struct acpi_nfit_desc *acpi_desc, struct device *dev) INIT_LIST_HEAD(&acpi_desc->flushes); INIT_LIST_HEAD(&acpi_desc->memdevs); INIT_LIST_HEAD(&acpi_desc->dimms); - mutex_init(&acpi_desc->spa_map_mutex); mutex_init(&acpi_desc->init_mutex); INIT_WORK(&acpi_desc->work, acpi_nfit_scrub); } diff --git a/drivers/acpi/nfit.h b/drivers/acpi/nfit.h index f06fa91c5abf..52078475d969 100644 --- a/drivers/acpi/nfit.h +++ b/drivers/acpi/nfit.h @@ -135,9 +135,7 @@ struct acpi_nfit_desc { struct nvdimm_bus_descriptor nd_desc; struct acpi_table_header acpi_header; struct acpi_nfit_header *nfit; - struct mutex spa_map_mutex; struct mutex init_mutex; - struct list_head spa_maps; struct list_head memdevs; struct list_head flushes; struct list_head dimms; @@ -188,25 +186,6 @@ struct nfit_blk { u32 dimm_flags; }; -enum spa_map_type { - SPA_MAP_CONTROL, - SPA_MAP_APERTURE, -}; - -struct nfit_spa_mapping { - struct acpi_nfit_desc *acpi_desc; - struct acpi_nfit_system_address *spa; - struct list_head list; - struct kref kref; - enum spa_map_type type; - struct nd_blk_addr addr; -}; - -static inline struct nfit_spa_mapping *to_spa_map(struct kref *kref) -{ - return container_of(kref, struct nfit_spa_mapping, kref); -} - static inline struct acpi_nfit_memory_map *__to_nfit_memdev( struct nfit_mem *nfit_mem) { diff --git a/drivers/nvdimm/nd.h b/drivers/nvdimm/nd.h index d0ac93c31dda..2819e886dfd2 100644 --- a/drivers/nvdimm/nd.h +++ b/drivers/nvdimm/nd.h @@ -119,7 +119,6 @@ struct nd_region { struct nd_blk_region { int (*enable)(struct nvdimm_bus *nvdimm_bus, struct device *dev); - void (*disable)(struct nvdimm_bus *nvdimm_bus, struct device *dev); int (*do_io)(struct nd_blk_region *ndbr, resource_size_t dpa, void *iobuf, u64 len, int rw); void *blk_provider_data; diff --git a/drivers/nvdimm/region_devs.c b/drivers/nvdimm/region_devs.c index 40fcfea26fbb..694b21024871 100644 --- a/drivers/nvdimm/region_devs.c +++ b/drivers/nvdimm/region_devs.c @@ -433,8 +433,6 @@ static void nd_region_notify_driver_action(struct nvdimm_bus *nvdimm_bus, if (is_nd_pmem(dev)) return; - - to_nd_blk_region(dev)->disable(nvdimm_bus, dev); } if (dev->parent && is_nd_blk(dev->parent) && probe) { nd_region = to_nd_region(dev->parent); @@ -698,7 +696,6 @@ static struct nd_region *nd_region_create(struct nvdimm_bus *nvdimm_bus, if (ndbr) { nd_region = &ndbr->nd_region; ndbr->enable = ndbr_desc->enable; - ndbr->disable = ndbr_desc->disable; ndbr->do_io = ndbr_desc->do_io; } region_buf = ndbr; diff --git a/include/linux/libnvdimm.h b/include/linux/libnvdimm.h index 18c3cc48a970..1050f9aa3a3e 100644 --- a/include/linux/libnvdimm.h +++ b/include/linux/libnvdimm.h @@ -114,7 +114,6 @@ struct device; struct nd_blk_region; struct nd_blk_region_desc { int (*enable)(struct nvdimm_bus *nvdimm_bus, struct device *dev); - void (*disable)(struct nvdimm_bus *nvdimm_bus, struct device *dev); int (*do_io)(struct nd_blk_region *ndbr, resource_size_t dpa, void *iobuf, u64 len, int rw); struct nd_region_desc ndr_desc; -- cgit v1.2.3-59-g8ed1b From e5ae3b252c6732f838f5695170bbf2ea9fb5b9ff Mon Sep 17 00:00:00 2001 From: Dan Williams Date: Tue, 7 Jun 2016 17:00:04 -0700 Subject: libnvdimm, nfit: move flush hint mapping to region-device driver-data In preparation for triggering flushes of a DIMM's writes-posted-queue (WPQ) via the pmem driver move mapping of flush hint addresses to the region driver. Since this uses devm_nvdimm_memremap() the flush addresses will remain mapped while any region to which the dimm belongs is active. We need to communicate more information to the nvdimm core to facilitate this mapping, namely each dimm object now carries an array of flush hint address resources. Signed-off-by: Dan Williams --- drivers/acpi/nfit.c | 21 +++++++++++- drivers/acpi/nfit.h | 1 + drivers/nvdimm/dimm_devs.c | 5 ++- drivers/nvdimm/nd-core.h | 3 +- drivers/nvdimm/nd.h | 8 +++-- drivers/nvdimm/region.c | 16 ++++----- drivers/nvdimm/region_devs.c | 79 ++++++++++++++++++++++++++++++++++++++++++-- include/linux/libnvdimm.h | 4 ++- 8 files changed, 119 insertions(+), 18 deletions(-) (limited to 'drivers/nvdimm') diff --git a/drivers/acpi/nfit.c b/drivers/acpi/nfit.c index b76c95981547..6796f780870a 100644 --- a/drivers/acpi/nfit.c +++ b/drivers/acpi/nfit.c @@ -714,9 +714,24 @@ static int nfit_mem_dcr_init(struct acpi_nfit_desc *acpi_desc, } list_for_each_entry(nfit_flush, &acpi_desc->flushes, list) { + struct acpi_nfit_flush_address *flush; + u16 i; + if (nfit_flush->flush->device_handle != device_handle) continue; nfit_mem->nfit_flush = nfit_flush; + flush = nfit_flush->flush; + nfit_mem->flush_wpq = devm_kzalloc(acpi_desc->dev, + flush->hint_count + * sizeof(struct resource), GFP_KERNEL); + if (!nfit_mem->flush_wpq) + return -ENOMEM; + for (i = 0; i < flush->hint_count; i++) { + struct resource *res = &nfit_mem->flush_wpq[i]; + + res->start = flush->hint_address[i]; + res->end = res->start + 8 - 1; + } break; } @@ -1171,6 +1186,7 @@ static int acpi_nfit_register_dimms(struct acpi_nfit_desc *acpi_desc) int dimm_count = 0; list_for_each_entry(nfit_mem, &acpi_desc->dimms, list) { + struct acpi_nfit_flush_address *flush; unsigned long flags = 0, cmd_mask; struct nvdimm *nvdimm; u32 device_handle; @@ -1204,9 +1220,12 @@ static int acpi_nfit_register_dimms(struct acpi_nfit_desc *acpi_desc) if (nfit_mem->family == NVDIMM_FAMILY_INTEL) cmd_mask |= nfit_mem->dsm_mask; + flush = nfit_mem->nfit_flush ? nfit_mem->nfit_flush->flush + : NULL; nvdimm = nvdimm_create(acpi_desc->nvdimm_bus, nfit_mem, acpi_nfit_dimm_attribute_groups, - flags, cmd_mask); + flags, cmd_mask, flush ? flush->hint_count : 0, + nfit_mem->flush_wpq); if (!nvdimm) return -ENOMEM; diff --git a/drivers/acpi/nfit.h b/drivers/acpi/nfit.h index 52078475d969..9282eb324dcc 100644 --- a/drivers/acpi/nfit.h +++ b/drivers/acpi/nfit.h @@ -127,6 +127,7 @@ struct nfit_mem { struct list_head list; struct acpi_device *adev; struct acpi_nfit_desc *acpi_desc; + struct resource *flush_wpq; unsigned long dsm_mask; int family; }; diff --git a/drivers/nvdimm/dimm_devs.c b/drivers/nvdimm/dimm_devs.c index bbde28d3dec5..d9bba5edd8dc 100644 --- a/drivers/nvdimm/dimm_devs.c +++ b/drivers/nvdimm/dimm_devs.c @@ -346,7 +346,8 @@ EXPORT_SYMBOL_GPL(nvdimm_attribute_group); struct nvdimm *nvdimm_create(struct nvdimm_bus *nvdimm_bus, void *provider_data, const struct attribute_group **groups, unsigned long flags, - unsigned long cmd_mask) + unsigned long cmd_mask, int num_flush, + struct resource *flush_wpq) { struct nvdimm *nvdimm = kzalloc(sizeof(*nvdimm), GFP_KERNEL); struct device *dev; @@ -362,6 +363,8 @@ struct nvdimm *nvdimm_create(struct nvdimm_bus *nvdimm_bus, void *provider_data, nvdimm->provider_data = provider_data; nvdimm->flags = flags; nvdimm->cmd_mask = cmd_mask; + nvdimm->num_flush = num_flush; + nvdimm->flush_wpq = flush_wpq; atomic_set(&nvdimm->busy, 0); dev = &nvdimm->dev; dev_set_name(dev, "nmem%d", nvdimm->id); diff --git a/drivers/nvdimm/nd-core.h b/drivers/nvdimm/nd-core.h index 790b62cc81ed..6e961f7f43e7 100644 --- a/drivers/nvdimm/nd-core.h +++ b/drivers/nvdimm/nd-core.h @@ -41,7 +41,8 @@ struct nvdimm { unsigned long cmd_mask; struct device dev; atomic_t busy; - int id; + int id, num_flush; + struct resource *flush_wpq; }; bool is_nvdimm(struct device *dev); diff --git a/drivers/nvdimm/nd.h b/drivers/nvdimm/nd.h index 2819e886dfd2..5912bd6b4234 100644 --- a/drivers/nvdimm/nd.h +++ b/drivers/nvdimm/nd.h @@ -49,9 +49,10 @@ struct nvdimm_drvdata { struct kref kref; }; -struct nd_region_namespaces { - int count; - int active; +struct nd_region_data { + int ns_count; + int ns_active; + void __iomem *flush_wpq[0][0]; }; static inline struct nd_namespace_index *to_namespace_index( @@ -324,6 +325,7 @@ static inline void devm_nsio_disable(struct device *dev, } #endif int nd_blk_region_init(struct nd_region *nd_region); +int nd_region_activate(struct nd_region *nd_region); void __nd_iostat_start(struct bio *bio, unsigned long *start); static inline bool nd_iostat_start(struct bio *bio, unsigned long *start) { diff --git a/drivers/nvdimm/region.c b/drivers/nvdimm/region.c index 05a912359939..333175dac8d5 100644 --- a/drivers/nvdimm/region.c +++ b/drivers/nvdimm/region.c @@ -20,7 +20,7 @@ static int nd_region_probe(struct device *dev) { int err, rc; static unsigned long once; - struct nd_region_namespaces *num_ns; + struct nd_region_data *ndrd; struct nd_region *nd_region = to_nd_region(dev); if (nd_region->num_lanes > num_online_cpus() @@ -33,21 +33,21 @@ static int nd_region_probe(struct device *dev) nd_region->num_lanes); } + rc = nd_region_activate(nd_region); + if (rc) + return rc; + rc = nd_blk_region_init(nd_region); if (rc) return rc; rc = nd_region_register_namespaces(nd_region, &err); - num_ns = devm_kzalloc(dev, sizeof(*num_ns), GFP_KERNEL); - if (!num_ns) - return -ENOMEM; - if (rc < 0) return rc; - num_ns->active = rc; - num_ns->count = rc + err; - dev_set_drvdata(dev, num_ns); + ndrd = dev_get_drvdata(dev); + ndrd->ns_active = rc; + ndrd->ns_count = rc + err; if (rc && err && rc == err) return -ENODEV; diff --git a/drivers/nvdimm/region_devs.c b/drivers/nvdimm/region_devs.c index 694b21024871..67022f74febc 100644 --- a/drivers/nvdimm/region_devs.c +++ b/drivers/nvdimm/region_devs.c @@ -22,6 +22,79 @@ static DEFINE_IDA(region_ida); +static int nvdimm_map_flush(struct device *dev, struct nvdimm *nvdimm, int dimm, + struct nd_region_data *ndrd) +{ + int i, j; + + dev_dbg(dev, "%s: map %d flush address%s\n", nvdimm_name(nvdimm), + nvdimm->num_flush, nvdimm->num_flush == 1 ? "" : "es"); + for (i = 0; i < nvdimm->num_flush; i++) { + struct resource *res = &nvdimm->flush_wpq[i]; + unsigned long pfn = PHYS_PFN(res->start); + void __iomem *flush_page; + + /* check if flush hints share a page */ + for (j = 0; j < i; j++) { + struct resource *res_j = &nvdimm->flush_wpq[j]; + unsigned long pfn_j = PHYS_PFN(res_j->start); + + if (pfn == pfn_j) + break; + } + + if (j < i) + flush_page = (void __iomem *) ((unsigned long) + ndrd->flush_wpq[dimm][j] & PAGE_MASK); + else + flush_page = devm_nvdimm_ioremap(dev, + PHYS_PFN(pfn), PAGE_SIZE); + if (!flush_page) + return -ENXIO; + ndrd->flush_wpq[dimm][i] = flush_page + + (res->start & ~PAGE_MASK); + } + + return 0; +} + +int nd_region_activate(struct nd_region *nd_region) +{ + int i; + struct nd_region_data *ndrd; + struct device *dev = &nd_region->dev; + size_t flush_data_size = sizeof(void *); + + nvdimm_bus_lock(&nd_region->dev); + for (i = 0; i < nd_region->ndr_mappings; i++) { + struct nd_mapping *nd_mapping = &nd_region->mapping[i]; + struct nvdimm *nvdimm = nd_mapping->nvdimm; + + /* at least one null hint slot per-dimm for the "no-hint" case */ + flush_data_size += sizeof(void *); + if (!nvdimm->num_flush) + continue; + flush_data_size += nvdimm->num_flush * sizeof(void *); + } + nvdimm_bus_unlock(&nd_region->dev); + + ndrd = devm_kzalloc(dev, sizeof(*ndrd) + flush_data_size, GFP_KERNEL); + if (!ndrd) + return -ENOMEM; + dev_set_drvdata(dev, ndrd); + + for (i = 0; i < nd_region->ndr_mappings; i++) { + struct nd_mapping *nd_mapping = &nd_region->mapping[i]; + struct nvdimm *nvdimm = nd_mapping->nvdimm; + int rc = nvdimm_map_flush(&nd_region->dev, nvdimm, i, ndrd); + + if (rc) + return rc; + } + + return 0; +} + static void nd_region_release(struct device *dev) { struct nd_region *nd_region = to_nd_region(dev); @@ -242,12 +315,12 @@ static DEVICE_ATTR_RO(available_size); static ssize_t init_namespaces_show(struct device *dev, struct device_attribute *attr, char *buf) { - struct nd_region_namespaces *num_ns = dev_get_drvdata(dev); + struct nd_region_data *ndrd = dev_get_drvdata(dev); ssize_t rc; nvdimm_bus_lock(dev); - if (num_ns) - rc = sprintf(buf, "%d/%d\n", num_ns->active, num_ns->count); + if (ndrd) + rc = sprintf(buf, "%d/%d\n", ndrd->ns_active, ndrd->ns_count); else rc = -ENXIO; nvdimm_bus_unlock(dev); diff --git a/include/linux/libnvdimm.h b/include/linux/libnvdimm.h index 1050f9aa3a3e..815b9b430ead 100644 --- a/include/linux/libnvdimm.h +++ b/include/linux/libnvdimm.h @@ -52,6 +52,7 @@ typedef int (*ndctl_fn)(struct nvdimm_bus_descriptor *nd_desc, struct nd_namespace_label; struct nvdimm_drvdata; + struct nd_mapping { struct nvdimm *nvdimm; struct nd_namespace_label **labels; @@ -142,7 +143,8 @@ unsigned long nvdimm_cmd_mask(struct nvdimm *nvdimm); void *nvdimm_provider_data(struct nvdimm *nvdimm); struct nvdimm *nvdimm_create(struct nvdimm_bus *nvdimm_bus, void *provider_data, const struct attribute_group **groups, unsigned long flags, - unsigned long cmd_mask); + unsigned long cmd_mask, int num_flush, + struct resource *flush_wpq); const struct nd_cmd_desc *nd_cmd_dimm_desc(int cmd); const struct nd_cmd_desc *nd_cmd_bus_desc(int cmd); u32 nd_cmd_in_size(struct nvdimm *nvdimm, int cmd, -- cgit v1.2.3-59-g8ed1b From a8f720224eff09ea684b7f74aa1c0dff3da03170 Mon Sep 17 00:00:00 2001 From: Dan Williams Date: Sat, 9 Jul 2016 16:46:29 -0700 Subject: libnvdimm: keep region data alive over namespace removal nd_region device driver data will be used in the namespace i/o path. Re-order nd_region_remove() to ensure this data stays live across namespace device removal Signed-off-by: Dan Williams --- drivers/nvdimm/region.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'drivers/nvdimm') diff --git a/drivers/nvdimm/region.c b/drivers/nvdimm/region.c index 333175dac8d5..8f241772ec0b 100644 --- a/drivers/nvdimm/region.c +++ b/drivers/nvdimm/region.c @@ -82,6 +82,8 @@ static int nd_region_remove(struct device *dev) { struct nd_region *nd_region = to_nd_region(dev); + device_for_each_child(dev, NULL, child_unregister); + /* flush attribute readers and disable */ nvdimm_bus_lock(dev); nd_region->ns_seed = NULL; @@ -91,7 +93,6 @@ static int nd_region_remove(struct device *dev) dev_set_drvdata(dev, NULL); nvdimm_bus_unlock(dev); - device_for_each_child(dev, NULL, child_unregister); return 0; } -- cgit v1.2.3-59-g8ed1b From f284a4f23752d0334e482d04e0a584d19c9c8cd0 Mon Sep 17 00:00:00 2001 From: Dan Williams Date: Thu, 7 Jul 2016 19:44:50 -0700 Subject: libnvdimm: introduce nvdimm_flush() and nvdimm_has_flush() nvdimm_flush() is a replacement for the x86 'pcommit' instruction. It is an optional write flushing mechanism that an nvdimm bus can provide for the pmem driver to consume. In the case of the NFIT nvdimm-bus-provider nvdimm_flush() is implemented as a series of flush-hint-address [1] writes to each dimm in the interleave set (region) that backs the namespace. The nvdimm_has_flush() routine relies on platform firmware to describe the flushing capabilities of a platform. It uses the heuristic of whether an nvdimm bus provider provides flush address data to return a ternary result: 1: flush addresses defined 0: dimm topology described without flush addresses (assume ADR) -errno: no topology information, unable to determine flush mechanism The pmem driver is expected to take the following actions on this ternary result: 1: nvdimm_flush() in response to REQ_FUA / REQ_FLUSH and shutdown 0: do not set, WC or FUA on the queue, take no further action -errno: warn and then operate as if nvdimm_has_flush() returned '0' The caveat of this heuristic is that it can not distinguish the "dimm does not have flush address" case from the "platform firmware is broken and failed to describe a flush address". Given we are already explicitly trusting the NFIT there's not much more we can do beyond blacklisting broken firmwares if they are ever encountered. Cc: Ross Zwisler Signed-off-by: Dan Williams --- drivers/acpi/nfit.c | 33 +++--------------------- drivers/acpi/nfit.h | 1 - drivers/nvdimm/pmem.c | 27 +++++++++++++++----- drivers/nvdimm/region_devs.c | 61 ++++++++++++++++++++++++++++++++++++++++++++ include/linux/libnvdimm.h | 2 ++ 5 files changed, 87 insertions(+), 37 deletions(-) (limited to 'drivers/nvdimm') diff --git a/drivers/acpi/nfit.c b/drivers/acpi/nfit.c index 6796f780870a..0497175ee6cb 100644 --- a/drivers/acpi/nfit.c +++ b/drivers/acpi/nfit.c @@ -1393,24 +1393,6 @@ static u64 to_interleave_offset(u64 offset, struct nfit_blk_mmio *mmio) return mmio->base_offset + line_offset + table_offset + sub_line_offset; } -static void wmb_blk(struct nfit_blk *nfit_blk) -{ - - if (nfit_blk->nvdimm_flush) { - /* - * The first wmb() is needed to 'sfence' all previous writes - * such that they are architecturally visible for the platform - * buffer flush. Note that we've already arranged for pmem - * writes to avoid the cache via arch_memcpy_to_pmem(). The - * final wmb() ensures ordering for the NVDIMM flush write. - */ - wmb(); - writeq(1, nfit_blk->nvdimm_flush); - wmb(); - } else - wmb_pmem(); -} - static u32 read_blk_stat(struct nfit_blk *nfit_blk, unsigned int bw) { struct nfit_blk_mmio *mmio = &nfit_blk->mmio[DCR]; @@ -1445,7 +1427,7 @@ static void write_blk_ctl(struct nfit_blk *nfit_blk, unsigned int bw, offset = to_interleave_offset(offset, mmio); writeq(cmd, mmio->addr.base + offset); - wmb_blk(nfit_blk); + nvdimm_flush(nfit_blk->nd_region); if (nfit_blk->dimm_flags & NFIT_BLK_DCR_LATCH) readq(mmio->addr.base + offset); @@ -1496,7 +1478,7 @@ static int acpi_nfit_blk_single_io(struct nfit_blk *nfit_blk, } if (rw) - wmb_blk(nfit_blk); + nvdimm_flush(nfit_blk->nd_region); rc = read_blk_stat(nfit_blk, lane) ? -EIO : 0; return rc; @@ -1570,7 +1552,6 @@ static int acpi_nfit_blk_region_enable(struct nvdimm_bus *nvdimm_bus, { struct nvdimm_bus_descriptor *nd_desc = to_nd_desc(nvdimm_bus); struct nd_blk_region *ndbr = to_nd_blk_region(dev); - struct nfit_flush *nfit_flush; struct nfit_blk_mmio *mmio; struct nfit_blk *nfit_blk; struct nfit_mem *nfit_mem; @@ -1645,15 +1626,7 @@ static int acpi_nfit_blk_region_enable(struct nvdimm_bus *nvdimm_bus, return rc; } - nfit_flush = nfit_mem->nfit_flush; - if (nfit_flush && nfit_flush->flush->hint_count != 0) { - nfit_blk->nvdimm_flush = devm_nvdimm_ioremap(dev, - nfit_flush->flush->hint_address[0], 8); - if (!nfit_blk->nvdimm_flush) - return -ENOMEM; - } - - if (!arch_has_wmb_pmem() && !nfit_blk->nvdimm_flush) + if (nvdimm_has_flush(nfit_blk->nd_region) < 0) dev_warn(dev, "unable to guarantee persistence of writes\n"); if (mmio->line_size == 0) diff --git a/drivers/acpi/nfit.h b/drivers/acpi/nfit.h index 9282eb324dcc..9fda77cf81da 100644 --- a/drivers/acpi/nfit.h +++ b/drivers/acpi/nfit.h @@ -183,7 +183,6 @@ struct nfit_blk { u64 bdw_offset; /* post interleave offset */ u64 stat_offset; u64 cmd_offset; - void __iomem *nvdimm_flush; u32 dimm_flags; }; diff --git a/drivers/nvdimm/pmem.c b/drivers/nvdimm/pmem.c index b6fcb97a601c..e303655f243e 100644 --- a/drivers/nvdimm/pmem.c +++ b/drivers/nvdimm/pmem.c @@ -33,10 +33,24 @@ #include "pfn.h" #include "nd.h" +static struct device *to_dev(struct pmem_device *pmem) +{ + /* + * nvdimm bus services need a 'dev' parameter, and we record the device + * at init in bb.dev. + */ + return pmem->bb.dev; +} + +static struct nd_region *to_region(struct pmem_device *pmem) +{ + return to_nd_region(to_dev(pmem)->parent); +} + static void pmem_clear_poison(struct pmem_device *pmem, phys_addr_t offset, unsigned int len) { - struct device *dev = pmem->bb.dev; + struct device *dev = to_dev(pmem); sector_t sector; long cleared; @@ -122,7 +136,7 @@ static blk_qc_t pmem_make_request(struct request_queue *q, struct bio *bio) nd_iostat_end(bio, start); if (bio_data_dir(bio)) - wmb_pmem(); + nvdimm_flush(to_region(pmem)); bio_endio(bio); return BLK_QC_T_NONE; @@ -136,7 +150,7 @@ static int pmem_rw_page(struct block_device *bdev, sector_t sector, rc = pmem_do_bvec(pmem, page, PAGE_SIZE, 0, rw, sector); if (rw & WRITE) - wmb_pmem(); + nvdimm_flush(to_region(pmem)); /* * The ->rw_page interface is subtle and tricky. The core @@ -193,6 +207,7 @@ static int pmem_attach_disk(struct device *dev, struct nd_namespace_common *ndns) { struct nd_namespace_io *nsio = to_nd_namespace_io(&ndns->dev); + struct nd_region *nd_region = to_nd_region(dev->parent); struct vmem_altmap __altmap, *altmap = NULL; struct resource *res = &nsio->res; struct nd_pfn *nd_pfn = NULL; @@ -222,7 +237,7 @@ static int pmem_attach_disk(struct device *dev, dev_set_drvdata(dev, pmem); pmem->phys_addr = res->start; pmem->size = resource_size(res); - if (!arch_has_wmb_pmem()) + if (nvdimm_has_flush(nd_region) < 0) dev_warn(dev, "unable to guarantee persistence of writes\n"); if (!devm_request_mem_region(dev, res->start, resource_size(res), @@ -284,7 +299,7 @@ static int pmem_attach_disk(struct device *dev, / 512); if (devm_init_badblocks(dev, &pmem->bb)) return -ENOMEM; - nvdimm_badblocks_populate(to_nd_region(dev->parent), &pmem->bb, res); + nvdimm_badblocks_populate(nd_region, &pmem->bb, res); disk->bb = &pmem->bb; add_disk(disk); @@ -331,8 +346,8 @@ static int nd_pmem_remove(struct device *dev) static void nd_pmem_notify(struct device *dev, enum nvdimm_event event) { - struct nd_region *nd_region = to_nd_region(dev->parent); struct pmem_device *pmem = dev_get_drvdata(dev); + struct nd_region *nd_region = to_region(pmem); resource_size_t offset = 0, end_trunc = 0; struct nd_namespace_common *ndns; struct nd_namespace_io *nsio; diff --git a/drivers/nvdimm/region_devs.c b/drivers/nvdimm/region_devs.c index 67022f74febc..5d97b127b715 100644 --- a/drivers/nvdimm/region_devs.c +++ b/drivers/nvdimm/region_devs.c @@ -14,12 +14,19 @@ #include #include #include +#include #include #include #include #include "nd-core.h" #include "nd.h" +/* + * For readq() and writeq() on 32-bit builds, the hi-lo, lo-hi order is + * irrelevant. + */ +#include + static DEFINE_IDA(region_ida); static int nvdimm_map_flush(struct device *dev, struct nvdimm *nvdimm, int dimm, @@ -864,6 +871,60 @@ struct nd_region *nvdimm_volatile_region_create(struct nvdimm_bus *nvdimm_bus, } EXPORT_SYMBOL_GPL(nvdimm_volatile_region_create); +/** + * nvdimm_flush - flush any posted write queues between the cpu and pmem media + * @nd_region: blk or interleaved pmem region + */ +void nvdimm_flush(struct nd_region *nd_region) +{ + struct nd_region_data *ndrd = dev_get_drvdata(&nd_region->dev); + int i; + + /* + * The first wmb() is needed to 'sfence' all previous writes + * such that they are architecturally visible for the platform + * buffer flush. Note that we've already arranged for pmem + * writes to avoid the cache via arch_memcpy_to_pmem(). The + * final wmb() ensures ordering for the NVDIMM flush write. + */ + wmb(); + for (i = 0; i < nd_region->ndr_mappings; i++) + if (ndrd->flush_wpq[i][0]) + writeq(1, ndrd->flush_wpq[i][0]); + wmb(); +} +EXPORT_SYMBOL_GPL(nvdimm_flush); + +/** + * nvdimm_has_flush - determine write flushing requirements + * @nd_region: blk or interleaved pmem region + * + * Returns 1 if writes require flushing + * Returns 0 if writes do not require flushing + * Returns -ENXIO if flushing capability can not be determined + */ +int nvdimm_has_flush(struct nd_region *nd_region) +{ + struct nd_region_data *ndrd = dev_get_drvdata(&nd_region->dev); + int i; + + /* no nvdimm == flushing capability unknown */ + if (nd_region->ndr_mappings == 0) + return -ENXIO; + + for (i = 0; i < nd_region->ndr_mappings; i++) + /* flush hints present, flushing required */ + if (ndrd->flush_wpq[i][0]) + return 1; + + /* + * The platform defines dimm devices without hints, assume + * platform persistence mechanism like ADR + */ + return 0; +} +EXPORT_SYMBOL_GPL(nvdimm_has_flush); + void __exit nd_region_devs_exit(void) { ida_destroy(®ion_ida); diff --git a/include/linux/libnvdimm.h b/include/linux/libnvdimm.h index 815b9b430ead..d37fda6dd64c 100644 --- a/include/linux/libnvdimm.h +++ b/include/linux/libnvdimm.h @@ -166,4 +166,6 @@ struct nvdimm *nd_blk_region_to_dimm(struct nd_blk_region *ndbr); unsigned int nd_region_acquire_lane(struct nd_region *nd_region); void nd_region_release_lane(struct nd_region *nd_region, unsigned int lane); u64 nd_fletcher64(void *addr, size_t len, bool le); +void nvdimm_flush(struct nd_region *nd_region); +int nvdimm_has_flush(struct nd_region *nd_region); #endif /* __LIBNVDIMM_H__ */ -- cgit v1.2.3-59-g8ed1b From 0c27af60d1bbd33c7d3dffb46a4c9f6aa103d754 Mon Sep 17 00:00:00 2001 From: Dan Williams Date: Fri, 27 May 2016 09:23:01 -0700 Subject: libnvdimm: cycle flush hints When the NFIT provides multiple flush hint addresses per-dimm it is expressing that the platform is capable of processing multiple flush requests in parallel. There is some fixed cost per flush request, let the cost be shared in parallel on multiple cpus. Since there may not be enough flush hint addresses for each cpu to have one, keep a per-cpu index of the last used hint, hash it with current pid, and assume that access pattern and scheduler randomness will keep the flush-hint usage somewhat staggered across cpus. Cc: Ross Zwisler Signed-off-by: Dan Williams --- drivers/nvdimm/nd.h | 1 + drivers/nvdimm/region_devs.c | 17 ++++++++++++++--- 2 files changed, 15 insertions(+), 3 deletions(-) (limited to 'drivers/nvdimm') diff --git a/drivers/nvdimm/nd.h b/drivers/nvdimm/nd.h index 5912bd6b4234..40476399d227 100644 --- a/drivers/nvdimm/nd.h +++ b/drivers/nvdimm/nd.h @@ -52,6 +52,7 @@ struct nvdimm_drvdata { struct nd_region_data { int ns_count; int ns_active; + unsigned int flush_mask; void __iomem *flush_wpq[0][0]; }; diff --git a/drivers/nvdimm/region_devs.c b/drivers/nvdimm/region_devs.c index 5d97b127b715..e8d5ba7b29af 100644 --- a/drivers/nvdimm/region_devs.c +++ b/drivers/nvdimm/region_devs.c @@ -14,6 +14,7 @@ #include #include #include +#include #include #include #include @@ -28,6 +29,7 @@ #include static DEFINE_IDA(region_ida); +static DEFINE_PER_CPU(int, flush_idx); static int nvdimm_map_flush(struct device *dev, struct nvdimm *nvdimm, int dimm, struct nd_region_data *ndrd) @@ -67,7 +69,7 @@ static int nvdimm_map_flush(struct device *dev, struct nvdimm *nvdimm, int dimm, int nd_region_activate(struct nd_region *nd_region) { - int i; + int i, num_flush = 0; struct nd_region_data *ndrd; struct device *dev = &nd_region->dev; size_t flush_data_size = sizeof(void *); @@ -79,6 +81,7 @@ int nd_region_activate(struct nd_region *nd_region) /* at least one null hint slot per-dimm for the "no-hint" case */ flush_data_size += sizeof(void *); + num_flush = min_not_zero(num_flush, nvdimm->num_flush); if (!nvdimm->num_flush) continue; flush_data_size += nvdimm->num_flush * sizeof(void *); @@ -90,6 +93,7 @@ int nd_region_activate(struct nd_region *nd_region) return -ENOMEM; dev_set_drvdata(dev, ndrd); + ndrd->flush_mask = (1 << ilog2(num_flush)) - 1; for (i = 0; i < nd_region->ndr_mappings; i++) { struct nd_mapping *nd_mapping = &nd_region->mapping[i]; struct nvdimm *nvdimm = nd_mapping->nvdimm; @@ -878,7 +882,14 @@ EXPORT_SYMBOL_GPL(nvdimm_volatile_region_create); void nvdimm_flush(struct nd_region *nd_region) { struct nd_region_data *ndrd = dev_get_drvdata(&nd_region->dev); - int i; + int i, idx; + + /* + * Try to encourage some diversity in flush hint addresses + * across cpus assuming a limited number of flush hints. + */ + idx = this_cpu_read(flush_idx); + idx = this_cpu_add_return(flush_idx, hash_32(current->pid + idx, 8)); /* * The first wmb() is needed to 'sfence' all previous writes @@ -890,7 +901,7 @@ void nvdimm_flush(struct nd_region *nd_region) wmb(); for (i = 0; i < nd_region->ndr_mappings; i++) if (ndrd->flush_wpq[i][0]) - writeq(1, ndrd->flush_wpq[i][0]); + writeq(1, ndrd->flush_wpq[i][idx & ndrd->flush_mask]); wmb(); } EXPORT_SYMBOL_GPL(nvdimm_flush); -- cgit v1.2.3-59-g8ed1b From 7e267a8c790edfde9b697cbe944ee566f41219c3 Mon Sep 17 00:00:00 2001 From: Dan Williams Date: Wed, 1 Jun 2016 20:48:15 -0700 Subject: libnvdimm, pmem: use REQ_FUA, REQ_FLUSH for nvdimm_flush() Given that nvdimm_flush() has higher overhead than wmb_pmem() (pointer chasing through nd_region), and that we otherwise assume a platform has ADR capability when flush hints are not present, move nvdimm_flush() to REQ_FLUSH context. Note that we still arrange for nvdimm_flush() to be called even in the ADR case. We need at least once wmb() fence to push buffered writes in the cpu out to the ADR protected domain. Cc: Toshi Kani Cc: Ross Zwisler Signed-off-by: Dan Williams --- drivers/nvdimm/pmem.c | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) (limited to 'drivers/nvdimm') diff --git a/drivers/nvdimm/pmem.c b/drivers/nvdimm/pmem.c index e303655f243e..9d9c1beef020 100644 --- a/drivers/nvdimm/pmem.c +++ b/drivers/nvdimm/pmem.c @@ -113,6 +113,11 @@ static int pmem_do_bvec(struct pmem_device *pmem, struct page *page, return rc; } +/* account for REQ_FLUSH rename, replace with REQ_PREFLUSH after v4.8-rc1 */ +#ifndef REQ_FLUSH +#define REQ_FLUSH REQ_PREFLUSH +#endif + static blk_qc_t pmem_make_request(struct request_queue *q, struct bio *bio) { int rc = 0; @@ -121,6 +126,10 @@ static blk_qc_t pmem_make_request(struct request_queue *q, struct bio *bio) struct bio_vec bvec; struct bvec_iter iter; struct pmem_device *pmem = q->queuedata; + struct nd_region *nd_region = to_region(pmem); + + if (bio->bi_rw & REQ_FLUSH) + nvdimm_flush(nd_region); do_acct = nd_iostat_start(bio, &start); bio_for_each_segment(bvec, bio, iter) { @@ -135,8 +144,8 @@ static blk_qc_t pmem_make_request(struct request_queue *q, struct bio *bio) if (do_acct) nd_iostat_end(bio, start); - if (bio_data_dir(bio)) - nvdimm_flush(to_region(pmem)); + if (bio->bi_rw & REQ_FUA) + nvdimm_flush(nd_region); bio_endio(bio); return BLK_QC_T_NONE; @@ -149,8 +158,6 @@ static int pmem_rw_page(struct block_device *bdev, sector_t sector, int rc; rc = pmem_do_bvec(pmem, page, PAGE_SIZE, 0, rw, sector); - if (rw & WRITE) - nvdimm_flush(to_region(pmem)); /* * The ->rw_page interface is subtle and tricky. The core @@ -279,6 +286,7 @@ static int pmem_attach_disk(struct device *dev, return PTR_ERR(addr); pmem->virt_addr = (void __pmem *) addr; + blk_queue_write_cache(q, true, true); blk_queue_make_request(q, pmem_make_request); blk_queue_physical_block_size(q, PAGE_SIZE); blk_queue_max_hw_sectors(q, UINT_MAX); -- cgit v1.2.3-59-g8ed1b From 476f848aaee466fd5d74f123fa652e757f2baeba Mon Sep 17 00:00:00 2001 From: Dan Williams Date: Sat, 9 Jul 2016 00:12:52 -0700 Subject: libnvdimm, pmem: flush posted-write queues on shutdown Commit writes to media on system shutdown or pmem driver unload. Signed-off-by: Dan Williams --- drivers/nvdimm/bus.c | 16 ++++++++++++++++ drivers/nvdimm/pmem.c | 8 ++++++++ include/linux/nd.h | 1 + 3 files changed, 25 insertions(+) (limited to 'drivers/nvdimm') diff --git a/drivers/nvdimm/bus.c b/drivers/nvdimm/bus.c index e4882e63bece..1cc7880320fe 100644 --- a/drivers/nvdimm/bus.c +++ b/drivers/nvdimm/bus.c @@ -136,6 +136,21 @@ static int nvdimm_bus_remove(struct device *dev) return rc; } +static void nvdimm_bus_shutdown(struct device *dev) +{ + struct nvdimm_bus *nvdimm_bus = walk_to_nvdimm_bus(dev); + struct nd_device_driver *nd_drv = NULL; + + if (dev->driver) + nd_drv = to_nd_device_driver(dev->driver); + + if (nd_drv && nd_drv->shutdown) { + nd_drv->shutdown(dev); + dev_dbg(&nvdimm_bus->dev, "%s.shutdown(%s)\n", + dev->driver->name, dev_name(dev)); + } +} + void nd_device_notify(struct device *dev, enum nvdimm_event event) { device_lock(dev); @@ -214,6 +229,7 @@ static struct bus_type nvdimm_bus_type = { .match = nvdimm_bus_match, .probe = nvdimm_bus_probe, .remove = nvdimm_bus_remove, + .shutdown = nvdimm_bus_shutdown, }; static ASYNC_DOMAIN_EXCLUSIVE(nd_async_domain); diff --git a/drivers/nvdimm/pmem.c b/drivers/nvdimm/pmem.c index 9d9c1beef020..8bfc6acc2e43 100644 --- a/drivers/nvdimm/pmem.c +++ b/drivers/nvdimm/pmem.c @@ -349,9 +349,16 @@ static int nd_pmem_remove(struct device *dev) { if (is_nd_btt(dev)) nvdimm_namespace_detach_btt(to_nd_btt(dev)); + nvdimm_flush(to_nd_region(dev->parent)); + return 0; } +static void nd_pmem_shutdown(struct device *dev) +{ + nvdimm_flush(to_nd_region(dev->parent)); +} + static void nd_pmem_notify(struct device *dev, enum nvdimm_event event) { struct pmem_device *pmem = dev_get_drvdata(dev); @@ -391,6 +398,7 @@ static struct nd_device_driver nd_pmem_driver = { .probe = nd_pmem_probe, .remove = nd_pmem_remove, .notify = nd_pmem_notify, + .shutdown = nd_pmem_shutdown, .drv = { .name = "nd_pmem", }, diff --git a/include/linux/nd.h b/include/linux/nd.h index aee2761d294c..1ecd64643512 100644 --- a/include/linux/nd.h +++ b/include/linux/nd.h @@ -26,6 +26,7 @@ struct nd_device_driver { unsigned long type; int (*probe)(struct device *dev); int (*remove)(struct device *dev); + void (*shutdown)(struct device *dev); void (*notify)(struct device *dev, enum nvdimm_event event); }; -- cgit v1.2.3-59-g8ed1b From 91131dbd1d50637dc338526502a1a2ec5a7f97df Mon Sep 17 00:00:00 2001 From: Dan Williams Date: Wed, 1 Jun 2016 23:14:22 -0700 Subject: libnvdimm, pmem: use nvdimm_flush() for namespace I/O writes nsio_rw_bytes() is used to write info block metadata to the namespace, so it should trigger a flush after every write. Replace wmb_pmem() with nvdimm_flush() in this path. Cc: Ross Zwisler Signed-off-by: Dan Williams --- drivers/nvdimm/claim.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'drivers/nvdimm') diff --git a/drivers/nvdimm/claim.c b/drivers/nvdimm/claim.c index 9997ff94a132..d5dc80c48b4c 100644 --- a/drivers/nvdimm/claim.c +++ b/drivers/nvdimm/claim.c @@ -240,7 +240,7 @@ static int nsio_rw_bytes(struct nd_namespace_common *ndns, return memcpy_from_pmem(buf, nsio->addr + offset, size); } else { memcpy_to_pmem(nsio->addr + offset, buf, size); - wmb_pmem(); + nvdimm_flush(to_nd_region(ndns->dev.parent)); } return 0; -- cgit v1.2.3-59-g8ed1b From 7a9eb20666317794d0279843fbd091af93907780 Mon Sep 17 00:00:00 2001 From: Dan Williams Date: Fri, 3 Jun 2016 18:06:47 -0700 Subject: pmem: kill __pmem address space The __pmem address space was meant to annotate codepaths that touch persistent memory and need to coordinate a call to wmb_pmem(). Now that wmb_pmem() is gone, there is little need to keep this annotation. Cc: Christoph Hellwig Cc: Ross Zwisler Signed-off-by: Dan Williams --- Documentation/filesystems/Locking | 2 +- arch/powerpc/sysdev/axonram.c | 4 +-- arch/x86/include/asm/pmem.h | 41 ++++++++++------------- drivers/acpi/nfit.h | 2 +- drivers/block/brd.c | 4 +-- drivers/nvdimm/pmem.c | 6 ++-- drivers/nvdimm/pmem.h | 4 +-- drivers/s390/block/dcssblk.c | 6 ++-- fs/dax.c | 6 ++-- include/linux/blkdev.h | 6 ++-- include/linux/compiler.h | 2 -- include/linux/nd.h | 2 +- include/linux/pmem.h | 70 ++++++++++----------------------------- scripts/checkpatch.pl | 1 - tools/testing/nvdimm/pmem-dax.c | 2 +- 15 files changed, 56 insertions(+), 102 deletions(-) (limited to 'drivers/nvdimm') diff --git a/Documentation/filesystems/Locking b/Documentation/filesystems/Locking index 75eea7ce3d7c..d9c37ec4c760 100644 --- a/Documentation/filesystems/Locking +++ b/Documentation/filesystems/Locking @@ -395,7 +395,7 @@ prototypes: int (*release) (struct gendisk *, fmode_t); int (*ioctl) (struct block_device *, fmode_t, unsigned, unsigned long); int (*compat_ioctl) (struct block_device *, fmode_t, unsigned, unsigned long); - int (*direct_access) (struct block_device *, sector_t, void __pmem **, + int (*direct_access) (struct block_device *, sector_t, void **, unsigned long *); int (*media_changed) (struct gendisk *); void (*unlock_native_capacity) (struct gendisk *); diff --git a/arch/powerpc/sysdev/axonram.c b/arch/powerpc/sysdev/axonram.c index ff75d70f7285..a87489d007dc 100644 --- a/arch/powerpc/sysdev/axonram.c +++ b/arch/powerpc/sysdev/axonram.c @@ -143,12 +143,12 @@ axon_ram_make_request(struct request_queue *queue, struct bio *bio) */ static long axon_ram_direct_access(struct block_device *device, sector_t sector, - void __pmem **kaddr, pfn_t *pfn, long size) + void **kaddr, pfn_t *pfn, long size) { struct axon_ram_bank *bank = device->bd_disk->private_data; loff_t offset = (loff_t)sector << AXON_RAM_SECTOR_SHIFT; - *kaddr = (void __pmem __force *) bank->io_addr + offset; + *kaddr = (void *) bank->io_addr + offset; *pfn = phys_to_pfn_t(bank->ph_addr + offset, PFN_DEV); return bank->size - offset; } diff --git a/arch/x86/include/asm/pmem.h b/arch/x86/include/asm/pmem.h index a8cf2a6b14d9..643eba42d620 100644 --- a/arch/x86/include/asm/pmem.h +++ b/arch/x86/include/asm/pmem.h @@ -28,10 +28,9 @@ * Copy data to persistent memory media via non-temporal stores so that * a subsequent pmem driver flush operation will drain posted write queues. */ -static inline void arch_memcpy_to_pmem(void __pmem *dst, const void *src, - size_t n) +static inline void arch_memcpy_to_pmem(void *dst, const void *src, size_t n) { - int unwritten; + int rem; /* * We are copying between two kernel buffers, if @@ -39,19 +38,17 @@ static inline void arch_memcpy_to_pmem(void __pmem *dst, const void *src, * fault) we would have already reported a general protection fault * before the WARN+BUG. */ - unwritten = __copy_from_user_inatomic_nocache((void __force *) dst, - (void __user *) src, n); - if (WARN(unwritten, "%s: fault copying %p <- %p unwritten: %d\n", - __func__, dst, src, unwritten)) + rem = __copy_from_user_inatomic_nocache(dst, (void __user *) src, n); + if (WARN(rem, "%s: fault copying %p <- %p unwritten: %d\n", + __func__, dst, src, rem)) BUG(); } -static inline int arch_memcpy_from_pmem(void *dst, const void __pmem *src, - size_t n) +static inline int arch_memcpy_from_pmem(void *dst, const void *src, size_t n) { if (static_cpu_has(X86_FEATURE_MCE_RECOVERY)) - return memcpy_mcsafe(dst, (void __force *) src, n); - memcpy(dst, (void __force *) src, n); + return memcpy_mcsafe(dst, src, n); + memcpy(dst, src, n); return 0; } @@ -63,15 +60,14 @@ static inline int arch_memcpy_from_pmem(void *dst, const void __pmem *src, * Write back a cache range using the CLWB (cache line write back) * instruction. */ -static inline void arch_wb_cache_pmem(void __pmem *addr, size_t size) +static inline void arch_wb_cache_pmem(void *addr, size_t size) { u16 x86_clflush_size = boot_cpu_data.x86_clflush_size; unsigned long clflush_mask = x86_clflush_size - 1; - void *vaddr = (void __force *)addr; - void *vend = vaddr + size; + void *vend = addr + size; void *p; - for (p = (void *)((unsigned long)vaddr & ~clflush_mask); + for (p = (void *)((unsigned long)addr & ~clflush_mask); p < vend; p += x86_clflush_size) clwb(p); } @@ -93,14 +89,13 @@ static inline bool __iter_needs_pmem_wb(struct iov_iter *i) * * Copy data from the iterator 'i' to the PMEM buffer starting at 'addr'. */ -static inline size_t arch_copy_from_iter_pmem(void __pmem *addr, size_t bytes, +static inline size_t arch_copy_from_iter_pmem(void *addr, size_t bytes, struct iov_iter *i) { - void *vaddr = (void __force *)addr; size_t len; /* TODO: skip the write-back by always using non-temporal stores */ - len = copy_from_iter_nocache(vaddr, bytes, i); + len = copy_from_iter_nocache(addr, bytes, i); if (__iter_needs_pmem_wb(i)) arch_wb_cache_pmem(addr, bytes); @@ -115,17 +110,15 @@ static inline size_t arch_copy_from_iter_pmem(void __pmem *addr, size_t bytes, * * Write zeros into the memory range starting at 'addr' for 'size' bytes. */ -static inline void arch_clear_pmem(void __pmem *addr, size_t size) +static inline void arch_clear_pmem(void *addr, size_t size) { - void *vaddr = (void __force *)addr; - - memset(vaddr, 0, size); + memset(addr, 0, size); arch_wb_cache_pmem(addr, size); } -static inline void arch_invalidate_pmem(void __pmem *addr, size_t size) +static inline void arch_invalidate_pmem(void *addr, size_t size) { - clflush_cache_range((void __force *) addr, size); + clflush_cache_range(addr, size); } #endif /* CONFIG_ARCH_HAS_PMEM_API */ #endif /* __ASM_X86_PMEM_H__ */ diff --git a/drivers/acpi/nfit.h b/drivers/acpi/nfit.h index 9fda77cf81da..80fb2c0ac8bf 100644 --- a/drivers/acpi/nfit.h +++ b/drivers/acpi/nfit.h @@ -164,7 +164,7 @@ enum nd_blk_mmio_selector { struct nd_blk_addr { union { void __iomem *base; - void __pmem *aperture; + void *aperture; }; }; diff --git a/drivers/block/brd.c b/drivers/block/brd.c index c04bd9bc39fd..5f1fe4e6208d 100644 --- a/drivers/block/brd.c +++ b/drivers/block/brd.c @@ -381,7 +381,7 @@ static int brd_rw_page(struct block_device *bdev, sector_t sector, #ifdef CONFIG_BLK_DEV_RAM_DAX static long brd_direct_access(struct block_device *bdev, sector_t sector, - void __pmem **kaddr, pfn_t *pfn, long size) + void **kaddr, pfn_t *pfn, long size) { struct brd_device *brd = bdev->bd_disk->private_data; struct page *page; @@ -391,7 +391,7 @@ static long brd_direct_access(struct block_device *bdev, sector_t sector, page = brd_insert_page(brd, sector); if (!page) return -ENOSPC; - *kaddr = (void __pmem *)page_address(page); + *kaddr = page_address(page); *pfn = page_to_pfn_t(page); return PAGE_SIZE; diff --git a/drivers/nvdimm/pmem.c b/drivers/nvdimm/pmem.c index 8bfc6acc2e43..7251b4b6da84 100644 --- a/drivers/nvdimm/pmem.c +++ b/drivers/nvdimm/pmem.c @@ -74,7 +74,7 @@ static int pmem_do_bvec(struct pmem_device *pmem, struct page *page, bool bad_pmem = false; void *mem = kmap_atomic(page); phys_addr_t pmem_off = sector * 512 + pmem->data_offset; - void __pmem *pmem_addr = pmem->virt_addr + pmem_off; + void *pmem_addr = pmem->virt_addr + pmem_off; if (unlikely(is_bad_pmem(&pmem->bb, sector, len))) bad_pmem = true; @@ -173,7 +173,7 @@ static int pmem_rw_page(struct block_device *bdev, sector_t sector, /* see "strong" declaration in tools/testing/nvdimm/pmem-dax.c */ __weak long pmem_direct_access(struct block_device *bdev, sector_t sector, - void __pmem **kaddr, pfn_t *pfn, long size) + void **kaddr, pfn_t *pfn, long size) { struct pmem_device *pmem = bdev->bd_queue->queuedata; resource_size_t offset = sector * 512 + pmem->data_offset; @@ -284,7 +284,7 @@ static int pmem_attach_disk(struct device *dev, if (IS_ERR(addr)) return PTR_ERR(addr); - pmem->virt_addr = (void __pmem *) addr; + pmem->virt_addr = addr; blk_queue_write_cache(q, true, true); blk_queue_make_request(q, pmem_make_request); diff --git a/drivers/nvdimm/pmem.h b/drivers/nvdimm/pmem.h index c48d4e3aa346..b4ee4f71b4a1 100644 --- a/drivers/nvdimm/pmem.h +++ b/drivers/nvdimm/pmem.h @@ -6,7 +6,7 @@ #include long pmem_direct_access(struct block_device *bdev, sector_t sector, - void __pmem **kaddr, pfn_t *pfn, long size); + void **kaddr, pfn_t *pfn, long size); /* this definition is in it's own header for tools/testing/nvdimm to consume */ struct pmem_device { /* One contiguous memory region per device */ @@ -14,7 +14,7 @@ struct pmem_device { /* when non-zero this device is hosting a 'pfn' instance */ phys_addr_t data_offset; u64 pfn_flags; - void __pmem *virt_addr; + void *virt_addr; /* immutable base size of the namespace */ size_t size; /* trim size when namespace capacity has been section aligned */ diff --git a/drivers/s390/block/dcssblk.c b/drivers/s390/block/dcssblk.c index bed53c46dd90..023c5c975dc0 100644 --- a/drivers/s390/block/dcssblk.c +++ b/drivers/s390/block/dcssblk.c @@ -31,7 +31,7 @@ static void dcssblk_release(struct gendisk *disk, fmode_t mode); static blk_qc_t dcssblk_make_request(struct request_queue *q, struct bio *bio); static long dcssblk_direct_access(struct block_device *bdev, sector_t secnum, - void __pmem **kaddr, pfn_t *pfn, long size); + void **kaddr, pfn_t *pfn, long size); static char dcssblk_segments[DCSSBLK_PARM_LEN] = "\0"; @@ -884,7 +884,7 @@ fail: static long dcssblk_direct_access (struct block_device *bdev, sector_t secnum, - void __pmem **kaddr, pfn_t *pfn, long size) + void **kaddr, pfn_t *pfn, long size) { struct dcssblk_dev_info *dev_info; unsigned long offset, dev_sz; @@ -894,7 +894,7 @@ dcssblk_direct_access (struct block_device *bdev, sector_t secnum, return -ENODEV; dev_sz = dev_info->end - dev_info->start; offset = secnum * 512; - *kaddr = (void __pmem *) (dev_info->start + offset); + *kaddr = (void *) dev_info->start + offset; *pfn = __pfn_to_pfn_t(PFN_DOWN(dev_info->start + offset), PFN_DEV); return dev_sz - offset; diff --git a/fs/dax.c b/fs/dax.c index 434f421da660..c8312f6441bc 100644 --- a/fs/dax.c +++ b/fs/dax.c @@ -75,13 +75,13 @@ static long dax_map_atomic(struct block_device *bdev, struct blk_dax_ctl *dax) struct request_queue *q = bdev->bd_queue; long rc = -EIO; - dax->addr = (void __pmem *) ERR_PTR(-EIO); + dax->addr = ERR_PTR(-EIO); if (blk_queue_enter(q, true) != 0) return rc; rc = bdev_direct_access(bdev, dax); if (rc < 0) { - dax->addr = (void __pmem *) ERR_PTR(rc); + dax->addr = ERR_PTR(rc); blk_queue_exit(q); return rc; } @@ -152,7 +152,7 @@ static ssize_t dax_io(struct inode *inode, struct iov_iter *iter, int rw = iov_iter_rw(iter), rc; long map_len = 0; struct blk_dax_ctl dax = { - .addr = (void __pmem *) ERR_PTR(-EIO), + .addr = ERR_PTR(-EIO), }; unsigned blkbits = inode->i_blkbits; sector_t file_blks = (i_size_read(inode) + (1 << blkbits) - 1) diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h index 3d9cf326574f..fde908b2836b 100644 --- a/include/linux/blkdev.h +++ b/include/linux/blkdev.h @@ -1659,7 +1659,7 @@ static inline bool integrity_req_gap_front_merge(struct request *req, */ struct blk_dax_ctl { sector_t sector; - void __pmem *addr; + void *addr; long size; pfn_t pfn; }; @@ -1670,8 +1670,8 @@ struct block_device_operations { int (*rw_page)(struct block_device *, sector_t, struct page *, int rw); int (*ioctl) (struct block_device *, fmode_t, unsigned, unsigned long); int (*compat_ioctl) (struct block_device *, fmode_t, unsigned, unsigned long); - long (*direct_access)(struct block_device *, sector_t, void __pmem **, - pfn_t *, long); + long (*direct_access)(struct block_device *, sector_t, void **, pfn_t *, + long); unsigned int (*check_events) (struct gendisk *disk, unsigned int clearing); /* ->media_changed() is DEPRECATED, use ->check_events() instead */ diff --git a/include/linux/compiler.h b/include/linux/compiler.h index 793c0829e3a3..b966974938ed 100644 --- a/include/linux/compiler.h +++ b/include/linux/compiler.h @@ -17,7 +17,6 @@ # define __release(x) __context__(x,-1) # define __cond_lock(x,c) ((c) ? ({ __acquire(x); 1; }) : 0) # define __percpu __attribute__((noderef, address_space(3))) -# define __pmem __attribute__((noderef, address_space(5))) #ifdef CONFIG_SPARSE_RCU_POINTER # define __rcu __attribute__((noderef, address_space(4))) #else /* CONFIG_SPARSE_RCU_POINTER */ @@ -45,7 +44,6 @@ extern void __chk_io_ptr(const volatile void __iomem *); # define __cond_lock(x,c) (c) # define __percpu # define __rcu -# define __pmem # define __private # define ACCESS_PRIVATE(p, member) ((p)->member) #endif /* __CHECKER__ */ diff --git a/include/linux/nd.h b/include/linux/nd.h index 1ecd64643512..f1ea426d6a5e 100644 --- a/include/linux/nd.h +++ b/include/linux/nd.h @@ -68,7 +68,7 @@ struct nd_namespace_io { struct nd_namespace_common common; struct resource res; resource_size_t size; - void __pmem *addr; + void *addr; struct badblocks bb; }; diff --git a/include/linux/pmem.h b/include/linux/pmem.h index 9e3ea94b8157..e856c2cb0fe8 100644 --- a/include/linux/pmem.h +++ b/include/linux/pmem.h @@ -26,37 +26,35 @@ * calling these symbols with arch_has_pmem_api() and redirect to the * implementation in asm/pmem.h. */ -static inline void arch_memcpy_to_pmem(void __pmem *dst, const void *src, - size_t n) +static inline void arch_memcpy_to_pmem(void *dst, const void *src, size_t n) { BUG(); } -static inline int arch_memcpy_from_pmem(void *dst, const void __pmem *src, - size_t n) +static inline int arch_memcpy_from_pmem(void *dst, const void *src, size_t n) { BUG(); return -EFAULT; } -static inline size_t arch_copy_from_iter_pmem(void __pmem *addr, size_t bytes, +static inline size_t arch_copy_from_iter_pmem(void *addr, size_t bytes, struct iov_iter *i) { BUG(); return 0; } -static inline void arch_clear_pmem(void __pmem *addr, size_t size) +static inline void arch_clear_pmem(void *addr, size_t size) { BUG(); } -static inline void arch_wb_cache_pmem(void __pmem *addr, size_t size) +static inline void arch_wb_cache_pmem(void *addr, size_t size) { BUG(); } -static inline void arch_invalidate_pmem(void __pmem *addr, size_t size) +static inline void arch_invalidate_pmem(void *addr, size_t size) { BUG(); } @@ -67,13 +65,6 @@ static inline bool arch_has_pmem_api(void) return IS_ENABLED(CONFIG_ARCH_HAS_PMEM_API); } -static inline int default_memcpy_from_pmem(void *dst, void __pmem const *src, - size_t size) -{ - memcpy(dst, (void __force *) src, size); - return 0; -} - /* * memcpy_from_pmem - read from persistent memory with error handling * @dst: destination buffer @@ -82,40 +73,13 @@ static inline int default_memcpy_from_pmem(void *dst, void __pmem const *src, * * Returns 0 on success negative error code on failure. */ -static inline int memcpy_from_pmem(void *dst, void __pmem const *src, - size_t size) +static inline int memcpy_from_pmem(void *dst, void const *src, size_t size) { if (arch_has_pmem_api()) return arch_memcpy_from_pmem(dst, src, size); else - return default_memcpy_from_pmem(dst, src, size); -} - -/* - * These defaults seek to offer decent performance and minimize the - * window between i/o completion and writes being durable on media. - * However, it is undefined / architecture specific whether - * ARCH_MEMREMAP_PMEM + default_memcpy_to_pmem is sufficient for - * making data durable relative to i/o completion. - */ -static inline void default_memcpy_to_pmem(void __pmem *dst, const void *src, - size_t size) -{ - memcpy((void __force *) dst, src, size); -} - -static inline size_t default_copy_from_iter_pmem(void __pmem *addr, - size_t bytes, struct iov_iter *i) -{ - return copy_from_iter_nocache((void __force *)addr, bytes, i); -} - -static inline void default_clear_pmem(void __pmem *addr, size_t size) -{ - if (size == PAGE_SIZE && ((unsigned long)addr & ~PAGE_MASK) == 0) - clear_page((void __force *)addr); - else - memset((void __force *)addr, 0, size); + memcpy(dst, src, size); + return 0; } /** @@ -130,12 +94,12 @@ static inline void default_clear_pmem(void __pmem *addr, size_t size) * data may still reside in cpu or platform buffers, so this operation * must be followed by a blkdev_issue_flush() on the pmem block device. */ -static inline void memcpy_to_pmem(void __pmem *dst, const void *src, size_t n) +static inline void memcpy_to_pmem(void *dst, const void *src, size_t n) { if (arch_has_pmem_api()) arch_memcpy_to_pmem(dst, src, n); else - default_memcpy_to_pmem(dst, src, n); + memcpy(dst, src, n); } /** @@ -147,12 +111,12 @@ static inline void memcpy_to_pmem(void __pmem *dst, const void *src, size_t n) * Copy data from the iterator 'i' to the PMEM buffer starting at 'addr'. * See blkdev_issue_flush() note for memcpy_to_pmem(). */ -static inline size_t copy_from_iter_pmem(void __pmem *addr, size_t bytes, +static inline size_t copy_from_iter_pmem(void *addr, size_t bytes, struct iov_iter *i) { if (arch_has_pmem_api()) return arch_copy_from_iter_pmem(addr, bytes, i); - return default_copy_from_iter_pmem(addr, bytes, i); + return copy_from_iter_nocache(addr, bytes, i); } /** @@ -163,12 +127,12 @@ static inline size_t copy_from_iter_pmem(void __pmem *addr, size_t bytes, * Write zeros into the memory range starting at 'addr' for 'size' bytes. * See blkdev_issue_flush() note for memcpy_to_pmem(). */ -static inline void clear_pmem(void __pmem *addr, size_t size) +static inline void clear_pmem(void *addr, size_t size) { if (arch_has_pmem_api()) arch_clear_pmem(addr, size); else - default_clear_pmem(addr, size); + memset(addr, 0, size); } /** @@ -179,7 +143,7 @@ static inline void clear_pmem(void __pmem *addr, size_t size) * For platforms that support clearing poison this flushes any poisoned * ranges out of the cache */ -static inline void invalidate_pmem(void __pmem *addr, size_t size) +static inline void invalidate_pmem(void *addr, size_t size) { if (arch_has_pmem_api()) arch_invalidate_pmem(addr, size); @@ -193,7 +157,7 @@ static inline void invalidate_pmem(void __pmem *addr, size_t size) * Write back the processor cache range starting at 'addr' for 'size' bytes. * See blkdev_issue_flush() note for memcpy_to_pmem(). */ -static inline void wb_cache_pmem(void __pmem *addr, size_t size) +static inline void wb_cache_pmem(void *addr, size_t size) { if (arch_has_pmem_api()) arch_wb_cache_pmem(addr, size); diff --git a/scripts/checkpatch.pl b/scripts/checkpatch.pl index 4904ced676d4..24a08363995a 100755 --- a/scripts/checkpatch.pl +++ b/scripts/checkpatch.pl @@ -313,7 +313,6 @@ our $Sparse = qr{ __kernel| __force| __iomem| - __pmem| __must_check| __init_refok| __kprobes| diff --git a/tools/testing/nvdimm/pmem-dax.c b/tools/testing/nvdimm/pmem-dax.c index 1e0218ce6a8b..c9b8c48f85fc 100644 --- a/tools/testing/nvdimm/pmem-dax.c +++ b/tools/testing/nvdimm/pmem-dax.c @@ -16,7 +16,7 @@ #include long pmem_direct_access(struct block_device *bdev, sector_t sector, - void __pmem **kaddr, pfn_t *pfn, long size) + void **kaddr, pfn_t *pfn, long size) { struct pmem_device *pmem = bdev->bd_queue->queuedata; resource_size_t offset = sector * 512 + pmem->data_offset; -- cgit v1.2.3-59-g8ed1b From bc9775d8697f57b333b6b316fb5145d6ca9dc36d Mon Sep 17 00:00:00 2001 From: Dan Williams Date: Thu, 21 Jul 2016 20:03:19 -0700 Subject: libnvdimm: move ->module to struct nvdimm_bus_descriptor Let the provider module be explicitly passed in rather than implicitly assumed by the module that calls nvdimm_bus_register(). This is in preparation for unifying the nfit and nfit_test driver teardown paths. Reviewed-by: Lee, Chun-Yi Signed-off-by: Dan Williams --- drivers/acpi/nfit.c | 1 + drivers/nvdimm/bus.c | 2 +- drivers/nvdimm/core.c | 7 +++---- drivers/nvdimm/e820.c | 1 + drivers/nvdimm/nd-core.h | 1 - include/linux/libnvdimm.h | 7 +++---- tools/testing/nvdimm/test/nfit.c | 1 + 7 files changed, 10 insertions(+), 10 deletions(-) (limited to 'drivers/nvdimm') diff --git a/drivers/acpi/nfit.c b/drivers/acpi/nfit.c index fb80f32db525..e7eb3b6f1514 100644 --- a/drivers/acpi/nfit.c +++ b/drivers/acpi/nfit.c @@ -2417,6 +2417,7 @@ void acpi_nfit_desc_init(struct acpi_nfit_desc *acpi_desc, struct device *dev) acpi_desc->blk_do_io = acpi_nfit_blk_region_do_io; nd_desc = &acpi_desc->nd_desc; nd_desc->provider_name = "ACPI.NFIT"; + nd_desc->module = THIS_MODULE; nd_desc->ndctl = acpi_nfit_ctl; nd_desc->flush_probe = acpi_nfit_flush_probe; nd_desc->clear_to_send = acpi_nfit_clear_to_send; diff --git a/drivers/nvdimm/bus.c b/drivers/nvdimm/bus.c index 1cc7880320fe..275dd5c0a301 100644 --- a/drivers/nvdimm/bus.c +++ b/drivers/nvdimm/bus.c @@ -73,7 +73,7 @@ static struct module *to_bus_provider(struct device *dev) if (is_nd_pmem(dev) || is_nd_blk(dev)) { struct nvdimm_bus *nvdimm_bus = walk_to_nvdimm_bus(dev); - return nvdimm_bus->module; + return nvdimm_bus->nd_desc->module; } return NULL; } diff --git a/drivers/nvdimm/core.c b/drivers/nvdimm/core.c index 757e0cf028bf..e8528756f54f 100644 --- a/drivers/nvdimm/core.c +++ b/drivers/nvdimm/core.c @@ -447,8 +447,8 @@ struct attribute_group nvdimm_bus_attribute_group = { }; EXPORT_SYMBOL_GPL(nvdimm_bus_attribute_group); -struct nvdimm_bus *__nvdimm_bus_register(struct device *parent, - struct nvdimm_bus_descriptor *nd_desc, struct module *module) +struct nvdimm_bus *nvdimm_bus_register(struct device *parent, + struct nvdimm_bus_descriptor *nd_desc) { struct nvdimm_bus *nvdimm_bus; int rc; @@ -467,7 +467,6 @@ struct nvdimm_bus *__nvdimm_bus_register(struct device *parent, return NULL; } nvdimm_bus->nd_desc = nd_desc; - nvdimm_bus->module = module; nvdimm_bus->dev.parent = parent; nvdimm_bus->dev.release = nvdimm_bus_release; nvdimm_bus->dev.groups = nd_desc->attr_groups; @@ -491,7 +490,7 @@ struct nvdimm_bus *__nvdimm_bus_register(struct device *parent, put_device(&nvdimm_bus->dev); return NULL; } -EXPORT_SYMBOL_GPL(__nvdimm_bus_register); +EXPORT_SYMBOL_GPL(nvdimm_bus_register); static void set_badblock(struct badblocks *bb, sector_t s, int num) { diff --git a/drivers/nvdimm/e820.c b/drivers/nvdimm/e820.c index 95825b38559a..11ea90120542 100644 --- a/drivers/nvdimm/e820.c +++ b/drivers/nvdimm/e820.c @@ -47,6 +47,7 @@ static int e820_pmem_probe(struct platform_device *pdev) nd_desc.attr_groups = e820_pmem_attribute_groups; nd_desc.provider_name = "e820"; + nd_desc.module = THIS_MODULE; nvdimm_bus = nvdimm_bus_register(dev, &nd_desc); if (!nvdimm_bus) goto err; diff --git a/drivers/nvdimm/nd-core.h b/drivers/nvdimm/nd-core.h index 6e961f7f43e7..38ce6bbbc170 100644 --- a/drivers/nvdimm/nd-core.h +++ b/drivers/nvdimm/nd-core.h @@ -26,7 +26,6 @@ extern int nvdimm_major; struct nvdimm_bus { struct nvdimm_bus_descriptor *nd_desc; wait_queue_head_t probe_wait; - struct module *module; struct list_head list; struct device dev; int id, probe_active; diff --git a/include/linux/libnvdimm.h b/include/linux/libnvdimm.h index d37fda6dd64c..2ab869dece4a 100644 --- a/include/linux/libnvdimm.h +++ b/include/linux/libnvdimm.h @@ -70,6 +70,7 @@ struct nd_mapping { struct nvdimm_bus_descriptor { const struct attribute_group **attr_groups; unsigned long cmd_mask; + struct module *module; char *provider_name; ndctl_fn ndctl; int (*flush_probe)(struct nvdimm_bus_descriptor *nd_desc); @@ -128,10 +129,8 @@ static inline struct nd_blk_region_desc *to_blk_region_desc( } int nvdimm_bus_add_poison(struct nvdimm_bus *nvdimm_bus, u64 addr, u64 length); -struct nvdimm_bus *__nvdimm_bus_register(struct device *parent, - struct nvdimm_bus_descriptor *nfit_desc, struct module *module); -#define nvdimm_bus_register(parent, desc) \ - __nvdimm_bus_register(parent, desc, THIS_MODULE) +struct nvdimm_bus *nvdimm_bus_register(struct device *parent, + struct nvdimm_bus_descriptor *nfit_desc); void nvdimm_bus_unregister(struct nvdimm_bus *nvdimm_bus); struct nvdimm_bus *to_nvdimm_bus(struct device *dev); struct nvdimm *to_nvdimm(struct device *dev); diff --git a/tools/testing/nvdimm/test/nfit.c b/tools/testing/nvdimm/test/nfit.c index 78cba1e3b1da..642713f15723 100644 --- a/tools/testing/nvdimm/test/nfit.c +++ b/tools/testing/nvdimm/test/nfit.c @@ -1463,6 +1463,7 @@ static int nfit_test_probe(struct platform_device *pdev) acpi_desc->blk_do_io = nfit_test_blk_do_io; nd_desc = &acpi_desc->nd_desc; nd_desc->provider_name = NULL; + nd_desc->module = THIS_MODULE; nd_desc->ndctl = nfit_test_ctl; acpi_desc->nvdimm_bus = nvdimm_bus_register(&pdev->dev, nd_desc); if (!acpi_desc->nvdimm_bus) -- cgit v1.2.3-59-g8ed1b From 5bf0b6e1af98a012e8871a89c2082353c3f70fee Mon Sep 17 00:00:00 2001 From: Vishal Verma Date: Fri, 22 Jul 2016 17:21:31 -0600 Subject: pmem: clarify a debug print in pmem_clear_poison Prefix the sector number being cleared with a '0x' to make it clear that this is a hex value. Signed-off-by: Vishal Verma Signed-off-by: Dan Williams --- drivers/nvdimm/pmem.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'drivers/nvdimm') diff --git a/drivers/nvdimm/pmem.c b/drivers/nvdimm/pmem.c index 7251b4b6da84..9f75eb85cf7c 100644 --- a/drivers/nvdimm/pmem.c +++ b/drivers/nvdimm/pmem.c @@ -58,7 +58,7 @@ static void pmem_clear_poison(struct pmem_device *pmem, phys_addr_t offset, cleared = nvdimm_clear_poison(dev, pmem->phys_addr + offset, len); if (cleared > 0 && cleared / 512) { - dev_dbg(dev, "%s: %llx clear %ld sector%s\n", + dev_dbg(dev, "%s: %#llx clear %ld sector%s\n", __func__, (unsigned long long) sector, cleared / 512, cleared / 512 > 1 ? "s" : ""); badblocks_clear(&pmem->bb, sector, cleared / 512); -- cgit v1.2.3-59-g8ed1b From 18515942d61bdfd4b31ea13f9fbb9c18650c6818 Mon Sep 17 00:00:00 2001 From: Dan Williams Date: Fri, 22 Jul 2016 23:46:08 -0700 Subject: libnvdimm: register nvdimm_bus devices with an nd_bus driver A recent effort to add a new nvdimm bus provider attribute highlighted a race between interrogating nvdimm_bus->nd_desc and nvdimm_bus tear down. The typical way to handle these races is to take the device_lock() in the attribute method and validate that the device is still active. In order for a device to be 'active' it needs to be associated with a driver. So, we create the small boilerplate for a driver and register nvdimm_bus devices on the 'nvdimm_bus_type' bus. A result of this change is that ndbusX devices now appear under /sys/bus/nd/devices. In fact this makes /sys/class/nd somewhat redundant, but removing that will need to take a long deprecation period given its use by ndctl binaries in the field. This change naturally pulls code from drivers/nvdimm/core.c to drivers/nvdimm/bus.c, so it is a nice code organization clean-up as well. Cc: Vishal Verma Signed-off-by: Dan Williams --- drivers/nvdimm/bus.c | 188 ++++++++++++++++++++++++++++++++++++++++++++++++-- drivers/nvdimm/core.c | 127 ---------------------------------- 2 files changed, 181 insertions(+), 134 deletions(-) (limited to 'drivers/nvdimm') diff --git a/drivers/nvdimm/bus.c b/drivers/nvdimm/bus.c index 275dd5c0a301..46d7e555b044 100644 --- a/drivers/nvdimm/bus.c +++ b/drivers/nvdimm/bus.c @@ -31,6 +31,7 @@ int nvdimm_major; static int nvdimm_bus_major; static struct class *nd_class; +static DEFINE_IDA(nd_ida); static int to_nd_device_type(struct device *dev) { @@ -60,13 +61,6 @@ static int nvdimm_bus_uevent(struct device *dev, struct kobj_uevent_env *env) to_nd_device_type(dev)); } -static int nvdimm_bus_match(struct device *dev, struct device_driver *drv) -{ - struct nd_device_driver *nd_drv = to_nd_device_driver(drv); - - return !!test_bit(to_nd_device_type(dev), &nd_drv->type); -} - static struct module *to_bus_provider(struct device *dev) { /* pin bus providers while regions are enabled */ @@ -223,6 +217,8 @@ long nvdimm_clear_poison(struct device *dev, phys_addr_t phys, } EXPORT_SYMBOL_GPL(nvdimm_clear_poison); +static int nvdimm_bus_match(struct device *dev, struct device_driver *drv); + static struct bus_type nvdimm_bus_type = { .name = "nd", .uevent = nvdimm_bus_uevent, @@ -232,6 +228,176 @@ static struct bus_type nvdimm_bus_type = { .shutdown = nvdimm_bus_shutdown, }; +static void nvdimm_bus_release(struct device *dev) +{ + struct nvdimm_bus *nvdimm_bus; + + nvdimm_bus = container_of(dev, struct nvdimm_bus, dev); + ida_simple_remove(&nd_ida, nvdimm_bus->id); + kfree(nvdimm_bus); +} + +static bool is_nvdimm_bus(struct device *dev) +{ + return dev->release == nvdimm_bus_release; +} + +struct nvdimm_bus *walk_to_nvdimm_bus(struct device *nd_dev) +{ + struct device *dev; + + for (dev = nd_dev; dev; dev = dev->parent) + if (is_nvdimm_bus(dev)) + break; + dev_WARN_ONCE(nd_dev, !dev, "invalid dev, not on nd bus\n"); + if (dev) + return to_nvdimm_bus(dev); + return NULL; +} + +struct nvdimm_bus *to_nvdimm_bus(struct device *dev) +{ + struct nvdimm_bus *nvdimm_bus; + + nvdimm_bus = container_of(dev, struct nvdimm_bus, dev); + WARN_ON(!is_nvdimm_bus(dev)); + return nvdimm_bus; +} +EXPORT_SYMBOL_GPL(to_nvdimm_bus); + +struct nvdimm_bus *nvdimm_bus_register(struct device *parent, + struct nvdimm_bus_descriptor *nd_desc) +{ + struct nvdimm_bus *nvdimm_bus; + int rc; + + nvdimm_bus = kzalloc(sizeof(*nvdimm_bus), GFP_KERNEL); + if (!nvdimm_bus) + return NULL; + INIT_LIST_HEAD(&nvdimm_bus->list); + INIT_LIST_HEAD(&nvdimm_bus->mapping_list); + INIT_LIST_HEAD(&nvdimm_bus->poison_list); + init_waitqueue_head(&nvdimm_bus->probe_wait); + nvdimm_bus->id = ida_simple_get(&nd_ida, 0, 0, GFP_KERNEL); + mutex_init(&nvdimm_bus->reconfig_mutex); + if (nvdimm_bus->id < 0) { + kfree(nvdimm_bus); + return NULL; + } + nvdimm_bus->nd_desc = nd_desc; + nvdimm_bus->dev.parent = parent; + nvdimm_bus->dev.release = nvdimm_bus_release; + nvdimm_bus->dev.groups = nd_desc->attr_groups; + nvdimm_bus->dev.bus = &nvdimm_bus_type; + dev_set_name(&nvdimm_bus->dev, "ndbus%d", nvdimm_bus->id); + rc = device_register(&nvdimm_bus->dev); + if (rc) { + dev_dbg(&nvdimm_bus->dev, "registration failed: %d\n", rc); + goto err; + } + + return nvdimm_bus; + err: + put_device(&nvdimm_bus->dev); + return NULL; +} +EXPORT_SYMBOL_GPL(nvdimm_bus_register); + +void nvdimm_bus_unregister(struct nvdimm_bus *nvdimm_bus) +{ + if (!nvdimm_bus) + return; + device_unregister(&nvdimm_bus->dev); +} +EXPORT_SYMBOL_GPL(nvdimm_bus_unregister); + +static int child_unregister(struct device *dev, void *data) +{ + /* + * the singular ndctl class device per bus needs to be + * "device_destroy"ed, so skip it here + * + * i.e. remove classless children + */ + if (dev->class) + /* pass */; + else + nd_device_unregister(dev, ND_SYNC); + return 0; +} + +static void free_poison_list(struct list_head *poison_list) +{ + struct nd_poison *pl, *next; + + list_for_each_entry_safe(pl, next, poison_list, list) { + list_del(&pl->list); + kfree(pl); + } + list_del_init(poison_list); +} + +static int nd_bus_remove(struct device *dev) +{ + struct nvdimm_bus *nvdimm_bus = to_nvdimm_bus(dev); + + mutex_lock(&nvdimm_bus_list_mutex); + list_del_init(&nvdimm_bus->list); + mutex_unlock(&nvdimm_bus_list_mutex); + + nd_synchronize(); + device_for_each_child(&nvdimm_bus->dev, NULL, child_unregister); + + nvdimm_bus_lock(&nvdimm_bus->dev); + free_poison_list(&nvdimm_bus->poison_list); + nvdimm_bus_unlock(&nvdimm_bus->dev); + + nvdimm_bus_destroy_ndctl(nvdimm_bus); + + return 0; +} + +static int nd_bus_probe(struct device *dev) +{ + struct nvdimm_bus *nvdimm_bus = to_nvdimm_bus(dev); + int rc; + + rc = nvdimm_bus_create_ndctl(nvdimm_bus); + if (rc) + return rc; + + mutex_lock(&nvdimm_bus_list_mutex); + list_add_tail(&nvdimm_bus->list, &nvdimm_bus_list); + mutex_unlock(&nvdimm_bus_list_mutex); + + /* enable bus provider attributes to look up their local context */ + dev_set_drvdata(dev, nvdimm_bus->nd_desc); + + return 0; +} + +static struct nd_device_driver nd_bus_driver = { + .probe = nd_bus_probe, + .remove = nd_bus_remove, + .drv = { + .name = "nd_bus", + .suppress_bind_attrs = true, + .bus = &nvdimm_bus_type, + .owner = THIS_MODULE, + .mod_name = KBUILD_MODNAME, + }, +}; + +static int nvdimm_bus_match(struct device *dev, struct device_driver *drv) +{ + struct nd_device_driver *nd_drv = to_nd_device_driver(drv); + + if (is_nvdimm_bus(dev) && nd_drv == &nd_bus_driver) + return true; + + return !!test_bit(to_nd_device_type(dev), &nd_drv->type); +} + static ASYNC_DOMAIN_EXCLUSIVE(nd_async_domain); void nd_synchronize(void) @@ -864,8 +1030,14 @@ int __init nvdimm_bus_init(void) goto err_class; } + rc = driver_register(&nd_bus_driver.drv); + if (rc) + goto err_nd_bus; + return 0; + err_nd_bus: + class_destroy(nd_class); err_class: unregister_chrdev(nvdimm_major, "dimmctl"); err_dimm_chrdev: @@ -878,8 +1050,10 @@ int __init nvdimm_bus_init(void) void nvdimm_bus_exit(void) { + driver_unregister(&nd_bus_driver.drv); class_destroy(nd_class); unregister_chrdev(nvdimm_bus_major, "ndctl"); unregister_chrdev(nvdimm_major, "dimmctl"); bus_unregister(&nvdimm_bus_type); + ida_destroy(&nd_ida); } diff --git a/drivers/nvdimm/core.c b/drivers/nvdimm/core.c index e8528756f54f..2c98f958fabb 100644 --- a/drivers/nvdimm/core.c +++ b/drivers/nvdimm/core.c @@ -26,7 +26,6 @@ LIST_HEAD(nvdimm_bus_list); DEFINE_MUTEX(nvdimm_bus_list_mutex); -static DEFINE_IDA(nd_ida); void nvdimm_bus_lock(struct device *dev) { @@ -195,25 +194,6 @@ u64 nd_fletcher64(void *addr, size_t len, bool le) } EXPORT_SYMBOL_GPL(nd_fletcher64); -static void nvdimm_bus_release(struct device *dev) -{ - struct nvdimm_bus *nvdimm_bus; - - nvdimm_bus = container_of(dev, struct nvdimm_bus, dev); - ida_simple_remove(&nd_ida, nvdimm_bus->id); - kfree(nvdimm_bus); -} - -struct nvdimm_bus *to_nvdimm_bus(struct device *dev) -{ - struct nvdimm_bus *nvdimm_bus; - - nvdimm_bus = container_of(dev, struct nvdimm_bus, dev); - WARN_ON(nvdimm_bus->dev.release != nvdimm_bus_release); - return nvdimm_bus; -} -EXPORT_SYMBOL_GPL(to_nvdimm_bus); - struct nvdimm_bus_descriptor *to_nd_desc(struct nvdimm_bus *nvdimm_bus) { /* struct nvdimm_bus definition is private to libnvdimm */ @@ -221,19 +201,6 @@ struct nvdimm_bus_descriptor *to_nd_desc(struct nvdimm_bus *nvdimm_bus) } EXPORT_SYMBOL_GPL(to_nd_desc); -struct nvdimm_bus *walk_to_nvdimm_bus(struct device *nd_dev) -{ - struct device *dev; - - for (dev = nd_dev; dev; dev = dev->parent) - if (dev->release == nvdimm_bus_release) - break; - dev_WARN_ONCE(nd_dev, !dev, "invalid dev, not on nd bus\n"); - if (dev) - return to_nvdimm_bus(dev); - return NULL; -} - static bool is_uuid_sep(char sep) { if (sep == '\n' || sep == '-' || sep == ':' || sep == '\0') @@ -447,51 +414,6 @@ struct attribute_group nvdimm_bus_attribute_group = { }; EXPORT_SYMBOL_GPL(nvdimm_bus_attribute_group); -struct nvdimm_bus *nvdimm_bus_register(struct device *parent, - struct nvdimm_bus_descriptor *nd_desc) -{ - struct nvdimm_bus *nvdimm_bus; - int rc; - - nvdimm_bus = kzalloc(sizeof(*nvdimm_bus), GFP_KERNEL); - if (!nvdimm_bus) - return NULL; - INIT_LIST_HEAD(&nvdimm_bus->list); - INIT_LIST_HEAD(&nvdimm_bus->mapping_list); - INIT_LIST_HEAD(&nvdimm_bus->poison_list); - init_waitqueue_head(&nvdimm_bus->probe_wait); - nvdimm_bus->id = ida_simple_get(&nd_ida, 0, 0, GFP_KERNEL); - mutex_init(&nvdimm_bus->reconfig_mutex); - if (nvdimm_bus->id < 0) { - kfree(nvdimm_bus); - return NULL; - } - nvdimm_bus->nd_desc = nd_desc; - nvdimm_bus->dev.parent = parent; - nvdimm_bus->dev.release = nvdimm_bus_release; - nvdimm_bus->dev.groups = nd_desc->attr_groups; - dev_set_name(&nvdimm_bus->dev, "ndbus%d", nvdimm_bus->id); - rc = device_register(&nvdimm_bus->dev); - if (rc) { - dev_dbg(&nvdimm_bus->dev, "registration failed: %d\n", rc); - goto err; - } - - rc = nvdimm_bus_create_ndctl(nvdimm_bus); - if (rc) - goto err; - - mutex_lock(&nvdimm_bus_list_mutex); - list_add_tail(&nvdimm_bus->list, &nvdimm_bus_list); - mutex_unlock(&nvdimm_bus_list_mutex); - - return nvdimm_bus; - err: - put_device(&nvdimm_bus->dev); - return NULL; -} -EXPORT_SYMBOL_GPL(nvdimm_bus_register); - static void set_badblock(struct badblocks *bb, sector_t s, int num) { dev_dbg(bb->dev, "Found a poison range (0x%llx, 0x%llx)\n", @@ -667,54 +589,6 @@ int nvdimm_bus_add_poison(struct nvdimm_bus *nvdimm_bus, u64 addr, u64 length) } EXPORT_SYMBOL_GPL(nvdimm_bus_add_poison); -static void free_poison_list(struct list_head *poison_list) -{ - struct nd_poison *pl, *next; - - list_for_each_entry_safe(pl, next, poison_list, list) { - list_del(&pl->list); - kfree(pl); - } - list_del_init(poison_list); -} - -static int child_unregister(struct device *dev, void *data) -{ - /* - * the singular ndctl class device per bus needs to be - * "device_destroy"ed, so skip it here - * - * i.e. remove classless children - */ - if (dev->class) - /* pass */; - else - nd_device_unregister(dev, ND_SYNC); - return 0; -} - -void nvdimm_bus_unregister(struct nvdimm_bus *nvdimm_bus) -{ - if (!nvdimm_bus) - return; - - mutex_lock(&nvdimm_bus_list_mutex); - list_del_init(&nvdimm_bus->list); - mutex_unlock(&nvdimm_bus_list_mutex); - - nd_synchronize(); - device_for_each_child(&nvdimm_bus->dev, NULL, child_unregister); - - nvdimm_bus_lock(&nvdimm_bus->dev); - free_poison_list(&nvdimm_bus->poison_list); - nvdimm_bus_unlock(&nvdimm_bus->dev); - - nvdimm_bus_destroy_ndctl(nvdimm_bus); - - device_unregister(&nvdimm_bus->dev); -} -EXPORT_SYMBOL_GPL(nvdimm_bus_unregister); - #ifdef CONFIG_BLK_DEV_INTEGRITY int nd_integrity_init(struct gendisk *disk, unsigned long meta_size) { @@ -773,7 +647,6 @@ static __exit void libnvdimm_exit(void) nvdimm_bus_exit(); nd_region_devs_exit(); nvdimm_devs_exit(); - ida_destroy(&nd_ida); } MODULE_LICENSE("GPL v2"); -- cgit v1.2.3-59-g8ed1b From 37b137ff8c833385b75ff2baf4bace25e52247d2 Mon Sep 17 00:00:00 2001 From: Vishal Verma Date: Sat, 23 Jul 2016 21:51:42 -0700 Subject: nfit, libnvdimm: allow an ARS scrub to be triggered on demand Normally, an ARS (Address Range Scrub) only happens at boot/initialization time. There can however arise situations where a bus-wide rescan is needed - notably, in the case of discovering a latent media error, we should do a full rescan to figure out what other sectors are bad, and thus potentially avoid triggering an mce on them in the future. Also provide a sysfs trigger to start a bus-wide scrub. Cc: Rafael J. Wysocki Signed-off-by: Vishal Verma Signed-off-by: Dan Williams --- drivers/acpi/nfit.c | 161 ++++++++++++++++++++++++++++++++++++++++++++-- drivers/acpi/nfit.h | 4 +- drivers/nvdimm/core.c | 7 ++ include/linux/libnvdimm.h | 1 + 4 files changed, 165 insertions(+), 8 deletions(-) (limited to 'drivers/nvdimm') diff --git a/drivers/acpi/nfit.c b/drivers/acpi/nfit.c index be7c2fde16e7..19d0dfdf9633 100644 --- a/drivers/acpi/nfit.c +++ b/drivers/acpi/nfit.c @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -874,14 +875,87 @@ static ssize_t revision_show(struct device *dev, } static DEVICE_ATTR_RO(revision); +/* + * This shows the number of full Address Range Scrubs that have been + * completed since driver load time. Userspace can wait on this using + * select/poll etc. A '+' at the end indicates an ARS is in progress + */ +static ssize_t scrub_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct nvdimm_bus_descriptor *nd_desc; + ssize_t rc = -ENXIO; + + device_lock(dev); + nd_desc = dev_get_drvdata(dev); + if (nd_desc) { + struct acpi_nfit_desc *acpi_desc = to_acpi_desc(nd_desc); + + rc = sprintf(buf, "%d%s", acpi_desc->scrub_count, + (work_busy(&acpi_desc->work)) ? "+\n" : "\n"); + } + device_unlock(dev); + return rc; +} + +static int acpi_nfit_ars_rescan(struct acpi_nfit_desc *acpi_desc); + +static ssize_t scrub_store(struct device *dev, + struct device_attribute *attr, const char *buf, size_t size) +{ + struct nvdimm_bus_descriptor *nd_desc; + ssize_t rc; + long val; + + rc = kstrtol(buf, 0, &val); + if (rc) + return rc; + if (val != 1) + return -EINVAL; + + device_lock(dev); + nd_desc = dev_get_drvdata(dev); + if (nd_desc) { + struct acpi_nfit_desc *acpi_desc = to_acpi_desc(nd_desc); + + rc = acpi_nfit_ars_rescan(acpi_desc); + } + device_unlock(dev); + if (rc) + return rc; + return size; +} +static DEVICE_ATTR_RW(scrub); + +static bool ars_supported(struct nvdimm_bus *nvdimm_bus) +{ + struct nvdimm_bus_descriptor *nd_desc = to_nd_desc(nvdimm_bus); + const unsigned long mask = 1 << ND_CMD_ARS_CAP | 1 << ND_CMD_ARS_START + | 1 << ND_CMD_ARS_STATUS; + + return (nd_desc->cmd_mask & mask) == mask; +} + +static umode_t nfit_visible(struct kobject *kobj, struct attribute *a, int n) +{ + struct device *dev = container_of(kobj, struct device, kobj); + struct nvdimm_bus *nvdimm_bus = to_nvdimm_bus(dev); + + if (a == &dev_attr_scrub.attr && !ars_supported(nvdimm_bus)) + return 0; + return a->mode; +} + static struct attribute *acpi_nfit_attributes[] = { &dev_attr_revision.attr, + &dev_attr_scrub.attr, NULL, }; static struct attribute_group acpi_nfit_attribute_group = { .name = "nfit", .attrs = acpi_nfit_attributes, + .is_visible = nfit_visible, }; static const struct attribute_group *acpi_nfit_attribute_groups[] = { @@ -2054,7 +2128,7 @@ static void acpi_nfit_async_scrub(struct acpi_nfit_desc *acpi_desc, unsigned int tmo = scrub_timeout; int rc; - if (nfit_spa->ars_done || !nfit_spa->nd_region) + if (!nfit_spa->ars_required || !nfit_spa->nd_region) return; rc = ars_start(acpi_desc, nfit_spa); @@ -2143,7 +2217,9 @@ static void acpi_nfit_scrub(struct work_struct *work) * firmware initiated scrubs to complete and then we go search for the * affected spa regions to mark them scanned. In the second phase we * initiate a directed scrub for every range that was not scrubbed in - * phase 1. + * phase 1. If we're called for a 'rescan', we harmlessly pass through + * the first phase, but really only care about running phase 2, where + * regions can be notified of new poison. */ /* process platform firmware initiated scrubs */ @@ -2246,14 +2322,17 @@ static void acpi_nfit_scrub(struct work_struct *work) * Flag all the ranges that still need scrubbing, but * register them now to make data available. */ - if (nfit_spa->nd_region) - nfit_spa->ars_done = 1; - else + if (!nfit_spa->nd_region) { + nfit_spa->ars_required = 1; acpi_nfit_register_region(acpi_desc, nfit_spa); + } } list_for_each_entry(nfit_spa, &acpi_desc->spas, list) acpi_nfit_async_scrub(acpi_desc, nfit_spa); + acpi_desc->scrub_count++; + if (acpi_desc->scrub_count_state) + sysfs_notify_dirent(acpi_desc->scrub_count_state); mutex_unlock(&acpi_desc->init_mutex); } @@ -2291,12 +2370,48 @@ static int acpi_nfit_check_deletions(struct acpi_nfit_desc *acpi_desc, return 0; } +static int acpi_nfit_desc_init_scrub_attr(struct acpi_nfit_desc *acpi_desc) +{ + struct device *dev = acpi_desc->dev; + struct kernfs_node *nfit; + struct device *bus_dev; + + if (!ars_supported(acpi_desc->nvdimm_bus)) + return 0; + + bus_dev = to_nvdimm_bus_dev(acpi_desc->nvdimm_bus); + nfit = sysfs_get_dirent(bus_dev->kobj.sd, "nfit"); + if (!nfit) { + dev_err(dev, "sysfs_get_dirent 'nfit' failed\n"); + return -ENODEV; + } + acpi_desc->scrub_count_state = sysfs_get_dirent(nfit, "scrub"); + sysfs_put(nfit); + if (!acpi_desc->scrub_count_state) { + dev_err(dev, "sysfs_get_dirent 'scrub' failed\n"); + return -ENODEV; + } + + return 0; +} + static void acpi_nfit_destruct(void *data) { struct acpi_nfit_desc *acpi_desc = data; + struct device *bus_dev = to_nvdimm_bus_dev(acpi_desc->nvdimm_bus); acpi_desc->cancel = 1; + /* + * Bounce the nvdimm bus lock to make sure any in-flight + * acpi_nfit_ars_rescan() submissions have had a chance to + * either submit or see ->cancel set. + */ + device_lock(bus_dev); + device_unlock(bus_dev); + flush_workqueue(nfit_wq); + if (acpi_desc->scrub_count_state) + sysfs_put(acpi_desc->scrub_count_state); nvdimm_bus_unregister(acpi_desc->nvdimm_bus); acpi_desc->nvdimm_bus = NULL; } @@ -2309,14 +2424,21 @@ int acpi_nfit_init(struct acpi_nfit_desc *acpi_desc, void *data, acpi_size sz) int rc; if (!acpi_desc->nvdimm_bus) { + acpi_nfit_init_dsms(acpi_desc); + acpi_desc->nvdimm_bus = nvdimm_bus_register(dev, &acpi_desc->nd_desc); if (!acpi_desc->nvdimm_bus) return -ENOMEM; + rc = devm_add_action_or_reset(dev, acpi_nfit_destruct, acpi_desc); if (rc) return rc; + + rc = acpi_nfit_desc_init_scrub_attr(acpi_desc); + if (rc) + return rc; } mutex_lock(&acpi_desc->init_mutex); @@ -2360,8 +2482,6 @@ int acpi_nfit_init(struct acpi_nfit_desc *acpi_desc, void *data, acpi_size sz) if (rc) goto out_unlock; - acpi_nfit_init_dsms(acpi_desc); - rc = acpi_nfit_register_dimms(acpi_desc); if (rc) goto out_unlock; @@ -2429,6 +2549,33 @@ static int acpi_nfit_clear_to_send(struct nvdimm_bus_descriptor *nd_desc, return 0; } +static int acpi_nfit_ars_rescan(struct acpi_nfit_desc *acpi_desc) +{ + struct device *dev = acpi_desc->dev; + struct nfit_spa *nfit_spa; + + if (work_busy(&acpi_desc->work)) + return -EBUSY; + + if (acpi_desc->cancel) + return 0; + + mutex_lock(&acpi_desc->init_mutex); + list_for_each_entry(nfit_spa, &acpi_desc->spas, list) { + struct acpi_nfit_system_address *spa = nfit_spa->spa; + + if (nfit_spa_type(spa) != NFIT_SPA_PM) + continue; + + nfit_spa->ars_required = 1; + } + queue_work(nfit_wq, &acpi_desc->work); + dev_dbg(dev, "%s: ars_scan triggered\n", __func__); + mutex_unlock(&acpi_desc->init_mutex); + + return 0; +} + void acpi_nfit_desc_init(struct acpi_nfit_desc *acpi_desc, struct device *dev) { struct nvdimm_bus_descriptor *nd_desc; diff --git a/drivers/acpi/nfit.h b/drivers/acpi/nfit.h index b63a583a678f..6ecf337c97aa 100644 --- a/drivers/acpi/nfit.h +++ b/drivers/acpi/nfit.h @@ -80,7 +80,7 @@ enum { struct nfit_spa { struct list_head list; struct nd_region *nd_region; - unsigned int ars_done:1; + unsigned int ars_required:1; u32 clear_err_unit; u32 max_ars; struct acpi_nfit_system_address spa[0]; @@ -148,6 +148,8 @@ struct acpi_nfit_desc { struct nd_cmd_ars_status *ars_status; size_t ars_status_size; struct work_struct work; + struct kernfs_node *scrub_count_state; + unsigned int scrub_count; unsigned int cancel:1; unsigned long dimm_cmd_force_en; unsigned long bus_cmd_force_en; diff --git a/drivers/nvdimm/core.c b/drivers/nvdimm/core.c index 2c98f958fabb..715583f69d28 100644 --- a/drivers/nvdimm/core.c +++ b/drivers/nvdimm/core.c @@ -201,6 +201,13 @@ struct nvdimm_bus_descriptor *to_nd_desc(struct nvdimm_bus *nvdimm_bus) } EXPORT_SYMBOL_GPL(to_nd_desc); +struct device *to_nvdimm_bus_dev(struct nvdimm_bus *nvdimm_bus) +{ + /* struct nvdimm_bus definition is private to libnvdimm */ + return &nvdimm_bus->dev; +} +EXPORT_SYMBOL_GPL(to_nvdimm_bus_dev); + static bool is_uuid_sep(char sep) { if (sep == '\n' || sep == '-' || sep == ':' || sep == '\0') diff --git a/include/linux/libnvdimm.h b/include/linux/libnvdimm.h index 2ab869dece4a..b519e137b9b7 100644 --- a/include/linux/libnvdimm.h +++ b/include/linux/libnvdimm.h @@ -137,6 +137,7 @@ struct nvdimm *to_nvdimm(struct device *dev); struct nd_region *to_nd_region(struct device *dev); struct nd_blk_region *to_nd_blk_region(struct device *dev); struct nvdimm_bus_descriptor *to_nd_desc(struct nvdimm_bus *nvdimm_bus); +struct device *to_nvdimm_bus_dev(struct nvdimm_bus *nvdimm_bus); const char *nvdimm_name(struct nvdimm *nvdimm); unsigned long nvdimm_cmd_mask(struct nvdimm *nvdimm); void *nvdimm_provider_data(struct nvdimm *nvdimm); -- cgit v1.2.3-59-g8ed1b From d4c5725d57323e2348940fcc6416072671a9b432 Mon Sep 17 00:00:00 2001 From: Markus Elfring Date: Sun, 24 Jul 2016 11:05:34 +0200 Subject: libnvdimm-btt: Delete an unnecessary check before the function call "__nd_device_register" The __nd_device_register() function tests whether its argument is NULL and then returns immediately. Thus the test around the call is not needed. This issue was detected by using the Coccinelle software. Signed-off-by: Markus Elfring Signed-off-by: Dan Williams --- drivers/nvdimm/btt_devs.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'drivers/nvdimm') diff --git a/drivers/nvdimm/btt_devs.c b/drivers/nvdimm/btt_devs.c index 816d0dae6398..3fa7919f94a8 100644 --- a/drivers/nvdimm/btt_devs.c +++ b/drivers/nvdimm/btt_devs.c @@ -198,8 +198,7 @@ struct device *nd_btt_create(struct nd_region *nd_region) { struct device *dev = __nd_btt_create(nd_region, 0, NULL, NULL); - if (dev) - __nd_device_register(dev); + __nd_device_register(dev); return dev; } -- cgit v1.2.3-59-g8ed1b