From d750c42ac265c00df3f0963a240a4440fa073603 Mon Sep 17 00:00:00 2001 From: Gang He Date: Tue, 22 Mar 2016 14:24:33 -0700 Subject: ocfs2: add feature document for online file check This document will describe OCFS2 online file check feature. OCFS2 is often used in high-availaibility systems. However, OCFS2 usually converts the filesystem to read-only when encounters an error. This may not be necessary, since turning the filesystem read-only would affect other running processes as well, decreasing availability. Then, a mount option (errors=continue) is introduced, which would return the -EIO errno to the calling process and terminate furhter processing so that the filesystem is not corrupted further. The filesystem is not converted to read-only, and the problematic file's inode number is reported in the kernel log. The user can try to check/fix this file via online filecheck feature. Signed-off-by: Gang He Cc: Mark Fasheh Cc: Joel Becker Cc: Junxiao Bi Cc: Joseph Qi Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- .../filesystems/ocfs2-online-filecheck.txt | 94 ++++++++++++++++++++++ 1 file changed, 94 insertions(+) create mode 100644 Documentation/filesystems/ocfs2-online-filecheck.txt (limited to 'Documentation') diff --git a/Documentation/filesystems/ocfs2-online-filecheck.txt b/Documentation/filesystems/ocfs2-online-filecheck.txt new file mode 100644 index 000000000000..1ab07860430d --- /dev/null +++ b/Documentation/filesystems/ocfs2-online-filecheck.txt @@ -0,0 +1,94 @@ + OCFS2 online file check + ----------------------- + +This document will describe OCFS2 online file check feature. + +Introduction +============ +OCFS2 is often used in high-availaibility systems. However, OCFS2 usually +converts the filesystem to read-only when encounters an error. This may not be +necessary, since turning the filesystem read-only would affect other running +processes as well, decreasing availability. +Then, a mount option (errors=continue) is introduced, which would return the +-EIO errno to the calling process and terminate furhter processing so that the +filesystem is not corrupted further. The filesystem is not converted to +read-only, and the problematic file's inode number is reported in the kernel +log. The user can try to check/fix this file via online filecheck feature. + +Scope +===== +This effort is to check/fix small issues which may hinder day-to-day operations +of a cluster filesystem by turning the filesystem read-only. The scope of +checking/fixing is at the file level, initially for regular files and eventually +to all files (including system files) of the filesystem. + +In case of directory to file links is incorrect, the directory inode is +reported as erroneous. + +This feature is not suited for extravagant checks which involve dependency of +other components of the filesystem, such as but not limited to, checking if the +bits for file blocks in the allocation has been set. In case of such an error, +the offline fsck should/would be recommended. + +Finally, such an operation/feature should not be automated lest the filesystem +may end up with more damage than before the repair attempt. So, this has to +be performed using user interaction and consent. + +User interface +============== +When there are errors in the OCFS2 filesystem, they are usually accompanied +by the inode number which caused the error. This inode number would be the +input to check/fix the file. + +There is a sysfs directory for each OCFS2 file system mounting: + + /sys/fs/ocfs2//filecheck + +Here, indicates the name of OCFS2 volumn device which has been already +mounted. The file above would accept inode numbers. This could be used to +communicate with kernel space, tell which file(inode number) will be checked or +fixed. Currently, three operations are supported, which includes checking +inode, fixing inode and setting the size of result record history. + +1. If you want to know what error exactly happened to before fixing, do + + # echo "" > /sys/fs/ocfs2//filecheck/check + # cat /sys/fs/ocfs2//filecheck/check + +The output is like this: + INO DONE ERROR +39502 1 GENERATION + + lists the inode numbers. + indicates whether the operation has been finished. + says what kind of errors was found. For the detailed error numbers, +please refer to the file linux/fs/ocfs2/filecheck.h. + +2. If you determine to fix this inode, do + + # echo "" > /sys/fs/ocfs2//filecheck/fix + # cat /sys/fs/ocfs2//filecheck/fix + +The output is like this: + INO DONE ERROR +39502 1 SUCCESS + +This time, the column indicates whether this fix is successful or not. + +3. The record cache is used to store the history of check/fix results. It's +defalut size is 10, and can be adjust between the range of 10 ~ 100. You can +adjust the size like this: + + # echo "" > /sys/fs/ocfs2//filecheck/set + +Fixing stuff +============ +On receivng the inode, the filesystem would read the inode and the +file metadata. In case of errors, the filesystem would fix the errors +and report the problems it fixed in the kernel log. As a precautionary measure, +the inode must first be checked for errors before performing a final fix. + +The inode and the result history will be maintained temporarily in a +small linked list buffer which would contain the last (N) inodes +fixed/checked, the detailed errors which were fixed/checked are printed in the +kernel log. -- cgit v1.2.3-59-g8ed1b From 38739380683795354b3f0f1a1e80614e311b8617 Mon Sep 17 00:00:00 2001 From: "Maciej S. Szmigiero" Date: Tue, 22 Mar 2016 14:25:30 -0700 Subject: fat: add config option to set UTF-8 mount option by default FAT has long supported its own default file name encoding config setting, separate from CONFIG_NLS_DEFAULT. However, if UTF-8 encoded file names are desired FAT character set should not be set to utf8 since this would make file names case sensitive even if case insensitive matching is requested. Instead, "utf8" mount options should be provided to enable UTF-8 file names in FAT file system. Unfortunately, there was no possibility to set the default value of this option so on UTF-8 system "utf8" mount option had to be added manually to most FAT mounts. This patch adds config option to set such default value. Signed-off-by: Maciej S. Szmigiero Acked-by: OGAWA Hirofumi Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- Documentation/filesystems/vfat.txt | 7 ++++--- fs/fat/Kconfig | 18 +++++++++++++++++- fs/fat/inode.c | 4 +++- 3 files changed, 24 insertions(+), 5 deletions(-) (limited to 'Documentation') diff --git a/Documentation/filesystems/vfat.txt b/Documentation/filesystems/vfat.txt index 223c32171dcc..cf51360e3a9f 100644 --- a/Documentation/filesystems/vfat.txt +++ b/Documentation/filesystems/vfat.txt @@ -56,9 +56,10 @@ iocharset= -- Character set to use for converting between the you should consider the following option instead. utf8= -- UTF-8 is the filesystem safe version of Unicode that - is used by the console. It can be enabled for the - filesystem with this option. If 'uni_xlate' gets set, - UTF-8 gets disabled. + is used by the console. It can be enabled or disabled + for the filesystem with this option. + If 'uni_xlate' gets set, UTF-8 gets disabled. + By default, FAT_DEFAULT_UTF8 setting is used. uni_xlate= -- Translate unhandled Unicode characters to special escaped sequences. This would let you backup and diff --git a/fs/fat/Kconfig b/fs/fat/Kconfig index 182f9ffe2b51..3ff1772f612e 100644 --- a/fs/fat/Kconfig +++ b/fs/fat/Kconfig @@ -93,8 +93,24 @@ config FAT_DEFAULT_IOCHARSET that most of your FAT filesystems use, and can be overridden with the "iocharset" mount option for FAT filesystems. Note that "utf8" is not recommended for FAT filesystems. - If unsure, you shouldn't set "utf8" here. + If unsure, you shouldn't set "utf8" here - select the next option + instead if you would like to use UTF-8 encoded file names by default. See for more information. Enable any character sets you need in File Systems/Native Language Support. + +config FAT_DEFAULT_UTF8 + bool "Enable FAT UTF-8 option by default" + depends on VFAT_FS + default n + help + Set this if you would like to have "utf8" mount option set + by default when mounting FAT filesystems. + + Even if you say Y here can always disable UTF-8 for + particular mount by adding "utf8=0" to mount options. + + Say Y if you use UTF-8 encoding for file names, N otherwise. + + See for more information. diff --git a/fs/fat/inode.c b/fs/fat/inode.c index a5599052116c..226281068a46 100644 --- a/fs/fat/inode.c +++ b/fs/fat/inode.c @@ -1127,7 +1127,7 @@ static int parse_options(struct super_block *sb, char *options, int is_vfat, } opts->name_check = 'n'; opts->quiet = opts->showexec = opts->sys_immutable = opts->dotsOK = 0; - opts->utf8 = opts->unicode_xlate = 0; + opts->unicode_xlate = 0; opts->numtail = 1; opts->usefree = opts->nocase = 0; opts->tz_set = 0; @@ -1135,6 +1135,8 @@ static int parse_options(struct super_block *sb, char *options, int is_vfat, opts->errors = FAT_ERRORS_RO; *debug = 0; + opts->utf8 = IS_ENABLED(CONFIG_FAT_DEFAULT_UTF8) && is_vfat; + if (!options) goto out; -- cgit v1.2.3-59-g8ed1b From 72d8a0d23083ba89fb00a7ad9b07419e34ebe47c Mon Sep 17 00:00:00 2001 From: Alexandre Bounine Date: Tue, 22 Mar 2016 14:26:56 -0700 Subject: rapidio/tsi721: add filtered debug output Replace "all-or-nothing" debug output with controlled debug output using functional block masks. This allows run time control of debug messages through 'dbg_level' module parameter. Signed-off-by: Alexandre Bounine Cc: Matt Porter Cc: Aurelien Jacquiot Cc: Andre van Herk Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- Documentation/rapidio/tsi721.txt | 9 ++ drivers/rapidio/devices/tsi721.c | 242 ++++++++++++++++++----------------- drivers/rapidio/devices/tsi721.h | 40 ++++++ drivers/rapidio/devices/tsi721_dma.c | 147 +++++++++++---------- 4 files changed, 249 insertions(+), 189 deletions(-) (limited to 'Documentation') diff --git a/Documentation/rapidio/tsi721.txt b/Documentation/rapidio/tsi721.txt index 626052f403bb..7c1c7bf48ec0 100644 --- a/Documentation/rapidio/tsi721.txt +++ b/Documentation/rapidio/tsi721.txt @@ -16,6 +16,15 @@ For inbound messages this driver uses destination ID matching to forward message into the corresponding message queue. Messaging callbacks are implemented to be fully compatible with RIONET driver (Ethernet over RapidIO messaging services). +1. Module parameters: +- 'dbg_level' - This parameter allows to control amount of debug information + generated by this device driver. This parameter is formed by set of + This parameter can be changed bit masks that correspond to the specific + functional block. + For mask definitions see 'drivers/rapidio/devices/tsi721.h' + This parameter can be changed dynamically. + Use CONFIG_RAPIDIO_DEBUG=y to enable debug output at the top level. + II. Known problems None. diff --git a/drivers/rapidio/devices/tsi721.c b/drivers/rapidio/devices/tsi721.c index 74bc1fbb784e..b5b455614f8a 100644 --- a/drivers/rapidio/devices/tsi721.c +++ b/drivers/rapidio/devices/tsi721.c @@ -36,6 +36,12 @@ #include "tsi721.h" +#ifdef DEBUG +u32 dbg_level = DBG_INIT | DBG_EXIT; +module_param(dbg_level, uint, S_IWUSR | S_IRUGO); +MODULE_PARM_DESC(dbg_level, "Debugging output level (default 0 = none)"); +#endif + static void tsi721_omsg_handler(struct tsi721_device *priv, int ch); static void tsi721_imsg_handler(struct tsi721_device *priv, int ch); @@ -141,9 +147,9 @@ static int tsi721_maint_dma(struct tsi721_device *priv, u32 sys_size, & TSI721_DMAC_STS_RUN) { udelay(1); if (++i >= 5000000) { - dev_dbg(&priv->pdev->dev, - "%s : DMA[%d] read timeout ch_status=%x\n", - __func__, priv->mdma.ch_id, ch_stat); + tsi_debug(MAINT, &priv->pdev->dev, + "DMA[%d] read timeout ch_status=%x", + priv->mdma.ch_id, ch_stat); if (!do_wr) *data = 0xffffffff; err = -EIO; @@ -155,10 +161,12 @@ static int tsi721_maint_dma(struct tsi721_device *priv, u32 sys_size, /* If DMA operation aborted due to error, * reinitialize DMA channel */ - dev_dbg(&priv->pdev->dev, "%s : DMA ABORT ch_stat=%x\n", - __func__, ch_stat); - dev_dbg(&priv->pdev->dev, "OP=%d : destid=%x hc=%x off=%x\n", - do_wr ? MAINT_WR : MAINT_RD, destid, hopcount, offset); + tsi_debug(MAINT, &priv->pdev->dev, "DMA ABORT ch_stat=%x", + ch_stat); + tsi_debug(MAINT, &priv->pdev->dev, + "OP=%d : destid=%x hc=%x off=%x", + do_wr ? MAINT_WR : MAINT_RD, + destid, hopcount, offset); iowrite32(TSI721_DMAC_INT_ALL, regs + TSI721_DMAC_INT); iowrite32(TSI721_DMAC_CTL_INIT, regs + TSI721_DMAC_CTL); udelay(10); @@ -336,8 +344,8 @@ static int tsi721_dsend(struct rio_mport *mport, int index, offset = (((mport->sys_size) ? RIO_TT_CODE_16 : RIO_TT_CODE_8) << 18) | (destid << 2); - dev_dbg(&priv->pdev->dev, - "Send Doorbell 0x%04x to destID 0x%x\n", data, destid); + tsi_debug(DBELL, &priv->pdev->dev, + "Send Doorbell 0x%04x to destID 0x%x", data, destid); iowrite16be(data, priv->odb_base + offset); return 0; @@ -411,10 +419,10 @@ static void tsi721_db_dpc(struct work_struct *work) dbell->dinb(mport, dbell->dev_id, DBELL_SID(idb.bytes), DBELL_TID(idb.bytes), DBELL_INF(idb.bytes)); } else { - dev_dbg(&priv->pdev->dev, - "spurious inb doorbell, sid %2.2x tid %2.2x" - " info %4.4x\n", DBELL_SID(idb.bytes), - DBELL_TID(idb.bytes), DBELL_INF(idb.bytes)); + tsi_debug(DBELL, &priv->pdev->dev, + "spurious IDB sid %2.2x tid %2.2x info %4.4x", + DBELL_SID(idb.bytes), DBELL_TID(idb.bytes), + DBELL_INF(idb.bytes)); } wr_ptr = ioread32(priv->regs + @@ -470,8 +478,8 @@ static irqreturn_t tsi721_irqhandler(int irq, void *ptr) if (intval & TSI721_SR_CHINT_IDBQRCV) tsi721_dbell_handler(priv); else - dev_info(&priv->pdev->dev, - "Unsupported SR_CH_INT %x\n", intval); + tsi_info(&priv->pdev->dev, + "Unsupported SR_CH_INT %x", intval); /* Clear interrupts */ iowrite32(intval, @@ -533,8 +541,8 @@ static irqreturn_t tsi721_irqhandler(int irq, void *ptr) int ch; if (dev_ch_int & TSI721_INT_BDMA_CHAN_M) { - dev_dbg(&priv->pdev->dev, - "IRQ from DMA channel 0x%08x\n", dev_ch_int); + tsi_debug(DMA, &priv->pdev->dev, + "IRQ from DMA channel 0x%08x", dev_ch_int); for (ch = 0; ch < TSI721_DMA_MAXCH; ch++) { if (!(dev_ch_int & TSI721_INT_BDMA_CHAN(ch))) @@ -749,8 +757,8 @@ static int tsi721_enable_msix(struct tsi721_device *priv) err = pci_enable_msix_exact(priv->pdev, entries, ARRAY_SIZE(entries)); if (err) { - dev_err(&priv->pdev->dev, - "Failed to enable MSI-X (err=%d)\n", err); + tsi_err(&priv->pdev->dev, + "Failed to enable MSI-X (err=%d)", err); return err; } @@ -824,8 +832,8 @@ static int tsi721_request_irq(struct tsi721_device *priv) DRV_NAME, (void *)priv); if (err) - dev_err(&priv->pdev->dev, - "Unable to allocate interrupt, Error: %d\n", err); + tsi_err(&priv->pdev->dev, + "Unable to allocate interrupt, err=%d", err); return err; } @@ -891,7 +899,7 @@ tsi721_obw_alloc(struct tsi721_device *priv, struct tsi721_obw_bar *pbar, return -ENOMEM; if (!new_win) { - dev_err(&priv->pdev->dev, "ERR: OBW count tracking failed\n"); + tsi_err(&priv->pdev->dev, "OBW count tracking failed"); return -EIO; } @@ -918,6 +926,9 @@ static int tsi721_map_outb_win(struct rio_mport *mport, u16 destid, u64 rstart, u32 zsize; int ret = -ENOMEM; + tsi_debug(OBW, &priv->pdev->dev, + "did=%d ra=0x%llx sz=0x%x", destid, rstart, size); + if (!is_power_of_2(size) || (size < 0x8000) || (rstart & (size - 1))) return -EINVAL; @@ -940,6 +951,8 @@ static int tsi721_map_outb_win(struct rio_mport *mport, u16 destid, u64 rstart, ob_win = &priv->ob_win[obw]; ob_win->destid = destid; ob_win->rstart = rstart; + tsi_debug(OBW, &priv->pdev->dev, + "allocated OBW%d @%llx", obw, ob_win->base); /* * Configure Outbound Window @@ -990,11 +1003,15 @@ static void tsi721_unmap_outb_win(struct rio_mport *mport, struct tsi721_ob_win *ob_win; int i; + tsi_debug(OBW, &priv->pdev->dev, "did=%d ra=0x%llx", destid, rstart); + for (i = 0; i < TSI721_OBWIN_NUM; i++) { ob_win = &priv->ob_win[i]; if (ob_win->active && ob_win->destid == destid && ob_win->rstart == rstart) { + tsi_debug(OBW, &priv->pdev->dev, + "free OBW%d @%llx", i, ob_win->base); ob_win->active = false; iowrite32(0, priv->regs + TSI721_OBWINLB(i)); ob_win->pbar->free += ob_win->size; @@ -1078,15 +1095,15 @@ static int tsi721_rio_map_inb_mem(struct rio_mport *mport, dma_addr_t lstart, int ret = -EBUSY; if (direct) { - dev_dbg(&priv->pdev->dev, - "Direct (RIO_0x%llx -> PCIe_0x%pad), size=0x%x", - rstart, &lstart, size); - /* Calculate minimal acceptable window size and base address */ ibw_size = roundup_pow_of_two(size); ibw_start = lstart & ~(ibw_size - 1); + tsi_debug(IBW, &priv->pdev->dev, + "Direct (RIO_0x%llx -> PCIe_0x%pad), size=0x%x, ibw_start = 0x%llx", + rstart, &lstart, size, ibw_start); + while ((lstart + size) > (ibw_start + ibw_size)) { ibw_size *= 2; ibw_start = lstart & ~(ibw_size - 1); @@ -1102,7 +1119,7 @@ static int tsi721_rio_map_inb_mem(struct rio_mport *mport, dma_addr_t lstart, return -ENOMEM; } else { - dev_dbg(&priv->pdev->dev, + tsi_debug(IBW, &priv->pdev->dev, "Translated (RIO_0x%llx -> PCIe_0x%pad), size=0x%x", rstart, &lstart, size); @@ -1197,9 +1214,9 @@ static int tsi721_rio_map_inb_mem(struct rio_mport *mport, dma_addr_t lstart, priv->ibwin_cnt--; - dev_dbg(&priv->pdev->dev, - "Configured IBWIN%d (RIO_0x%llx -> PCIe_0x%llx), size=0x%llx\n", - i, ibw_start, (unsigned long long)loc_start, ibw_size); + tsi_debug(IBW, &priv->pdev->dev, + "Configured IBWIN%d (RIO_0x%llx -> PCIe_0x%pad), size=0x%llx", + i, ibw_start, &loc_start, ibw_size); return 0; out: @@ -1219,7 +1236,7 @@ static void tsi721_rio_unmap_inb_mem(struct rio_mport *mport, struct tsi721_ib_win *ib_win; int i; - dev_dbg(&priv->pdev->dev, + tsi_debug(IBW, &priv->pdev->dev, "Unmap IBW mapped to PCIe_0x%pad", &lstart); /* Search for matching active inbound translation window */ @@ -1255,7 +1272,7 @@ static void tsi721_rio_unmap_inb_mem(struct rio_mport *mport, break; } - dev_dbg(&priv->pdev->dev, "Disable IBWIN_%d", i); + tsi_debug(IBW, &priv->pdev->dev, "Disable IBWIN_%d", i); iowrite32(0, priv->regs + TSI721_IBWIN_LB(i)); ib_win->active = false; priv->ibwin_cnt++; @@ -1264,7 +1281,7 @@ static void tsi721_rio_unmap_inb_mem(struct rio_mport *mport, } if (i == TSI721_IBWIN_NUM) - dev_dbg(&priv->pdev->dev, + tsi_debug(IBW, &priv->pdev->dev, "IB window mapped to %pad not found", &lstart); } @@ -1319,7 +1336,7 @@ static int tsi721_port_write_init(struct tsi721_device *priv) spin_lock_init(&priv->pw_fifo_lock); if (kfifo_alloc(&priv->pw_fifo, TSI721_RIO_PW_MSG_SIZE * 32, GFP_KERNEL)) { - dev_err(&priv->pdev->dev, "PW FIFO allocation failed\n"); + tsi_err(&priv->pdev->dev, "PW FIFO allocation failed"); return -ENOMEM; } @@ -1351,8 +1368,9 @@ static int tsi721_doorbell_init(struct tsi721_device *priv) if (!priv->idb_base) return -ENOMEM; - dev_dbg(&priv->pdev->dev, "Allocated IDB buffer @ %p (phys = %llx)\n", - priv->idb_base, (unsigned long long)priv->idb_dma); + tsi_debug(DBELL, &priv->pdev->dev, + "Allocated IDB buffer @ %p (phys = %pad)", + priv->idb_base, &priv->idb_dma); iowrite32(TSI721_IDQ_SIZE_VAL(IDB_QSIZE), priv->regs + TSI721_IDQ_SIZE(IDB_QUEUE)); @@ -1398,9 +1416,8 @@ static int tsi721_bdma_maint_init(struct tsi721_device *priv) int bd_num = 2; void __iomem *regs; - dev_dbg(&priv->pdev->dev, - "Init Block DMA Engine for Maintenance requests, CH%d\n", - TSI721_DMACH_MAINT); + tsi_debug(MAINT, &priv->pdev->dev, + "Init BDMA_%d Maintenance requests", TSI721_DMACH_MAINT); /* * Initialize DMA channel for maintenance requests @@ -1420,8 +1437,8 @@ static int tsi721_bdma_maint_init(struct tsi721_device *priv) priv->mdma.bd_phys = bd_phys; priv->mdma.bd_base = bd_ptr; - dev_dbg(&priv->pdev->dev, "DMA descriptors @ %p (phys = %llx)\n", - bd_ptr, (unsigned long long)bd_phys); + tsi_debug(MAINT, &priv->pdev->dev, "DMA descriptors @ %p (phys = %pad)", + bd_ptr, &bd_phys); /* Allocate space for descriptor status FIFO */ sts_size = (bd_num >= TSI721_DMA_MINSTSSZ) ? @@ -1443,9 +1460,9 @@ static int tsi721_bdma_maint_init(struct tsi721_device *priv) priv->mdma.sts_base = sts_ptr; priv->mdma.sts_size = sts_size; - dev_dbg(&priv->pdev->dev, - "desc status FIFO @ %p (phys = %llx) size=0x%x\n", - sts_ptr, (unsigned long long)sts_phys, sts_size); + tsi_debug(MAINT, &priv->pdev->dev, + "desc status FIFO @ %p (phys = %pad) size=0x%x", + sts_ptr, &sts_phys, sts_size); /* Initialize DMA descriptors ring */ bd_ptr[bd_num - 1].type_id = cpu_to_le32(DTYPE3 << 29); @@ -1720,8 +1737,8 @@ static void tsi721_omsg_handler(struct tsi721_device *priv, int ch) omsg_int = ioread32(priv->regs + TSI721_OBDMAC_INT(ch)); if (omsg_int & TSI721_OBDMAC_INT_ST_FULL) - dev_info(&priv->pdev->dev, - "OB MBOX%d: Status FIFO is full\n", ch); + tsi_info(&priv->pdev->dev, + "OB MBOX%d: Status FIFO is full", ch); if (omsg_int & (TSI721_OBDMAC_INT_DONE | TSI721_OBDMAC_INT_IOF_DONE)) { u32 srd_ptr; @@ -1777,7 +1794,7 @@ static void tsi721_omsg_handler(struct tsi721_device *priv, int ch) } if (tx_slot >= priv->omsg_ring[ch].size) - dev_dbg(&priv->pdev->dev, + tsi_debug(OMSG, &priv->pdev->dev, "OB_MSG tx_slot=%x > size=%x", tx_slot, priv->omsg_ring[ch].size); WARN_ON(tx_slot >= priv->omsg_ring[ch].size); @@ -1799,8 +1816,8 @@ no_sts_update: * reinitialize OB MSG channel */ - dev_dbg(&priv->pdev->dev, "OB MSG ABORT ch_stat=%x\n", - ioread32(priv->regs + TSI721_OBDMAC_STS(ch))); + tsi_debug(OMSG, &priv->pdev->dev, "OB MSG ABORT ch_stat=%x", + ioread32(priv->regs + TSI721_OBDMAC_STS(ch))); iowrite32(TSI721_OBDMAC_INT_ERROR, priv->regs + TSI721_OBDMAC_INT(ch)); @@ -1874,9 +1891,8 @@ static int tsi721_open_outb_mbox(struct rio_mport *mport, void *dev_id, &priv->omsg_ring[mbox].omq_phys[i], GFP_KERNEL); if (priv->omsg_ring[mbox].omq_base[i] == NULL) { - dev_dbg(&priv->pdev->dev, - "Unable to allocate OB MSG data buffer for" - " MBOX%d\n", mbox); + tsi_debug(OMSG, &priv->pdev->dev, + "ENOMEM for OB_MSG_%d data buffer", mbox); rc = -ENOMEM; goto out_buf; } @@ -1888,9 +1904,8 @@ static int tsi721_open_outb_mbox(struct rio_mport *mport, void *dev_id, (entries + 1) * sizeof(struct tsi721_omsg_desc), &priv->omsg_ring[mbox].omd_phys, GFP_KERNEL); if (priv->omsg_ring[mbox].omd_base == NULL) { - dev_dbg(&priv->pdev->dev, - "Unable to allocate OB MSG descriptor memory " - "for MBOX%d\n", mbox); + tsi_debug(OMSG, &priv->pdev->dev, + "ENOMEM for OB_MSG_%d descriptor memory", mbox); rc = -ENOMEM; goto out_buf; } @@ -1904,9 +1919,8 @@ static int tsi721_open_outb_mbox(struct rio_mport *mport, void *dev_id, sizeof(struct tsi721_dma_sts), &priv->omsg_ring[mbox].sts_phys, GFP_KERNEL); if (priv->omsg_ring[mbox].sts_base == NULL) { - dev_dbg(&priv->pdev->dev, - "Unable to allocate OB MSG descriptor status FIFO " - "for MBOX%d\n", mbox); + tsi_debug(OMSG, &priv->pdev->dev, + "ENOMEM for OB_MSG_%d status FIFO", mbox); rc = -ENOMEM; goto out_desc; } @@ -1942,9 +1956,9 @@ static int tsi721_open_outb_mbox(struct rio_mport *mport, void *dev_id, priv->msix[idx].irq_name, (void *)priv); if (rc) { - dev_dbg(&priv->pdev->dev, - "Unable to allocate MSI-X interrupt for " - "OBOX%d-DONE\n", mbox); + tsi_debug(OMSG, &priv->pdev->dev, + "Unable to get MSI-X IRQ for OBOX%d-DONE", + mbox); goto out_stat; } @@ -1953,9 +1967,8 @@ static int tsi721_open_outb_mbox(struct rio_mport *mport, void *dev_id, priv->msix[idx].irq_name, (void *)priv); if (rc) { - dev_dbg(&priv->pdev->dev, - "Unable to allocate MSI-X interrupt for " - "MBOX%d-INT\n", mbox); + tsi_debug(OMSG, &priv->pdev->dev, + "Unable to get MSI-X IRQ for MBOX%d-INT", mbox); idx = TSI721_VECT_OMB0_DONE + mbox; free_irq(priv->msix[idx].vector, (void *)priv); goto out_stat; @@ -2096,16 +2109,13 @@ static void tsi721_imsg_handler(struct tsi721_device *priv, int ch) imsg_int = ioread32(priv->regs + TSI721_IBDMAC_INT(ch)); if (imsg_int & TSI721_IBDMAC_INT_SRTO) - dev_info(&priv->pdev->dev, "IB MBOX%d SRIO timeout\n", - mbox); + tsi_info(&priv->pdev->dev, "IB MBOX%d SRIO timeout", mbox); if (imsg_int & TSI721_IBDMAC_INT_PC_ERROR) - dev_info(&priv->pdev->dev, "IB MBOX%d PCIe error\n", - mbox); + tsi_info(&priv->pdev->dev, "IB MBOX%d PCIe error", mbox); if (imsg_int & TSI721_IBDMAC_INT_FQ_LOW) - dev_info(&priv->pdev->dev, - "IB MBOX%d IB free queue low\n", mbox); + tsi_info(&priv->pdev->dev, "IB MBOX%d IB free queue low", mbox); /* Clear IB channel interrupts */ iowrite32(imsg_int, priv->regs + TSI721_IBDMAC_INT(ch)); @@ -2169,8 +2179,8 @@ static int tsi721_open_inb_mbox(struct rio_mport *mport, void *dev_id, GFP_KERNEL); if (priv->imsg_ring[mbox].buf_base == NULL) { - dev_err(&priv->pdev->dev, - "Failed to allocate buffers for IB MBOX%d\n", mbox); + tsi_err(&priv->pdev->dev, + "Failed to allocate buffers for IB MBOX%d", mbox); rc = -ENOMEM; goto out; } @@ -2183,8 +2193,8 @@ static int tsi721_open_inb_mbox(struct rio_mport *mport, void *dev_id, GFP_KERNEL); if (priv->imsg_ring[mbox].imfq_base == NULL) { - dev_err(&priv->pdev->dev, - "Failed to allocate free queue for IB MBOX%d\n", mbox); + tsi_err(&priv->pdev->dev, + "Failed to allocate free queue for IB MBOX%d", mbox); rc = -ENOMEM; goto out_buf; } @@ -2196,8 +2206,8 @@ static int tsi721_open_inb_mbox(struct rio_mport *mport, void *dev_id, &priv->imsg_ring[mbox].imd_phys, GFP_KERNEL); if (priv->imsg_ring[mbox].imd_base == NULL) { - dev_err(&priv->pdev->dev, - "Failed to allocate descriptor memory for IB MBOX%d\n", + tsi_err(&priv->pdev->dev, + "Failed to allocate descriptor memory for IB MBOX%d", mbox); rc = -ENOMEM; goto out_dma; @@ -2256,9 +2266,9 @@ static int tsi721_open_inb_mbox(struct rio_mport *mport, void *dev_id, priv->msix[idx].irq_name, (void *)priv); if (rc) { - dev_dbg(&priv->pdev->dev, - "Unable to allocate MSI-X interrupt for " - "IBOX%d-DONE\n", mbox); + tsi_debug(IMSG, &priv->pdev->dev, + "Unable to get MSI-X IRQ for IBOX%d-DONE", + mbox); goto out_desc; } @@ -2267,9 +2277,8 @@ static int tsi721_open_inb_mbox(struct rio_mport *mport, void *dev_id, priv->msix[idx].irq_name, (void *)priv); if (rc) { - dev_dbg(&priv->pdev->dev, - "Unable to allocate MSI-X interrupt for " - "IBOX%d-INT\n", mbox); + tsi_debug(IMSG, &priv->pdev->dev, + "Unable to get MSI-X IRQ for IBOX%d-INT", mbox); free_irq( priv->msix[TSI721_VECT_IMB0_RCV + mbox].vector, (void *)priv); @@ -2392,8 +2401,8 @@ static int tsi721_add_inb_buffer(struct rio_mport *mport, int mbox, void *buf) rx_slot = priv->imsg_ring[mbox].rx_slot; if (priv->imsg_ring[mbox].imq_base[rx_slot]) { - dev_err(&priv->pdev->dev, - "Error adding inbound buffer %d, buffer exists\n", + tsi_err(&priv->pdev->dev, + "Error adding inbound buffer %d, buffer exists", rx_slot); rc = -EINVAL; goto out; @@ -2619,7 +2628,7 @@ static void tsi721_mport_release(struct device *dev) { struct rio_mport *mport = to_rio_mport(dev); - dev_dbg(dev, "RIO: %s %s id=%d\n", __func__, mport->name, mport->id); + tsi_debug(EXIT, dev, "%s id=%d", mport->name, mport->id); } /** @@ -2663,15 +2672,15 @@ static int tsi721_setup_mport(struct tsi721_device *priv) else if (!pci_enable_msi(pdev)) priv->flags |= TSI721_USING_MSI; else - dev_info(&pdev->dev, - "MSI/MSI-X is not available. Using legacy INTx.\n"); + tsi_debug(MPORT, &pdev->dev, + "MSI/MSI-X is not available. Using legacy INTx."); #endif /* CONFIG_PCI_MSI */ err = tsi721_request_irq(priv); if (err) { - dev_err(&pdev->dev, "Unable to get assigned PCI IRQ " - "vector %02X err=0x%x\n", pdev->irq, err); + tsi_err(&pdev->dev, "Unable to get PCI IRQ %02X (err=0x%x)", + pdev->irq, err); return err; } @@ -2712,15 +2721,14 @@ static int tsi721_probe(struct pci_dev *pdev, int err; priv = kzalloc(sizeof(struct tsi721_device), GFP_KERNEL); - if (priv == NULL) { - dev_err(&pdev->dev, "Failed to allocate memory for device\n"); + if (!priv) { err = -ENOMEM; goto err_exit; } err = pci_enable_device(pdev); if (err) { - dev_err(&pdev->dev, "Failed to enable PCI device\n"); + tsi_err(&pdev->dev, "Failed to enable PCI device"); goto err_clean; } @@ -2728,13 +2736,12 @@ static int tsi721_probe(struct pci_dev *pdev, #ifdef DEBUG { - int i; - for (i = 0; i <= PCI_STD_RESOURCE_END; i++) { - dev_dbg(&pdev->dev, "res[%d] @ 0x%llx (0x%lx, 0x%lx)\n", - i, (unsigned long long)pci_resource_start(pdev, i), - (unsigned long)pci_resource_len(pdev, i), - pci_resource_flags(pdev, i)); - } + int i; + + for (i = 0; i <= PCI_STD_RESOURCE_END; i++) { + tsi_debug(INIT, &pdev->dev, "res%d %pR", + i, &pdev->resource[i]); + } } #endif /* @@ -2745,8 +2752,7 @@ static int tsi721_probe(struct pci_dev *pdev, if (!(pci_resource_flags(pdev, BAR_0) & IORESOURCE_MEM) || pci_resource_flags(pdev, BAR_0) & IORESOURCE_MEM_64 || pci_resource_len(pdev, BAR_0) < TSI721_REG_SPACE_SIZE) { - dev_err(&pdev->dev, - "Missing or misconfigured CSR BAR0, aborting.\n"); + tsi_err(&pdev->dev, "Missing or misconfigured CSR BAR0"); err = -ENODEV; goto err_disable_pdev; } @@ -2755,8 +2761,7 @@ static int tsi721_probe(struct pci_dev *pdev, if (!(pci_resource_flags(pdev, BAR_1) & IORESOURCE_MEM) || pci_resource_flags(pdev, BAR_1) & IORESOURCE_MEM_64 || pci_resource_len(pdev, BAR_1) < TSI721_DB_WIN_SIZE) { - dev_err(&pdev->dev, - "Missing or misconfigured Doorbell BAR1, aborting.\n"); + tsi_err(&pdev->dev, "Missing or misconfigured Doorbell BAR1"); err = -ENODEV; goto err_disable_pdev; } @@ -2773,8 +2778,8 @@ static int tsi721_probe(struct pci_dev *pdev, if (pci_resource_flags(pdev, BAR_2) & IORESOURCE_MEM_64) { if (pci_resource_flags(pdev, BAR_2) & IORESOURCE_PREFETCH) - dev_info(&pdev->dev, - "Prefetchable OBW BAR2 will not be used\n"); + tsi_debug(INIT, &pdev->dev, + "Prefetchable OBW BAR2 will not be used"); else { priv->p2r_bar[0].base = pci_resource_start(pdev, BAR_2); priv->p2r_bar[0].size = pci_resource_len(pdev, BAR_2); @@ -2783,8 +2788,8 @@ static int tsi721_probe(struct pci_dev *pdev, if (pci_resource_flags(pdev, BAR_4) & IORESOURCE_MEM_64) { if (pci_resource_flags(pdev, BAR_4) & IORESOURCE_PREFETCH) - dev_info(&pdev->dev, - "Prefetchable OBW BAR4 will not be used\n"); + tsi_debug(INIT, &pdev->dev, + "Prefetchable OBW BAR4 will not be used"); else { priv->p2r_bar[1].base = pci_resource_start(pdev, BAR_4); priv->p2r_bar[1].size = pci_resource_len(pdev, BAR_4); @@ -2793,8 +2798,7 @@ static int tsi721_probe(struct pci_dev *pdev, err = pci_request_regions(pdev, DRV_NAME); if (err) { - dev_err(&pdev->dev, "Cannot obtain PCI resources, " - "aborting.\n"); + tsi_err(&pdev->dev, "Unable to obtain PCI resources"); goto err_disable_pdev; } @@ -2802,16 +2806,14 @@ static int tsi721_probe(struct pci_dev *pdev, priv->regs = pci_ioremap_bar(pdev, BAR_0); if (!priv->regs) { - dev_err(&pdev->dev, - "Unable to map device registers space, aborting\n"); + tsi_err(&pdev->dev, "Unable to map device registers space"); err = -ENOMEM; goto err_free_res; } priv->odb_base = pci_ioremap_bar(pdev, BAR_1); if (!priv->odb_base) { - dev_err(&pdev->dev, - "Unable to map outbound doorbells space, aborting\n"); + tsi_err(&pdev->dev, "Unable to map outbound doorbells space"); err = -ENOMEM; goto err_unmap_bars; } @@ -2820,16 +2822,16 @@ static int tsi721_probe(struct pci_dev *pdev, if (pci_set_dma_mask(pdev, DMA_BIT_MASK(64))) { err = pci_set_dma_mask(pdev, DMA_BIT_MASK(32)); if (err) { - dev_info(&pdev->dev, "Unable to set DMA mask\n"); + tsi_err(&pdev->dev, "Unable to set DMA mask"); goto err_unmap_bars; } if (pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32))) - dev_info(&pdev->dev, "Unable to set consistent DMA mask\n"); + tsi_info(&pdev->dev, "Unable to set consistent DMA mask"); } else { err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64)); if (err) - dev_info(&pdev->dev, "Unable to set consistent DMA mask\n"); + tsi_info(&pdev->dev, "Unable to set consistent DMA mask"); } BUG_ON(!pci_is_pcie(pdev)); @@ -2858,7 +2860,7 @@ static int tsi721_probe(struct pci_dev *pdev, tsi721_init_sr2pc_mapping(priv); if (tsi721_bdma_maint_init(priv)) { - dev_err(&pdev->dev, "BDMA initialization failed, aborting\n"); + tsi_err(&pdev->dev, "BDMA initialization failed"); err = -ENOMEM; goto err_unmap_bars; } @@ -2907,7 +2909,7 @@ static void tsi721_remove(struct pci_dev *pdev) { struct tsi721_device *priv = pci_get_drvdata(pdev); - dev_dbg(&pdev->dev, "%s enter\n", __func__); + tsi_debug(EXIT, &pdev->dev, "enter"); tsi721_disable_ints(priv); tsi721_free_irq(priv); @@ -2935,14 +2937,14 @@ static void tsi721_remove(struct pci_dev *pdev) pci_disable_device(pdev); pci_set_drvdata(pdev, NULL); kfree(priv); - dev_dbg(&pdev->dev, "%s exit\n", __func__); + tsi_debug(EXIT, &pdev->dev, "exit"); } static void tsi721_shutdown(struct pci_dev *pdev) { struct tsi721_device *priv = pci_get_drvdata(pdev); - dev_dbg(&pdev->dev, "RIO: %s\n", __func__); + tsi_debug(EXIT, &pdev->dev, "enter"); tsi721_disable_ints(priv); tsi721_dma_stop_all(priv); diff --git a/drivers/rapidio/devices/tsi721.h b/drivers/rapidio/devices/tsi721.h index 57b46d025630..5456dbddc929 100644 --- a/drivers/rapidio/devices/tsi721.h +++ b/drivers/rapidio/devices/tsi721.h @@ -21,6 +21,46 @@ #ifndef __TSI721_H #define __TSI721_H +/* Debug output filtering masks */ +enum { + DBG_NONE = 0, + DBG_INIT = BIT(0), /* driver init */ + DBG_EXIT = BIT(1), /* driver exit */ + DBG_MPORT = BIT(2), /* mport add/remove */ + DBG_MAINT = BIT(3), /* maintenance ops messages */ + DBG_DMA = BIT(4), /* DMA transfer messages */ + DBG_DMAV = BIT(5), /* verbose DMA transfer messages */ + DBG_IBW = BIT(6), /* inbound window */ + DBG_EVENT = BIT(7), /* event handling messages */ + DBG_OBW = BIT(8), /* outbound window messages */ + DBG_DBELL = BIT(9), /* doorbell messages */ + DBG_OMSG = BIT(10), /* doorbell messages */ + DBG_IMSG = BIT(11), /* doorbell messages */ + DBG_ALL = ~0, +}; + +#ifdef DEBUG +extern u32 dbg_level; + +#define tsi_debug(level, dev, fmt, arg...) \ + do { \ + if (DBG_##level & dbg_level) \ + dev_dbg(dev, "%s: " fmt "\n", __func__, ##arg); \ + } while (0) +#else +#define tsi_debug(level, dev, fmt, arg...) \ + no_printk(KERN_DEBUG "%s: " fmt "\n", __func__, ##arg) +#endif + +#define tsi_info(dev, fmt, arg...) \ + dev_info(dev, "%s: " fmt "\n", __func__, ##arg) + +#define tsi_warn(dev, fmt, arg...) \ + dev_warn(dev, "%s: WARNING " fmt "\n", __func__, ##arg) + +#define tsi_err(dev, fmt, arg...) \ + dev_err(dev, "%s: ERROR " fmt "\n", __func__, ##arg) + #define DRV_NAME "tsi721" #define DEFAULT_HOPCOUNT 0xff diff --git a/drivers/rapidio/devices/tsi721_dma.c b/drivers/rapidio/devices/tsi721_dma.c index 31bb61b43c3f..494482e4ab0a 100644 --- a/drivers/rapidio/devices/tsi721_dma.c +++ b/drivers/rapidio/devices/tsi721_dma.c @@ -30,6 +30,7 @@ #include #include #include +#include #include #include "../../dma/dmaengine.h" @@ -75,7 +76,7 @@ static int tsi721_bdma_ch_init(struct tsi721_bdma_chan *bdma_chan, int bd_num) struct tsi721_device *priv = to_tsi721(bdma_chan->dchan.device); #endif - dev_dbg(dev, "Init Block DMA Engine, CH%d\n", bdma_chan->id); + tsi_debug(DMA, &bdma_chan->dchan.dev->device, "DMAC%d", bdma_chan->id); /* * Allocate space for DMA descriptors @@ -91,8 +92,9 @@ static int tsi721_bdma_ch_init(struct tsi721_bdma_chan *bdma_chan, int bd_num) bdma_chan->bd_phys = bd_phys; bdma_chan->bd_base = bd_ptr; - dev_dbg(dev, "DMA descriptors @ %p (phys = %llx)\n", - bd_ptr, (unsigned long long)bd_phys); + tsi_debug(DMA, &bdma_chan->dchan.dev->device, + "DMAC%d descriptors @ %p (phys = %pad)", + bdma_chan->id, bd_ptr, &bd_phys); /* Allocate space for descriptor status FIFO */ sts_size = ((bd_num + 1) >= TSI721_DMA_MINSTSSZ) ? @@ -114,9 +116,9 @@ static int tsi721_bdma_ch_init(struct tsi721_bdma_chan *bdma_chan, int bd_num) bdma_chan->sts_base = sts_ptr; bdma_chan->sts_size = sts_size; - dev_dbg(dev, - "desc status FIFO @ %p (phys = %llx) size=0x%x\n", - sts_ptr, (unsigned long long)sts_phys, sts_size); + tsi_debug(DMA, &bdma_chan->dchan.dev->device, + "DMAC%d desc status FIFO @ %p (phys = %pad) size=0x%x", + bdma_chan->id, sts_ptr, &sts_phys, sts_size); /* Initialize DMA descriptors ring using added link descriptor */ bd_ptr[bd_num].type_id = cpu_to_le32(DTYPE3 << 29); @@ -155,8 +157,9 @@ static int tsi721_bdma_ch_init(struct tsi721_bdma_chan *bdma_chan, int bd_num) priv->msix[idx].irq_name, (void *)bdma_chan); if (rc) { - dev_dbg(dev, "Unable to get MSI-X for BDMA%d-DONE\n", - bdma_chan->id); + tsi_debug(DMA, &bdma_chan->dchan.dev->device, + "Unable to get MSI-X for DMAC%d-DONE", + bdma_chan->id); goto err_out; } @@ -166,8 +169,9 @@ static int tsi721_bdma_ch_init(struct tsi721_bdma_chan *bdma_chan, int bd_num) priv->msix[idx].irq_name, (void *)bdma_chan); if (rc) { - dev_dbg(dev, "Unable to get MSI-X for BDMA%d-INT\n", - bdma_chan->id); + tsi_debug(DMA, &bdma_chan->dchan.dev->device, + "Unable to get MSI-X for DMAC%d-INT", + bdma_chan->id); free_irq( priv->msix[TSI721_VECT_DMA0_DONE + bdma_chan->id].vector, @@ -302,20 +306,22 @@ static irqreturn_t tsi721_bdma_msix(int irq, void *ptr) static void tsi721_start_dma(struct tsi721_bdma_chan *bdma_chan) { if (!tsi721_dma_is_idle(bdma_chan)) { - dev_err(bdma_chan->dchan.device->dev, - "BUG: Attempt to start non-idle channel\n"); + tsi_err(&bdma_chan->dchan.dev->device, + "DMAC%d Attempt to start non-idle channel", + bdma_chan->id); return; } if (bdma_chan->wr_count == bdma_chan->wr_count_next) { - dev_err(bdma_chan->dchan.device->dev, - "BUG: Attempt to start DMA with no BDs ready\n"); + tsi_err(&bdma_chan->dchan.dev->device, + "DMAC%d Attempt to start DMA with no BDs ready %d", + bdma_chan->id, task_pid_nr(current)); return; } - dev_dbg(bdma_chan->dchan.device->dev, - "%s: chan_%d (wrc=%d)\n", __func__, bdma_chan->id, - bdma_chan->wr_count_next); + tsi_debug(DMA, &bdma_chan->dchan.dev->device, "DMAC%d (wrc=%d) %d", + bdma_chan->id, bdma_chan->wr_count_next, + task_pid_nr(current)); iowrite32(bdma_chan->wr_count_next, bdma_chan->regs + TSI721_DMAC_DWRCNT); @@ -417,10 +423,11 @@ static int tsi721_submit_sg(struct tsi721_tx_desc *desc) struct tsi721_dma_desc *bd_ptr = NULL; u32 idx, rd_idx; u32 add_count = 0; + struct device *ch_dev = &dchan->dev->device; if (!tsi721_dma_is_idle(bdma_chan)) { - dev_err(bdma_chan->dchan.device->dev, - "BUG: Attempt to use non-idle channel\n"); + tsi_err(ch_dev, "DMAC%d ERR: Attempt to use non-idle channel", + bdma_chan->id); return -EIO; } @@ -431,7 +438,7 @@ static int tsi721_submit_sg(struct tsi721_tx_desc *desc) rio_addr = desc->rio_addr; next_addr = -1; bcount = 0; - sys_size = dma_to_mport(bdma_chan->dchan.device)->sys_size; + sys_size = dma_to_mport(dchan->device)->sys_size; rd_idx = ioread32(bdma_chan->regs + TSI721_DMAC_DRDCNT); rd_idx %= (bdma_chan->bd_num + 1); @@ -443,18 +450,18 @@ static int tsi721_submit_sg(struct tsi721_tx_desc *desc) add_count++; } - dev_dbg(dchan->device->dev, "%s: BD ring status: rdi=%d wri=%d\n", - __func__, rd_idx, idx); + tsi_debug(DMA, ch_dev, "DMAC%d BD ring status: rdi=%d wri=%d", + bdma_chan->id, rd_idx, idx); for_each_sg(desc->sg, sg, desc->sg_len, i) { - dev_dbg(dchan->device->dev, "sg%d/%d addr: 0x%llx len: %d\n", - i, desc->sg_len, + tsi_debug(DMAV, ch_dev, "DMAC%d sg%d/%d addr: 0x%llx len: %d", + bdma_chan->id, i, desc->sg_len, (unsigned long long)sg_dma_address(sg), sg_dma_len(sg)); if (sg_dma_len(sg) > TSI721_BDMA_MAX_BCOUNT) { - dev_err(dchan->device->dev, - "%s: SG entry %d is too large\n", __func__, i); + tsi_err(ch_dev, "DMAC%d SG entry %d is too large", + bdma_chan->id, i); err = -EINVAL; break; } @@ -471,17 +478,16 @@ static int tsi721_submit_sg(struct tsi721_tx_desc *desc) } else if (next_addr != -1) { /* Finalize descriptor using total byte count value */ tsi721_desc_fill_end(bd_ptr, bcount, 0); - dev_dbg(dchan->device->dev, - "%s: prev desc final len: %d\n", - __func__, bcount); + tsi_debug(DMAV, ch_dev, "DMAC%d prev desc final len: %d", + bdma_chan->id, bcount); } desc->rio_addr = rio_addr; if (i && idx == rd_idx) { - dev_dbg(dchan->device->dev, - "%s: HW descriptor ring is full @ %d\n", - __func__, i); + tsi_debug(DMAV, ch_dev, + "DMAC%d HW descriptor ring is full @ %d", + bdma_chan->id, i); desc->sg = sg; desc->sg_len -= i; break; @@ -490,13 +496,12 @@ static int tsi721_submit_sg(struct tsi721_tx_desc *desc) bd_ptr = &((struct tsi721_dma_desc *)bdma_chan->bd_base)[idx]; err = tsi721_desc_fill_init(desc, bd_ptr, sg, sys_size); if (err) { - dev_err(dchan->device->dev, - "Failed to build desc: err=%d\n", err); + tsi_err(ch_dev, "Failed to build desc: err=%d", err); break; } - dev_dbg(dchan->device->dev, "bd_ptr = %p did=%d raddr=0x%llx\n", - bd_ptr, desc->destid, desc->rio_addr); + tsi_debug(DMAV, ch_dev, "DMAC%d bd_ptr = %p did=%d raddr=0x%llx", + bdma_chan->id, bd_ptr, desc->destid, desc->rio_addr); next_addr = sg_dma_address(sg); bcount = sg_dma_len(sg); @@ -511,8 +516,9 @@ static int tsi721_submit_sg(struct tsi721_tx_desc *desc) entry_done: if (sg_is_last(sg)) { tsi721_desc_fill_end(bd_ptr, bcount, 0); - dev_dbg(dchan->device->dev, "%s: last desc final len: %d\n", - __func__, bcount); + tsi_debug(DMAV, ch_dev, + "DMAC%d last desc final len: %d", + bdma_chan->id, bcount); desc->sg_len = 0; } else { rio_addr += sg_dma_len(sg); @@ -531,7 +537,7 @@ static void tsi721_advance_work(struct tsi721_bdma_chan *bdma_chan, { int err; - dev_dbg(bdma_chan->dchan.device->dev, "%s: Enter\n", __func__); + tsi_debug(DMA, &bdma_chan->dchan.dev->device, "DMAC%d", bdma_chan->id); if (!tsi721_dma_is_idle(bdma_chan)) return; @@ -555,13 +561,14 @@ static void tsi721_advance_work(struct tsi721_bdma_chan *bdma_chan, tsi721_start_dma(bdma_chan); else { tsi721_dma_tx_err(bdma_chan, desc); - dev_dbg(bdma_chan->dchan.device->dev, - "ERR: tsi721_submit_sg failed with err=%d\n", - err); + tsi_debug(DMA, &bdma_chan->dchan.dev->device, + "DMAC%d ERR: tsi721_submit_sg failed with err=%d", + bdma_chan->id, err); } } - dev_dbg(bdma_chan->dchan.device->dev, "%s: Exit\n", __func__); + tsi_debug(DMA, &bdma_chan->dchan.dev->device, "DMAC%d Exit", + bdma_chan->id); } static void tsi721_dma_tasklet(unsigned long data) @@ -570,16 +577,16 @@ static void tsi721_dma_tasklet(unsigned long data) u32 dmac_int, dmac_sts; dmac_int = ioread32(bdma_chan->regs + TSI721_DMAC_INT); - dev_dbg(bdma_chan->dchan.device->dev, "%s: DMAC%d_INT = 0x%x\n", - __func__, bdma_chan->id, dmac_int); + tsi_debug(DMA, &bdma_chan->dchan.dev->device, "DMAC%d_INT = 0x%x", + bdma_chan->id, dmac_int); /* Clear channel interrupts */ iowrite32(dmac_int, bdma_chan->regs + TSI721_DMAC_INT); if (dmac_int & TSI721_DMAC_INT_ERR) { dmac_sts = ioread32(bdma_chan->regs + TSI721_DMAC_STS); - dev_err(bdma_chan->dchan.device->dev, - "%s: DMA ERROR - DMAC%d_STS = 0x%x\n", - __func__, bdma_chan->id, dmac_sts); + tsi_err(&bdma_chan->dchan.dev->device, + "ERR - DMAC%d_STS = 0x%x", + bdma_chan->id, dmac_sts); spin_lock(&bdma_chan->lock); bdma_chan->active_tx = NULL; @@ -587,9 +594,9 @@ static void tsi721_dma_tasklet(unsigned long data) } if (dmac_int & TSI721_DMAC_INT_STFULL) { - dev_err(bdma_chan->dchan.device->dev, - "%s: DMAC%d descriptor status FIFO is full\n", - __func__, bdma_chan->id); + tsi_err(&bdma_chan->dchan.dev->device, + "DMAC%d descriptor status FIFO is full", + bdma_chan->id); } if (dmac_int & (TSI721_DMAC_INT_DONE | TSI721_DMAC_INT_IOFDONE)) { @@ -633,8 +640,9 @@ static dma_cookie_t tsi721_tx_submit(struct dma_async_tx_descriptor *txd) /* Check if the descriptor is detached from any lists */ if (!list_empty(&desc->desc_node)) { - dev_err(bdma_chan->dchan.device->dev, - "%s: wrong state of descriptor %p\n", __func__, txd); + tsi_err(&bdma_chan->dchan.dev->device, + "DMAC%d wrong state of descriptor %p", + bdma_chan->id, txd); return -EIO; } @@ -659,16 +667,15 @@ static int tsi721_alloc_chan_resources(struct dma_chan *dchan) struct tsi721_tx_desc *desc = NULL; int i; - dev_dbg(dchan->device->dev, "%s: for channel %d\n", - __func__, bdma_chan->id); + tsi_debug(DMA, &dchan->dev->device, "DMAC%d", bdma_chan->id); if (bdma_chan->bd_base) return TSI721_DMA_TX_QUEUE_SZ; /* Initialize BDMA channel */ if (tsi721_bdma_ch_init(bdma_chan, dma_desc_per_channel)) { - dev_err(dchan->device->dev, "Unable to initialize data DMA" - " channel %d, aborting\n", bdma_chan->id); + tsi_err(&dchan->dev->device, "Unable to initialize DMAC%d", + bdma_chan->id); return -ENODEV; } @@ -676,8 +683,9 @@ static int tsi721_alloc_chan_resources(struct dma_chan *dchan) desc = kcalloc(TSI721_DMA_TX_QUEUE_SZ, sizeof(struct tsi721_tx_desc), GFP_KERNEL); if (!desc) { - dev_err(dchan->device->dev, - "Failed to allocate logical descriptors\n"); + tsi_err(&dchan->dev->device, + "DMAC%d Failed to allocate logical descriptors", + bdma_chan->id); tsi721_bdma_ch_free(bdma_chan); return -ENOMEM; } @@ -718,8 +726,7 @@ static void tsi721_free_chan_resources(struct dma_chan *dchan) { struct tsi721_bdma_chan *bdma_chan = to_tsi721_chan(dchan); - dev_dbg(dchan->device->dev, "%s: for channel %d\n", - __func__, bdma_chan->id); + tsi_debug(DMA, &dchan->dev->device, "DMAC%d", bdma_chan->id); if (bdma_chan->bd_base == NULL) return; @@ -744,7 +751,7 @@ static void tsi721_issue_pending(struct dma_chan *dchan) { struct tsi721_bdma_chan *bdma_chan = to_tsi721_chan(dchan); - dev_dbg(dchan->device->dev, "%s: Enter\n", __func__); + tsi_debug(DMA, &dchan->dev->device, "DMAC%d", bdma_chan->id); spin_lock_bh(&bdma_chan->lock); if (tsi721_dma_is_idle(bdma_chan) && bdma_chan->active) { @@ -766,12 +773,13 @@ struct dma_async_tx_descriptor *tsi721_prep_rio_sg(struct dma_chan *dchan, struct dma_async_tx_descriptor *txd = NULL; if (!sgl || !sg_len) { - dev_err(dchan->device->dev, "%s: No SG list\n", __func__); + tsi_err(&dchan->dev->device, "DMAC%d No SG list", + bdma_chan->id); return NULL; } - dev_dbg(dchan->device->dev, "%s: %s\n", __func__, - (dir == DMA_DEV_TO_MEM)?"READ":"WRITE"); + tsi_debug(DMA, &dchan->dev->device, "DMAC%d %s", bdma_chan->id, + (dir == DMA_DEV_TO_MEM)?"READ":"WRITE"); if (dir == DMA_DEV_TO_MEM) rtype = NREAD; @@ -789,8 +797,9 @@ struct dma_async_tx_descriptor *tsi721_prep_rio_sg(struct dma_chan *dchan, break; } } else { - dev_err(dchan->device->dev, - "%s: Unsupported DMA direction option\n", __func__); + tsi_err(&dchan->dev->device, + "DMAC%d Unsupported DMA direction option", + bdma_chan->id); return NULL; } @@ -823,7 +832,7 @@ static int tsi721_terminate_all(struct dma_chan *dchan) u32 dmac_int; LIST_HEAD(list); - dev_dbg(dchan->device->dev, "%s: Entry\n", __func__); + tsi_debug(DMA, &dchan->dev->device, "DMAC%d", bdma_chan->id); spin_lock_bh(&bdma_chan->lock); @@ -933,7 +942,7 @@ int tsi721_register_dma(struct tsi721_device *priv) err = dma_async_device_register(&mport->dma); if (err) - dev_err(&priv->pdev->dev, "Failed to register DMA device\n"); + tsi_err(&priv->pdev->dev, "Failed to register DMA device"); return err; } -- cgit v1.2.3-59-g8ed1b From e8de370188d098bb49483c287b44925957c3c9b6 Mon Sep 17 00:00:00 2001 From: Alexandre Bounine Date: Tue, 22 Mar 2016 14:27:08 -0700 Subject: rapidio: add mport char device driver Add mport character device driver to provide user space interface to basic RapidIO subsystem operations. See included Documentation/rapidio/mport_cdev.txt for more details. [akpm@linux-foundation.org: fix printk warning on i386] [dan.carpenter@oracle.com: mport_cdev: fix some error codes] Signed-off-by: Alexandre Bounine Signed-off-by: Dan Carpenter Tested-by: Barry Wood Cc: Matt Porter Cc: Aurelien Jacquiot Cc: Andre van Herk Cc: Barry Wood Cc: Randy Dunlap Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- Documentation/rapidio/mport_cdev.txt | 104 ++ drivers/rapidio/Kconfig | 8 + drivers/rapidio/devices/Makefile | 1 + drivers/rapidio/devices/rio_mport_cdev.c | 2720 ++++++++++++++++++++++++++++++ include/linux/rio_mport_cdev.h | 271 +++ include/uapi/linux/Kbuild | 1 + 6 files changed, 3105 insertions(+) create mode 100644 Documentation/rapidio/mport_cdev.txt create mode 100644 drivers/rapidio/devices/rio_mport_cdev.c create mode 100644 include/linux/rio_mport_cdev.h (limited to 'Documentation') diff --git a/Documentation/rapidio/mport_cdev.txt b/Documentation/rapidio/mport_cdev.txt new file mode 100644 index 000000000000..20c120d4b3b8 --- /dev/null +++ b/Documentation/rapidio/mport_cdev.txt @@ -0,0 +1,104 @@ +RapidIO subsystem mport character device driver (rio_mport_cdev.c) +================================================================== + +Version History: +---------------- + 1.0.0 - Initial driver release. + +================================================================== + +I. Overview + +This device driver is the result of collaboration within the RapidIO.org +Software Task Group (STG) between Texas Instruments, Freescale, +Prodrive Technologies, Nokia Networks, BAE and IDT. Additional input was +received from other members of RapidIO.org. The objective was to create a +character mode driver interface which exposes the capabilities of RapidIO +devices directly to applications, in a manner that allows the numerous and +varied RapidIO implementations to interoperate. + +This driver (MPORT_CDEV) provides access to basic RapidIO subsystem operations +for user-space applications. Most of RapidIO operations are supported through +'ioctl' system calls. + +When loaded this device driver creates filesystem nodes named rio_mportX in /dev +directory for each registered RapidIO mport device. 'X' in the node name matches +to unique port ID assigned to each local mport device. + +Using available set of ioctl commands user-space applications can perform +following RapidIO bus and subsystem operations: + +- Reads and writes from/to configuration registers of mport devices + (RIO_MPORT_MAINT_READ_LOCAL/RIO_MPORT_MAINT_WRITE_LOCAL) +- Reads and writes from/to configuration registers of remote RapidIO devices. + This operations are defined as RapidIO Maintenance reads/writes in RIO spec. + (RIO_MPORT_MAINT_READ_REMOTE/RIO_MPORT_MAINT_WRITE_REMOTE) +- Set RapidIO Destination ID for mport devices (RIO_MPORT_MAINT_HDID_SET) +- Set RapidIO Component Tag for mport devices (RIO_MPORT_MAINT_COMPTAG_SET) +- Query logical index of mport devices (RIO_MPORT_MAINT_PORT_IDX_GET) +- Query capabilities and RapidIO link configuration of mport devices + (RIO_MPORT_GET_PROPERTIES) +- Enable/Disable reporting of RapidIO doorbell events to user-space applications + (RIO_ENABLE_DOORBELL_RANGE/RIO_DISABLE_DOORBELL_RANGE) +- Enable/Disable reporting of RIO port-write events to user-space applications + (RIO_ENABLE_PORTWRITE_RANGE/RIO_DISABLE_PORTWRITE_RANGE) +- Query/Control type of events reported through this driver: doorbells, + port-writes or both (RIO_SET_EVENT_MASK/RIO_GET_EVENT_MASK) +- Configure/Map mport's outbound requests window(s) for specific size, + RapidIO destination ID, hopcount and request type + (RIO_MAP_OUTBOUND/RIO_UNMAP_OUTBOUND) +- Configure/Map mport's inbound requests window(s) for specific size, + RapidIO base address and local memory base address + (RIO_MAP_INBOUND/RIO_UNMAP_INBOUND) +- Allocate/Free contiguous DMA coherent memory buffer for DMA data transfers + to/from remote RapidIO devices (RIO_ALLOC_DMA/RIO_FREE_DMA) +- Initiate DMA data transfers to/from remote RapidIO devices (RIO_TRANSFER). + Supports blocking, asynchronous and posted (a.k.a 'fire-and-forget') data + transfer modes. +- Check/Wait for completion of asynchronous DMA data transfer + (RIO_WAIT_FOR_ASYNC) +- Manage device objects supported by RapidIO subsystem (RIO_DEV_ADD/RIO_DEV_DEL). + This allows implementation of various RapidIO fabric enumeration algorithms + as user-space applications while using remaining functionality provided by + kernel RapidIO subsystem. + +II. Hardware Compatibility + +This device driver uses standard interfaces defined by kernel RapidIO subsystem +and therefore it can be used with any mport device driver registered by RapidIO +subsystem with limitations set by available mport implementation. + +At this moment the most common limitation is availability of RapidIO-specific +DMA engine framework for specific mport device. Users should verify available +functionality of their platform when planning to use this driver: + +- IDT Tsi721 PCIe-to-RapidIO bridge device and its mport device driver are fully + compatible with this driver. +- Freescale SoCs 'fsl_rio' mport driver does not have implementation for RapidIO + specific DMA engine support and therefore DMA data transfers mport_cdev driver + are not available. + +III. Module parameters + +- 'dbg_level' - This parameter allows to control amount of debug information + generated by this device driver. This parameter is formed by set of + This parameter can be changed bit masks that correspond to the specific + functional block. + For mask definitions see 'drivers/rapidio/devices/rio_mport_cdev.c' + This parameter can be changed dynamically. + Use CONFIG_RAPIDIO_DEBUG=y to enable debug output at the top level. + +IV. Known problems + + None. + +V. User-space Applications and API + +API library and applications that use this device driver are available from +RapidIO.org. + +VI. TODO List + +- Add support for sending/receiving "raw" RapidIO messaging packets. +- Add memory mapped DMA data transfers as an option when RapidIO-specific DMA + is not available. diff --git a/drivers/rapidio/Kconfig b/drivers/rapidio/Kconfig index 3e3be57e9a1a..b5a10d3c92c7 100644 --- a/drivers/rapidio/Kconfig +++ b/drivers/rapidio/Kconfig @@ -67,6 +67,14 @@ config RAPIDIO_ENUM_BASIC endchoice +config RAPIDIO_MPORT_CDEV + tristate "RapidIO /dev mport device driver" + depends on RAPIDIO + help + This option includes generic RapidIO mport device driver which + allows to user space applications to perform RapidIO-specific + operations through selected RapidIO mport. + menu "RapidIO Switch drivers" depends on RAPIDIO diff --git a/drivers/rapidio/devices/Makefile b/drivers/rapidio/devices/Makefile index 9432c494cf57..927dbf89592b 100644 --- a/drivers/rapidio/devices/Makefile +++ b/drivers/rapidio/devices/Makefile @@ -5,3 +5,4 @@ obj-$(CONFIG_RAPIDIO_TSI721) += tsi721_mport.o tsi721_mport-y := tsi721.o tsi721_mport-$(CONFIG_RAPIDIO_DMA_ENGINE) += tsi721_dma.o +obj-$(CONFIG_RAPIDIO_MPORT_CDEV) += rio_mport_cdev.o diff --git a/drivers/rapidio/devices/rio_mport_cdev.c b/drivers/rapidio/devices/rio_mport_cdev.c new file mode 100644 index 000000000000..9607bc826460 --- /dev/null +++ b/drivers/rapidio/devices/rio_mport_cdev.c @@ -0,0 +1,2720 @@ +/* + * RapidIO mport character device + * + * Copyright 2014-2015 Integrated Device Technology, Inc. + * Alexandre Bounine + * Copyright 2014-2015 Prodrive Technologies + * Andre van Herk + * Jerry Jacobs + * Copyright (C) 2014 Texas Instruments Incorporated + * Aurelien Jacquiot + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the + * Free Software Foundation; either version 2 of the License, or (at your + * option) any later version. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include +#ifdef CONFIG_RAPIDIO_DMA_ENGINE +#include +#endif + +#include +#include +#include +#include + +#include "../rio.h" + +#define DRV_NAME "rio_mport" +#define DRV_PREFIX DRV_NAME ": " +#define DEV_NAME "rio_mport" +#define DRV_VERSION "1.0.0" + +/* Debug output filtering masks */ +enum { + DBG_NONE = 0, + DBG_INIT = BIT(0), /* driver init */ + DBG_EXIT = BIT(1), /* driver exit */ + DBG_MPORT = BIT(2), /* mport add/remove */ + DBG_RDEV = BIT(3), /* RapidIO device add/remove */ + DBG_DMA = BIT(4), /* DMA transfer messages */ + DBG_MMAP = BIT(5), /* mapping messages */ + DBG_IBW = BIT(6), /* inbound window */ + DBG_EVENT = BIT(7), /* event handling messages */ + DBG_OBW = BIT(8), /* outbound window messages */ + DBG_DBELL = BIT(9), /* doorbell messages */ + DBG_ALL = ~0, +}; + +#ifdef DEBUG +#define rmcd_debug(level, fmt, arg...) \ + do { \ + if (DBG_##level & dbg_level) \ + pr_debug(DRV_PREFIX "%s: " fmt "\n", __func__, ##arg); \ + } while (0) +#else +#define rmcd_debug(level, fmt, arg...) \ + no_printk(KERN_DEBUG pr_fmt(DRV_PREFIX fmt "\n"), ##arg) +#endif + +#define rmcd_warn(fmt, arg...) \ + pr_warn(DRV_PREFIX "%s WARNING " fmt "\n", __func__, ##arg) + +#define rmcd_error(fmt, arg...) \ + pr_err(DRV_PREFIX "%s ERROR " fmt "\n", __func__, ##arg) + +MODULE_AUTHOR("Jerry Jacobs "); +MODULE_AUTHOR("Aurelien Jacquiot "); +MODULE_AUTHOR("Alexandre Bounine "); +MODULE_AUTHOR("Andre van Herk "); +MODULE_DESCRIPTION("RapidIO mport character device driver"); +MODULE_LICENSE("GPL"); +MODULE_VERSION(DRV_VERSION); + +static int dma_timeout = 3000; /* DMA transfer timeout in msec */ +module_param(dma_timeout, int, S_IRUGO); +MODULE_PARM_DESC(dma_timeout, "DMA Transfer Timeout in msec (default: 3000)"); + +#ifdef DEBUG +static u32 dbg_level = DBG_NONE; +module_param(dbg_level, uint, S_IWUSR | S_IWGRP | S_IRUGO); +MODULE_PARM_DESC(dbg_level, "Debugging output level (default 0 = none)"); +#endif + +/* + * An internal DMA coherent buffer + */ +struct mport_dma_buf { + void *ib_base; + dma_addr_t ib_phys; + u32 ib_size; + u64 ib_rio_base; + bool ib_map; + struct file *filp; +}; + +/* + * Internal memory mapping structure + */ +enum rio_mport_map_dir { + MAP_INBOUND, + MAP_OUTBOUND, + MAP_DMA, +}; + +struct rio_mport_mapping { + struct list_head node; + struct mport_dev *md; + enum rio_mport_map_dir dir; + u32 rioid; + u64 rio_addr; + dma_addr_t phys_addr; /* for mmap */ + void *virt_addr; /* kernel address, for dma_free_coherent */ + u64 size; + struct kref ref; /* refcount of vmas sharing the mapping */ + struct file *filp; +}; + +struct rio_mport_dma_map { + int valid; + uint64_t length; + void *vaddr; + dma_addr_t paddr; +}; + +#define MPORT_MAX_DMA_BUFS 16 +#define MPORT_EVENT_DEPTH 10 + +/* + * mport_dev driver-specific structure that represents mport device + * @active mport device status flag + * @node list node to maintain list of registered mports + * @cdev character device + * @dev associated device object + * @mport associated subsystem's master port device object + * @buf_mutex lock for buffer handling + * @file_mutex - lock for open files list + * @file_list - list of open files on given mport + * @properties properties of this mport + * @portwrites queue of inbound portwrites + * @pw_lock lock for port write queue + * @mappings queue for memory mappings + * @dma_chan DMA channels associated with this device + * @dma_ref: + * @comp: + */ +struct mport_dev { + atomic_t active; + struct list_head node; + struct cdev cdev; + struct device dev; + struct rio_mport *mport; + struct mutex buf_mutex; + struct mutex file_mutex; + struct list_head file_list; + struct rio_mport_properties properties; + struct list_head doorbells; + spinlock_t db_lock; + struct list_head portwrites; + spinlock_t pw_lock; + struct list_head mappings; +#ifdef CONFIG_RAPIDIO_DMA_ENGINE + struct dma_chan *dma_chan; + struct kref dma_ref; + struct completion comp; +#endif +}; + +/* + * mport_cdev_priv - data structure specific to individual file object + * associated with an open device + * @md master port character device object + * @async_queue - asynchronous notification queue + * @list - file objects tracking list + * @db_filters inbound doorbell filters for this descriptor + * @pw_filters portwrite filters for this descriptor + * @event_fifo event fifo for this descriptor + * @event_rx_wait wait queue for this descriptor + * @fifo_lock lock for event_fifo + * @event_mask event mask for this descriptor + * @dmach DMA engine channel allocated for specific file object + */ +struct mport_cdev_priv { + struct mport_dev *md; + struct fasync_struct *async_queue; + struct list_head list; + struct list_head db_filters; + struct list_head pw_filters; + struct kfifo event_fifo; + wait_queue_head_t event_rx_wait; + spinlock_t fifo_lock; + unsigned int event_mask; /* RIO_DOORBELL, RIO_PORTWRITE */ +#ifdef CONFIG_RAPIDIO_DMA_ENGINE + struct dma_chan *dmach; + struct list_head async_list; + struct list_head pend_list; + spinlock_t req_lock; + struct mutex dma_lock; + struct kref dma_ref; + struct completion comp; +#endif +}; + +/* + * rio_mport_pw_filter - structure to describe a portwrite filter + * md_node node in mport device's list + * priv_node node in private file object's list + * priv reference to private data + * filter actual portwrite filter + */ +struct rio_mport_pw_filter { + struct list_head md_node; + struct list_head priv_node; + struct mport_cdev_priv *priv; + struct rio_pw_filter filter; +}; + +/* + * rio_mport_db_filter - structure to describe a doorbell filter + * @data_node reference to device node + * @priv_node node in private data + * @priv reference to private data + * @filter actual doorbell filter + */ +struct rio_mport_db_filter { + struct list_head data_node; + struct list_head priv_node; + struct mport_cdev_priv *priv; + struct rio_doorbell_filter filter; +}; + +static LIST_HEAD(mport_devs); +static DEFINE_MUTEX(mport_devs_lock); + +#if (0) /* used by commented out portion of poll function : FIXME */ +static DECLARE_WAIT_QUEUE_HEAD(mport_cdev_wait); +#endif + +static struct class *dev_class; +static dev_t dev_number; + +static struct workqueue_struct *dma_wq; + +static void mport_release_mapping(struct kref *ref); + +static int rio_mport_maint_rd(struct mport_cdev_priv *priv, void __user *arg, + int local) +{ + struct rio_mport *mport = priv->md->mport; + struct rio_mport_maint_io maint_io; + u32 *buffer; + u32 offset; + size_t length; + int ret, i; + + if (unlikely(copy_from_user(&maint_io, arg, sizeof(maint_io)))) + return -EFAULT; + + if ((maint_io.offset % 4) || + (maint_io.length == 0) || (maint_io.length % 4)) + return -EINVAL; + + buffer = vmalloc(maint_io.length); + if (buffer == NULL) + return -ENOMEM; + length = maint_io.length/sizeof(u32); + offset = maint_io.offset; + + for (i = 0; i < length; i++) { + if (local) + ret = __rio_local_read_config_32(mport, + offset, &buffer[i]); + else + ret = rio_mport_read_config_32(mport, maint_io.rioid, + maint_io.hopcount, offset, &buffer[i]); + if (ret) + goto out; + + offset += 4; + } + + if (unlikely(copy_to_user(maint_io.buffer, buffer, maint_io.length))) + ret = -EFAULT; +out: + vfree(buffer); + return ret; +} + +static int rio_mport_maint_wr(struct mport_cdev_priv *priv, void __user *arg, + int local) +{ + struct rio_mport *mport = priv->md->mport; + struct rio_mport_maint_io maint_io; + u32 *buffer; + u32 offset; + size_t length; + int ret = -EINVAL, i; + + if (unlikely(copy_from_user(&maint_io, arg, sizeof(maint_io)))) + return -EFAULT; + + if ((maint_io.offset % 4) || + (maint_io.length == 0) || (maint_io.length % 4)) + return -EINVAL; + + buffer = vmalloc(maint_io.length); + if (buffer == NULL) + return -ENOMEM; + length = maint_io.length; + + if (unlikely(copy_from_user(buffer, maint_io.buffer, length))) { + ret = -EFAULT; + goto out; + } + + offset = maint_io.offset; + length /= sizeof(u32); + + for (i = 0; i < length; i++) { + if (local) + ret = __rio_local_write_config_32(mport, + offset, buffer[i]); + else + ret = rio_mport_write_config_32(mport, maint_io.rioid, + maint_io.hopcount, + offset, buffer[i]); + if (ret) + goto out; + + offset += 4; + } + +out: + vfree(buffer); + return ret; +} + + +/* + * Inbound/outbound memory mapping functions + */ +static int +rio_mport_create_outbound_mapping(struct mport_dev *md, struct file *filp, + u32 rioid, u64 raddr, u32 size, + dma_addr_t *paddr) +{ + struct rio_mport *mport = md->mport; + struct rio_mport_mapping *map; + int ret; + + rmcd_debug(OBW, "did=%d ra=0x%llx sz=0x%x", rioid, raddr, size); + + map = kzalloc(sizeof(struct rio_mport_mapping), GFP_KERNEL); + if (map == NULL) + return -ENOMEM; + + ret = rio_map_outb_region(mport, rioid, raddr, size, 0, paddr); + if (ret < 0) + goto err_map_outb; + + map->dir = MAP_OUTBOUND; + map->rioid = rioid; + map->rio_addr = raddr; + map->size = size; + map->phys_addr = *paddr; + map->filp = filp; + map->md = md; + kref_init(&map->ref); + list_add_tail(&map->node, &md->mappings); + return 0; +err_map_outb: + kfree(map); + return ret; +} + +static int +rio_mport_get_outbound_mapping(struct mport_dev *md, struct file *filp, + u32 rioid, u64 raddr, u32 size, + dma_addr_t *paddr) +{ + struct rio_mport_mapping *map; + int err = -ENOMEM; + + mutex_lock(&md->buf_mutex); + list_for_each_entry(map, &md->mappings, node) { + if (map->dir != MAP_OUTBOUND) + continue; + if (rioid == map->rioid && + raddr == map->rio_addr && size == map->size) { + *paddr = map->phys_addr; + err = 0; + break; + } else if (rioid == map->rioid && + raddr < (map->rio_addr + map->size - 1) && + (raddr + size) > map->rio_addr) { + err = -EBUSY; + break; + } + } + + /* If not found, create new */ + if (err == -ENOMEM) + err = rio_mport_create_outbound_mapping(md, filp, rioid, raddr, + size, paddr); + mutex_unlock(&md->buf_mutex); + return err; +} + +static int rio_mport_obw_map(struct file *filp, void __user *arg) +{ + struct mport_cdev_priv *priv = filp->private_data; + struct mport_dev *data = priv->md; + struct rio_mmap map; + dma_addr_t paddr; + int ret; + + if (unlikely(copy_from_user(&map, arg, sizeof(struct rio_mmap)))) + return -EFAULT; + + rmcd_debug(OBW, "did=%d ra=0x%llx sz=0x%llx", + map.rioid, map.rio_addr, map.length); + + ret = rio_mport_get_outbound_mapping(data, filp, map.rioid, + map.rio_addr, map.length, &paddr); + if (ret < 0) { + rmcd_error("Failed to set OBW err= %d", ret); + return ret; + } + + map.handle = paddr; + + if (unlikely(copy_to_user(arg, &map, sizeof(struct rio_mmap)))) + return -EFAULT; + return 0; +} + +/* + * rio_mport_obw_free() - unmap an OutBound Window from RapidIO address space + * + * @priv: driver private data + * @arg: buffer handle returned by allocation routine + */ +static int rio_mport_obw_free(struct file *filp, void __user *arg) +{ + struct mport_cdev_priv *priv = filp->private_data; + struct mport_dev *md = priv->md; + u64 handle; + struct rio_mport_mapping *map, *_map; + + if (!md->mport->ops->unmap_outb) + return -EPROTONOSUPPORT; + + if (copy_from_user(&handle, arg, sizeof(u64))) + return -EFAULT; + + rmcd_debug(OBW, "h=0x%llx", handle); + + mutex_lock(&md->buf_mutex); + list_for_each_entry_safe(map, _map, &md->mappings, node) { + if (map->dir == MAP_OUTBOUND && map->phys_addr == handle) { + if (map->filp == filp) { + rmcd_debug(OBW, "kref_put h=0x%llx", handle); + map->filp = NULL; + kref_put(&map->ref, mport_release_mapping); + } + break; + } + } + mutex_unlock(&md->buf_mutex); + + return 0; +} + +/* + * maint_hdid_set() - Set the host Device ID + * @priv: driver private data + * @arg: Device Id + */ +static int maint_hdid_set(struct mport_cdev_priv *priv, void __user *arg) +{ + struct mport_dev *md = priv->md; + uint16_t hdid; + + if (copy_from_user(&hdid, arg, sizeof(uint16_t))) + return -EFAULT; + + md->mport->host_deviceid = hdid; + md->properties.hdid = hdid; + rio_local_set_device_id(md->mport, hdid); + + rmcd_debug(MPORT, "Set host device Id to %d", hdid); + + return 0; +} + +/* + * maint_comptag_set() - Set the host Component Tag + * @priv: driver private data + * @arg: Component Tag + */ +static int maint_comptag_set(struct mport_cdev_priv *priv, void __user *arg) +{ + struct mport_dev *md = priv->md; + uint32_t comptag; + + if (copy_from_user(&comptag, arg, sizeof(uint32_t))) + return -EFAULT; + + rio_local_write_config_32(md->mport, RIO_COMPONENT_TAG_CSR, comptag); + + rmcd_debug(MPORT, "Set host Component Tag to %d", comptag); + + return 0; +} + +#ifdef CONFIG_RAPIDIO_DMA_ENGINE + +struct mport_dma_req { + struct list_head node; + struct file *filp; + struct mport_cdev_priv *priv; + enum rio_transfer_sync sync; + struct sg_table sgt; + struct page **page_list; + unsigned int nr_pages; + struct rio_mport_mapping *map; + struct dma_chan *dmach; + enum dma_data_direction dir; + dma_cookie_t cookie; + enum dma_status status; + struct completion req_comp; +}; + +struct mport_faf_work { + struct work_struct work; + struct mport_dma_req *req; +}; + +static void mport_release_def_dma(struct kref *dma_ref) +{ + struct mport_dev *md = + container_of(dma_ref, struct mport_dev, dma_ref); + + rmcd_debug(EXIT, "DMA_%d", md->dma_chan->chan_id); + rio_release_dma(md->dma_chan); + md->dma_chan = NULL; +} + +static void mport_release_dma(struct kref *dma_ref) +{ + struct mport_cdev_priv *priv = + container_of(dma_ref, struct mport_cdev_priv, dma_ref); + + rmcd_debug(EXIT, "DMA_%d", priv->dmach->chan_id); + complete(&priv->comp); +} + +static void dma_req_free(struct mport_dma_req *req) +{ + struct mport_cdev_priv *priv = req->priv; + unsigned int i; + + dma_unmap_sg(req->dmach->device->dev, + req->sgt.sgl, req->sgt.nents, req->dir); + sg_free_table(&req->sgt); + if (req->page_list) { + for (i = 0; i < req->nr_pages; i++) + put_page(req->page_list[i]); + kfree(req->page_list); + } + + if (req->map) { + mutex_lock(&req->map->md->buf_mutex); + kref_put(&req->map->ref, mport_release_mapping); + mutex_unlock(&req->map->md->buf_mutex); + } + + kref_put(&priv->dma_ref, mport_release_dma); + + kfree(req); +} + +static void dma_xfer_callback(void *param) +{ + struct mport_dma_req *req = (struct mport_dma_req *)param; + struct mport_cdev_priv *priv = req->priv; + + req->status = dma_async_is_tx_complete(priv->dmach, req->cookie, + NULL, NULL); + complete(&req->req_comp); +} + +static void dma_faf_cleanup(struct work_struct *_work) +{ + struct mport_faf_work *work = container_of(_work, + struct mport_faf_work, work); + struct mport_dma_req *req = work->req; + + dma_req_free(req); + kfree(work); +} + +static void dma_faf_callback(void *param) +{ + struct mport_dma_req *req = (struct mport_dma_req *)param; + struct mport_faf_work *work; + + work = kmalloc(sizeof(*work), GFP_ATOMIC); + if (!work) + return; + + INIT_WORK(&work->work, dma_faf_cleanup); + work->req = req; + queue_work(dma_wq, &work->work); +} + +/* + * prep_dma_xfer() - Configure and send request to DMAengine to prepare DMA + * transfer object. + * Returns pointer to DMA transaction descriptor allocated by DMA driver on + * success or ERR_PTR (and/or NULL) if failed. Caller must check returned + * non-NULL pointer using IS_ERR macro. + */ +static struct dma_async_tx_descriptor +*prep_dma_xfer(struct dma_chan *chan, struct rio_transfer_io *transfer, + struct sg_table *sgt, int nents, enum dma_transfer_direction dir, + enum dma_ctrl_flags flags) +{ + struct rio_dma_data tx_data; + + tx_data.sg = sgt->sgl; + tx_data.sg_len = nents; + tx_data.rio_addr_u = 0; + tx_data.rio_addr = transfer->rio_addr; + if (dir == DMA_MEM_TO_DEV) { + switch (transfer->method) { + case RIO_EXCHANGE_NWRITE: + tx_data.wr_type = RDW_ALL_NWRITE; + break; + case RIO_EXCHANGE_NWRITE_R_ALL: + tx_data.wr_type = RDW_ALL_NWRITE_R; + break; + case RIO_EXCHANGE_NWRITE_R: + tx_data.wr_type = RDW_LAST_NWRITE_R; + break; + case RIO_EXCHANGE_DEFAULT: + tx_data.wr_type = RDW_DEFAULT; + break; + default: + return ERR_PTR(-EINVAL); + } + } + + return rio_dma_prep_xfer(chan, transfer->rioid, &tx_data, dir, flags); +} + +/* Request DMA channel associated with this mport device. + * Try to request DMA channel for every new process that opened given + * mport. If a new DMA channel is not available use default channel + * which is the first DMA channel opened on mport device. + */ +static int get_dma_channel(struct mport_cdev_priv *priv) +{ + mutex_lock(&priv->dma_lock); + if (!priv->dmach) { + priv->dmach = rio_request_mport_dma(priv->md->mport); + if (!priv->dmach) { + /* Use default DMA channel if available */ + if (priv->md->dma_chan) { + priv->dmach = priv->md->dma_chan; + kref_get(&priv->md->dma_ref); + } else { + rmcd_error("Failed to get DMA channel"); + mutex_unlock(&priv->dma_lock); + return -ENODEV; + } + } else if (!priv->md->dma_chan) { + /* Register default DMA channel if we do not have one */ + priv->md->dma_chan = priv->dmach; + kref_init(&priv->md->dma_ref); + rmcd_debug(DMA, "Register DMA_chan %d as default", + priv->dmach->chan_id); + } + + kref_init(&priv->dma_ref); + init_completion(&priv->comp); + } + + kref_get(&priv->dma_ref); + mutex_unlock(&priv->dma_lock); + return 0; +} + +static void put_dma_channel(struct mport_cdev_priv *priv) +{ + kref_put(&priv->dma_ref, mport_release_dma); +} + +/* + * DMA transfer functions + */ +static int do_dma_request(struct mport_dma_req *req, + struct rio_transfer_io *xfer, + enum rio_transfer_sync sync, int nents) +{ + struct mport_cdev_priv *priv; + struct sg_table *sgt; + struct dma_chan *chan; + struct dma_async_tx_descriptor *tx; + dma_cookie_t cookie; + unsigned long tmo = msecs_to_jiffies(dma_timeout); + enum dma_transfer_direction dir; + long wret; + int ret = 0; + + priv = req->priv; + sgt = &req->sgt; + + chan = priv->dmach; + dir = (req->dir == DMA_FROM_DEVICE) ? DMA_DEV_TO_MEM : DMA_MEM_TO_DEV; + + rmcd_debug(DMA, "%s(%d) uses %s for DMA_%s", + current->comm, task_pid_nr(current), + dev_name(&chan->dev->device), + (dir == DMA_DEV_TO_MEM)?"READ":"WRITE"); + + /* Initialize DMA transaction request */ + tx = prep_dma_xfer(chan, xfer, sgt, nents, dir, + DMA_CTRL_ACK | DMA_PREP_INTERRUPT); + + if (!tx) { + rmcd_debug(DMA, "prep error for %s A:0x%llx L:0x%llx", + (dir == DMA_DEV_TO_MEM)?"READ":"WRITE", + xfer->rio_addr, xfer->length); + ret = -EIO; + goto err_out; + } else if (IS_ERR(tx)) { + ret = PTR_ERR(tx); + rmcd_debug(DMA, "prep error %d for %s A:0x%llx L:0x%llx", ret, + (dir == DMA_DEV_TO_MEM)?"READ":"WRITE", + xfer->rio_addr, xfer->length); + goto err_out; + } + + if (sync == RIO_TRANSFER_FAF) + tx->callback = dma_faf_callback; + else + tx->callback = dma_xfer_callback; + tx->callback_param = req; + + req->dmach = chan; + req->sync = sync; + req->status = DMA_IN_PROGRESS; + init_completion(&req->req_comp); + + cookie = dmaengine_submit(tx); + req->cookie = cookie; + + rmcd_debug(DMA, "pid=%d DMA_%s tx_cookie = %d", task_pid_nr(current), + (dir == DMA_DEV_TO_MEM)?"READ":"WRITE", cookie); + + if (dma_submit_error(cookie)) { + rmcd_error("submit err=%d (addr:0x%llx len:0x%llx)", + cookie, xfer->rio_addr, xfer->length); + ret = -EIO; + goto err_out; + } + + dma_async_issue_pending(chan); + + if (sync == RIO_TRANSFER_ASYNC) { + spin_lock(&priv->req_lock); + list_add_tail(&req->node, &priv->async_list); + spin_unlock(&priv->req_lock); + return cookie; + } else if (sync == RIO_TRANSFER_FAF) + return 0; + + wret = wait_for_completion_interruptible_timeout(&req->req_comp, tmo); + + if (wret == 0) { + /* Timeout on wait occurred */ + rmcd_error("%s(%d) timed out waiting for DMA_%s %d", + current->comm, task_pid_nr(current), + (dir == DMA_DEV_TO_MEM)?"READ":"WRITE", cookie); + return -ETIMEDOUT; + } else if (wret == -ERESTARTSYS) { + /* Wait_for_completion was interrupted by a signal but DMA may + * be in progress + */ + rmcd_error("%s(%d) wait for DMA_%s %d was interrupted", + current->comm, task_pid_nr(current), + (dir == DMA_DEV_TO_MEM)?"READ":"WRITE", cookie); + return -EINTR; + } + + if (req->status != DMA_COMPLETE) { + /* DMA transaction completion was signaled with error */ + rmcd_error("%s(%d) DMA_%s %d completed with status %d (ret=%d)", + current->comm, task_pid_nr(current), + (dir == DMA_DEV_TO_MEM)?"READ":"WRITE", + cookie, req->status, ret); + ret = -EIO; + } + +err_out: + return ret; +} + +/* + * rio_dma_transfer() - Perform RapidIO DMA data transfer to/from + * the remote RapidIO device + * @filp: file pointer associated with the call + * @transfer_mode: DMA transfer mode + * @sync: synchronization mode + * @dir: DMA transfer direction (DMA_MEM_TO_DEV = write OR + * DMA_DEV_TO_MEM = read) + * @xfer: data transfer descriptor structure + */ +static int +rio_dma_transfer(struct file *filp, uint32_t transfer_mode, + enum rio_transfer_sync sync, enum dma_data_direction dir, + struct rio_transfer_io *xfer) +{ + struct mport_cdev_priv *priv = filp->private_data; + unsigned long nr_pages = 0; + struct page **page_list = NULL; + struct mport_dma_req *req; + struct mport_dev *md = priv->md; + struct dma_chan *chan; + int i, ret; + int nents; + + if (xfer->length == 0) + return -EINVAL; + req = kzalloc(sizeof(*req), GFP_KERNEL); + if (!req) + return -ENOMEM; + + ret = get_dma_channel(priv); + if (ret) { + kfree(req); + return ret; + } + + /* + * If parameter loc_addr != NULL, we are transferring data from/to + * data buffer allocated in user-space: lock in memory user-space + * buffer pages and build an SG table for DMA transfer request + * + * Otherwise (loc_addr == NULL) contiguous kernel-space buffer is + * used for DMA data transfers: build single entry SG table using + * offset within the internal buffer specified by handle parameter. + */ + if (xfer->loc_addr) { + unsigned long offset; + long pinned; + + offset = (unsigned long)xfer->loc_addr & ~PAGE_MASK; + nr_pages = PAGE_ALIGN(xfer->length + offset) >> PAGE_SHIFT; + + page_list = kmalloc_array(nr_pages, + sizeof(*page_list), GFP_KERNEL); + if (page_list == NULL) { + ret = -ENOMEM; + goto err_req; + } + + down_read(¤t->mm->mmap_sem); + pinned = get_user_pages(current, current->mm, + (unsigned long)xfer->loc_addr & PAGE_MASK, + nr_pages, dir == DMA_FROM_DEVICE, 0, + page_list, NULL); + up_read(¤t->mm->mmap_sem); + + if (pinned != nr_pages) { + if (pinned < 0) { + rmcd_error("get_user_pages err=%ld", pinned); + nr_pages = 0; + } else + rmcd_error("pinned %ld out of %ld pages", + pinned, nr_pages); + ret = -EFAULT; + goto err_pg; + } + + ret = sg_alloc_table_from_pages(&req->sgt, page_list, nr_pages, + offset, xfer->length, GFP_KERNEL); + if (ret) { + rmcd_error("sg_alloc_table failed with err=%d", ret); + goto err_pg; + } + + req->page_list = page_list; + req->nr_pages = nr_pages; + } else { + dma_addr_t baddr; + struct rio_mport_mapping *map; + + baddr = (dma_addr_t)xfer->handle; + + mutex_lock(&md->buf_mutex); + list_for_each_entry(map, &md->mappings, node) { + if (baddr >= map->phys_addr && + baddr < (map->phys_addr + map->size)) { + kref_get(&map->ref); + req->map = map; + break; + } + } + mutex_unlock(&md->buf_mutex); + + if (req->map == NULL) { + ret = -ENOMEM; + goto err_req; + } + + if (xfer->length + xfer->offset > map->size) { + ret = -EINVAL; + goto err_req; + } + + ret = sg_alloc_table(&req->sgt, 1, GFP_KERNEL); + if (unlikely(ret)) { + rmcd_error("sg_alloc_table failed for internal buf"); + goto err_req; + } + + sg_set_buf(req->sgt.sgl, + map->virt_addr + (baddr - map->phys_addr) + + xfer->offset, xfer->length); + } + + req->dir = dir; + req->filp = filp; + req->priv = priv; + chan = priv->dmach; + + nents = dma_map_sg(chan->device->dev, + req->sgt.sgl, req->sgt.nents, dir); + if (nents == -EFAULT) { + rmcd_error("Failed to map SG list"); + return -EFAULT; + } + + ret = do_dma_request(req, xfer, sync, nents); + + if (ret >= 0) { + if (sync == RIO_TRANSFER_SYNC) + goto sync_out; + return ret; /* return ASYNC cookie */ + } + + if (ret == -ETIMEDOUT || ret == -EINTR) { + /* + * This can happen only in case of SYNC transfer. + * Do not free unfinished request structure immediately. + * Place it into pending list and deal with it later + */ + spin_lock(&priv->req_lock); + list_add_tail(&req->node, &priv->pend_list); + spin_unlock(&priv->req_lock); + return ret; + } + + + rmcd_debug(DMA, "do_dma_request failed with err=%d", ret); +sync_out: + dma_unmap_sg(chan->device->dev, req->sgt.sgl, req->sgt.nents, dir); + sg_free_table(&req->sgt); +err_pg: + if (page_list) { + for (i = 0; i < nr_pages; i++) + put_page(page_list[i]); + kfree(page_list); + } +err_req: + if (req->map) { + mutex_lock(&md->buf_mutex); + kref_put(&req->map->ref, mport_release_mapping); + mutex_unlock(&md->buf_mutex); + } + put_dma_channel(priv); + kfree(req); + return ret; +} + +static int rio_mport_transfer_ioctl(struct file *filp, void __user *arg) +{ + struct mport_cdev_priv *priv = filp->private_data; + struct rio_transaction transaction; + struct rio_transfer_io *transfer; + enum dma_data_direction dir; + int i, ret = 0; + + if (unlikely(copy_from_user(&transaction, arg, sizeof(transaction)))) + return -EFAULT; + + if (transaction.count != 1) + return -EINVAL; + + if ((transaction.transfer_mode & + priv->md->properties.transfer_mode) == 0) + return -ENODEV; + + transfer = vmalloc(transaction.count * sizeof(struct rio_transfer_io)); + if (!transfer) + return -ENOMEM; + + if (unlikely(copy_from_user(transfer, transaction.block, + transaction.count * sizeof(struct rio_transfer_io)))) { + ret = -EFAULT; + goto out_free; + } + + dir = (transaction.dir == RIO_TRANSFER_DIR_READ) ? + DMA_FROM_DEVICE : DMA_TO_DEVICE; + for (i = 0; i < transaction.count && ret == 0; i++) + ret = rio_dma_transfer(filp, transaction.transfer_mode, + transaction.sync, dir, &transfer[i]); + + if (unlikely(copy_to_user(transaction.block, transfer, + transaction.count * sizeof(struct rio_transfer_io)))) + ret = -EFAULT; + +out_free: + vfree(transfer); + + return ret; +} + +static int rio_mport_wait_for_async_dma(struct file *filp, void __user *arg) +{ + struct mport_cdev_priv *priv; + struct mport_dev *md; + struct rio_async_tx_wait w_param; + struct mport_dma_req *req; + dma_cookie_t cookie; + unsigned long tmo; + long wret; + int found = 0; + int ret; + + priv = (struct mport_cdev_priv *)filp->private_data; + md = priv->md; + + if (unlikely(copy_from_user(&w_param, arg, sizeof(w_param)))) + return -EFAULT; + + cookie = w_param.token; + if (w_param.timeout) + tmo = msecs_to_jiffies(w_param.timeout); + else /* Use default DMA timeout */ + tmo = msecs_to_jiffies(dma_timeout); + + spin_lock(&priv->req_lock); + list_for_each_entry(req, &priv->async_list, node) { + if (req->cookie == cookie) { + list_del(&req->node); + found = 1; + break; + } + } + spin_unlock(&priv->req_lock); + + if (!found) + return -EAGAIN; + + wret = wait_for_completion_interruptible_timeout(&req->req_comp, tmo); + + if (wret == 0) { + /* Timeout on wait occurred */ + rmcd_error("%s(%d) timed out waiting for ASYNC DMA_%s", + current->comm, task_pid_nr(current), + (req->dir == DMA_FROM_DEVICE)?"READ":"WRITE"); + ret = -ETIMEDOUT; + goto err_tmo; + } else if (wret == -ERESTARTSYS) { + /* Wait_for_completion was interrupted by a signal but DMA may + * be still in progress + */ + rmcd_error("%s(%d) wait for ASYNC DMA_%s was interrupted", + current->comm, task_pid_nr(current), + (req->dir == DMA_FROM_DEVICE)?"READ":"WRITE"); + ret = -EINTR; + goto err_tmo; + } + + if (req->status != DMA_COMPLETE) { + /* DMA transaction completion signaled with transfer error */ + rmcd_error("%s(%d) ASYNC DMA_%s completion with status %d", + current->comm, task_pid_nr(current), + (req->dir == DMA_FROM_DEVICE)?"READ":"WRITE", + req->status); + ret = -EIO; + } else + ret = 0; + + if (req->status != DMA_IN_PROGRESS && req->status != DMA_PAUSED) + dma_req_free(req); + + return ret; + +err_tmo: + /* Return request back into async queue */ + spin_lock(&priv->req_lock); + list_add_tail(&req->node, &priv->async_list); + spin_unlock(&priv->req_lock); + return ret; +} + +static int rio_mport_create_dma_mapping(struct mport_dev *md, struct file *filp, + uint64_t size, struct rio_mport_mapping **mapping) +{ + struct rio_mport_mapping *map; + + map = kzalloc(sizeof(struct rio_mport_mapping), GFP_KERNEL); + if (map == NULL) + return -ENOMEM; + + map->virt_addr = dma_alloc_coherent(md->mport->dev.parent, size, + &map->phys_addr, GFP_KERNEL); + if (map->virt_addr == NULL) { + kfree(map); + return -ENOMEM; + } + + map->dir = MAP_DMA; + map->size = size; + map->filp = filp; + map->md = md; + kref_init(&map->ref); + mutex_lock(&md->buf_mutex); + list_add_tail(&map->node, &md->mappings); + mutex_unlock(&md->buf_mutex); + *mapping = map; + + return 0; +} + +static int rio_mport_alloc_dma(struct file *filp, void __user *arg) +{ + struct mport_cdev_priv *priv = filp->private_data; + struct mport_dev *md = priv->md; + struct rio_dma_mem map; + struct rio_mport_mapping *mapping = NULL; + int ret; + + if (unlikely(copy_from_user(&map, arg, sizeof(struct rio_dma_mem)))) + return -EFAULT; + + ret = rio_mport_create_dma_mapping(md, filp, map.length, &mapping); + if (ret) + return ret; + + map.dma_handle = mapping->phys_addr; + + if (unlikely(copy_to_user(arg, &map, sizeof(struct rio_dma_mem)))) { + mutex_lock(&md->buf_mutex); + kref_put(&mapping->ref, mport_release_mapping); + mutex_unlock(&md->buf_mutex); + return -EFAULT; + } + + return 0; +} + +static int rio_mport_free_dma(struct file *filp, void __user *arg) +{ + struct mport_cdev_priv *priv = filp->private_data; + struct mport_dev *md = priv->md; + u64 handle; + int ret = -EFAULT; + struct rio_mport_mapping *map, *_map; + + if (copy_from_user(&handle, arg, sizeof(u64))) + return -EFAULT; + rmcd_debug(EXIT, "filp=%p", filp); + + mutex_lock(&md->buf_mutex); + list_for_each_entry_safe(map, _map, &md->mappings, node) { + if (map->dir == MAP_DMA && map->phys_addr == handle && + map->filp == filp) { + kref_put(&map->ref, mport_release_mapping); + ret = 0; + break; + } + } + mutex_unlock(&md->buf_mutex); + + if (ret == -EFAULT) { + rmcd_debug(DMA, "ERR no matching mapping"); + return ret; + } + + return 0; +} +#else +static int rio_mport_transfer_ioctl(struct file *filp, void *arg) +{ + return -ENODEV; +} + +static int rio_mport_wait_for_async_dma(struct file *filp, void __user *arg) +{ + return -ENODEV; +} + +static int rio_mport_alloc_dma(struct file *filp, void __user *arg) +{ + return -ENODEV; +} + +static int rio_mport_free_dma(struct file *filp, void __user *arg) +{ + return -ENODEV; +} +#endif /* CONFIG_RAPIDIO_DMA_ENGINE */ + +/* + * Inbound/outbound memory mapping functions + */ + +static int +rio_mport_create_inbound_mapping(struct mport_dev *md, struct file *filp, + u64 raddr, u32 size, + struct rio_mport_mapping **mapping) +{ + struct rio_mport *mport = md->mport; + struct rio_mport_mapping *map; + int ret; + + map = kzalloc(sizeof(struct rio_mport_mapping), GFP_KERNEL); + if (map == NULL) + return -ENOMEM; + + map->virt_addr = dma_alloc_coherent(mport->dev.parent, size, + &map->phys_addr, GFP_KERNEL); + if (map->virt_addr == NULL) { + ret = -ENOMEM; + goto err_dma_alloc; + } + + if (raddr == RIO_MAP_ANY_ADDR) + raddr = map->phys_addr; + ret = rio_map_inb_region(mport, map->phys_addr, raddr, size, 0); + if (ret < 0) + goto err_map_inb; + + map->dir = MAP_INBOUND; + map->rio_addr = raddr; + map->size = size; + map->filp = filp; + map->md = md; + kref_init(&map->ref); + mutex_lock(&md->buf_mutex); + list_add_tail(&map->node, &md->mappings); + mutex_unlock(&md->buf_mutex); + *mapping = map; + return 0; + +err_map_inb: + dma_free_coherent(mport->dev.parent, size, + map->virt_addr, map->phys_addr); +err_dma_alloc: + kfree(map); + return ret; +} + +static int +rio_mport_get_inbound_mapping(struct mport_dev *md, struct file *filp, + u64 raddr, u32 size, + struct rio_mport_mapping **mapping) +{ + struct rio_mport_mapping *map; + int err = -ENOMEM; + + if (raddr == RIO_MAP_ANY_ADDR) + goto get_new; + + mutex_lock(&md->buf_mutex); + list_for_each_entry(map, &md->mappings, node) { + if (map->dir != MAP_INBOUND) + continue; + if (raddr == map->rio_addr && size == map->size) { + /* allow exact match only */ + *mapping = map; + err = 0; + break; + } else if (raddr < (map->rio_addr + map->size - 1) && + (raddr + size) > map->rio_addr) { + err = -EBUSY; + break; + } + } + mutex_unlock(&md->buf_mutex); + + if (err != -ENOMEM) + return err; +get_new: + /* not found, create new */ + return rio_mport_create_inbound_mapping(md, filp, raddr, size, mapping); +} + +static int rio_mport_map_inbound(struct file *filp, void __user *arg) +{ + struct mport_cdev_priv *priv = filp->private_data; + struct mport_dev *md = priv->md; + struct rio_mmap map; + struct rio_mport_mapping *mapping = NULL; + int ret; + + if (!md->mport->ops->map_inb) + return -EPROTONOSUPPORT; + if (unlikely(copy_from_user(&map, arg, sizeof(struct rio_mmap)))) + return -EFAULT; + + rmcd_debug(IBW, "%s filp=%p", dev_name(&priv->md->dev), filp); + + ret = rio_mport_get_inbound_mapping(md, filp, map.rio_addr, + map.length, &mapping); + if (ret) + return ret; + + map.handle = mapping->phys_addr; + map.rio_addr = mapping->rio_addr; + + if (unlikely(copy_to_user(arg, &map, sizeof(struct rio_mmap)))) { + /* Delete mapping if it was created by this request */ + if (ret == 0 && mapping->filp == filp) { + mutex_lock(&md->buf_mutex); + kref_put(&mapping->ref, mport_release_mapping); + mutex_unlock(&md->buf_mutex); + } + return -EFAULT; + } + + return 0; +} + +/* + * rio_mport_inbound_free() - unmap from RapidIO address space and free + * previously allocated inbound DMA coherent buffer + * @priv: driver private data + * @arg: buffer handle returned by allocation routine + */ +static int rio_mport_inbound_free(struct file *filp, void __user *arg) +{ + struct mport_cdev_priv *priv = filp->private_data; + struct mport_dev *md = priv->md; + u64 handle; + struct rio_mport_mapping *map, *_map; + + rmcd_debug(IBW, "%s filp=%p", dev_name(&priv->md->dev), filp); + + if (!md->mport->ops->unmap_inb) + return -EPROTONOSUPPORT; + + if (copy_from_user(&handle, arg, sizeof(u64))) + return -EFAULT; + + mutex_lock(&md->buf_mutex); + list_for_each_entry_safe(map, _map, &md->mappings, node) { + if (map->dir == MAP_INBOUND && map->phys_addr == handle) { + if (map->filp == filp) { + map->filp = NULL; + kref_put(&map->ref, mport_release_mapping); + } + break; + } + } + mutex_unlock(&md->buf_mutex); + + return 0; +} + +/* + * maint_port_idx_get() - Get the port index of the mport instance + * @priv: driver private data + * @arg: port index + */ +static int maint_port_idx_get(struct mport_cdev_priv *priv, void __user *arg) +{ + struct mport_dev *md = priv->md; + uint32_t port_idx = md->mport->index; + + rmcd_debug(MPORT, "port_index=%d", port_idx); + + if (copy_to_user(arg, &port_idx, sizeof(port_idx))) + return -EFAULT; + + return 0; +} + +static int rio_mport_add_event(struct mport_cdev_priv *priv, + struct rio_event *event) +{ + int overflow; + + if (!(priv->event_mask & event->header)) + return -EACCES; + + spin_lock(&priv->fifo_lock); + overflow = kfifo_avail(&priv->event_fifo) < sizeof(*event) + || kfifo_in(&priv->event_fifo, (unsigned char *)event, + sizeof(*event)) != sizeof(*event); + spin_unlock(&priv->fifo_lock); + + wake_up_interruptible(&priv->event_rx_wait); + + if (overflow) { + dev_warn(&priv->md->dev, DRV_NAME ": event fifo overflow\n"); + return -EBUSY; + } + + return 0; +} + +static void rio_mport_doorbell_handler(struct rio_mport *mport, void *dev_id, + u16 src, u16 dst, u16 info) +{ + struct mport_dev *data = dev_id; + struct mport_cdev_priv *priv; + struct rio_mport_db_filter *db_filter; + struct rio_event event; + int handled; + + event.header = RIO_DOORBELL; + event.u.doorbell.rioid = src; + event.u.doorbell.payload = info; + + handled = 0; + spin_lock(&data->db_lock); + list_for_each_entry(db_filter, &data->doorbells, data_node) { + if (((db_filter->filter.rioid == 0xffffffff || + db_filter->filter.rioid == src)) && + info >= db_filter->filter.low && + info <= db_filter->filter.high) { + priv = db_filter->priv; + rio_mport_add_event(priv, &event); + handled = 1; + } + } + spin_unlock(&data->db_lock); + + if (!handled) + dev_warn(&data->dev, + "%s: spurious DB received from 0x%x, info=0x%04x\n", + __func__, src, info); +} + +static int rio_mport_add_db_filter(struct mport_cdev_priv *priv, + void __user *arg) +{ + struct mport_dev *md = priv->md; + struct rio_mport_db_filter *db_filter; + struct rio_doorbell_filter filter; + unsigned long flags; + int ret; + + if (copy_from_user(&filter, arg, sizeof(filter))) + return -EFAULT; + + if (filter.low > filter.high) + return -EINVAL; + + ret = rio_request_inb_dbell(md->mport, md, filter.low, filter.high, + rio_mport_doorbell_handler); + if (ret) { + rmcd_error("%s failed to register IBDB, err=%d", + dev_name(&md->dev), ret); + return ret; + } + + db_filter = kzalloc(sizeof(*db_filter), GFP_KERNEL); + if (db_filter == NULL) { + rio_release_inb_dbell(md->mport, filter.low, filter.high); + return -ENOMEM; + } + + db_filter->filter = filter; + db_filter->priv = priv; + spin_lock_irqsave(&md->db_lock, flags); + list_add_tail(&db_filter->priv_node, &priv->db_filters); + list_add_tail(&db_filter->data_node, &md->doorbells); + spin_unlock_irqrestore(&md->db_lock, flags); + + return 0; +} + +static void rio_mport_delete_db_filter(struct rio_mport_db_filter *db_filter) +{ + list_del(&db_filter->data_node); + list_del(&db_filter->priv_node); + kfree(db_filter); +} + +static int rio_mport_remove_db_filter(struct mport_cdev_priv *priv, + void __user *arg) +{ + struct rio_mport_db_filter *db_filter; + struct rio_doorbell_filter filter; + unsigned long flags; + int ret = -EINVAL; + + if (copy_from_user(&filter, arg, sizeof(filter))) + return -EFAULT; + + spin_lock_irqsave(&priv->md->db_lock, flags); + list_for_each_entry(db_filter, &priv->db_filters, priv_node) { + if (db_filter->filter.rioid == filter.rioid && + db_filter->filter.low == filter.low && + db_filter->filter.high == filter.high) { + rio_mport_delete_db_filter(db_filter); + ret = 0; + break; + } + } + spin_unlock_irqrestore(&priv->md->db_lock, flags); + + if (!ret) + rio_release_inb_dbell(priv->md->mport, filter.low, filter.high); + + return ret; +} + +static int rio_mport_match_pw(union rio_pw_msg *msg, + struct rio_pw_filter *filter) +{ + if ((msg->em.comptag & filter->mask) < filter->low || + (msg->em.comptag & filter->mask) > filter->high) + return 0; + return 1; +} + +static int rio_mport_pw_handler(struct rio_mport *mport, void *context, + union rio_pw_msg *msg, int step) +{ + struct mport_dev *md = context; + struct mport_cdev_priv *priv; + struct rio_mport_pw_filter *pw_filter; + struct rio_event event; + int handled; + + event.header = RIO_PORTWRITE; + memcpy(event.u.portwrite.payload, msg->raw, RIO_PW_MSG_SIZE); + + handled = 0; + spin_lock(&md->pw_lock); + list_for_each_entry(pw_filter, &md->portwrites, md_node) { + if (rio_mport_match_pw(msg, &pw_filter->filter)) { + priv = pw_filter->priv; + rio_mport_add_event(priv, &event); + handled = 1; + } + } + spin_unlock(&md->pw_lock); + + if (!handled) { + printk_ratelimited(KERN_WARNING DRV_NAME + ": mport%d received spurious PW from 0x%08x\n", + mport->id, msg->em.comptag); + } + + return 0; +} + +static int rio_mport_add_pw_filter(struct mport_cdev_priv *priv, + void __user *arg) +{ + struct mport_dev *md = priv->md; + struct rio_mport_pw_filter *pw_filter; + struct rio_pw_filter filter; + unsigned long flags; + int hadd = 0; + + if (copy_from_user(&filter, arg, sizeof(filter))) + return -EFAULT; + + pw_filter = kzalloc(sizeof(*pw_filter), GFP_KERNEL); + if (pw_filter == NULL) + return -ENOMEM; + + pw_filter->filter = filter; + pw_filter->priv = priv; + spin_lock_irqsave(&md->pw_lock, flags); + if (list_empty(&md->portwrites)) + hadd = 1; + list_add_tail(&pw_filter->priv_node, &priv->pw_filters); + list_add_tail(&pw_filter->md_node, &md->portwrites); + spin_unlock_irqrestore(&md->pw_lock, flags); + + if (hadd) { + int ret; + + ret = rio_add_mport_pw_handler(md->mport, md, + rio_mport_pw_handler); + if (ret) { + dev_err(&md->dev, + "%s: failed to add IB_PW handler, err=%d\n", + __func__, ret); + return ret; + } + rio_pw_enable(md->mport, 1); + } + + return 0; +} + +static void rio_mport_delete_pw_filter(struct rio_mport_pw_filter *pw_filter) +{ + list_del(&pw_filter->md_node); + list_del(&pw_filter->priv_node); + kfree(pw_filter); +} + +static int rio_mport_match_pw_filter(struct rio_pw_filter *a, + struct rio_pw_filter *b) +{ + if ((a->mask == b->mask) && (a->low == b->low) && (a->high == b->high)) + return 1; + return 0; +} + +static int rio_mport_remove_pw_filter(struct mport_cdev_priv *priv, + void __user *arg) +{ + struct mport_dev *md = priv->md; + struct rio_mport_pw_filter *pw_filter; + struct rio_pw_filter filter; + unsigned long flags; + int ret = -EINVAL; + int hdel = 0; + + if (copy_from_user(&filter, arg, sizeof(filter))) + return -EFAULT; + + spin_lock_irqsave(&md->pw_lock, flags); + list_for_each_entry(pw_filter, &priv->pw_filters, priv_node) { + if (rio_mport_match_pw_filter(&pw_filter->filter, &filter)) { + rio_mport_delete_pw_filter(pw_filter); + ret = 0; + break; + } + } + + if (list_empty(&md->portwrites)) + hdel = 1; + spin_unlock_irqrestore(&md->pw_lock, flags); + + if (hdel) { + rio_del_mport_pw_handler(md->mport, priv->md, + rio_mport_pw_handler); + rio_pw_enable(md->mport, 0); + } + + return ret; +} + +/* + * rio_release_dev - release routine for kernel RIO device object + * @dev: kernel device object associated with a RIO device structure + * + * Frees a RIO device struct associated a RIO device struct. + * The RIO device struct is freed. + */ +static void rio_release_dev(struct device *dev) +{ + struct rio_dev *rdev; + + rdev = to_rio_dev(dev); + pr_info(DRV_PREFIX "%s: %s\n", __func__, rio_name(rdev)); + kfree(rdev); +} + + +static void rio_release_net(struct device *dev) +{ + struct rio_net *net; + + net = to_rio_net(dev); + rmcd_debug(RDEV, "net_%d", net->id); + kfree(net); +} + + +/* + * rio_mport_add_riodev - creates a kernel RIO device object + * + * Allocates a RIO device data structure and initializes required fields based + * on device's configuration space contents. + * If the device has switch capabilities, then a switch specific portion is + * allocated and configured. + */ +static int rio_mport_add_riodev(struct mport_cdev_priv *priv, + void __user *arg) +{ + struct mport_dev *md = priv->md; + struct rio_rdev_info dev_info; + struct rio_dev *rdev; + struct rio_switch *rswitch = NULL; + struct rio_mport *mport; + size_t size; + u32 rval; + u32 swpinfo = 0; + u16 destid; + u8 hopcount; + int err; + + if (copy_from_user(&dev_info, arg, sizeof(dev_info))) + return -EFAULT; + + rmcd_debug(RDEV, "name:%s ct:0x%x did:0x%x hc:0x%x", dev_info.name, + dev_info.comptag, dev_info.destid, dev_info.hopcount); + + if (bus_find_device_by_name(&rio_bus_type, NULL, dev_info.name)) { + rmcd_debug(RDEV, "device %s already exists", dev_info.name); + return -EEXIST; + } + + size = sizeof(struct rio_dev); + mport = md->mport; + destid = (u16)dev_info.destid; + hopcount = (u8)dev_info.hopcount; + + if (rio_mport_read_config_32(mport, destid, hopcount, + RIO_PEF_CAR, &rval)) + return -EIO; + + if (rval & RIO_PEF_SWITCH) { + rio_mport_read_config_32(mport, destid, hopcount, + RIO_SWP_INFO_CAR, &swpinfo); + size += (RIO_GET_TOTAL_PORTS(swpinfo) * + sizeof(rswitch->nextdev[0])) + sizeof(*rswitch); + } + + rdev = kzalloc(size, GFP_KERNEL); + if (rdev == NULL) + return -ENOMEM; + + if (mport->net == NULL) { + struct rio_net *net; + + net = rio_alloc_net(mport); + if (!net) { + err = -ENOMEM; + rmcd_debug(RDEV, "failed to allocate net object"); + goto cleanup; + } + + net->id = mport->id; + net->hport = mport; + dev_set_name(&net->dev, "rnet_%d", net->id); + net->dev.parent = &mport->dev; + net->dev.release = rio_release_net; + err = rio_add_net(net); + if (err) { + rmcd_debug(RDEV, "failed to register net, err=%d", err); + kfree(net); + goto cleanup; + } + } + + rdev->net = mport->net; + rdev->pef = rval; + rdev->swpinfo = swpinfo; + rio_mport_read_config_32(mport, destid, hopcount, + RIO_DEV_ID_CAR, &rval); + rdev->did = rval >> 16; + rdev->vid = rval & 0xffff; + rio_mport_read_config_32(mport, destid, hopcount, RIO_DEV_INFO_CAR, + &rdev->device_rev); + rio_mport_read_config_32(mport, destid, hopcount, RIO_ASM_ID_CAR, + &rval); + rdev->asm_did = rval >> 16; + rdev->asm_vid = rval & 0xffff; + rio_mport_read_config_32(mport, destid, hopcount, RIO_ASM_INFO_CAR, + &rval); + rdev->asm_rev = rval >> 16; + + if (rdev->pef & RIO_PEF_EXT_FEATURES) { + rdev->efptr = rval & 0xffff; + rdev->phys_efptr = rio_mport_get_physefb(mport, 0, destid, + hopcount); + + rdev->em_efptr = rio_mport_get_feature(mport, 0, destid, + hopcount, RIO_EFB_ERR_MGMNT); + } + + rio_mport_read_config_32(mport, destid, hopcount, RIO_SRC_OPS_CAR, + &rdev->src_ops); + rio_mport_read_config_32(mport, destid, hopcount, RIO_DST_OPS_CAR, + &rdev->dst_ops); + + rdev->comp_tag = dev_info.comptag; + rdev->destid = destid; + /* hopcount is stored as specified by a caller, regardles of EP or SW */ + rdev->hopcount = hopcount; + + if (rdev->pef & RIO_PEF_SWITCH) { + rswitch = rdev->rswitch; + rswitch->route_table = NULL; + } + + if (strlen(dev_info.name)) + dev_set_name(&rdev->dev, "%s", dev_info.name); + else if (rdev->pef & RIO_PEF_SWITCH) + dev_set_name(&rdev->dev, "%02x:s:%04x", mport->id, + rdev->comp_tag & RIO_CTAG_UDEVID); + else + dev_set_name(&rdev->dev, "%02x:e:%04x", mport->id, + rdev->comp_tag & RIO_CTAG_UDEVID); + + INIT_LIST_HEAD(&rdev->net_list); + rdev->dev.parent = &mport->net->dev; + rio_attach_device(rdev); + rdev->dev.release = rio_release_dev; + + if (rdev->dst_ops & RIO_DST_OPS_DOORBELL) + rio_init_dbell_res(&rdev->riores[RIO_DOORBELL_RESOURCE], + 0, 0xffff); + err = rio_add_device(rdev); + if (err) + goto cleanup; + rio_dev_get(rdev); + + return 0; +cleanup: + kfree(rdev); + return err; +} + +static int rio_mport_del_riodev(struct mport_cdev_priv *priv, void __user *arg) +{ + struct rio_rdev_info dev_info; + struct rio_dev *rdev = NULL; + struct device *dev; + struct rio_mport *mport; + struct rio_net *net; + + if (copy_from_user(&dev_info, arg, sizeof(dev_info))) + return -EFAULT; + + mport = priv->md->mport; + + /* If device name is specified, removal by name has priority */ + if (strlen(dev_info.name)) { + dev = bus_find_device_by_name(&rio_bus_type, NULL, + dev_info.name); + if (dev) + rdev = to_rio_dev(dev); + } else { + do { + rdev = rio_get_comptag(dev_info.comptag, rdev); + if (rdev && rdev->dev.parent == &mport->net->dev && + rdev->destid == (u16)dev_info.destid && + rdev->hopcount == (u8)dev_info.hopcount) + break; + } while (rdev); + } + + if (!rdev) { + rmcd_debug(RDEV, + "device name:%s ct:0x%x did:0x%x hc:0x%x not found", + dev_info.name, dev_info.comptag, dev_info.destid, + dev_info.hopcount); + return -ENODEV; + } + + net = rdev->net; + rio_dev_put(rdev); + rio_del_device(rdev, RIO_DEVICE_SHUTDOWN); + + if (list_empty(&net->devices)) { + rio_free_net(net); + mport->net = NULL; + } + + return 0; +} + +/* + * Mport cdev management + */ + +/* + * mport_cdev_open() - Open character device (mport) + */ +static int mport_cdev_open(struct inode *inode, struct file *filp) +{ + int ret; + int minor = iminor(inode); + struct mport_dev *chdev; + struct mport_cdev_priv *priv; + + /* Test for valid device */ + if (minor >= RIO_MAX_MPORTS) { + rmcd_error("Invalid minor device number"); + return -EINVAL; + } + + chdev = container_of(inode->i_cdev, struct mport_dev, cdev); + + rmcd_debug(INIT, "%s filp=%p", dev_name(&chdev->dev), filp); + + if (atomic_read(&chdev->active) == 0) + return -ENODEV; + + get_device(&chdev->dev); + + priv = kzalloc(sizeof(*priv), GFP_KERNEL); + if (!priv) { + put_device(&chdev->dev); + return -ENOMEM; + } + + priv->md = chdev; + + mutex_lock(&chdev->file_mutex); + list_add_tail(&priv->list, &chdev->file_list); + mutex_unlock(&chdev->file_mutex); + + INIT_LIST_HEAD(&priv->db_filters); + INIT_LIST_HEAD(&priv->pw_filters); + spin_lock_init(&priv->fifo_lock); + init_waitqueue_head(&priv->event_rx_wait); + ret = kfifo_alloc(&priv->event_fifo, + sizeof(struct rio_event) * MPORT_EVENT_DEPTH, + GFP_KERNEL); + if (ret < 0) { + dev_err(&chdev->dev, DRV_NAME ": kfifo_alloc failed\n"); + ret = -ENOMEM; + goto err_fifo; + } + +#ifdef CONFIG_RAPIDIO_DMA_ENGINE + INIT_LIST_HEAD(&priv->async_list); + INIT_LIST_HEAD(&priv->pend_list); + spin_lock_init(&priv->req_lock); + mutex_init(&priv->dma_lock); +#endif + + filp->private_data = priv; + goto out; +err_fifo: + kfree(priv); +out: + return ret; +} + +static int mport_cdev_fasync(int fd, struct file *filp, int mode) +{ + struct mport_cdev_priv *priv = filp->private_data; + + return fasync_helper(fd, filp, mode, &priv->async_queue); +} + +#ifdef CONFIG_RAPIDIO_DMA_ENGINE +static void mport_cdev_release_dma(struct file *filp) +{ + struct mport_cdev_priv *priv = filp->private_data; + struct mport_dev *md; + struct mport_dma_req *req, *req_next; + unsigned long tmo = msecs_to_jiffies(dma_timeout); + long wret; + LIST_HEAD(list); + + rmcd_debug(EXIT, "from filp=%p %s(%d)", + filp, current->comm, task_pid_nr(current)); + + if (!priv->dmach) { + rmcd_debug(EXIT, "No DMA channel for filp=%p", filp); + return; + } + + md = priv->md; + + flush_workqueue(dma_wq); + + spin_lock(&priv->req_lock); + if (!list_empty(&priv->async_list)) { + rmcd_debug(EXIT, "async list not empty filp=%p %s(%d)", + filp, current->comm, task_pid_nr(current)); + list_splice_init(&priv->async_list, &list); + } + spin_unlock(&priv->req_lock); + + if (!list_empty(&list)) { + rmcd_debug(EXIT, "temp list not empty"); + list_for_each_entry_safe(req, req_next, &list, node) { + rmcd_debug(EXIT, "free req->filp=%p cookie=%d compl=%s", + req->filp, req->cookie, + completion_done(&req->req_comp)?"yes":"no"); + list_del(&req->node); + dma_req_free(req); + } + } + + if (!list_empty(&priv->pend_list)) { + rmcd_debug(EXIT, "Free pending DMA requests for filp=%p %s(%d)", + filp, current->comm, task_pid_nr(current)); + list_for_each_entry_safe(req, + req_next, &priv->pend_list, node) { + rmcd_debug(EXIT, "free req->filp=%p cookie=%d compl=%s", + req->filp, req->cookie, + completion_done(&req->req_comp)?"yes":"no"); + list_del(&req->node); + dma_req_free(req); + } + } + + put_dma_channel(priv); + wret = wait_for_completion_interruptible_timeout(&priv->comp, tmo); + + if (wret <= 0) { + rmcd_error("%s(%d) failed waiting for DMA release err=%ld", + current->comm, task_pid_nr(current), wret); + } + + spin_lock(&priv->req_lock); + + if (!list_empty(&priv->pend_list)) { + rmcd_debug(EXIT, "ATTN: pending DMA requests, filp=%p %s(%d)", + filp, current->comm, task_pid_nr(current)); + } + + spin_unlock(&priv->req_lock); + + if (priv->dmach != priv->md->dma_chan) { + rmcd_debug(EXIT, "Release DMA channel for filp=%p %s(%d)", + filp, current->comm, task_pid_nr(current)); + rio_release_dma(priv->dmach); + } else { + rmcd_debug(EXIT, "Adjust default DMA channel refcount"); + kref_put(&md->dma_ref, mport_release_def_dma); + } + + priv->dmach = NULL; +} +#else +#define mport_cdev_release_dma(priv) do {} while (0) +#endif + +/* + * mport_cdev_release() - Release character device + */ +static int mport_cdev_release(struct inode *inode, struct file *filp) +{ + struct mport_cdev_priv *priv = filp->private_data; + struct mport_dev *chdev; + struct rio_mport_pw_filter *pw_filter, *pw_filter_next; + struct rio_mport_db_filter *db_filter, *db_filter_next; + struct rio_mport_mapping *map, *_map; + unsigned long flags; + + rmcd_debug(EXIT, "%s filp=%p", dev_name(&priv->md->dev), filp); + + chdev = priv->md; + mport_cdev_release_dma(filp); + + priv->event_mask = 0; + + spin_lock_irqsave(&chdev->pw_lock, flags); + if (!list_empty(&priv->pw_filters)) { + list_for_each_entry_safe(pw_filter, pw_filter_next, + &priv->pw_filters, priv_node) + rio_mport_delete_pw_filter(pw_filter); + } + spin_unlock_irqrestore(&chdev->pw_lock, flags); + + spin_lock_irqsave(&chdev->db_lock, flags); + list_for_each_entry_safe(db_filter, db_filter_next, + &priv->db_filters, priv_node) { + rio_mport_delete_db_filter(db_filter); + } + spin_unlock_irqrestore(&chdev->db_lock, flags); + + kfifo_free(&priv->event_fifo); + + mutex_lock(&chdev->buf_mutex); + list_for_each_entry_safe(map, _map, &chdev->mappings, node) { + if (map->filp == filp) { + rmcd_debug(EXIT, "release mapping %p filp=%p", + map->virt_addr, filp); + kref_put(&map->ref, mport_release_mapping); + } + } + mutex_unlock(&chdev->buf_mutex); + + mport_cdev_fasync(-1, filp, 0); + filp->private_data = NULL; + mutex_lock(&chdev->file_mutex); + list_del(&priv->list); + mutex_unlock(&chdev->file_mutex); + put_device(&chdev->dev); + kfree(priv); + return 0; +} + +/* + * mport_cdev_ioctl() - IOCTLs for character device + */ +static long mport_cdev_ioctl(struct file *filp, + unsigned int cmd, unsigned long arg) +{ + int err = -EINVAL; + struct mport_cdev_priv *data = filp->private_data; + struct mport_dev *md = data->md; + + if (atomic_read(&md->active) == 0) + return -ENODEV; + + switch (cmd) { + case RIO_MPORT_MAINT_READ_LOCAL: + return rio_mport_maint_rd(data, (void __user *)arg, 1); + case RIO_MPORT_MAINT_WRITE_LOCAL: + return rio_mport_maint_wr(data, (void __user *)arg, 1); + case RIO_MPORT_MAINT_READ_REMOTE: + return rio_mport_maint_rd(data, (void __user *)arg, 0); + case RIO_MPORT_MAINT_WRITE_REMOTE: + return rio_mport_maint_wr(data, (void __user *)arg, 0); + case RIO_MPORT_MAINT_HDID_SET: + return maint_hdid_set(data, (void __user *)arg); + case RIO_MPORT_MAINT_COMPTAG_SET: + return maint_comptag_set(data, (void __user *)arg); + case RIO_MPORT_MAINT_PORT_IDX_GET: + return maint_port_idx_get(data, (void __user *)arg); + case RIO_MPORT_GET_PROPERTIES: + md->properties.hdid = md->mport->host_deviceid; + if (copy_to_user((void __user *)arg, &(data->md->properties), + sizeof(data->md->properties))) + return -EFAULT; + return 0; + case RIO_ENABLE_DOORBELL_RANGE: + return rio_mport_add_db_filter(data, (void __user *)arg); + case RIO_DISABLE_DOORBELL_RANGE: + return rio_mport_remove_db_filter(data, (void __user *)arg); + case RIO_ENABLE_PORTWRITE_RANGE: + return rio_mport_add_pw_filter(data, (void __user *)arg); + case RIO_DISABLE_PORTWRITE_RANGE: + return rio_mport_remove_pw_filter(data, (void __user *)arg); + case RIO_SET_EVENT_MASK: + data->event_mask = arg; + return 0; + case RIO_GET_EVENT_MASK: + if (copy_to_user((void __user *)arg, &data->event_mask, + sizeof(data->event_mask))) + return -EFAULT; + return 0; + case RIO_MAP_OUTBOUND: + return rio_mport_obw_map(filp, (void __user *)arg); + case RIO_MAP_INBOUND: + return rio_mport_map_inbound(filp, (void __user *)arg); + case RIO_UNMAP_OUTBOUND: + return rio_mport_obw_free(filp, (void __user *)arg); + case RIO_UNMAP_INBOUND: + return rio_mport_inbound_free(filp, (void __user *)arg); + case RIO_ALLOC_DMA: + return rio_mport_alloc_dma(filp, (void __user *)arg); + case RIO_FREE_DMA: + return rio_mport_free_dma(filp, (void __user *)arg); + case RIO_WAIT_FOR_ASYNC: + return rio_mport_wait_for_async_dma(filp, (void __user *)arg); + case RIO_TRANSFER: + return rio_mport_transfer_ioctl(filp, (void __user *)arg); + case RIO_DEV_ADD: + return rio_mport_add_riodev(data, (void __user *)arg); + case RIO_DEV_DEL: + return rio_mport_del_riodev(data, (void __user *)arg); + default: + break; + } + + return err; +} + +/* + * mport_release_mapping - free mapping resources and info structure + * @ref: a pointer to the kref within struct rio_mport_mapping + * + * NOTE: Shall be called while holding buf_mutex. + */ +static void mport_release_mapping(struct kref *ref) +{ + struct rio_mport_mapping *map = + container_of(ref, struct rio_mport_mapping, ref); + struct rio_mport *mport = map->md->mport; + + rmcd_debug(MMAP, "type %d mapping @ %p (phys = %pad) for %s", + map->dir, map->virt_addr, + &map->phys_addr, mport->name); + + list_del(&map->node); + + switch (map->dir) { + case MAP_INBOUND: + rio_unmap_inb_region(mport, map->phys_addr); + case MAP_DMA: + dma_free_coherent(mport->dev.parent, map->size, + map->virt_addr, map->phys_addr); + break; + case MAP_OUTBOUND: + rio_unmap_outb_region(mport, map->rioid, map->rio_addr); + break; + } + kfree(map); +} + +static void mport_mm_open(struct vm_area_struct *vma) +{ + struct rio_mport_mapping *map = vma->vm_private_data; + +rmcd_debug(MMAP, "0x%pad", &map->phys_addr); + kref_get(&map->ref); +} + +static void mport_mm_close(struct vm_area_struct *vma) +{ + struct rio_mport_mapping *map = vma->vm_private_data; + +rmcd_debug(MMAP, "0x%pad", &map->phys_addr); + mutex_lock(&map->md->buf_mutex); + kref_put(&map->ref, mport_release_mapping); + mutex_unlock(&map->md->buf_mutex); +} + +static const struct vm_operations_struct vm_ops = { + .open = mport_mm_open, + .close = mport_mm_close, +}; + +static int mport_cdev_mmap(struct file *filp, struct vm_area_struct *vma) +{ + struct mport_cdev_priv *priv = filp->private_data; + struct mport_dev *md; + size_t size = vma->vm_end - vma->vm_start; + dma_addr_t baddr; + unsigned long offset; + int found = 0, ret; + struct rio_mport_mapping *map; + + rmcd_debug(MMAP, "0x%x bytes at offset 0x%lx", + (unsigned int)size, vma->vm_pgoff); + + md = priv->md; + baddr = ((dma_addr_t)vma->vm_pgoff << PAGE_SHIFT); + + mutex_lock(&md->buf_mutex); + list_for_each_entry(map, &md->mappings, node) { + if (baddr >= map->phys_addr && + baddr < (map->phys_addr + map->size)) { + found = 1; + break; + } + } + mutex_unlock(&md->buf_mutex); + + if (!found) + return -ENOMEM; + + offset = baddr - map->phys_addr; + + if (size + offset > map->size) + return -EINVAL; + + vma->vm_pgoff = offset >> PAGE_SHIFT; + rmcd_debug(MMAP, "MMAP adjusted offset = 0x%lx", vma->vm_pgoff); + + if (map->dir == MAP_INBOUND || map->dir == MAP_DMA) + ret = dma_mmap_coherent(md->mport->dev.parent, vma, + map->virt_addr, map->phys_addr, map->size); + else if (map->dir == MAP_OUTBOUND) { + vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); + ret = vm_iomap_memory(vma, map->phys_addr, map->size); + } else { + rmcd_error("Attempt to mmap unsupported mapping type"); + ret = -EIO; + } + + if (!ret) { + vma->vm_private_data = map; + vma->vm_ops = &vm_ops; + mport_mm_open(vma); + } else { + rmcd_error("MMAP exit with err=%d", ret); + } + + return ret; +} + +static unsigned int mport_cdev_poll(struct file *filp, poll_table *wait) +{ + struct mport_cdev_priv *priv = filp->private_data; + + poll_wait(filp, &priv->event_rx_wait, wait); + if (kfifo_len(&priv->event_fifo)) + return POLLIN | POLLRDNORM; + + return 0; +} + +static ssize_t mport_read(struct file *filp, char __user *buf, size_t count, + loff_t *ppos) +{ + struct mport_cdev_priv *priv = filp->private_data; + int copied; + ssize_t ret; + + if (!count) + return 0; + + if (kfifo_is_empty(&priv->event_fifo) && + (filp->f_flags & O_NONBLOCK)) + return -EAGAIN; + + if (count % sizeof(struct rio_event)) + return -EINVAL; + + ret = wait_event_interruptible(priv->event_rx_wait, + kfifo_len(&priv->event_fifo) != 0); + if (ret) + return ret; + + while (ret < count) { + if (kfifo_to_user(&priv->event_fifo, buf, + sizeof(struct rio_event), &copied)) + return -EFAULT; + ret += copied; + buf += copied; + } + + return ret; +} + +static ssize_t mport_write(struct file *filp, const char __user *buf, + size_t count, loff_t *ppos) +{ + struct mport_cdev_priv *priv = filp->private_data; + struct rio_mport *mport = priv->md->mport; + struct rio_event event; + int len, ret; + + if (!count) + return 0; + + if (count % sizeof(event)) + return -EINVAL; + + len = 0; + while ((count - len) >= (int)sizeof(event)) { + if (copy_from_user(&event, buf, sizeof(event))) + return -EFAULT; + + if (event.header != RIO_DOORBELL) + return -EINVAL; + + ret = rio_mport_send_doorbell(mport, + (u16)event.u.doorbell.rioid, + event.u.doorbell.payload); + if (ret < 0) + return ret; + + len += sizeof(event); + buf += sizeof(event); + } + + return len; +} + +static const struct file_operations mport_fops = { + .owner = THIS_MODULE, + .open = mport_cdev_open, + .release = mport_cdev_release, + .poll = mport_cdev_poll, + .read = mport_read, + .write = mport_write, + .mmap = mport_cdev_mmap, + .fasync = mport_cdev_fasync, + .unlocked_ioctl = mport_cdev_ioctl +}; + +/* + * Character device management + */ + +static void mport_device_release(struct device *dev) +{ + struct mport_dev *md; + + rmcd_debug(EXIT, "%s", dev_name(dev)); + md = container_of(dev, struct mport_dev, dev); + kfree(md); +} + +/* + * mport_cdev_add() - Create mport_dev from rio_mport + * @mport: RapidIO master port + */ +static struct mport_dev *mport_cdev_add(struct rio_mport *mport) +{ + int ret = 0; + struct mport_dev *md; + struct rio_mport_attr attr; + + md = kzalloc(sizeof(struct mport_dev), GFP_KERNEL); + if (!md) { + rmcd_error("Unable allocate a device object"); + return NULL; + } + + md->mport = mport; + mutex_init(&md->buf_mutex); + mutex_init(&md->file_mutex); + INIT_LIST_HEAD(&md->file_list); + cdev_init(&md->cdev, &mport_fops); + md->cdev.owner = THIS_MODULE; + ret = cdev_add(&md->cdev, MKDEV(MAJOR(dev_number), mport->id), 1); + if (ret < 0) { + kfree(md); + rmcd_error("Unable to register a device, err=%d", ret); + return NULL; + } + + md->dev.devt = md->cdev.dev; + md->dev.class = dev_class; + md->dev.parent = &mport->dev; + md->dev.release = mport_device_release; + dev_set_name(&md->dev, DEV_NAME "%d", mport->id); + atomic_set(&md->active, 1); + + ret = device_register(&md->dev); + if (ret) { + rmcd_error("Failed to register mport %d (err=%d)", + mport->id, ret); + goto err_cdev; + } + + get_device(&md->dev); + + INIT_LIST_HEAD(&md->doorbells); + spin_lock_init(&md->db_lock); + INIT_LIST_HEAD(&md->portwrites); + spin_lock_init(&md->pw_lock); + INIT_LIST_HEAD(&md->mappings); + + md->properties.id = mport->id; + md->properties.sys_size = mport->sys_size; + md->properties.hdid = mport->host_deviceid; + md->properties.index = mport->index; + + /* The transfer_mode property will be returned through mport query + * interface + */ +#ifdef CONFIG_PPC /* for now: only on Freescale's SoCs */ + md->properties.transfer_mode |= RIO_TRANSFER_MODE_MAPPED; +#else + md->properties.transfer_mode |= RIO_TRANSFER_MODE_TRANSFER; +#endif + ret = rio_query_mport(mport, &attr); + if (!ret) { + md->properties.flags = attr.flags; + md->properties.link_speed = attr.link_speed; + md->properties.link_width = attr.link_width; + md->properties.dma_max_sge = attr.dma_max_sge; + md->properties.dma_max_size = attr.dma_max_size; + md->properties.dma_align = attr.dma_align; + md->properties.cap_sys_size = 0; + md->properties.cap_transfer_mode = 0; + md->properties.cap_addr_size = 0; + } else + pr_info(DRV_PREFIX "Failed to obtain info for %s cdev(%d:%d)\n", + mport->name, MAJOR(dev_number), mport->id); + + mutex_lock(&mport_devs_lock); + list_add_tail(&md->node, &mport_devs); + mutex_unlock(&mport_devs_lock); + + pr_info(DRV_PREFIX "Added %s cdev(%d:%d)\n", + mport->name, MAJOR(dev_number), mport->id); + + return md; + +err_cdev: + cdev_del(&md->cdev); + kfree(md); + return NULL; +} + +/* + * mport_cdev_terminate_dma() - Stop all active DMA data transfers and release + * associated DMA channels. + */ +static void mport_cdev_terminate_dma(struct mport_dev *md) +{ +#ifdef CONFIG_RAPIDIO_DMA_ENGINE + struct mport_cdev_priv *client; + + rmcd_debug(DMA, "%s", dev_name(&md->dev)); + + mutex_lock(&md->file_mutex); + list_for_each_entry(client, &md->file_list, list) { + if (client->dmach) { + dmaengine_terminate_all(client->dmach); + rio_release_dma(client->dmach); + } + } + mutex_unlock(&md->file_mutex); + + if (md->dma_chan) { + dmaengine_terminate_all(md->dma_chan); + rio_release_dma(md->dma_chan); + md->dma_chan = NULL; + } +#endif +} + + +/* + * mport_cdev_kill_fasync() - Send SIGIO signal to all processes with open + * mport_cdev files. + */ +static int mport_cdev_kill_fasync(struct mport_dev *md) +{ + unsigned int files = 0; + struct mport_cdev_priv *client; + + mutex_lock(&md->file_mutex); + list_for_each_entry(client, &md->file_list, list) { + if (client->async_queue) + kill_fasync(&client->async_queue, SIGIO, POLL_HUP); + files++; + } + mutex_unlock(&md->file_mutex); + return files; +} + +/* + * mport_cdev_remove() - Remove mport character device + * @dev: Mport device to remove + */ +static void mport_cdev_remove(struct mport_dev *md) +{ + struct rio_mport_mapping *map, *_map; + + rmcd_debug(EXIT, "Remove %s cdev", md->mport->name); + atomic_set(&md->active, 0); + mport_cdev_terminate_dma(md); + rio_del_mport_pw_handler(md->mport, md, rio_mport_pw_handler); + cdev_del(&(md->cdev)); + mport_cdev_kill_fasync(md); + + flush_workqueue(dma_wq); + + /* TODO: do we need to give clients some time to close file + * descriptors? Simple wait for XX, or kref? + */ + + /* + * Release DMA buffers allocated for the mport device. + * Disable associated inbound Rapidio requests mapping if applicable. + */ + mutex_lock(&md->buf_mutex); + list_for_each_entry_safe(map, _map, &md->mappings, node) { + kref_put(&map->ref, mport_release_mapping); + } + mutex_unlock(&md->buf_mutex); + + if (!list_empty(&md->mappings)) + rmcd_warn("WARNING: %s pending mappings on removal", + md->mport->name); + + rio_release_inb_dbell(md->mport, 0, 0x0fff); + + device_unregister(&md->dev); + put_device(&md->dev); +} + +/* + * RIO rio_mport_interface driver + */ + +/* + * mport_add_mport() - Add rio_mport from LDM device struct + * @dev: Linux device model struct + * @class_intf: Linux class_interface + */ +static int mport_add_mport(struct device *dev, + struct class_interface *class_intf) +{ + struct rio_mport *mport = NULL; + struct mport_dev *chdev = NULL; + + mport = to_rio_mport(dev); + if (!mport) + return -ENODEV; + + chdev = mport_cdev_add(mport); + if (!chdev) + return -ENODEV; + + return 0; +} + +/* + * mport_remove_mport() - Remove rio_mport from global list + * TODO remove device from global mport_dev list + */ +static void mport_remove_mport(struct device *dev, + struct class_interface *class_intf) +{ + struct rio_mport *mport = NULL; + struct mport_dev *chdev; + int found = 0; + + mport = to_rio_mport(dev); + rmcd_debug(EXIT, "Remove %s", mport->name); + + mutex_lock(&mport_devs_lock); + list_for_each_entry(chdev, &mport_devs, node) { + if (chdev->mport->id == mport->id) { + atomic_set(&chdev->active, 0); + list_del(&chdev->node); + found = 1; + break; + } + } + mutex_unlock(&mport_devs_lock); + + if (found) + mport_cdev_remove(chdev); +} + +/* the rio_mport_interface is used to handle local mport devices */ +static struct class_interface rio_mport_interface __refdata = { + .class = &rio_mport_class, + .add_dev = mport_add_mport, + .remove_dev = mport_remove_mport, +}; + +/* + * Linux kernel module + */ + +/* + * mport_init - Driver module loading + */ +static int __init mport_init(void) +{ + int ret; + + /* Create device class needed by udev */ + dev_class = class_create(THIS_MODULE, DRV_NAME); + if (!dev_class) { + rmcd_error("Unable to create " DRV_NAME " class"); + return -EINVAL; + } + + ret = alloc_chrdev_region(&dev_number, 0, RIO_MAX_MPORTS, DRV_NAME); + if (ret < 0) + goto err_chr; + + rmcd_debug(INIT, "Registered class with major=%d", MAJOR(dev_number)); + + /* Register to rio_mport_interface */ + ret = class_interface_register(&rio_mport_interface); + if (ret) { + rmcd_error("class_interface_register() failed, err=%d", ret); + goto err_cli; + } + + dma_wq = create_singlethread_workqueue("dma_wq"); + if (!dma_wq) { + rmcd_error("failed to create DMA work queue"); + ret = -ENOMEM; + goto err_wq; + } + + return 0; + +err_wq: + class_interface_unregister(&rio_mport_interface); +err_cli: + unregister_chrdev_region(dev_number, RIO_MAX_MPORTS); +err_chr: + class_destroy(dev_class); + return ret; +} + +/** + * mport_exit - Driver module unloading + */ +static void __exit mport_exit(void) +{ + class_interface_unregister(&rio_mport_interface); + class_destroy(dev_class); + unregister_chrdev_region(dev_number, RIO_MAX_MPORTS); + destroy_workqueue(dma_wq); +} + +module_init(mport_init); +module_exit(mport_exit); diff --git a/include/linux/rio_mport_cdev.h b/include/linux/rio_mport_cdev.h new file mode 100644 index 000000000000..b65d19df76d2 --- /dev/null +++ b/include/linux/rio_mport_cdev.h @@ -0,0 +1,271 @@ +/* + * Copyright (c) 2015-2016, Integrated Device Technology Inc. + * Copyright (c) 2015, Prodrive Technologies + * Copyright (c) 2015, Texas Instruments Incorporated + * Copyright (c) 2015, RapidIO Trade Association + * All rights reserved. + * + * This software is available to you under a choice of one of two licenses. + * You may choose to be licensed under the terms of the GNU General Public + * License(GPL) Version 2, or the BSD-3 Clause license below: + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors + * may be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR + * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _RIO_MPORT_CDEV_H_ +#define _RIO_MPORT_CDEV_H_ + +#ifndef __user +#define __user +#endif + +struct rio_mport_maint_io { + uint32_t rioid; /* destID of remote device */ + uint32_t hopcount; /* hopcount to remote device */ + uint32_t offset; /* offset in register space */ + size_t length; /* length in bytes */ + void __user *buffer; /* data buffer */ +}; + +/* + * Definitions for RapidIO data transfers: + * - memory mapped (MAPPED) + * - packet generation from memory (TRANSFER) + */ +#define RIO_TRANSFER_MODE_MAPPED (1 << 0) +#define RIO_TRANSFER_MODE_TRANSFER (1 << 1) +#define RIO_CAP_DBL_SEND (1 << 2) +#define RIO_CAP_DBL_RECV (1 << 3) +#define RIO_CAP_PW_SEND (1 << 4) +#define RIO_CAP_PW_RECV (1 << 5) +#define RIO_CAP_MAP_OUTB (1 << 6) +#define RIO_CAP_MAP_INB (1 << 7) + +struct rio_mport_properties { + uint16_t hdid; + uint8_t id; /* Physical port ID */ + uint8_t index; + uint32_t flags; + uint32_t sys_size; /* Default addressing size */ + uint8_t port_ok; + uint8_t link_speed; + uint8_t link_width; + uint32_t dma_max_sge; + uint32_t dma_max_size; + uint32_t dma_align; + uint32_t transfer_mode; /* Default transfer mode */ + uint32_t cap_sys_size; /* Capable system sizes */ + uint32_t cap_addr_size; /* Capable addressing sizes */ + uint32_t cap_transfer_mode; /* Capable transfer modes */ + uint32_t cap_mport; /* Mport capabilities */ +}; + +/* + * Definitions for RapidIO events; + * - incoming port-writes + * - incoming doorbells + */ +#define RIO_DOORBELL (1 << 0) +#define RIO_PORTWRITE (1 << 1) + +struct rio_doorbell { + uint32_t rioid; + uint16_t payload; +}; + +struct rio_doorbell_filter { + uint32_t rioid; /* 0xffffffff to match all ids */ + uint16_t low; + uint16_t high; +}; + + +struct rio_portwrite { + uint32_t payload[16]; +}; + +struct rio_pw_filter { + uint32_t mask; + uint32_t low; + uint32_t high; +}; + +/* RapidIO base address for inbound requests set to value defined below + * indicates that no specific RIO-to-local address translation is requested + * and driver should use direct (one-to-one) address mapping. +*/ +#define RIO_MAP_ANY_ADDR (uint64_t)(~((uint64_t) 0)) + +struct rio_mmap { + uint32_t rioid; + uint64_t rio_addr; + uint64_t length; + uint64_t handle; + void *address; +}; + +struct rio_dma_mem { + uint64_t length; /* length of DMA memory */ + uint64_t dma_handle; /* handle associated with this memory */ + void *buffer; /* pointer to this memory */ +}; + + +struct rio_event { + unsigned int header; /* event type RIO_DOORBELL or RIO_PORTWRITE */ + union { + struct rio_doorbell doorbell; /* header for RIO_DOORBELL */ + struct rio_portwrite portwrite; /* header for RIO_PORTWRITE */ + } u; +}; + +enum rio_transfer_sync { + RIO_TRANSFER_SYNC, /* synchronous transfer */ + RIO_TRANSFER_ASYNC, /* asynchronous transfer */ + RIO_TRANSFER_FAF, /* fire-and-forget transfer */ +}; + +enum rio_transfer_dir { + RIO_TRANSFER_DIR_READ, /* Read operation */ + RIO_TRANSFER_DIR_WRITE, /* Write operation */ +}; + +/* + * RapidIO data exchange transactions are lists of individual transfers. Each + * transfer exchanges data between two RapidIO devices by remote direct memory + * access and has its own completion code. + * + * The RapidIO specification defines four types of data exchange requests: + * NREAD, NWRITE, SWRITE and NWRITE_R. The RapidIO DMA channel interface allows + * to specify the required type of write operation or combination of them when + * only the last data packet requires response. + * + * NREAD: read up to 256 bytes from remote device memory into local memory + * NWRITE: write up to 256 bytes from local memory to remote device memory + * without confirmation + * SWRITE: as NWRITE, but all addresses and payloads must be 64-bit aligned + * NWRITE_R: as NWRITE, but expect acknowledgment from remote device. + * + * The default exchange is chosen from NREAD and any of the WRITE modes as the + * driver sees fit. For write requests the user can explicitly choose between + * any of the write modes for each transaction. + */ +enum rio_exchange { + RIO_EXCHANGE_DEFAULT, /* Default method */ + RIO_EXCHANGE_NWRITE, /* All packets using NWRITE */ + RIO_EXCHANGE_SWRITE, /* All packets using SWRITE */ + RIO_EXCHANGE_NWRITE_R, /* Last packet NWRITE_R, others NWRITE */ + RIO_EXCHANGE_SWRITE_R, /* Last packet NWRITE_R, others SWRITE */ + RIO_EXCHANGE_NWRITE_R_ALL, /* All packets using NWRITE_R */ +}; + +struct rio_transfer_io { + uint32_t rioid; /* Target destID */ + uint64_t rio_addr; /* Address in target's RIO mem space */ + enum rio_exchange method; /* Data exchange method */ + void __user *loc_addr; + uint64_t handle; + uint64_t offset; /* Offset in buffer */ + uint64_t length; /* Length in bytes */ + uint32_t completion_code; /* Completion code for this transfer */ +}; + +struct rio_transaction { + uint32_t transfer_mode; /* Data transfer mode */ + enum rio_transfer_sync sync; /* Synchronization method */ + enum rio_transfer_dir dir; /* Transfer direction */ + size_t count; /* Number of transfers */ + struct rio_transfer_io __user *block; /* Array of transfers */ +}; + +struct rio_async_tx_wait { + uint32_t token; /* DMA transaction ID token */ + uint32_t timeout; /* Wait timeout in msec, if 0 use default TO */ +}; + +#define RIO_MAX_DEVNAME_SZ 20 + +struct rio_rdev_info { + uint32_t destid; + uint8_t hopcount; + uint32_t comptag; + char name[RIO_MAX_DEVNAME_SZ + 1]; +}; + +/* Driver IOCTL codes */ +#define RIO_MPORT_DRV_MAGIC 'm' + +#define RIO_MPORT_MAINT_HDID_SET \ + _IOW(RIO_MPORT_DRV_MAGIC, 1, uint16_t) +#define RIO_MPORT_MAINT_COMPTAG_SET \ + _IOW(RIO_MPORT_DRV_MAGIC, 2, uint32_t) +#define RIO_MPORT_MAINT_PORT_IDX_GET \ + _IOR(RIO_MPORT_DRV_MAGIC, 3, uint32_t) +#define RIO_MPORT_GET_PROPERTIES \ + _IOR(RIO_MPORT_DRV_MAGIC, 4, struct rio_mport_properties) +#define RIO_MPORT_MAINT_READ_LOCAL \ + _IOR(RIO_MPORT_DRV_MAGIC, 5, struct rio_mport_maint_io) +#define RIO_MPORT_MAINT_WRITE_LOCAL \ + _IOW(RIO_MPORT_DRV_MAGIC, 6, struct rio_mport_maint_io) +#define RIO_MPORT_MAINT_READ_REMOTE \ + _IOR(RIO_MPORT_DRV_MAGIC, 7, struct rio_mport_maint_io) +#define RIO_MPORT_MAINT_WRITE_REMOTE \ + _IOW(RIO_MPORT_DRV_MAGIC, 8, struct rio_mport_maint_io) +#define RIO_ENABLE_DOORBELL_RANGE \ + _IOW(RIO_MPORT_DRV_MAGIC, 9, struct rio_doorbell_filter) +#define RIO_DISABLE_DOORBELL_RANGE \ + _IOW(RIO_MPORT_DRV_MAGIC, 10, struct rio_doorbell_filter) +#define RIO_ENABLE_PORTWRITE_RANGE \ + _IOW(RIO_MPORT_DRV_MAGIC, 11, struct rio_pw_filter) +#define RIO_DISABLE_PORTWRITE_RANGE \ + _IOW(RIO_MPORT_DRV_MAGIC, 12, struct rio_pw_filter) +#define RIO_SET_EVENT_MASK \ + _IOW(RIO_MPORT_DRV_MAGIC, 13, unsigned int) +#define RIO_GET_EVENT_MASK \ + _IOR(RIO_MPORT_DRV_MAGIC, 14, unsigned int) +#define RIO_MAP_OUTBOUND \ + _IOWR(RIO_MPORT_DRV_MAGIC, 15, struct rio_mmap) +#define RIO_UNMAP_OUTBOUND \ + _IOW(RIO_MPORT_DRV_MAGIC, 16, struct rio_mmap) +#define RIO_MAP_INBOUND \ + _IOWR(RIO_MPORT_DRV_MAGIC, 17, struct rio_mmap) +#define RIO_UNMAP_INBOUND \ + _IOW(RIO_MPORT_DRV_MAGIC, 18, uint64_t) +#define RIO_ALLOC_DMA \ + _IOWR(RIO_MPORT_DRV_MAGIC, 19, struct rio_dma_mem) +#define RIO_FREE_DMA \ + _IOW(RIO_MPORT_DRV_MAGIC, 20, uint64_t) +#define RIO_TRANSFER \ + _IOWR(RIO_MPORT_DRV_MAGIC, 21, struct rio_transaction) +#define RIO_WAIT_FOR_ASYNC \ + _IOW(RIO_MPORT_DRV_MAGIC, 22, struct rio_async_tx_wait) +#define RIO_DEV_ADD \ + _IOW(RIO_MPORT_DRV_MAGIC, 23, struct rio_rdev_info) +#define RIO_DEV_DEL \ + _IOW(RIO_MPORT_DRV_MAGIC, 24, struct rio_rdev_info) + +#endif /* _RIO_MPORT_CDEV_H_ */ diff --git a/include/uapi/linux/Kbuild b/include/uapi/linux/Kbuild index 0495884defc1..b71fd0b5cbad 100644 --- a/include/uapi/linux/Kbuild +++ b/include/uapi/linux/Kbuild @@ -354,6 +354,7 @@ header-y += reiserfs_fs.h header-y += reiserfs_xattr.h header-y += resource.h header-y += rfkill.h +header-y += rio_mport_cdev.h header-y += romfs_fs.h header-y += rose.h header-y += route.h -- cgit v1.2.3-59-g8ed1b From 5c9a8750a6409c63a0f01d51a9024861022f6593 Mon Sep 17 00:00:00 2001 From: Dmitry Vyukov Date: Tue, 22 Mar 2016 14:27:30 -0700 Subject: kernel: add kcov code coverage kcov provides code coverage collection for coverage-guided fuzzing (randomized testing). Coverage-guided fuzzing is a testing technique that uses coverage feedback to determine new interesting inputs to a system. A notable user-space example is AFL (http://lcamtuf.coredump.cx/afl/). However, this technique is not widely used for kernel testing due to missing compiler and kernel support. kcov does not aim to collect as much coverage as possible. It aims to collect more or less stable coverage that is function of syscall inputs. To achieve this goal it does not collect coverage in soft/hard interrupts and instrumentation of some inherently non-deterministic or non-interesting parts of kernel is disbled (e.g. scheduler, locking). Currently there is a single coverage collection mode (tracing), but the API anticipates additional collection modes. Initially I also implemented a second mode which exposes coverage in a fixed-size hash table of counters (what Quentin used in his original patch). I've dropped the second mode for simplicity. This patch adds the necessary support on kernel side. The complimentary compiler support was added in gcc revision 231296. We've used this support to build syzkaller system call fuzzer, which has found 90 kernel bugs in just 2 months: https://github.com/google/syzkaller/wiki/Found-Bugs We've also found 30+ bugs in our internal systems with syzkaller. Another (yet unexplored) direction where kcov coverage would greatly help is more traditional "blob mutation". For example, mounting a random blob as a filesystem, or receiving a random blob over wire. Why not gcov. Typical fuzzing loop looks as follows: (1) reset coverage, (2) execute a bit of code, (3) collect coverage, repeat. A typical coverage can be just a dozen of basic blocks (e.g. an invalid input). In such context gcov becomes prohibitively expensive as reset/collect coverage steps depend on total number of basic blocks/edges in program (in case of kernel it is about 2M). Cost of kcov depends only on number of executed basic blocks/edges. On top of that, kernel requires per-thread coverage because there are always background threads and unrelated processes that also produce coverage. With inlined gcov instrumentation per-thread coverage is not possible. kcov exposes kernel PCs and control flow to user-space which is insecure. But debugfs should not be mapped as user accessible. Based on a patch by Quentin Casasnovas. [akpm@linux-foundation.org: make task_struct.kcov_mode have type `enum kcov_mode'] [akpm@linux-foundation.org: unbreak allmodconfig] [akpm@linux-foundation.org: follow x86 Makefile layout standards] Signed-off-by: Dmitry Vyukov Reviewed-by: Kees Cook Cc: syzkaller Cc: Vegard Nossum Cc: Catalin Marinas Cc: Tavis Ormandy Cc: Will Deacon Cc: Quentin Casasnovas Cc: Kostya Serebryany Cc: Eric Dumazet Cc: Alexander Potapenko Cc: Kees Cook Cc: Bjorn Helgaas Cc: Sasha Levin Cc: David Drysdale Cc: Ard Biesheuvel Cc: Andrey Ryabinin Cc: Kirill A. Shutemov Cc: Jiri Slaby Cc: Ingo Molnar Cc: Thomas Gleixner Cc: "H. Peter Anvin" Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- Documentation/kcov.txt | 111 ++++++++++++++ Makefile | 11 +- arch/x86/Kconfig | 1 + arch/x86/boot/Makefile | 7 + arch/x86/boot/compressed/Makefile | 3 + arch/x86/entry/vdso/Makefile | 3 + arch/x86/kernel/Makefile | 6 + arch/x86/kernel/apic/Makefile | 4 + arch/x86/kernel/cpu/Makefile | 4 + arch/x86/lib/Makefile | 3 + arch/x86/mm/Makefile | 3 + arch/x86/realmode/rm/Makefile | 3 + drivers/firmware/efi/libstub/Makefile | 3 + include/linux/kcov.h | 29 ++++ include/linux/sched.h | 11 ++ include/uapi/linux/kcov.h | 10 ++ kernel/Makefile | 12 ++ kernel/exit.c | 2 + kernel/fork.c | 3 + kernel/kcov.c | 273 ++++++++++++++++++++++++++++++++++ kernel/locking/Makefile | 3 + kernel/rcu/Makefile | 4 + kernel/sched/Makefile | 4 + lib/Kconfig.debug | 21 +++ lib/Makefile | 12 ++ mm/Makefile | 15 ++ mm/kasan/Makefile | 1 + scripts/Makefile.lib | 6 + 28 files changed, 567 insertions(+), 1 deletion(-) create mode 100644 Documentation/kcov.txt create mode 100644 include/linux/kcov.h create mode 100644 include/uapi/linux/kcov.h create mode 100644 kernel/kcov.c (limited to 'Documentation') diff --git a/Documentation/kcov.txt b/Documentation/kcov.txt new file mode 100644 index 000000000000..779ff4ab1c1d --- /dev/null +++ b/Documentation/kcov.txt @@ -0,0 +1,111 @@ +kcov: code coverage for fuzzing +=============================== + +kcov exposes kernel code coverage information in a form suitable for coverage- +guided fuzzing (randomized testing). Coverage data of a running kernel is +exported via the "kcov" debugfs file. Coverage collection is enabled on a task +basis, and thus it can capture precise coverage of a single system call. + +Note that kcov does not aim to collect as much coverage as possible. It aims +to collect more or less stable coverage that is function of syscall inputs. +To achieve this goal it does not collect coverage in soft/hard interrupts +and instrumentation of some inherently non-deterministic parts of kernel is +disbled (e.g. scheduler, locking). + +Usage: +====== + +Configure kernel with: + + CONFIG_KCOV=y + +CONFIG_KCOV requires gcc built on revision 231296 or later. +Profiling data will only become accessible once debugfs has been mounted: + + mount -t debugfs none /sys/kernel/debug + +The following program demonstrates kcov usage from within a test program: + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define KCOV_INIT_TRACE _IOR('c', 1, unsigned long) +#define KCOV_ENABLE _IO('c', 100) +#define KCOV_DISABLE _IO('c', 101) +#define COVER_SIZE (64<<10) + +int main(int argc, char **argv) +{ + int fd; + unsigned long *cover, n, i; + + /* A single fd descriptor allows coverage collection on a single + * thread. + */ + fd = open("/sys/kernel/debug/kcov", O_RDWR); + if (fd == -1) + perror("open"), exit(1); + /* Setup trace mode and trace size. */ + if (ioctl(fd, KCOV_INIT_TRACE, COVER_SIZE)) + perror("ioctl"), exit(1); + /* Mmap buffer shared between kernel- and user-space. */ + cover = (unsigned long*)mmap(NULL, COVER_SIZE * sizeof(unsigned long), + PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); + if ((void*)cover == MAP_FAILED) + perror("mmap"), exit(1); + /* Enable coverage collection on the current thread. */ + if (ioctl(fd, KCOV_ENABLE, 0)) + perror("ioctl"), exit(1); + /* Reset coverage from the tail of the ioctl() call. */ + __atomic_store_n(&cover[0], 0, __ATOMIC_RELAXED); + /* That's the target syscal call. */ + read(-1, NULL, 0); + /* Read number of PCs collected. */ + n = __atomic_load_n(&cover[0], __ATOMIC_RELAXED); + for (i = 0; i < n; i++) + printf("0x%lx\n", cover[i + 1]); + /* Disable coverage collection for the current thread. After this call + * coverage can be enabled for a different thread. + */ + if (ioctl(fd, KCOV_DISABLE, 0)) + perror("ioctl"), exit(1); + /* Free resources. */ + if (munmap(cover, COVER_SIZE * sizeof(unsigned long))) + perror("munmap"), exit(1); + if (close(fd)) + perror("close"), exit(1); + return 0; +} + +After piping through addr2line output of the program looks as follows: + +SyS_read +fs/read_write.c:562 +__fdget_pos +fs/file.c:774 +__fget_light +fs/file.c:746 +__fget_light +fs/file.c:750 +__fget_light +fs/file.c:760 +__fdget_pos +fs/file.c:784 +SyS_read +fs/read_write.c:562 + +If a program needs to collect coverage from several threads (independently), +it needs to open /sys/kernel/debug/kcov in each thread separately. + +The interface is fine-grained to allow efficient forking of test processes. +That is, a parent process opens /sys/kernel/debug/kcov, enables trace mode, +mmaps coverage buffer and then forks child processes in a loop. Child processes +only need to enable coverage (disable happens automatically on thread end). diff --git a/Makefile b/Makefile index e055b969c325..b98a4f70d1b5 100644 --- a/Makefile +++ b/Makefile @@ -365,6 +365,7 @@ LDFLAGS_MODULE = CFLAGS_KERNEL = AFLAGS_KERNEL = CFLAGS_GCOV = -fprofile-arcs -ftest-coverage +CFLAGS_KCOV = -fsanitize-coverage=trace-pc # Use USERINCLUDE when you must reference the UAPI directories only. @@ -411,7 +412,7 @@ export MAKE AWK GENKSYMS INSTALLKERNEL PERL PYTHON UTS_MACHINE export HOSTCXX HOSTCXXFLAGS LDFLAGS_MODULE CHECK CHECKFLAGS export KBUILD_CPPFLAGS NOSTDINC_FLAGS LINUXINCLUDE OBJCOPYFLAGS LDFLAGS -export KBUILD_CFLAGS CFLAGS_KERNEL CFLAGS_MODULE CFLAGS_GCOV CFLAGS_KASAN CFLAGS_UBSAN +export KBUILD_CFLAGS CFLAGS_KERNEL CFLAGS_MODULE CFLAGS_GCOV CFLAGS_KCOV CFLAGS_KASAN CFLAGS_UBSAN export KBUILD_AFLAGS AFLAGS_KERNEL AFLAGS_MODULE export KBUILD_AFLAGS_MODULE KBUILD_CFLAGS_MODULE KBUILD_LDFLAGS_MODULE export KBUILD_AFLAGS_KERNEL KBUILD_CFLAGS_KERNEL @@ -673,6 +674,14 @@ endif endif KBUILD_CFLAGS += $(stackp-flag) +ifdef CONFIG_KCOV + ifeq ($(call cc-option, $(CFLAGS_KCOV)),) + $(warning Cannot use CONFIG_KCOV: \ + -fsanitize-coverage=trace-pc is not supported by compiler) + CFLAGS_KCOV = + endif +endif + ifeq ($(cc-name),clang) KBUILD_CPPFLAGS += $(call cc-option,-Qunused-arguments,) KBUILD_CPPFLAGS += $(call cc-option,-Wno-unknown-warning-option,) diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index 8b680a5cb25b..54478b7635de 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig @@ -28,6 +28,7 @@ config X86 select ARCH_HAS_ELF_RANDOMIZE select ARCH_HAS_FAST_MULTIPLIER select ARCH_HAS_GCOV_PROFILE_ALL + select ARCH_HAS_KCOV if X86_64 select ARCH_HAS_PMEM_API if X86_64 select ARCH_HAS_MMIO_FLUSH select ARCH_HAS_SG_CHAIN diff --git a/arch/x86/boot/Makefile b/arch/x86/boot/Makefile index 0bf6749522d9..b1ef9e489084 100644 --- a/arch/x86/boot/Makefile +++ b/arch/x86/boot/Makefile @@ -12,6 +12,13 @@ KASAN_SANITIZE := n OBJECT_FILES_NON_STANDARD := y +# Kernel does not boot with kcov instrumentation here. +# One of the problems observed was insertion of __sanitizer_cov_trace_pc() +# callback into middle of per-cpu data enabling code. Thus the callback observed +# inconsistent state and crashed. We are interested mostly in syscall coverage, +# so boot code is not interesting anyway. +KCOV_INSTRUMENT := n + # If you want to preset the SVGA mode, uncomment the next line and # set SVGA_MODE to whatever number you want. # Set it to -DSVGA_MODE=NORMAL_VGA if you just want the EGA/VGA mode. diff --git a/arch/x86/boot/compressed/Makefile b/arch/x86/boot/compressed/Makefile index 5e1d26e09407..6915ff2bd996 100644 --- a/arch/x86/boot/compressed/Makefile +++ b/arch/x86/boot/compressed/Makefile @@ -19,6 +19,9 @@ KASAN_SANITIZE := n OBJECT_FILES_NON_STANDARD := y +# Prevents link failures: __sanitizer_cov_trace_pc() is not linked in. +KCOV_INSTRUMENT := n + targets := vmlinux vmlinux.bin vmlinux.bin.gz vmlinux.bin.bz2 vmlinux.bin.lzma \ vmlinux.bin.xz vmlinux.bin.lzo vmlinux.bin.lz4 diff --git a/arch/x86/entry/vdso/Makefile b/arch/x86/entry/vdso/Makefile index f9fb859c98b9..6874da5f67fc 100644 --- a/arch/x86/entry/vdso/Makefile +++ b/arch/x86/entry/vdso/Makefile @@ -7,6 +7,9 @@ KASAN_SANITIZE := n UBSAN_SANITIZE := n OBJECT_FILES_NON_STANDARD := y +# Prevents link failures: __sanitizer_cov_trace_pc() is not linked in. +KCOV_INSTRUMENT := n + VDSO64-$(CONFIG_X86_64) := y VDSOX32-$(CONFIG_X86_X32_ABI) := y VDSO32-$(CONFIG_X86_32) := y diff --git a/arch/x86/kernel/Makefile b/arch/x86/kernel/Makefile index d5fb0871aba3..adaae2c781c1 100644 --- a/arch/x86/kernel/Makefile +++ b/arch/x86/kernel/Makefile @@ -25,6 +25,12 @@ OBJECT_FILES_NON_STANDARD_relocate_kernel_$(BITS).o := y OBJECT_FILES_NON_STANDARD_mcount_$(BITS).o := y OBJECT_FILES_NON_STANDARD_test_nx.o := y +# If instrumentation of this dir is enabled, boot hangs during first second. +# Probably could be more selective here, but note that files related to irqs, +# boot, dumpstack/stacktrace, etc are either non-interesting or can lead to +# non-deterministic coverage. +KCOV_INSTRUMENT := n + CFLAGS_irq.o := -I$(src)/../include/asm/trace obj-y := process_$(BITS).o signal.o diff --git a/arch/x86/kernel/apic/Makefile b/arch/x86/kernel/apic/Makefile index 8bb12ddc5db8..8e63ebdcbd0b 100644 --- a/arch/x86/kernel/apic/Makefile +++ b/arch/x86/kernel/apic/Makefile @@ -2,6 +2,10 @@ # Makefile for local APIC drivers and for the IO-APIC code # +# Leads to non-deterministic coverage that is not a function of syscall inputs. +# In particualr, smp_apic_timer_interrupt() is called in random places. +KCOV_INSTRUMENT := n + obj-$(CONFIG_X86_LOCAL_APIC) += apic.o apic_noop.o ipi.o vector.o obj-y += hw_nmi.o diff --git a/arch/x86/kernel/cpu/Makefile b/arch/x86/kernel/cpu/Makefile index 0d373d7affc8..4a8697f7d4ef 100644 --- a/arch/x86/kernel/cpu/Makefile +++ b/arch/x86/kernel/cpu/Makefile @@ -8,6 +8,10 @@ CFLAGS_REMOVE_common.o = -pg CFLAGS_REMOVE_perf_event.o = -pg endif +# If these files are instrumented, boot hangs during the first second. +KCOV_INSTRUMENT_common.o := n +KCOV_INSTRUMENT_perf_event.o := n + # Make sure load_percpu_segment has no stackprotector nostackp := $(call cc-option, -fno-stack-protector) CFLAGS_common.o := $(nostackp) diff --git a/arch/x86/lib/Makefile b/arch/x86/lib/Makefile index a501fa25da41..72a576752a7e 100644 --- a/arch/x86/lib/Makefile +++ b/arch/x86/lib/Makefile @@ -2,6 +2,9 @@ # Makefile for x86 specific library files. # +# Produces uninteresting flaky coverage. +KCOV_INSTRUMENT_delay.o := n + inat_tables_script = $(srctree)/arch/x86/tools/gen-insn-attr-x86.awk inat_tables_maps = $(srctree)/arch/x86/lib/x86-opcode-map.txt quiet_cmd_inat_tables = GEN $@ diff --git a/arch/x86/mm/Makefile b/arch/x86/mm/Makefile index 67cf2e1e557b..f98913258c63 100644 --- a/arch/x86/mm/Makefile +++ b/arch/x86/mm/Makefile @@ -1,3 +1,6 @@ +# Kernel does not boot with instrumentation of tlb.c. +KCOV_INSTRUMENT_tlb.o := n + obj-y := init.o init_$(BITS).o fault.o ioremap.o extable.o pageattr.o mmap.o \ pat.o pgtable.o physaddr.o gup.o setup_nx.o diff --git a/arch/x86/realmode/rm/Makefile b/arch/x86/realmode/rm/Makefile index 053abe7b0ef7..b95964610ea7 100644 --- a/arch/x86/realmode/rm/Makefile +++ b/arch/x86/realmode/rm/Makefile @@ -9,6 +9,9 @@ KASAN_SANITIZE := n OBJECT_FILES_NON_STANDARD := y +# Prevents link failures: __sanitizer_cov_trace_pc() is not linked in. +KCOV_INSTRUMENT := n + always := realmode.bin realmode.relocs wakeup-objs := wakeup_asm.o wakemain.o video-mode.o diff --git a/drivers/firmware/efi/libstub/Makefile b/drivers/firmware/efi/libstub/Makefile index a15841eced4e..da99bbb74aeb 100644 --- a/drivers/firmware/efi/libstub/Makefile +++ b/drivers/firmware/efi/libstub/Makefile @@ -25,6 +25,9 @@ KASAN_SANITIZE := n UBSAN_SANITIZE := n OBJECT_FILES_NON_STANDARD := y +# Prevents link failures: __sanitizer_cov_trace_pc() is not linked in. +KCOV_INSTRUMENT := n + lib-y := efi-stub-helper.o # include the stub's generic dependencies from lib/ when building for ARM/arm64 diff --git a/include/linux/kcov.h b/include/linux/kcov.h new file mode 100644 index 000000000000..2883ac98c280 --- /dev/null +++ b/include/linux/kcov.h @@ -0,0 +1,29 @@ +#ifndef _LINUX_KCOV_H +#define _LINUX_KCOV_H + +#include + +struct task_struct; + +#ifdef CONFIG_KCOV + +void kcov_task_init(struct task_struct *t); +void kcov_task_exit(struct task_struct *t); + +enum kcov_mode { + /* Coverage collection is not enabled yet. */ + KCOV_MODE_DISABLED = 0, + /* + * Tracing coverage collection mode. + * Covered PCs are collected in a per-task buffer. + */ + KCOV_MODE_TRACE = 1, +}; + +#else + +static inline void kcov_task_init(struct task_struct *t) {} +static inline void kcov_task_exit(struct task_struct *t) {} + +#endif /* CONFIG_KCOV */ +#endif /* _LINUX_KCOV_H */ diff --git a/include/linux/sched.h b/include/linux/sched.h index 084ed9fba620..34495d2d2d7b 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -51,6 +51,7 @@ struct sched_param { #include #include #include +#include #include #include #include @@ -1818,6 +1819,16 @@ struct task_struct { /* bitmask and counter of trace recursion */ unsigned long trace_recursion; #endif /* CONFIG_TRACING */ +#ifdef CONFIG_KCOV + /* Coverage collection mode enabled for this task (0 if disabled). */ + enum kcov_mode kcov_mode; + /* Size of the kcov_area. */ + unsigned kcov_size; + /* Buffer for coverage collection. */ + void *kcov_area; + /* kcov desciptor wired with this task or NULL. */ + struct kcov *kcov; +#endif #ifdef CONFIG_MEMCG struct mem_cgroup *memcg_in_oom; gfp_t memcg_oom_gfp_mask; diff --git a/include/uapi/linux/kcov.h b/include/uapi/linux/kcov.h new file mode 100644 index 000000000000..574e22ec640d --- /dev/null +++ b/include/uapi/linux/kcov.h @@ -0,0 +1,10 @@ +#ifndef _LINUX_KCOV_IOCTLS_H +#define _LINUX_KCOV_IOCTLS_H + +#include + +#define KCOV_INIT_TRACE _IOR('c', 1, unsigned long) +#define KCOV_ENABLE _IO('c', 100) +#define KCOV_DISABLE _IO('c', 101) + +#endif /* _LINUX_KCOV_IOCTLS_H */ diff --git a/kernel/Makefile b/kernel/Makefile index baa55e50a315..f0c40bf49d9f 100644 --- a/kernel/Makefile +++ b/kernel/Makefile @@ -18,6 +18,17 @@ ifdef CONFIG_FUNCTION_TRACER CFLAGS_REMOVE_irq_work.o = $(CC_FLAGS_FTRACE) endif +# Prevents flicker of uninteresting __do_softirq()/__local_bh_disable_ip() +# in coverage traces. +KCOV_INSTRUMENT_softirq.o := n +# These are called from save_stack_trace() on slub debug path, +# and produce insane amounts of uninteresting coverage. +KCOV_INSTRUMENT_module.o := n +KCOV_INSTRUMENT_extable.o := n +# Don't self-instrument. +KCOV_INSTRUMENT_kcov.o := n +KASAN_SANITIZE_kcov.o := n + # cond_syscall is currently not LTO compatible CFLAGS_sys_ni.o = $(DISABLE_LTO) @@ -68,6 +79,7 @@ obj-$(CONFIG_AUDITSYSCALL) += auditsc.o obj-$(CONFIG_AUDIT_WATCH) += audit_watch.o audit_fsnotify.o obj-$(CONFIG_AUDIT_TREE) += audit_tree.o obj-$(CONFIG_GCOV_KERNEL) += gcov/ +obj-$(CONFIG_KCOV) += kcov.o obj-$(CONFIG_KPROBES) += kprobes.o obj-$(CONFIG_KGDB) += debug/ obj-$(CONFIG_DETECT_HUNG_TASK) += hung_task.o diff --git a/kernel/exit.c b/kernel/exit.c index 10e088237fed..953d1a1c0387 100644 --- a/kernel/exit.c +++ b/kernel/exit.c @@ -53,6 +53,7 @@ #include #include #include +#include #include #include @@ -655,6 +656,7 @@ void do_exit(long code) TASKS_RCU(int tasks_rcu_i); profile_task_exit(tsk); + kcov_task_exit(tsk); WARN_ON(blk_needs_flush_plug(tsk)); diff --git a/kernel/fork.c b/kernel/fork.c index 5b8d1e7ceeea..d277e83ed3e0 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -75,6 +75,7 @@ #include #include #include +#include #include #include @@ -392,6 +393,8 @@ static struct task_struct *dup_task_struct(struct task_struct *orig) account_kernel_stack(ti, 1); + kcov_task_init(tsk); + return tsk; free_ti: diff --git a/kernel/kcov.c b/kernel/kcov.c new file mode 100644 index 000000000000..3efbee0834a8 --- /dev/null +++ b/kernel/kcov.c @@ -0,0 +1,273 @@ +#define pr_fmt(fmt) "kcov: " fmt + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* + * kcov descriptor (one per opened debugfs file). + * State transitions of the descriptor: + * - initial state after open() + * - then there must be a single ioctl(KCOV_INIT_TRACE) call + * - then, mmap() call (several calls are allowed but not useful) + * - then, repeated enable/disable for a task (only one task a time allowed) + */ +struct kcov { + /* + * Reference counter. We keep one for: + * - opened file descriptor + * - task with enabled coverage (we can't unwire it from another task) + */ + atomic_t refcount; + /* The lock protects mode, size, area and t. */ + spinlock_t lock; + enum kcov_mode mode; + /* Size of arena (in long's for KCOV_MODE_TRACE). */ + unsigned size; + /* Coverage buffer shared with user space. */ + void *area; + /* Task for which we collect coverage, or NULL. */ + struct task_struct *t; +}; + +/* + * Entry point from instrumented code. + * This is called once per basic-block/edge. + */ +void __sanitizer_cov_trace_pc(void) +{ + struct task_struct *t; + enum kcov_mode mode; + + t = current; + /* + * We are interested in code coverage as a function of a syscall inputs, + * so we ignore code executed in interrupts. + */ + if (!t || in_interrupt()) + return; + mode = READ_ONCE(t->kcov_mode); + if (mode == KCOV_MODE_TRACE) { + unsigned long *area; + unsigned long pos; + + /* + * There is some code that runs in interrupts but for which + * in_interrupt() returns false (e.g. preempt_schedule_irq()). + * READ_ONCE()/barrier() effectively provides load-acquire wrt + * interrupts, there are paired barrier()/WRITE_ONCE() in + * kcov_ioctl_locked(). + */ + barrier(); + area = t->kcov_area; + /* The first word is number of subsequent PCs. */ + pos = READ_ONCE(area[0]) + 1; + if (likely(pos < t->kcov_size)) { + area[pos] = _RET_IP_; + WRITE_ONCE(area[0], pos); + } + } +} +EXPORT_SYMBOL(__sanitizer_cov_trace_pc); + +static void kcov_get(struct kcov *kcov) +{ + atomic_inc(&kcov->refcount); +} + +static void kcov_put(struct kcov *kcov) +{ + if (atomic_dec_and_test(&kcov->refcount)) { + vfree(kcov->area); + kfree(kcov); + } +} + +void kcov_task_init(struct task_struct *t) +{ + t->kcov_mode = KCOV_MODE_DISABLED; + t->kcov_size = 0; + t->kcov_area = NULL; + t->kcov = NULL; +} + +void kcov_task_exit(struct task_struct *t) +{ + struct kcov *kcov; + + kcov = t->kcov; + if (kcov == NULL) + return; + spin_lock(&kcov->lock); + if (WARN_ON(kcov->t != t)) { + spin_unlock(&kcov->lock); + return; + } + /* Just to not leave dangling references behind. */ + kcov_task_init(t); + kcov->t = NULL; + spin_unlock(&kcov->lock); + kcov_put(kcov); +} + +static int kcov_mmap(struct file *filep, struct vm_area_struct *vma) +{ + int res = 0; + void *area; + struct kcov *kcov = vma->vm_file->private_data; + unsigned long size, off; + struct page *page; + + area = vmalloc_user(vma->vm_end - vma->vm_start); + if (!area) + return -ENOMEM; + + spin_lock(&kcov->lock); + size = kcov->size * sizeof(unsigned long); + if (kcov->mode == KCOV_MODE_DISABLED || vma->vm_pgoff != 0 || + vma->vm_end - vma->vm_start != size) { + res = -EINVAL; + goto exit; + } + if (!kcov->area) { + kcov->area = area; + vma->vm_flags |= VM_DONTEXPAND; + spin_unlock(&kcov->lock); + for (off = 0; off < size; off += PAGE_SIZE) { + page = vmalloc_to_page(kcov->area + off); + if (vm_insert_page(vma, vma->vm_start + off, page)) + WARN_ONCE(1, "vm_insert_page() failed"); + } + return 0; + } +exit: + spin_unlock(&kcov->lock); + vfree(area); + return res; +} + +static int kcov_open(struct inode *inode, struct file *filep) +{ + struct kcov *kcov; + + kcov = kzalloc(sizeof(*kcov), GFP_KERNEL); + if (!kcov) + return -ENOMEM; + atomic_set(&kcov->refcount, 1); + spin_lock_init(&kcov->lock); + filep->private_data = kcov; + return nonseekable_open(inode, filep); +} + +static int kcov_close(struct inode *inode, struct file *filep) +{ + kcov_put(filep->private_data); + return 0; +} + +static int kcov_ioctl_locked(struct kcov *kcov, unsigned int cmd, + unsigned long arg) +{ + struct task_struct *t; + unsigned long size, unused; + + switch (cmd) { + case KCOV_INIT_TRACE: + /* + * Enable kcov in trace mode and setup buffer size. + * Must happen before anything else. + */ + if (kcov->mode != KCOV_MODE_DISABLED) + return -EBUSY; + /* + * Size must be at least 2 to hold current position and one PC. + * Later we allocate size * sizeof(unsigned long) memory, + * that must not overflow. + */ + size = arg; + if (size < 2 || size > INT_MAX / sizeof(unsigned long)) + return -EINVAL; + kcov->size = size; + kcov->mode = KCOV_MODE_TRACE; + return 0; + case KCOV_ENABLE: + /* + * Enable coverage for the current task. + * At this point user must have been enabled trace mode, + * and mmapped the file. Coverage collection is disabled only + * at task exit or voluntary by KCOV_DISABLE. After that it can + * be enabled for another task. + */ + unused = arg; + if (unused != 0 || kcov->mode == KCOV_MODE_DISABLED || + kcov->area == NULL) + return -EINVAL; + if (kcov->t != NULL) + return -EBUSY; + t = current; + /* Cache in task struct for performance. */ + t->kcov_size = kcov->size; + t->kcov_area = kcov->area; + /* See comment in __sanitizer_cov_trace_pc(). */ + barrier(); + WRITE_ONCE(t->kcov_mode, kcov->mode); + t->kcov = kcov; + kcov->t = t; + /* This is put either in kcov_task_exit() or in KCOV_DISABLE. */ + kcov_get(kcov); + return 0; + case KCOV_DISABLE: + /* Disable coverage for the current task. */ + unused = arg; + if (unused != 0 || current->kcov != kcov) + return -EINVAL; + t = current; + if (WARN_ON(kcov->t != t)) + return -EINVAL; + kcov_task_init(t); + kcov->t = NULL; + kcov_put(kcov); + return 0; + default: + return -ENOTTY; + } +} + +static long kcov_ioctl(struct file *filep, unsigned int cmd, unsigned long arg) +{ + struct kcov *kcov; + int res; + + kcov = filep->private_data; + spin_lock(&kcov->lock); + res = kcov_ioctl_locked(kcov, cmd, arg); + spin_unlock(&kcov->lock); + return res; +} + +static const struct file_operations kcov_fops = { + .open = kcov_open, + .unlocked_ioctl = kcov_ioctl, + .mmap = kcov_mmap, + .release = kcov_close, +}; + +static int __init kcov_init(void) +{ + if (!debugfs_create_file("kcov", 0600, NULL, NULL, &kcov_fops)) { + pr_err("failed to create kcov in debugfs\n"); + return -ENOMEM; + } + return 0; +} + +device_initcall(kcov_init); diff --git a/kernel/locking/Makefile b/kernel/locking/Makefile index 8e96f6cc2a4a..31322a4275cd 100644 --- a/kernel/locking/Makefile +++ b/kernel/locking/Makefile @@ -1,3 +1,6 @@ +# Any varying coverage in these files is non-deterministic +# and is generally not a function of system call inputs. +KCOV_INSTRUMENT := n obj-y += mutex.o semaphore.o rwsem.o percpu-rwsem.o diff --git a/kernel/rcu/Makefile b/kernel/rcu/Makefile index 61a16569ffbf..032b2c015beb 100644 --- a/kernel/rcu/Makefile +++ b/kernel/rcu/Makefile @@ -1,3 +1,7 @@ +# Any varying coverage in these files is non-deterministic +# and is generally not a function of system call inputs. +KCOV_INSTRUMENT := n + obj-y += update.o sync.o obj-$(CONFIG_SRCU) += srcu.o obj-$(CONFIG_RCU_TORTURE_TEST) += rcutorture.o diff --git a/kernel/sched/Makefile b/kernel/sched/Makefile index 302d6ebd64f7..414d9c16da42 100644 --- a/kernel/sched/Makefile +++ b/kernel/sched/Makefile @@ -2,6 +2,10 @@ ifdef CONFIG_FUNCTION_TRACER CFLAGS_REMOVE_clock.o = $(CC_FLAGS_FTRACE) endif +# These files are disabled because they produce non-interesting flaky coverage +# that is not a function of syscall inputs. E.g. involuntary context switches. +KCOV_INSTRUMENT := n + ifneq ($(CONFIG_SCHED_OMIT_FRAME_POINTER),y) # According to Alan Modra , the -fno-omit-frame-pointer is # needed for x86 only. Why this used to be enabled for all architectures is beyond diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug index 5a60f45cd9bb..532d4d52d1df 100644 --- a/lib/Kconfig.debug +++ b/lib/Kconfig.debug @@ -696,6 +696,27 @@ source "lib/Kconfig.kasan" endmenu # "Memory Debugging" +config ARCH_HAS_KCOV + bool + help + KCOV does not have any arch-specific code, but currently it is enabled + only for x86_64. KCOV requires testing on other archs, and most likely + disabling of instrumentation for some early boot code. + +config KCOV + bool "Code coverage for fuzzing" + depends on ARCH_HAS_KCOV + select DEBUG_FS + help + KCOV exposes kernel code coverage information in a form suitable + for coverage-guided fuzzing (randomized testing). + + If RANDOMIZE_BASE is enabled, PC values will not be stable across + different machines and across reboots. If you need stable PC values, + disable RANDOMIZE_BASE. + + For more details, see Documentation/kcov.txt. + config DEBUG_SHIRQ bool "Debug shared IRQ handlers" depends on DEBUG_KERNEL diff --git a/lib/Makefile b/lib/Makefile index 4962d14c450f..a1de5b61ff40 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -7,6 +7,18 @@ ORIG_CFLAGS := $(KBUILD_CFLAGS) KBUILD_CFLAGS = $(subst $(CC_FLAGS_FTRACE),,$(ORIG_CFLAGS)) endif +# These files are disabled because they produce lots of non-interesting and/or +# flaky coverage that is not a function of syscall inputs. For example, +# rbtree can be global and individual rotations don't correlate with inputs. +KCOV_INSTRUMENT_string.o := n +KCOV_INSTRUMENT_rbtree.o := n +KCOV_INSTRUMENT_list_debug.o := n +KCOV_INSTRUMENT_debugobjects.o := n +KCOV_INSTRUMENT_dynamic_debug.o := n +# Kernel does not boot if we instrument this file as it uses custom calling +# convention (see CONFIG_ARCH_HWEIGHT_CFLAGS). +KCOV_INSTRUMENT_hweight.o := n + lib-y := ctype.o string.o vsprintf.o cmdline.o \ rbtree.o radix-tree.o dump_stack.o timerqueue.o\ idr.o int_sqrt.o extable.o \ diff --git a/mm/Makefile b/mm/Makefile index 6da300a1414b..f5e797cbd128 100644 --- a/mm/Makefile +++ b/mm/Makefile @@ -5,6 +5,21 @@ KASAN_SANITIZE_slab_common.o := n KASAN_SANITIZE_slub.o := n +# These files are disabled because they produce non-interesting and/or +# flaky coverage that is not a function of syscall inputs. E.g. slab is out of +# free pages, or a task is migrated between nodes. +KCOV_INSTRUMENT_slab_common.o := n +KCOV_INSTRUMENT_slob.o := n +KCOV_INSTRUMENT_slab.o := n +KCOV_INSTRUMENT_slub.o := n +KCOV_INSTRUMENT_page_alloc.o := n +KCOV_INSTRUMENT_debug-pagealloc.o := n +KCOV_INSTRUMENT_kmemleak.o := n +KCOV_INSTRUMENT_kmemcheck.o := n +KCOV_INSTRUMENT_memcontrol.o := n +KCOV_INSTRUMENT_mmzone.o := n +KCOV_INSTRUMENT_vmstat.o := n + mmu-y := nommu.o mmu-$(CONFIG_MMU) := gup.o highmem.o memory.o mincore.o \ mlock.o mmap.o mprotect.o mremap.o msync.o rmap.o \ diff --git a/mm/kasan/Makefile b/mm/kasan/Makefile index a61460d9f5b0..131daadf40e4 100644 --- a/mm/kasan/Makefile +++ b/mm/kasan/Makefile @@ -1,5 +1,6 @@ KASAN_SANITIZE := n UBSAN_SANITIZE_kasan.o := n +KCOV_INSTRUMENT := n CFLAGS_REMOVE_kasan.o = -pg # Function splitter causes unnecessary splits in __asan_load1/__asan_store1 diff --git a/scripts/Makefile.lib b/scripts/Makefile.lib index ad50d5859ac4..ddf83d0181e7 100644 --- a/scripts/Makefile.lib +++ b/scripts/Makefile.lib @@ -136,6 +136,12 @@ _c_flags += $(if $(patsubst n%,, \ $(CFLAGS_UBSAN)) endif +ifeq ($(CONFIG_KCOV),y) +_c_flags += $(if $(patsubst n%,, \ + $(KCOV_INSTRUMENT_$(basetarget).o)$(KCOV_INSTRUMENT)y), \ + $(CFLAGS_KCOV)) +endif + # If building the kernel in a separate objtree expand all occurrences # of -Idir to -I$(srctree)/dir except for absolute paths (starting with '/'). -- cgit v1.2.3-59-g8ed1b