From d98793b5d4256faae76177178456214f55bc7083 Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Tue, 27 Oct 2020 14:34:09 -0700 Subject: dmaengine: idxd: fix wq config registers offset programming DSA spec v1.1 [1] updated to include a stride size register for WQ configuration that will specify how much space is reserved for the WQ configuration register set. This change is expected to be in the final gen1 DSA hardware. Fix the driver to use WQCFG_OFFSET() for all WQ offset calculation and fixup WQCFG_OFFSET() to use the new calculated wq size. [1]: https://software.intel.com/content/www/us/en/develop/download/intel-data-streaming-accelerator-preliminary-architecture-specification.html Fixes: bfe1d56091c1 ("dmaengine: idxd: Init and probe for Intel data accelerators") Signed-off-by: Dave Jiang Link: https://lore.kernel.org/r/160383444959.48058.14249265538404901781.stgit@djiang5-desk3.ch.intel.com Signed-off-by: Vinod Koul --- drivers/dma/idxd/device.c | 29 ++++++++++++++--------------- drivers/dma/idxd/idxd.h | 3 ++- drivers/dma/idxd/init.c | 5 +++++ drivers/dma/idxd/registers.h | 23 ++++++++++++++++++++++- 4 files changed, 43 insertions(+), 17 deletions(-) diff --git a/drivers/dma/idxd/device.c b/drivers/dma/idxd/device.c index 200b9109cacf..506ac85c4a7f 100644 --- a/drivers/dma/idxd/device.c +++ b/drivers/dma/idxd/device.c @@ -295,7 +295,7 @@ void idxd_wq_disable_cleanup(struct idxd_wq *wq) int i, wq_offset; lockdep_assert_held(&idxd->dev_lock); - memset(&wq->wqcfg, 0, sizeof(wq->wqcfg)); + memset(wq->wqcfg, 0, idxd->wqcfg_size); wq->type = IDXD_WQT_NONE; wq->size = 0; wq->group = NULL; @@ -304,8 +304,8 @@ void idxd_wq_disable_cleanup(struct idxd_wq *wq) clear_bit(WQ_FLAG_DEDICATED, &wq->flags); memset(wq->name, 0, WQ_NAME_SIZE); - for (i = 0; i < 8; i++) { - wq_offset = idxd->wqcfg_offset + wq->id * 32 + i * sizeof(u32); + for (i = 0; i < WQCFG_STRIDES(idxd); i++) { + wq_offset = WQCFG_OFFSET(idxd, wq->id, i); iowrite32(0, idxd->reg_base + wq_offset); dev_dbg(dev, "WQ[%d][%d][%#x]: %#x\n", wq->id, i, wq_offset, @@ -539,10 +539,10 @@ static int idxd_wq_config_write(struct idxd_wq *wq) if (!wq->group) return 0; - memset(&wq->wqcfg, 0, sizeof(union wqcfg)); + memset(wq->wqcfg, 0, idxd->wqcfg_size); /* byte 0-3 */ - wq->wqcfg.wq_size = wq->size; + wq->wqcfg->wq_size = wq->size; if (wq->size == 0) { dev_warn(dev, "Incorrect work queue size: 0\n"); @@ -550,22 +550,21 @@ static int idxd_wq_config_write(struct idxd_wq *wq) } /* bytes 4-7 */ - wq->wqcfg.wq_thresh = wq->threshold; + wq->wqcfg->wq_thresh = wq->threshold; /* byte 8-11 */ - wq->wqcfg.priv = !!(wq->type == IDXD_WQT_KERNEL); - wq->wqcfg.mode = 1; - - wq->wqcfg.priority = wq->priority; + wq->wqcfg->priv = !!(wq->type == IDXD_WQT_KERNEL); + wq->wqcfg->mode = 1; + wq->wqcfg->priority = wq->priority; /* bytes 12-15 */ - wq->wqcfg.max_xfer_shift = ilog2(wq->max_xfer_bytes); - wq->wqcfg.max_batch_shift = ilog2(wq->max_batch_size); + wq->wqcfg->max_xfer_shift = ilog2(wq->max_xfer_bytes); + wq->wqcfg->max_batch_shift = ilog2(wq->max_batch_size); dev_dbg(dev, "WQ %d CFGs\n", wq->id); - for (i = 0; i < 8; i++) { - wq_offset = idxd->wqcfg_offset + wq->id * 32 + i * sizeof(u32); - iowrite32(wq->wqcfg.bits[i], idxd->reg_base + wq_offset); + for (i = 0; i < WQCFG_STRIDES(idxd); i++) { + wq_offset = WQCFG_OFFSET(idxd, wq->id, i); + iowrite32(wq->wqcfg->bits[i], idxd->reg_base + wq_offset); dev_dbg(dev, "WQ[%d][%d][%#x]: %#x\n", wq->id, i, wq_offset, ioread32(idxd->reg_base + wq_offset)); diff --git a/drivers/dma/idxd/idxd.h b/drivers/dma/idxd/idxd.h index c64df197e724..d48f193daacc 100644 --- a/drivers/dma/idxd/idxd.h +++ b/drivers/dma/idxd/idxd.h @@ -103,7 +103,7 @@ struct idxd_wq { u32 priority; enum idxd_wq_state state; unsigned long flags; - union wqcfg wqcfg; + union wqcfg *wqcfg; u32 vec_ptr; /* interrupt steering */ struct dsa_hw_desc **hw_descs; int num_descs; @@ -183,6 +183,7 @@ struct idxd_device { int max_wq_size; int token_limit; int nr_tokens; /* non-reserved tokens */ + unsigned int wqcfg_size; union sw_err_reg sw_err; wait_queue_head_t cmd_waitq; diff --git a/drivers/dma/idxd/init.c b/drivers/dma/idxd/init.c index 11e5ce168177..0a4432b063b5 100644 --- a/drivers/dma/idxd/init.c +++ b/drivers/dma/idxd/init.c @@ -178,6 +178,9 @@ static int idxd_setup_internals(struct idxd_device *idxd) wq->idxd_cdev.minor = -1; wq->max_xfer_bytes = idxd->max_xfer_bytes; wq->max_batch_size = idxd->max_batch_size; + wq->wqcfg = devm_kzalloc(dev, idxd->wqcfg_size, GFP_KERNEL); + if (!wq->wqcfg) + return -ENOMEM; } for (i = 0; i < idxd->max_engines; i++) { @@ -251,6 +254,8 @@ static void idxd_read_caps(struct idxd_device *idxd) dev_dbg(dev, "total workqueue size: %u\n", idxd->max_wq_size); idxd->max_wqs = idxd->hw.wq_cap.num_wqs; dev_dbg(dev, "max workqueues: %u\n", idxd->max_wqs); + idxd->wqcfg_size = 1 << (idxd->hw.wq_cap.wqcfg_size + IDXD_WQCFG_MIN); + dev_dbg(dev, "wqcfg size: %u\n", idxd->wqcfg_size); /* reading operation capabilities */ for (i = 0; i < 4; i++) { diff --git a/drivers/dma/idxd/registers.h b/drivers/dma/idxd/registers.h index a39e7ae6b3d9..aef5a902829e 100644 --- a/drivers/dma/idxd/registers.h +++ b/drivers/dma/idxd/registers.h @@ -43,7 +43,8 @@ union wq_cap_reg { struct { u64 total_wq_size:16; u64 num_wqs:8; - u64 rsvd:24; + u64 wqcfg_size:4; + u64 rsvd:20; u64 shared_mode:1; u64 dedicated_mode:1; u64 rsvd2:1; @@ -55,6 +56,7 @@ union wq_cap_reg { u64 bits; } __packed; #define IDXD_WQCAP_OFFSET 0x20 +#define IDXD_WQCFG_MIN 5 union group_cap_reg { struct { @@ -333,4 +335,23 @@ union wqcfg { }; u32 bits[8]; } __packed; + +/* + * This macro calculates the offset into the WQCFG register + * idxd - struct idxd * + * n - wq id + * ofs - the index of the 32b dword for the config register + * + * The WQCFG register block is divided into groups per each wq. The n index + * allows us to move to the register group that's for that particular wq. + * Each register is 32bits. The ofs gives us the number of register to access. + */ +#define WQCFG_OFFSET(_idxd_dev, n, ofs) \ +({\ + typeof(_idxd_dev) __idxd_dev = (_idxd_dev); \ + (__idxd_dev)->wqcfg_offset + (n) * (__idxd_dev)->wqcfg_size + sizeof(u32) * (ofs); \ +}) + +#define WQCFG_STRIDES(_idxd_dev) ((_idxd_dev)->wqcfg_size / sizeof(u32)) + #endif -- cgit v1.2.3-59-g8ed1b From 8145dce88a788f178dfe19376e5d8645d1b87e05 Mon Sep 17 00:00:00 2001 From: "Gustavo A. R. Silva" Date: Thu, 8 Oct 2020 09:18:28 -0500 Subject: dmaengine: stm32-mdma: Use struct_size() in kzalloc() Make use of the new struct_size() helper instead of the offsetof() idiom. Signed-off-by: Gustavo A. R. Silva Link: https://lore.kernel.org/r/20201008141828.GA20325@embeddedor Signed-off-by: Vinod Koul --- drivers/dma/stm32-mdma.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/dma/stm32-mdma.c b/drivers/dma/stm32-mdma.c index 08cfbfab837b..29e5e30524bb 100644 --- a/drivers/dma/stm32-mdma.c +++ b/drivers/dma/stm32-mdma.c @@ -339,7 +339,7 @@ static struct stm32_mdma_desc *stm32_mdma_alloc_desc( struct stm32_mdma_desc *desc; int i; - desc = kzalloc(offsetof(typeof(*desc), node[count]), GFP_NOWAIT); + desc = kzalloc(struct_size(desc, node, count), GFP_NOWAIT); if (!desc) return NULL; -- cgit v1.2.3-59-g8ed1b From dafd8fe27a9999ef14094dddfd5aaccd49750e1d Mon Sep 17 00:00:00 2001 From: Surendran K Date: Fri, 16 Oct 2020 16:03:47 +0530 Subject: dmaengine: pl330: Remove unreachable code _setup_req(..) never returns negative value. Hence the condition ret < 0 is never met Signed-off-by: Surendran K Link: https://lore.kernel.org/r/20201016103347.63084-1-surendran.k@samsung.com Signed-off-by: Vinod Koul --- drivers/dma/pl330.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/drivers/dma/pl330.c b/drivers/dma/pl330.c index e9f0101d92fa..8355586c9788 100644 --- a/drivers/dma/pl330.c +++ b/drivers/dma/pl330.c @@ -1527,8 +1527,6 @@ static int pl330_submit_req(struct pl330_thread *thrd, /* First dry run to check if req is acceptable */ ret = _setup_req(pl330, 1, thrd, idx, &xs); - if (ret < 0) - goto xfer_exit; if (ret > pl330->mcbufsz / 2) { dev_info(pl330->ddma.dev, "%s:%d Try increasing mcbufsz (%i/%i)\n", -- cgit v1.2.3-59-g8ed1b From cb0362b6ec537619b842b54e794ae8a87a5122bb Mon Sep 17 00:00:00 2001 From: Eugen Hristev Date: Fri, 16 Oct 2020 11:17:51 +0300 Subject: dt-bindings: dmaengine: at_xdmac: add compatible with microchip,sama7g5 Add compatible to sama7g5 SoC. Signed-off-by: Eugen Hristev Acked-by: Rob Herring Link: https://lore.kernel.org/r/20201016081754.288488-1-eugen.hristev@microchip.com Signed-off-by: Vinod Koul --- Documentation/devicetree/bindings/dma/atmel-xdma.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Documentation/devicetree/bindings/dma/atmel-xdma.txt b/Documentation/devicetree/bindings/dma/atmel-xdma.txt index 4dc398e1a371..510b7f25ba24 100644 --- a/Documentation/devicetree/bindings/dma/atmel-xdma.txt +++ b/Documentation/devicetree/bindings/dma/atmel-xdma.txt @@ -2,7 +2,8 @@ * XDMA Controller Required properties: -- compatible: Should be "atmel,sama5d4-dma" or "microchip,sam9x60-dma". +- compatible: Should be "atmel,sama5d4-dma", "microchip,sam9x60-dma" or + "microchip,sama7g5-dma". - reg: Should contain DMA registers location and length. - interrupts: Should contain DMA interrupt. - #dma-cells: Must be <1>, used to represent the number of integer cells in -- cgit v1.2.3-59-g8ed1b From 60f88c031d94f906f01df1cb155f7a866812a716 Mon Sep 17 00:00:00 2001 From: Eugen Hristev Date: Fri, 16 Oct 2020 12:37:25 +0300 Subject: dmaengine: at_xdmac: adapt perid for mem2mem operations The PERID in the CC register for mem2mem operations must match an unused PERID. The PERID field is 7 bits, but the selected value is 0x3f. On later products we can have more reserved PERIDs for actual peripherals, thus this needs to be increased to maximum size. Changing the value to 0x7f, which is the maximum for 7 bits field. Signed-off-by: Eugen Hristev Reviewed-by: Tudor Ambarus Link: https://lore.kernel.org/r/20201016093725.289880-1-eugen.hristev@microchip.com Signed-off-by: Vinod Koul --- drivers/dma/at_xdmac.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/dma/at_xdmac.c b/drivers/dma/at_xdmac.c index 3b53115db268..9f8fc9271a74 100644 --- a/drivers/dma/at_xdmac.c +++ b/drivers/dma/at_xdmac.c @@ -867,7 +867,7 @@ at_xdmac_interleaved_queue_desc(struct dma_chan *chan, * match the one of another channel. If not, it could lead to spurious * flag status. */ - u32 chan_cc = AT_XDMAC_CC_PERID(0x3f) + u32 chan_cc = AT_XDMAC_CC_PERID(0x7f) | AT_XDMAC_CC_DIF(0) | AT_XDMAC_CC_SIF(0) | AT_XDMAC_CC_MBSIZE_SIXTEEN @@ -1049,7 +1049,7 @@ at_xdmac_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src, * match the one of another channel. If not, it could lead to spurious * flag status. */ - u32 chan_cc = AT_XDMAC_CC_PERID(0x3f) + u32 chan_cc = AT_XDMAC_CC_PERID(0x7f) | AT_XDMAC_CC_DAM_INCREMENTED_AM | AT_XDMAC_CC_SAM_INCREMENTED_AM | AT_XDMAC_CC_DIF(0) @@ -1155,7 +1155,7 @@ static struct at_xdmac_desc *at_xdmac_memset_create_desc(struct dma_chan *chan, * match the one of another channel. If not, it could lead to spurious * flag status. */ - u32 chan_cc = AT_XDMAC_CC_PERID(0x3f) + u32 chan_cc = AT_XDMAC_CC_PERID(0x7f) | AT_XDMAC_CC_DAM_UBS_AM | AT_XDMAC_CC_SAM_INCREMENTED_AM | AT_XDMAC_CC_DIF(0) -- cgit v1.2.3-59-g8ed1b From 2bec35a529b7d3bf28ee0d90ec157560fdf8d4e4 Mon Sep 17 00:00:00 2001 From: Eugen Hristev Date: Fri, 16 Oct 2020 12:38:50 +0300 Subject: dmaengine: at_xdmac: add support for sama7g5 based at_xdmac SAMA7G5 SoC uses a slightly different variant of the AT_XDMAC. Added support by a new compatible and a layout struct that copes to the specific version considering the compatible string. Only the differences in register map are present in the layout struct. I reworked the register access for this part that has the differences. Also the Source/Destination Interface bits are no longer valid for this variant of the XDMAC. Thus, the layout also has a bool for specifying whether these bits are required or not. Signed-off-by: Eugen Hristev Link: https://lore.kernel.org/r/20201016093850.290053-1-eugen.hristev@microchip.com Signed-off-by: Vinod Koul --- drivers/dma/at_xdmac.c | 110 +++++++++++++++++++++++++++++++++++++------------ 1 file changed, 84 insertions(+), 26 deletions(-) diff --git a/drivers/dma/at_xdmac.c b/drivers/dma/at_xdmac.c index 9f8fc9271a74..5a5c621206f2 100644 --- a/drivers/dma/at_xdmac.c +++ b/drivers/dma/at_xdmac.c @@ -38,13 +38,6 @@ #define AT_XDMAC_GE 0x1C /* Global Channel Enable Register */ #define AT_XDMAC_GD 0x20 /* Global Channel Disable Register */ #define AT_XDMAC_GS 0x24 /* Global Channel Status Register */ -#define AT_XDMAC_GRS 0x28 /* Global Channel Read Suspend Register */ -#define AT_XDMAC_GWS 0x2C /* Global Write Suspend Register */ -#define AT_XDMAC_GRWS 0x30 /* Global Channel Read Write Suspend Register */ -#define AT_XDMAC_GRWR 0x34 /* Global Channel Read Write Resume Register */ -#define AT_XDMAC_GSWR 0x38 /* Global Channel Software Request Register */ -#define AT_XDMAC_GSWS 0x3C /* Global channel Software Request Status Register */ -#define AT_XDMAC_GSWF 0x40 /* Global Channel Software Flush Request Register */ #define AT_XDMAC_VERSION 0xFFC /* XDMAC Version Register */ /* Channel relative registers offsets */ @@ -150,8 +143,6 @@ #define AT_XDMAC_CSUS 0x30 /* Channel Source Microblock Stride */ #define AT_XDMAC_CDUS 0x34 /* Channel Destination Microblock Stride */ -#define AT_XDMAC_CHAN_REG_BASE 0x50 /* Channel registers base address */ - /* Microblock control members */ #define AT_XDMAC_MBR_UBC_UBLEN_MAX 0xFFFFFFUL /* Maximum Microblock Length */ #define AT_XDMAC_MBR_UBC_NDE (0x1 << 24) /* Next Descriptor Enable */ @@ -179,6 +170,27 @@ enum atc_status { AT_XDMAC_CHAN_IS_PAUSED, }; +struct at_xdmac_layout { + /* Global Channel Read Suspend Register */ + u8 grs; + /* Global Write Suspend Register */ + u8 gws; + /* Global Channel Read Write Suspend Register */ + u8 grws; + /* Global Channel Read Write Resume Register */ + u8 grwr; + /* Global Channel Software Request Register */ + u8 gswr; + /* Global channel Software Request Status Register */ + u8 gsws; + /* Global Channel Software Flush Request Register */ + u8 gswf; + /* Channel reg base */ + u8 chan_cc_reg_base; + /* Source/Destination Interface must be specified or not */ + bool sdif; +}; + /* ----- Channels ----- */ struct at_xdmac_chan { struct dma_chan chan; @@ -212,6 +224,7 @@ struct at_xdmac { struct clk *clk; u32 save_gim; struct dma_pool *at_xdmac_desc_pool; + const struct at_xdmac_layout *layout; struct at_xdmac_chan chan[]; }; @@ -244,9 +257,33 @@ struct at_xdmac_desc { struct list_head xfer_node; } __aligned(sizeof(u64)); +static const struct at_xdmac_layout at_xdmac_sama5d4_layout = { + .grs = 0x28, + .gws = 0x2C, + .grws = 0x30, + .grwr = 0x34, + .gswr = 0x38, + .gsws = 0x3C, + .gswf = 0x40, + .chan_cc_reg_base = 0x50, + .sdif = true, +}; + +static const struct at_xdmac_layout at_xdmac_sama7g5_layout = { + .grs = 0x30, + .gws = 0x38, + .grws = 0x40, + .grwr = 0x44, + .gswr = 0x48, + .gsws = 0x4C, + .gswf = 0x50, + .chan_cc_reg_base = 0x60, + .sdif = false, +}; + static inline void __iomem *at_xdmac_chan_reg_base(struct at_xdmac *atxdmac, unsigned int chan_nb) { - return atxdmac->regs + (AT_XDMAC_CHAN_REG_BASE + chan_nb * 0x40); + return atxdmac->regs + (atxdmac->layout->chan_cc_reg_base + chan_nb * 0x40); } #define at_xdmac_read(atxdmac, reg) readl_relaxed((atxdmac)->regs + (reg)) @@ -345,8 +382,10 @@ static void at_xdmac_start_xfer(struct at_xdmac_chan *atchan, first->active_xfer = true; /* Tell xdmac where to get the first descriptor. */ - reg = AT_XDMAC_CNDA_NDA(first->tx_dma_desc.phys) - | AT_XDMAC_CNDA_NDAIF(atchan->memif); + reg = AT_XDMAC_CNDA_NDA(first->tx_dma_desc.phys); + if (atxdmac->layout->sdif) + reg |= AT_XDMAC_CNDA_NDAIF(atchan->memif); + at_xdmac_chan_write(atchan, AT_XDMAC_CNDA, reg); /* @@ -541,6 +580,7 @@ static int at_xdmac_compute_chan_conf(struct dma_chan *chan, enum dma_transfer_direction direction) { struct at_xdmac_chan *atchan = to_at_xdmac_chan(chan); + struct at_xdmac *atxdmac = to_at_xdmac(atchan->chan.device); int csize, dwidth; if (direction == DMA_DEV_TO_MEM) { @@ -548,12 +588,14 @@ static int at_xdmac_compute_chan_conf(struct dma_chan *chan, AT91_XDMAC_DT_PERID(atchan->perid) | AT_XDMAC_CC_DAM_INCREMENTED_AM | AT_XDMAC_CC_SAM_FIXED_AM - | AT_XDMAC_CC_DIF(atchan->memif) - | AT_XDMAC_CC_SIF(atchan->perif) | AT_XDMAC_CC_SWREQ_HWR_CONNECTED | AT_XDMAC_CC_DSYNC_PER2MEM | AT_XDMAC_CC_MBSIZE_SIXTEEN | AT_XDMAC_CC_TYPE_PER_TRAN; + if (atxdmac->layout->sdif) + atchan->cfg |= AT_XDMAC_CC_DIF(atchan->memif) | + AT_XDMAC_CC_SIF(atchan->perif); + csize = ffs(atchan->sconfig.src_maxburst) - 1; if (csize < 0) { dev_err(chan2dev(chan), "invalid src maxburst value\n"); @@ -571,12 +613,14 @@ static int at_xdmac_compute_chan_conf(struct dma_chan *chan, AT91_XDMAC_DT_PERID(atchan->perid) | AT_XDMAC_CC_DAM_FIXED_AM | AT_XDMAC_CC_SAM_INCREMENTED_AM - | AT_XDMAC_CC_DIF(atchan->perif) - | AT_XDMAC_CC_SIF(atchan->memif) | AT_XDMAC_CC_SWREQ_HWR_CONNECTED | AT_XDMAC_CC_DSYNC_MEM2PER | AT_XDMAC_CC_MBSIZE_SIXTEEN | AT_XDMAC_CC_TYPE_PER_TRAN; + if (atxdmac->layout->sdif) + atchan->cfg |= AT_XDMAC_CC_DIF(atchan->perif) | + AT_XDMAC_CC_SIF(atchan->memif); + csize = ffs(atchan->sconfig.dst_maxburst) - 1; if (csize < 0) { dev_err(chan2dev(chan), "invalid src maxburst value\n"); @@ -866,10 +910,12 @@ at_xdmac_interleaved_queue_desc(struct dma_chan *chan, * ERRATA: Even if useless for memory transfers, the PERID has to not * match the one of another channel. If not, it could lead to spurious * flag status. + * For SAMA7G5x case, the SIF and DIF fields are no longer used. + * Thus, no need to have the SIF/DIF interfaces here. + * For SAMA5D4x and SAMA5D2x the SIF and DIF are already configured as + * zero. */ u32 chan_cc = AT_XDMAC_CC_PERID(0x7f) - | AT_XDMAC_CC_DIF(0) - | AT_XDMAC_CC_SIF(0) | AT_XDMAC_CC_MBSIZE_SIXTEEN | AT_XDMAC_CC_TYPE_MEM_TRAN; @@ -1048,12 +1094,14 @@ at_xdmac_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src, * ERRATA: Even if useless for memory transfers, the PERID has to not * match the one of another channel. If not, it could lead to spurious * flag status. + * For SAMA7G5x case, the SIF and DIF fields are no longer used. + * Thus, no need to have the SIF/DIF interfaces here. + * For SAMA5D4x and SAMA5D2x the SIF and DIF are already configured as + * zero. */ u32 chan_cc = AT_XDMAC_CC_PERID(0x7f) | AT_XDMAC_CC_DAM_INCREMENTED_AM | AT_XDMAC_CC_SAM_INCREMENTED_AM - | AT_XDMAC_CC_DIF(0) - | AT_XDMAC_CC_SIF(0) | AT_XDMAC_CC_MBSIZE_SIXTEEN | AT_XDMAC_CC_TYPE_MEM_TRAN; unsigned long irqflags; @@ -1154,12 +1202,14 @@ static struct at_xdmac_desc *at_xdmac_memset_create_desc(struct dma_chan *chan, * ERRATA: Even if useless for memory transfers, the PERID has to not * match the one of another channel. If not, it could lead to spurious * flag status. + * For SAMA7G5x case, the SIF and DIF fields are no longer used. + * Thus, no need to have the SIF/DIF interfaces here. + * For SAMA5D4x and SAMA5D2x the SIF and DIF are already configured as + * zero. */ u32 chan_cc = AT_XDMAC_CC_PERID(0x7f) | AT_XDMAC_CC_DAM_UBS_AM | AT_XDMAC_CC_SAM_INCREMENTED_AM - | AT_XDMAC_CC_DIF(0) - | AT_XDMAC_CC_SIF(0) | AT_XDMAC_CC_MBSIZE_SIXTEEN | AT_XDMAC_CC_MEMSET_HW_MODE | AT_XDMAC_CC_TYPE_MEM_TRAN; @@ -1438,7 +1488,7 @@ at_xdmac_tx_status(struct dma_chan *chan, dma_cookie_t cookie, mask = AT_XDMAC_CC_TYPE | AT_XDMAC_CC_DSYNC; value = AT_XDMAC_CC_TYPE_PER_TRAN | AT_XDMAC_CC_DSYNC_PER2MEM; if ((desc->lld.mbr_cfg & mask) == value) { - at_xdmac_write(atxdmac, AT_XDMAC_GSWF, atchan->mask); + at_xdmac_write(atxdmac, atxdmac->layout->gswf, atchan->mask); while (!(at_xdmac_chan_read(atchan, AT_XDMAC_CIS) & AT_XDMAC_CIS_FIS)) cpu_relax(); } @@ -1496,7 +1546,7 @@ at_xdmac_tx_status(struct dma_chan *chan, dma_cookie_t cookie, * FIFO flush ensures that data are really written. */ if ((desc->lld.mbr_cfg & mask) == value) { - at_xdmac_write(atxdmac, AT_XDMAC_GSWF, atchan->mask); + at_xdmac_write(atxdmac, atxdmac->layout->gswf, atchan->mask); while (!(at_xdmac_chan_read(atchan, AT_XDMAC_CIS) & AT_XDMAC_CIS_FIS)) cpu_relax(); } @@ -1761,7 +1811,7 @@ static int at_xdmac_device_pause(struct dma_chan *chan) return 0; spin_lock_irqsave(&atchan->lock, flags); - at_xdmac_write(atxdmac, AT_XDMAC_GRWS, atchan->mask); + at_xdmac_write(atxdmac, atxdmac->layout->grws, atchan->mask); while (at_xdmac_chan_read(atchan, AT_XDMAC_CC) & (AT_XDMAC_CC_WRIP | AT_XDMAC_CC_RDIP)) cpu_relax(); @@ -1784,7 +1834,7 @@ static int at_xdmac_device_resume(struct dma_chan *chan) return 0; } - at_xdmac_write(atxdmac, AT_XDMAC_GRWR, atchan->mask); + at_xdmac_write(atxdmac, atxdmac->layout->grwr, atchan->mask); clear_bit(AT_XDMAC_CHAN_IS_PAUSED, &atchan->status); spin_unlock_irqrestore(&atchan->lock, flags); @@ -1986,6 +2036,10 @@ static int at_xdmac_probe(struct platform_device *pdev) atxdmac->regs = base; atxdmac->irq = irq; + atxdmac->layout = of_device_get_match_data(&pdev->dev); + if (!atxdmac->layout) + return -ENODEV; + atxdmac->clk = devm_clk_get(&pdev->dev, "dma_clk"); if (IS_ERR(atxdmac->clk)) { dev_err(&pdev->dev, "can't get dma_clk\n"); @@ -2128,6 +2182,10 @@ static const struct dev_pm_ops atmel_xdmac_dev_pm_ops = { static const struct of_device_id atmel_xdmac_dt_ids[] = { { .compatible = "atmel,sama5d4-dma", + .data = &at_xdmac_sama5d4_layout, + }, { + .compatible = "microchip,sama7g5-dma", + .data = &at_xdmac_sama7g5_layout, }, { /* sentinel */ } -- cgit v1.2.3-59-g8ed1b From f40566f220a1a7ab3ac4de8d594a4a038bace156 Mon Sep 17 00:00:00 2001 From: Eugen Hristev Date: Fri, 16 Oct 2020 12:39:18 +0300 Subject: dmaengine: at_xdmac: add AXI priority support and recommended settings The sama7g5 version of the XDMAC supports priority configuration and outstanding capabilities. Add defines for the specific registers for this configuration, together with recommended settings. However the settings are very different if the XDMAC is a mem2mem or a per2mem controller. Thus, we need to differentiate according to device tree property. Signed-off-by: Eugen Hristev Link: https://lore.kernel.org/r/20201016093918.290137-1-eugen.hristev@microchip.com Signed-off-by: Vinod Koul --- drivers/dma/at_xdmac.c | 47 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/drivers/dma/at_xdmac.c b/drivers/dma/at_xdmac.c index 5a5c621206f2..fe45ad5d06c4 100644 --- a/drivers/dma/at_xdmac.c +++ b/drivers/dma/at_xdmac.c @@ -30,7 +30,24 @@ #define AT_XDMAC_FIFO_SZ(i) (((i) >> 5) & 0x7FF) /* Number of Bytes */ #define AT_XDMAC_NB_REQ(i) ((((i) >> 16) & 0x3F) + 1) /* Number of Peripheral Requests Minus One */ #define AT_XDMAC_GCFG 0x04 /* Global Configuration Register */ +#define AT_XDMAC_WRHP(i) (((i) & 0xF) << 4) +#define AT_XDMAC_WRMP(i) (((i) & 0xF) << 8) +#define AT_XDMAC_WRLP(i) (((i) & 0xF) << 12) +#define AT_XDMAC_RDHP(i) (((i) & 0xF) << 16) +#define AT_XDMAC_RDMP(i) (((i) & 0xF) << 20) +#define AT_XDMAC_RDLP(i) (((i) & 0xF) << 24) +#define AT_XDMAC_RDSG(i) (((i) & 0xF) << 28) +#define AT_XDMAC_GCFG_M2M (AT_XDMAC_RDLP(0xF) | AT_XDMAC_WRLP(0xF)) +#define AT_XDMAC_GCFG_P2M (AT_XDMAC_RDSG(0x1) | AT_XDMAC_RDHP(0x3) | \ + AT_XDMAC_WRHP(0x5)) #define AT_XDMAC_GWAC 0x08 /* Global Weighted Arbiter Configuration Register */ +#define AT_XDMAC_PW0(i) (((i) & 0xF) << 0) +#define AT_XDMAC_PW1(i) (((i) & 0xF) << 4) +#define AT_XDMAC_PW2(i) (((i) & 0xF) << 8) +#define AT_XDMAC_PW3(i) (((i) & 0xF) << 12) +#define AT_XDMAC_GWAC_M2M 0 +#define AT_XDMAC_GWAC_P2M (AT_XDMAC_PW0(0xF) | AT_XDMAC_PW2(0xF)) + #define AT_XDMAC_GIE 0x0C /* Global Interrupt Enable Register */ #define AT_XDMAC_GID 0x10 /* Global Interrupt Disable Register */ #define AT_XDMAC_GIM 0x14 /* Global Interrupt Mask Register */ @@ -189,6 +206,8 @@ struct at_xdmac_layout { u8 chan_cc_reg_base; /* Source/Destination Interface must be specified or not */ bool sdif; + /* AXI queue priority configuration supported */ + bool axi_config; }; /* ----- Channels ----- */ @@ -267,6 +286,7 @@ static const struct at_xdmac_layout at_xdmac_sama5d4_layout = { .gswf = 0x40, .chan_cc_reg_base = 0x50, .sdif = true, + .axi_config = false, }; static const struct at_xdmac_layout at_xdmac_sama7g5_layout = { @@ -279,6 +299,7 @@ static const struct at_xdmac_layout at_xdmac_sama7g5_layout = { .gswf = 0x50, .chan_cc_reg_base = 0x60, .sdif = false, + .axi_config = true, }; static inline void __iomem *at_xdmac_chan_reg_base(struct at_xdmac *atxdmac, unsigned int chan_nb) @@ -1997,6 +2018,30 @@ static int atmel_xdmac_resume(struct device *dev) } #endif /* CONFIG_PM_SLEEP */ +static void at_xdmac_axi_config(struct platform_device *pdev) +{ + struct at_xdmac *atxdmac = (struct at_xdmac *)platform_get_drvdata(pdev); + bool dev_m2m = false; + u32 dma_requests; + + if (!atxdmac->layout->axi_config) + return; /* Not supported */ + + if (!of_property_read_u32(pdev->dev.of_node, "dma-requests", + &dma_requests)) { + dev_info(&pdev->dev, "controller in mem2mem mode.\n"); + dev_m2m = true; + } + + if (dev_m2m) { + at_xdmac_write(atxdmac, AT_XDMAC_GCFG, AT_XDMAC_GCFG_M2M); + at_xdmac_write(atxdmac, AT_XDMAC_GWAC, AT_XDMAC_GWAC_M2M); + } else { + at_xdmac_write(atxdmac, AT_XDMAC_GCFG, AT_XDMAC_GCFG_P2M); + at_xdmac_write(atxdmac, AT_XDMAC_GWAC, AT_XDMAC_GWAC_P2M); + } +} + static int at_xdmac_probe(struct platform_device *pdev) { struct at_xdmac *atxdmac; @@ -2141,6 +2186,8 @@ static int at_xdmac_probe(struct platform_device *pdev) dev_info(&pdev->dev, "%d channels, mapped at 0x%p\n", nr_channels, atxdmac->regs); + at_xdmac_axi_config(pdev); + return 0; err_dma_unregister: -- cgit v1.2.3-59-g8ed1b From 68f35add4ba4c850a33878633e10c02d7ba32d84 Mon Sep 17 00:00:00 2001 From: Krzysztof Kozlowski Date: Mon, 19 Oct 2020 17:57:55 +0200 Subject: dmaengine: ppc4xx: make ppc440spe_adma_chan_list static The ppc440spe_adma_chan_list file-scope variable is not used outside of the unit so it can be made static. Reported-by: kernel test robot Signed-off-by: Krzysztof Kozlowski Link: https://lore.kernel.org/r/20201019155756.21445-1-krzk@kernel.org Signed-off-by: Vinod Koul --- drivers/dma/ppc4xx/adma.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/dma/ppc4xx/adma.c b/drivers/dma/ppc4xx/adma.c index 71cdaaa8134c..fea598550582 100644 --- a/drivers/dma/ppc4xx/adma.c +++ b/drivers/dma/ppc4xx/adma.c @@ -69,7 +69,7 @@ struct ppc_dma_chan_ref { }; /* The list of channels exported by ppc440spe ADMA */ -struct list_head +static struct list_head ppc440spe_adma_chan_list = LIST_HEAD_INIT(ppc440spe_adma_chan_list); /* This flag is set when want to refetch the xor chain in the interrupt -- cgit v1.2.3-59-g8ed1b From 212a93ca435ec847bdfe238f225dd6e8b77927e9 Mon Sep 17 00:00:00 2001 From: Krzysztof Kozlowski Date: Mon, 19 Oct 2020 17:57:56 +0200 Subject: dmaengine: ppc4xx: remove xor_hw_desc assignment without reading MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The xor_hw_desc local variable is assigned but never read: drivers/dma/ppc4xx/adma.c: In function ‘ppc440spe_desc_set_src_mult’: drivers/dma/ppc4xx/adma.c:562:17: warning: variable ‘xor_hw_desc’ set but not used [-Wunused-but-set-variable] Signed-off-by: Krzysztof Kozlowski Link: https://lore.kernel.org/r/20201019155756.21445-2-krzk@kernel.org Signed-off-by: Vinod Koul --- drivers/dma/ppc4xx/adma.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/drivers/dma/ppc4xx/adma.c b/drivers/dma/ppc4xx/adma.c index fea598550582..df7704053d91 100644 --- a/drivers/dma/ppc4xx/adma.c +++ b/drivers/dma/ppc4xx/adma.c @@ -559,7 +559,6 @@ static void ppc440spe_desc_set_src_mult(struct ppc440spe_adma_desc_slot *desc, int sg_index, unsigned char mult_value) { struct dma_cdb *dma_hw_desc; - struct xor_cb *xor_hw_desc; u32 *psgu; switch (chan->device->id) { @@ -590,7 +589,6 @@ static void ppc440spe_desc_set_src_mult(struct ppc440spe_adma_desc_slot *desc, *psgu |= cpu_to_le32(mult_value << mult_index); break; case PPC440SPE_XOR_ID: - xor_hw_desc = desc->hw_desc; break; default: BUG(); -- cgit v1.2.3-59-g8ed1b From 8e50d392652f20616a136165dff516b86baf5e49 Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Tue, 27 Oct 2020 10:34:35 -0700 Subject: dmaengine: idxd: Add shared workqueue support Add shared workqueue support that includes the support of Shared Virtual memory (SVM) or in similar terms On Demand Paging (ODP). The shared workqueue uses the enqcmds command in kernel and will respond with retry if the workqueue is full. Shared workqueue only works when there is PASID support from the IOMMU. Signed-off-by: Dave Jiang Reviewed-by: Tony Luck Reviewed-by: Dan Williams Link: https://lore.kernel.org/r/160382007499.3911367.26043087963708134.stgit@djiang5-desk3.ch.intel.com Signed-off-by: Vinod Koul --- drivers/dma/Kconfig | 10 ++++ drivers/dma/idxd/cdev.c | 49 ++++++++++++++++- drivers/dma/idxd/device.c | 90 +++++++++++++++++++++++++++--- drivers/dma/idxd/dma.c | 9 --- drivers/dma/idxd/idxd.h | 28 +++++++++- drivers/dma/idxd/init.c | 91 ++++++++++++++++++++++++------- drivers/dma/idxd/registers.h | 2 + drivers/dma/idxd/submit.c | 35 ++++++++++-- drivers/dma/idxd/sysfs.c | 127 +++++++++++++++++++++++++++++++++++++++++++ 9 files changed, 398 insertions(+), 43 deletions(-) diff --git a/drivers/dma/Kconfig b/drivers/dma/Kconfig index 518a1437862a..6a908785a5f7 100644 --- a/drivers/dma/Kconfig +++ b/drivers/dma/Kconfig @@ -296,6 +296,16 @@ config INTEL_IDXD If unsure, say N. +# Config symbol that collects all the dependencies that's necessary to +# support shared virtual memory for the devices supported by idxd. +config INTEL_IDXD_SVM + bool "Accelerator Shared Virtual Memory Support" + depends on INTEL_IDXD + depends on INTEL_IOMMU_SVM + depends on PCI_PRI + depends on PCI_PASID + depends on PCI_IOV + config INTEL_IOATDMA tristate "Intel I/OAT DMA support" depends on PCI && X86_64 diff --git a/drivers/dma/idxd/cdev.c b/drivers/dma/idxd/cdev.c index c3976156db2f..010b820d8f74 100644 --- a/drivers/dma/idxd/cdev.c +++ b/drivers/dma/idxd/cdev.c @@ -11,6 +11,7 @@ #include #include #include +#include #include #include "registers.h" #include "idxd.h" @@ -32,7 +33,9 @@ static struct idxd_cdev_context ictx[IDXD_TYPE_MAX] = { struct idxd_user_context { struct idxd_wq *wq; struct task_struct *task; + unsigned int pasid; unsigned int flags; + struct iommu_sva *sva; }; enum idxd_cdev_cleanup { @@ -75,6 +78,8 @@ static int idxd_cdev_open(struct inode *inode, struct file *filp) struct idxd_wq *wq; struct device *dev; int rc = 0; + struct iommu_sva *sva; + unsigned int pasid; wq = inode_wq(inode); idxd = wq->idxd; @@ -95,6 +100,34 @@ static int idxd_cdev_open(struct inode *inode, struct file *filp) ctx->wq = wq; filp->private_data = ctx; + + if (device_pasid_enabled(idxd)) { + sva = iommu_sva_bind_device(dev, current->mm, NULL); + if (IS_ERR(sva)) { + rc = PTR_ERR(sva); + dev_err(dev, "pasid allocation failed: %d\n", rc); + goto failed; + } + + pasid = iommu_sva_get_pasid(sva); + if (pasid == IOMMU_PASID_INVALID) { + iommu_sva_unbind_device(sva); + goto failed; + } + + ctx->sva = sva; + ctx->pasid = pasid; + + if (wq_dedicated(wq)) { + rc = idxd_wq_set_pasid(wq, pasid); + if (rc < 0) { + iommu_sva_unbind_device(sva); + dev_err(dev, "wq set pasid failed: %d\n", rc); + goto failed; + } + } + } + idxd_wq_get(wq); mutex_unlock(&wq->wq_lock); return 0; @@ -111,13 +144,27 @@ static int idxd_cdev_release(struct inode *node, struct file *filep) struct idxd_wq *wq = ctx->wq; struct idxd_device *idxd = wq->idxd; struct device *dev = &idxd->pdev->dev; + int rc; dev_dbg(dev, "%s called\n", __func__); filep->private_data = NULL; /* Wait for in-flight operations to complete. */ - idxd_wq_drain(wq); + if (wq_shared(wq)) { + idxd_device_drain_pasid(idxd, ctx->pasid); + } else { + if (device_pasid_enabled(idxd)) { + /* The wq disable in the disable pasid function will drain the wq */ + rc = idxd_wq_disable_pasid(wq); + if (rc < 0) + dev_err(dev, "wq disable pasid failed.\n"); + } else { + idxd_wq_drain(wq); + } + } + if (ctx->sva) + iommu_sva_unbind_device(ctx->sva); kfree(ctx); mutex_lock(&wq->wq_lock); idxd_wq_put(wq); diff --git a/drivers/dma/idxd/device.c b/drivers/dma/idxd/device.c index 506ac85c4a7f..2f09eb89a906 100644 --- a/drivers/dma/idxd/device.c +++ b/drivers/dma/idxd/device.c @@ -273,10 +273,9 @@ int idxd_wq_map_portal(struct idxd_wq *wq) start = pci_resource_start(pdev, IDXD_WQ_BAR); start = start + wq->id * IDXD_PORTAL_SIZE; - wq->dportal = devm_ioremap(dev, start, IDXD_PORTAL_SIZE); - if (!wq->dportal) + wq->portal = devm_ioremap(dev, start, IDXD_PORTAL_SIZE); + if (!wq->portal) return -ENOMEM; - dev_dbg(dev, "wq %d portal mapped at %p\n", wq->id, wq->dportal); return 0; } @@ -285,7 +284,61 @@ void idxd_wq_unmap_portal(struct idxd_wq *wq) { struct device *dev = &wq->idxd->pdev->dev; - devm_iounmap(dev, wq->dportal); + devm_iounmap(dev, wq->portal); +} + +int idxd_wq_set_pasid(struct idxd_wq *wq, int pasid) +{ + struct idxd_device *idxd = wq->idxd; + int rc; + union wqcfg wqcfg; + unsigned int offset; + unsigned long flags; + + rc = idxd_wq_disable(wq); + if (rc < 0) + return rc; + + offset = WQCFG_OFFSET(idxd, wq->id, WQCFG_PASID_IDX); + spin_lock_irqsave(&idxd->dev_lock, flags); + wqcfg.bits[WQCFG_PASID_IDX] = ioread32(idxd->reg_base + offset); + wqcfg.pasid_en = 1; + wqcfg.pasid = pasid; + iowrite32(wqcfg.bits[WQCFG_PASID_IDX], idxd->reg_base + offset); + spin_unlock_irqrestore(&idxd->dev_lock, flags); + + rc = idxd_wq_enable(wq); + if (rc < 0) + return rc; + + return 0; +} + +int idxd_wq_disable_pasid(struct idxd_wq *wq) +{ + struct idxd_device *idxd = wq->idxd; + int rc; + union wqcfg wqcfg; + unsigned int offset; + unsigned long flags; + + rc = idxd_wq_disable(wq); + if (rc < 0) + return rc; + + offset = WQCFG_OFFSET(idxd, wq->id, WQCFG_PASID_IDX); + spin_lock_irqsave(&idxd->dev_lock, flags); + wqcfg.bits[WQCFG_PASID_IDX] = ioread32(idxd->reg_base + offset); + wqcfg.pasid_en = 0; + wqcfg.pasid = 0; + iowrite32(wqcfg.bits[WQCFG_PASID_IDX], idxd->reg_base + offset); + spin_unlock_irqrestore(&idxd->dev_lock, flags); + + rc = idxd_wq_enable(wq); + if (rc < 0) + return rc; + + return 0; } void idxd_wq_disable_cleanup(struct idxd_wq *wq) @@ -468,6 +521,17 @@ void idxd_device_reset(struct idxd_device *idxd) spin_unlock_irqrestore(&idxd->dev_lock, flags); } +void idxd_device_drain_pasid(struct idxd_device *idxd, int pasid) +{ + struct device *dev = &idxd->pdev->dev; + u32 operand; + + operand = pasid; + dev_dbg(dev, "cmd: %u operand: %#x\n", IDXD_CMD_DRAIN_PASID, operand); + idxd_cmd_exec(idxd, IDXD_CMD_DRAIN_PASID, operand, NULL); + dev_dbg(dev, "pasid %d drained\n", pasid); +} + /* Device configuration bits */ static void idxd_group_config_write(struct idxd_group *group) { @@ -554,9 +618,21 @@ static int idxd_wq_config_write(struct idxd_wq *wq) /* byte 8-11 */ wq->wqcfg->priv = !!(wq->type == IDXD_WQT_KERNEL); - wq->wqcfg->mode = 1; + if (wq_dedicated(wq)) + wq->wqcfg->mode = 1; + + if (device_pasid_enabled(idxd)) { + wq->wqcfg->pasid_en = 1; + if (wq->type == IDXD_WQT_KERNEL && wq_dedicated(wq)) + wq->wqcfg->pasid = idxd->pasid; + } + wq->wqcfg->priority = wq->priority; + if (idxd->hw.gen_cap.block_on_fault && + test_bit(WQ_FLAG_BLOCK_ON_FAULT, &wq->flags)) + wq->wqcfg->bof = 1; + /* bytes 12-15 */ wq->wqcfg->max_xfer_shift = ilog2(wq->max_xfer_bytes); wq->wqcfg->max_batch_shift = ilog2(wq->max_batch_size); @@ -664,8 +740,8 @@ static int idxd_wqs_setup(struct idxd_device *idxd) if (!wq->size) continue; - if (!wq_dedicated(wq)) { - dev_warn(dev, "No shared workqueue support.\n"); + if (wq_shared(wq) && !device_swq_supported(idxd)) { + dev_warn(dev, "No shared wq support but configured.\n"); return -EINVAL; } diff --git a/drivers/dma/idxd/dma.c b/drivers/dma/idxd/dma.c index 0c892cbd72e0..8ed2773d8285 100644 --- a/drivers/dma/idxd/dma.c +++ b/drivers/dma/idxd/dma.c @@ -61,8 +61,6 @@ static inline void idxd_prep_desc_common(struct idxd_wq *wq, u64 addr_f1, u64 addr_f2, u64 len, u64 compl, u32 flags) { - struct idxd_device *idxd = wq->idxd; - hw->flags = flags; hw->opcode = opcode; hw->src_addr = addr_f1; @@ -70,13 +68,6 @@ static inline void idxd_prep_desc_common(struct idxd_wq *wq, hw->xfer_size = len; hw->priv = !!(wq->type == IDXD_WQT_KERNEL); hw->completion_addr = compl; - - /* - * Descriptor completion vectors are 1-8 for MSIX. We will round - * robin through the 8 vectors. - */ - wq->vec_ptr = (wq->vec_ptr % idxd->num_wq_irqs) + 1; - hw->int_handle = wq->vec_ptr; } static struct dma_async_tx_descriptor * diff --git a/drivers/dma/idxd/idxd.h b/drivers/dma/idxd/idxd.h index d48f193daacc..dcac3bb5a0d0 100644 --- a/drivers/dma/idxd/idxd.h +++ b/drivers/dma/idxd/idxd.h @@ -59,6 +59,7 @@ enum idxd_wq_state { enum idxd_wq_flag { WQ_FLAG_DEDICATED = 0, + WQ_FLAG_BLOCK_ON_FAULT, }; enum idxd_wq_type { @@ -86,10 +87,11 @@ enum idxd_op_type { enum idxd_complete_type { IDXD_COMPLETE_NORMAL = 0, IDXD_COMPLETE_ABORT, + IDXD_COMPLETE_DEV_FAIL, }; struct idxd_wq { - void __iomem *dportal; + void __iomem *portal; struct device conf_dev; struct idxd_cdev idxd_cdev; struct idxd_device *idxd; @@ -145,6 +147,7 @@ enum idxd_device_state { enum idxd_device_flag { IDXD_FLAG_CONFIGURABLE = 0, IDXD_FLAG_CMD_RUNNING, + IDXD_FLAG_PASID_ENABLED, }; struct idxd_device { @@ -167,6 +170,9 @@ struct idxd_device { struct idxd_wq *wqs; struct idxd_engine *engines; + struct iommu_sva *sva; + unsigned int pasid; + int num_groups; u32 msix_perm_offset; @@ -215,11 +221,28 @@ struct idxd_desc { extern struct bus_type dsa_bus_type; +extern bool support_enqcmd; + static inline bool wq_dedicated(struct idxd_wq *wq) { return test_bit(WQ_FLAG_DEDICATED, &wq->flags); } +static inline bool wq_shared(struct idxd_wq *wq) +{ + return !test_bit(WQ_FLAG_DEDICATED, &wq->flags); +} + +static inline bool device_pasid_enabled(struct idxd_device *idxd) +{ + return test_bit(IDXD_FLAG_PASID_ENABLED, &idxd->flags); +} + +static inline bool device_swq_supported(struct idxd_device *idxd) +{ + return (support_enqcmd && device_pasid_enabled(idxd)); +} + enum idxd_portal_prot { IDXD_PORTAL_UNLIMITED = 0, IDXD_PORTAL_LIMITED, @@ -288,6 +311,7 @@ void idxd_device_reset(struct idxd_device *idxd); void idxd_device_cleanup(struct idxd_device *idxd); int idxd_device_config(struct idxd_device *idxd); void idxd_device_wqs_clear_state(struct idxd_device *idxd); +void idxd_device_drain_pasid(struct idxd_device *idxd, int pasid); /* work queue control */ int idxd_wq_alloc_resources(struct idxd_wq *wq); @@ -298,6 +322,8 @@ void idxd_wq_drain(struct idxd_wq *wq); int idxd_wq_map_portal(struct idxd_wq *wq); void idxd_wq_unmap_portal(struct idxd_wq *wq); void idxd_wq_disable_cleanup(struct idxd_wq *wq); +int idxd_wq_set_pasid(struct idxd_wq *wq, int pasid); +int idxd_wq_disable_pasid(struct idxd_wq *wq); /* submission */ int idxd_submit_desc(struct idxd_wq *wq, struct idxd_desc *desc); diff --git a/drivers/dma/idxd/init.c b/drivers/dma/idxd/init.c index 0a4432b063b5..1639f3b2aa58 100644 --- a/drivers/dma/idxd/init.c +++ b/drivers/dma/idxd/init.c @@ -14,6 +14,8 @@ #include #include #include +#include +#include #include #include #include "../dmaengine.h" @@ -26,6 +28,8 @@ MODULE_AUTHOR("Intel Corporation"); #define DRV_NAME "idxd" +bool support_enqcmd; + static struct idr idxd_idrs[IDXD_TYPE_MAX]; static struct mutex idxd_idr_lock; @@ -53,6 +57,7 @@ static int idxd_setup_interrupts(struct idxd_device *idxd) struct idxd_irq_entry *irq_entry; int i, msixcnt; int rc = 0; + union msix_perm mperm; msixcnt = pci_msix_vec_count(pdev); if (msixcnt < 0) { @@ -131,6 +136,13 @@ static int idxd_setup_interrupts(struct idxd_device *idxd) idxd_unmask_error_interrupts(idxd); + /* Setup MSIX permission table */ + mperm.bits = 0; + mperm.pasid = idxd->pasid; + mperm.pasid_en = device_pasid_enabled(idxd); + for (i = 1; i < msixcnt; i++) + iowrite32(mperm.bits, idxd->reg_base + idxd->msix_perm_offset + i * 8); + return 0; err_no_irq: @@ -265,8 +277,7 @@ static void idxd_read_caps(struct idxd_device *idxd) } } -static struct idxd_device *idxd_alloc(struct pci_dev *pdev, - void __iomem * const *iomap) +static struct idxd_device *idxd_alloc(struct pci_dev *pdev) { struct device *dev = &pdev->dev; struct idxd_device *idxd; @@ -276,12 +287,45 @@ static struct idxd_device *idxd_alloc(struct pci_dev *pdev, return NULL; idxd->pdev = pdev; - idxd->reg_base = iomap[IDXD_MMIO_BAR]; spin_lock_init(&idxd->dev_lock); return idxd; } +static int idxd_enable_system_pasid(struct idxd_device *idxd) +{ + int flags; + unsigned int pasid; + struct iommu_sva *sva; + + flags = SVM_FLAG_SUPERVISOR_MODE; + + sva = iommu_sva_bind_device(&idxd->pdev->dev, NULL, &flags); + if (IS_ERR(sva)) { + dev_warn(&idxd->pdev->dev, + "iommu sva bind failed: %ld\n", PTR_ERR(sva)); + return PTR_ERR(sva); + } + + pasid = iommu_sva_get_pasid(sva); + if (pasid == IOMMU_PASID_INVALID) { + iommu_sva_unbind_device(sva); + return -ENODEV; + } + + idxd->sva = sva; + idxd->pasid = pasid; + dev_dbg(&idxd->pdev->dev, "system pasid: %u\n", pasid); + return 0; +} + +static void idxd_disable_system_pasid(struct idxd_device *idxd) +{ + + iommu_sva_unbind_device(idxd->sva); + idxd->sva = NULL; +} + static int idxd_probe(struct idxd_device *idxd) { struct pci_dev *pdev = idxd->pdev; @@ -292,6 +336,14 @@ static int idxd_probe(struct idxd_device *idxd) idxd_device_init_reset(idxd); dev_dbg(dev, "IDXD reset complete\n"); + if (IS_ENABLED(CONFIG_INTEL_IDXD_SVM)) { + rc = idxd_enable_system_pasid(idxd); + if (rc < 0) + dev_warn(dev, "Failed to enable PASID. No SVA support: %d\n", rc); + else + set_bit(IDXD_FLAG_PASID_ENABLED, &idxd->flags); + } + idxd_read_caps(idxd); idxd_read_table_offsets(idxd); @@ -322,29 +374,29 @@ static int idxd_probe(struct idxd_device *idxd) idxd_mask_error_interrupts(idxd); idxd_mask_msix_vectors(idxd); err_setup: + if (device_pasid_enabled(idxd)) + idxd_disable_system_pasid(idxd); return rc; } static int idxd_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) { - void __iomem * const *iomap; struct device *dev = &pdev->dev; struct idxd_device *idxd; int rc; - unsigned int mask; rc = pcim_enable_device(pdev); if (rc) return rc; - dev_dbg(dev, "Mapping BARs\n"); - mask = (1 << IDXD_MMIO_BAR); - rc = pcim_iomap_regions(pdev, mask, DRV_NAME); - if (rc) - return rc; + dev_dbg(dev, "Alloc IDXD context\n"); + idxd = idxd_alloc(pdev); + if (!idxd) + return -ENOMEM; - iomap = pcim_iomap_table(pdev); - if (!iomap) + dev_dbg(dev, "Mapping BARs\n"); + idxd->reg_base = pcim_iomap(pdev, IDXD_MMIO_BAR, 0); + if (!idxd->reg_base) return -ENOMEM; dev_dbg(dev, "Set DMA masks\n"); @@ -360,11 +412,6 @@ static int idxd_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) if (rc) return rc; - dev_dbg(dev, "Alloc IDXD context\n"); - idxd = idxd_alloc(pdev, iomap); - if (!idxd) - return -ENOMEM; - idxd_set_type(idxd); dev_dbg(dev, "Set PCI master\n"); @@ -452,6 +499,8 @@ static void idxd_remove(struct pci_dev *pdev) dev_dbg(&pdev->dev, "%s called\n", __func__); idxd_cleanup_sysfs(idxd); idxd_shutdown(pdev); + if (device_pasid_enabled(idxd)) + idxd_disable_system_pasid(idxd); mutex_lock(&idxd_idr_lock); idr_remove(&idxd_idrs[idxd->type], idxd->id); mutex_unlock(&idxd_idr_lock); @@ -470,7 +519,7 @@ static int __init idxd_init_module(void) int err, i; /* - * If the CPU does not support write512, there's no point in + * If the CPU does not support MOVDIR64B or ENQCMDS, there's no point in * enumerating the device. We can not utilize it. */ if (!boot_cpu_has(X86_FEATURE_MOVDIR64B)) { @@ -478,8 +527,10 @@ static int __init idxd_init_module(void) return -ENODEV; } - pr_info("%s: Intel(R) Accelerator Devices Driver %s\n", - DRV_NAME, IDXD_DRIVER_VERSION); + if (!boot_cpu_has(X86_FEATURE_ENQCMD)) + pr_warn("Platform does not have ENQCMD(S) support.\n"); + else + support_enqcmd = true; mutex_init(&idxd_idr_lock); for (i = 0; i < IDXD_TYPE_MAX; i++) diff --git a/drivers/dma/idxd/registers.h b/drivers/dma/idxd/registers.h index aef5a902829e..1041c2779f9b 100644 --- a/drivers/dma/idxd/registers.h +++ b/drivers/dma/idxd/registers.h @@ -336,6 +336,8 @@ union wqcfg { u32 bits[8]; } __packed; +#define WQCFG_PASID_IDX 2 + /* * This macro calculates the offset into the WQCFG register * idxd - struct idxd * diff --git a/drivers/dma/idxd/submit.c b/drivers/dma/idxd/submit.c index 156a1ee233aa..efca5d8468a6 100644 --- a/drivers/dma/idxd/submit.c +++ b/drivers/dma/idxd/submit.c @@ -11,11 +11,22 @@ static struct idxd_desc *__get_desc(struct idxd_wq *wq, int idx, int cpu) { struct idxd_desc *desc; + struct idxd_device *idxd = wq->idxd; desc = wq->descs[idx]; memset(desc->hw, 0, sizeof(struct dsa_hw_desc)); memset(desc->completion, 0, sizeof(struct dsa_completion_record)); desc->cpu = cpu; + + if (device_pasid_enabled(idxd)) + desc->hw->pasid = idxd->pasid; + + /* + * Descriptor completion vectors are 1-8 for MSIX. We will round + * robin through the 8 vectors. + */ + wq->vec_ptr = (wq->vec_ptr % idxd->num_wq_irqs) + 1; + desc->hw->int_handle = wq->vec_ptr; return desc; } @@ -70,18 +81,32 @@ int idxd_submit_desc(struct idxd_wq *wq, struct idxd_desc *desc) struct idxd_device *idxd = wq->idxd; int vec = desc->hw->int_handle; void __iomem *portal; + int rc; if (idxd->state != IDXD_DEV_ENABLED) return -EIO; - portal = wq->dportal + idxd_get_wq_portal_offset(IDXD_PORTAL_UNLIMITED); + portal = wq->portal + idxd_get_wq_portal_offset(IDXD_PORTAL_LIMITED); + /* - * The wmb() flushes writes to coherent DMA data before possibly - * triggering a DMA read. The wmb() is necessary even on UP because - * the recipient is a device. + * The wmb() flushes writes to coherent DMA data before + * possibly triggering a DMA read. The wmb() is necessary + * even on UP because the recipient is a device. */ wmb(); - iosubmit_cmds512(portal, desc->hw, 1); + if (wq_dedicated(wq)) { + iosubmit_cmds512(portal, desc->hw, 1); + } else { + /* + * It's not likely that we would receive queue full rejection + * since the descriptor allocation gates at wq size. If we + * receive a -EAGAIN, that means something went wrong such as the + * device is not accepting descriptor at all. + */ + rc = enqcmds(portal, desc->hw); + if (rc < 0) + return rc; + } /* * Pending the descriptor to the lockless list for the irq_entry diff --git a/drivers/dma/idxd/sysfs.c b/drivers/dma/idxd/sysfs.c index 07a5db06a29a..6d292eb79bf3 100644 --- a/drivers/dma/idxd/sysfs.c +++ b/drivers/dma/idxd/sysfs.c @@ -175,6 +175,30 @@ static int idxd_config_bus_probe(struct device *dev) return -EINVAL; } + /* Shared WQ checks */ + if (wq_shared(wq)) { + if (!device_swq_supported(idxd)) { + dev_warn(dev, + "PASID not enabled and shared WQ.\n"); + mutex_unlock(&wq->wq_lock); + return -ENXIO; + } + /* + * Shared wq with the threshold set to 0 means the user + * did not set the threshold or transitioned from a + * dedicated wq but did not set threshold. A value + * of 0 would effectively disable the shared wq. The + * driver does not allow a value of 0 to be set for + * threshold via sysfs. + */ + if (wq->threshold == 0) { + dev_warn(dev, + "Shared WQ and threshold 0.\n"); + mutex_unlock(&wq->wq_lock); + return -EINVAL; + } + } + rc = idxd_wq_alloc_resources(wq); if (rc < 0) { mutex_unlock(&wq->wq_lock); @@ -875,6 +899,8 @@ static ssize_t wq_mode_store(struct device *dev, if (sysfs_streq(buf, "dedicated")) { set_bit(WQ_FLAG_DEDICATED, &wq->flags); wq->threshold = 0; + } else if (sysfs_streq(buf, "shared") && device_swq_supported(idxd)) { + clear_bit(WQ_FLAG_DEDICATED, &wq->flags); } else { return -EINVAL; } @@ -973,6 +999,87 @@ static ssize_t wq_priority_store(struct device *dev, static struct device_attribute dev_attr_wq_priority = __ATTR(priority, 0644, wq_priority_show, wq_priority_store); +static ssize_t wq_block_on_fault_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct idxd_wq *wq = container_of(dev, struct idxd_wq, conf_dev); + + return sprintf(buf, "%u\n", + test_bit(WQ_FLAG_BLOCK_ON_FAULT, &wq->flags)); +} + +static ssize_t wq_block_on_fault_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) +{ + struct idxd_wq *wq = container_of(dev, struct idxd_wq, conf_dev); + struct idxd_device *idxd = wq->idxd; + bool bof; + int rc; + + if (!test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags)) + return -EPERM; + + if (wq->state != IDXD_WQ_DISABLED) + return -ENXIO; + + rc = kstrtobool(buf, &bof); + if (rc < 0) + return rc; + + if (bof) + set_bit(WQ_FLAG_BLOCK_ON_FAULT, &wq->flags); + else + clear_bit(WQ_FLAG_BLOCK_ON_FAULT, &wq->flags); + + return count; +} + +static struct device_attribute dev_attr_wq_block_on_fault = + __ATTR(block_on_fault, 0644, wq_block_on_fault_show, + wq_block_on_fault_store); + +static ssize_t wq_threshold_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct idxd_wq *wq = container_of(dev, struct idxd_wq, conf_dev); + + return sprintf(buf, "%u\n", wq->threshold); +} + +static ssize_t wq_threshold_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) +{ + struct idxd_wq *wq = container_of(dev, struct idxd_wq, conf_dev); + struct idxd_device *idxd = wq->idxd; + unsigned int val; + int rc; + + rc = kstrtouint(buf, 0, &val); + if (rc < 0) + return -EINVAL; + + if (val > wq->size || val <= 0) + return -EINVAL; + + if (!test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags)) + return -EPERM; + + if (wq->state != IDXD_WQ_DISABLED) + return -ENXIO; + + if (test_bit(WQ_FLAG_DEDICATED, &wq->flags)) + return -EINVAL; + + wq->threshold = val; + + return count; +} + +static struct device_attribute dev_attr_wq_threshold = + __ATTR(threshold, 0644, wq_threshold_show, wq_threshold_store); + static ssize_t wq_type_show(struct device *dev, struct device_attribute *attr, char *buf) { @@ -1044,6 +1151,13 @@ static ssize_t wq_name_store(struct device *dev, if (strlen(buf) > WQ_NAME_SIZE || strlen(buf) == 0) return -EINVAL; + /* + * This is temporarily placed here until we have SVM support for + * dmaengine. + */ + if (wq->type == IDXD_WQT_KERNEL && device_pasid_enabled(wq->idxd)) + return -EOPNOTSUPP; + memset(wq->name, 0, WQ_NAME_SIZE + 1); strncpy(wq->name, buf, WQ_NAME_SIZE); strreplace(wq->name, '\n', '\0'); @@ -1154,6 +1268,8 @@ static struct attribute *idxd_wq_attributes[] = { &dev_attr_wq_mode.attr, &dev_attr_wq_size.attr, &dev_attr_wq_priority.attr, + &dev_attr_wq_block_on_fault.attr, + &dev_attr_wq_threshold.attr, &dev_attr_wq_type.attr, &dev_attr_wq_name.attr, &dev_attr_wq_cdev_minor.attr, @@ -1305,6 +1421,16 @@ static ssize_t clients_show(struct device *dev, } static DEVICE_ATTR_RO(clients); +static ssize_t pasid_enabled_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct idxd_device *idxd = + container_of(dev, struct idxd_device, conf_dev); + + return sprintf(buf, "%u\n", device_pasid_enabled(idxd)); +} +static DEVICE_ATTR_RO(pasid_enabled); + static ssize_t state_show(struct device *dev, struct device_attribute *attr, char *buf) { @@ -1424,6 +1550,7 @@ static struct attribute *idxd_device_attributes[] = { &dev_attr_gen_cap.attr, &dev_attr_configurable.attr, &dev_attr_clients.attr, + &dev_attr_pasid_enabled.attr, &dev_attr_state.attr, &dev_attr_errors.attr, &dev_attr_max_tokens.attr, -- cgit v1.2.3-59-g8ed1b From e4f4d8cdeb9a2fe746411c0b9a7538b5c0232c1e Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Tue, 27 Oct 2020 10:34:40 -0700 Subject: dmaengine: idxd: Clean up descriptors with fault error Add code to "complete" a descriptor when the descriptor or its completion address hit a fault error when SVA mode is being used. This error can be triggered due to bad programming by the user. A lock is introduced in order to protect the descriptor completion lists since the fault handler will run from the system work queue after being scheduled in the interrupt handler. Signed-off-by: Dave Jiang Reviewed-by: Tony Luck Reviewed-by: Dan Williams Link: https://lore.kernel.org/r/160382008092.3911367.12766483427643278985.stgit@djiang5-desk3.ch.intel.com Signed-off-by: Vinod Koul --- drivers/dma/idxd/idxd.h | 5 ++ drivers/dma/idxd/init.c | 1 + drivers/dma/idxd/irq.c | 146 ++++++++++++++++++++++++++++++++++++++++++++---- 3 files changed, 140 insertions(+), 12 deletions(-) diff --git a/drivers/dma/idxd/idxd.h b/drivers/dma/idxd/idxd.h index dcac3bb5a0d0..7e54209c433a 100644 --- a/drivers/dma/idxd/idxd.h +++ b/drivers/dma/idxd/idxd.h @@ -34,6 +34,11 @@ struct idxd_irq_entry { int id; struct llist_head pending_llist; struct list_head work_list; + /* + * Lock to protect access between irq thread process descriptor + * and irq thread processing error descriptor. + */ + spinlock_t list_lock; }; struct idxd_group { diff --git a/drivers/dma/idxd/init.c b/drivers/dma/idxd/init.c index 1639f3b2aa58..c24106efc16e 100644 --- a/drivers/dma/idxd/init.c +++ b/drivers/dma/idxd/init.c @@ -97,6 +97,7 @@ static int idxd_setup_interrupts(struct idxd_device *idxd) for (i = 0; i < msixcnt; i++) { idxd->irq_entries[i].id = i; idxd->irq_entries[i].idxd = idxd; + spin_lock_init(&idxd->irq_entries[i].list_lock); } msix = &idxd->msix_entries[0]; diff --git a/drivers/dma/idxd/irq.c b/drivers/dma/idxd/irq.c index 17a65a13fb64..593a2f6ed16c 100644 --- a/drivers/dma/idxd/irq.c +++ b/drivers/dma/idxd/irq.c @@ -11,6 +11,24 @@ #include "idxd.h" #include "registers.h" +enum irq_work_type { + IRQ_WORK_NORMAL = 0, + IRQ_WORK_PROCESS_FAULT, +}; + +struct idxd_fault { + struct work_struct work; + u64 addr; + struct idxd_device *idxd; +}; + +static int irq_process_work_list(struct idxd_irq_entry *irq_entry, + enum irq_work_type wtype, + int *processed, u64 data); +static int irq_process_pending_llist(struct idxd_irq_entry *irq_entry, + enum irq_work_type wtype, + int *processed, u64 data); + static void idxd_device_reinit(struct work_struct *work) { struct idxd_device *idxd = container_of(work, struct idxd_device, work); @@ -44,6 +62,46 @@ static void idxd_device_reinit(struct work_struct *work) idxd_device_wqs_clear_state(idxd); } +static void idxd_device_fault_work(struct work_struct *work) +{ + struct idxd_fault *fault = container_of(work, struct idxd_fault, work); + struct idxd_irq_entry *ie; + int i; + int processed; + int irqcnt = fault->idxd->num_wq_irqs + 1; + + for (i = 1; i < irqcnt; i++) { + ie = &fault->idxd->irq_entries[i]; + irq_process_work_list(ie, IRQ_WORK_PROCESS_FAULT, + &processed, fault->addr); + if (processed) + break; + + irq_process_pending_llist(ie, IRQ_WORK_PROCESS_FAULT, + &processed, fault->addr); + if (processed) + break; + } + + kfree(fault); +} + +static int idxd_device_schedule_fault_process(struct idxd_device *idxd, + u64 fault_addr) +{ + struct idxd_fault *fault; + + fault = kmalloc(sizeof(*fault), GFP_ATOMIC); + if (!fault) + return -ENOMEM; + + fault->addr = fault_addr; + fault->idxd = idxd; + INIT_WORK(&fault->work, idxd_device_fault_work); + queue_work(idxd->wq, &fault->work); + return 0; +} + irqreturn_t idxd_irq_handler(int vec, void *data) { struct idxd_irq_entry *irq_entry = data; @@ -125,6 +183,15 @@ irqreturn_t idxd_misc_thread(int vec, void *data) if (!err) goto out; + /* + * This case should rarely happen and typically is due to software + * programming error by the driver. + */ + if (idxd->sw_err.valid && + idxd->sw_err.desc_valid && + idxd->sw_err.fault_addr) + idxd_device_schedule_fault_process(idxd, idxd->sw_err.fault_addr); + gensts.bits = ioread32(idxd->reg_base + IDXD_GENSTATS_OFFSET); if (gensts.state == IDXD_DEVICE_STATE_HALT) { idxd->state = IDXD_DEV_HALTED; @@ -152,57 +219,110 @@ irqreturn_t idxd_misc_thread(int vec, void *data) return IRQ_HANDLED; } +static bool process_fault(struct idxd_desc *desc, u64 fault_addr) +{ + /* + * Completion address can be bad as well. Check fault address match for descriptor + * and completion address. + */ + if ((u64)desc->hw == fault_addr || + (u64)desc->completion == fault_addr) { + idxd_dma_complete_txd(desc, IDXD_COMPLETE_DEV_FAIL); + return true; + } + + return false; +} + +static bool complete_desc(struct idxd_desc *desc) +{ + if (desc->completion->status) { + idxd_dma_complete_txd(desc, IDXD_COMPLETE_NORMAL); + return true; + } + + return false; +} + static int irq_process_pending_llist(struct idxd_irq_entry *irq_entry, - int *processed) + enum irq_work_type wtype, + int *processed, u64 data) { struct idxd_desc *desc, *t; struct llist_node *head; int queued = 0; + bool completed = false; + unsigned long flags; *processed = 0; head = llist_del_all(&irq_entry->pending_llist); if (!head) - return 0; + goto out; llist_for_each_entry_safe(desc, t, head, llnode) { - if (desc->completion->status) { - idxd_dma_complete_txd(desc, IDXD_COMPLETE_NORMAL); + if (wtype == IRQ_WORK_NORMAL) + completed = complete_desc(desc); + else if (wtype == IRQ_WORK_PROCESS_FAULT) + completed = process_fault(desc, data); + + if (completed) { idxd_free_desc(desc->wq, desc); (*processed)++; + if (wtype == IRQ_WORK_PROCESS_FAULT) + break; } else { - list_add_tail(&desc->list, &irq_entry->work_list); + spin_lock_irqsave(&irq_entry->list_lock, flags); + list_add_tail(&desc->list, + &irq_entry->work_list); + spin_unlock_irqrestore(&irq_entry->list_lock, flags); queued++; } } + out: return queued; } static int irq_process_work_list(struct idxd_irq_entry *irq_entry, - int *processed) + enum irq_work_type wtype, + int *processed, u64 data) { struct list_head *node, *next; int queued = 0; + bool completed = false; + unsigned long flags; *processed = 0; + spin_lock_irqsave(&irq_entry->list_lock, flags); if (list_empty(&irq_entry->work_list)) - return 0; + goto out; list_for_each_safe(node, next, &irq_entry->work_list) { struct idxd_desc *desc = container_of(node, struct idxd_desc, list); - if (desc->completion->status) { + spin_unlock_irqrestore(&irq_entry->list_lock, flags); + if (wtype == IRQ_WORK_NORMAL) + completed = complete_desc(desc); + else if (wtype == IRQ_WORK_PROCESS_FAULT) + completed = process_fault(desc, data); + + if (completed) { + spin_lock_irqsave(&irq_entry->list_lock, flags); list_del(&desc->list); - /* process and callback */ - idxd_dma_complete_txd(desc, IDXD_COMPLETE_NORMAL); + spin_unlock_irqrestore(&irq_entry->list_lock, flags); idxd_free_desc(desc->wq, desc); (*processed)++; + if (wtype == IRQ_WORK_PROCESS_FAULT) + return queued; } else { queued++; } + spin_lock_irqsave(&irq_entry->list_lock, flags); } + out: + spin_unlock_irqrestore(&irq_entry->list_lock, flags); return queued; } @@ -230,12 +350,14 @@ static int idxd_desc_process(struct idxd_irq_entry *irq_entry) * 5. Repeat until no more descriptors. */ do { - rc = irq_process_work_list(irq_entry, &processed); + rc = irq_process_work_list(irq_entry, IRQ_WORK_NORMAL, + &processed, 0); total += processed; if (rc != 0) continue; - rc = irq_process_pending_llist(irq_entry, &processed); + rc = irq_process_pending_llist(irq_entry, IRQ_WORK_NORMAL, + &processed, 0); total += processed; } while (rc != 0); -- cgit v1.2.3-59-g8ed1b From 4749f51ddd8aee5c7aa11e6593a54f96374a77ad Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Tue, 27 Oct 2020 10:34:46 -0700 Subject: dmaengine: idxd: Add ABI documentation for shared wq Add the sysfs attribute bits in ABI/stable for shared wq support. Signed-off-by: Jing Lin Signed-off-by: Dave Jiang Reviewed-by: Tony Luck Reviewed-by: Dan Williams Link: https://lore.kernel.org/r/160382008649.3911367.10851752182908509837.stgit@djiang5-desk3.ch.intel.com Signed-off-by: Vinod Koul --- Documentation/ABI/stable/sysfs-driver-dma-idxd | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/Documentation/ABI/stable/sysfs-driver-dma-idxd b/Documentation/ABI/stable/sysfs-driver-dma-idxd index b44183880935..5ea81ffd3c1a 100644 --- a/Documentation/ABI/stable/sysfs-driver-dma-idxd +++ b/Documentation/ABI/stable/sysfs-driver-dma-idxd @@ -77,6 +77,13 @@ Contact: dmaengine@vger.kernel.org Description: The operation capability bit mask specify the operation types supported by the this device. +What: /sys/bus/dsa/devices/dsa/pasid_enabled +Date: Oct 27, 2020 +KernelVersion: 5.11.0 +Contact: dmaengine@vger.kernel.org +Description: To indicate if PASID (process address space identifier) is + enabled or not for this device. + What: /sys/bus/dsa/devices/dsa/state Date: Oct 25, 2019 KernelVersion: 5.6.0 @@ -122,6 +129,13 @@ KernelVersion: 5.10.0 Contact: dmaengine@vger.kernel.org Description: The last executed device administrative command's status/error. +What: /sys/bus/dsa/devices/wq./block_on_fault +Date: Oct 27, 2020 +KernelVersion: 5.11.0 +Contact: dmaengine@vger.kernel.org +Description: To indicate block on fault is allowed or not for the work queue + to support on demand paging. + What: /sys/bus/dsa/devices/wq./group_id Date: Oct 25, 2019 KernelVersion: 5.6.0 -- cgit v1.2.3-59-g8ed1b From 5a71270197f39ff3e526cf130bc5d6e84db8f2d7 Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Fri, 30 Oct 2020 08:49:06 -0700 Subject: dmaengine: idxd: Update calculation of group offset to be more readable Create helper macros to make group offset calculation more readable. Signed-off-by: Dave Jiang Link: https://lore.kernel.org/r/160407294683.839093.10740868559754142070.stgit@djiang5-desk3.ch.intel.com Signed-off-by: Vinod Koul --- drivers/dma/idxd/device.c | 12 +++++------- drivers/dma/idxd/registers.h | 18 ++++++++++++++++++ 2 files changed, 23 insertions(+), 7 deletions(-) diff --git a/drivers/dma/idxd/device.c b/drivers/dma/idxd/device.c index 2f09eb89a906..d6f551dcbcb6 100644 --- a/drivers/dma/idxd/device.c +++ b/drivers/dma/idxd/device.c @@ -543,24 +543,22 @@ static void idxd_group_config_write(struct idxd_group *group) dev_dbg(dev, "Writing group %d cfg registers\n", group->id); /* setup GRPWQCFG */ - for (i = 0; i < 4; i++) { - grpcfg_offset = idxd->grpcfg_offset + - group->id * 64 + i * sizeof(u64); - iowrite64(group->grpcfg.wqs[i], - idxd->reg_base + grpcfg_offset); + for (i = 0; i < GRPWQCFG_STRIDES; i++) { + grpcfg_offset = GRPWQCFG_OFFSET(idxd, group->id, i); + iowrite64(group->grpcfg.wqs[i], idxd->reg_base + grpcfg_offset); dev_dbg(dev, "GRPCFG wq[%d:%d: %#x]: %#llx\n", group->id, i, grpcfg_offset, ioread64(idxd->reg_base + grpcfg_offset)); } /* setup GRPENGCFG */ - grpcfg_offset = idxd->grpcfg_offset + group->id * 64 + 32; + grpcfg_offset = GRPENGCFG_OFFSET(idxd, group->id); iowrite64(group->grpcfg.engines, idxd->reg_base + grpcfg_offset); dev_dbg(dev, "GRPCFG engs[%d: %#x]: %#llx\n", group->id, grpcfg_offset, ioread64(idxd->reg_base + grpcfg_offset)); /* setup GRPFLAGS */ - grpcfg_offset = idxd->grpcfg_offset + group->id * 64 + 40; + grpcfg_offset = GRPFLGCFG_OFFSET(idxd, group->id); iowrite32(group->grpcfg.flags.bits, idxd->reg_base + grpcfg_offset); dev_dbg(dev, "GRPFLAGS flags[%d: %#x]: %#x\n", group->id, grpcfg_offset, diff --git a/drivers/dma/idxd/registers.h b/drivers/dma/idxd/registers.h index 1041c2779f9b..6f2f736097e5 100644 --- a/drivers/dma/idxd/registers.h +++ b/drivers/dma/idxd/registers.h @@ -356,4 +356,22 @@ union wqcfg { #define WQCFG_STRIDES(_idxd_dev) ((_idxd_dev)->wqcfg_size / sizeof(u32)) +#define GRPCFG_SIZE 64 +#define GRPWQCFG_STRIDES 4 + +/* + * This macro calculates the offset into the GRPCFG register + * idxd - struct idxd * + * n - wq id + * ofs - the index of the 32b dword for the config register + * + * The WQCFG register block is divided into groups per each wq. The n index + * allows us to move to the register group that's for that particular wq. + * Each register is 32bits. The ofs gives us the number of register to access. + */ +#define GRPWQCFG_OFFSET(idxd_dev, n, ofs) ((idxd_dev)->grpcfg_offset +\ + (n) * GRPCFG_SIZE + sizeof(u64) * (ofs)) +#define GRPENGCFG_OFFSET(idxd_dev, n) ((idxd_dev)->grpcfg_offset + (n) * GRPCFG_SIZE + 32) +#define GRPFLGCFG_OFFSET(idxd_dev, n) ((idxd_dev)->grpcfg_offset + (n) * GRPCFG_SIZE + 40) + #endif -- cgit v1.2.3-59-g8ed1b From 2f8417a967d571bf8fb81cba95d7acf508ed334f Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Fri, 30 Oct 2020 08:51:56 -0700 Subject: dmaengine: idxd: define table offset multiplier Convert table offset multiplier magic number to a define. Signed-off-by: Dave Jiang Link: https://lore.kernel.org/r/160407311690.839435.6941865731867828234.stgit@djiang5-desk3.ch.intel.com Signed-off-by: Vinod Koul --- drivers/dma/idxd/init.c | 17 +++++++---------- drivers/dma/idxd/registers.h | 2 ++ 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/drivers/dma/idxd/init.c b/drivers/dma/idxd/init.c index c24106efc16e..45b0eac640c3 100644 --- a/drivers/dma/idxd/init.c +++ b/drivers/dma/idxd/init.c @@ -214,17 +214,14 @@ static void idxd_read_table_offsets(struct idxd_device *idxd) struct device *dev = &idxd->pdev->dev; offsets.bits[0] = ioread64(idxd->reg_base + IDXD_TABLE_OFFSET); - offsets.bits[1] = ioread64(idxd->reg_base + IDXD_TABLE_OFFSET - + sizeof(u64)); - idxd->grpcfg_offset = offsets.grpcfg * 0x100; + offsets.bits[1] = ioread64(idxd->reg_base + IDXD_TABLE_OFFSET + sizeof(u64)); + idxd->grpcfg_offset = offsets.grpcfg * IDXD_TABLE_MULT; dev_dbg(dev, "IDXD Group Config Offset: %#x\n", idxd->grpcfg_offset); - idxd->wqcfg_offset = offsets.wqcfg * 0x100; - dev_dbg(dev, "IDXD Work Queue Config Offset: %#x\n", - idxd->wqcfg_offset); - idxd->msix_perm_offset = offsets.msix_perm * 0x100; - dev_dbg(dev, "IDXD MSIX Permission Offset: %#x\n", - idxd->msix_perm_offset); - idxd->perfmon_offset = offsets.perfmon * 0x100; + idxd->wqcfg_offset = offsets.wqcfg * IDXD_TABLE_MULT; + dev_dbg(dev, "IDXD Work Queue Config Offset: %#x\n", idxd->wqcfg_offset); + idxd->msix_perm_offset = offsets.msix_perm * IDXD_TABLE_MULT; + dev_dbg(dev, "IDXD MSIX Permission Offset: %#x\n", idxd->msix_perm_offset); + idxd->perfmon_offset = offsets.perfmon * IDXD_TABLE_MULT; dev_dbg(dev, "IDXD Perfmon Offset: %#x\n", idxd->perfmon_offset); } diff --git a/drivers/dma/idxd/registers.h b/drivers/dma/idxd/registers.h index 6f2f736097e5..d29a58ee2651 100644 --- a/drivers/dma/idxd/registers.h +++ b/drivers/dma/idxd/registers.h @@ -102,6 +102,8 @@ union offsets_reg { u64 bits[2]; } __packed; +#define IDXD_TABLE_MULT 0x100 + #define IDXD_GENCFG_OFFSET 0x80 union gencfg_reg { struct { -- cgit v1.2.3-59-g8ed1b From 842067940a3e3fc008a60fee388e000219b32632 Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Tue, 3 Nov 2020 20:39:38 +0200 Subject: dmaengine: dw: Enable runtime PM When consumer requests channel power on the DMA controller device and otherwise on the freeing channel resources. Note, in some cases consumer acquires channel at the ->probe() stage and releases it at the ->remove() stage. It will mean that DMA controller device will be powered during all this time if there is no assist from hardware to idle it. The above mentioned cases should be investigated separately and individually. Signed-off-by: Andy Shevchenko Acked-by: Viresh Kumar Link: https://lore.kernel.org/r/20201103183938.64752-1-andriy.shevchenko@linux.intel.com Signed-off-by: Vinod Koul --- drivers/dma/dw/core.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/drivers/dma/dw/core.c b/drivers/dma/dw/core.c index 7ab83fe601ed..19a23767533a 100644 --- a/drivers/dma/dw/core.c +++ b/drivers/dma/dw/core.c @@ -982,8 +982,11 @@ static int dwc_alloc_chan_resources(struct dma_chan *chan) dev_vdbg(chan2dev(chan), "%s\n", __func__); + pm_runtime_get_sync(dw->dma.dev); + /* ASSERT: channel is idle */ if (dma_readl(dw, CH_EN) & dwc->mask) { + pm_runtime_put_sync_suspend(dw->dma.dev); dev_dbg(chan2dev(chan), "DMA channel not idle?\n"); return -EIO; } @@ -1000,6 +1003,7 @@ static int dwc_alloc_chan_resources(struct dma_chan *chan) * We need controller-specific data to set up slave transfers. */ if (chan->private && !dw_dma_filter(chan, chan->private)) { + pm_runtime_put_sync_suspend(dw->dma.dev); dev_warn(chan2dev(chan), "Wrong controller-specific data\n"); return -EINVAL; } @@ -1043,6 +1047,8 @@ static void dwc_free_chan_resources(struct dma_chan *chan) if (!dw->in_use) do_dw_dma_off(dw); + pm_runtime_put_sync_suspend(dw->dma.dev); + dev_vdbg(chan2dev(chan), "%s: done\n", __func__); } -- cgit v1.2.3-59-g8ed1b From 6349753276a657b423460b93c2f511c73bb0a118 Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Wed, 4 Nov 2020 12:31:31 +0200 Subject: dmaengine: idma64: Switch to use __maybe_unused instead of ifdeffery ifdeffery is prone to errors and makes code harder to read. Switch to use __maybe_unused instead of ifdeffery. Signed-off-by: Andy Shevchenko Link: https://lore.kernel.org/r/20201104103131.89907-1-andriy.shevchenko@linux.intel.com Signed-off-by: Vinod Koul --- drivers/dma/idma64.c | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/drivers/dma/idma64.c b/drivers/dma/idma64.c index f5a84c846394..f4c07ad3be15 100644 --- a/drivers/dma/idma64.c +++ b/drivers/dma/idma64.c @@ -667,9 +667,7 @@ static int idma64_platform_remove(struct platform_device *pdev) return idma64_remove(chip); } -#ifdef CONFIG_PM_SLEEP - -static int idma64_pm_suspend(struct device *dev) +static int __maybe_unused idma64_pm_suspend(struct device *dev) { struct idma64_chip *chip = dev_get_drvdata(dev); @@ -677,7 +675,7 @@ static int idma64_pm_suspend(struct device *dev) return 0; } -static int idma64_pm_resume(struct device *dev) +static int __maybe_unused idma64_pm_resume(struct device *dev) { struct idma64_chip *chip = dev_get_drvdata(dev); @@ -685,8 +683,6 @@ static int idma64_pm_resume(struct device *dev) return 0; } -#endif /* CONFIG_PM_SLEEP */ - static const struct dev_pm_ops idma64_dev_pm_ops = { SET_SYSTEM_SLEEP_PM_OPS(idma64_pm_suspend, idma64_pm_resume) }; -- cgit v1.2.3-59-g8ed1b From 69973b4895b35a67409c8ce1c3d62090470ef47b Mon Sep 17 00:00:00 2001 From: Grygorii Strashko Date: Fri, 30 Oct 2020 22:30:00 +0200 Subject: dmaengine: ti: k3-udma-glue: move psi-l pairing in channel en/dis functions The NAVSS UDMA will stuck if target IP module is disabled by PM while PSI-L threads are paired UDMA<->IP and no further transfers is possible. This could be the case for IPs J721E Main CPSW (cpsw9g). Hence, to avoid such situation do PSI-L threads pairing only when UDMA channel is going to be enabled as at this time DMA consumer module expected to be active already. Signed-off-by: Grygorii Strashko Acked-by: Peter Ujfalusi Link: https://lore.kernel.org/r/20201030203000.4281-1-grygorii.strashko@ti.com Signed-off-by: Vinod Koul --- drivers/dma/ti/k3-udma-glue.c | 64 +++++++++++++++++++++++++------------------ 1 file changed, 38 insertions(+), 26 deletions(-) diff --git a/drivers/dma/ti/k3-udma-glue.c b/drivers/dma/ti/k3-udma-glue.c index a367584f0d7b..dfb65e382ab9 100644 --- a/drivers/dma/ti/k3-udma-glue.c +++ b/drivers/dma/ti/k3-udma-glue.c @@ -303,19 +303,6 @@ struct k3_udma_glue_tx_channel *k3_udma_glue_request_tx_chn(struct device *dev, goto err; } - ret = xudma_navss_psil_pair(tx_chn->common.udmax, - tx_chn->common.src_thread, - tx_chn->common.dst_thread); - if (ret) { - dev_err(dev, "PSI-L request err %d\n", ret); - goto err; - } - - tx_chn->psil_paired = true; - - /* reset TX RT registers */ - k3_udma_glue_disable_tx_chn(tx_chn); - k3_udma_glue_dump_tx_chn(tx_chn); return tx_chn; @@ -378,6 +365,18 @@ EXPORT_SYMBOL_GPL(k3_udma_glue_pop_tx_chn); int k3_udma_glue_enable_tx_chn(struct k3_udma_glue_tx_channel *tx_chn) { + int ret; + + ret = xudma_navss_psil_pair(tx_chn->common.udmax, + tx_chn->common.src_thread, + tx_chn->common.dst_thread); + if (ret) { + dev_err(tx_chn->common.dev, "PSI-L request err %d\n", ret); + return ret; + } + + tx_chn->psil_paired = true; + xudma_tchanrt_write(tx_chn->udma_tchanx, UDMA_CHAN_RT_PEER_RT_EN_REG, UDMA_PEER_RT_EN_ENABLE); @@ -398,6 +397,13 @@ void k3_udma_glue_disable_tx_chn(struct k3_udma_glue_tx_channel *tx_chn) xudma_tchanrt_write(tx_chn->udma_tchanx, UDMA_CHAN_RT_PEER_RT_EN_REG, 0); k3_udma_glue_dump_tx_rt_chn(tx_chn, "txchn dis2"); + + if (tx_chn->psil_paired) { + xudma_navss_psil_unpair(tx_chn->common.udmax, + tx_chn->common.src_thread, + tx_chn->common.dst_thread); + tx_chn->psil_paired = false; + } } EXPORT_SYMBOL_GPL(k3_udma_glue_disable_tx_chn); @@ -815,19 +821,6 @@ k3_udma_glue_request_rx_chn_priv(struct device *dev, const char *name, goto err; } - ret = xudma_navss_psil_pair(rx_chn->common.udmax, - rx_chn->common.src_thread, - rx_chn->common.dst_thread); - if (ret) { - dev_err(dev, "PSI-L request err %d\n", ret); - goto err; - } - - rx_chn->psil_paired = true; - - /* reset RX RT registers */ - k3_udma_glue_disable_rx_chn(rx_chn); - k3_udma_glue_dump_rx_chn(rx_chn); return rx_chn; @@ -1052,12 +1045,24 @@ EXPORT_SYMBOL_GPL(k3_udma_glue_rx_flow_disable); int k3_udma_glue_enable_rx_chn(struct k3_udma_glue_rx_channel *rx_chn) { + int ret; + if (rx_chn->remote) return -EINVAL; if (rx_chn->flows_ready < rx_chn->flow_num) return -EINVAL; + ret = xudma_navss_psil_pair(rx_chn->common.udmax, + rx_chn->common.src_thread, + rx_chn->common.dst_thread); + if (ret) { + dev_err(rx_chn->common.dev, "PSI-L request err %d\n", ret); + return ret; + } + + rx_chn->psil_paired = true; + xudma_rchanrt_write(rx_chn->udma_rchanx, UDMA_CHAN_RT_CTL_REG, UDMA_CHAN_RT_CTL_EN); @@ -1078,6 +1083,13 @@ void k3_udma_glue_disable_rx_chn(struct k3_udma_glue_rx_channel *rx_chn) xudma_rchanrt_write(rx_chn->udma_rchanx, UDMA_CHAN_RT_CTL_REG, 0); k3_udma_glue_dump_rx_rt_chn(rx_chn, "rxrt dis2"); + + if (rx_chn->psil_paired) { + xudma_navss_psil_unpair(rx_chn->common.udmax, + rx_chn->common.src_thread, + rx_chn->common.dst_thread); + rx_chn->psil_paired = false; + } } EXPORT_SYMBOL_GPL(k3_udma_glue_disable_rx_chn); -- cgit v1.2.3-59-g8ed1b From f3b1024908ec02d7af4d01a2659901828844bdd2 Mon Sep 17 00:00:00 2001 From: Barry Song Date: Wed, 28 Oct 2020 10:52:43 +1300 Subject: dmaengine: ipu_idmac: remove redundant irqsave and restore in hardIRQ Running in hardIRQ, disabling IRQ is redundant since hardIRQ has disabled IRQ. This patch removes the irqsave and irqstore to save some instruction cycles. Signed-off-by: Barry Song Link: https://lore.kernel.org/r/20201027215252.25820-2-song.bao.hua@hisilicon.com Signed-off-by: Vinod Koul --- drivers/dma/ipu/ipu_idmac.c | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/drivers/dma/ipu/ipu_idmac.c b/drivers/dma/ipu/ipu_idmac.c index 38036db284cb..104ad420abbe 100644 --- a/drivers/dma/ipu/ipu_idmac.c +++ b/drivers/dma/ipu/ipu_idmac.c @@ -1160,14 +1160,13 @@ static irqreturn_t idmac_interrupt(int irq, void *dev_id) struct idmac_tx_desc *desc, *descnew; bool done = false; u32 ready0, ready1, curbuf, err; - unsigned long flags; struct dmaengine_desc_callback cb; /* IDMAC has cleared the respective BUFx_RDY bit, we manage the buffer */ dev_dbg(dev, "IDMAC irq %d, buf %d\n", irq, ichan->active_buffer); - spin_lock_irqsave(&ipu_data.lock, flags); + spin_lock(&ipu_data.lock); ready0 = idmac_read_ipureg(&ipu_data, IPU_CHA_BUF0_RDY); ready1 = idmac_read_ipureg(&ipu_data, IPU_CHA_BUF1_RDY); @@ -1176,7 +1175,7 @@ static irqreturn_t idmac_interrupt(int irq, void *dev_id) if (err & (1 << chan_id)) { idmac_write_ipureg(&ipu_data, 1 << chan_id, IPU_INT_STAT_4); - spin_unlock_irqrestore(&ipu_data.lock, flags); + spin_unlock(&ipu_data.lock); /* * Doing this * ichan->sg[0] = ichan->sg[1] = NULL; @@ -1188,7 +1187,7 @@ static irqreturn_t idmac_interrupt(int irq, void *dev_id) chan_id, ready0, ready1, curbuf); return IRQ_HANDLED; } - spin_unlock_irqrestore(&ipu_data.lock, flags); + spin_unlock(&ipu_data.lock); /* Other interrupts do not interfere with this channel */ spin_lock(&ichan->lock); @@ -1251,9 +1250,9 @@ static irqreturn_t idmac_interrupt(int irq, void *dev_id) if (unlikely(sgnew)) { ipu_submit_buffer(ichan, descnew, sgnew, !ichan->active_buffer); } else { - spin_lock_irqsave(&ipu_data.lock, flags); + spin_lock(&ipu_data.lock); ipu_ic_disable_task(&ipu_data, chan_id); - spin_unlock_irqrestore(&ipu_data.lock, flags); + spin_unlock(&ipu_data.lock); ichan->status = IPU_CHANNEL_READY; /* Continue to check for complete descriptor */ } -- cgit v1.2.3-59-g8ed1b From e991c06ed714c64d3d79696513370f894a4ab751 Mon Sep 17 00:00:00 2001 From: Barry Song Date: Wed, 28 Oct 2020 10:52:44 +1300 Subject: dmaengine: ti: k3-udma: remove redundant irqsave and irqrestore in hardIRQ Running in hardIRQ, disabling IRQ is redundant since hardIRQ has disabled IRQ. This patch removes the irqsave and irqstore to save some instruction cycles. Signed-off-by: Barry Song Acked-by: Peter Ujfalusi Link: https://lore.kernel.org/r/20201027215252.25820-3-song.bao.hua@hisilicon.com Signed-off-by: Vinod Koul --- drivers/dma/ti/k3-udma.c | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/drivers/dma/ti/k3-udma.c b/drivers/dma/ti/k3-udma.c index 82cf6c77f5c9..e508280b3d70 100644 --- a/drivers/dma/ti/k3-udma.c +++ b/drivers/dma/ti/k3-udma.c @@ -1020,13 +1020,12 @@ static irqreturn_t udma_ring_irq_handler(int irq, void *data) { struct udma_chan *uc = data; struct udma_desc *d; - unsigned long flags; dma_addr_t paddr = 0; if (udma_pop_from_ring(uc, &paddr) || !paddr) return IRQ_HANDLED; - spin_lock_irqsave(&uc->vc.lock, flags); + spin_lock(&uc->vc.lock); /* Teardown completion message */ if (cppi5_desc_is_tdcm(paddr)) { @@ -1077,7 +1076,7 @@ static irqreturn_t udma_ring_irq_handler(int irq, void *data) } } out: - spin_unlock_irqrestore(&uc->vc.lock, flags); + spin_unlock(&uc->vc.lock); return IRQ_HANDLED; } @@ -1086,9 +1085,8 @@ static irqreturn_t udma_udma_irq_handler(int irq, void *data) { struct udma_chan *uc = data; struct udma_desc *d; - unsigned long flags; - spin_lock_irqsave(&uc->vc.lock, flags); + spin_lock(&uc->vc.lock); d = uc->desc; if (d) { d->tr_idx = (d->tr_idx + 1) % d->sglen; @@ -1103,7 +1101,7 @@ static irqreturn_t udma_udma_irq_handler(int irq, void *data) } } - spin_unlock_irqrestore(&uc->vc.lock, flags); + spin_unlock(&uc->vc.lock); return IRQ_HANDLED; } -- cgit v1.2.3-59-g8ed1b From 302b3b38236a70bffb13f3a90a52bd630841aa90 Mon Sep 17 00:00:00 2001 From: Barry Song Date: Wed, 28 Oct 2020 10:52:45 +1300 Subject: dmaengine: sf-pdma: remove redundant irqsave and irqrestore in hardIRQ Running in hardIRQ, disabling IRQ is redundant since hardIRQ has disabled IRQ. This patch removes the irqsave and irqstore to save some instruction cycles. Signed-off-by: Barry Song Cc: Green Wan Link: https://lore.kernel.org/r/20201027215252.25820-4-song.bao.hua@hisilicon.com Signed-off-by: Vinod Koul --- drivers/dma/sf-pdma/sf-pdma.c | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/drivers/dma/sf-pdma/sf-pdma.c b/drivers/dma/sf-pdma/sf-pdma.c index 528deb5d9f31..981ddcc42d70 100644 --- a/drivers/dma/sf-pdma/sf-pdma.c +++ b/drivers/dma/sf-pdma/sf-pdma.c @@ -326,10 +326,9 @@ static irqreturn_t sf_pdma_done_isr(int irq, void *dev_id) { struct sf_pdma_chan *chan = dev_id; struct pdma_regs *regs = &chan->regs; - unsigned long flags; u64 residue; - spin_lock_irqsave(&chan->vchan.lock, flags); + spin_lock(&chan->vchan.lock); writel((readl(regs->ctrl)) & ~PDMA_DONE_STATUS_MASK, regs->ctrl); residue = readq(regs->residue); @@ -346,7 +345,7 @@ static irqreturn_t sf_pdma_done_isr(int irq, void *dev_id) sf_pdma_xfer_desc(chan); } - spin_unlock_irqrestore(&chan->vchan.lock, flags); + spin_unlock(&chan->vchan.lock); return IRQ_HANDLED; } @@ -355,11 +354,10 @@ static irqreturn_t sf_pdma_err_isr(int irq, void *dev_id) { struct sf_pdma_chan *chan = dev_id; struct pdma_regs *regs = &chan->regs; - unsigned long flags; - spin_lock_irqsave(&chan->lock, flags); + spin_lock(&chan->lock); writel((readl(regs->ctrl)) & ~PDMA_ERR_STATUS_MASK, regs->ctrl); - spin_unlock_irqrestore(&chan->lock, flags); + spin_unlock(&chan->lock); tasklet_schedule(&chan->err_tasklet); -- cgit v1.2.3-59-g8ed1b From 654115e3f62671a6b8bcee702697dbdfc6fce1a3 Mon Sep 17 00:00:00 2001 From: Barry Song Date: Wed, 28 Oct 2020 10:52:46 +1300 Subject: dmaengine: tegra210-adma: remove redundant irqsave and irqrestore in hardIRQ Running in hardIRQ, disabling IRQ is redundant since hardIRQ has disabled IRQ. This patch removes the irqsave and irqstore to save some instruction cycles. Signed-off-by: Barry Song Acked-by: Jon Hunter Cc: Laxman Dewangan Cc: Thierry Reding Link: https://lore.kernel.org/r/20201027215252.25820-5-song.bao.hua@hisilicon.com Signed-off-by: Vinod Koul --- drivers/dma/tegra210-adma.c | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/drivers/dma/tegra210-adma.c b/drivers/dma/tegra210-adma.c index c5fa2ef74abc..4735742e826d 100644 --- a/drivers/dma/tegra210-adma.c +++ b/drivers/dma/tegra210-adma.c @@ -408,19 +408,18 @@ static irqreturn_t tegra_adma_isr(int irq, void *dev_id) { struct tegra_adma_chan *tdc = dev_id; unsigned long status; - unsigned long flags; - spin_lock_irqsave(&tdc->vc.lock, flags); + spin_lock(&tdc->vc.lock); status = tegra_adma_irq_clear(tdc); if (status == 0 || !tdc->desc) { - spin_unlock_irqrestore(&tdc->vc.lock, flags); + spin_unlock(&tdc->vc.lock); return IRQ_NONE; } vchan_cyclic_callback(&tdc->desc->vd); - spin_unlock_irqrestore(&tdc->vc.lock, flags); + spin_unlock(&tdc->vc.lock); return IRQ_HANDLED; } -- cgit v1.2.3-59-g8ed1b From 280e7f90d4520682d0a18b33fa43fd8cccab3439 Mon Sep 17 00:00:00 2001 From: Barry Song Date: Wed, 28 Oct 2020 10:52:47 +1300 Subject: dmaengine: milbeaut-xdmac: remove redundant irqsave and irqrestore in hardIRQ Running in hardIRQ, disabling IRQ is redundant since hardIRQ has disabled IRQ. This patch removes the irqsave and irqstore to save some instruction cycles. Signed-off-by: Barry Song Link: https://lore.kernel.org/r/20201027215252.25820-6-song.bao.hua@hisilicon.com Signed-off-by: Vinod Koul --- drivers/dma/milbeaut-xdmac.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/drivers/dma/milbeaut-xdmac.c b/drivers/dma/milbeaut-xdmac.c index 85a597228fb0..584c931e807a 100644 --- a/drivers/dma/milbeaut-xdmac.c +++ b/drivers/dma/milbeaut-xdmac.c @@ -160,10 +160,9 @@ static irqreturn_t milbeaut_xdmac_interrupt(int irq, void *dev_id) { struct milbeaut_xdmac_chan *mc = dev_id; struct milbeaut_xdmac_desc *md; - unsigned long flags; u32 val; - spin_lock_irqsave(&mc->vc.lock, flags); + spin_lock(&mc->vc.lock); /* Ack and Stop */ val = FIELD_PREP(M10V_XDDSD_IS_MASK, 0x0); @@ -177,7 +176,7 @@ static irqreturn_t milbeaut_xdmac_interrupt(int irq, void *dev_id) milbeaut_xdmac_start(mc); out: - spin_unlock_irqrestore(&mc->vc.lock, flags); + spin_unlock(&mc->vc.lock); return IRQ_HANDLED; } -- cgit v1.2.3-59-g8ed1b From 1ff206561920c9998058e52ea465347d4944b635 Mon Sep 17 00:00:00 2001 From: Barry Song Date: Wed, 28 Oct 2020 10:52:48 +1300 Subject: dmaengine: k3dma: remove redundant irqsave and irqrestore in hardIRQ Running in hardIRQ, disabling IRQ is redundant since hardIRQ has disabled IRQ. This patch removes the irqsave and irqstore to save some instruction cycles. Signed-off-by: Barry Song Link: https://lore.kernel.org/r/20201027215252.25820-7-song.bao.hua@hisilicon.com Signed-off-by: Vinod Koul --- drivers/dma/k3dma.c | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/drivers/dma/k3dma.c b/drivers/dma/k3dma.c index f609a84c493c..d0b2e601e3e5 100644 --- a/drivers/dma/k3dma.c +++ b/drivers/dma/k3dma.c @@ -223,24 +223,23 @@ static irqreturn_t k3_dma_int_handler(int irq, void *dev_id) i = __ffs(stat); stat &= ~BIT(i); if (likely(tc1 & BIT(i)) || (tc2 & BIT(i))) { - unsigned long flags; p = &d->phy[i]; c = p->vchan; if (c && (tc1 & BIT(i))) { - spin_lock_irqsave(&c->vc.lock, flags); + spin_lock(&c->vc.lock); if (p->ds_run != NULL) { vchan_cookie_complete(&p->ds_run->vd); p->ds_done = p->ds_run; p->ds_run = NULL; } - spin_unlock_irqrestore(&c->vc.lock, flags); + spin_unlock(&c->vc.lock); } if (c && (tc2 & BIT(i))) { - spin_lock_irqsave(&c->vc.lock, flags); + spin_lock(&c->vc.lock); if (p->ds_run != NULL) vchan_cyclic_callback(&p->ds_run->vd); - spin_unlock_irqrestore(&c->vc.lock, flags); + spin_unlock(&c->vc.lock); } irq_chan |= BIT(i); } -- cgit v1.2.3-59-g8ed1b From d9c8d4b278d167bf7cba83ec6fc15a0a17dfc407 Mon Sep 17 00:00:00 2001 From: Barry Song Date: Wed, 28 Oct 2020 10:52:49 +1300 Subject: dmaengine: hisi_dma: remove redundant irqsave and irqrestore in hardIRQ Running in hardIRQ, disabling IRQ is redundant since hardIRQ has disabled IRQ. This patch removes the irqsave and irqstore to save some instruction cycles. Signed-off-by: Barry Song Acked-by: Zhou Wang Link: https://lore.kernel.org/r/20201027215252.25820-8-song.bao.hua@hisilicon.com Signed-off-by: Vinod Koul --- drivers/dma/hisi_dma.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/drivers/dma/hisi_dma.c b/drivers/dma/hisi_dma.c index e1a958ae7925..a259ee010e9b 100644 --- a/drivers/dma/hisi_dma.c +++ b/drivers/dma/hisi_dma.c @@ -431,9 +431,8 @@ static irqreturn_t hisi_dma_irq(int irq, void *data) struct hisi_dma_dev *hdma_dev = chan->hdma_dev; struct hisi_dma_desc *desc; struct hisi_dma_cqe *cqe; - unsigned long flags; - spin_lock_irqsave(&chan->vc.lock, flags); + spin_lock(&chan->vc.lock); desc = chan->desc; cqe = chan->cq + chan->cq_head; @@ -452,7 +451,7 @@ static irqreturn_t hisi_dma_irq(int irq, void *data) chan->desc = NULL; } - spin_unlock_irqrestore(&chan->vc.lock, flags); + spin_unlock(&chan->vc.lock); return IRQ_HANDLED; } -- cgit v1.2.3-59-g8ed1b From 8c94b83e0c370e72c131f5da8cb19c2cb858a1bf Mon Sep 17 00:00:00 2001 From: Barry Song Date: Wed, 28 Oct 2020 10:52:50 +1300 Subject: dmaengine: moxart-dma: remove redundant irqsave and irqrestore in hardIRQ Running in hardIRQ, disabling IRQ is redundant since hardIRQ has disabled IRQ. This patch removes the irqsave and irqstore to save some instruction cycles. Signed-off-by: Barry Song Link: https://lore.kernel.org/r/20201027215252.25820-9-song.bao.hua@hisilicon.com Signed-off-by: Vinod Koul --- drivers/dma/moxart-dma.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/drivers/dma/moxart-dma.c b/drivers/dma/moxart-dma.c index 347146a6e1d0..74755093e14b 100644 --- a/drivers/dma/moxart-dma.c +++ b/drivers/dma/moxart-dma.c @@ -524,7 +524,6 @@ static irqreturn_t moxart_dma_interrupt(int irq, void *devid) struct moxart_dmadev *mc = devid; struct moxart_chan *ch = &mc->slave_chans[0]; unsigned int i; - unsigned long flags; u32 ctrl; dev_dbg(chan2dev(&ch->vc.chan), "%s\n", __func__); @@ -541,14 +540,14 @@ static irqreturn_t moxart_dma_interrupt(int irq, void *devid) if (ctrl & APB_DMA_FIN_INT_STS) { ctrl &= ~APB_DMA_FIN_INT_STS; if (ch->desc) { - spin_lock_irqsave(&ch->vc.lock, flags); + spin_lock(&ch->vc.lock); if (++ch->sgidx < ch->desc->sglen) { moxart_dma_start_sg(ch, ch->sgidx); } else { vchan_cookie_complete(&ch->desc->vd); moxart_dma_start_desc(&ch->vc.chan); } - spin_unlock_irqrestore(&ch->vc.lock, flags); + spin_unlock(&ch->vc.lock); } } -- cgit v1.2.3-59-g8ed1b From 618a8e383bbdf9ce5a252797046dda694bf40389 Mon Sep 17 00:00:00 2001 From: Barry Song Date: Wed, 28 Oct 2020 10:52:51 +1300 Subject: dmaengine: ste_dma40: remove redundant irqsave and irqrestore in hardIRQ Running in hardIRQ, disabling IRQ is redundant since hardIRQ has disabled IRQ. This patch removes the irqsave and irqstore to save some instruction cycles. Signed-off-by: Barry Song Reviewed-by: Linus Walleij Cc: Linus Walleij Link: https://lore.kernel.org/r/20201027215252.25820-10-song.bao.hua@hisilicon.com Signed-off-by: Vinod Koul --- drivers/dma/ste_dma40.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/drivers/dma/ste_dma40.c b/drivers/dma/ste_dma40.c index 77ab1f4730be..4256e55bbf25 100644 --- a/drivers/dma/ste_dma40.c +++ b/drivers/dma/ste_dma40.c @@ -1643,13 +1643,12 @@ static irqreturn_t d40_handle_interrupt(int irq, void *data) u32 row; long chan = -1; struct d40_chan *d40c; - unsigned long flags; struct d40_base *base = data; u32 *regs = base->regs_interrupt; struct d40_interrupt_lookup *il = base->gen_dmac.il; u32 il_size = base->gen_dmac.il_size; - spin_lock_irqsave(&base->interrupt_lock, flags); + spin_lock(&base->interrupt_lock); /* Read interrupt status of both logical and physical channels */ for (i = 0; i < il_size; i++) @@ -1694,7 +1693,7 @@ static irqreturn_t d40_handle_interrupt(int irq, void *data) spin_unlock(&d40c->lock); } - spin_unlock_irqrestore(&base->interrupt_lock, flags); + spin_unlock(&base->interrupt_lock); return IRQ_HANDLED; } -- cgit v1.2.3-59-g8ed1b From 0e15ca5fe2240e9f1e4c408a29505b60dbdfaddd Mon Sep 17 00:00:00 2001 From: Barry Song Date: Wed, 28 Oct 2020 10:52:52 +1300 Subject: dmaengine: pxa_dma: remove redundant irqsave and irqrestore in hardIRQ Running in hardIRQ, disabling IRQ is redundant since hardIRQ has disabled IRQ. This patch removes the irqsave and irqstore to save some instruction cycles. Signed-off-by: Barry Song Acked-by: Robert Jarzmik Cc: Daniel Mack Cc: Haojian Zhuang Link: https://lore.kernel.org/r/20201027215252.25820-11-song.bao.hua@hisilicon.com Signed-off-by: Vinod Koul --- drivers/dma/pxa_dma.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/drivers/dma/pxa_dma.c b/drivers/dma/pxa_dma.c index 349fb312c872..4a2a796e348c 100644 --- a/drivers/dma/pxa_dma.c +++ b/drivers/dma/pxa_dma.c @@ -606,7 +606,6 @@ static irqreturn_t pxad_chan_handler(int irq, void *dev_id) struct pxad_chan *chan = phy->vchan; struct virt_dma_desc *vd, *tmp; unsigned int dcsr; - unsigned long flags; bool vd_completed; dma_cookie_t last_started = 0; @@ -616,7 +615,7 @@ static irqreturn_t pxad_chan_handler(int irq, void *dev_id) if (dcsr & PXA_DCSR_RUN) return IRQ_NONE; - spin_lock_irqsave(&chan->vc.lock, flags); + spin_lock(&chan->vc.lock); list_for_each_entry_safe(vd, tmp, &chan->vc.desc_issued, node) { vd_completed = is_desc_completed(vd); dev_dbg(&chan->vc.chan.dev->device, @@ -658,7 +657,7 @@ static irqreturn_t pxad_chan_handler(int irq, void *dev_id) pxad_launch_chan(chan, to_pxad_sw_desc(vd)); } } - spin_unlock_irqrestore(&chan->vc.lock, flags); + spin_unlock(&chan->vc.lock); wake_up(&chan->wq_state); return IRQ_HANDLED; -- cgit v1.2.3-59-g8ed1b From 5b7b41cbf2f9b473ccd13f69337d7c26f4d138c2 Mon Sep 17 00:00:00 2001 From: "周琰杰 (Zhou Yanjie)" Date: Sat, 7 Nov 2020 20:20:15 +0800 Subject: dt-bindings: dmaengine: Add JZ4775 bindings. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add the dmaengine bindings for the JZ4775 SoC from Ingenic. Signed-off-by: 周琰杰 (Zhou Yanjie) Acked-by: Rob Herring Link: https://lore.kernel.org/r/20201107122016.89859-2-zhouyanjie@wanyeetech.com Signed-off-by: Vinod Koul --- include/dt-bindings/dma/jz4775-dma.h | 44 ++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 include/dt-bindings/dma/jz4775-dma.h diff --git a/include/dt-bindings/dma/jz4775-dma.h b/include/dt-bindings/dma/jz4775-dma.h new file mode 100644 index 000000000000..8d27e2c69dca --- /dev/null +++ b/include/dt-bindings/dma/jz4775-dma.h @@ -0,0 +1,44 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * This header provides macros for JZ4775 DMA bindings. + * + * Copyright (c) 2020 周琰杰 (Zhou Yanjie) + */ + +#ifndef __DT_BINDINGS_DMA_JZ4775_DMA_H__ +#define __DT_BINDINGS_DMA_JZ4775_DMA_H__ + +/* + * Request type numbers for the JZ4775 DMA controller (written to the DRTn + * register for the channel). + */ +#define JZ4775_DMA_I2S0_TX 0x6 +#define JZ4775_DMA_I2S0_RX 0x7 +#define JZ4775_DMA_AUTO 0x8 +#define JZ4775_DMA_SADC_RX 0x9 +#define JZ4775_DMA_UART3_TX 0x0e +#define JZ4775_DMA_UART3_RX 0x0f +#define JZ4775_DMA_UART2_TX 0x10 +#define JZ4775_DMA_UART2_RX 0x11 +#define JZ4775_DMA_UART1_TX 0x12 +#define JZ4775_DMA_UART1_RX 0x13 +#define JZ4775_DMA_UART0_TX 0x14 +#define JZ4775_DMA_UART0_RX 0x15 +#define JZ4775_DMA_SSI0_TX 0x16 +#define JZ4775_DMA_SSI0_RX 0x17 +#define JZ4775_DMA_MSC0_TX 0x1a +#define JZ4775_DMA_MSC0_RX 0x1b +#define JZ4775_DMA_MSC1_TX 0x1c +#define JZ4775_DMA_MSC1_RX 0x1d +#define JZ4775_DMA_MSC2_TX 0x1e +#define JZ4775_DMA_MSC2_RX 0x1f +#define JZ4775_DMA_PCM0_TX 0x20 +#define JZ4775_DMA_PCM0_RX 0x21 +#define JZ4775_DMA_SMB0_TX 0x24 +#define JZ4775_DMA_SMB0_RX 0x25 +#define JZ4775_DMA_SMB1_TX 0x26 +#define JZ4775_DMA_SMB1_RX 0x27 +#define JZ4775_DMA_SMB2_TX 0x28 +#define JZ4775_DMA_SMB2_RX 0x29 + +#endif /* __DT_BINDINGS_DMA_JZ4775_DMA_H__ */ -- cgit v1.2.3-59-g8ed1b From 46d613fd8da84d41fb2adb2ccb97230979d38af8 Mon Sep 17 00:00:00 2001 From: "周琰杰 (Zhou Yanjie)" Date: Sat, 7 Nov 2020 20:20:16 +0800 Subject: dt-bindings: dmaengine: Add X2000 bindings. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add the dmaengine bindings for the X2000 SoC from Ingenic. Signed-off-by: 周琰杰 (Zhou Yanjie) Acked-by: Rob Herring Link: https://lore.kernel.org/r/20201107122016.89859-3-zhouyanjie@wanyeetech.com Signed-off-by: Vinod Koul --- include/dt-bindings/dma/x2000-dma.h | 54 +++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 include/dt-bindings/dma/x2000-dma.h diff --git a/include/dt-bindings/dma/x2000-dma.h b/include/dt-bindings/dma/x2000-dma.h new file mode 100644 index 000000000000..db2cd4830b00 --- /dev/null +++ b/include/dt-bindings/dma/x2000-dma.h @@ -0,0 +1,54 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * This header provides macros for X2000 DMA bindings. + * + * Copyright (c) 2020 周琰杰 (Zhou Yanjie) + */ + +#ifndef __DT_BINDINGS_DMA_X2000_DMA_H__ +#define __DT_BINDINGS_DMA_X2000_DMA_H__ + +/* + * Request type numbers for the X2000 DMA controller (written to the DRTn + * register for the channel). + */ +#define X2000_DMA_AUTO 0x8 +#define X2000_DMA_UART5_TX 0xa +#define X2000_DMA_UART5_RX 0xb +#define X2000_DMA_UART4_TX 0xc +#define X2000_DMA_UART4_RX 0xd +#define X2000_DMA_UART3_TX 0xe +#define X2000_DMA_UART3_RX 0xf +#define X2000_DMA_UART2_TX 0x10 +#define X2000_DMA_UART2_RX 0x11 +#define X2000_DMA_UART1_TX 0x12 +#define X2000_DMA_UART1_RX 0x13 +#define X2000_DMA_UART0_TX 0x14 +#define X2000_DMA_UART0_RX 0x15 +#define X2000_DMA_SSI0_TX 0x16 +#define X2000_DMA_SSI0_RX 0x17 +#define X2000_DMA_SSI1_TX 0x18 +#define X2000_DMA_SSI1_RX 0x19 +#define X2000_DMA_I2C0_TX 0x24 +#define X2000_DMA_I2C0_RX 0x25 +#define X2000_DMA_I2C1_TX 0x26 +#define X2000_DMA_I2C1_RX 0x27 +#define X2000_DMA_I2C2_TX 0x28 +#define X2000_DMA_I2C2_RX 0x29 +#define X2000_DMA_I2C3_TX 0x2a +#define X2000_DMA_I2C3_RX 0x2b +#define X2000_DMA_I2C4_TX 0x2c +#define X2000_DMA_I2C4_RX 0x2d +#define X2000_DMA_I2C5_TX 0x2e +#define X2000_DMA_I2C5_RX 0x2f +#define X2000_DMA_UART6_TX 0x30 +#define X2000_DMA_UART6_RX 0x31 +#define X2000_DMA_UART7_TX 0x32 +#define X2000_DMA_UART7_RX 0x33 +#define X2000_DMA_UART8_TX 0x34 +#define X2000_DMA_UART8_RX 0x35 +#define X2000_DMA_UART9_TX 0x36 +#define X2000_DMA_UART9_RX 0x37 +#define X2000_DMA_SADC_RX 0x38 + +#endif /* __DT_BINDINGS_DMA_X2000_DMA_H__ */ -- cgit v1.2.3-59-g8ed1b From 613ff7e19c5877d62118cb6612d4e336272620e7 Mon Sep 17 00:00:00 2001 From: Yangtao Li Date: Tue, 10 Nov 2020 14:26:38 +0800 Subject: dt-bindings: dma: allwinner,sun50i-a64-dma: Add A100 compatible Add a binding for A100's dma controller. Signed-off-by: Yangtao Li Acked-by: Rob Herring Link: https://lore.kernel.org/r/f15a18e9b8868e8853db1b5a3d1e411b0ac1c63a.1604988979.git.frank@allwinnertech.com Signed-off-by: Vinod Koul --- Documentation/devicetree/bindings/dma/allwinner,sun50i-a64-dma.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Documentation/devicetree/bindings/dma/allwinner,sun50i-a64-dma.yaml b/Documentation/devicetree/bindings/dma/allwinner,sun50i-a64-dma.yaml index 372679dbd216..b6e1ebfaf366 100644 --- a/Documentation/devicetree/bindings/dma/allwinner,sun50i-a64-dma.yaml +++ b/Documentation/devicetree/bindings/dma/allwinner,sun50i-a64-dma.yaml @@ -21,6 +21,7 @@ properties: compatible: oneOf: - const: allwinner,sun50i-a64-dma + - const: allwinner,sun50i-a100-dma - const: allwinner,sun50i-h6-dma - items: - const: allwinner,sun8i-r40-dma @@ -56,7 +57,9 @@ required: if: properties: compatible: - const: allwinner,sun50i-h6-dma + enum: + - allwinner,sun50i-a100-dma + - allwinner,sun50i-h6-dma then: properties: -- cgit v1.2.3-59-g8ed1b From 07b552732edd1b09aecc0f57d479e5eccf11c295 Mon Sep 17 00:00:00 2001 From: Yangtao Li Date: Tue, 10 Nov 2020 14:28:02 +0800 Subject: dmaengine: sun6i: Add support for A100 DMA The dma of a100 is similar to h6, with some minor changes to support greater addressing capabilities. Add support for it. Signed-off-by: Yangtao Li Link: https://lore.kernel.org/r/719852c6a9a597bd2e82d01a268ca02b9dee826c.1604988979.git.frank@allwinnertech.com Signed-off-by: Vinod Koul --- drivers/dma/sun6i-dma.c | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/drivers/dma/sun6i-dma.c b/drivers/dma/sun6i-dma.c index f5f9c86c50bc..5cadd4d2b824 100644 --- a/drivers/dma/sun6i-dma.c +++ b/drivers/dma/sun6i-dma.c @@ -1173,6 +1173,30 @@ static struct sun6i_dma_config sun50i_a64_dma_cfg = { BIT(DMA_SLAVE_BUSWIDTH_8_BYTES), }; +/* + * TODO: Add support for more than 4g physical addressing. + * + * The A100 binding uses the number of dma channels from the + * device tree node. + */ +static struct sun6i_dma_config sun50i_a100_dma_cfg = { + .clock_autogate_enable = sun6i_enable_clock_autogate_h3, + .set_burst_length = sun6i_set_burst_length_h3, + .set_drq = sun6i_set_drq_h6, + .set_mode = sun6i_set_mode_h6, + .src_burst_lengths = BIT(1) | BIT(4) | BIT(8) | BIT(16), + .dst_burst_lengths = BIT(1) | BIT(4) | BIT(8) | BIT(16), + .src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | + BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | + BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) | + BIT(DMA_SLAVE_BUSWIDTH_8_BYTES), + .dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | + BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | + BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) | + BIT(DMA_SLAVE_BUSWIDTH_8_BYTES), + .has_mbus_clk = true, +}; + /* * The H6 binding uses the number of dma channels from the * device tree node. @@ -1225,6 +1249,7 @@ static const struct of_device_id sun6i_dma_match[] = { { .compatible = "allwinner,sun8i-h3-dma", .data = &sun8i_h3_dma_cfg }, { .compatible = "allwinner,sun8i-v3s-dma", .data = &sun8i_v3s_dma_cfg }, { .compatible = "allwinner,sun50i-a64-dma", .data = &sun50i_a64_dma_cfg }, + { .compatible = "allwinner,sun50i-a100-dma", .data = &sun50i_a100_dma_cfg }, { .compatible = "allwinner,sun50i-h6-dma", .data = &sun50i_h6_dma_cfg }, { /* sentinel */ } }; -- cgit v1.2.3-59-g8ed1b From f74faa0ca3d56df7d135602bca80f6e39be9f4ad Mon Sep 17 00:00:00 2001 From: Fabio Estevam Date: Mon, 16 Nov 2020 17:24:03 -0300 Subject: dmaengine: imx-sdma: Remove unused .id_table support Since 5.10-rc1 i.MX is a devicetree-only platform and the existing .id_table support in this driver was only useful for old non-devicetree platforms. Signed-off-by: Fabio Estevam Link: https://lore.kernel.org/r/20201116202403.29749-1-festevam@gmail.com Signed-off-by: Vinod Koul --- drivers/dma/imx-sdma.c | 38 +------------------------------------- 1 file changed, 1 insertion(+), 37 deletions(-) diff --git a/drivers/dma/imx-sdma.c b/drivers/dma/imx-sdma.c index 16b908c77db3..41ba21eea7c8 100644 --- a/drivers/dma/imx-sdma.c +++ b/drivers/dma/imx-sdma.c @@ -566,37 +566,6 @@ static struct sdma_driver_data sdma_imx8mq = { .check_ratio = 1, }; -static const struct platform_device_id sdma_devtypes[] = { - { - .name = "imx25-sdma", - .driver_data = (unsigned long)&sdma_imx25, - }, { - .name = "imx31-sdma", - .driver_data = (unsigned long)&sdma_imx31, - }, { - .name = "imx35-sdma", - .driver_data = (unsigned long)&sdma_imx35, - }, { - .name = "imx51-sdma", - .driver_data = (unsigned long)&sdma_imx51, - }, { - .name = "imx53-sdma", - .driver_data = (unsigned long)&sdma_imx53, - }, { - .name = "imx6q-sdma", - .driver_data = (unsigned long)&sdma_imx6q, - }, { - .name = "imx7d-sdma", - .driver_data = (unsigned long)&sdma_imx7d, - }, { - .name = "imx8mq-sdma", - .driver_data = (unsigned long)&sdma_imx8mq, - }, { - /* sentinel */ - } -}; -MODULE_DEVICE_TABLE(platform, sdma_devtypes); - static const struct of_device_id sdma_dt_ids[] = { { .compatible = "fsl,imx6q-sdma", .data = &sdma_imx6q, }, { .compatible = "fsl,imx53-sdma", .data = &sdma_imx53, }, @@ -1998,11 +1967,7 @@ static int sdma_probe(struct platform_device *pdev) s32 *saddr_arr; const struct sdma_driver_data *drvdata = NULL; - if (of_id) - drvdata = of_id->data; - else if (pdev->id_entry) - drvdata = (void *)pdev->id_entry->driver_data; - + drvdata = of_id->data; if (!drvdata) { dev_err(&pdev->dev, "unable to find driver data\n"); return -EINVAL; @@ -2211,7 +2176,6 @@ static struct platform_driver sdma_driver = { .name = "imx-sdma", .of_match_table = sdma_dt_ids, }, - .id_table = sdma_devtypes, .remove = sdma_remove, .probe = sdma_probe, }; -- cgit v1.2.3-59-g8ed1b From 5c9f8c2dbdbe53818bcde6aa6695e1331e5f841f Mon Sep 17 00:00:00 2001 From: Jonathan McDowell Date: Sat, 14 Nov 2020 14:02:33 +0000 Subject: dmaengine: qcom: Add ADM driver Add the DMA engine driver for the QCOM Application Data Mover (ADM) DMA controller found in the MSM8x60 and IPQ/APQ8064 platforms. The ADM supports both memory to memory transactions and memory to/from peripheral device transactions. The controller also provides flow control capabilities for transactions to/from peripheral devices. The initial release of this driver supports slave transfers to/from peripherals and also incorporates CRCI (client rate control interface) flow control. The hardware only supports a 32 bit physical address, so specifying !PHYS_ADDR_T_64BIT gives maximum COMPILE_TEST coverage without having to spend effort on kludging things in the code that will never actually be needed on real hardware. Signed-off-by: Andy Gross Signed-off-by: Thomas Pedersen Signed-off-by: Jonathan McDowell Link: https://lore.kernel.org/r/20201114140233.GM32650@earth.li Signed-off-by: Vinod Koul --- drivers/dma/qcom/Kconfig | 11 + drivers/dma/qcom/Makefile | 1 + drivers/dma/qcom/qcom_adm.c | 903 ++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 915 insertions(+) create mode 100644 drivers/dma/qcom/qcom_adm.c diff --git a/drivers/dma/qcom/Kconfig b/drivers/dma/qcom/Kconfig index 3bcb689162c6..0389d60d2604 100644 --- a/drivers/dma/qcom/Kconfig +++ b/drivers/dma/qcom/Kconfig @@ -1,4 +1,15 @@ # SPDX-License-Identifier: GPL-2.0-only +config QCOM_ADM + tristate "Qualcomm ADM support" + depends on (ARCH_QCOM || COMPILE_TEST) && !PHYS_ADDR_T_64BIT + select DMA_ENGINE + select DMA_VIRTUAL_CHANNELS + help + Enable support for the Qualcomm Application Data Mover (ADM) DMA + controller, as present on MSM8x60, APQ8064, and IPQ8064 devices. + This controller provides DMA capabilities for both general purpose + and on-chip peripheral devices. + config QCOM_BAM_DMA tristate "QCOM BAM DMA support" depends on ARCH_QCOM || (COMPILE_TEST && OF && ARM) diff --git a/drivers/dma/qcom/Makefile b/drivers/dma/qcom/Makefile index 1ae92da88b0c..346e643fbb6d 100644 --- a/drivers/dma/qcom/Makefile +++ b/drivers/dma/qcom/Makefile @@ -1,4 +1,5 @@ # SPDX-License-Identifier: GPL-2.0 +obj-$(CONFIG_QCOM_ADM) += qcom_adm.o obj-$(CONFIG_QCOM_BAM_DMA) += bam_dma.o obj-$(CONFIG_QCOM_HIDMA_MGMT) += hdma_mgmt.o hdma_mgmt-objs := hidma_mgmt.o hidma_mgmt_sys.o diff --git a/drivers/dma/qcom/qcom_adm.c b/drivers/dma/qcom/qcom_adm.c new file mode 100644 index 000000000000..9b6f8e050ecc --- /dev/null +++ b/drivers/dma/qcom/qcom_adm.c @@ -0,0 +1,903 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2013-2015, The Linux Foundation. All rights reserved. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "../dmaengine.h" +#include "../virt-dma.h" + +/* ADM registers - calculated from channel number and security domain */ +#define ADM_CHAN_MULTI 0x4 +#define ADM_CI_MULTI 0x4 +#define ADM_CRCI_MULTI 0x4 +#define ADM_EE_MULTI 0x800 +#define ADM_CHAN_OFFS(chan) (ADM_CHAN_MULTI * (chan)) +#define ADM_EE_OFFS(ee) (ADM_EE_MULTI * (ee)) +#define ADM_CHAN_EE_OFFS(chan, ee) (ADM_CHAN_OFFS(chan) + ADM_EE_OFFS(ee)) +#define ADM_CHAN_OFFS(chan) (ADM_CHAN_MULTI * (chan)) +#define ADM_CI_OFFS(ci) (ADM_CHAN_OFF(ci)) +#define ADM_CH_CMD_PTR(chan, ee) (ADM_CHAN_EE_OFFS(chan, ee)) +#define ADM_CH_RSLT(chan, ee) (0x40 + ADM_CHAN_EE_OFFS(chan, ee)) +#define ADM_CH_FLUSH_STATE0(chan, ee) (0x80 + ADM_CHAN_EE_OFFS(chan, ee)) +#define ADM_CH_STATUS_SD(chan, ee) (0x200 + ADM_CHAN_EE_OFFS(chan, ee)) +#define ADM_CH_CONF(chan) (0x240 + ADM_CHAN_OFFS(chan)) +#define ADM_CH_RSLT_CONF(chan, ee) (0x300 + ADM_CHAN_EE_OFFS(chan, ee)) +#define ADM_SEC_DOMAIN_IRQ_STATUS(ee) (0x380 + ADM_EE_OFFS(ee)) +#define ADM_CI_CONF(ci) (0x390 + (ci) * ADM_CI_MULTI) +#define ADM_GP_CTL 0x3d8 +#define ADM_CRCI_CTL(crci, ee) (0x400 + (crci) * ADM_CRCI_MULTI + \ + ADM_EE_OFFS(ee)) + +/* channel status */ +#define ADM_CH_STATUS_VALID BIT(1) + +/* channel result */ +#define ADM_CH_RSLT_VALID BIT(31) +#define ADM_CH_RSLT_ERR BIT(3) +#define ADM_CH_RSLT_FLUSH BIT(2) +#define ADM_CH_RSLT_TPD BIT(1) + +/* channel conf */ +#define ADM_CH_CONF_SHADOW_EN BIT(12) +#define ADM_CH_CONF_MPU_DISABLE BIT(11) +#define ADM_CH_CONF_PERM_MPU_CONF BIT(9) +#define ADM_CH_CONF_FORCE_RSLT_EN BIT(7) +#define ADM_CH_CONF_SEC_DOMAIN(ee) ((((ee) & 0x3) << 4) | (((ee) & 0x4) << 11)) + +/* channel result conf */ +#define ADM_CH_RSLT_CONF_FLUSH_EN BIT(1) +#define ADM_CH_RSLT_CONF_IRQ_EN BIT(0) + +/* CRCI CTL */ +#define ADM_CRCI_CTL_MUX_SEL BIT(18) +#define ADM_CRCI_CTL_RST BIT(17) + +/* CI configuration */ +#define ADM_CI_RANGE_END(x) ((x) << 24) +#define ADM_CI_RANGE_START(x) ((x) << 16) +#define ADM_CI_BURST_4_WORDS BIT(2) +#define ADM_CI_BURST_8_WORDS BIT(3) + +/* GP CTL */ +#define ADM_GP_CTL_LP_EN BIT(12) +#define ADM_GP_CTL_LP_CNT(x) ((x) << 8) + +/* Command pointer list entry */ +#define ADM_CPLE_LP BIT(31) +#define ADM_CPLE_CMD_PTR_LIST BIT(29) + +/* Command list entry */ +#define ADM_CMD_LC BIT(31) +#define ADM_CMD_DST_CRCI(n) (((n) & 0xf) << 7) +#define ADM_CMD_SRC_CRCI(n) (((n) & 0xf) << 3) + +#define ADM_CMD_TYPE_SINGLE 0x0 +#define ADM_CMD_TYPE_BOX 0x3 + +#define ADM_CRCI_MUX_SEL BIT(4) +#define ADM_DESC_ALIGN 8 +#define ADM_MAX_XFER (SZ_64K - 1) +#define ADM_MAX_ROWS (SZ_64K - 1) +#define ADM_MAX_CHANNELS 16 + +struct adm_desc_hw_box { + u32 cmd; + u32 src_addr; + u32 dst_addr; + u32 row_len; + u32 num_rows; + u32 row_offset; +}; + +struct adm_desc_hw_single { + u32 cmd; + u32 src_addr; + u32 dst_addr; + u32 len; +}; + +struct adm_async_desc { + struct virt_dma_desc vd; + struct adm_device *adev; + + size_t length; + enum dma_transfer_direction dir; + dma_addr_t dma_addr; + size_t dma_len; + + void *cpl; + dma_addr_t cp_addr; + u32 crci; + u32 mux; + u32 blk_size; +}; + +struct adm_chan { + struct virt_dma_chan vc; + struct adm_device *adev; + + /* parsed from DT */ + u32 id; /* channel id */ + + struct adm_async_desc *curr_txd; + struct dma_slave_config slave; + struct list_head node; + + int error; + int initialized; +}; + +static inline struct adm_chan *to_adm_chan(struct dma_chan *common) +{ + return container_of(common, struct adm_chan, vc.chan); +} + +struct adm_device { + void __iomem *regs; + struct device *dev; + struct dma_device common; + struct device_dma_parameters dma_parms; + struct adm_chan *channels; + + u32 ee; + + struct clk *core_clk; + struct clk *iface_clk; + + struct reset_control *clk_reset; + struct reset_control *c0_reset; + struct reset_control *c1_reset; + struct reset_control *c2_reset; + int irq; +}; + +/** + * adm_free_chan - Frees dma resources associated with the specific channel + * + * Free all allocated descriptors associated with this channel + * + */ +static void adm_free_chan(struct dma_chan *chan) +{ + /* free all queued descriptors */ + vchan_free_chan_resources(to_virt_chan(chan)); +} + +/** + * adm_get_blksize - Get block size from burst value + * + */ +static int adm_get_blksize(unsigned int burst) +{ + int ret; + + switch (burst) { + case 16: + case 32: + case 64: + case 128: + ret = ffs(burst >> 4) - 1; + break; + case 192: + ret = 4; + break; + case 256: + ret = 5; + break; + default: + ret = -EINVAL; + break; + } + + return ret; +} + +/** + * adm_process_fc_descriptors - Process descriptors for flow controlled xfers + * + * @achan: ADM channel + * @desc: Descriptor memory pointer + * @sg: Scatterlist entry + * @crci: CRCI value + * @burst: Burst size of transaction + * @direction: DMA transfer direction + */ +static void *adm_process_fc_descriptors(struct adm_chan *achan, void *desc, + struct scatterlist *sg, u32 crci, + u32 burst, + enum dma_transfer_direction direction) +{ + struct adm_desc_hw_box *box_desc = NULL; + struct adm_desc_hw_single *single_desc; + u32 remainder = sg_dma_len(sg); + u32 rows, row_offset, crci_cmd; + u32 mem_addr = sg_dma_address(sg); + u32 *incr_addr = &mem_addr; + u32 *src, *dst; + + if (direction == DMA_DEV_TO_MEM) { + crci_cmd = ADM_CMD_SRC_CRCI(crci); + row_offset = burst; + src = &achan->slave.src_addr; + dst = &mem_addr; + } else { + crci_cmd = ADM_CMD_DST_CRCI(crci); + row_offset = burst << 16; + src = &mem_addr; + dst = &achan->slave.dst_addr; + } + + while (remainder >= burst) { + box_desc = desc; + box_desc->cmd = ADM_CMD_TYPE_BOX | crci_cmd; + box_desc->row_offset = row_offset; + box_desc->src_addr = *src; + box_desc->dst_addr = *dst; + + rows = remainder / burst; + rows = min_t(u32, rows, ADM_MAX_ROWS); + box_desc->num_rows = rows << 16 | rows; + box_desc->row_len = burst << 16 | burst; + + *incr_addr += burst * rows; + remainder -= burst * rows; + desc += sizeof(*box_desc); + } + + /* if leftover bytes, do one single descriptor */ + if (remainder) { + single_desc = desc; + single_desc->cmd = ADM_CMD_TYPE_SINGLE | crci_cmd; + single_desc->len = remainder; + single_desc->src_addr = *src; + single_desc->dst_addr = *dst; + desc += sizeof(*single_desc); + + if (sg_is_last(sg)) + single_desc->cmd |= ADM_CMD_LC; + } else { + if (box_desc && sg_is_last(sg)) + box_desc->cmd |= ADM_CMD_LC; + } + + return desc; +} + +/** + * adm_process_non_fc_descriptors - Process descriptors for non-fc xfers + * + * @achan: ADM channel + * @desc: Descriptor memory pointer + * @sg: Scatterlist entry + * @direction: DMA transfer direction + */ +static void *adm_process_non_fc_descriptors(struct adm_chan *achan, void *desc, + struct scatterlist *sg, + enum dma_transfer_direction direction) +{ + struct adm_desc_hw_single *single_desc; + u32 remainder = sg_dma_len(sg); + u32 mem_addr = sg_dma_address(sg); + u32 *incr_addr = &mem_addr; + u32 *src, *dst; + + if (direction == DMA_DEV_TO_MEM) { + src = &achan->slave.src_addr; + dst = &mem_addr; + } else { + src = &mem_addr; + dst = &achan->slave.dst_addr; + } + + do { + single_desc = desc; + single_desc->cmd = ADM_CMD_TYPE_SINGLE; + single_desc->src_addr = *src; + single_desc->dst_addr = *dst; + single_desc->len = (remainder > ADM_MAX_XFER) ? + ADM_MAX_XFER : remainder; + + remainder -= single_desc->len; + *incr_addr += single_desc->len; + desc += sizeof(*single_desc); + } while (remainder); + + /* set last command if this is the end of the whole transaction */ + if (sg_is_last(sg)) + single_desc->cmd |= ADM_CMD_LC; + + return desc; +} + +/** + * adm_prep_slave_sg - Prep slave sg transaction + * + * @chan: dma channel + * @sgl: scatter gather list + * @sg_len: length of sg + * @direction: DMA transfer direction + * @flags: DMA flags + * @context: transfer context (unused) + */ +static struct dma_async_tx_descriptor *adm_prep_slave_sg(struct dma_chan *chan, + struct scatterlist *sgl, + unsigned int sg_len, + enum dma_transfer_direction direction, + unsigned long flags, + void *context) +{ + struct adm_chan *achan = to_adm_chan(chan); + struct adm_device *adev = achan->adev; + struct adm_async_desc *async_desc; + struct scatterlist *sg; + dma_addr_t cple_addr; + u32 i, burst; + u32 single_count = 0, box_count = 0, crci = 0; + void *desc; + u32 *cple; + int blk_size = 0; + + if (!is_slave_direction(direction)) { + dev_err(adev->dev, "invalid dma direction\n"); + return NULL; + } + + /* + * get burst value from slave configuration + */ + burst = (direction == DMA_MEM_TO_DEV) ? + achan->slave.dst_maxburst : + achan->slave.src_maxburst; + + /* if using flow control, validate burst and crci values */ + if (achan->slave.device_fc) { + blk_size = adm_get_blksize(burst); + if (blk_size < 0) { + dev_err(adev->dev, "invalid burst value: %d\n", + burst); + return ERR_PTR(-EINVAL); + } + + crci = achan->slave.slave_id & 0xf; + if (!crci || achan->slave.slave_id > 0x1f) { + dev_err(adev->dev, "invalid crci value\n"); + return ERR_PTR(-EINVAL); + } + } + + /* iterate through sgs and compute allocation size of structures */ + for_each_sg(sgl, sg, sg_len, i) { + if (achan->slave.device_fc) { + box_count += DIV_ROUND_UP(sg_dma_len(sg) / burst, + ADM_MAX_ROWS); + if (sg_dma_len(sg) % burst) + single_count++; + } else { + single_count += DIV_ROUND_UP(sg_dma_len(sg), + ADM_MAX_XFER); + } + } + + async_desc = kzalloc(sizeof(*async_desc), GFP_NOWAIT); + if (!async_desc) + return ERR_PTR(-ENOMEM); + + if (crci) + async_desc->mux = achan->slave.slave_id & ADM_CRCI_MUX_SEL ? + ADM_CRCI_CTL_MUX_SEL : 0; + async_desc->crci = crci; + async_desc->blk_size = blk_size; + async_desc->dma_len = single_count * sizeof(struct adm_desc_hw_single) + + box_count * sizeof(struct adm_desc_hw_box) + + sizeof(*cple) + 2 * ADM_DESC_ALIGN; + + async_desc->cpl = kzalloc(async_desc->dma_len, GFP_NOWAIT); + if (!async_desc->cpl) + goto free; + + async_desc->adev = adev; + + /* both command list entry and descriptors must be 8 byte aligned */ + cple = PTR_ALIGN(async_desc->cpl, ADM_DESC_ALIGN); + desc = PTR_ALIGN(cple + 1, ADM_DESC_ALIGN); + + for_each_sg(sgl, sg, sg_len, i) { + async_desc->length += sg_dma_len(sg); + + if (achan->slave.device_fc) + desc = adm_process_fc_descriptors(achan, desc, sg, crci, + burst, direction); + else + desc = adm_process_non_fc_descriptors(achan, desc, sg, + direction); + } + + async_desc->dma_addr = dma_map_single(adev->dev, async_desc->cpl, + async_desc->dma_len, + DMA_TO_DEVICE); + if (dma_mapping_error(adev->dev, async_desc->dma_addr)) + goto free; + + cple_addr = async_desc->dma_addr + ((void *)cple - async_desc->cpl); + + /* init cmd list */ + dma_sync_single_for_cpu(adev->dev, cple_addr, sizeof(*cple), + DMA_TO_DEVICE); + *cple = ADM_CPLE_LP; + *cple |= (async_desc->dma_addr + ADM_DESC_ALIGN) >> 3; + dma_sync_single_for_device(adev->dev, cple_addr, sizeof(*cple), + DMA_TO_DEVICE); + + return vchan_tx_prep(&achan->vc, &async_desc->vd, flags); + +free: + kfree(async_desc); + return ERR_PTR(-ENOMEM); +} + +/** + * adm_terminate_all - terminate all transactions on a channel + * @achan: adm dma channel + * + * Dequeues and frees all transactions, aborts current transaction + * No callbacks are done + * + */ +static int adm_terminate_all(struct dma_chan *chan) +{ + struct adm_chan *achan = to_adm_chan(chan); + struct adm_device *adev = achan->adev; + unsigned long flags; + LIST_HEAD(head); + + spin_lock_irqsave(&achan->vc.lock, flags); + vchan_get_all_descriptors(&achan->vc, &head); + + /* send flush command to terminate current transaction */ + writel_relaxed(0x0, + adev->regs + ADM_CH_FLUSH_STATE0(achan->id, adev->ee)); + + spin_unlock_irqrestore(&achan->vc.lock, flags); + + vchan_dma_desc_free_list(&achan->vc, &head); + + return 0; +} + +static int adm_slave_config(struct dma_chan *chan, struct dma_slave_config *cfg) +{ + struct adm_chan *achan = to_adm_chan(chan); + unsigned long flag; + + spin_lock_irqsave(&achan->vc.lock, flag); + memcpy(&achan->slave, cfg, sizeof(struct dma_slave_config)); + spin_unlock_irqrestore(&achan->vc.lock, flag); + + return 0; +} + +/** + * adm_start_dma - start next transaction + * @achan - ADM dma channel + */ +static void adm_start_dma(struct adm_chan *achan) +{ + struct virt_dma_desc *vd = vchan_next_desc(&achan->vc); + struct adm_device *adev = achan->adev; + struct adm_async_desc *async_desc; + + lockdep_assert_held(&achan->vc.lock); + + if (!vd) + return; + + list_del(&vd->node); + + /* write next command list out to the CMD FIFO */ + async_desc = container_of(vd, struct adm_async_desc, vd); + achan->curr_txd = async_desc; + + /* reset channel error */ + achan->error = 0; + + if (!achan->initialized) { + /* enable interrupts */ + writel(ADM_CH_CONF_SHADOW_EN | + ADM_CH_CONF_PERM_MPU_CONF | + ADM_CH_CONF_MPU_DISABLE | + ADM_CH_CONF_SEC_DOMAIN(adev->ee), + adev->regs + ADM_CH_CONF(achan->id)); + + writel(ADM_CH_RSLT_CONF_IRQ_EN | ADM_CH_RSLT_CONF_FLUSH_EN, + adev->regs + ADM_CH_RSLT_CONF(achan->id, adev->ee)); + + achan->initialized = 1; + } + + /* set the crci block size if this transaction requires CRCI */ + if (async_desc->crci) { + writel(async_desc->mux | async_desc->blk_size, + adev->regs + ADM_CRCI_CTL(async_desc->crci, adev->ee)); + } + + /* make sure IRQ enable doesn't get reordered */ + wmb(); + + /* write next command list out to the CMD FIFO */ + writel(ALIGN(async_desc->dma_addr, ADM_DESC_ALIGN) >> 3, + adev->regs + ADM_CH_CMD_PTR(achan->id, adev->ee)); +} + +/** + * adm_dma_irq - irq handler for ADM controller + * @irq: IRQ of interrupt + * @data: callback data + * + * IRQ handler for the bam controller + */ +static irqreturn_t adm_dma_irq(int irq, void *data) +{ + struct adm_device *adev = data; + u32 srcs, i; + struct adm_async_desc *async_desc; + unsigned long flags; + + srcs = readl_relaxed(adev->regs + + ADM_SEC_DOMAIN_IRQ_STATUS(adev->ee)); + + for (i = 0; i < ADM_MAX_CHANNELS; i++) { + struct adm_chan *achan = &adev->channels[i]; + u32 status, result; + + if (srcs & BIT(i)) { + status = readl_relaxed(adev->regs + + ADM_CH_STATUS_SD(i, adev->ee)); + + /* if no result present, skip */ + if (!(status & ADM_CH_STATUS_VALID)) + continue; + + result = readl_relaxed(adev->regs + + ADM_CH_RSLT(i, adev->ee)); + + /* no valid results, skip */ + if (!(result & ADM_CH_RSLT_VALID)) + continue; + + /* flag error if transaction was flushed or failed */ + if (result & (ADM_CH_RSLT_ERR | ADM_CH_RSLT_FLUSH)) + achan->error = 1; + + spin_lock_irqsave(&achan->vc.lock, flags); + async_desc = achan->curr_txd; + + achan->curr_txd = NULL; + + if (async_desc) { + vchan_cookie_complete(&async_desc->vd); + + /* kick off next DMA */ + adm_start_dma(achan); + } + + spin_unlock_irqrestore(&achan->vc.lock, flags); + } + } + + return IRQ_HANDLED; +} + +/** + * adm_tx_status - returns status of transaction + * @chan: dma channel + * @cookie: transaction cookie + * @txstate: DMA transaction state + * + * Return status of dma transaction + */ +static enum dma_status adm_tx_status(struct dma_chan *chan, dma_cookie_t cookie, + struct dma_tx_state *txstate) +{ + struct adm_chan *achan = to_adm_chan(chan); + struct virt_dma_desc *vd; + enum dma_status ret; + unsigned long flags; + size_t residue = 0; + + ret = dma_cookie_status(chan, cookie, txstate); + if (ret == DMA_COMPLETE || !txstate) + return ret; + + spin_lock_irqsave(&achan->vc.lock, flags); + + vd = vchan_find_desc(&achan->vc, cookie); + if (vd) + residue = container_of(vd, struct adm_async_desc, vd)->length; + + spin_unlock_irqrestore(&achan->vc.lock, flags); + + /* + * residue is either the full length if it is in the issued list, or 0 + * if it is in progress. We have no reliable way of determining + * anything inbetween + */ + dma_set_residue(txstate, residue); + + if (achan->error) + return DMA_ERROR; + + return ret; +} + +/** + * adm_issue_pending - starts pending transactions + * @chan: dma channel + * + * Issues all pending transactions and starts DMA + */ +static void adm_issue_pending(struct dma_chan *chan) +{ + struct adm_chan *achan = to_adm_chan(chan); + unsigned long flags; + + spin_lock_irqsave(&achan->vc.lock, flags); + + if (vchan_issue_pending(&achan->vc) && !achan->curr_txd) + adm_start_dma(achan); + spin_unlock_irqrestore(&achan->vc.lock, flags); +} + +/** + * adm_dma_free_desc - free descriptor memory + * @vd: virtual descriptor + * + */ +static void adm_dma_free_desc(struct virt_dma_desc *vd) +{ + struct adm_async_desc *async_desc = container_of(vd, + struct adm_async_desc, vd); + + dma_unmap_single(async_desc->adev->dev, async_desc->dma_addr, + async_desc->dma_len, DMA_TO_DEVICE); + kfree(async_desc->cpl); + kfree(async_desc); +} + +static void adm_channel_init(struct adm_device *adev, struct adm_chan *achan, + u32 index) +{ + achan->id = index; + achan->adev = adev; + + vchan_init(&achan->vc, &adev->common); + achan->vc.desc_free = adm_dma_free_desc; +} + +static int adm_dma_probe(struct platform_device *pdev) +{ + struct adm_device *adev; + int ret; + u32 i; + + adev = devm_kzalloc(&pdev->dev, sizeof(*adev), GFP_KERNEL); + if (!adev) + return -ENOMEM; + + adev->dev = &pdev->dev; + + adev->regs = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(adev->regs)) + return PTR_ERR(adev->regs); + + adev->irq = platform_get_irq(pdev, 0); + if (adev->irq < 0) + return adev->irq; + + ret = of_property_read_u32(pdev->dev.of_node, "qcom,ee", &adev->ee); + if (ret) { + dev_err(adev->dev, "Execution environment unspecified\n"); + return ret; + } + + adev->core_clk = devm_clk_get(adev->dev, "core"); + if (IS_ERR(adev->core_clk)) + return PTR_ERR(adev->core_clk); + + adev->iface_clk = devm_clk_get(adev->dev, "iface"); + if (IS_ERR(adev->iface_clk)) + return PTR_ERR(adev->iface_clk); + + adev->clk_reset = devm_reset_control_get_exclusive(&pdev->dev, "clk"); + if (IS_ERR(adev->clk_reset)) { + dev_err(adev->dev, "failed to get ADM0 reset\n"); + return PTR_ERR(adev->clk_reset); + } + + adev->c0_reset = devm_reset_control_get_exclusive(&pdev->dev, "c0"); + if (IS_ERR(adev->c0_reset)) { + dev_err(adev->dev, "failed to get ADM0 C0 reset\n"); + return PTR_ERR(adev->c0_reset); + } + + adev->c1_reset = devm_reset_control_get_exclusive(&pdev->dev, "c1"); + if (IS_ERR(adev->c1_reset)) { + dev_err(adev->dev, "failed to get ADM0 C1 reset\n"); + return PTR_ERR(adev->c1_reset); + } + + adev->c2_reset = devm_reset_control_get_exclusive(&pdev->dev, "c2"); + if (IS_ERR(adev->c2_reset)) { + dev_err(adev->dev, "failed to get ADM0 C2 reset\n"); + return PTR_ERR(adev->c2_reset); + } + + ret = clk_prepare_enable(adev->core_clk); + if (ret) { + dev_err(adev->dev, "failed to prepare/enable core clock\n"); + return ret; + } + + ret = clk_prepare_enable(adev->iface_clk); + if (ret) { + dev_err(adev->dev, "failed to prepare/enable iface clock\n"); + goto err_disable_core_clk; + } + + reset_control_assert(adev->clk_reset); + reset_control_assert(adev->c0_reset); + reset_control_assert(adev->c1_reset); + reset_control_assert(adev->c2_reset); + + udelay(2); + + reset_control_deassert(adev->clk_reset); + reset_control_deassert(adev->c0_reset); + reset_control_deassert(adev->c1_reset); + reset_control_deassert(adev->c2_reset); + + adev->channels = devm_kcalloc(adev->dev, ADM_MAX_CHANNELS, + sizeof(*adev->channels), GFP_KERNEL); + + if (!adev->channels) { + ret = -ENOMEM; + goto err_disable_clks; + } + + /* allocate and initialize channels */ + INIT_LIST_HEAD(&adev->common.channels); + + for (i = 0; i < ADM_MAX_CHANNELS; i++) + adm_channel_init(adev, &adev->channels[i], i); + + /* reset CRCIs */ + for (i = 0; i < 16; i++) + writel(ADM_CRCI_CTL_RST, adev->regs + + ADM_CRCI_CTL(i, adev->ee)); + + /* configure client interfaces */ + writel(ADM_CI_RANGE_START(0x40) | ADM_CI_RANGE_END(0xb0) | + ADM_CI_BURST_8_WORDS, adev->regs + ADM_CI_CONF(0)); + writel(ADM_CI_RANGE_START(0x2a) | ADM_CI_RANGE_END(0x2c) | + ADM_CI_BURST_8_WORDS, adev->regs + ADM_CI_CONF(1)); + writel(ADM_CI_RANGE_START(0x12) | ADM_CI_RANGE_END(0x28) | + ADM_CI_BURST_8_WORDS, adev->regs + ADM_CI_CONF(2)); + writel(ADM_GP_CTL_LP_EN | ADM_GP_CTL_LP_CNT(0xf), + adev->regs + ADM_GP_CTL); + + ret = devm_request_irq(adev->dev, adev->irq, adm_dma_irq, + 0, "adm_dma", adev); + if (ret) + goto err_disable_clks; + + platform_set_drvdata(pdev, adev); + + adev->common.dev = adev->dev; + adev->common.dev->dma_parms = &adev->dma_parms; + + /* set capabilities */ + dma_cap_zero(adev->common.cap_mask); + dma_cap_set(DMA_SLAVE, adev->common.cap_mask); + dma_cap_set(DMA_PRIVATE, adev->common.cap_mask); + + /* initialize dmaengine apis */ + adev->common.directions = BIT(DMA_DEV_TO_MEM | DMA_MEM_TO_DEV); + adev->common.residue_granularity = DMA_RESIDUE_GRANULARITY_DESCRIPTOR; + adev->common.src_addr_widths = DMA_SLAVE_BUSWIDTH_4_BYTES; + adev->common.dst_addr_widths = DMA_SLAVE_BUSWIDTH_4_BYTES; + adev->common.device_free_chan_resources = adm_free_chan; + adev->common.device_prep_slave_sg = adm_prep_slave_sg; + adev->common.device_issue_pending = adm_issue_pending; + adev->common.device_tx_status = adm_tx_status; + adev->common.device_terminate_all = adm_terminate_all; + adev->common.device_config = adm_slave_config; + + ret = dma_async_device_register(&adev->common); + if (ret) { + dev_err(adev->dev, "failed to register dma async device\n"); + goto err_disable_clks; + } + + ret = of_dma_controller_register(pdev->dev.of_node, + of_dma_xlate_by_chan_id, + &adev->common); + if (ret) + goto err_unregister_dma; + + return 0; + +err_unregister_dma: + dma_async_device_unregister(&adev->common); +err_disable_clks: + clk_disable_unprepare(adev->iface_clk); +err_disable_core_clk: + clk_disable_unprepare(adev->core_clk); + + return ret; +} + +static int adm_dma_remove(struct platform_device *pdev) +{ + struct adm_device *adev = platform_get_drvdata(pdev); + struct adm_chan *achan; + u32 i; + + of_dma_controller_free(pdev->dev.of_node); + dma_async_device_unregister(&adev->common); + + for (i = 0; i < ADM_MAX_CHANNELS; i++) { + achan = &adev->channels[i]; + + /* mask IRQs for this channel/EE pair */ + writel(0, adev->regs + ADM_CH_RSLT_CONF(achan->id, adev->ee)); + + tasklet_kill(&adev->channels[i].vc.task); + adm_terminate_all(&adev->channels[i].vc.chan); + } + + devm_free_irq(adev->dev, adev->irq, adev); + + clk_disable_unprepare(adev->core_clk); + clk_disable_unprepare(adev->iface_clk); + + return 0; +} + +static const struct of_device_id adm_of_match[] = { + { .compatible = "qcom,adm", }, + {} +}; +MODULE_DEVICE_TABLE(of, adm_of_match); + +static struct platform_driver adm_dma_driver = { + .probe = adm_dma_probe, + .remove = adm_dma_remove, + .driver = { + .name = "adm-dma-engine", + .of_match_table = adm_of_match, + }, +}; + +module_platform_driver(adm_dma_driver); + +MODULE_AUTHOR("Andy Gross "); +MODULE_DESCRIPTION("QCOM ADM DMA engine driver"); +MODULE_LICENSE("GPL v2"); -- cgit v1.2.3-59-g8ed1b From 678198f21135b0aaf9782049d0a6bbbeefff1804 Mon Sep 17 00:00:00 2001 From: Vinod Koul Date: Mon, 9 Nov 2020 14:24:48 +0530 Subject: dt-bindings: dmaengine: Document qcom,gpi dma binding Add devicetree binding documentation for GPI DMA controller implemented on Qualcomm SoCs Reviewed-by: Rob Herring Link: https://lore.kernel.org/r/20201109085450.24843-2-vkoul@kernel.org Signed-off-by: Vinod Koul --- .../devicetree/bindings/dma/qcom,gpi.yaml | 88 ++++++++++++++++++++++ include/dt-bindings/dma/qcom-gpi.h | 11 +++ 2 files changed, 99 insertions(+) create mode 100644 Documentation/devicetree/bindings/dma/qcom,gpi.yaml create mode 100644 include/dt-bindings/dma/qcom-gpi.h diff --git a/Documentation/devicetree/bindings/dma/qcom,gpi.yaml b/Documentation/devicetree/bindings/dma/qcom,gpi.yaml new file mode 100644 index 000000000000..f8142adf9aea --- /dev/null +++ b/Documentation/devicetree/bindings/dma/qcom,gpi.yaml @@ -0,0 +1,88 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/dma/qcom,gpi.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Qualcomm Technologies Inc GPI DMA controller + +maintainers: + - Vinod Koul + +description: | + QCOM GPI DMA controller provides DMA capabilities for + peripheral buses such as I2C, UART, and SPI. + +allOf: + - $ref: "dma-controller.yaml#" + +properties: + compatible: + enum: + - qcom,sdm845-gpi-dma + + reg: + maxItems: 1 + + interrupts: + description: + Interrupt lines for each GPI instance + maxItems: 13 + + "#dma-cells": + const: 3 + description: > + DMA clients must use the format described in dma.txt, giving a phandle + to the DMA controller plus the following 3 integer cells: + - channel: if set to 0xffffffff, any available channel will be allocated + for the client. Otherwise, the exact channel specified will be used. + - seid: serial id of the client as defined in the SoC documentation. + - client: type of the client as defined in dt-bindings/dma/qcom-gpi.h + + iommus: + maxItems: 1 + + dma-channels: + maximum: 31 + + dma-channel-mask: + maxItems: 1 + +required: + - compatible + - reg + - interrupts + - "#dma-cells" + - iommus + - dma-channels + - dma-channel-mask + +additionalProperties: false + +examples: + - | + #include + #include + gpi_dma0: dma-controller@800000 { + compatible = "qcom,gpi-dma"; + #dma-cells = <3>; + reg = <0x00800000 0x60000>; + iommus = <&apps_smmu 0x0016 0x0>; + dma-channels = <13>; + dma-channel-mask = <0xfa>; + interrupts = , + , + , + , + , + , + , + , + , + , + , + , + ; + }; + +... diff --git a/include/dt-bindings/dma/qcom-gpi.h b/include/dt-bindings/dma/qcom-gpi.h new file mode 100644 index 000000000000..ebda2a37f52a --- /dev/null +++ b/include/dt-bindings/dma/qcom-gpi.h @@ -0,0 +1,11 @@ +/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */ +/* Copyright (c) 2020, Linaro Ltd. */ + +#ifndef __DT_BINDINGS_DMA_QCOM_GPI_H__ +#define __DT_BINDINGS_DMA_QCOM_GPI_H__ + +#define QCOM_GPI_SPI 1 +#define QCOM_GPI_UART 2 +#define QCOM_GPI_I2C 3 + +#endif /* __DT_BINDINGS_DMA_QCOM_GPI_H__ */ -- cgit v1.2.3-59-g8ed1b From e7bbb7acabf47d74672e0e314bed4d452d2097b4 Mon Sep 17 00:00:00 2001 From: Vinod Koul Date: Mon, 9 Nov 2020 14:24:49 +0530 Subject: dmaengine: add peripheral configuration Some complex dmaengine controllers have capability to program the peripheral device, so pass on the peripheral configuration as part of dma_slave_config Link: https://lore.kernel.org/r/20201109085450.24843-3-vkoul@kernel.org Signed-off-by: Vinod Koul --- include/linux/dmaengine.h | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/include/linux/dmaengine.h b/include/linux/dmaengine.h index dd357a747780..493a047ed0a2 100644 --- a/include/linux/dmaengine.h +++ b/include/linux/dmaengine.h @@ -418,6 +418,9 @@ enum dma_slave_buswidth { * @slave_id: Slave requester id. Only valid for slave channels. The dma * slave peripheral will have unique id as dma requester which need to be * pass as slave config. + * @peripheral_config: peripheral configuration for programming peripheral + * for dmaengine transfer + * @peripheral_size: peripheral configuration buffer size * * This struct is passed in as configuration data to a DMA engine * in order to set up a certain channel for DMA transport at runtime. @@ -443,6 +446,8 @@ struct dma_slave_config { u32 dst_port_window_size; bool device_fc; unsigned int slave_id; + void *peripheral_config; + size_t peripheral_size; }; /** -- cgit v1.2.3-59-g8ed1b From 5d0c3533a19f48e5e7e73806a3e4b29cd4364130 Mon Sep 17 00:00:00 2001 From: Vinod Koul Date: Mon, 9 Nov 2020 14:24:50 +0530 Subject: dmaengine: qcom: Add GPI dma driver This controller provides DMAengine capabilities for a variety of peripheral buses such as I2C, UART, and SPI. By using GPI dmaengine driver, bus drivers can use a standardize interface that is protocol independent to transfer data between memory and peripheral. Link: https://lore.kernel.org/r/20201109085450.24843-4-vkoul@kernel.org Signed-off-by: Vinod Koul --- drivers/dma/qcom/Kconfig | 12 + drivers/dma/qcom/Makefile | 1 + drivers/dma/qcom/gpi.c | 2303 ++++++++++++++++++++++++++++++++++++++ include/linux/dma/qcom-gpi-dma.h | 83 ++ 4 files changed, 2399 insertions(+) create mode 100644 drivers/dma/qcom/gpi.c create mode 100644 include/linux/dma/qcom-gpi-dma.h diff --git a/drivers/dma/qcom/Kconfig b/drivers/dma/qcom/Kconfig index 0389d60d2604..365f94eb3b08 100644 --- a/drivers/dma/qcom/Kconfig +++ b/drivers/dma/qcom/Kconfig @@ -19,6 +19,18 @@ config QCOM_BAM_DMA Enable support for the QCOM BAM DMA controller. This controller provides DMA capabilities for a variety of on-chip devices. +config QCOM_GPI_DMA + tristate "Qualcomm Technologies GPI DMA support" + depends on ARCH_QCOM + select DMA_ENGINE + select DMA_VIRTUAL_CHANNELS + help + Enable support for the QCOM GPI DMA controller. This controller + provides DMA capabilities for a variety of peripheral buses such + as I2C, UART, and SPI. By using GPI dmaengine driver, bus drivers + can use a standardize interface that is protocol independent to + transfer data between DDR and peripheral. + config QCOM_HIDMA_MGMT tristate "Qualcomm Technologies HIDMA Management support" select DMA_ENGINE diff --git a/drivers/dma/qcom/Makefile b/drivers/dma/qcom/Makefile index 346e643fbb6d..50f1e7014693 100644 --- a/drivers/dma/qcom/Makefile +++ b/drivers/dma/qcom/Makefile @@ -1,6 +1,7 @@ # SPDX-License-Identifier: GPL-2.0 obj-$(CONFIG_QCOM_ADM) += qcom_adm.o obj-$(CONFIG_QCOM_BAM_DMA) += bam_dma.o +obj-$(CONFIG_QCOM_GPI_DMA) += gpi.o obj-$(CONFIG_QCOM_HIDMA_MGMT) += hdma_mgmt.o hdma_mgmt-objs := hidma_mgmt.o hidma_mgmt_sys.o obj-$(CONFIG_QCOM_HIDMA) += hdma.o diff --git a/drivers/dma/qcom/gpi.c b/drivers/dma/qcom/gpi.c new file mode 100644 index 000000000000..d2334f535de2 --- /dev/null +++ b/drivers/dma/qcom/gpi.c @@ -0,0 +1,2303 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2017-2020, The Linux Foundation. All rights reserved. + * Copyright (c) 2020, Linaro Limited + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "../dmaengine.h" +#include "../virt-dma.h" + +#define TRE_TYPE_DMA 0x10 +#define TRE_TYPE_GO 0x20 +#define TRE_TYPE_CONFIG0 0x22 + +/* TRE flags */ +#define TRE_FLAGS_CHAIN BIT(0) +#define TRE_FLAGS_IEOB BIT(8) +#define TRE_FLAGS_IEOT BIT(9) +#define TRE_FLAGS_BEI BIT(10) +#define TRE_FLAGS_LINK BIT(11) +#define TRE_FLAGS_TYPE GENMASK(23, 16) + +/* SPI CONFIG0 WD0 */ +#define TRE_SPI_C0_WORD_SZ GENMASK(4, 0) +#define TRE_SPI_C0_LOOPBACK BIT(8) +#define TRE_SPI_C0_CS BIT(11) +#define TRE_SPI_C0_CPHA BIT(12) +#define TRE_SPI_C0_CPOL BIT(13) +#define TRE_SPI_C0_TX_PACK BIT(24) +#define TRE_SPI_C0_RX_PACK BIT(25) + +/* CONFIG0 WD2 */ +#define TRE_C0_CLK_DIV GENMASK(11, 0) +#define TRE_C0_CLK_SRC GENMASK(19, 16) + +/* SPI GO WD0 */ +#define TRE_SPI_GO_CMD GENMASK(4, 0) +#define TRE_SPI_GO_CS GENMASK(10, 8) +#define TRE_SPI_GO_FRAG BIT(26) + +/* GO WD2 */ +#define TRE_RX_LEN GENMASK(23, 0) + +/* I2C Config0 WD0 */ +#define TRE_I2C_C0_TLOW GENMASK(7, 0) +#define TRE_I2C_C0_THIGH GENMASK(15, 8) +#define TRE_I2C_C0_TCYL GENMASK(23, 16) +#define TRE_I2C_C0_TX_PACK BIT(24) +#define TRE_I2C_C0_RX_PACK BIT(25) + +/* I2C GO WD0 */ +#define TRE_I2C_GO_CMD GENMASK(4, 0) +#define TRE_I2C_GO_ADDR GENMASK(14, 8) +#define TRE_I2C_GO_STRETCH BIT(26) + +/* DMA TRE */ +#define TRE_DMA_LEN GENMASK(23, 0) + +/* Register offsets from gpi-top */ +#define GPII_n_CH_k_CNTXT_0_OFFS(n, k) (0x20000 + (0x4000 * (n)) + (0x80 * (k))) +#define GPII_n_CH_k_CNTXT_0_EL_SIZE GENMASK(31, 24) +#define GPII_n_CH_k_CNTXT_0_CHSTATE GENMASK(23, 20) +#define GPII_n_CH_k_CNTXT_0_ERIDX GENMASK(18, 14) +#define GPII_n_CH_k_CNTXT_0_DIR BIT(3) +#define GPII_n_CH_k_CNTXT_0_PROTO GENMASK(2, 0) + +#define GPII_n_CH_k_CNTXT_0(el_size, erindex, dir, chtype_proto) \ + (FIELD_PREP(GPII_n_CH_k_CNTXT_0_EL_SIZE, el_size) | \ + FIELD_PREP(GPII_n_CH_k_CNTXT_0_ERIDX, erindex) | \ + FIELD_PREP(GPII_n_CH_k_CNTXT_0_DIR, dir) | \ + FIELD_PREP(GPII_n_CH_k_CNTXT_0_PROTO, chtype_proto)) + +#define GPI_CHTYPE_DIR_IN (0) +#define GPI_CHTYPE_DIR_OUT (1) + +#define GPI_CHTYPE_PROTO_GPI (0x2) + +#define GPII_n_CH_k_DOORBELL_0_OFFS(n, k) (0x22000 + (0x4000 * (n)) + (0x8 * (k))) +#define GPII_n_CH_CMD_OFFS(n) (0x23008 + (0x4000 * (n))) +#define GPII_n_CH_CMD_OPCODE GENMASK(31, 24) +#define GPII_n_CH_CMD_CHID GENMASK(7, 0) +#define GPII_n_CH_CMD(opcode, chid) \ + (FIELD_PREP(GPII_n_CH_CMD_OPCODE, opcode) | \ + FIELD_PREP(GPII_n_CH_CMD_CHID, chid)) + +#define GPII_n_CH_CMD_ALLOCATE (0) +#define GPII_n_CH_CMD_START (1) +#define GPII_n_CH_CMD_STOP (2) +#define GPII_n_CH_CMD_RESET (9) +#define GPII_n_CH_CMD_DE_ALLOC (10) +#define GPII_n_CH_CMD_UART_SW_STALE (32) +#define GPII_n_CH_CMD_UART_RFR_READY (33) +#define GPII_n_CH_CMD_UART_RFR_NOT_READY (34) + +/* EV Context Array */ +#define GPII_n_EV_CH_k_CNTXT_0_OFFS(n, k) (0x21000 + (0x4000 * (n)) + (0x80 * (k))) +#define GPII_n_EV_k_CNTXT_0_EL_SIZE GENMASK(31, 24) +#define GPII_n_EV_k_CNTXT_0_CHSTATE GENMASK(23, 20) +#define GPII_n_EV_k_CNTXT_0_INTYPE BIT(16) +#define GPII_n_EV_k_CNTXT_0_CHTYPE GENMASK(3, 0) + +#define GPII_n_EV_k_CNTXT_0(el_size, inttype, chtype) \ + (FIELD_PREP(GPII_n_EV_k_CNTXT_0_EL_SIZE, el_size) | \ + FIELD_PREP(GPII_n_EV_k_CNTXT_0_INTYPE, inttype) | \ + FIELD_PREP(GPII_n_EV_k_CNTXT_0_CHTYPE, chtype)) + +#define GPI_INTTYPE_IRQ (1) +#define GPI_CHTYPE_GPI_EV (0x2) + +enum CNTXT_OFFS { + CNTXT_0_CONFIG = 0x0, + CNTXT_1_R_LENGTH = 0x4, + CNTXT_2_RING_BASE_LSB = 0x8, + CNTXT_3_RING_BASE_MSB = 0xC, + CNTXT_4_RING_RP_LSB = 0x10, + CNTXT_5_RING_RP_MSB = 0x14, + CNTXT_6_RING_WP_LSB = 0x18, + CNTXT_7_RING_WP_MSB = 0x1C, + CNTXT_8_RING_INT_MOD = 0x20, + CNTXT_9_RING_INTVEC = 0x24, + CNTXT_10_RING_MSI_LSB = 0x28, + CNTXT_11_RING_MSI_MSB = 0x2C, + CNTXT_12_RING_RP_UPDATE_LSB = 0x30, + CNTXT_13_RING_RP_UPDATE_MSB = 0x34, +}; + +#define GPII_n_EV_CH_k_DOORBELL_0_OFFS(n, k) (0x22100 + (0x4000 * (n)) + (0x8 * (k))) +#define GPII_n_EV_CH_CMD_OFFS(n) (0x23010 + (0x4000 * (n))) +#define GPII_n_EV_CMD_OPCODE GENMASK(31, 24) +#define GPII_n_EV_CMD_CHID GENMASK(7, 0) +#define GPII_n_EV_CMD(opcode, chid) \ + (FIELD_PREP(GPII_n_EV_CMD_OPCODE, opcode) | \ + FIELD_PREP(GPII_n_EV_CMD_CHID, chid)) + +#define GPII_n_EV_CH_CMD_ALLOCATE (0x00) +#define GPII_n_EV_CH_CMD_RESET (0x09) +#define GPII_n_EV_CH_CMD_DE_ALLOC (0x0A) + +#define GPII_n_CNTXT_TYPE_IRQ_OFFS(n) (0x23080 + (0x4000 * (n))) + +/* mask type register */ +#define GPII_n_CNTXT_TYPE_IRQ_MSK_OFFS(n) (0x23088 + (0x4000 * (n))) +#define GPII_n_CNTXT_TYPE_IRQ_MSK_BMSK GENMASK(6, 0) +#define GPII_n_CNTXT_TYPE_IRQ_MSK_GENERAL BIT(6) +#define GPII_n_CNTXT_TYPE_IRQ_MSK_IEOB BIT(3) +#define GPII_n_CNTXT_TYPE_IRQ_MSK_GLOB BIT(2) +#define GPII_n_CNTXT_TYPE_IRQ_MSK_EV_CTRL BIT(1) +#define GPII_n_CNTXT_TYPE_IRQ_MSK_CH_CTRL BIT(0) + +#define GPII_n_CNTXT_SRC_GPII_CH_IRQ_OFFS(n) (0x23090 + (0x4000 * (n))) +#define GPII_n_CNTXT_SRC_EV_CH_IRQ_OFFS(n) (0x23094 + (0x4000 * (n))) + +/* Mask channel control interrupt register */ +#define GPII_n_CNTXT_SRC_CH_IRQ_MSK_OFFS(n) (0x23098 + (0x4000 * (n))) +#define GPII_n_CNTXT_SRC_CH_IRQ_MSK_BMSK GENMASK(1, 0) + +/* Mask event control interrupt register */ +#define GPII_n_CNTXT_SRC_EV_CH_IRQ_MSK_OFFS(n) (0x2309C + (0x4000 * (n))) +#define GPII_n_CNTXT_SRC_EV_CH_IRQ_MSK_BMSK BIT(0) + +#define GPII_n_CNTXT_SRC_CH_IRQ_CLR_OFFS(n) (0x230A0 + (0x4000 * (n))) +#define GPII_n_CNTXT_SRC_EV_CH_IRQ_CLR_OFFS(n) (0x230A4 + (0x4000 * (n))) + +/* Mask event interrupt register */ +#define GPII_n_CNTXT_SRC_IEOB_IRQ_MSK_OFFS(n) (0x230B8 + (0x4000 * (n))) +#define GPII_n_CNTXT_SRC_IEOB_IRQ_MSK_BMSK BIT(0) + +#define GPII_n_CNTXT_SRC_IEOB_IRQ_CLR_OFFS(n) (0x230C0 + (0x4000 * (n))) +#define GPII_n_CNTXT_GLOB_IRQ_STTS_OFFS(n) (0x23100 + (0x4000 * (n))) +#define GPI_GLOB_IRQ_ERROR_INT_MSK BIT(0) + +/* GPII specific Global - Enable bit register */ +#define GPII_n_CNTXT_GLOB_IRQ_EN_OFFS(n) (0x23108 + (0x4000 * (n))) +#define GPII_n_CNTXT_GLOB_IRQ_CLR_OFFS(n) (0x23110 + (0x4000 * (n))) +#define GPII_n_CNTXT_GPII_IRQ_STTS_OFFS(n) (0x23118 + (0x4000 * (n))) + +/* GPII general interrupt - Enable bit register */ +#define GPII_n_CNTXT_GPII_IRQ_EN_OFFS(n) (0x23120 + (0x4000 * (n))) +#define GPII_n_CNTXT_GPII_IRQ_EN_BMSK GENMASK(3, 0) + +#define GPII_n_CNTXT_GPII_IRQ_CLR_OFFS(n) (0x23128 + (0x4000 * (n))) + +/* GPII Interrupt Type register */ +#define GPII_n_CNTXT_INTSET_OFFS(n) (0x23180 + (0x4000 * (n))) +#define GPII_n_CNTXT_INTSET_BMSK BIT(0) + +#define GPII_n_CNTXT_MSI_BASE_LSB_OFFS(n) (0x23188 + (0x4000 * (n))) +#define GPII_n_CNTXT_MSI_BASE_MSB_OFFS(n) (0x2318C + (0x4000 * (n))) +#define GPII_n_CNTXT_SCRATCH_0_OFFS(n) (0x23400 + (0x4000 * (n))) +#define GPII_n_CNTXT_SCRATCH_1_OFFS(n) (0x23404 + (0x4000 * (n))) + +#define GPII_n_ERROR_LOG_OFFS(n) (0x23200 + (0x4000 * (n))) + +/* QOS Registers */ +#define GPII_n_CH_k_QOS_OFFS(n, k) (0x2005C + (0x4000 * (n)) + (0x80 * (k))) + +/* Scratch registers */ +#define GPII_n_CH_k_SCRATCH_0_OFFS(n, k) (0x20060 + (0x4000 * (n)) + (0x80 * (k))) +#define GPII_n_CH_k_SCRATCH_0_SEID GENMASK(2, 0) +#define GPII_n_CH_k_SCRATCH_0_PROTO GENMASK(7, 4) +#define GPII_n_CH_k_SCRATCH_0_PAIR GENMASK(20, 16) +#define GPII_n_CH_k_SCRATCH_0(pair, proto, seid) \ + (FIELD_PREP(GPII_n_CH_k_SCRATCH_0_PAIR, pair) | \ + FIELD_PREP(GPII_n_CH_k_SCRATCH_0_PROTO, proto) | \ + FIELD_PREP(GPII_n_CH_k_SCRATCH_0_SEID, seid)) +#define GPII_n_CH_k_SCRATCH_1_OFFS(n, k) (0x20064 + (0x4000 * (n)) + (0x80 * (k))) +#define GPII_n_CH_k_SCRATCH_2_OFFS(n, k) (0x20068 + (0x4000 * (n)) + (0x80 * (k))) +#define GPII_n_CH_k_SCRATCH_3_OFFS(n, k) (0x2006C + (0x4000 * (n)) + (0x80 * (k))) + +struct __packed gpi_tre { + u32 dword[4]; +}; + +enum msm_gpi_tce_code { + MSM_GPI_TCE_SUCCESS = 1, + MSM_GPI_TCE_EOT = 2, + MSM_GPI_TCE_EOB = 4, + MSM_GPI_TCE_UNEXP_ERR = 16, +}; + +#define CMD_TIMEOUT_MS (250) + +#define MAX_CHANNELS_PER_GPII (2) +#define GPI_TX_CHAN (0) +#define GPI_RX_CHAN (1) +#define STATE_IGNORE (U32_MAX) +#define EV_FACTOR (2) +#define REQ_OF_DMA_ARGS (5) /* # of arguments required from client */ +#define CHAN_TRES 64 + +struct __packed xfer_compl_event { + u64 ptr; + u32 length:24; + u8 code; + u16 status; + u8 type; + u8 chid; +}; + +struct __packed immediate_data_event { + u8 data_bytes[8]; + u8 length:4; + u8 resvd:4; + u16 tre_index; + u8 code; + u16 status; + u8 type; + u8 chid; +}; + +struct __packed qup_notif_event { + u32 status; + u32 time; + u32 count:24; + u8 resvd; + u16 resvd1; + u8 type; + u8 chid; +}; + +struct __packed gpi_ere { + u32 dword[4]; +}; + +enum GPI_EV_TYPE { + XFER_COMPLETE_EV_TYPE = 0x22, + IMMEDIATE_DATA_EV_TYPE = 0x30, + QUP_NOTIF_EV_TYPE = 0x31, + STALE_EV_TYPE = 0xFF, +}; + +union __packed gpi_event { + struct __packed xfer_compl_event xfer_compl_event; + struct __packed immediate_data_event immediate_data_event; + struct __packed qup_notif_event qup_notif_event; + struct __packed gpi_ere gpi_ere; +}; + +enum gpii_irq_settings { + DEFAULT_IRQ_SETTINGS, + MASK_IEOB_SETTINGS, +}; + +enum gpi_ev_state { + DEFAULT_EV_CH_STATE = 0, + EV_STATE_NOT_ALLOCATED = DEFAULT_EV_CH_STATE, + EV_STATE_ALLOCATED, + MAX_EV_STATES +}; + +static const char *const gpi_ev_state_str[MAX_EV_STATES] = { + [EV_STATE_NOT_ALLOCATED] = "NOT ALLOCATED", + [EV_STATE_ALLOCATED] = "ALLOCATED", +}; + +#define TO_GPI_EV_STATE_STR(_state) (((_state) >= MAX_EV_STATES) ? \ + "INVALID" : gpi_ev_state_str[(_state)]) + +enum gpi_ch_state { + DEFAULT_CH_STATE = 0x0, + CH_STATE_NOT_ALLOCATED = DEFAULT_CH_STATE, + CH_STATE_ALLOCATED = 0x1, + CH_STATE_STARTED = 0x2, + CH_STATE_STOPPED = 0x3, + CH_STATE_STOP_IN_PROC = 0x4, + CH_STATE_ERROR = 0xf, + MAX_CH_STATES +}; + +enum gpi_cmd { + GPI_CH_CMD_BEGIN, + GPI_CH_CMD_ALLOCATE = GPI_CH_CMD_BEGIN, + GPI_CH_CMD_START, + GPI_CH_CMD_STOP, + GPI_CH_CMD_RESET, + GPI_CH_CMD_DE_ALLOC, + GPI_CH_CMD_UART_SW_STALE, + GPI_CH_CMD_UART_RFR_READY, + GPI_CH_CMD_UART_RFR_NOT_READY, + GPI_CH_CMD_END = GPI_CH_CMD_UART_RFR_NOT_READY, + GPI_EV_CMD_BEGIN, + GPI_EV_CMD_ALLOCATE = GPI_EV_CMD_BEGIN, + GPI_EV_CMD_RESET, + GPI_EV_CMD_DEALLOC, + GPI_EV_CMD_END = GPI_EV_CMD_DEALLOC, + GPI_MAX_CMD, +}; + +#define IS_CHAN_CMD(_cmd) ((_cmd) <= GPI_CH_CMD_END) + +static const char *const gpi_cmd_str[GPI_MAX_CMD] = { + [GPI_CH_CMD_ALLOCATE] = "CH ALLOCATE", + [GPI_CH_CMD_START] = "CH START", + [GPI_CH_CMD_STOP] = "CH STOP", + [GPI_CH_CMD_RESET] = "CH_RESET", + [GPI_CH_CMD_DE_ALLOC] = "DE ALLOC", + [GPI_CH_CMD_UART_SW_STALE] = "UART SW STALE", + [GPI_CH_CMD_UART_RFR_READY] = "UART RFR READY", + [GPI_CH_CMD_UART_RFR_NOT_READY] = "UART RFR NOT READY", + [GPI_EV_CMD_ALLOCATE] = "EV ALLOCATE", + [GPI_EV_CMD_RESET] = "EV RESET", + [GPI_EV_CMD_DEALLOC] = "EV DEALLOC", +}; + +#define TO_GPI_CMD_STR(_cmd) (((_cmd) >= GPI_MAX_CMD) ? "INVALID" : \ + gpi_cmd_str[(_cmd)]) + +/* + * @DISABLE_STATE: no register access allowed + * @CONFIG_STATE: client has configured the channel + * @PREP_HARDWARE: register access is allowed + * however, no processing EVENTS + * @ACTIVE_STATE: channels are fully operational + * @PREPARE_TERMINATE: graceful termination of channels + * register access is allowed + * @PAUSE_STATE: channels are active, but not processing any events + */ +enum gpi_pm_state { + DISABLE_STATE, + CONFIG_STATE, + PREPARE_HARDWARE, + ACTIVE_STATE, + PREPARE_TERMINATE, + PAUSE_STATE, + MAX_PM_STATE +}; + +#define REG_ACCESS_VALID(_pm_state) ((_pm_state) >= PREPARE_HARDWARE) + +static const char *const gpi_pm_state_str[MAX_PM_STATE] = { + [DISABLE_STATE] = "DISABLE", + [CONFIG_STATE] = "CONFIG", + [PREPARE_HARDWARE] = "PREPARE HARDWARE", + [ACTIVE_STATE] = "ACTIVE", + [PREPARE_TERMINATE] = "PREPARE TERMINATE", + [PAUSE_STATE] = "PAUSE", +}; + +#define TO_GPI_PM_STR(_state) (((_state) >= MAX_PM_STATE) ? \ + "INVALID" : gpi_pm_state_str[(_state)]) + +static const struct { + enum gpi_cmd gpi_cmd; + u32 opcode; + u32 state; +} gpi_cmd_info[GPI_MAX_CMD] = { + { + GPI_CH_CMD_ALLOCATE, + GPII_n_CH_CMD_ALLOCATE, + CH_STATE_ALLOCATED, + }, + { + GPI_CH_CMD_START, + GPII_n_CH_CMD_START, + CH_STATE_STARTED, + }, + { + GPI_CH_CMD_STOP, + GPII_n_CH_CMD_STOP, + CH_STATE_STOPPED, + }, + { + GPI_CH_CMD_RESET, + GPII_n_CH_CMD_RESET, + CH_STATE_ALLOCATED, + }, + { + GPI_CH_CMD_DE_ALLOC, + GPII_n_CH_CMD_DE_ALLOC, + CH_STATE_NOT_ALLOCATED, + }, + { + GPI_CH_CMD_UART_SW_STALE, + GPII_n_CH_CMD_UART_SW_STALE, + STATE_IGNORE, + }, + { + GPI_CH_CMD_UART_RFR_READY, + GPII_n_CH_CMD_UART_RFR_READY, + STATE_IGNORE, + }, + { + GPI_CH_CMD_UART_RFR_NOT_READY, + GPII_n_CH_CMD_UART_RFR_NOT_READY, + STATE_IGNORE, + }, + { + GPI_EV_CMD_ALLOCATE, + GPII_n_EV_CH_CMD_ALLOCATE, + EV_STATE_ALLOCATED, + }, + { + GPI_EV_CMD_RESET, + GPII_n_EV_CH_CMD_RESET, + EV_STATE_ALLOCATED, + }, + { + GPI_EV_CMD_DEALLOC, + GPII_n_EV_CH_CMD_DE_ALLOC, + EV_STATE_NOT_ALLOCATED, + }, +}; + +struct gpi_ring { + void *pre_aligned; + size_t alloc_size; + phys_addr_t phys_addr; + dma_addr_t dma_handle; + void *base; + void *wp; + void *rp; + u32 len; + u32 el_size; + u32 elements; + bool configured; +}; + +struct gpi_dev { + struct dma_device dma_device; + struct device *dev; + struct resource *res; + void __iomem *regs; + void __iomem *ee_base; /*ee register base address*/ + u32 max_gpii; /* maximum # of gpii instances available per gpi block */ + u32 gpii_mask; /* gpii instances available for apps */ + u32 ev_factor; /* ev ring length factor */ + struct gpii *gpiis; +}; + +struct reg_info { + char *name; + u32 offset; + u32 val; +}; + +struct gchan { + struct virt_dma_chan vc; + u32 chid; + u32 seid; + u32 protocol; + struct gpii *gpii; + enum gpi_ch_state ch_state; + enum gpi_pm_state pm_state; + void __iomem *ch_cntxt_base_reg; + void __iomem *ch_cntxt_db_reg; + void __iomem *ch_cmd_reg; + u32 dir; + struct gpi_ring ch_ring; + void *config; +}; + +struct gpii { + u32 gpii_id; + struct gchan gchan[MAX_CHANNELS_PER_GPII]; + struct gpi_dev *gpi_dev; + int irq; + void __iomem *regs; /* points to gpi top */ + void __iomem *ev_cntxt_base_reg; + void __iomem *ev_cntxt_db_reg; + void __iomem *ev_ring_rp_lsb_reg; + void __iomem *ev_cmd_reg; + void __iomem *ieob_clr_reg; + struct mutex ctrl_lock; + enum gpi_ev_state ev_state; + bool configured_irq; + enum gpi_pm_state pm_state; + rwlock_t pm_lock; + struct gpi_ring ev_ring; + struct tasklet_struct ev_task; /* event processing tasklet */ + struct completion cmd_completion; + enum gpi_cmd gpi_cmd; + u32 cntxt_type_irq_msk; + bool ieob_set; +}; + +#define MAX_TRE 3 + +struct gpi_desc { + struct virt_dma_desc vd; + size_t len; + void *db; /* DB register to program */ + struct gchan *gchan; + struct gpi_tre tre[MAX_TRE]; + u32 num_tre; +}; + +static const u32 GPII_CHAN_DIR[MAX_CHANNELS_PER_GPII] = { + GPI_CHTYPE_DIR_OUT, GPI_CHTYPE_DIR_IN +}; + +static irqreturn_t gpi_handle_irq(int irq, void *data); +static void gpi_ring_recycle_ev_element(struct gpi_ring *ring); +static int gpi_ring_add_element(struct gpi_ring *ring, void **wp); +static void gpi_process_events(struct gpii *gpii); + +static inline struct gchan *to_gchan(struct dma_chan *dma_chan) +{ + return container_of(dma_chan, struct gchan, vc.chan); +} + +static inline struct gpi_desc *to_gpi_desc(struct virt_dma_desc *vd) +{ + return container_of(vd, struct gpi_desc, vd); +} + +static inline phys_addr_t to_physical(const struct gpi_ring *const ring, + void *addr) +{ + return ring->phys_addr + (addr - ring->base); +} + +static inline void *to_virtual(const struct gpi_ring *const ring, phys_addr_t addr) +{ + return ring->base + (addr - ring->phys_addr); +} + +static inline u32 gpi_read_reg(struct gpii *gpii, void __iomem *addr) +{ + return readl_relaxed(addr); +} + +static inline void gpi_write_reg(struct gpii *gpii, void __iomem *addr, u32 val) +{ + writel_relaxed(val, addr); +} + +/* gpi_write_reg_field - write to specific bit field */ +static inline void gpi_write_reg_field(struct gpii *gpii, void __iomem *addr, + u32 mask, u32 shift, u32 val) +{ + u32 tmp = gpi_read_reg(gpii, addr); + + tmp &= ~mask; + val = tmp | ((val << shift) & mask); + gpi_write_reg(gpii, addr, val); +} + +static inline void +gpi_update_reg(struct gpii *gpii, u32 offset, u32 mask, u32 val) +{ + void __iomem *addr = gpii->regs + offset; + u32 tmp = gpi_read_reg(gpii, addr); + + tmp &= ~mask; + tmp |= u32_encode_bits(val, mask); + + gpi_write_reg(gpii, addr, tmp); +} + +static void gpi_disable_interrupts(struct gpii *gpii) +{ + gpi_update_reg(gpii, GPII_n_CNTXT_TYPE_IRQ_MSK_OFFS(gpii->gpii_id), + GPII_n_CNTXT_TYPE_IRQ_MSK_BMSK, 0); + gpi_update_reg(gpii, GPII_n_CNTXT_SRC_IEOB_IRQ_MSK_OFFS(gpii->gpii_id), + GPII_n_CNTXT_SRC_IEOB_IRQ_MSK_BMSK, 0); + gpi_update_reg(gpii, GPII_n_CNTXT_SRC_CH_IRQ_MSK_OFFS(gpii->gpii_id), + GPII_n_CNTXT_SRC_CH_IRQ_MSK_BMSK, 0); + gpi_update_reg(gpii, GPII_n_CNTXT_SRC_EV_CH_IRQ_MSK_OFFS(gpii->gpii_id), + GPII_n_CNTXT_SRC_EV_CH_IRQ_MSK_BMSK, 0); + gpi_update_reg(gpii, GPII_n_CNTXT_GLOB_IRQ_EN_OFFS(gpii->gpii_id), + GPII_n_CNTXT_GPII_IRQ_EN_BMSK, 0); + gpi_update_reg(gpii, GPII_n_CNTXT_GPII_IRQ_EN_OFFS(gpii->gpii_id), + GPII_n_CNTXT_GPII_IRQ_EN_BMSK, 0); + gpi_update_reg(gpii, GPII_n_CNTXT_INTSET_OFFS(gpii->gpii_id), + GPII_n_CNTXT_INTSET_BMSK, 0); + + gpii->cntxt_type_irq_msk = 0; + devm_free_irq(gpii->gpi_dev->dev, gpii->irq, gpii); + gpii->configured_irq = false; +} + +/* configure and enable interrupts */ +static int gpi_config_interrupts(struct gpii *gpii, enum gpii_irq_settings settings, bool mask) +{ + const u32 enable = (GPII_n_CNTXT_TYPE_IRQ_MSK_GENERAL | + GPII_n_CNTXT_TYPE_IRQ_MSK_IEOB | + GPII_n_CNTXT_TYPE_IRQ_MSK_GLOB | + GPII_n_CNTXT_TYPE_IRQ_MSK_EV_CTRL | + GPII_n_CNTXT_TYPE_IRQ_MSK_CH_CTRL); + int ret; + + if (!gpii->configured_irq) { + ret = devm_request_irq(gpii->gpi_dev->dev, gpii->irq, + gpi_handle_irq, IRQF_TRIGGER_HIGH, + "gpi-dma", gpii); + if (ret < 0) { + dev_err(gpii->gpi_dev->dev, "error request irq:%d ret:%d\n", + gpii->irq, ret); + return ret; + } + } + + if (settings == MASK_IEOB_SETTINGS) { + /* + * GPII only uses one EV ring per gpii so we can globally + * enable/disable IEOB interrupt + */ + if (mask) + gpii->cntxt_type_irq_msk |= GPII_n_CNTXT_TYPE_IRQ_MSK_IEOB; + else + gpii->cntxt_type_irq_msk &= ~(GPII_n_CNTXT_TYPE_IRQ_MSK_IEOB); + gpi_update_reg(gpii, GPII_n_CNTXT_TYPE_IRQ_MSK_OFFS(gpii->gpii_id), + GPII_n_CNTXT_TYPE_IRQ_MSK_BMSK, gpii->cntxt_type_irq_msk); + } else { + gpi_update_reg(gpii, GPII_n_CNTXT_TYPE_IRQ_MSK_OFFS(gpii->gpii_id), + GPII_n_CNTXT_TYPE_IRQ_MSK_BMSK, enable); + gpi_update_reg(gpii, GPII_n_CNTXT_SRC_IEOB_IRQ_MSK_OFFS(gpii->gpii_id), + GPII_n_CNTXT_SRC_IEOB_IRQ_MSK_BMSK, + GPII_n_CNTXT_SRC_IEOB_IRQ_MSK_BMSK); + gpi_update_reg(gpii, GPII_n_CNTXT_SRC_CH_IRQ_MSK_OFFS(gpii->gpii_id), + GPII_n_CNTXT_SRC_CH_IRQ_MSK_BMSK, + GPII_n_CNTXT_SRC_CH_IRQ_MSK_BMSK); + gpi_update_reg(gpii, GPII_n_CNTXT_SRC_EV_CH_IRQ_MSK_OFFS(gpii->gpii_id), + GPII_n_CNTXT_SRC_EV_CH_IRQ_MSK_BMSK, + GPII_n_CNTXT_SRC_EV_CH_IRQ_MSK_BMSK); + gpi_update_reg(gpii, GPII_n_CNTXT_GLOB_IRQ_EN_OFFS(gpii->gpii_id), + GPII_n_CNTXT_GPII_IRQ_EN_BMSK, + GPII_n_CNTXT_GPII_IRQ_EN_BMSK); + gpi_update_reg(gpii, GPII_n_CNTXT_GPII_IRQ_EN_OFFS(gpii->gpii_id), + GPII_n_CNTXT_GPII_IRQ_EN_BMSK, GPII_n_CNTXT_GPII_IRQ_EN_BMSK); + gpi_update_reg(gpii, GPII_n_CNTXT_MSI_BASE_LSB_OFFS(gpii->gpii_id), U32_MAX, 0); + gpi_update_reg(gpii, GPII_n_CNTXT_MSI_BASE_MSB_OFFS(gpii->gpii_id), U32_MAX, 0); + gpi_update_reg(gpii, GPII_n_CNTXT_SCRATCH_0_OFFS(gpii->gpii_id), U32_MAX, 0); + gpi_update_reg(gpii, GPII_n_CNTXT_SCRATCH_1_OFFS(gpii->gpii_id), U32_MAX, 0); + gpi_update_reg(gpii, GPII_n_CNTXT_INTSET_OFFS(gpii->gpii_id), + GPII_n_CNTXT_INTSET_BMSK, 1); + gpi_update_reg(gpii, GPII_n_ERROR_LOG_OFFS(gpii->gpii_id), U32_MAX, 0); + + gpii->cntxt_type_irq_msk = enable; + } + + gpii->configured_irq = true; + return 0; +} + +/* Sends gpii event or channel command */ +static int gpi_send_cmd(struct gpii *gpii, struct gchan *gchan, + enum gpi_cmd gpi_cmd) +{ + u32 chid = MAX_CHANNELS_PER_GPII; + unsigned long timeout; + void __iomem *cmd_reg; + u32 cmd; + + if (gpi_cmd >= GPI_MAX_CMD) + return -EINVAL; + if (IS_CHAN_CMD(gpi_cmd)) + chid = gchan->chid; + + dev_dbg(gpii->gpi_dev->dev, + "sending cmd: %s:%u\n", TO_GPI_CMD_STR(gpi_cmd), chid); + + /* send opcode and wait for completion */ + reinit_completion(&gpii->cmd_completion); + gpii->gpi_cmd = gpi_cmd; + + cmd_reg = IS_CHAN_CMD(gpi_cmd) ? gchan->ch_cmd_reg : gpii->ev_cmd_reg; + cmd = IS_CHAN_CMD(gpi_cmd) ? GPII_n_CH_CMD(gpi_cmd_info[gpi_cmd].opcode, chid) : + GPII_n_EV_CMD(gpi_cmd_info[gpi_cmd].opcode, 0); + gpi_write_reg(gpii, cmd_reg, cmd); + timeout = wait_for_completion_timeout(&gpii->cmd_completion, + msecs_to_jiffies(CMD_TIMEOUT_MS)); + if (!timeout) { + dev_err(gpii->gpi_dev->dev, "cmd: %s completion timeout:%u\n", + TO_GPI_CMD_STR(gpi_cmd), chid); + return -EIO; + } + + /* confirm new ch state is correct , if the cmd is a state change cmd */ + if (gpi_cmd_info[gpi_cmd].state == STATE_IGNORE) + return 0; + + if (IS_CHAN_CMD(gpi_cmd) && gchan->ch_state == gpi_cmd_info[gpi_cmd].state) + return 0; + + if (!IS_CHAN_CMD(gpi_cmd) && gpii->ev_state == gpi_cmd_info[gpi_cmd].state) + return 0; + + return -EIO; +} + +/* program transfer ring DB register */ +static inline void gpi_write_ch_db(struct gchan *gchan, + struct gpi_ring *ring, void *wp) +{ + struct gpii *gpii = gchan->gpii; + phys_addr_t p_wp; + + p_wp = to_physical(ring, wp); + gpi_write_reg(gpii, gchan->ch_cntxt_db_reg, p_wp); +} + +/* program event ring DB register */ +static inline void gpi_write_ev_db(struct gpii *gpii, + struct gpi_ring *ring, void *wp) +{ + phys_addr_t p_wp; + + p_wp = ring->phys_addr + (wp - ring->base); + gpi_write_reg(gpii, gpii->ev_cntxt_db_reg, p_wp); +} + +/* process transfer completion interrupt */ +static void gpi_process_ieob(struct gpii *gpii) +{ + gpi_write_reg(gpii, gpii->ieob_clr_reg, BIT(0)); + + gpi_config_interrupts(gpii, MASK_IEOB_SETTINGS, 0); + tasklet_hi_schedule(&gpii->ev_task); +} + +/* process channel control interrupt */ +static void gpi_process_ch_ctrl_irq(struct gpii *gpii) +{ + u32 gpii_id = gpii->gpii_id; + u32 offset = GPII_n_CNTXT_SRC_GPII_CH_IRQ_OFFS(gpii_id); + u32 ch_irq = gpi_read_reg(gpii, gpii->regs + offset); + struct gchan *gchan; + u32 chid, state; + + /* clear the status */ + offset = GPII_n_CNTXT_SRC_CH_IRQ_CLR_OFFS(gpii_id); + gpi_write_reg(gpii, gpii->regs + offset, (u32)ch_irq); + + for (chid = 0; chid < MAX_CHANNELS_PER_GPII; chid++) { + if (!(BIT(chid) & ch_irq)) + continue; + + gchan = &gpii->gchan[chid]; + state = gpi_read_reg(gpii, gchan->ch_cntxt_base_reg + + CNTXT_0_CONFIG); + state = FIELD_GET(GPII_n_CH_k_CNTXT_0_CHSTATE, state); + + /* + * CH_CMD_DEALLOC cmd always successful. However cmd does + * not change hardware status. So overwriting software state + * to default state. + */ + if (gpii->gpi_cmd == GPI_CH_CMD_DE_ALLOC) + state = DEFAULT_CH_STATE; + gchan->ch_state = state; + + /* + * Triggering complete all if ch_state is not a stop in process. + * Stop in process is a transition state and we will wait for + * stop interrupt before notifying. + */ + if (gchan->ch_state != CH_STATE_STOP_IN_PROC) + complete_all(&gpii->cmd_completion); + } +} + +/* processing gpi general error interrupts */ +static void gpi_process_gen_err_irq(struct gpii *gpii) +{ + u32 gpii_id = gpii->gpii_id; + u32 offset = GPII_n_CNTXT_GPII_IRQ_STTS_OFFS(gpii_id); + u32 irq_stts = gpi_read_reg(gpii, gpii->regs + offset); + + /* clear the status */ + dev_dbg(gpii->gpi_dev->dev, "irq_stts:0x%x\n", irq_stts); + + /* Clear the register */ + offset = GPII_n_CNTXT_GPII_IRQ_CLR_OFFS(gpii_id); + gpi_write_reg(gpii, gpii->regs + offset, irq_stts); +} + +/* processing gpi level error interrupts */ +static void gpi_process_glob_err_irq(struct gpii *gpii) +{ + u32 gpii_id = gpii->gpii_id; + u32 offset = GPII_n_CNTXT_GLOB_IRQ_STTS_OFFS(gpii_id); + u32 irq_stts = gpi_read_reg(gpii, gpii->regs + offset); + + offset = GPII_n_CNTXT_GLOB_IRQ_CLR_OFFS(gpii_id); + gpi_write_reg(gpii, gpii->regs + offset, irq_stts); + + /* only error interrupt should be set */ + if (irq_stts & ~GPI_GLOB_IRQ_ERROR_INT_MSK) { + dev_err(gpii->gpi_dev->dev, "invalid error status:0x%x\n", irq_stts); + return; + } + + offset = GPII_n_ERROR_LOG_OFFS(gpii_id); + gpi_write_reg(gpii, gpii->regs + offset, 0); +} + +/* gpii interrupt handler */ +static irqreturn_t gpi_handle_irq(int irq, void *data) +{ + struct gpii *gpii = data; + u32 gpii_id = gpii->gpii_id; + u32 type, offset; + unsigned long flags; + + read_lock_irqsave(&gpii->pm_lock, flags); + + /* + * States are out of sync to receive interrupt + * while software state is in DISABLE state, bailing out. + */ + if (!REG_ACCESS_VALID(gpii->pm_state)) { + dev_err(gpii->gpi_dev->dev, "receive interrupt while in %s state\n", + TO_GPI_PM_STR(gpii->pm_state)); + goto exit_irq; + } + + offset = GPII_n_CNTXT_TYPE_IRQ_OFFS(gpii->gpii_id); + type = gpi_read_reg(gpii, gpii->regs + offset); + + do { + /* global gpii error */ + if (type & GPII_n_CNTXT_TYPE_IRQ_MSK_GLOB) { + gpi_process_glob_err_irq(gpii); + type &= ~(GPII_n_CNTXT_TYPE_IRQ_MSK_GLOB); + } + + /* transfer complete interrupt */ + if (type & GPII_n_CNTXT_TYPE_IRQ_MSK_IEOB) { + gpi_process_ieob(gpii); + type &= ~GPII_n_CNTXT_TYPE_IRQ_MSK_IEOB; + } + + /* event control irq */ + if (type & GPII_n_CNTXT_TYPE_IRQ_MSK_EV_CTRL) { + u32 ev_state; + u32 ev_ch_irq; + + dev_dbg(gpii->gpi_dev->dev, + "processing EV CTRL interrupt\n"); + offset = GPII_n_CNTXT_SRC_EV_CH_IRQ_OFFS(gpii_id); + ev_ch_irq = gpi_read_reg(gpii, gpii->regs + offset); + + offset = GPII_n_CNTXT_SRC_EV_CH_IRQ_CLR_OFFS + (gpii_id); + gpi_write_reg(gpii, gpii->regs + offset, ev_ch_irq); + ev_state = gpi_read_reg(gpii, gpii->ev_cntxt_base_reg + + CNTXT_0_CONFIG); + ev_state = FIELD_GET(GPII_n_EV_k_CNTXT_0_CHSTATE, ev_state); + + /* + * CMD EV_CMD_DEALLOC is always successful. However + * cmd does not change hardware status. So overwriting + * software state to default state. + */ + if (gpii->gpi_cmd == GPI_EV_CMD_DEALLOC) + ev_state = DEFAULT_EV_CH_STATE; + + gpii->ev_state = ev_state; + dev_dbg(gpii->gpi_dev->dev, "setting EV state to %s\n", + TO_GPI_EV_STATE_STR(gpii->ev_state)); + complete_all(&gpii->cmd_completion); + type &= ~(GPII_n_CNTXT_TYPE_IRQ_MSK_EV_CTRL); + } + + /* channel control irq */ + if (type & GPII_n_CNTXT_TYPE_IRQ_MSK_CH_CTRL) { + dev_dbg(gpii->gpi_dev->dev, "process CH CTRL interrupts\n"); + gpi_process_ch_ctrl_irq(gpii); + type &= ~(GPII_n_CNTXT_TYPE_IRQ_MSK_CH_CTRL); + } + + if (type) { + dev_err(gpii->gpi_dev->dev, "Unhandled interrupt status:0x%x\n", type); + gpi_process_gen_err_irq(gpii); + goto exit_irq; + } + + offset = GPII_n_CNTXT_TYPE_IRQ_OFFS(gpii->gpii_id); + type = gpi_read_reg(gpii, gpii->regs + offset); + } while (type); + +exit_irq: + read_unlock_irqrestore(&gpii->pm_lock, flags); + + return IRQ_HANDLED; +} + +/* process DMA Immediate completion data events */ +static void gpi_process_imed_data_event(struct gchan *gchan, + struct immediate_data_event *imed_event) +{ + struct gpii *gpii = gchan->gpii; + struct gpi_ring *ch_ring = &gchan->ch_ring; + void *tre = ch_ring->base + (ch_ring->el_size * imed_event->tre_index); + struct dmaengine_result result; + struct gpi_desc *gpi_desc; + struct virt_dma_desc *vd; + unsigned long flags; + u32 chid; + + /* + * If channel not active don't process event + */ + if (gchan->pm_state != ACTIVE_STATE) { + dev_err(gpii->gpi_dev->dev, "skipping processing event because ch @ %s state\n", + TO_GPI_PM_STR(gchan->pm_state)); + return; + } + + spin_lock_irqsave(&gchan->vc.lock, flags); + vd = vchan_next_desc(&gchan->vc); + if (!vd) { + struct gpi_ere *gpi_ere; + struct gpi_tre *gpi_tre; + + spin_unlock_irqrestore(&gchan->vc.lock, flags); + dev_dbg(gpii->gpi_dev->dev, "event without a pending descriptor!\n"); + gpi_ere = (struct gpi_ere *)imed_event; + dev_dbg(gpii->gpi_dev->dev, + "Event: %08x %08x %08x %08x\n", + gpi_ere->dword[0], gpi_ere->dword[1], + gpi_ere->dword[2], gpi_ere->dword[3]); + gpi_tre = tre; + dev_dbg(gpii->gpi_dev->dev, + "Pending TRE: %08x %08x %08x %08x\n", + gpi_tre->dword[0], gpi_tre->dword[1], + gpi_tre->dword[2], gpi_tre->dword[3]); + return; + } + gpi_desc = to_gpi_desc(vd); + spin_unlock_irqrestore(&gchan->vc.lock, flags); + + /* + * RP pointed by Event is to last TRE processed, + * we need to update ring rp to tre + 1 + */ + tre += ch_ring->el_size; + if (tre >= (ch_ring->base + ch_ring->len)) + tre = ch_ring->base; + ch_ring->rp = tre; + + /* make sure rp updates are immediately visible to all cores */ + smp_wmb(); + + chid = imed_event->chid; + if (imed_event->code == MSM_GPI_TCE_EOT && gpii->ieob_set) { + if (chid == GPI_RX_CHAN) + goto gpi_free_desc; + else + return; + } + + if (imed_event->code == MSM_GPI_TCE_UNEXP_ERR) + result.result = DMA_TRANS_ABORTED; + else + result.result = DMA_TRANS_NOERROR; + result.residue = gpi_desc->len - imed_event->length; + + dma_cookie_complete(&vd->tx); + dmaengine_desc_get_callback_invoke(&vd->tx, &result); + +gpi_free_desc: + spin_lock_irqsave(&gchan->vc.lock, flags); + list_del(&vd->node); + spin_unlock_irqrestore(&gchan->vc.lock, flags); + kfree(gpi_desc); + gpi_desc = NULL; +} + +/* processing transfer completion events */ +static void gpi_process_xfer_compl_event(struct gchan *gchan, + struct xfer_compl_event *compl_event) +{ + struct gpii *gpii = gchan->gpii; + struct gpi_ring *ch_ring = &gchan->ch_ring; + void *ev_rp = to_virtual(ch_ring, compl_event->ptr); + struct virt_dma_desc *vd; + struct gpi_desc *gpi_desc; + struct dmaengine_result result; + unsigned long flags; + u32 chid; + + /* only process events on active channel */ + if (unlikely(gchan->pm_state != ACTIVE_STATE)) { + dev_err(gpii->gpi_dev->dev, "skipping processing event because ch @ %s state\n", + TO_GPI_PM_STR(gchan->pm_state)); + return; + } + + spin_lock_irqsave(&gchan->vc.lock, flags); + vd = vchan_next_desc(&gchan->vc); + if (!vd) { + struct gpi_ere *gpi_ere; + + spin_unlock_irqrestore(&gchan->vc.lock, flags); + dev_err(gpii->gpi_dev->dev, "Event without a pending descriptor!\n"); + gpi_ere = (struct gpi_ere *)compl_event; + dev_err(gpii->gpi_dev->dev, + "Event: %08x %08x %08x %08x\n", + gpi_ere->dword[0], gpi_ere->dword[1], + gpi_ere->dword[2], gpi_ere->dword[3]); + return; + } + + gpi_desc = to_gpi_desc(vd); + spin_unlock_irqrestore(&gchan->vc.lock, flags); + + /* + * RP pointed by Event is to last TRE processed, + * we need to update ring rp to ev_rp + 1 + */ + ev_rp += ch_ring->el_size; + if (ev_rp >= (ch_ring->base + ch_ring->len)) + ev_rp = ch_ring->base; + ch_ring->rp = ev_rp; + + /* update must be visible to other cores */ + smp_wmb(); + + chid = compl_event->chid; + if (compl_event->code == MSM_GPI_TCE_EOT && gpii->ieob_set) { + if (chid == GPI_RX_CHAN) + goto gpi_free_desc; + else + return; + } + + if (compl_event->code == MSM_GPI_TCE_UNEXP_ERR) { + dev_err(gpii->gpi_dev->dev, "Error in Transaction\n"); + result.result = DMA_TRANS_ABORTED; + } else { + dev_dbg(gpii->gpi_dev->dev, "Transaction Success\n"); + result.result = DMA_TRANS_NOERROR; + } + result.residue = gpi_desc->len - compl_event->length; + dev_dbg(gpii->gpi_dev->dev, "Residue %d\n", result.residue); + + dma_cookie_complete(&vd->tx); + dmaengine_desc_get_callback_invoke(&vd->tx, &result); + +gpi_free_desc: + spin_lock_irqsave(&gchan->vc.lock, flags); + list_del(&vd->node); + spin_unlock_irqrestore(&gchan->vc.lock, flags); + kfree(gpi_desc); + gpi_desc = NULL; +} + +/* process all events */ +static void gpi_process_events(struct gpii *gpii) +{ + struct gpi_ring *ev_ring = &gpii->ev_ring; + phys_addr_t cntxt_rp; + void *rp; + union gpi_event *gpi_event; + struct gchan *gchan; + u32 chid, type; + + cntxt_rp = gpi_read_reg(gpii, gpii->ev_ring_rp_lsb_reg); + rp = to_virtual(ev_ring, cntxt_rp); + + do { + while (rp != ev_ring->rp) { + gpi_event = ev_ring->rp; + chid = gpi_event->xfer_compl_event.chid; + type = gpi_event->xfer_compl_event.type; + + dev_dbg(gpii->gpi_dev->dev, + "Event: CHID:%u, type:%x %08x %08x %08x %08x\n", + chid, type, gpi_event->gpi_ere.dword[0], + gpi_event->gpi_ere.dword[1], gpi_event->gpi_ere.dword[2], + gpi_event->gpi_ere.dword[3]); + + switch (type) { + case XFER_COMPLETE_EV_TYPE: + gchan = &gpii->gchan[chid]; + gpi_process_xfer_compl_event(gchan, + &gpi_event->xfer_compl_event); + break; + case STALE_EV_TYPE: + dev_dbg(gpii->gpi_dev->dev, "stale event, not processing\n"); + break; + case IMMEDIATE_DATA_EV_TYPE: + gchan = &gpii->gchan[chid]; + gpi_process_imed_data_event(gchan, + &gpi_event->immediate_data_event); + break; + case QUP_NOTIF_EV_TYPE: + dev_dbg(gpii->gpi_dev->dev, "QUP_NOTIF_EV_TYPE\n"); + break; + default: + dev_dbg(gpii->gpi_dev->dev, + "not supported event type:0x%x\n", type); + } + gpi_ring_recycle_ev_element(ev_ring); + } + gpi_write_ev_db(gpii, ev_ring, ev_ring->wp); + + /* clear pending IEOB events */ + gpi_write_reg(gpii, gpii->ieob_clr_reg, BIT(0)); + + cntxt_rp = gpi_read_reg(gpii, gpii->ev_ring_rp_lsb_reg); + rp = to_virtual(ev_ring, cntxt_rp); + + } while (rp != ev_ring->rp); +} + +/* processing events using tasklet */ +static void gpi_ev_tasklet(unsigned long data) +{ + struct gpii *gpii = (struct gpii *)data; + + read_lock_bh(&gpii->pm_lock); + if (!REG_ACCESS_VALID(gpii->pm_state)) { + read_unlock_bh(&gpii->pm_lock); + dev_err(gpii->gpi_dev->dev, "not processing any events, pm_state:%s\n", + TO_GPI_PM_STR(gpii->pm_state)); + return; + } + + /* process the events */ + gpi_process_events(gpii); + + /* enable IEOB, switching back to interrupts */ + gpi_config_interrupts(gpii, MASK_IEOB_SETTINGS, 1); + read_unlock_bh(&gpii->pm_lock); +} + +/* marks all pending events for the channel as stale */ +static void gpi_mark_stale_events(struct gchan *gchan) +{ + struct gpii *gpii = gchan->gpii; + struct gpi_ring *ev_ring = &gpii->ev_ring; + u32 cntxt_rp, local_rp; + void *ev_rp; + + cntxt_rp = gpi_read_reg(gpii, gpii->ev_ring_rp_lsb_reg); + + ev_rp = ev_ring->rp; + local_rp = (u32)to_physical(ev_ring, ev_rp); + while (local_rp != cntxt_rp) { + union gpi_event *gpi_event = ev_rp; + u32 chid = gpi_event->xfer_compl_event.chid; + + if (chid == gchan->chid) + gpi_event->xfer_compl_event.type = STALE_EV_TYPE; + ev_rp += ev_ring->el_size; + if (ev_rp >= (ev_ring->base + ev_ring->len)) + ev_rp = ev_ring->base; + cntxt_rp = gpi_read_reg(gpii, gpii->ev_ring_rp_lsb_reg); + local_rp = (u32)to_physical(ev_ring, ev_rp); + } +} + +/* reset sw state and issue channel reset or de-alloc */ +static int gpi_reset_chan(struct gchan *gchan, enum gpi_cmd gpi_cmd) +{ + struct gpii *gpii = gchan->gpii; + struct gpi_ring *ch_ring = &gchan->ch_ring; + unsigned long flags; + LIST_HEAD(list); + int ret; + + ret = gpi_send_cmd(gpii, gchan, gpi_cmd); + if (ret) { + dev_err(gpii->gpi_dev->dev, "Error with cmd:%s ret:%d\n", + TO_GPI_CMD_STR(gpi_cmd), ret); + return ret; + } + + /* initialize the local ring ptrs */ + ch_ring->rp = ch_ring->base; + ch_ring->wp = ch_ring->base; + + /* visible to other cores */ + smp_wmb(); + + /* check event ring for any stale events */ + write_lock_irq(&gpii->pm_lock); + gpi_mark_stale_events(gchan); + + /* remove all async descriptors */ + spin_lock_irqsave(&gchan->vc.lock, flags); + vchan_get_all_descriptors(&gchan->vc, &list); + spin_unlock_irqrestore(&gchan->vc.lock, flags); + write_unlock_irq(&gpii->pm_lock); + vchan_dma_desc_free_list(&gchan->vc, &list); + + return 0; +} + +static int gpi_start_chan(struct gchan *gchan) +{ + struct gpii *gpii = gchan->gpii; + int ret; + + ret = gpi_send_cmd(gpii, gchan, GPI_CH_CMD_START); + if (ret) { + dev_err(gpii->gpi_dev->dev, "Error with cmd:%s ret:%d\n", + TO_GPI_CMD_STR(GPI_CH_CMD_START), ret); + return ret; + } + + /* gpii CH is active now */ + write_lock_irq(&gpii->pm_lock); + gchan->pm_state = ACTIVE_STATE; + write_unlock_irq(&gpii->pm_lock); + + return 0; +} + +static int gpi_stop_chan(struct gchan *gchan) +{ + struct gpii *gpii = gchan->gpii; + int ret; + + ret = gpi_send_cmd(gpii, gchan, GPI_CH_CMD_STOP); + if (ret) { + dev_err(gpii->gpi_dev->dev, "Error with cmd:%s ret:%d\n", + TO_GPI_CMD_STR(GPI_CH_CMD_STOP), ret); + return ret; + } + + return 0; +} + +/* allocate and configure the transfer channel */ +static int gpi_alloc_chan(struct gchan *chan, bool send_alloc_cmd) +{ + struct gpii *gpii = chan->gpii; + struct gpi_ring *ring = &chan->ch_ring; + int ret; + u32 id = gpii->gpii_id; + u32 chid = chan->chid; + u32 pair_chid = !chid; + + if (send_alloc_cmd) { + ret = gpi_send_cmd(gpii, chan, GPI_CH_CMD_ALLOCATE); + if (ret) { + dev_err(gpii->gpi_dev->dev, "Error with cmd:%s ret:%d\n", + TO_GPI_CMD_STR(GPI_CH_CMD_ALLOCATE), ret); + return ret; + } + } + + gpi_write_reg(gpii, chan->ch_cntxt_base_reg + CNTXT_0_CONFIG, + GPII_n_CH_k_CNTXT_0(ring->el_size, 0, chan->dir, GPI_CHTYPE_PROTO_GPI)); + gpi_write_reg(gpii, chan->ch_cntxt_base_reg + CNTXT_1_R_LENGTH, ring->len); + gpi_write_reg(gpii, chan->ch_cntxt_base_reg + CNTXT_2_RING_BASE_LSB, ring->phys_addr); + gpi_write_reg(gpii, chan->ch_cntxt_base_reg + CNTXT_3_RING_BASE_MSB, + upper_32_bits(ring->phys_addr)); + gpi_write_reg(gpii, chan->ch_cntxt_db_reg + CNTXT_5_RING_RP_MSB - CNTXT_4_RING_RP_LSB, + upper_32_bits(ring->phys_addr)); + gpi_write_reg(gpii, gpii->regs + GPII_n_CH_k_SCRATCH_0_OFFS(id, chid), + GPII_n_CH_k_SCRATCH_0(pair_chid, chan->protocol, chan->seid)); + gpi_write_reg(gpii, gpii->regs + GPII_n_CH_k_SCRATCH_1_OFFS(id, chid), 0); + gpi_write_reg(gpii, gpii->regs + GPII_n_CH_k_SCRATCH_2_OFFS(id, chid), 0); + gpi_write_reg(gpii, gpii->regs + GPII_n_CH_k_SCRATCH_3_OFFS(id, chid), 0); + gpi_write_reg(gpii, gpii->regs + GPII_n_CH_k_QOS_OFFS(id, chid), 1); + + /* flush all the writes */ + wmb(); + return 0; +} + +/* allocate and configure event ring */ +static int gpi_alloc_ev_chan(struct gpii *gpii) +{ + struct gpi_ring *ring = &gpii->ev_ring; + void __iomem *base = gpii->ev_cntxt_base_reg; + int ret; + + ret = gpi_send_cmd(gpii, NULL, GPI_EV_CMD_ALLOCATE); + if (ret) { + dev_err(gpii->gpi_dev->dev, "error with cmd:%s ret:%d\n", + TO_GPI_CMD_STR(GPI_EV_CMD_ALLOCATE), ret); + return ret; + } + + /* program event context */ + gpi_write_reg(gpii, base + CNTXT_0_CONFIG, + GPII_n_EV_k_CNTXT_0(ring->el_size, GPI_INTTYPE_IRQ, GPI_CHTYPE_GPI_EV)); + gpi_write_reg(gpii, base + CNTXT_1_R_LENGTH, ring->len); + gpi_write_reg(gpii, base + CNTXT_2_RING_BASE_LSB, lower_32_bits(ring->phys_addr)); + gpi_write_reg(gpii, base + CNTXT_3_RING_BASE_MSB, upper_32_bits(ring->phys_addr)); + gpi_write_reg(gpii, gpii->ev_cntxt_db_reg + CNTXT_5_RING_RP_MSB - CNTXT_4_RING_RP_LSB, + upper_32_bits(ring->phys_addr)); + gpi_write_reg(gpii, base + CNTXT_8_RING_INT_MOD, 0); + gpi_write_reg(gpii, base + CNTXT_10_RING_MSI_LSB, 0); + gpi_write_reg(gpii, base + CNTXT_11_RING_MSI_MSB, 0); + gpi_write_reg(gpii, base + CNTXT_8_RING_INT_MOD, 0); + gpi_write_reg(gpii, base + CNTXT_12_RING_RP_UPDATE_LSB, 0); + gpi_write_reg(gpii, base + CNTXT_13_RING_RP_UPDATE_MSB, 0); + + /* add events to ring */ + ring->wp = (ring->base + ring->len - ring->el_size); + + /* flush all the writes */ + wmb(); + + /* gpii is active now */ + write_lock_irq(&gpii->pm_lock); + gpii->pm_state = ACTIVE_STATE; + write_unlock_irq(&gpii->pm_lock); + gpi_write_ev_db(gpii, ring, ring->wp); + + return 0; +} + +/* calculate # of ERE/TRE available to queue */ +static int gpi_ring_num_elements_avail(const struct gpi_ring * const ring) +{ + int elements = 0; + + if (ring->wp < ring->rp) { + elements = ((ring->rp - ring->wp) / ring->el_size) - 1; + } else { + elements = (ring->rp - ring->base) / ring->el_size; + elements += ((ring->base + ring->len - ring->wp) / ring->el_size) - 1; + } + + return elements; +} + +static int gpi_ring_add_element(struct gpi_ring *ring, void **wp) +{ + if (gpi_ring_num_elements_avail(ring) <= 0) + return -ENOMEM; + + *wp = ring->wp; + ring->wp += ring->el_size; + if (ring->wp >= (ring->base + ring->len)) + ring->wp = ring->base; + + /* visible to other cores */ + smp_wmb(); + + return 0; +} + +static void gpi_ring_recycle_ev_element(struct gpi_ring *ring) +{ + /* Update the WP */ + ring->wp += ring->el_size; + if (ring->wp >= (ring->base + ring->len)) + ring->wp = ring->base; + + /* Update the RP */ + ring->rp += ring->el_size; + if (ring->rp >= (ring->base + ring->len)) + ring->rp = ring->base; + + /* visible to other cores */ + smp_wmb(); +} + +static void gpi_free_ring(struct gpi_ring *ring, + struct gpii *gpii) +{ + dma_free_coherent(gpii->gpi_dev->dev, ring->alloc_size, + ring->pre_aligned, ring->dma_handle); + memset(ring, 0, sizeof(*ring)); +} + +/* allocate memory for transfer and event rings */ +static int gpi_alloc_ring(struct gpi_ring *ring, u32 elements, + u32 el_size, struct gpii *gpii) +{ + u64 len = elements * el_size; + int bit; + + /* ring len must be power of 2 */ + bit = find_last_bit((unsigned long *)&len, 32); + if (((1 << bit) - 1) & len) + bit++; + len = 1 << bit; + ring->alloc_size = (len + (len - 1)); + dev_dbg(gpii->gpi_dev->dev, + "#el:%u el_size:%u len:%u actual_len:%llu alloc_size:%lu\n", + elements, el_size, (elements * el_size), len, + ring->alloc_size); + + ring->pre_aligned = dma_alloc_coherent(gpii->gpi_dev->dev, + ring->alloc_size, + &ring->dma_handle, GFP_KERNEL); + if (!ring->pre_aligned) { + dev_err(gpii->gpi_dev->dev, "could not alloc size:%lu mem for ring\n", + ring->alloc_size); + return -ENOMEM; + } + + /* align the physical mem */ + ring->phys_addr = (ring->dma_handle + (len - 1)) & ~(len - 1); + ring->base = ring->pre_aligned + (ring->phys_addr - ring->dma_handle); + ring->rp = ring->base; + ring->wp = ring->base; + ring->len = len; + ring->el_size = el_size; + ring->elements = ring->len / ring->el_size; + memset(ring->base, 0, ring->len); + ring->configured = true; + + /* update to other cores */ + smp_wmb(); + + dev_dbg(gpii->gpi_dev->dev, + "phy_pre:0x%0llx phy_alig:0x%0llx len:%u el_size:%u elements:%u\n", + ring->dma_handle, ring->phys_addr, ring->len, + ring->el_size, ring->elements); + + return 0; +} + +/* copy tre into transfer ring */ +static void gpi_queue_xfer(struct gpii *gpii, struct gchan *gchan, + struct gpi_tre *gpi_tre, void **wp) +{ + struct gpi_tre *ch_tre; + int ret; + + /* get next tre location we can copy */ + ret = gpi_ring_add_element(&gchan->ch_ring, (void **)&ch_tre); + if (unlikely(ret)) { + dev_err(gpii->gpi_dev->dev, "Error adding ring element to xfer ring\n"); + return; + } + + /* copy the tre info */ + memcpy(ch_tre, gpi_tre, sizeof(*ch_tre)); + *wp = ch_tre; +} + +/* reset and restart transfer channel */ +static int gpi_terminate_all(struct dma_chan *chan) +{ + struct gchan *gchan = to_gchan(chan); + struct gpii *gpii = gchan->gpii; + int schid, echid, i; + int ret = 0; + + mutex_lock(&gpii->ctrl_lock); + + /* + * treat both channels as a group if its protocol is not UART + * STOP, RESET, or START needs to be in lockstep + */ + schid = (gchan->protocol == QCOM_GPI_UART) ? gchan->chid : 0; + echid = (gchan->protocol == QCOM_GPI_UART) ? schid + 1 : MAX_CHANNELS_PER_GPII; + + /* stop the channel */ + for (i = schid; i < echid; i++) { + gchan = &gpii->gchan[i]; + + /* disable ch state so no more TRE processing */ + write_lock_irq(&gpii->pm_lock); + gchan->pm_state = PREPARE_TERMINATE; + write_unlock_irq(&gpii->pm_lock); + + /* send command to Stop the channel */ + ret = gpi_stop_chan(gchan); + } + + /* reset the channels (clears any pending tre) */ + for (i = schid; i < echid; i++) { + gchan = &gpii->gchan[i]; + + ret = gpi_reset_chan(gchan, GPI_CH_CMD_RESET); + if (ret) { + dev_err(gpii->gpi_dev->dev, "Error resetting channel ret:%d\n", ret); + goto terminate_exit; + } + + /* reprogram channel CNTXT */ + ret = gpi_alloc_chan(gchan, false); + if (ret) { + dev_err(gpii->gpi_dev->dev, "Error alloc_channel ret:%d\n", ret); + goto terminate_exit; + } + } + + /* restart the channels */ + for (i = schid; i < echid; i++) { + gchan = &gpii->gchan[i]; + + ret = gpi_start_chan(gchan); + if (ret) { + dev_err(gpii->gpi_dev->dev, "Error Starting Channel ret:%d\n", ret); + goto terminate_exit; + } + } + +terminate_exit: + mutex_unlock(&gpii->ctrl_lock); + return ret; +} + +/* pause dma transfer for all channels */ +static int gpi_pause(struct dma_chan *chan) +{ + struct gchan *gchan = to_gchan(chan); + struct gpii *gpii = gchan->gpii; + int i, ret; + + mutex_lock(&gpii->ctrl_lock); + + /* + * pause/resume are per gpii not per channel, so + * client needs to call pause only once + */ + if (gpii->pm_state == PAUSE_STATE) { + dev_dbg(gpii->gpi_dev->dev, "channel is already paused\n"); + mutex_unlock(&gpii->ctrl_lock); + return 0; + } + + /* send stop command to stop the channels */ + for (i = 0; i < MAX_CHANNELS_PER_GPII; i++) { + ret = gpi_stop_chan(&gpii->gchan[i]); + if (ret) { + mutex_unlock(&gpii->ctrl_lock); + return ret; + } + } + + disable_irq(gpii->irq); + + /* Wait for threads to complete out */ + tasklet_kill(&gpii->ev_task); + + write_lock_irq(&gpii->pm_lock); + gpii->pm_state = PAUSE_STATE; + write_unlock_irq(&gpii->pm_lock); + mutex_unlock(&gpii->ctrl_lock); + + return 0; +} + +/* resume dma transfer */ +static int gpi_resume(struct dma_chan *chan) +{ + struct gchan *gchan = to_gchan(chan); + struct gpii *gpii = gchan->gpii; + int i, ret; + + mutex_lock(&gpii->ctrl_lock); + if (gpii->pm_state == ACTIVE_STATE) { + dev_dbg(gpii->gpi_dev->dev, "channel is already active\n"); + mutex_unlock(&gpii->ctrl_lock); + return 0; + } + + enable_irq(gpii->irq); + + /* send start command to start the channels */ + for (i = 0; i < MAX_CHANNELS_PER_GPII; i++) { + ret = gpi_send_cmd(gpii, &gpii->gchan[i], GPI_CH_CMD_START); + if (ret) { + dev_err(gpii->gpi_dev->dev, "Error starting chan, ret:%d\n", ret); + mutex_unlock(&gpii->ctrl_lock); + return ret; + } + } + + write_lock_irq(&gpii->pm_lock); + gpii->pm_state = ACTIVE_STATE; + write_unlock_irq(&gpii->pm_lock); + mutex_unlock(&gpii->ctrl_lock); + + return 0; +} + +static void gpi_desc_free(struct virt_dma_desc *vd) +{ + struct gpi_desc *gpi_desc = to_gpi_desc(vd); + + kfree(gpi_desc); + gpi_desc = NULL; +} + +static int +gpi_peripheral_config(struct dma_chan *chan, struct dma_slave_config *config) +{ + struct gchan *gchan = to_gchan(chan); + + if (!config->peripheral_config) + return -EINVAL; + + gchan->config = krealloc(gchan->config, config->peripheral_size, GFP_NOWAIT); + if (!gchan->config) + return -ENOMEM; + + memcpy(gchan->config, config->peripheral_config, config->peripheral_size); + + return 0; +} + +static int gpi_create_i2c_tre(struct gchan *chan, struct gpi_desc *desc, + struct scatterlist *sgl, enum dma_transfer_direction direction) +{ + struct gpi_i2c_config *i2c = chan->config; + struct device *dev = chan->gpii->gpi_dev->dev; + unsigned int tre_idx = 0; + dma_addr_t address; + struct gpi_tre *tre; + unsigned int i; + + /* first create config tre if applicable */ + if (i2c->set_config) { + tre = &desc->tre[tre_idx]; + tre_idx++; + + tre->dword[0] = u32_encode_bits(i2c->low_count, TRE_I2C_C0_TLOW); + tre->dword[0] |= u32_encode_bits(i2c->high_count, TRE_I2C_C0_THIGH); + tre->dword[0] |= u32_encode_bits(i2c->cycle_count, TRE_I2C_C0_TCYL); + tre->dword[0] |= u32_encode_bits(i2c->pack_enable, TRE_I2C_C0_TX_PACK); + tre->dword[0] |= u32_encode_bits(i2c->pack_enable, TRE_I2C_C0_RX_PACK); + + tre->dword[1] = 0; + + tre->dword[2] = u32_encode_bits(i2c->clk_div, TRE_C0_CLK_DIV); + + tre->dword[3] = u32_encode_bits(TRE_TYPE_CONFIG0, TRE_FLAGS_TYPE); + tre->dword[3] |= u32_encode_bits(1, TRE_FLAGS_CHAIN); + } + + /* create the GO tre for Tx */ + if (i2c->op == I2C_WRITE) { + tre = &desc->tre[tre_idx]; + tre_idx++; + + if (i2c->multi_msg) + tre->dword[0] = u32_encode_bits(I2C_READ, TRE_I2C_GO_CMD); + else + tre->dword[0] = u32_encode_bits(i2c->op, TRE_I2C_GO_CMD); + + tre->dword[0] |= u32_encode_bits(i2c->addr, TRE_I2C_GO_ADDR); + tre->dword[0] |= u32_encode_bits(i2c->stretch, TRE_I2C_GO_STRETCH); + + tre->dword[1] = 0; + tre->dword[2] = u32_encode_bits(i2c->rx_len, TRE_RX_LEN); + + tre->dword[3] = u32_encode_bits(TRE_TYPE_GO, TRE_FLAGS_TYPE); + + if (i2c->multi_msg) + tre->dword[3] |= u32_encode_bits(1, TRE_FLAGS_LINK); + else + tre->dword[3] |= u32_encode_bits(1, TRE_FLAGS_CHAIN); + } + + if (i2c->op == I2C_READ || i2c->multi_msg == false) { + /* create the DMA TRE */ + tre = &desc->tre[tre_idx]; + tre_idx++; + + address = sg_dma_address(sgl); + tre->dword[0] = lower_32_bits(address); + tre->dword[1] = upper_32_bits(address); + + tre->dword[2] = u32_encode_bits(sg_dma_len(sgl), TRE_DMA_LEN); + + tre->dword[3] = u32_encode_bits(TRE_TYPE_DMA, TRE_FLAGS_TYPE); + tre->dword[3] |= u32_encode_bits(1, TRE_FLAGS_IEOT); + }; + + for (i = 0; i < tre_idx; i++) + dev_dbg(dev, "TRE:%d %x:%x:%x:%x\n", i, desc->tre[i].dword[0], + desc->tre[i].dword[1], desc->tre[i].dword[2], desc->tre[i].dword[3]); + + return tre_idx; +} + +static int gpi_create_spi_tre(struct gchan *chan, struct gpi_desc *desc, + struct scatterlist *sgl, enum dma_transfer_direction direction) +{ + struct gpi_spi_config *spi = chan->config; + struct device *dev = chan->gpii->gpi_dev->dev; + unsigned int tre_idx = 0; + dma_addr_t address; + struct gpi_tre *tre; + unsigned int i; + + /* first create config tre if applicable */ + if (direction == DMA_MEM_TO_DEV && spi->set_config) { + tre = &desc->tre[tre_idx]; + tre_idx++; + + tre->dword[0] = u32_encode_bits(spi->word_len, TRE_SPI_C0_WORD_SZ); + tre->dword[0] |= u32_encode_bits(spi->loopback_en, TRE_SPI_C0_LOOPBACK); + tre->dword[0] |= u32_encode_bits(spi->clock_pol_high, TRE_SPI_C0_CPOL); + tre->dword[0] |= u32_encode_bits(spi->data_pol_high, TRE_SPI_C0_CPHA); + tre->dword[0] |= u32_encode_bits(spi->pack_en, TRE_SPI_C0_TX_PACK); + tre->dword[0] |= u32_encode_bits(spi->pack_en, TRE_SPI_C0_RX_PACK); + + tre->dword[1] = 0; + + tre->dword[2] = u32_encode_bits(spi->clk_div, TRE_C0_CLK_DIV); + tre->dword[2] |= u32_encode_bits(spi->clk_src, TRE_C0_CLK_SRC); + + tre->dword[3] = u32_encode_bits(TRE_TYPE_CONFIG0, TRE_FLAGS_TYPE); + tre->dword[3] |= u32_encode_bits(1, TRE_FLAGS_CHAIN); + } + + /* create the GO tre for Tx */ + if (direction == DMA_MEM_TO_DEV) { + tre = &desc->tre[tre_idx]; + tre_idx++; + + tre->dword[0] = u32_encode_bits(spi->fragmentation, TRE_SPI_GO_FRAG); + tre->dword[0] |= u32_encode_bits(spi->cs, TRE_SPI_GO_CS); + tre->dword[0] |= u32_encode_bits(spi->cmd, TRE_SPI_GO_CMD); + + tre->dword[1] = 0; + + tre->dword[2] = u32_encode_bits(spi->rx_len, TRE_RX_LEN); + + tre->dword[3] = u32_encode_bits(TRE_TYPE_GO, TRE_FLAGS_TYPE); + if (spi->cmd == SPI_RX) + tre->dword[3] |= u32_encode_bits(1, TRE_FLAGS_IEOB); + else + tre->dword[3] |= u32_encode_bits(1, TRE_FLAGS_CHAIN); + } + + /* create the dma tre */ + tre = &desc->tre[tre_idx]; + tre_idx++; + + address = sg_dma_address(sgl); + tre->dword[0] = lower_32_bits(address); + tre->dword[1] = upper_32_bits(address); + + tre->dword[2] = u32_encode_bits(sg_dma_len(sgl), TRE_DMA_LEN); + + tre->dword[3] = u32_encode_bits(TRE_TYPE_DMA, TRE_FLAGS_TYPE); + if (direction == DMA_MEM_TO_DEV) + tre->dword[3] |= u32_encode_bits(1, TRE_FLAGS_IEOT); + + for (i = 0; i < tre_idx; i++) + dev_dbg(dev, "TRE:%d %x:%x:%x:%x\n", i, desc->tre[i].dword[0], + desc->tre[i].dword[1], desc->tre[i].dword[2], desc->tre[i].dword[3]); + + return tre_idx; +} + +/* copy tre into transfer ring */ +static struct dma_async_tx_descriptor * +gpi_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl, + unsigned int sg_len, enum dma_transfer_direction direction, + unsigned long flags, void *context) +{ + struct gchan *gchan = to_gchan(chan); + struct gpii *gpii = gchan->gpii; + struct device *dev = gpii->gpi_dev->dev; + struct gpi_ring *ch_ring = &gchan->ch_ring; + struct gpi_desc *gpi_desc; + u32 nr, nr_tre = 0; + u8 set_config; + int i; + + gpii->ieob_set = false; + if (!is_slave_direction(direction)) { + dev_err(gpii->gpi_dev->dev, "invalid dma direction: %d\n", direction); + return NULL; + } + + if (sg_len > 1) { + dev_err(dev, "Multi sg sent, we support only one atm: %d\n", sg_len); + return NULL; + } + + nr_tre = 3; + set_config = *(u32 *)gchan->config; + if (!set_config) + nr_tre = 2; + if (direction == DMA_DEV_TO_MEM) /* rx */ + nr_tre = 1; + + /* calculate # of elements required & available */ + nr = gpi_ring_num_elements_avail(ch_ring); + if (nr < nr_tre) { + dev_err(dev, "not enough space in ring, avail:%u required:%u\n", nr, nr_tre); + return NULL; + } + + gpi_desc = kzalloc(sizeof(*gpi_desc), GFP_NOWAIT); + if (!gpi_desc) + return NULL; + + /* create TREs for xfer */ + if (gchan->protocol == QCOM_GPI_SPI) { + i = gpi_create_spi_tre(gchan, gpi_desc, sgl, direction); + } else if (gchan->protocol == QCOM_GPI_I2C) { + i = gpi_create_i2c_tre(gchan, gpi_desc, sgl, direction); + } else { + dev_err(dev, "invalid peripheral: %d\n", gchan->protocol); + kfree(gpi_desc); + return NULL; + } + + /* set up the descriptor */ + gpi_desc->gchan = gchan; + gpi_desc->len = sg_dma_len(sgl); + gpi_desc->num_tre = i; + + return vchan_tx_prep(&gchan->vc, &gpi_desc->vd, flags); +} + +/* rings transfer ring db to being transfer */ +static void gpi_issue_pending(struct dma_chan *chan) +{ + struct gchan *gchan = to_gchan(chan); + struct gpii *gpii = gchan->gpii; + unsigned long flags, pm_lock_flags; + struct virt_dma_desc *vd = NULL; + struct gpi_desc *gpi_desc; + struct gpi_ring *ch_ring = &gchan->ch_ring; + void *tre, *wp = NULL; + int i; + + read_lock_irqsave(&gpii->pm_lock, pm_lock_flags); + + /* move all submitted discriptors to issued list */ + spin_lock_irqsave(&gchan->vc.lock, flags); + if (vchan_issue_pending(&gchan->vc)) + vd = list_last_entry(&gchan->vc.desc_issued, + struct virt_dma_desc, node); + spin_unlock_irqrestore(&gchan->vc.lock, flags); + + /* nothing to do list is empty */ + if (!vd) { + read_unlock_irqrestore(&gpii->pm_lock, pm_lock_flags); + return; + } + + gpi_desc = to_gpi_desc(vd); + for (i = 0; i < gpi_desc->num_tre; i++) { + tre = &gpi_desc->tre[i]; + gpi_queue_xfer(gpii, gchan, tre, &wp); + } + + gpi_desc->db = ch_ring->wp; + gpi_write_ch_db(gchan, &gchan->ch_ring, gpi_desc->db); + read_unlock_irqrestore(&gpii->pm_lock, pm_lock_flags); +} + +static int gpi_ch_init(struct gchan *gchan) +{ + struct gpii *gpii = gchan->gpii; + const int ev_factor = gpii->gpi_dev->ev_factor; + u32 elements; + int i = 0, ret = 0; + + gchan->pm_state = CONFIG_STATE; + + /* check if both channels are configured before continue */ + for (i = 0; i < MAX_CHANNELS_PER_GPII; i++) + if (gpii->gchan[i].pm_state != CONFIG_STATE) + goto exit_gpi_init; + + /* protocol must be same for both channels */ + if (gpii->gchan[0].protocol != gpii->gchan[1].protocol) { + dev_err(gpii->gpi_dev->dev, "protocol did not match protocol %u != %u\n", + gpii->gchan[0].protocol, gpii->gchan[1].protocol); + ret = -EINVAL; + goto exit_gpi_init; + } + + /* allocate memory for event ring */ + elements = CHAN_TRES << ev_factor; + ret = gpi_alloc_ring(&gpii->ev_ring, elements, + sizeof(union gpi_event), gpii); + if (ret) + goto exit_gpi_init; + + /* configure interrupts */ + write_lock_irq(&gpii->pm_lock); + gpii->pm_state = PREPARE_HARDWARE; + write_unlock_irq(&gpii->pm_lock); + ret = gpi_config_interrupts(gpii, DEFAULT_IRQ_SETTINGS, 0); + if (ret) { + dev_err(gpii->gpi_dev->dev, "error config. interrupts, ret:%d\n", ret); + goto error_config_int; + } + + /* allocate event rings */ + ret = gpi_alloc_ev_chan(gpii); + if (ret) { + dev_err(gpii->gpi_dev->dev, "error alloc_ev_chan:%d\n", ret); + goto error_alloc_ev_ring; + } + + /* Allocate all channels */ + for (i = 0; i < MAX_CHANNELS_PER_GPII; i++) { + ret = gpi_alloc_chan(&gpii->gchan[i], true); + if (ret) { + dev_err(gpii->gpi_dev->dev, "Error allocating chan:%d\n", ret); + goto error_alloc_chan; + } + } + + /* start channels */ + for (i = 0; i < MAX_CHANNELS_PER_GPII; i++) { + ret = gpi_start_chan(&gpii->gchan[i]); + if (ret) { + dev_err(gpii->gpi_dev->dev, "Error start chan:%d\n", ret); + goto error_start_chan; + } + } + return ret; + +error_start_chan: + for (i = i - 1; i >= 0; i++) { + gpi_stop_chan(&gpii->gchan[i]); + gpi_send_cmd(gpii, gchan, GPI_CH_CMD_RESET); + } + i = 2; +error_alloc_chan: + for (i = i - 1; i >= 0; i--) + gpi_reset_chan(gchan, GPI_CH_CMD_DE_ALLOC); +error_alloc_ev_ring: + gpi_disable_interrupts(gpii); +error_config_int: + gpi_free_ring(&gpii->ev_ring, gpii); +exit_gpi_init: + mutex_unlock(&gpii->ctrl_lock); + return ret; +} + +/* release all channel resources */ +static void gpi_free_chan_resources(struct dma_chan *chan) +{ + struct gchan *gchan = to_gchan(chan); + struct gpii *gpii = gchan->gpii; + enum gpi_pm_state cur_state; + int ret, i; + + mutex_lock(&gpii->ctrl_lock); + + cur_state = gchan->pm_state; + + /* disable ch state so no more TRE processing for this channel */ + write_lock_irq(&gpii->pm_lock); + gchan->pm_state = PREPARE_TERMINATE; + write_unlock_irq(&gpii->pm_lock); + + /* attempt to do graceful hardware shutdown */ + if (cur_state == ACTIVE_STATE) { + gpi_stop_chan(gchan); + + ret = gpi_send_cmd(gpii, gchan, GPI_CH_CMD_RESET); + if (ret) + dev_err(gpii->gpi_dev->dev, "error resetting channel:%d\n", ret); + + gpi_reset_chan(gchan, GPI_CH_CMD_DE_ALLOC); + } + + /* free all allocated memory */ + gpi_free_ring(&gchan->ch_ring, gpii); + vchan_free_chan_resources(&gchan->vc); + kfree(gchan->config); + + write_lock_irq(&gpii->pm_lock); + gchan->pm_state = DISABLE_STATE; + write_unlock_irq(&gpii->pm_lock); + + /* if other rings are still active exit */ + for (i = 0; i < MAX_CHANNELS_PER_GPII; i++) + if (gpii->gchan[i].ch_ring.configured) + goto exit_free; + + /* deallocate EV Ring */ + cur_state = gpii->pm_state; + write_lock_irq(&gpii->pm_lock); + gpii->pm_state = PREPARE_TERMINATE; + write_unlock_irq(&gpii->pm_lock); + + /* wait for threads to complete out */ + tasklet_kill(&gpii->ev_task); + + /* send command to de allocate event ring */ + if (cur_state == ACTIVE_STATE) + gpi_send_cmd(gpii, NULL, GPI_EV_CMD_DEALLOC); + + gpi_free_ring(&gpii->ev_ring, gpii); + + /* disable interrupts */ + if (cur_state == ACTIVE_STATE) + gpi_disable_interrupts(gpii); + + /* set final state to disable */ + write_lock_irq(&gpii->pm_lock); + gpii->pm_state = DISABLE_STATE; + write_unlock_irq(&gpii->pm_lock); + +exit_free: + mutex_unlock(&gpii->ctrl_lock); +} + +/* allocate channel resources */ +static int gpi_alloc_chan_resources(struct dma_chan *chan) +{ + struct gchan *gchan = to_gchan(chan); + struct gpii *gpii = gchan->gpii; + int ret; + + mutex_lock(&gpii->ctrl_lock); + + /* allocate memory for transfer ring */ + ret = gpi_alloc_ring(&gchan->ch_ring, CHAN_TRES, + sizeof(struct gpi_tre), gpii); + if (ret) + goto xfer_alloc_err; + + ret = gpi_ch_init(gchan); + + mutex_unlock(&gpii->ctrl_lock); + + return ret; +xfer_alloc_err: + mutex_unlock(&gpii->ctrl_lock); + + return ret; +} + +static int gpi_find_avail_gpii(struct gpi_dev *gpi_dev, u32 seid) +{ + struct gchan *tx_chan, *rx_chan; + unsigned int gpii; + + /* check if same seid is already configured for another chid */ + for (gpii = 0; gpii < gpi_dev->max_gpii; gpii++) { + if (!((1 << gpii) & gpi_dev->gpii_mask)) + continue; + + tx_chan = &gpi_dev->gpiis[gpii].gchan[GPI_TX_CHAN]; + rx_chan = &gpi_dev->gpiis[gpii].gchan[GPI_RX_CHAN]; + + if (rx_chan->vc.chan.client_count && rx_chan->seid == seid) + return gpii; + if (tx_chan->vc.chan.client_count && tx_chan->seid == seid) + return gpii; + } + + /* no channels configured with same seid, return next avail gpii */ + for (gpii = 0; gpii < gpi_dev->max_gpii; gpii++) { + if (!((1 << gpii) & gpi_dev->gpii_mask)) + continue; + + tx_chan = &gpi_dev->gpiis[gpii].gchan[GPI_TX_CHAN]; + rx_chan = &gpi_dev->gpiis[gpii].gchan[GPI_RX_CHAN]; + + /* check if gpii is configured */ + if (tx_chan->vc.chan.client_count || + rx_chan->vc.chan.client_count) + continue; + + /* found a free gpii */ + return gpii; + } + + /* no gpii instance available to use */ + return -EIO; +} + +/* gpi_of_dma_xlate: open client requested channel */ +static struct dma_chan *gpi_of_dma_xlate(struct of_phandle_args *args, + struct of_dma *of_dma) +{ + struct gpi_dev *gpi_dev = (struct gpi_dev *)of_dma->of_dma_data; + u32 seid, chid; + int gpii; + struct gchan *gchan; + + if (args->args_count < 3) { + dev_err(gpi_dev->dev, "gpii require minimum 2 args, client passed:%d args\n", + args->args_count); + return NULL; + } + + chid = args->args[0]; + if (chid >= MAX_CHANNELS_PER_GPII) { + dev_err(gpi_dev->dev, "gpii channel:%d not valid\n", chid); + return NULL; + } + + seid = args->args[1]; + + /* find next available gpii to use */ + gpii = gpi_find_avail_gpii(gpi_dev, seid); + if (gpii < 0) { + dev_err(gpi_dev->dev, "no available gpii instances\n"); + return NULL; + } + + gchan = &gpi_dev->gpiis[gpii].gchan[chid]; + if (gchan->vc.chan.client_count) { + dev_err(gpi_dev->dev, "gpii:%d chid:%d seid:%d already configured\n", + gpii, chid, gchan->seid); + return NULL; + } + + gchan->seid = seid; + gchan->protocol = args->args[2]; + + return dma_get_slave_channel(&gchan->vc.chan); +} + +static int gpi_probe(struct platform_device *pdev) +{ + struct gpi_dev *gpi_dev; + unsigned int i; + int ret; + + gpi_dev = devm_kzalloc(&pdev->dev, sizeof(*gpi_dev), GFP_KERNEL); + if (!gpi_dev) + return -ENOMEM; + + gpi_dev->dev = &pdev->dev; + gpi_dev->res = platform_get_resource(pdev, IORESOURCE_MEM, 0); + gpi_dev->regs = devm_ioremap_resource(gpi_dev->dev, gpi_dev->res); + if (IS_ERR(gpi_dev->regs)) + return PTR_ERR(gpi_dev->regs); + gpi_dev->ee_base = gpi_dev->regs; + + ret = of_property_read_u32(gpi_dev->dev->of_node, "dma-channels", + &gpi_dev->max_gpii); + if (ret) { + dev_err(gpi_dev->dev, "missing 'max-no-gpii' DT node\n"); + return ret; + } + + ret = of_property_read_u32(gpi_dev->dev->of_node, "dma-channel-mask", + &gpi_dev->gpii_mask); + if (ret) { + dev_err(gpi_dev->dev, "missing 'gpii-mask' DT node\n"); + return ret; + } + + gpi_dev->ev_factor = EV_FACTOR; + + ret = dma_set_mask(gpi_dev->dev, DMA_BIT_MASK(64)); + if (ret) { + dev_err(gpi_dev->dev, "Error setting dma_mask to 64, ret:%d\n", ret); + return ret; + } + + gpi_dev->gpiis = devm_kzalloc(gpi_dev->dev, sizeof(*gpi_dev->gpiis) * + gpi_dev->max_gpii, GFP_KERNEL); + if (!gpi_dev->gpiis) + return -ENOMEM; + + /* setup all the supported gpii */ + INIT_LIST_HEAD(&gpi_dev->dma_device.channels); + for (i = 0; i < gpi_dev->max_gpii; i++) { + struct gpii *gpii = &gpi_dev->gpiis[i]; + int chan; + + if (!((1 << i) & gpi_dev->gpii_mask)) + continue; + + /* set up ev cntxt register map */ + gpii->ev_cntxt_base_reg = gpi_dev->ee_base + GPII_n_EV_CH_k_CNTXT_0_OFFS(i, 0); + gpii->ev_cntxt_db_reg = gpi_dev->ee_base + GPII_n_EV_CH_k_DOORBELL_0_OFFS(i, 0); + gpii->ev_ring_rp_lsb_reg = gpii->ev_cntxt_base_reg + CNTXT_4_RING_RP_LSB; + gpii->ev_cmd_reg = gpi_dev->ee_base + GPII_n_EV_CH_CMD_OFFS(i); + gpii->ieob_clr_reg = gpi_dev->ee_base + GPII_n_CNTXT_SRC_IEOB_IRQ_CLR_OFFS(i); + + /* set up irq */ + ret = platform_get_irq(pdev, i); + if (ret < 0) { + dev_err(gpi_dev->dev, "platform_get_irq failed for %d:%d\n", i, ret); + return ret; + } + gpii->irq = ret; + + /* set up channel specific register info */ + for (chan = 0; chan < MAX_CHANNELS_PER_GPII; chan++) { + struct gchan *gchan = &gpii->gchan[chan]; + + /* set up ch cntxt register map */ + gchan->ch_cntxt_base_reg = gpi_dev->ee_base + + GPII_n_CH_k_CNTXT_0_OFFS(i, chan); + gchan->ch_cntxt_db_reg = gpi_dev->ee_base + + GPII_n_CH_k_DOORBELL_0_OFFS(i, chan); + gchan->ch_cmd_reg = gpi_dev->ee_base + GPII_n_CH_CMD_OFFS(i); + + /* vchan setup */ + vchan_init(&gchan->vc, &gpi_dev->dma_device); + gchan->vc.desc_free = gpi_desc_free; + gchan->chid = chan; + gchan->gpii = gpii; + gchan->dir = GPII_CHAN_DIR[chan]; + } + mutex_init(&gpii->ctrl_lock); + rwlock_init(&gpii->pm_lock); + tasklet_init(&gpii->ev_task, gpi_ev_tasklet, + (unsigned long)gpii); + init_completion(&gpii->cmd_completion); + gpii->gpii_id = i; + gpii->regs = gpi_dev->ee_base; + gpii->gpi_dev = gpi_dev; + } + + platform_set_drvdata(pdev, gpi_dev); + + /* clear and Set capabilities */ + dma_cap_zero(gpi_dev->dma_device.cap_mask); + dma_cap_set(DMA_SLAVE, gpi_dev->dma_device.cap_mask); + + /* configure dmaengine apis */ + gpi_dev->dma_device.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV); + gpi_dev->dma_device.residue_granularity = DMA_RESIDUE_GRANULARITY_DESCRIPTOR; + gpi_dev->dma_device.src_addr_widths = DMA_SLAVE_BUSWIDTH_8_BYTES; + gpi_dev->dma_device.dst_addr_widths = DMA_SLAVE_BUSWIDTH_8_BYTES; + gpi_dev->dma_device.device_alloc_chan_resources = gpi_alloc_chan_resources; + gpi_dev->dma_device.device_free_chan_resources = gpi_free_chan_resources; + gpi_dev->dma_device.device_tx_status = dma_cookie_status; + gpi_dev->dma_device.device_issue_pending = gpi_issue_pending; + gpi_dev->dma_device.device_prep_slave_sg = gpi_prep_slave_sg; + gpi_dev->dma_device.device_config = gpi_peripheral_config; + gpi_dev->dma_device.device_terminate_all = gpi_terminate_all; + gpi_dev->dma_device.dev = gpi_dev->dev; + gpi_dev->dma_device.device_pause = gpi_pause; + gpi_dev->dma_device.device_resume = gpi_resume; + + /* register with dmaengine framework */ + ret = dma_async_device_register(&gpi_dev->dma_device); + if (ret) { + dev_err(gpi_dev->dev, "async_device_register failed ret:%d", ret); + return ret; + } + + ret = of_dma_controller_register(gpi_dev->dev->of_node, + gpi_of_dma_xlate, gpi_dev); + if (ret) { + dev_err(gpi_dev->dev, "of_dma_controller_reg failed ret:%d", ret); + return ret; + } + + return ret; +} + +static const struct of_device_id gpi_of_match[] = { + { .compatible = "qcom,sdm845-gpi-dma" }, + { }, +}; +MODULE_DEVICE_TABLE(of, gpi_of_match); + +static struct platform_driver gpi_driver = { + .probe = gpi_probe, + .driver = { + .name = KBUILD_MODNAME, + .of_match_table = gpi_of_match, + }, +}; + +static int __init gpi_init(void) +{ + return platform_driver_register(&gpi_driver); +} +subsys_initcall(gpi_init) + +MODULE_DESCRIPTION("QCOM GPI DMA engine driver"); +MODULE_LICENSE("GPL v2"); diff --git a/include/linux/dma/qcom-gpi-dma.h b/include/linux/dma/qcom-gpi-dma.h new file mode 100644 index 000000000000..f46dc3372f11 --- /dev/null +++ b/include/linux/dma/qcom-gpi-dma.h @@ -0,0 +1,83 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2020, Linaro Limited + */ + +#ifndef QCOM_GPI_DMA_H +#define QCOM_GPI_DMA_H + +/** + * enum spi_transfer_cmd - spi transfer commands + */ +enum spi_transfer_cmd { + SPI_TX = 1, + SPI_RX, + SPI_DUPLEX, +}; + +/** + * struct gpi_spi_config - spi config for peripheral + * + * @loopback_en: spi loopback enable when set + * @clock_pol_high: clock polarity + * @data_pol_high: data polarity + * @pack_en: process tx/rx buffers as packed + * @word_len: spi word length + * @clk_div: source clock divider + * @clk_src: serial clock + * @cmd: spi cmd + * @fragmentation: keep CS assserted at end of sequence + * @cs: chip select toggle + * @set_config: set peripheral config + * @rx_len: receive length for buffer + */ +struct gpi_spi_config { + u8 set_config; + u8 loopback_en; + u8 clock_pol_high; + u8 data_pol_high; + u8 pack_en; + u8 word_len; + u8 fragmentation; + u8 cs; + u32 clk_div; + u32 clk_src; + enum spi_transfer_cmd cmd; + u32 rx_len; +}; + +enum i2c_op { + I2C_WRITE = 1, + I2C_READ, +}; + +/** + * struct gpi_i2c_config - i2c config for peripheral + * + * @pack_enable: process tx/rx buffers as packed + * @cycle_count: clock cycles to be sent + * @high_count: high period of clock + * @low_count: low period of clock + * @clk_div: source clock divider + * @addr: i2c bus address + * @stretch: stretch the clock at eot + * @set_config: set peripheral config + * @rx_len: receive length for buffer + * @op: i2c cmd + * @muli-msg: is part of multi i2c r-w msgs + */ +struct gpi_i2c_config { + u8 set_config; + u8 pack_enable; + u8 cycle_count; + u8 high_count; + u8 low_count; + u8 addr; + u8 stretch; + u16 clk_div; + u32 rx_len; + enum i2c_op op; + bool multi_msg; +}; + +#endif /* QCOM_GPI_DMA_H */ -- cgit v1.2.3-59-g8ed1b From 0ab785c894e618587e83bb67e8a8e96649868ad1 Mon Sep 17 00:00:00 2001 From: Fabio Estevam Date: Tue, 24 Nov 2020 11:34:05 -0300 Subject: dmaengine: imx-dma: Remove unused .id_table Since 5.10-rc1 i.MX is a devicetree-only platform, so simplify the code by removing the unused non-DT support. Signed-off-by: Fabio Estevam Link: https://lore.kernel.org/r/20201124143405.2764-1-festevam@gmail.com Signed-off-by: Vinod Koul --- drivers/dma/imx-dma.c | 33 ++++----------------------------- 1 file changed, 4 insertions(+), 29 deletions(-) diff --git a/drivers/dma/imx-dma.c b/drivers/dma/imx-dma.c index 670db04b0757..7f116bbcfad2 100644 --- a/drivers/dma/imx-dma.c +++ b/drivers/dma/imx-dma.c @@ -191,32 +191,13 @@ struct imxdma_filter_data { int request; }; -static const struct platform_device_id imx_dma_devtype[] = { - { - .name = "imx1-dma", - .driver_data = IMX1_DMA, - }, { - .name = "imx21-dma", - .driver_data = IMX21_DMA, - }, { - .name = "imx27-dma", - .driver_data = IMX27_DMA, - }, { - /* sentinel */ - } -}; -MODULE_DEVICE_TABLE(platform, imx_dma_devtype); - static const struct of_device_id imx_dma_of_dev_id[] = { { - .compatible = "fsl,imx1-dma", - .data = &imx_dma_devtype[IMX1_DMA], + .compatible = "fsl,imx1-dma", .data = (const void *)IMX1_DMA, }, { - .compatible = "fsl,imx21-dma", - .data = &imx_dma_devtype[IMX21_DMA], + .compatible = "fsl,imx21-dma", .data = (const void *)IMX21_DMA, }, { - .compatible = "fsl,imx27-dma", - .data = &imx_dma_devtype[IMX27_DMA], + .compatible = "fsl,imx27-dma", .data = (const void *)IMX27_DMA, }, { /* sentinel */ } @@ -1056,20 +1037,15 @@ static int __init imxdma_probe(struct platform_device *pdev) { struct imxdma_engine *imxdma; struct resource *res; - const struct of_device_id *of_id; int ret, i; int irq, irq_err; - of_id = of_match_device(imx_dma_of_dev_id, &pdev->dev); - if (of_id) - pdev->id_entry = of_id->data; - imxdma = devm_kzalloc(&pdev->dev, sizeof(*imxdma), GFP_KERNEL); if (!imxdma) return -ENOMEM; imxdma->dev = &pdev->dev; - imxdma->devtype = pdev->id_entry->driver_data; + imxdma->devtype = (enum imx_dma_type)of_device_get_match_data(&pdev->dev); res = platform_get_resource(pdev, IORESOURCE_MEM, 0); imxdma->base = devm_ioremap_resource(&pdev->dev, res); @@ -1263,7 +1239,6 @@ static struct platform_driver imxdma_driver = { .name = "imx-dma", .of_match_table = imx_dma_of_dev_id, }, - .id_table = imx_dma_devtype, .remove = imxdma_remove, }; -- cgit v1.2.3-59-g8ed1b From c95e6515a8c065862361f7e0e452978ade7f94ec Mon Sep 17 00:00:00 2001 From: Zhihao Cheng Date: Tue, 24 Nov 2020 09:08:13 +0800 Subject: dmaengine: mv_xor_v2: Fix error return code in mv_xor_v2_probe() Return the corresponding error code when first_msi_entry() returns NULL in mv_xor_v2_probe(). Fixes: 19a340b1a820430 ("dmaengine: mv_xor_v2: new driver") Reported-by: Hulk Robot Signed-off-by: Zhihao Cheng Link: https://lore.kernel.org/r/20201124010813.1939095-1-chengzhihao1@huawei.com Signed-off-by: Vinod Koul --- drivers/dma/mv_xor_v2.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/drivers/dma/mv_xor_v2.c b/drivers/dma/mv_xor_v2.c index 2753a6b916f6..9b0d463f89bb 100644 --- a/drivers/dma/mv_xor_v2.c +++ b/drivers/dma/mv_xor_v2.c @@ -771,8 +771,10 @@ static int mv_xor_v2_probe(struct platform_device *pdev) goto disable_clk; msi_desc = first_msi_entry(&pdev->dev); - if (!msi_desc) + if (!msi_desc) { + ret = -ENODEV; goto free_msi_irqs; + } xor_dev->msi_desc = msi_desc; ret = devm_request_irq(&pdev->dev, msi_desc->irq, -- cgit v1.2.3-59-g8ed1b From cc2afb0d4c7cbba6743ed6d9564f0883cab6bae1 Mon Sep 17 00:00:00 2001 From: Fabio Estevam Date: Mon, 23 Nov 2020 16:30:51 -0300 Subject: dmaengine: mxs-dma: Remove the unused .id_table The mxs-dma driver is only used by DT platforms and the .id_table is unused. Get rid of it to simplify the code. Signed-off-by: Fabio Estevam Link: https://lore.kernel.org/r/20201123193051.17285-1-festevam@gmail.com Signed-off-by: Vinod Koul --- drivers/dma/mxs-dma.c | 37 +++++-------------------------------- 1 file changed, 5 insertions(+), 32 deletions(-) diff --git a/drivers/dma/mxs-dma.c b/drivers/dma/mxs-dma.c index 65f816b40c32..994fc4d2aca4 100644 --- a/drivers/dma/mxs-dma.c +++ b/drivers/dma/mxs-dma.c @@ -167,29 +167,11 @@ static struct mxs_dma_type mxs_dma_types[] = { } }; -static const struct platform_device_id mxs_dma_ids[] = { - { - .name = "imx23-dma-apbh", - .driver_data = (kernel_ulong_t) &mxs_dma_types[0], - }, { - .name = "imx23-dma-apbx", - .driver_data = (kernel_ulong_t) &mxs_dma_types[1], - }, { - .name = "imx28-dma-apbh", - .driver_data = (kernel_ulong_t) &mxs_dma_types[2], - }, { - .name = "imx28-dma-apbx", - .driver_data = (kernel_ulong_t) &mxs_dma_types[3], - }, { - /* end of list */ - } -}; - static const struct of_device_id mxs_dma_dt_ids[] = { - { .compatible = "fsl,imx23-dma-apbh", .data = &mxs_dma_ids[0], }, - { .compatible = "fsl,imx23-dma-apbx", .data = &mxs_dma_ids[1], }, - { .compatible = "fsl,imx28-dma-apbh", .data = &mxs_dma_ids[2], }, - { .compatible = "fsl,imx28-dma-apbx", .data = &mxs_dma_ids[3], }, + { .compatible = "fsl,imx23-dma-apbh", .data = &mxs_dma_types[0], }, + { .compatible = "fsl,imx23-dma-apbx", .data = &mxs_dma_types[1], }, + { .compatible = "fsl,imx28-dma-apbh", .data = &mxs_dma_types[2], }, + { .compatible = "fsl,imx28-dma-apbx", .data = &mxs_dma_types[3], }, { /* sentinel */ } }; MODULE_DEVICE_TABLE(of, mxs_dma_dt_ids); @@ -762,8 +744,6 @@ static struct dma_chan *mxs_dma_xlate(struct of_phandle_args *dma_spec, static int __init mxs_dma_probe(struct platform_device *pdev) { struct device_node *np = pdev->dev.of_node; - const struct platform_device_id *id_entry; - const struct of_device_id *of_id; const struct mxs_dma_type *dma_type; struct mxs_dma_engine *mxs_dma; struct resource *iores; @@ -779,13 +759,7 @@ static int __init mxs_dma_probe(struct platform_device *pdev) return ret; } - of_id = of_match_device(mxs_dma_dt_ids, &pdev->dev); - if (of_id) - id_entry = of_id->data; - else - id_entry = platform_get_device_id(pdev); - - dma_type = (struct mxs_dma_type *)id_entry->driver_data; + dma_type = (struct mxs_dma_type *)of_device_get_match_data(&pdev->dev); mxs_dma->type = dma_type->type; mxs_dma->dev_id = dma_type->id; @@ -865,7 +839,6 @@ static struct platform_driver mxs_dma_driver = { .name = "mxs-dma", .of_match_table = mxs_dma_dt_ids, }, - .id_table = mxs_dma_ids, }; static int __init mxs_dma_module_init(void) -- cgit v1.2.3-59-g8ed1b From 255c2cc896b1009e7df4c4e32d9090fc22d28293 Mon Sep 17 00:00:00 2001 From: Krzysztof Kozlowski Date: Fri, 20 Nov 2020 17:22:58 +0100 Subject: dmaengine: jz4780: drop of_match_ptr from of_device_id table MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The driver can match only via the DT table so the table should be always used and the of_match_ptr does not have any sense (this also allows ACPI matching via PRP0001, even though it is not relevant here). This fixes compile warning (!CONFIG_OF on x86_64): drivers/dma/dma-jz4780.c:1031:34: warning: ‘jz4780_dma_dt_match’ defined but not used [-Wunused-const-variable=] Signed-off-by: Krzysztof Kozlowski Link: https://lore.kernel.org/r/20201120162303.482126-1-krzk@kernel.org Signed-off-by: Vinod Koul --- drivers/dma/dma-jz4780.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/dma/dma-jz4780.c b/drivers/dma/dma-jz4780.c index a608efaa435f..612d353648cf 100644 --- a/drivers/dma/dma-jz4780.c +++ b/drivers/dma/dma-jz4780.c @@ -1044,7 +1044,7 @@ static struct platform_driver jz4780_dma_driver = { .remove = jz4780_dma_remove, .driver = { .name = "jz4780-dma", - .of_match_table = of_match_ptr(jz4780_dma_dt_match), + .of_match_table = jz4780_dma_dt_match, }, }; -- cgit v1.2.3-59-g8ed1b From 60b6122e86403fae58099dc08ecd319a86d03906 Mon Sep 17 00:00:00 2001 From: Krzysztof Kozlowski Date: Fri, 20 Nov 2020 17:22:59 +0100 Subject: dmaengine: dw-axi-dmac: drop of_match_ptr from of_device_id table MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The driver can match only via the DT table so the table should be always used and the of_match_ptr does not have any sense (this also allows ACPI matching via PRP0001, even though it is not relevant here). This fixes compile warning (!CONFIG_OF on x86_64): drivers/dma/dw-axi-dmac/dw-axi-dmac-platform.c:984:34: warning: ‘dw_dma_of_id_table’ defined but not used [-Wunused-const-variable=] Signed-off-by: Krzysztof Kozlowski Link: https://lore.kernel.org/r/20201120162303.482126-2-krzk@kernel.org Signed-off-by: Vinod Koul --- drivers/dma/dw-axi-dmac/dw-axi-dmac-platform.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/dma/dw-axi-dmac/dw-axi-dmac-platform.c b/drivers/dma/dw-axi-dmac/dw-axi-dmac-platform.c index 14c1ac26f866..e164f3295f5d 100644 --- a/drivers/dma/dw-axi-dmac/dw-axi-dmac-platform.c +++ b/drivers/dma/dw-axi-dmac/dw-axi-dmac-platform.c @@ -992,7 +992,7 @@ static struct platform_driver dw_driver = { .remove = dw_remove, .driver = { .name = KBUILD_MODNAME, - .of_match_table = of_match_ptr(dw_dma_of_id_table), + .of_match_table = dw_dma_of_id_table, .pm = &dw_axi_dma_pm_ops, }, }; -- cgit v1.2.3-59-g8ed1b From 890bcd49d8a6d8cb943c8b844bdc1547c1806ad7 Mon Sep 17 00:00:00 2001 From: Krzysztof Kozlowski Date: Fri, 20 Nov 2020 17:23:00 +0100 Subject: dmaengine: mv_xor: drop of_match_ptr from of_device_id table MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The driver can match only via the DT table so the table should be always used and the of_match_ptr does not have any sense (this also allows ACPI matching via PRP0001, even though it is not relevant here). This fixes compile warning (!CONFIG_OF on x86_64): drivers/dma/mv_xor.c:1281:34: warning: ‘mv_xor_dt_ids’ defined but not used [-Wunused-const-variable=] Signed-off-by: Krzysztof Kozlowski Link: https://lore.kernel.org/r/20201120162303.482126-3-krzk@kernel.org Signed-off-by: Vinod Koul --- drivers/dma/mv_xor.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/dma/mv_xor.c b/drivers/dma/mv_xor.c index 00cd1335eeba..23b232b57518 100644 --- a/drivers/dma/mv_xor.c +++ b/drivers/dma/mv_xor.c @@ -1455,7 +1455,7 @@ static struct platform_driver mv_xor_driver = { .resume = mv_xor_resume, .driver = { .name = MV_XOR_NAME, - .of_match_table = of_match_ptr(mv_xor_dt_ids), + .of_match_table = mv_xor_dt_ids, }, }; -- cgit v1.2.3-59-g8ed1b From 75fa2d4218c7df7added689dc941ba7b52519b2e Mon Sep 17 00:00:00 2001 From: Krzysztof Kozlowski Date: Fri, 20 Nov 2020 17:23:01 +0100 Subject: dmaengine: sf: drop of_match_ptr from of_device_id table MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The driver can match only via the DT table so the table should be always used and the of_match_ptr does not have any sense (this also allows ACPI matching via PRP0001, even though it is not relevant here). This fixes compile warning (!CONFIG_OF on x86_64): drivers/dma/sf-pdma/sf-pdma.c:576:34: warning: ‘sf_pdma_dt_ids’ defined but not used [-Wunused-const-variable=] Signed-off-by: Krzysztof Kozlowski Link: https://lore.kernel.org/r/20201120162303.482126-4-krzk@kernel.org Signed-off-by: Vinod Koul --- drivers/dma/sf-pdma/sf-pdma.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/dma/sf-pdma/sf-pdma.c b/drivers/dma/sf-pdma/sf-pdma.c index 981ddcc42d70..c4c4e8575764 100644 --- a/drivers/dma/sf-pdma/sf-pdma.c +++ b/drivers/dma/sf-pdma/sf-pdma.c @@ -582,7 +582,7 @@ static struct platform_driver sf_pdma_driver = { .remove = sf_pdma_remove, .driver = { .name = "sf-pdma", - .of_match_table = of_match_ptr(sf_pdma_dt_ids), + .of_match_table = sf_pdma_dt_ids, }, }; -- cgit v1.2.3-59-g8ed1b From 25d39b590dac0b8230cc3663c9f8f1a3a21831fd Mon Sep 17 00:00:00 2001 From: Krzysztof Kozlowski Date: Fri, 20 Nov 2020 17:23:02 +0100 Subject: dmaengine: stm32: mark of_device_id table as maybe unused MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The driver uses a second of_device_id table in the probe() function by passing it to of_match_node(). This code will be a no-op for compile testing (!CONFIG_OF on x86_64): drivers/dma/stm32-dmamux.c:171:34: warning: ‘stm32_stm32dma_master_match’ defined but not used [-Wunused-const-variable=] Signed-off-by: Krzysztof Kozlowski Link: https://lore.kernel.org/r/20201120162303.482126-5-krzk@kernel.org Signed-off-by: Vinod Koul --- drivers/dma/stm32-dmamux.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/dma/stm32-dmamux.c b/drivers/dma/stm32-dmamux.c index a10ccd964376..ef0d0555103d 100644 --- a/drivers/dma/stm32-dmamux.c +++ b/drivers/dma/stm32-dmamux.c @@ -168,7 +168,7 @@ error_chan_id: return ERR_PTR(ret); } -static const struct of_device_id stm32_stm32dma_master_match[] = { +static const struct of_device_id stm32_stm32dma_master_match[] __maybe_unused = { { .compatible = "st,stm32-dma", }, {}, }; -- cgit v1.2.3-59-g8ed1b From 5d051f37f49d5bf04dca15fadea3a90a6a6f0f15 Mon Sep 17 00:00:00 2001 From: Krzysztof Kozlowski Date: Fri, 20 Nov 2020 17:23:03 +0100 Subject: dmaengine: ti: drop of_match_ptr and mark of_device_id table as maybe unused MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The driver can match only via the DT table so the main table should be always used and the of_match_ptr does not have any sense (this also allows ACPI matching via PRP0001, even though it is not relevant here). The secondary match of_device_id tables (passed to of_match_node) should be marked as maybe unused to fix compile testing (!CONFIG_OF on x86_64) warnings: drivers/dma/ti/dma-crossbar.c:125:34: warning: ‘ti_am335x_master_match’ defined but not used [-Wunused-const-variable=] drivers/dma/ti/dma-crossbar.c:22:34: warning: ‘ti_dma_xbar_match’ defined but not used [-Wunused-const-variable=] Signed-off-by: Krzysztof Kozlowski Link: https://lore.kernel.org/r/20201120162303.482126-6-krzk@kernel.org Signed-off-by: Vinod Koul --- drivers/dma/ti/dma-crossbar.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/dma/ti/dma-crossbar.c b/drivers/dma/ti/dma-crossbar.c index 4ba8fa5d9c36..71d24fc07c00 100644 --- a/drivers/dma/ti/dma-crossbar.c +++ b/drivers/dma/ti/dma-crossbar.c @@ -122,7 +122,7 @@ static void *ti_am335x_xbar_route_allocate(struct of_phandle_args *dma_spec, return map; } -static const struct of_device_id ti_am335x_master_match[] = { +static const struct of_device_id ti_am335x_master_match[] __maybe_unused = { { .compatible = "ti,edma3-tpcc", }, {}, }; @@ -292,7 +292,7 @@ static const u32 ti_dma_offset[] = { [TI_XBAR_SDMA_OFFSET] = 1, }; -static const struct of_device_id ti_dra7_master_match[] = { +static const struct of_device_id ti_dra7_master_match[] __maybe_unused = { { .compatible = "ti,omap4430-sdma", .data = &ti_dma_offset[TI_XBAR_SDMA_OFFSET], @@ -460,7 +460,7 @@ static int ti_dma_xbar_probe(struct platform_device *pdev) static struct platform_driver ti_dma_xbar_driver = { .driver = { .name = "ti-dma-crossbar", - .of_match_table = of_match_ptr(ti_dma_xbar_match), + .of_match_table = ti_dma_xbar_match, }, .probe = ti_dma_xbar_probe, }; -- cgit v1.2.3-59-g8ed1b From 92de5fa2dc39c3fba0704f7ac914e7f02eb732f2 Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Fri, 13 Nov 2020 15:55:05 -0700 Subject: dmaengine: idxd: add ATS disable knob for work queues With the DSA spec 1.1 update, a knob to disable ATS for individually is introduced. Add enabling code to allow a system admin to make the configuration through sysfs. Signed-off-by: Dave Jiang Link: https://lore.kernel.org/r/160530810593.1288392.2561048329116529566.stgit@djiang5-desk3.ch.intel.com Signed-off-by: Vinod Koul --- Documentation/ABI/stable/sysfs-driver-dma-idxd | 7 ++++++ drivers/dma/idxd/device.c | 4 +++ drivers/dma/idxd/idxd.h | 1 + drivers/dma/idxd/registers.h | 5 ++-- drivers/dma/idxd/sysfs.c | 34 ++++++++++++++++++++++++++ 5 files changed, 49 insertions(+), 2 deletions(-) diff --git a/Documentation/ABI/stable/sysfs-driver-dma-idxd b/Documentation/ABI/stable/sysfs-driver-dma-idxd index 5ea81ffd3c1a..55285c136cf0 100644 --- a/Documentation/ABI/stable/sysfs-driver-dma-idxd +++ b/Documentation/ABI/stable/sysfs-driver-dma-idxd @@ -204,6 +204,13 @@ Contact: dmaengine@vger.kernel.org Description: The max batch size for this workqueue. Cannot exceed device max batch size. Configurable parameter. +What: /sys/bus/dsa/devices/wq./ats_disable +Date: Nov 13, 2020 +KernelVersion: 5.11.0 +Contact: dmaengine@vger.kernel.org +Description: Indicate whether ATS disable is turned on for the workqueue. + 0 indicates ATS is on, and 1 indicates ATS is off for the workqueue. + What: /sys/bus/dsa/devices/engine./group_id Date: Oct 25, 2019 KernelVersion: 5.6.0 diff --git a/drivers/dma/idxd/device.c b/drivers/dma/idxd/device.c index d6f551dcbcb6..b75f9a09666e 100644 --- a/drivers/dma/idxd/device.c +++ b/drivers/dma/idxd/device.c @@ -354,6 +354,7 @@ void idxd_wq_disable_cleanup(struct idxd_wq *wq) wq->group = NULL; wq->threshold = 0; wq->priority = 0; + wq->ats_dis = 0; clear_bit(WQ_FLAG_DEDICATED, &wq->flags); memset(wq->name, 0, WQ_NAME_SIZE); @@ -631,6 +632,9 @@ static int idxd_wq_config_write(struct idxd_wq *wq) test_bit(WQ_FLAG_BLOCK_ON_FAULT, &wq->flags)) wq->wqcfg->bof = 1; + if (idxd->hw.wq_cap.wq_ats_support) + wq->wqcfg->wq_ats_disable = wq->ats_dis; + /* bytes 12-15 */ wq->wqcfg->max_xfer_shift = ilog2(wq->max_xfer_bytes); wq->wqcfg->max_batch_shift = ilog2(wq->max_batch_size); diff --git a/drivers/dma/idxd/idxd.h b/drivers/dma/idxd/idxd.h index 7e54209c433a..149934f8d097 100644 --- a/drivers/dma/idxd/idxd.h +++ b/drivers/dma/idxd/idxd.h @@ -123,6 +123,7 @@ struct idxd_wq { char name[WQ_NAME_SIZE + 1]; u64 max_xfer_bytes; u32 max_batch_size; + bool ats_dis; }; struct idxd_engine { diff --git a/drivers/dma/idxd/registers.h b/drivers/dma/idxd/registers.h index d29a58ee2651..0cdc5405bc53 100644 --- a/drivers/dma/idxd/registers.h +++ b/drivers/dma/idxd/registers.h @@ -47,7 +47,7 @@ union wq_cap_reg { u64 rsvd:20; u64 shared_mode:1; u64 dedicated_mode:1; - u64 rsvd2:1; + u64 wq_ats_support:1; u64 priority:1; u64 occupancy:1; u64 occupancy_int:1; @@ -303,7 +303,8 @@ union wqcfg { /* bytes 8-11 */ u32 mode:1; /* shared or dedicated */ u32 bof:1; /* block on fault */ - u32 rsvd2:2; + u32 wq_ats_disable:1; + u32 rsvd2:1; u32 priority:4; u32 pasid:20; u32 pasid_en:1; diff --git a/drivers/dma/idxd/sysfs.c b/drivers/dma/idxd/sysfs.c index 6d292eb79bf3..3af83f1fd36e 100644 --- a/drivers/dma/idxd/sysfs.c +++ b/drivers/dma/idxd/sysfs.c @@ -1261,6 +1261,39 @@ static ssize_t wq_max_batch_size_store(struct device *dev, struct device_attribu static struct device_attribute dev_attr_wq_max_batch_size = __ATTR(max_batch_size, 0644, wq_max_batch_size_show, wq_max_batch_size_store); +static ssize_t wq_ats_disable_show(struct device *dev, struct device_attribute *attr, char *buf) +{ + struct idxd_wq *wq = container_of(dev, struct idxd_wq, conf_dev); + + return sprintf(buf, "%u\n", wq->ats_dis); +} + +static ssize_t wq_ats_disable_store(struct device *dev, struct device_attribute *attr, + const char *buf, size_t count) +{ + struct idxd_wq *wq = container_of(dev, struct idxd_wq, conf_dev); + struct idxd_device *idxd = wq->idxd; + bool ats_dis; + int rc; + + if (wq->state != IDXD_WQ_DISABLED) + return -EPERM; + + if (!idxd->hw.wq_cap.wq_ats_support) + return -EOPNOTSUPP; + + rc = kstrtobool(buf, &ats_dis); + if (rc < 0) + return rc; + + wq->ats_dis = ats_dis; + + return count; +} + +static struct device_attribute dev_attr_wq_ats_disable = + __ATTR(ats_disable, 0644, wq_ats_disable_show, wq_ats_disable_store); + static struct attribute *idxd_wq_attributes[] = { &dev_attr_wq_clients.attr, &dev_attr_wq_state.attr, @@ -1275,6 +1308,7 @@ static struct attribute *idxd_wq_attributes[] = { &dev_attr_wq_cdev_minor.attr, &dev_attr_wq_max_transfer_size.attr, &dev_attr_wq_max_batch_size.attr, + &dev_attr_wq_ats_disable.attr, NULL, }; -- cgit v1.2.3-59-g8ed1b From 4facce4c12638c5636cbe9d4b8a4a879bebe8570 Mon Sep 17 00:00:00 2001 From: Jonathan McDowell Date: Thu, 26 Nov 2020 18:46:02 +0000 Subject: dmaengine: qcom: Fix ADM driver kerneldoc markup Update the kerneldoc function headers to fix build warnings: drivers/dma/qcom/qcom_adm.c:180: warning: Function parameter or member 'chan' not described in 'adm_free_chan' drivers/dma/qcom/qcom_adm.c:190: warning: Function parameter or member 'burst' not described in 'adm_get_blksize' drivers/dma/qcom/qcom_adm.c:466: warning: Function parameter or member 'chan' not described in 'adm_terminate_all' drivers/dma/qcom/qcom_adm.c:466: warning: Excess function parameter 'achan' description in 'adm_terminate_all' drivers/dma/qcom/qcom_adm.c:503: warning: Function parameter or member 'achan' not described in 'adm_start_dma' Signed-off-by: Jonathan McDowell Link: https://lore.kernel.org/r/20201126184602.GA1008@earth.li Signed-off-by: Vinod Koul --- drivers/dma/qcom/qcom_adm.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/drivers/dma/qcom/qcom_adm.c b/drivers/dma/qcom/qcom_adm.c index 9b6f8e050ecc..ee78bed8d60d 100644 --- a/drivers/dma/qcom/qcom_adm.c +++ b/drivers/dma/qcom/qcom_adm.c @@ -173,8 +173,9 @@ struct adm_device { /** * adm_free_chan - Frees dma resources associated with the specific channel * - * Free all allocated descriptors associated with this channel + * @chan: dma channel * + * Free all allocated descriptors associated with this channel */ static void adm_free_chan(struct dma_chan *chan) { @@ -185,6 +186,7 @@ static void adm_free_chan(struct dma_chan *chan) /** * adm_get_blksize - Get block size from burst value * + * @burst: Burst size of transaction */ static int adm_get_blksize(unsigned int burst) { @@ -456,7 +458,7 @@ free: /** * adm_terminate_all - terminate all transactions on a channel - * @achan: adm dma channel + * @chan: dma channel * * Dequeues and frees all transactions, aborts current transaction * No callbacks are done @@ -497,7 +499,7 @@ static int adm_slave_config(struct dma_chan *chan, struct dma_slave_config *cfg) /** * adm_start_dma - start next transaction - * @achan - ADM dma channel + * @achan: ADM dma channel */ static void adm_start_dma(struct adm_chan *achan) { -- cgit v1.2.3-59-g8ed1b From 51b69c9679de9bcb45b846807d75bab7ce9c6fda Mon Sep 17 00:00:00 2001 From: Fabien Parent Date: Wed, 9 Dec 2020 12:47:35 +0100 Subject: dt-bindings: dma: mtk-apdma: add bindings for MT8516 SOC Add bindings to APDMA for MT8516 SoC. MT8516 is compatible with MT6577. Signed-off-by: Fabien Parent Reviewed-by: Matthias Brugger Acked-by: Rob Herring Link: https://lore.kernel.org/r/20201209114736.70625-1-fparent@baylibre.com Signed-off-by: Vinod Koul --- Documentation/devicetree/bindings/dma/mtk-uart-apdma.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/Documentation/devicetree/bindings/dma/mtk-uart-apdma.txt b/Documentation/devicetree/bindings/dma/mtk-uart-apdma.txt index 2117db0ce4f2..fef9c1eeb264 100644 --- a/Documentation/devicetree/bindings/dma/mtk-uart-apdma.txt +++ b/Documentation/devicetree/bindings/dma/mtk-uart-apdma.txt @@ -4,6 +4,7 @@ Required properties: - compatible should contain: * "mediatek,mt2712-uart-dma" for MT2712 compatible APDMA * "mediatek,mt6577-uart-dma" for MT6577 and all of the above + * "mediatek,mt8516-uart-dma", "mediatek,mt6577" for MT8516 SoC - reg: The base address of the APDMA register bank. -- cgit v1.2.3-59-g8ed1b From f25b463883a8a2d1b7303a63339c0d589fc94f1e Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Tue, 17 Nov 2020 13:39:14 -0700 Subject: dmaengine: idxd: add IAX configuration support in the IDXD driver Add support to allow configuration of Intel Analytics Accelerator (IAX) in addition to the Intel Data Streaming Accelerator (DSA). The IAX hardware has the same configuration interface as DSA. The main difference is the type of operations it performs. We can support the DSA and IAX devices on the same driver with some tweaks. IAX has a 64B completion record that needs to be 64B aligned, as opposed to a 32B completion record that is 32B aligned for DSA. IAX also does not support token management. Signed-off-by: Dave Jiang Link: https://lore.kernel.org/r/160564555488.1834439.4261958859935360473.stgit@djiang5-desk3.ch.intel.com Signed-off-by: Vinod Koul --- drivers/dma/idxd/cdev.c | 1 + drivers/dma/idxd/device.c | 37 ++++++++++++++++----- drivers/dma/idxd/idxd.h | 24 +++++++++++--- drivers/dma/idxd/init.c | 14 ++++++++ drivers/dma/idxd/registers.h | 1 + drivers/dma/idxd/submit.c | 2 +- drivers/dma/idxd/sysfs.c | 46 ++++++++++++++++++++++++-- include/uapi/linux/idxd.h | 79 ++++++++++++++++++++++++++++++++++++++++++++ 8 files changed, 187 insertions(+), 17 deletions(-) diff --git a/drivers/dma/idxd/cdev.c b/drivers/dma/idxd/cdev.c index 010b820d8f74..0db9b82ed8cf 100644 --- a/drivers/dma/idxd/cdev.c +++ b/drivers/dma/idxd/cdev.c @@ -28,6 +28,7 @@ struct idxd_cdev_context { */ static struct idxd_cdev_context ictx[IDXD_TYPE_MAX] = { { .name = "dsa" }, + { .name = "iax" } }; struct idxd_user_context { diff --git a/drivers/dma/idxd/device.c b/drivers/dma/idxd/device.c index b75f9a09666e..47ff8e387172 100644 --- a/drivers/dma/idxd/device.c +++ b/drivers/dma/idxd/device.c @@ -131,6 +131,8 @@ int idxd_wq_alloc_resources(struct idxd_wq *wq) struct idxd_device *idxd = wq->idxd; struct device *dev = &idxd->pdev->dev; int rc, num_descs, i; + int align; + u64 tmp; if (wq->type != IDXD_WQT_KERNEL) return 0; @@ -142,14 +144,27 @@ int idxd_wq_alloc_resources(struct idxd_wq *wq) if (rc < 0) return rc; - wq->compls_size = num_descs * sizeof(struct dsa_completion_record); - wq->compls = dma_alloc_coherent(dev, wq->compls_size, - &wq->compls_addr, GFP_KERNEL); - if (!wq->compls) { + if (idxd->type == IDXD_TYPE_DSA) + align = 32; + else if (idxd->type == IDXD_TYPE_IAX) + align = 64; + else + return -ENODEV; + + wq->compls_size = num_descs * idxd->compl_size + align; + wq->compls_raw = dma_alloc_coherent(dev, wq->compls_size, + &wq->compls_addr_raw, GFP_KERNEL); + if (!wq->compls_raw) { rc = -ENOMEM; goto fail_alloc_compls; } + /* Adjust alignment */ + wq->compls_addr = (wq->compls_addr_raw + (align - 1)) & ~(align - 1); + tmp = (u64)wq->compls_raw; + tmp = (tmp + (align - 1)) & ~(align - 1); + wq->compls = (struct dsa_completion_record *)tmp; + rc = alloc_descs(wq, num_descs); if (rc < 0) goto fail_alloc_descs; @@ -163,9 +178,11 @@ int idxd_wq_alloc_resources(struct idxd_wq *wq) struct idxd_desc *desc = wq->descs[i]; desc->hw = wq->hw_descs[i]; - desc->completion = &wq->compls[i]; - desc->compl_dma = wq->compls_addr + - sizeof(struct dsa_completion_record) * i; + if (idxd->type == IDXD_TYPE_DSA) + desc->completion = &wq->compls[i]; + else if (idxd->type == IDXD_TYPE_IAX) + desc->iax_completion = &wq->iax_compls[i]; + desc->compl_dma = wq->compls_addr + idxd->compl_size * i; desc->id = i; desc->wq = wq; desc->cpu = -1; @@ -178,7 +195,8 @@ int idxd_wq_alloc_resources(struct idxd_wq *wq) fail_sbitmap_init: free_descs(wq); fail_alloc_descs: - dma_free_coherent(dev, wq->compls_size, wq->compls, wq->compls_addr); + dma_free_coherent(dev, wq->compls_size, wq->compls_raw, + wq->compls_addr_raw); fail_alloc_compls: free_hw_descs(wq); return rc; @@ -193,7 +211,8 @@ void idxd_wq_free_resources(struct idxd_wq *wq) free_hw_descs(wq); free_descs(wq); - dma_free_coherent(dev, wq->compls_size, wq->compls, wq->compls_addr); + dma_free_coherent(dev, wq->compls_size, wq->compls_raw, + wq->compls_addr_raw); sbitmap_queue_free(&wq->sbq); } diff --git a/drivers/dma/idxd/idxd.h b/drivers/dma/idxd/idxd.h index 149934f8d097..5a50e91c71bf 100644 --- a/drivers/dma/idxd/idxd.h +++ b/drivers/dma/idxd/idxd.h @@ -20,7 +20,8 @@ extern struct kmem_cache *idxd_desc_pool; enum idxd_type { IDXD_TYPE_UNKNOWN = -1, IDXD_TYPE_DSA = 0, - IDXD_TYPE_MAX + IDXD_TYPE_IAX, + IDXD_TYPE_MAX, }; #define IDXD_NAME_SIZE 128 @@ -114,8 +115,13 @@ struct idxd_wq { u32 vec_ptr; /* interrupt steering */ struct dsa_hw_desc **hw_descs; int num_descs; - struct dsa_completion_record *compls; + union { + struct dsa_completion_record *compls; + struct iax_completion_record *iax_compls; + }; + void *compls_raw; dma_addr_t compls_addr; + dma_addr_t compls_addr_raw; int compls_size; struct idxd_desc **descs; struct sbitmap_queue sbq; @@ -196,6 +202,7 @@ struct idxd_device { int token_limit; int nr_tokens; /* non-reserved tokens */ unsigned int wqcfg_size; + int compl_size; union sw_err_reg sw_err; wait_queue_head_t cmd_waitq; @@ -210,9 +217,15 @@ struct idxd_device { /* IDXD software descriptor */ struct idxd_desc { - struct dsa_hw_desc *hw; + union { + struct dsa_hw_desc *hw; + struct iax_hw_desc *iax_hw; + }; dma_addr_t desc_dma; - struct dsa_completion_record *completion; + union { + struct dsa_completion_record *completion; + struct iax_completion_record *iax_completion; + }; dma_addr_t compl_dma; struct dma_async_tx_descriptor txd; struct llist_node llnode; @@ -226,6 +239,7 @@ struct idxd_desc { #define confdev_to_wq(dev) container_of(dev, struct idxd_wq, conf_dev) extern struct bus_type dsa_bus_type; +extern struct bus_type iax_bus_type; extern bool support_enqcmd; @@ -271,6 +285,8 @@ static inline void idxd_set_type(struct idxd_device *idxd) if (pdev->device == PCI_DEVICE_ID_INTEL_DSA_SPR0) idxd->type = IDXD_TYPE_DSA; + else if (pdev->device == PCI_DEVICE_ID_INTEL_IAX_SPR0) + idxd->type = IDXD_TYPE_IAX; else idxd->type = IDXD_TYPE_UNKNOWN; } diff --git a/drivers/dma/idxd/init.c b/drivers/dma/idxd/init.c index 45b0eac640c3..2c051e07c34c 100644 --- a/drivers/dma/idxd/init.c +++ b/drivers/dma/idxd/init.c @@ -36,12 +36,16 @@ static struct mutex idxd_idr_lock; static struct pci_device_id idxd_pci_tbl[] = { /* DSA ver 1.0 platforms */ { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_DSA_SPR0) }, + + /* IAX ver 1.0 platforms */ + { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IAX_SPR0) }, { 0, } }; MODULE_DEVICE_TABLE(pci, idxd_pci_tbl); static char *idxd_name[] = { "dsa", + "iax" }; const char *idxd_get_dev_name(struct idxd_device *idxd) @@ -377,6 +381,14 @@ static int idxd_probe(struct idxd_device *idxd) return rc; } +static void idxd_type_init(struct idxd_device *idxd) +{ + if (idxd->type == IDXD_TYPE_DSA) + idxd->compl_size = sizeof(struct dsa_completion_record); + else if (idxd->type == IDXD_TYPE_IAX) + idxd->compl_size = sizeof(struct iax_completion_record); +} + static int idxd_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) { struct device *dev = &pdev->dev; @@ -412,6 +424,8 @@ static int idxd_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) idxd_set_type(idxd); + idxd_type_init(idxd); + dev_dbg(dev, "Set PCI master\n"); pci_set_master(pdev); pci_set_drvdata(pdev, idxd); diff --git a/drivers/dma/idxd/registers.h b/drivers/dma/idxd/registers.h index 0cdc5405bc53..23c41fe52215 100644 --- a/drivers/dma/idxd/registers.h +++ b/drivers/dma/idxd/registers.h @@ -5,6 +5,7 @@ /* PCI Config */ #define PCI_DEVICE_ID_INTEL_DSA_SPR0 0x0b25 +#define PCI_DEVICE_ID_INTEL_IAX_SPR0 0x0cfe #define IDXD_MMIO_BAR 0 #define IDXD_WQ_BAR 2 diff --git a/drivers/dma/idxd/submit.c b/drivers/dma/idxd/submit.c index efca5d8468a6..0ff64eeb84be 100644 --- a/drivers/dma/idxd/submit.c +++ b/drivers/dma/idxd/submit.c @@ -15,7 +15,7 @@ static struct idxd_desc *__get_desc(struct idxd_wq *wq, int idx, int cpu) desc = wq->descs[idx]; memset(desc->hw, 0, sizeof(struct dsa_hw_desc)); - memset(desc->completion, 0, sizeof(struct dsa_completion_record)); + memset(desc->completion, 0, idxd->compl_size); desc->cpu = cpu; if (device_pasid_enabled(idxd)) diff --git a/drivers/dma/idxd/sysfs.c b/drivers/dma/idxd/sysfs.c index 3af83f1fd36e..266423a2cabc 100644 --- a/drivers/dma/idxd/sysfs.c +++ b/drivers/dma/idxd/sysfs.c @@ -41,14 +41,24 @@ static struct device_type dsa_device_type = { .release = idxd_conf_device_release, }; +static struct device_type iax_device_type = { + .name = "iax", + .release = idxd_conf_device_release, +}; + static inline bool is_dsa_dev(struct device *dev) { return dev ? dev->type == &dsa_device_type : false; } +static inline bool is_iax_dev(struct device *dev) +{ + return dev ? dev->type == &iax_device_type : false; +} + static inline bool is_idxd_dev(struct device *dev) { - return is_dsa_dev(dev); + return is_dsa_dev(dev) || is_iax_dev(dev); } static inline bool is_idxd_wq_dev(struct device *dev) @@ -359,8 +369,17 @@ struct bus_type dsa_bus_type = { .shutdown = idxd_config_bus_shutdown, }; +struct bus_type iax_bus_type = { + .name = "iax", + .match = idxd_config_bus_match, + .probe = idxd_config_bus_probe, + .remove = idxd_config_bus_remove, + .shutdown = idxd_config_bus_shutdown, +}; + static struct bus_type *idxd_bus_types[] = { - &dsa_bus_type + &dsa_bus_type, + &iax_bus_type }; static struct idxd_device_driver dsa_drv = { @@ -372,8 +391,18 @@ static struct idxd_device_driver dsa_drv = { }, }; +static struct idxd_device_driver iax_drv = { + .drv = { + .name = "iax", + .bus = &iax_bus_type, + .owner = THIS_MODULE, + .mod_name = KBUILD_MODNAME, + }, +}; + static struct idxd_device_driver *idxd_drvs[] = { - &dsa_drv + &dsa_drv, + &iax_drv }; struct bus_type *idxd_get_bus_type(struct idxd_device *idxd) @@ -385,6 +414,8 @@ static struct device_type *idxd_get_device_type(struct idxd_device *idxd) { if (idxd->type == IDXD_TYPE_DSA) return &dsa_device_type; + else if (idxd->type == IDXD_TYPE_IAX) + return &iax_device_type; else return NULL; } @@ -525,6 +556,9 @@ static ssize_t group_tokens_reserved_store(struct device *dev, if (rc < 0) return -EINVAL; + if (idxd->type == IDXD_TYPE_IAX) + return -EOPNOTSUPP; + if (!test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags)) return -EPERM; @@ -570,6 +604,9 @@ static ssize_t group_tokens_allowed_store(struct device *dev, if (rc < 0) return -EINVAL; + if (idxd->type == IDXD_TYPE_IAX) + return -EOPNOTSUPP; + if (!test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags)) return -EPERM; @@ -612,6 +649,9 @@ static ssize_t group_use_token_limit_store(struct device *dev, if (rc < 0) return -EINVAL; + if (idxd->type == IDXD_TYPE_IAX) + return -EOPNOTSUPP; + if (!test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags)) return -EPERM; diff --git a/include/uapi/linux/idxd.h b/include/uapi/linux/idxd.h index fdcdfe414223..236d437947bc 100644 --- a/include/uapi/linux/idxd.h +++ b/include/uapi/linux/idxd.h @@ -26,6 +26,9 @@ #define IDXD_OP_FLAG_DRDBK 0x4000 #define IDXD_OP_FLAG_DSTS 0x8000 +/* IAX */ +#define IDXD_OP_FLAG_RD_SRC2_AECS 0x010000 + /* Opcode */ enum dsa_opcode { DSA_OPCODE_NOOP = 0, @@ -47,6 +50,14 @@ enum dsa_opcode { DSA_OPCODE_CFLUSH = 0x20, }; +enum iax_opcode { + IAX_OPCODE_NOOP = 0, + IAX_OPCODE_DRAIN = 2, + IAX_OPCODE_MEMMOVE, + IAX_OPCODE_DECOMPRESS = 0x42, + IAX_OPCODE_COMPRESS, +}; + /* Completion record status */ enum dsa_completion_status { DSA_COMP_NONE = 0, @@ -80,6 +91,33 @@ enum dsa_completion_status { DSA_COMP_TRANSLATION_FAIL, }; +enum iax_completion_status { + IAX_COMP_NONE = 0, + IAX_COMP_SUCCESS, + IAX_COMP_PAGE_FAULT_IR = 0x04, + IAX_COMP_OUTBUF_OVERFLOW, + IAX_COMP_BAD_OPCODE = 0x10, + IAX_COMP_INVALID_FLAGS, + IAX_COMP_NOZERO_RESERVE, + IAX_COMP_INVALID_SIZE, + IAX_COMP_OVERLAP_BUFFERS = 0x16, + IAX_COMP_INT_HANDLE_INVAL = 0x19, + IAX_COMP_CRA_XLAT, + IAX_COMP_CRA_ALIGN, + IAX_COMP_ADDR_ALIGN, + IAX_COMP_PRIV_BAD, + IAX_COMP_TRAFFIC_CLASS_CONF, + IAX_COMP_PFAULT_RDBA, + IAX_COMP_HW_ERR1, + IAX_COMP_HW_ERR_DRB, + IAX_COMP_TRANSLATION_FAIL, + IAX_COMP_PRS_TIMEOUT, + IAX_COMP_WATCHDOG, + IAX_COMP_INVALID_COMP_FLAG = 0x30, + IAX_COMP_INVALID_FILTER_FLAG, + IAX_COMP_INVALID_NUM_ELEMS = 0x33, +}; + #define DSA_COMP_STATUS_MASK 0x7f #define DSA_COMP_STATUS_WRITE 0x80 @@ -163,6 +201,28 @@ struct dsa_hw_desc { }; } __attribute__((packed)); +struct iax_hw_desc { + uint32_t pasid:20; + uint32_t rsvd:11; + uint32_t priv:1; + uint32_t flags:24; + uint32_t opcode:8; + uint64_t completion_addr; + uint64_t src1_addr; + uint64_t dst_addr; + uint32_t src1_size; + uint16_t int_handle; + union { + uint16_t compr_flags; + uint16_t decompr_flags; + }; + uint64_t src2_addr; + uint32_t max_dst_size; + uint32_t src2_size; + uint32_t filter_flags; + uint32_t num_inputs; +} __attribute__((packed)); + struct dsa_raw_desc { uint64_t field[8]; } __attribute__((packed)); @@ -223,4 +283,23 @@ struct dsa_raw_completion_record { uint64_t field[4]; } __attribute__((packed)); +struct iax_completion_record { + volatile uint8_t status; + uint8_t error_code; + uint16_t rsvd; + uint32_t bytes_completed; + uint64_t fault_addr; + uint32_t invalid_flags; + uint32_t rsvd2; + uint32_t output_size; + uint8_t output_bits; + uint8_t rsvd3; + uint16_t rsvd4; + uint64_t rsvd5[4]; +} __attribute__((packed)); + +struct iax_raw_completion_record { + uint64_t field[8]; +} __attribute__((packed)); + #endif -- cgit v1.2.3-59-g8ed1b From 4421fe533296e070359573ab6d320d74f73c80b9 Mon Sep 17 00:00:00 2001 From: Parth Y Shah Date: Mon, 7 Dec 2020 12:03:40 +0530 Subject: dmaengine: bam_dma: fix return of bam_dma_irq() While performing suspend/resume, we were getting below kernel crash. [ 54.541672] [FTS][Info]gesture suspend... [ 54.605256] [FTS][Error][GESTURE]Enter into gesture(suspend) failed! [ 54.605256] [ 58.345850] irq event 10: bogus return value fffffff3 ...... [ 58.345966] [] el1_irq+0xb0/0x124 [ 58.345971] [] arch_cpu_idle+0x10/0x18 [ 58.345975] [] do_idle+0x1ac/0x1e0 [ 58.345979] [] cpu_startup_entry+0x20/0x28 [ 58.345983] [] rest_init+0xd0/0xdc [ 58.345988] [] start_kernel+0x390/0x3a4 [ 58.345990] handlers: [ 58.345994] [] bam_dma_irq The reason for the crash we found is, bam_dma_irq() was returning negative value when the device resumes in some conditions. In addition, the irq handler should have one of the below return values. IRQ_NONE interrupt was not from this device or was not handled IRQ_HANDLED interrupt was handled by this device IRQ_WAKE_THREAD handler requests to wake the handler thread Therefore, to resolve this crash, we have changed the return value to IRQ_NONE. Signed-off-by: Parth Y Shah Link: https://lore.kernel.org/r/1607322820-7450-1-git-send-email-sparth1292@gmail.com Signed-off-by: Vinod Koul --- drivers/dma/qcom/bam_dma.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/dma/qcom/bam_dma.c b/drivers/dma/qcom/bam_dma.c index 4eeb8bb27279..d5773d474d8f 100644 --- a/drivers/dma/qcom/bam_dma.c +++ b/drivers/dma/qcom/bam_dma.c @@ -875,7 +875,7 @@ static irqreturn_t bam_dma_irq(int irq, void *data) ret = bam_pm_runtime_get_sync(bdev->dev); if (ret < 0) - return ret; + return IRQ_NONE; if (srcs & BAM_IRQ) { clr_mask = readl_relaxed(bam_addr(bdev, 0, BAM_IRQ_STTS)); -- cgit v1.2.3-59-g8ed1b From a44d9d72453ea6b064380d4835e712e574e58d9b Mon Sep 17 00:00:00 2001 From: Amelie Delaunay Date: Fri, 20 Nov 2020 15:33:17 +0100 Subject: dmaengine: stm32-dma: rework irq handler to manage error before xfer events To better understand error that can be detected by the DMA controller, manage the error flags before the transfer flags. This way, it is possible to know if the FIFO error flag is set for an over/underrun condition or a FIFO level error. When a FIFO over/underrun condition occurs, the data is not lost because peripheral request is not acknowledged by the stream until the over/ underrun condition is cleared. If this acknowledge takes too much time, the peripheral itself may detect an over/underrun condition of its internal buffer and data might be lost. That's why in case the FIFO error flag is set, we check if the channel is disabled or not, and if a Transfer Complete flag is set, which means that the channel is disabled because of the end of transfer. Because channel is disabled by hardware either by a FIFO level error, or by an end of transfer. Signed-off-by: Amelie Delaunay Link: https://lore.kernel.org/r/20201120143320.30367-2-amelie.delaunay@st.com Signed-off-by: Vinod Koul --- drivers/dma/stm32-dma.c | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/drivers/dma/stm32-dma.c b/drivers/dma/stm32-dma.c index d0055d2f0b9a..55a6bd381219 100644 --- a/drivers/dma/stm32-dma.c +++ b/drivers/dma/stm32-dma.c @@ -648,21 +648,12 @@ static irqreturn_t stm32_dma_chan_irq(int irq, void *devid) scr = stm32_dma_read(dmadev, STM32_DMA_SCR(chan->id)); sfcr = stm32_dma_read(dmadev, STM32_DMA_SFCR(chan->id)); - if (status & STM32_DMA_TCI) { - stm32_dma_irq_clear(chan, STM32_DMA_TCI); - if (scr & STM32_DMA_SCR_TCIE) - stm32_dma_handle_chan_done(chan); - status &= ~STM32_DMA_TCI; - } - if (status & STM32_DMA_HTI) { - stm32_dma_irq_clear(chan, STM32_DMA_HTI); - status &= ~STM32_DMA_HTI; - } if (status & STM32_DMA_FEI) { stm32_dma_irq_clear(chan, STM32_DMA_FEI); status &= ~STM32_DMA_FEI; if (sfcr & STM32_DMA_SFCR_FEIE) { - if (!(scr & STM32_DMA_SCR_EN)) + if (!(scr & STM32_DMA_SCR_EN) && + !(status & STM32_DMA_TCI)) dev_err(chan2dev(chan), "FIFO Error\n"); else dev_dbg(chan2dev(chan), "FIFO over/underrun\n"); @@ -674,6 +665,19 @@ static irqreturn_t stm32_dma_chan_irq(int irq, void *devid) if (sfcr & STM32_DMA_SCR_DMEIE) dev_dbg(chan2dev(chan), "Direct mode overrun\n"); } + + if (status & STM32_DMA_TCI) { + stm32_dma_irq_clear(chan, STM32_DMA_TCI); + if (scr & STM32_DMA_SCR_TCIE) + stm32_dma_handle_chan_done(chan); + status &= ~STM32_DMA_TCI; + } + + if (status & STM32_DMA_HTI) { + stm32_dma_irq_clear(chan, STM32_DMA_HTI); + status &= ~STM32_DMA_HTI; + } + if (status) { stm32_dma_irq_clear(chan, status); dev_err(chan2dev(chan), "DMA error: status=0x%08x\n", status); -- cgit v1.2.3-59-g8ed1b From 5d4d4dfbda18063231a95dea28fdeab148f23301 Mon Sep 17 00:00:00 2001 From: Amelie Delaunay Date: Fri, 20 Nov 2020 15:33:18 +0100 Subject: dmaengine: stm32-dma: clean channel configuration when channel is freed When dma_channel_release is called, it means that the channel won't be used anymore with the configuration it had. To ensure a future client can safely use the channel after it has been released, clean the configuration done when channel was requested. Signed-off-by: Amelie Delaunay Link: https://lore.kernel.org/r/20201120143320.30367-3-amelie.delaunay@st.com Signed-off-by: Vinod Koul --- drivers/dma/stm32-dma.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/drivers/dma/stm32-dma.c b/drivers/dma/stm32-dma.c index 55a6bd381219..62501e5d9e9d 100644 --- a/drivers/dma/stm32-dma.c +++ b/drivers/dma/stm32-dma.c @@ -1220,6 +1220,8 @@ static void stm32_dma_free_chan_resources(struct dma_chan *c) pm_runtime_put(dmadev->ddev.dev); vchan_free_chan_resources(to_virt_chan(c)); + stm32_dma_clear_reg(&chan->chan_reg); + chan->threshold = 0; } static void stm32_dma_desc_free(struct virt_dma_desc *vdesc) -- cgit v1.2.3-59-g8ed1b From e0ebdbdcb42a66f49b7587dc50cc6f528ec55cad Mon Sep 17 00:00:00 2001 From: Amelie Delaunay Date: Fri, 20 Nov 2020 15:33:19 +0100 Subject: dmaengine: stm32-dma: take address into account when computing max width DMA_SxPAR or DMA_SxM0AR/M1AR registers have to be aligned on PSIZE or MSIZE respectively. This means that bus width needs to be forced to 1 byte when computed width is not aligned with address. Signed-off-by: Amelie Delaunay Link: https://lore.kernel.org/r/20201120143320.30367-4-amelie.delaunay@st.com Signed-off-by: Vinod Koul --- drivers/dma/stm32-dma.c | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/drivers/dma/stm32-dma.c b/drivers/dma/stm32-dma.c index 62501e5d9e9d..f54ecb123a52 100644 --- a/drivers/dma/stm32-dma.c +++ b/drivers/dma/stm32-dma.c @@ -264,9 +264,11 @@ static int stm32_dma_get_width(struct stm32_dma_chan *chan, } static enum dma_slave_buswidth stm32_dma_get_max_width(u32 buf_len, + dma_addr_t buf_addr, u32 threshold) { enum dma_slave_buswidth max_width; + u64 addr = buf_addr; if (threshold == STM32_DMA_FIFO_THRESHOLD_FULL) max_width = DMA_SLAVE_BUSWIDTH_4_BYTES; @@ -277,6 +279,9 @@ static enum dma_slave_buswidth stm32_dma_get_max_width(u32 buf_len, max_width > DMA_SLAVE_BUSWIDTH_1_BYTE) max_width = max_width >> 1; + if (do_div(addr, max_width)) + max_width = DMA_SLAVE_BUSWIDTH_1_BYTE; + return max_width; } @@ -707,7 +712,7 @@ static void stm32_dma_issue_pending(struct dma_chan *c) static int stm32_dma_set_xfer_param(struct stm32_dma_chan *chan, enum dma_transfer_direction direction, enum dma_slave_buswidth *buswidth, - u32 buf_len) + u32 buf_len, dma_addr_t buf_addr) { enum dma_slave_buswidth src_addr_width, dst_addr_width; int src_bus_width, dst_bus_width; @@ -739,7 +744,8 @@ static int stm32_dma_set_xfer_param(struct stm32_dma_chan *chan, return dst_burst_size; /* Set memory data size */ - src_addr_width = stm32_dma_get_max_width(buf_len, fifoth); + src_addr_width = stm32_dma_get_max_width(buf_len, buf_addr, + fifoth); chan->mem_width = src_addr_width; src_bus_width = stm32_dma_get_width(chan, src_addr_width); if (src_bus_width < 0) @@ -788,7 +794,8 @@ static int stm32_dma_set_xfer_param(struct stm32_dma_chan *chan, return src_burst_size; /* Set memory data size */ - dst_addr_width = stm32_dma_get_max_width(buf_len, fifoth); + dst_addr_width = stm32_dma_get_max_width(buf_len, buf_addr, + fifoth); chan->mem_width = dst_addr_width; dst_bus_width = stm32_dma_get_width(chan, dst_addr_width); if (dst_bus_width < 0) @@ -876,7 +883,8 @@ static struct dma_async_tx_descriptor *stm32_dma_prep_slave_sg( for_each_sg(sgl, sg, sg_len, i) { ret = stm32_dma_set_xfer_param(chan, direction, &buswidth, - sg_dma_len(sg)); + sg_dma_len(sg), + sg_dma_address(sg)); if (ret < 0) goto err; @@ -944,7 +952,8 @@ static struct dma_async_tx_descriptor *stm32_dma_prep_dma_cyclic( return NULL; } - ret = stm32_dma_set_xfer_param(chan, direction, &buswidth, period_len); + ret = stm32_dma_set_xfer_param(chan, direction, &buswidth, period_len, + buf_addr); if (ret < 0) return NULL; -- cgit v1.2.3-59-g8ed1b From 1d3dd68749b9f4a4da272f39608d03b4bae0b69f Mon Sep 17 00:00:00 2001 From: Amelie Delaunay Date: Fri, 20 Nov 2020 15:33:20 +0100 Subject: dmaengine: stm32-mdma: rework interrupt handler To avoid multiple entries in MDMA interrupt handler for each flag&interrupt enable, manage all flags set at once. Signed-off-by: Amelie Delaunay Link: https://lore.kernel.org/r/20201120143320.30367-5-amelie.delaunay@st.com Signed-off-by: Vinod Koul --- drivers/dma/stm32-mdma.c | 64 +++++++++++++++++++++++++----------------------- 1 file changed, 34 insertions(+), 30 deletions(-) diff --git a/drivers/dma/stm32-mdma.c b/drivers/dma/stm32-mdma.c index 29e5e30524bb..e4637ec786d3 100644 --- a/drivers/dma/stm32-mdma.c +++ b/drivers/dma/stm32-mdma.c @@ -1346,7 +1346,7 @@ static irqreturn_t stm32_mdma_irq_handler(int irq, void *devid) { struct stm32_mdma_device *dmadev = devid; struct stm32_mdma_chan *chan = devid; - u32 reg, id, ien, status, flag; + u32 reg, id, ccr, ien, status; /* Find out which channel generates the interrupt */ status = readl_relaxed(dmadev->base + STM32_MDMA_GISR0); @@ -1368,67 +1368,71 @@ static irqreturn_t stm32_mdma_irq_handler(int irq, void *devid) chan = &dmadev->chan[id]; if (!chan) { - dev_dbg(mdma2dev(dmadev), "MDMA channel not initialized\n"); - goto exit; + dev_warn(mdma2dev(dmadev), "MDMA channel not initialized\n"); + return IRQ_NONE; } /* Handle interrupt for the channel */ spin_lock(&chan->vchan.lock); - status = stm32_mdma_read(dmadev, STM32_MDMA_CISR(chan->id)); - ien = stm32_mdma_read(dmadev, STM32_MDMA_CCR(chan->id)); - ien &= STM32_MDMA_CCR_IRQ_MASK; - ien >>= 1; + status = stm32_mdma_read(dmadev, STM32_MDMA_CISR(id)); + /* Mask Channel ReQuest Active bit which can be set in case of MEM2MEM */ + status &= ~STM32_MDMA_CISR_CRQA; + ccr = stm32_mdma_read(dmadev, STM32_MDMA_CCR(id)); + ien = (ccr & STM32_MDMA_CCR_IRQ_MASK) >> 1; if (!(status & ien)) { spin_unlock(&chan->vchan.lock); - dev_dbg(chan2dev(chan), - "spurious it (status=0x%04x, ien=0x%04x)\n", - status, ien); + dev_warn(chan2dev(chan), + "spurious it (status=0x%04x, ien=0x%04x)\n", + status, ien); return IRQ_NONE; } - flag = __ffs(status & ien); - reg = STM32_MDMA_CIFCR(chan->id); + reg = STM32_MDMA_CIFCR(id); - switch (1 << flag) { - case STM32_MDMA_CISR_TEIF: - id = chan->id; - status = readl_relaxed(dmadev->base + STM32_MDMA_CESR(id)); - dev_err(chan2dev(chan), "Transfer Err: stat=0x%08x\n", status); + if (status & STM32_MDMA_CISR_TEIF) { + dev_err(chan2dev(chan), "Transfer Err: stat=0x%08x\n", + readl_relaxed(dmadev->base + STM32_MDMA_CESR(id))); stm32_mdma_set_bits(dmadev, reg, STM32_MDMA_CIFCR_CTEIF); - break; + status &= ~STM32_MDMA_CISR_TEIF; + } - case STM32_MDMA_CISR_CTCIF: + if (status & STM32_MDMA_CISR_CTCIF) { stm32_mdma_set_bits(dmadev, reg, STM32_MDMA_CIFCR_CCTCIF); + status &= ~STM32_MDMA_CISR_CTCIF; stm32_mdma_xfer_end(chan); - break; + } - case STM32_MDMA_CISR_BRTIF: + if (status & STM32_MDMA_CISR_BRTIF) { stm32_mdma_set_bits(dmadev, reg, STM32_MDMA_CIFCR_CBRTIF); - break; + status &= ~STM32_MDMA_CISR_BRTIF; + } - case STM32_MDMA_CISR_BTIF: + if (status & STM32_MDMA_CISR_BTIF) { stm32_mdma_set_bits(dmadev, reg, STM32_MDMA_CIFCR_CBTIF); + status &= ~STM32_MDMA_CISR_BTIF; chan->curr_hwdesc++; if (chan->desc && chan->desc->cyclic) { if (chan->curr_hwdesc == chan->desc->count) chan->curr_hwdesc = 0; vchan_cyclic_callback(&chan->desc->vdesc); } - break; + } - case STM32_MDMA_CISR_TCIF: + if (status & STM32_MDMA_CISR_TCIF) { stm32_mdma_set_bits(dmadev, reg, STM32_MDMA_CIFCR_CLTCIF); - break; + status &= ~STM32_MDMA_CISR_TCIF; + } - default: - dev_err(chan2dev(chan), "it %d unhandled (status=0x%04x)\n", - 1 << flag, status); + if (status) { + stm32_mdma_set_bits(dmadev, reg, status); + dev_err(chan2dev(chan), "DMA error: status=0x%08x\n", status); + if (!(ccr & STM32_MDMA_CCR_EN)) + dev_err(chan2dev(chan), "chan disabled by HW\n"); } spin_unlock(&chan->vchan.lock); -exit: return IRQ_HANDLED; } -- cgit v1.2.3-59-g8ed1b From e2de925bbfe321ba0588c99f577c59386ab1f428 Mon Sep 17 00:00:00 2001 From: Peter Ujfalusi Date: Tue, 8 Dec 2020 11:04:21 +0200 Subject: dmaengine: ti: k3-udma: Correct normal channel offset when uchan_cnt is not 0 According to different sections of the TRM, the hchan_cnt of CAP3 includes the number of uchan in UDMA, thus the start offset of the normal channels are hchan_cnt. Fixes: daf4ad0499aa4 ("dmaengine: ti: k3-udma: Query throughput level information from hardware") Signed-off-by: Peter Ujfalusi Link: https://lore.kernel.org/r/20201208090440.31792-2-peter.ujfalusi@ti.com Signed-off-by: Vinod Koul --- drivers/dma/ti/k3-udma.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/drivers/dma/ti/k3-udma.c b/drivers/dma/ti/k3-udma.c index e508280b3d70..0e8426dd18a7 100644 --- a/drivers/dma/ti/k3-udma.c +++ b/drivers/dma/ti/k3-udma.c @@ -3199,8 +3199,7 @@ static int udma_setup_resources(struct udma_dev *ud) } else if (UDMA_CAP3_UCHAN_CNT(cap3)) { ud->tpl_levels = 3; ud->tpl_start_idx[1] = UDMA_CAP3_UCHAN_CNT(cap3); - ud->tpl_start_idx[0] = ud->tpl_start_idx[1] + - UDMA_CAP3_HCHAN_CNT(cap3); + ud->tpl_start_idx[0] = UDMA_CAP3_HCHAN_CNT(cap3); } else if (UDMA_CAP3_HCHAN_CNT(cap3)) { ud->tpl_levels = 2; ud->tpl_start_idx[0] = UDMA_CAP3_HCHAN_CNT(cap3); -- cgit v1.2.3-59-g8ed1b From 5e1cb1cb0f9fe670900d736822a7dbcd7c11dbba Mon Sep 17 00:00:00 2001 From: Peter Ujfalusi Date: Tue, 8 Dec 2020 11:04:22 +0200 Subject: dmaengine: ti: k3-udma: Wait for peer teardown completion if supported Set the TDTYPE if it is supported on the platform (j721e) which will cause UDMAP to wait for the remote peer to finish the teardown before returning the teardown completed message. Signed-off-by: Peter Ujfalusi Tested-by: Keerthy Reviewed-by: Grygorii Strashko Link: https://lore.kernel.org/r/20201208090440.31792-3-peter.ujfalusi@ti.com Signed-off-by: Vinod Koul --- drivers/dma/ti/k3-udma.c | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/drivers/dma/ti/k3-udma.c b/drivers/dma/ti/k3-udma.c index 0e8426dd18a7..eee43757e774 100644 --- a/drivers/dma/ti/k3-udma.c +++ b/drivers/dma/ti/k3-udma.c @@ -86,6 +86,7 @@ struct udma_rchan { #define UDMA_FLAG_PDMA_ACC32 BIT(0) #define UDMA_FLAG_PDMA_BURST BIT(1) +#define UDMA_FLAG_TDTYPE BIT(2) struct udma_match_data { u32 psil_base; @@ -1589,6 +1590,13 @@ static int udma_tisci_tx_channel_config(struct udma_chan *uc) req_tx.tx_fetch_size = fetch_size >> 2; req_tx.txcq_qnum = tc_ring; req_tx.tx_atype = uc->config.atype; + if (uc->config.ep_type == PSIL_EP_PDMA_XY && + ud->match_data->flags & UDMA_FLAG_TDTYPE) { + /* wait for peer to complete the teardown for PDMAs */ + req_tx.valid_params |= + TI_SCI_MSG_VALUE_RM_UDMAP_CH_TX_TDTYPE_VALID; + req_tx.tx_tdtype = 1; + } ret = tisci_ops->tx_ch_cfg(tisci_rm->tisci, &req_tx); if (ret) @@ -3105,14 +3113,14 @@ static struct udma_match_data am654_mcu_data = { static struct udma_match_data j721e_main_data = { .psil_base = 0x1000, .enable_memcpy_support = true, - .flags = UDMA_FLAG_PDMA_ACC32 | UDMA_FLAG_PDMA_BURST, + .flags = UDMA_FLAG_PDMA_ACC32 | UDMA_FLAG_PDMA_BURST | UDMA_FLAG_TDTYPE, .statictr_z_mask = GENMASK(23, 0), }; static struct udma_match_data j721e_mcu_data = { .psil_base = 0x6000, .enable_memcpy_support = false, /* MEM_TO_MEM is slow via MCU UDMA */ - .flags = UDMA_FLAG_PDMA_ACC32 | UDMA_FLAG_PDMA_BURST, + .flags = UDMA_FLAG_PDMA_ACC32 | UDMA_FLAG_PDMA_BURST | UDMA_FLAG_TDTYPE, .statictr_z_mask = GENMASK(23, 0), }; -- cgit v1.2.3-59-g8ed1b From 1609c15a20b8e0c1adc2a26ad81fd7e2b968f81a Mon Sep 17 00:00:00 2001 From: Peter Ujfalusi Date: Tue, 8 Dec 2020 11:04:23 +0200 Subject: dmaengine: ti: k3-udma: Add support for second resource range from sysfw Resource allocation via sysfw can use up to two ranges per resource subtype to support more complex resource assignment, mainly for DMA channels. Take the second range also into consideration when setting up the maps for available resources. Signed-off-by: Peter Ujfalusi Link: https://lore.kernel.org/r/20201208090440.31792-4-peter.ujfalusi@ti.com Signed-off-by: Vinod Koul --- drivers/dma/ti/k3-udma.c | 55 +++++++++++++++++++++++++++--------------------- 1 file changed, 31 insertions(+), 24 deletions(-) diff --git a/drivers/dma/ti/k3-udma.c b/drivers/dma/ti/k3-udma.c index eee43757e774..b89afa602532 100644 --- a/drivers/dma/ti/k3-udma.c +++ b/drivers/dma/ti/k3-udma.c @@ -3174,12 +3174,22 @@ static int udma_get_mmrs(struct platform_device *pdev, struct udma_dev *ud) return 0; } +static void udma_mark_resource_ranges(struct udma_dev *ud, unsigned long *map, + struct ti_sci_resource_desc *rm_desc, + char *name) +{ + bitmap_clear(map, rm_desc->start, rm_desc->num); + bitmap_clear(map, rm_desc->start_sec, rm_desc->num_sec); + dev_dbg(ud->dev, "ti_sci resource range for %s: %d:%d | %d:%d\n", name, + rm_desc->start, rm_desc->num, rm_desc->start_sec, + rm_desc->num_sec); +} + static int udma_setup_resources(struct udma_dev *ud) { struct device *dev = ud->dev; int ch_count, ret, i, j; u32 cap2, cap3; - struct ti_sci_resource_desc *rm_desc; struct ti_sci_resource *rm_res, irq_res; struct udma_tisci_rm *tisci_rm = &ud->tisci_rm; static const char * const range_names[] = { "ti,sci-rm-range-tchan", @@ -3264,13 +3274,9 @@ static int udma_setup_resources(struct udma_dev *ud) bitmap_zero(ud->tchan_map, ud->tchan_cnt); } else { bitmap_fill(ud->tchan_map, ud->tchan_cnt); - for (i = 0; i < rm_res->sets; i++) { - rm_desc = &rm_res->desc[i]; - bitmap_clear(ud->tchan_map, rm_desc->start, - rm_desc->num); - dev_dbg(dev, "ti-sci-res: tchan: %d:%d\n", - rm_desc->start, rm_desc->num); - } + for (i = 0; i < rm_res->sets; i++) + udma_mark_resource_ranges(ud, ud->tchan_map, + &rm_res->desc[i], "tchan"); } irq_res.sets = rm_res->sets; @@ -3280,13 +3286,9 @@ static int udma_setup_resources(struct udma_dev *ud) bitmap_zero(ud->rchan_map, ud->rchan_cnt); } else { bitmap_fill(ud->rchan_map, ud->rchan_cnt); - for (i = 0; i < rm_res->sets; i++) { - rm_desc = &rm_res->desc[i]; - bitmap_clear(ud->rchan_map, rm_desc->start, - rm_desc->num); - dev_dbg(dev, "ti-sci-res: rchan: %d:%d\n", - rm_desc->start, rm_desc->num); - } + for (i = 0; i < rm_res->sets; i++) + udma_mark_resource_ranges(ud, ud->rchan_map, + &rm_res->desc[i], "rchan"); } irq_res.sets += rm_res->sets; @@ -3295,12 +3297,21 @@ static int udma_setup_resources(struct udma_dev *ud) for (i = 0; i < rm_res->sets; i++) { irq_res.desc[i].start = rm_res->desc[i].start; irq_res.desc[i].num = rm_res->desc[i].num; + irq_res.desc[i].start_sec = rm_res->desc[i].start_sec; + irq_res.desc[i].num_sec = rm_res->desc[i].num_sec; } rm_res = tisci_rm->rm_ranges[RM_RANGE_RCHAN]; for (j = 0; j < rm_res->sets; j++, i++) { - irq_res.desc[i].start = rm_res->desc[j].start + + if (rm_res->desc[j].num) { + irq_res.desc[i].start = rm_res->desc[j].start + ud->soc_data->rchan_oes_offset; - irq_res.desc[i].num = rm_res->desc[j].num; + irq_res.desc[i].num = rm_res->desc[j].num; + } + if (rm_res->desc[j].num_sec) { + irq_res.desc[i].start_sec = rm_res->desc[j].start_sec + + ud->soc_data->rchan_oes_offset; + irq_res.desc[i].num_sec = rm_res->desc[j].num_sec; + } } ret = ti_sci_inta_msi_domain_alloc_irqs(ud->dev, &irq_res); kfree(irq_res.desc); @@ -3316,13 +3327,9 @@ static int udma_setup_resources(struct udma_dev *ud) bitmap_clear(ud->rflow_gp_map, ud->rchan_cnt, ud->rflow_cnt - ud->rchan_cnt); } else { - for (i = 0; i < rm_res->sets; i++) { - rm_desc = &rm_res->desc[i]; - bitmap_clear(ud->rflow_gp_map, rm_desc->start, - rm_desc->num); - dev_dbg(dev, "ti-sci-res: rflow: %d:%d\n", - rm_desc->start, rm_desc->num); - } + for (i = 0; i < rm_res->sets; i++) + udma_mark_resource_ranges(ud, ud->rflow_gp_map, + &rm_res->desc[i], "gp-rflow"); } ch_count -= bitmap_weight(ud->tchan_map, ud->tchan_cnt); -- cgit v1.2.3-59-g8ed1b From 426506a7e0f1902268c3edbdc7e5475624a9d18b Mon Sep 17 00:00:00 2001 From: Peter Ujfalusi Date: Tue, 8 Dec 2020 11:04:24 +0200 Subject: dmaengine: ti: k3-udma-glue: Add function to get device pointer for DMA API Glue layer users should use the device of the DMA for DMA mapping and allocations as it is the DMA which accesses to descriptors and buffers, not the clients Signed-off-by: Peter Ujfalusi Reviewed-by: Grygorii Strashko Link: https://lore.kernel.org/r/20201208090440.31792-5-peter.ujfalusi@ti.com Signed-off-by: Vinod Koul --- drivers/dma/ti/k3-udma-glue.c | 14 ++++++++++++++ drivers/dma/ti/k3-udma-private.c | 6 ++++++ drivers/dma/ti/k3-udma.h | 1 + include/linux/dma/k3-udma-glue.h | 4 ++++ 4 files changed, 25 insertions(+) diff --git a/drivers/dma/ti/k3-udma-glue.c b/drivers/dma/ti/k3-udma-glue.c index dfb65e382ab9..29d1524d1916 100644 --- a/drivers/dma/ti/k3-udma-glue.c +++ b/drivers/dma/ti/k3-udma-glue.c @@ -493,6 +493,13 @@ int k3_udma_glue_tx_get_irq(struct k3_udma_glue_tx_channel *tx_chn) } EXPORT_SYMBOL_GPL(k3_udma_glue_tx_get_irq); +struct device * + k3_udma_glue_tx_get_dma_device(struct k3_udma_glue_tx_channel *tx_chn) +{ + return xudma_get_device(tx_chn->common.udmax); +} +EXPORT_SYMBOL_GPL(k3_udma_glue_tx_get_dma_device); + static int k3_udma_glue_cfg_rx_chn(struct k3_udma_glue_rx_channel *rx_chn) { const struct udma_tisci_rm *tisci_rm = rx_chn->common.tisci_rm; @@ -1201,3 +1208,10 @@ int k3_udma_glue_rx_get_irq(struct k3_udma_glue_rx_channel *rx_chn, return flow->virq; } EXPORT_SYMBOL_GPL(k3_udma_glue_rx_get_irq); + +struct device * + k3_udma_glue_rx_get_dma_device(struct k3_udma_glue_rx_channel *rx_chn) +{ + return xudma_get_device(rx_chn->common.udmax); +} +EXPORT_SYMBOL_GPL(k3_udma_glue_rx_get_dma_device); diff --git a/drivers/dma/ti/k3-udma-private.c b/drivers/dma/ti/k3-udma-private.c index aa24e554f7b4..8ff7a264be03 100644 --- a/drivers/dma/ti/k3-udma-private.c +++ b/drivers/dma/ti/k3-udma-private.c @@ -50,6 +50,12 @@ struct udma_dev *of_xudma_dev_get(struct device_node *np, const char *property) } EXPORT_SYMBOL(of_xudma_dev_get); +struct device *xudma_get_device(struct udma_dev *ud) +{ + return ud->dev; +} +EXPORT_SYMBOL(xudma_get_device); + u32 xudma_dev_get_psil_base(struct udma_dev *ud) { return ud->psil_base; diff --git a/drivers/dma/ti/k3-udma.h b/drivers/dma/ti/k3-udma.h index 09c4529e013d..d1cace0cb43b 100644 --- a/drivers/dma/ti/k3-udma.h +++ b/drivers/dma/ti/k3-udma.h @@ -112,6 +112,7 @@ int xudma_navss_psil_unpair(struct udma_dev *ud, u32 src_thread, u32 dst_thread); struct udma_dev *of_xudma_dev_get(struct device_node *np, const char *property); +struct device *xudma_get_device(struct udma_dev *ud); void xudma_dev_put(struct udma_dev *ud); u32 xudma_dev_get_psil_base(struct udma_dev *ud); struct udma_tisci_rm *xudma_dev_get_tisci_rm(struct udma_dev *ud); diff --git a/include/linux/dma/k3-udma-glue.h b/include/linux/dma/k3-udma-glue.h index 5eb34ad973a7..d7c12f31377c 100644 --- a/include/linux/dma/k3-udma-glue.h +++ b/include/linux/dma/k3-udma-glue.h @@ -41,6 +41,8 @@ void k3_udma_glue_reset_tx_chn(struct k3_udma_glue_tx_channel *tx_chn, u32 k3_udma_glue_tx_get_hdesc_size(struct k3_udma_glue_tx_channel *tx_chn); u32 k3_udma_glue_tx_get_txcq_id(struct k3_udma_glue_tx_channel *tx_chn); int k3_udma_glue_tx_get_irq(struct k3_udma_glue_tx_channel *tx_chn); +struct device * + k3_udma_glue_tx_get_dma_device(struct k3_udma_glue_tx_channel *tx_chn); enum { K3_UDMA_GLUE_SRC_TAG_LO_KEEP = 0, @@ -130,5 +132,7 @@ int k3_udma_glue_rx_flow_enable(struct k3_udma_glue_rx_channel *rx_chn, u32 flow_idx); int k3_udma_glue_rx_flow_disable(struct k3_udma_glue_rx_channel *rx_chn, u32 flow_idx); +struct device * + k3_udma_glue_rx_get_dma_device(struct k3_udma_glue_rx_channel *rx_chn); #endif /* K3_UDMA_GLUE_H_ */ -- cgit v1.2.3-59-g8ed1b From aa8a4c4edad0bed7aaf3a7cfcae9fa555d847955 Mon Sep 17 00:00:00 2001 From: Peter Ujfalusi Date: Tue, 8 Dec 2020 11:04:25 +0200 Subject: dmaengine: ti: k3-udma-glue: Get the ringacc from udma_dev If of_xudma_dev_get() returns with the valid udma_dev then the driver already got the ringacc, there is no need to execute of_k3_ringacc_get_by_phandle() for each channel via the glue layer. Signed-off-by: Peter Ujfalusi Reviewed-by: Grygorii Strashko Link: https://lore.kernel.org/r/20201208090440.31792-6-peter.ujfalusi@ti.com Signed-off-by: Vinod Koul --- drivers/dma/ti/k3-udma-glue.c | 6 +----- drivers/dma/ti/k3-udma-private.c | 6 ++++++ drivers/dma/ti/k3-udma.h | 1 + 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/drivers/dma/ti/k3-udma-glue.c b/drivers/dma/ti/k3-udma-glue.c index 29d1524d1916..8a8988be4175 100644 --- a/drivers/dma/ti/k3-udma-glue.c +++ b/drivers/dma/ti/k3-udma-glue.c @@ -86,15 +86,11 @@ struct k3_udma_glue_rx_channel { static int of_k3_udma_glue_parse(struct device_node *udmax_np, struct k3_udma_glue_common *common) { - common->ringacc = of_k3_ringacc_get_by_phandle(udmax_np, - "ti,ringacc"); - if (IS_ERR(common->ringacc)) - return PTR_ERR(common->ringacc); - common->udmax = of_xudma_dev_get(udmax_np, NULL); if (IS_ERR(common->udmax)) return PTR_ERR(common->udmax); + common->ringacc = xudma_get_ringacc(common->udmax); common->tisci_rm = xudma_dev_get_tisci_rm(common->udmax); return 0; diff --git a/drivers/dma/ti/k3-udma-private.c b/drivers/dma/ti/k3-udma-private.c index 8ff7a264be03..346a4dd9640a 100644 --- a/drivers/dma/ti/k3-udma-private.c +++ b/drivers/dma/ti/k3-udma-private.c @@ -56,6 +56,12 @@ struct device *xudma_get_device(struct udma_dev *ud) } EXPORT_SYMBOL(xudma_get_device); +struct k3_ringacc *xudma_get_ringacc(struct udma_dev *ud) +{ + return ud->ringacc; +} +EXPORT_SYMBOL(xudma_get_ringacc); + u32 xudma_dev_get_psil_base(struct udma_dev *ud) { return ud->psil_base; diff --git a/drivers/dma/ti/k3-udma.h b/drivers/dma/ti/k3-udma.h index d1cace0cb43b..b4334b1b7b14 100644 --- a/drivers/dma/ti/k3-udma.h +++ b/drivers/dma/ti/k3-udma.h @@ -113,6 +113,7 @@ int xudma_navss_psil_unpair(struct udma_dev *ud, u32 src_thread, struct udma_dev *of_xudma_dev_get(struct device_node *np, const char *property); struct device *xudma_get_device(struct udma_dev *ud); +struct k3_ringacc *xudma_get_ringacc(struct udma_dev *ud); void xudma_dev_put(struct udma_dev *ud); u32 xudma_dev_get_psil_base(struct udma_dev *ud); struct udma_tisci_rm *xudma_dev_get_tisci_rm(struct udma_dev *ud); -- cgit v1.2.3-59-g8ed1b From d553e2ab0137ae489b41824b1e8283053c363ed1 Mon Sep 17 00:00:00 2001 From: Peter Ujfalusi Date: Tue, 8 Dec 2020 11:04:26 +0200 Subject: dmaengine: ti: k3-udma-glue: Configure the dma_dev for rings Rings in RING mode should be using the DMA device for DMA API as in this mode the ringacc will not access the ring memory in any ways, but the DMA is. Fix up the ring configuration and set the dma_dev unconditionally and let the ringacc driver to select the correct device to use for DMA API. Signed-off-by: Peter Ujfalusi Reviewed-by: Grygorii Strashko Link: https://lore.kernel.org/r/20201208090440.31792-7-peter.ujfalusi@ti.com Signed-off-by: Vinod Koul --- drivers/dma/ti/k3-udma-glue.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/drivers/dma/ti/k3-udma-glue.c b/drivers/dma/ti/k3-udma-glue.c index 8a8988be4175..e6ebcd98c02a 100644 --- a/drivers/dma/ti/k3-udma-glue.c +++ b/drivers/dma/ti/k3-udma-glue.c @@ -276,6 +276,10 @@ struct k3_udma_glue_tx_channel *k3_udma_glue_request_tx_chn(struct device *dev, goto err; } + /* Set the dma_dev for the rings to be configured */ + cfg->tx_cfg.dma_dev = k3_udma_glue_tx_get_dma_device(tx_chn); + cfg->txcq_cfg.dma_dev = cfg->tx_cfg.dma_dev; + ret = k3_ringacc_ring_cfg(tx_chn->ringtx, &cfg->tx_cfg); if (ret) { dev_err(dev, "Failed to cfg ringtx %d\n", ret); @@ -591,6 +595,10 @@ static int k3_udma_glue_cfg_rx_flow(struct k3_udma_glue_rx_channel *rx_chn, goto err_rflow_put; } + /* Set the dma_dev for the rings to be configured */ + flow_cfg->rx_cfg.dma_dev = k3_udma_glue_rx_get_dma_device(rx_chn); + flow_cfg->rxfdq_cfg.dma_dev = flow_cfg->rx_cfg.dma_dev; + ret = k3_ringacc_ring_cfg(flow->ringrx, &flow_cfg->rx_cfg); if (ret) { dev_err(dev, "Failed to cfg ringrx %d\n", ret); -- cgit v1.2.3-59-g8ed1b From 4f910c035f38053ac8eb63a672c78862c535cd0f Mon Sep 17 00:00:00 2001 From: Peter Ujfalusi Date: Tue, 8 Dec 2020 11:04:27 +0200 Subject: dmaengine: of-dma: Add support for optional router configuration callback Additional configuration for the DMA event router might be needed for a channel which can not be done during device_alloc_chan_resources callback since the router information is not yet present for the drivers. If there is a need for additional configuration for the channel if DMA router is in use, then the driver can implement the device_router_config callback. Signed-off-by: Peter Ujfalusi Link: https://lore.kernel.org/r/20201208090440.31792-8-peter.ujfalusi@ti.com Signed-off-by: Vinod Koul --- drivers/dma/of-dma.c | 10 ++++++++++ include/linux/dmaengine.h | 2 ++ 2 files changed, 12 insertions(+) diff --git a/drivers/dma/of-dma.c b/drivers/dma/of-dma.c index 8a4f608904b9..ec00b20ae8e4 100644 --- a/drivers/dma/of-dma.c +++ b/drivers/dma/of-dma.c @@ -75,8 +75,18 @@ static struct dma_chan *of_dma_router_xlate(struct of_phandle_args *dma_spec, ofdma->dma_router->route_free(ofdma->dma_router->dev, route_data); } else { + int ret = 0; + chan->router = ofdma->dma_router; chan->route_data = route_data; + + if (chan->device->device_router_config) + ret = chan->device->device_router_config(chan); + + if (ret) { + dma_release_channel(chan); + chan = ERR_PTR(ret); + } } /* diff --git a/include/linux/dmaengine.h b/include/linux/dmaengine.h index 493a047ed0a2..aed44888cad3 100644 --- a/include/linux/dmaengine.h +++ b/include/linux/dmaengine.h @@ -805,6 +805,7 @@ struct dma_filter { * by tx_status * @device_alloc_chan_resources: allocate resources and return the * number of allocated descriptors + * @device_router_config: optional callback for DMA router configuration * @device_free_chan_resources: release DMA channel's resources * @device_prep_dma_memcpy: prepares a memcpy operation * @device_prep_dma_xor: prepares a xor operation @@ -879,6 +880,7 @@ struct dma_device { enum dma_residue_granularity residue_granularity; int (*device_alloc_chan_resources)(struct dma_chan *chan); + int (*device_router_config)(struct dma_chan *chan); void (*device_free_chan_resources)(struct dma_chan *chan); struct dma_async_tx_descriptor *(*device_prep_dma_memcpy)( -- cgit v1.2.3-59-g8ed1b From ab650ef6d548153862119e1bf3bf267510707f48 Mon Sep 17 00:00:00 2001 From: Peter Ujfalusi Date: Tue, 8 Dec 2020 11:04:28 +0200 Subject: dmaengine: Add support for per channel coherency handling If the DMA device supports per channel coherency configuration (a channel can be configured to have coherent or not coherent view) then a single device (the DMA controller's device) can not be used for dma_api for all channels as channels can have different coherency. Introduce custom_dma_mapping flag for the dma_chan and a new helper to get the device pointer to be used for dma_api for the given channel. Client drivers should be updated to be able to support per channel coherency by: - dma_map_single(chan->device->dev, ptr, size, DMA_TO_DEVICE); + struct device *dma_dev = dmaengine_get_dma_device(chan); + + dma_map_single(dma_dev, ptr, size, DMA_TO_DEVICE); Signed-off-by: Peter Ujfalusi Link: https://lore.kernel.org/r/20201208090440.31792-9-peter.ujfalusi@ti.com Signed-off-by: Vinod Koul --- include/linux/dmaengine.h | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/include/linux/dmaengine.h b/include/linux/dmaengine.h index aed44888cad3..68130f5f599e 100644 --- a/include/linux/dmaengine.h +++ b/include/linux/dmaengine.h @@ -357,11 +357,14 @@ struct dma_chan { * @chan: driver channel device * @device: sysfs device * @dev_id: parent dma_device dev_id + * @chan_dma_dev: The channel is using custom/different dma-mapping + * compared to the parent dma_device */ struct dma_chan_dev { struct dma_chan *chan; struct device device; int dev_id; + bool chan_dma_dev; }; /** @@ -1618,4 +1621,13 @@ dmaengine_get_direction_text(enum dma_transfer_direction dir) return "invalid"; } } + +static inline struct device *dmaengine_get_dma_device(struct dma_chan *chan) +{ + if (chan->dev->chan_dma_dev) + return &chan->dev->device; + + return chan->device->dev; +} + #endif /* DMAENGINE_H */ -- cgit v1.2.3-59-g8ed1b From f082c6df970e6e9aa97af35e826fec824007fbae Mon Sep 17 00:00:00 2001 From: Peter Ujfalusi Date: Tue, 8 Dec 2020 11:04:29 +0200 Subject: dmaengine: doc: client: Update for dmaengine_get_dma_device() usage Client drivers should use the dmaengine_get_dma_device(chan) to get the device pointer which should be used for DMA API for allocations and mapping. Signed-off-by: Peter Ujfalusi Link: https://lore.kernel.org/r/20201208090440.31792-10-peter.ujfalusi@ti.com Signed-off-by: Vinod Koul --- Documentation/driver-api/dmaengine/client.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Documentation/driver-api/dmaengine/client.rst b/Documentation/driver-api/dmaengine/client.rst index 09a3f66dcd26..bfd057b21a00 100644 --- a/Documentation/driver-api/dmaengine/client.rst +++ b/Documentation/driver-api/dmaengine/client.rst @@ -120,7 +120,9 @@ The details of these operations are: .. code-block:: c - nr_sg = dma_map_sg(chan->device->dev, sgl, sg_len); + struct device *dma_dev = dmaengine_get_dma_device(chan); + + nr_sg = dma_map_sg(dma_dev, sgl, sg_len); if (nr_sg == 0) /* error */ -- cgit v1.2.3-59-g8ed1b From adc0f941f9a8fae7e79b249b0abcdbf86017ea53 Mon Sep 17 00:00:00 2001 From: Peter Ujfalusi Date: Tue, 8 Dec 2020 11:04:30 +0200 Subject: dmaengine: dmatest: Use dmaengine_get_dma_device By using the dmaengine_get_dma_device() to get the device for dma_api use, the dmatest can support per channel coherency if it is supported by the DMA controller. Signed-off-by: Peter Ujfalusi Link: https://lore.kernel.org/r/20201208090440.31792-11-peter.ujfalusi@ti.com Signed-off-by: Vinod Koul --- drivers/dma/dmatest.c | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/drivers/dma/dmatest.c b/drivers/dma/dmatest.c index a3a172173e34..f696246f57fd 100644 --- a/drivers/dma/dmatest.c +++ b/drivers/dma/dmatest.c @@ -573,6 +573,7 @@ static int dmatest_func(void *data) struct dmatest_params *params; struct dma_chan *chan; struct dma_device *dev; + struct device *dma_dev; unsigned int error_count; unsigned int failed_tests = 0; unsigned int total_tests = 0; @@ -606,6 +607,8 @@ static int dmatest_func(void *data) params = &info->params; chan = thread->chan; dev = chan->device; + dma_dev = dmaengine_get_dma_device(chan); + src = &thread->src; dst = &thread->dst; if (thread->type == DMA_MEMCPY) { @@ -730,7 +733,7 @@ static int dmatest_func(void *data) filltime = ktime_add(filltime, diff); } - um = dmaengine_get_unmap_data(dev->dev, src->cnt + dst->cnt, + um = dmaengine_get_unmap_data(dma_dev, src->cnt + dst->cnt, GFP_KERNEL); if (!um) { failed_tests++; @@ -745,10 +748,10 @@ static int dmatest_func(void *data) struct page *pg = virt_to_page(buf); unsigned long pg_off = offset_in_page(buf); - um->addr[i] = dma_map_page(dev->dev, pg, pg_off, + um->addr[i] = dma_map_page(dma_dev, pg, pg_off, um->len, DMA_TO_DEVICE); srcs[i] = um->addr[i] + src->off; - ret = dma_mapping_error(dev->dev, um->addr[i]); + ret = dma_mapping_error(dma_dev, um->addr[i]); if (ret) { result("src mapping error", total_tests, src->off, dst->off, len, ret); @@ -763,9 +766,9 @@ static int dmatest_func(void *data) struct page *pg = virt_to_page(buf); unsigned long pg_off = offset_in_page(buf); - dsts[i] = dma_map_page(dev->dev, pg, pg_off, um->len, + dsts[i] = dma_map_page(dma_dev, pg, pg_off, um->len, DMA_BIDIRECTIONAL); - ret = dma_mapping_error(dev->dev, dsts[i]); + ret = dma_mapping_error(dma_dev, dsts[i]); if (ret) { result("dst mapping error", total_tests, src->off, dst->off, len, ret); -- cgit v1.2.3-59-g8ed1b From 991b96e0f12234e2a7e66d67275981b53572acd8 Mon Sep 17 00:00:00 2001 From: Peter Ujfalusi Date: Tue, 8 Dec 2020 11:04:31 +0200 Subject: dt-bindings: dma: ti: Add document for K3 BCDMA New binding document for Texas Instruments K3 Block Copy DMA (BCDMA). BCDMA is introduced as part of AM64. Signed-off-by: Peter Ujfalusi Reviewed-by: Rob Herring Link: https://lore.kernel.org/r/20201208090440.31792-12-peter.ujfalusi@ti.com Signed-off-by: Vinod Koul --- .../devicetree/bindings/dma/ti/k3-bcdma.yaml | 164 +++++++++++++++++++++ 1 file changed, 164 insertions(+) create mode 100644 Documentation/devicetree/bindings/dma/ti/k3-bcdma.yaml diff --git a/Documentation/devicetree/bindings/dma/ti/k3-bcdma.yaml b/Documentation/devicetree/bindings/dma/ti/k3-bcdma.yaml new file mode 100644 index 000000000000..b15f68c499cb --- /dev/null +++ b/Documentation/devicetree/bindings/dma/ti/k3-bcdma.yaml @@ -0,0 +1,164 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/dma/ti/k3-bcdma.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Texas Instruments K3 DMSS BCDMA Device Tree Bindings + +maintainers: + - Peter Ujfalusi + +description: | + The Block Copy DMA (BCDMA) is intended to perform similar functions as the TR + mode channels of K3 UDMA-P. + BCDMA includes block copy channels and Split channels. + + Block copy channels mainly used for memory to memory transfers, but with + optional triggers a block copy channel can service peripherals by accessing + directly to memory mapped registers or area. + + Split channels can be used to service PSI-L based peripherals. + The peripherals can be PSI-L native or legacy, non PSI-L native peripherals + with PDMAs. PDMA is tasked to act as a bridge between the PSI-L fabric and the + legacy peripheral. + + PDMAs can be configured via BCDMA split channel's peer registers to match with + the configuration of the legacy peripheral. + +allOf: + - $ref: /schemas/dma/dma-controller.yaml# + +properties: + compatible: + const: ti,am64-dmss-bcdma + + "#dma-cells": + const: 3 + description: | + cell 1: type of the BCDMA channel to be used to service the peripheral: + 0 - split channel + 1 - block copy channel using global trigger 1 + 2 - block copy channel using global trigger 2 + 3 - block copy channel using local trigger + + cell 2: parameter for the channel: + if cell 1 is 0 (split channel): + PSI-L thread ID of the remote (to BCDMA) end. + Valid ranges for thread ID depends on the data movement direction: + for source thread IDs (rx): 0 - 0x7fff + for destination thread IDs (tx): 0x8000 - 0xffff + + Please refer to the device documentation for the PSI-L thread map and + also the PSI-L peripheral chapter for the correct thread ID. + if cell 1 is 1 or 2 (block copy channel using global trigger): + Unused, ignored + + The trigger must be configured for the channel externally to BCDMA, + channels using global triggers should not be requested directly, but + via DMA event router. + if cell 1 is 3 (block copy channel using local trigger): + bchan number of the locally triggered channel + + cell 3: ASEL value for the channel + + reg: + maxItems: 5 + + reg-names: + items: + - const: gcfg + - const: bchanrt + - const: rchanrt + - const: tchanrt + - const: ringrt + + msi-parent: true + + ti,asel: + $ref: /schemas/types.yaml#/definitions/uint32 + description: ASEL value for non slave channels + + ti,sci-rm-range-bchan: + $ref: /schemas/types.yaml#/definitions/uint32-array + description: | + Array of BCDMA block-copy channel resource subtypes for resource + allocation for this host + minItems: 1 + # Should be enough + maxItems: 255 + items: + maximum: 0x3f + + ti,sci-rm-range-tchan: + $ref: /schemas/types.yaml#/definitions/uint32-array + description: | + Array of BCDMA split tx channel resource subtypes for resource allocation + for this host + minItems: 1 + # Should be enough + maxItems: 255 + items: + maximum: 0x3f + + ti,sci-rm-range-rchan: + $ref: /schemas/types.yaml#/definitions/uint32-array + description: | + Array of BCDMA split rx channel resource subtypes for resource allocation + for this host + minItems: 1 + # Should be enough + maxItems: 255 + items: + maximum: 0x3f + +required: + - compatible + - "#dma-cells" + - reg + - reg-names + - msi-parent + - ti,sci + - ti,sci-dev-id + - ti,sci-rm-range-bchan + - ti,sci-rm-range-tchan + - ti,sci-rm-range-rchan + +unevaluatedProperties: false + +examples: + - |+ + cbass_main { + #address-cells = <2>; + #size-cells = <2>; + + main_dmss { + compatible = "simple-mfd"; + #address-cells = <2>; + #size-cells = <2>; + dma-ranges; + ranges; + + ti,sci-dev-id = <25>; + + main_bcdma: dma-controller@485c0100 { + compatible = "ti,am64-dmss-bcdma"; + + reg = <0x0 0x485c0100 0x0 0x100>, + <0x0 0x4c000000 0x0 0x20000>, + <0x0 0x4a820000 0x0 0x20000>, + <0x0 0x4aa40000 0x0 0x20000>, + <0x0 0x4bc00000 0x0 0x100000>; + reg-names = "gcfg", "bchanrt", "rchanrt", "tchanrt", "ringrt"; + msi-parent = <&inta_main_dmss>; + #dma-cells = <3>; + + ti,sci = <&dmsc>; + ti,sci-dev-id = <26>; + + ti,sci-rm-range-bchan = <0x20>; /* BLOCK_COPY_CHAN */ + ti,sci-rm-range-rchan = <0x21>; /* SPLIT_TR_RX_CHAN */ + ti,sci-rm-range-tchan = <0x22>; /* SPLIT_TR_TX_CHAN */ + }; + }; + }; -- cgit v1.2.3-59-g8ed1b From 1d92cec649a38f9f4f488d5e359a27a4bb5f182d Mon Sep 17 00:00:00 2001 From: Peter Ujfalusi Date: Tue, 8 Dec 2020 11:04:32 +0200 Subject: dt-bindings: dma: ti: Add document for K3 PKTDMA New binding document for Texas Instruments K3 Packet DMA (PKTDMA). PKTDMA is introduced as part of AM64. Signed-off-by: Peter Ujfalusi Reviewed-by: Rob Herring Link: https://lore.kernel.org/r/20201208090440.31792-13-peter.ujfalusi@ti.com Signed-off-by: Vinod Koul --- .../devicetree/bindings/dma/ti/k3-pktdma.yaml | 172 +++++++++++++++++++++ 1 file changed, 172 insertions(+) create mode 100644 Documentation/devicetree/bindings/dma/ti/k3-pktdma.yaml diff --git a/Documentation/devicetree/bindings/dma/ti/k3-pktdma.yaml b/Documentation/devicetree/bindings/dma/ti/k3-pktdma.yaml new file mode 100644 index 000000000000..b13ab60cd740 --- /dev/null +++ b/Documentation/devicetree/bindings/dma/ti/k3-pktdma.yaml @@ -0,0 +1,172 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/dma/ti/k3-pktdma.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Texas Instruments K3 DMSS PKTDMA Device Tree Bindings + +maintainers: + - Peter Ujfalusi + +description: | + The Packet DMA (PKTDMA) is intended to perform similar functions as the packet + mode channels of K3 UDMA-P. + PKTDMA only includes Split channels to service PSI-L based peripherals. + + The peripherals can be PSI-L native or legacy, non PSI-L native peripherals + with PDMAs. PDMA is tasked to act as a bridge between the PSI-L fabric and the + legacy peripheral. + + PDMAs can be configured via PKTDMA split channel's peer registers to match + with the configuration of the legacy peripheral. + +allOf: + - $ref: /schemas/dma/dma-controller.yaml# + +properties: + compatible: + const: ti,am64-dmss-pktdma + + "#dma-cells": + const: 2 + description: | + The first cell is the PSI-L thread ID of the remote (to PKTDMA) end. + Valid ranges for thread ID depends on the data movement direction: + for source thread IDs (rx): 0 - 0x7fff + for destination thread IDs (tx): 0x8000 - 0xffff + + Please refer to the device documentation for the PSI-L thread map and also + the PSI-L peripheral chapter for the correct thread ID. + + The second cell is the ASEL value for the channel + + reg: + maxItems: 4 + + reg-names: + items: + - const: gcfg + - const: rchanrt + - const: tchanrt + - const: ringrt + + msi-parent: true + + ti,sci-rm-range-tchan: + $ref: /schemas/types.yaml#/definitions/uint32-array + description: | + Array of PKTDMA split tx channel resource subtypes for resource allocation + for this host + minItems: 1 + # Should be enough + maxItems: 255 + items: + maximum: 0x3f + + ti,sci-rm-range-tflow: + $ref: /schemas/types.yaml#/definitions/uint32-array + description: | + Array of PKTDMA split tx flow resource subtypes for resource allocation + for this host + minItems: 1 + # Should be enough + maxItems: 255 + items: + maximum: 0x3f + + ti,sci-rm-range-rchan: + $ref: /schemas/types.yaml#/definitions/uint32-array + description: | + Array of PKTDMA split rx channel resource subtypes for resource allocation + for this host + minItems: 1 + # Should be enough + maxItems: 255 + items: + maximum: 0x3f + + ti,sci-rm-range-rflow: + $ref: /schemas/types.yaml#/definitions/uint32-array + description: | + Array of PKTDMA split rx flow resource subtypes for resource allocation + for this host + minItems: 1 + # Should be enough + maxItems: 255 + items: + maximum: 0x3f + +required: + - compatible + - "#dma-cells" + - reg + - reg-names + - msi-parent + - ti,sci + - ti,sci-dev-id + - ti,sci-rm-range-tchan + - ti,sci-rm-range-tflow + - ti,sci-rm-range-rchan + - ti,sci-rm-range-rflow + +unevaluatedProperties: false + +examples: + - |+ + cbass_main { + #address-cells = <2>; + #size-cells = <2>; + + main_dmss { + compatible = "simple-mfd"; + #address-cells = <2>; + #size-cells = <2>; + dma-ranges; + ranges; + + ti,sci-dev-id = <25>; + + main_pktdma: dma-controller@485c0000 { + compatible = "ti,am64-dmss-pktdma"; + + reg = <0x0 0x485c0000 0x0 0x100>, + <0x0 0x4a800000 0x0 0x20000>, + <0x0 0x4aa00000 0x0 0x40000>, + <0x0 0x4b800000 0x0 0x400000>; + reg-names = "gcfg", "rchanrt", "tchanrt", "ringrt"; + msi-parent = <&inta_main_dmss>; + #dma-cells = <2>; + + ti,sci = <&dmsc>; + ti,sci-dev-id = <30>; + + ti,sci-rm-range-tchan = <0x23>, /* UNMAPPED_TX_CHAN */ + <0x24>, /* CPSW_TX_CHAN */ + <0x25>, /* SAUL_TX_0_CHAN */ + <0x26>, /* SAUL_TX_1_CHAN */ + <0x27>, /* ICSSG_0_TX_CHAN */ + <0x28>; /* ICSSG_1_TX_CHAN */ + ti,sci-rm-range-tflow = <0x10>, /* RING_UNMAPPED_TX_CHAN */ + <0x11>, /* RING_CPSW_TX_CHAN */ + <0x12>, /* RING_SAUL_TX_0_CHAN */ + <0x13>, /* RING_SAUL_TX_1_CHAN */ + <0x14>, /* RING_ICSSG_0_TX_CHAN */ + <0x15>; /* RING_ICSSG_1_TX_CHAN */ + ti,sci-rm-range-rchan = <0x29>, /* UNMAPPED_RX_CHAN */ + <0x2b>, /* CPSW_RX_CHAN */ + <0x2d>, /* SAUL_RX_0_CHAN */ + <0x2f>, /* SAUL_RX_1_CHAN */ + <0x31>, /* SAUL_RX_2_CHAN */ + <0x33>, /* SAUL_RX_3_CHAN */ + <0x35>, /* ICSSG_0_RX_CHAN */ + <0x37>; /* ICSSG_1_RX_CHAN */ + ti,sci-rm-range-rflow = <0x2a>, /* FLOW_UNMAPPED_RX_CHAN */ + <0x2c>, /* FLOW_CPSW_RX_CHAN */ + <0x2e>, /* FLOW_SAUL_RX_0/1_CHAN */ + <0x32>, /* FLOW_SAUL_RX_2/3_CHAN */ + <0x36>, /* FLOW_ICSSG_0_RX_CHAN */ + <0x38>; /* FLOW_ICSSG_1_RX_CHAN */ + }; + }; + }; -- cgit v1.2.3-59-g8ed1b From b9366e2577a38ca5322f326cff9752c2008597c6 Mon Sep 17 00:00:00 2001 From: Peter Ujfalusi Date: Tue, 8 Dec 2020 11:04:33 +0200 Subject: dmaengine: ti: k3-psil: Extend psil_endpoint_config for K3 PKTDMA Additional fields needed for K3 PKTDMA to be able to handle the mapped channels (channels are locked to handle specific threads) and flow ranges for these mapped threads. PKTDMA also introduces tflow for tx channels which can not be found in K3 UDMA architecture. Signed-off-by: Peter Ujfalusi Reviewed-by: Grygorii Strashko Link: https://lore.kernel.org/r/20201208090440.31792-14-peter.ujfalusi@ti.com Signed-off-by: Vinod Koul --- include/linux/dma/k3-psil.h | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/include/linux/dma/k3-psil.h b/include/linux/dma/k3-psil.h index 1962f75fa2d3..36e22c5a0f29 100644 --- a/include/linux/dma/k3-psil.h +++ b/include/linux/dma/k3-psil.h @@ -50,6 +50,15 @@ enum psil_endpoint_type { * @channel_tpl: Desired throughput level for the channel * @pdma_acc32: ACC32 must be enabled on the PDMA side * @pdma_burst: BURST must be enabled on the PDMA side + * @mapped_channel_id: PKTDMA thread to channel mapping for mapped channels. + * The thread must be serviced by the specified channel if + * mapped_channel_id is >= 0 in case of PKTDMA + * @flow_start: PKDMA flow range start of mapped channel. Unmapped + * channels use flow_id == chan_id + * @flow_num: PKDMA flow count of mapped channel. Unmapped channels + * use flow_id == chan_id + * @default_flow_id: PKDMA default (r)flow index of mapped channel. + * Must be within the flow range of the mapped channel. */ struct psil_endpoint_config { enum psil_endpoint_type ep_type; @@ -63,6 +72,13 @@ struct psil_endpoint_config { /* PDMA properties, valid for PSIL_EP_PDMA_* */ unsigned pdma_acc32:1; unsigned pdma_burst:1; + + /* PKDMA mapped channel */ + int mapped_channel_id; + /* PKTDMA tflow and rflow ranges for mapped channel */ + u16 flow_start; + u16 flow_num; + u16 default_flow_id; }; int psil_set_new_ep_config(struct device *dev, const char *name, -- cgit v1.2.3-59-g8ed1b From 2329725d1a228fb0d6424ee8d499f266020cc113 Mon Sep 17 00:00:00 2001 From: Peter Ujfalusi Date: Tue, 8 Dec 2020 11:04:34 +0200 Subject: dmaengine: ti: k3-psil: Add initial map for AM64 Add initial PSI-L map file for AM64. Signed-off-by: Peter Ujfalusi Link: https://lore.kernel.org/r/20201208090440.31792-15-peter.ujfalusi@ti.com Signed-off-by: Vinod Koul --- drivers/dma/ti/Makefile | 3 +- drivers/dma/ti/k3-psil-am64.c | 158 ++++++++++++++++++++++++++++++++++++++++++ drivers/dma/ti/k3-psil-priv.h | 1 + drivers/dma/ti/k3-psil.c | 1 + 4 files changed, 162 insertions(+), 1 deletion(-) create mode 100644 drivers/dma/ti/k3-psil-am64.c diff --git a/drivers/dma/ti/Makefile b/drivers/dma/ti/Makefile index 0c67254caee6..bd496efadff7 100644 --- a/drivers/dma/ti/Makefile +++ b/drivers/dma/ti/Makefile @@ -7,5 +7,6 @@ obj-$(CONFIG_TI_K3_UDMA_GLUE_LAYER) += k3-udma-glue.o obj-$(CONFIG_TI_K3_PSIL) += k3-psil.o \ k3-psil-am654.o \ k3-psil-j721e.o \ - k3-psil-j7200.o + k3-psil-j7200.o \ + k3-psil-am64.o obj-$(CONFIG_TI_DMA_CROSSBAR) += dma-crossbar.o diff --git a/drivers/dma/ti/k3-psil-am64.c b/drivers/dma/ti/k3-psil-am64.c new file mode 100644 index 000000000000..9fdeaa11a4fc --- /dev/null +++ b/drivers/dma/ti/k3-psil-am64.c @@ -0,0 +1,158 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2020 Texas Instruments Incorporated - https://www.ti.com + * Author: Peter Ujfalusi + */ + +#include + +#include "k3-psil-priv.h" + +#define PSIL_PDMA_XY_TR(x) \ + { \ + .thread_id = x, \ + .ep_config = { \ + .ep_type = PSIL_EP_PDMA_XY, \ + .mapped_channel_id = -1, \ + .default_flow_id = -1, \ + }, \ + } + +#define PSIL_PDMA_XY_PKT(x) \ + { \ + .thread_id = x, \ + .ep_config = { \ + .ep_type = PSIL_EP_PDMA_XY, \ + .mapped_channel_id = -1, \ + .default_flow_id = -1, \ + .pkt_mode = 1, \ + }, \ + } + +#define PSIL_ETHERNET(x, ch, flow_base, flow_cnt) \ + { \ + .thread_id = x, \ + .ep_config = { \ + .ep_type = PSIL_EP_NATIVE, \ + .pkt_mode = 1, \ + .needs_epib = 1, \ + .psd_size = 16, \ + .mapped_channel_id = ch, \ + .flow_start = flow_base, \ + .flow_num = flow_cnt, \ + .default_flow_id = flow_base, \ + }, \ + } + +#define PSIL_SAUL(x, ch, flow_base, flow_cnt, default_flow, tx) \ + { \ + .thread_id = x, \ + .ep_config = { \ + .ep_type = PSIL_EP_NATIVE, \ + .pkt_mode = 1, \ + .needs_epib = 1, \ + .psd_size = 64, \ + .mapped_channel_id = ch, \ + .flow_start = flow_base, \ + .flow_num = flow_cnt, \ + .default_flow_id = default_flow, \ + .notdpkt = tx, \ + }, \ + } + +/* PSI-L source thread IDs, used for RX (DMA_DEV_TO_MEM) */ +static struct psil_ep am64_src_ep_map[] = { + /* SAUL */ + PSIL_SAUL(0x4000, 17, 32, 8, 32, 0), + PSIL_SAUL(0x4001, 18, 32, 8, 33, 0), + PSIL_SAUL(0x4002, 19, 40, 8, 40, 0), + PSIL_SAUL(0x4003, 20, 40, 8, 41, 0), + /* ICSS_G0 */ + PSIL_ETHERNET(0x4100, 21, 48, 16), + PSIL_ETHERNET(0x4101, 22, 64, 16), + PSIL_ETHERNET(0x4102, 23, 80, 16), + PSIL_ETHERNET(0x4103, 24, 96, 16), + /* ICSS_G1 */ + PSIL_ETHERNET(0x4200, 25, 112, 16), + PSIL_ETHERNET(0x4201, 26, 128, 16), + PSIL_ETHERNET(0x4202, 27, 144, 16), + PSIL_ETHERNET(0x4203, 28, 160, 16), + /* PDMA_MAIN0 - SPI0-3 */ + PSIL_PDMA_XY_PKT(0x4300), + PSIL_PDMA_XY_PKT(0x4301), + PSIL_PDMA_XY_PKT(0x4302), + PSIL_PDMA_XY_PKT(0x4303), + PSIL_PDMA_XY_PKT(0x4304), + PSIL_PDMA_XY_PKT(0x4305), + PSIL_PDMA_XY_PKT(0x4306), + PSIL_PDMA_XY_PKT(0x4307), + PSIL_PDMA_XY_PKT(0x4308), + PSIL_PDMA_XY_PKT(0x4309), + PSIL_PDMA_XY_PKT(0x430a), + PSIL_PDMA_XY_PKT(0x430b), + PSIL_PDMA_XY_PKT(0x430c), + PSIL_PDMA_XY_PKT(0x430d), + PSIL_PDMA_XY_PKT(0x430e), + PSIL_PDMA_XY_PKT(0x430f), + /* PDMA_MAIN0 - USART0-1 */ + PSIL_PDMA_XY_PKT(0x4310), + PSIL_PDMA_XY_PKT(0x4311), + /* PDMA_MAIN1 - SPI4 */ + PSIL_PDMA_XY_PKT(0x4400), + PSIL_PDMA_XY_PKT(0x4401), + PSIL_PDMA_XY_PKT(0x4402), + PSIL_PDMA_XY_PKT(0x4403), + /* PDMA_MAIN1 - USART2-6 */ + PSIL_PDMA_XY_PKT(0x4404), + PSIL_PDMA_XY_PKT(0x4405), + PSIL_PDMA_XY_PKT(0x4406), + PSIL_PDMA_XY_PKT(0x4407), + PSIL_PDMA_XY_PKT(0x4408), + /* PDMA_MAIN1 - ADCs */ + PSIL_PDMA_XY_TR(0x440f), + PSIL_PDMA_XY_TR(0x4410), + /* CPSW2 */ + PSIL_ETHERNET(0x4500, 16, 16, 16), +}; + +/* PSI-L destination thread IDs, used for TX (DMA_MEM_TO_DEV) */ +static struct psil_ep am64_dst_ep_map[] = { + /* SAUL */ + PSIL_SAUL(0xc000, 24, 80, 8, 80, 1), + PSIL_SAUL(0xc001, 25, 88, 8, 88, 1), + /* ICSS_G0 */ + PSIL_ETHERNET(0xc100, 26, 96, 1), + PSIL_ETHERNET(0xc101, 27, 97, 1), + PSIL_ETHERNET(0xc102, 28, 98, 1), + PSIL_ETHERNET(0xc103, 29, 99, 1), + PSIL_ETHERNET(0xc104, 30, 100, 1), + PSIL_ETHERNET(0xc105, 31, 101, 1), + PSIL_ETHERNET(0xc106, 32, 102, 1), + PSIL_ETHERNET(0xc107, 33, 103, 1), + /* ICSS_G1 */ + PSIL_ETHERNET(0xc200, 34, 104, 1), + PSIL_ETHERNET(0xc201, 35, 105, 1), + PSIL_ETHERNET(0xc202, 36, 106, 1), + PSIL_ETHERNET(0xc203, 37, 107, 1), + PSIL_ETHERNET(0xc204, 38, 108, 1), + PSIL_ETHERNET(0xc205, 39, 109, 1), + PSIL_ETHERNET(0xc206, 40, 110, 1), + PSIL_ETHERNET(0xc207, 41, 111, 1), + /* CPSW2 */ + PSIL_ETHERNET(0xc500, 16, 16, 8), + PSIL_ETHERNET(0xc501, 17, 24, 8), + PSIL_ETHERNET(0xc502, 18, 32, 8), + PSIL_ETHERNET(0xc503, 19, 40, 8), + PSIL_ETHERNET(0xc504, 20, 48, 8), + PSIL_ETHERNET(0xc505, 21, 56, 8), + PSIL_ETHERNET(0xc506, 22, 64, 8), + PSIL_ETHERNET(0xc507, 23, 72, 8), +}; + +struct psil_ep_map am64_ep_map = { + .name = "am64", + .src = am64_src_ep_map, + .src_count = ARRAY_SIZE(am64_src_ep_map), + .dst = am64_dst_ep_map, + .dst_count = ARRAY_SIZE(am64_dst_ep_map), +}; diff --git a/drivers/dma/ti/k3-psil-priv.h b/drivers/dma/ti/k3-psil-priv.h index b4b0fb359eff..b74e192e3c2d 100644 --- a/drivers/dma/ti/k3-psil-priv.h +++ b/drivers/dma/ti/k3-psil-priv.h @@ -40,5 +40,6 @@ struct psil_endpoint_config *psil_get_ep_config(u32 thread_id); extern struct psil_ep_map am654_ep_map; extern struct psil_ep_map j721e_ep_map; extern struct psil_ep_map j7200_ep_map; +extern struct psil_ep_map am64_ep_map; #endif /* K3_PSIL_PRIV_H_ */ diff --git a/drivers/dma/ti/k3-psil.c b/drivers/dma/ti/k3-psil.c index 837853aab95a..13ce7367d870 100644 --- a/drivers/dma/ti/k3-psil.c +++ b/drivers/dma/ti/k3-psil.c @@ -20,6 +20,7 @@ static const struct soc_device_attribute k3_soc_devices[] = { { .family = "AM65X", .data = &am654_ep_map }, { .family = "J721E", .data = &j721e_ep_map }, { .family = "J7200", .data = &j7200_ep_map }, + { .family = "AM64X", .data = &am64_ep_map }, { /* sentinel */ } }; -- cgit v1.2.3-59-g8ed1b From fc373e47d72605cc3f5012ddda49d2dca430d51f Mon Sep 17 00:00:00 2001 From: Peter Ujfalusi Date: Tue, 8 Dec 2020 11:04:35 +0200 Subject: dmaengine: ti: Add support for k3 event routers In k3 architecture a DMA channel (in TR momde) can be triggered by global events, origination from different modules. The events for triggers can be sent from any module which is connected to PSI-L fabric, but the event number to be sent is DMA channel specific, it is only known after the channel itself is requested. The router operation needs to be split up: - route_allocate: configure the dma_spec for the DMA and store the configuration which is needed for the router's input - set_event: callback used by the DMA driver to set the event number for the channel and enable the routing Signed-off-by: Peter Ujfalusi Link: https://lore.kernel.org/r/20201208090440.31792-16-peter.ujfalusi@ti.com Signed-off-by: Vinod Koul --- include/linux/dma/k3-event-router.h | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 include/linux/dma/k3-event-router.h diff --git a/include/linux/dma/k3-event-router.h b/include/linux/dma/k3-event-router.h new file mode 100644 index 000000000000..e3f88b2f87be --- /dev/null +++ b/include/linux/dma/k3-event-router.h @@ -0,0 +1,16 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2020 Texas Instruments Incorporated - https://www.ti.com + */ + +#ifndef K3_EVENT_ROUTER_ +#define K3_EVENT_ROUTER_ + +#include + +struct k3_event_route_data { + void *priv; + int (*set_event)(void *priv, u32 event); +}; + +#endif /* K3_EVENT_ROUTER_ */ -- cgit v1.2.3-59-g8ed1b From d782298c6f6b854452965b56d91616dfb60490c5 Mon Sep 17 00:00:00 2001 From: Grygorii Strashko Date: Tue, 8 Dec 2020 11:04:36 +0200 Subject: soc: ti: k3-ringacc: add AM64 DMA rings support. The DMAs in AM64 have built in rings compared to AM654/J721e/J7200 where a separate and generic ringacc is used. The ring SW interface is similar to ringacc with some major architectural differences, like They are part of the DMA (BCDMA or PKTDMA). They are dual mode rings are modeled as pair of Rings objects which has common configuration and memory buffer, but separate real-time control register sets for each direction mem2dev (forward) and dev2mem (reverse). The ringacc driver must be initialized for DMA rings use with k3_ringacc_dmarings_init() as it is not an independent device as ringacc is. AM64 rings must be requested only using k3_ringacc_request_rings_pair(), and forward ring must always be initialized/configured. After this any other Ringacc APIs can be used without any callers changes. Signed-off-by: Grygorii Strashko Signed-off-by: Peter Ujfalusi Link: https://lore.kernel.org/r/20201208090440.31792-17-peter.ujfalusi@ti.com Signed-off-by: Vinod Koul --- drivers/soc/ti/k3-ringacc.c | 325 +++++++++++++++++++++++++++++++++++++- include/linux/soc/ti/k3-ringacc.h | 17 ++ 2 files changed, 335 insertions(+), 7 deletions(-) diff --git a/drivers/soc/ti/k3-ringacc.c b/drivers/soc/ti/k3-ringacc.c index 119164abcb41..c88c305ba367 100644 --- a/drivers/soc/ti/k3-ringacc.c +++ b/drivers/soc/ti/k3-ringacc.c @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include @@ -21,6 +22,7 @@ static LIST_HEAD(k3_ringacc_list); static DEFINE_MUTEX(k3_ringacc_list_lock); #define K3_RINGACC_CFG_RING_SIZE_ELCNT_MASK GENMASK(19, 0) +#define K3_DMARING_CFG_RING_SIZE_ELCNT_MASK GENMASK(15, 0) /** * struct k3_ring_rt_regs - The RA realtime Control/Status Registers region @@ -43,7 +45,13 @@ struct k3_ring_rt_regs { u32 hwindx; }; -#define K3_RINGACC_RT_REGS_STEP 0x1000 +#define K3_RINGACC_RT_REGS_STEP 0x1000 +#define K3_DMARING_RT_REGS_STEP 0x2000 +#define K3_DMARING_RT_REGS_REVERSE_OFS 0x1000 +#define K3_RINGACC_RT_OCC_MASK GENMASK(20, 0) +#define K3_DMARING_RT_OCC_TDOWN_COMPLETE BIT(31) +#define K3_DMARING_RT_DB_ENTRY_MASK GENMASK(7, 0) +#define K3_DMARING_RT_DB_TDOWN_ACK BIT(31) /** * struct k3_ring_fifo_regs - The Ring Accelerator Queues Registers region @@ -122,6 +130,7 @@ struct k3_ring_state { u32 occ; u32 windex; u32 rindex; + u32 tdown_complete:1; }; /** @@ -143,6 +152,7 @@ struct k3_ring_state { * @use_count: Use count for shared rings * @proxy_id: RA Ring Proxy Id (only if @K3_RINGACC_RING_USE_PROXY) * @dma_dev: device to be used for DMA API (allocation, mapping) + * @asel: Address Space Select value for physical addresses */ struct k3_ring { struct k3_ring_rt_regs __iomem *rt; @@ -157,12 +167,15 @@ struct k3_ring { u32 flags; #define K3_RING_FLAG_BUSY BIT(1) #define K3_RING_FLAG_SHARED BIT(2) +#define K3_RING_FLAG_REVERSE BIT(3) struct k3_ring_state state; u32 ring_id; struct k3_ringacc *parent; u32 use_count; int proxy_id; struct device *dma_dev; + u32 asel; +#define K3_ADDRESS_ASEL_SHIFT 48 }; struct k3_ringacc_ops { @@ -188,6 +201,7 @@ struct k3_ringacc_ops { * @tisci_ring_ops: ti-sci rings ops * @tisci_dev_id: ti-sci device id * @ops: SoC specific ringacc operation + * @dma_rings: indicate DMA ring (dual ring within BCDMA/PKTDMA) */ struct k3_ringacc { struct device *dev; @@ -210,6 +224,7 @@ struct k3_ringacc { u32 tisci_dev_id; const struct k3_ringacc_ops *ops; + bool dma_rings; }; /** @@ -221,6 +236,21 @@ struct k3_ringacc_soc_data { unsigned dma_ring_reset_quirk:1; }; +static int k3_ringacc_ring_read_occ(struct k3_ring *ring) +{ + return readl(&ring->rt->occ) & K3_RINGACC_RT_OCC_MASK; +} + +static void k3_ringacc_ring_update_occ(struct k3_ring *ring) +{ + u32 val; + + val = readl(&ring->rt->occ); + + ring->state.occ = val & K3_RINGACC_RT_OCC_MASK; + ring->state.tdown_complete = !!(val & K3_DMARING_RT_OCC_TDOWN_COMPLETE); +} + static long k3_ringacc_ring_get_fifo_pos(struct k3_ring *ring) { return K3_RINGACC_FIFO_WINDOW_SIZE_BYTES - @@ -234,12 +264,24 @@ static void *k3_ringacc_get_elm_addr(struct k3_ring *ring, u32 idx) static int k3_ringacc_ring_push_mem(struct k3_ring *ring, void *elem); static int k3_ringacc_ring_pop_mem(struct k3_ring *ring, void *elem); +static int k3_dmaring_fwd_pop(struct k3_ring *ring, void *elem); +static int k3_dmaring_reverse_pop(struct k3_ring *ring, void *elem); static struct k3_ring_ops k3_ring_mode_ring_ops = { .push_tail = k3_ringacc_ring_push_mem, .pop_head = k3_ringacc_ring_pop_mem, }; +static struct k3_ring_ops k3_dmaring_fwd_ops = { + .push_tail = k3_ringacc_ring_push_mem, + .pop_head = k3_dmaring_fwd_pop, +}; + +static struct k3_ring_ops k3_dmaring_reverse_ops = { + /* Reverse side of the DMA ring can only be popped by SW */ + .pop_head = k3_dmaring_reverse_pop, +}; + static int k3_ringacc_ring_push_io(struct k3_ring *ring, void *elem); static int k3_ringacc_ring_pop_io(struct k3_ring *ring, void *elem); static int k3_ringacc_ring_push_head_io(struct k3_ring *ring, void *elem); @@ -342,6 +384,40 @@ error: } EXPORT_SYMBOL_GPL(k3_ringacc_request_ring); +static int k3_dmaring_request_dual_ring(struct k3_ringacc *ringacc, int fwd_id, + struct k3_ring **fwd_ring, + struct k3_ring **compl_ring) +{ + int ret = 0; + + /* + * DMA rings must be requested by ID, completion ring is the reverse + * side of the forward ring + */ + if (fwd_id < 0) + return -EINVAL; + + mutex_lock(&ringacc->req_lock); + + if (test_bit(fwd_id, ringacc->rings_inuse)) { + ret = -EBUSY; + goto error; + } + + *fwd_ring = &ringacc->rings[fwd_id]; + *compl_ring = &ringacc->rings[fwd_id + ringacc->num_rings]; + set_bit(fwd_id, ringacc->rings_inuse); + ringacc->rings[fwd_id].use_count++; + dev_dbg(ringacc->dev, "Giving ring#%d\n", fwd_id); + + mutex_unlock(&ringacc->req_lock); + return 0; + +error: + mutex_unlock(&ringacc->req_lock); + return ret; +} + int k3_ringacc_request_rings_pair(struct k3_ringacc *ringacc, int fwd_id, int compl_id, struct k3_ring **fwd_ring, @@ -352,6 +428,10 @@ int k3_ringacc_request_rings_pair(struct k3_ringacc *ringacc, if (!fwd_ring || !compl_ring) return -EINVAL; + if (ringacc->dma_rings) + return k3_dmaring_request_dual_ring(ringacc, fwd_id, + fwd_ring, compl_ring); + *fwd_ring = k3_ringacc_request_ring(ringacc, fwd_id, 0); if (!(*fwd_ring)) return -ENODEV; @@ -421,7 +501,7 @@ void k3_ringacc_ring_reset_dma(struct k3_ring *ring, u32 occ) goto reset; if (!occ) - occ = readl(&ring->rt->occ); + occ = k3_ringacc_ring_read_occ(ring); if (occ) { u32 db_ring_cnt, db_ring_cnt_cur; @@ -496,6 +576,13 @@ int k3_ringacc_ring_free(struct k3_ring *ring) ringacc = ring->parent; + /* + * DMA rings: rings shared memory and configuration, only forward ring + * is configured and reverse ring considered as slave. + */ + if (ringacc->dma_rings && (ring->flags & K3_RING_FLAG_REVERSE)) + return 0; + dev_dbg(ring->parent->dev, "flags: 0x%08x\n", ring->flags); if (!test_bit(ring->ring_id, ringacc->rings_inuse)) @@ -517,6 +604,8 @@ int k3_ringacc_ring_free(struct k3_ring *ring) ring->flags = 0; ring->ops = NULL; ring->dma_dev = NULL; + ring->asel = 0; + if (ring->proxy_id != K3_RINGACC_PROXY_NOT_USED) { clear_bit(ring->proxy_id, ringacc->proxy_inuse); ring->proxy = NULL; @@ -581,6 +670,7 @@ static int k3_ringacc_ring_cfg_sci(struct k3_ring *ring) ring_cfg.count = ring->size; ring_cfg.mode = ring->mode; ring_cfg.size = ring->elm_size; + ring_cfg.asel = ring->asel; ret = ringacc->tisci_ring_ops->set_cfg(ringacc->tisci, &ring_cfg); if (ret) @@ -590,6 +680,90 @@ static int k3_ringacc_ring_cfg_sci(struct k3_ring *ring) return ret; } +static int k3_dmaring_cfg(struct k3_ring *ring, struct k3_ring_cfg *cfg) +{ + struct k3_ringacc *ringacc; + struct k3_ring *reverse_ring; + int ret = 0; + + if (cfg->elm_size != K3_RINGACC_RING_ELSIZE_8 || + cfg->mode != K3_RINGACC_RING_MODE_RING || + cfg->size & ~K3_DMARING_CFG_RING_SIZE_ELCNT_MASK) + return -EINVAL; + + ringacc = ring->parent; + + /* + * DMA rings: rings shared memory and configuration, only forward ring + * is configured and reverse ring considered as slave. + */ + if (ringacc->dma_rings && (ring->flags & K3_RING_FLAG_REVERSE)) + return 0; + + if (!test_bit(ring->ring_id, ringacc->rings_inuse)) + return -EINVAL; + + ring->size = cfg->size; + ring->elm_size = cfg->elm_size; + ring->mode = cfg->mode; + ring->asel = cfg->asel; + ring->dma_dev = cfg->dma_dev; + if (!ring->dma_dev) { + dev_warn(ringacc->dev, "dma_dev is not provided for ring%d\n", + ring->ring_id); + ring->dma_dev = ringacc->dev; + } + + memset(&ring->state, 0, sizeof(ring->state)); + + ring->ops = &k3_dmaring_fwd_ops; + + ring->ring_mem_virt = dma_alloc_coherent(ring->dma_dev, + ring->size * (4 << ring->elm_size), + &ring->ring_mem_dma, GFP_KERNEL); + if (!ring->ring_mem_virt) { + dev_err(ringacc->dev, "Failed to alloc ring mem\n"); + ret = -ENOMEM; + goto err_free_ops; + } + + ret = k3_ringacc_ring_cfg_sci(ring); + if (ret) + goto err_free_mem; + + ring->flags |= K3_RING_FLAG_BUSY; + + k3_ringacc_ring_dump(ring); + + /* DMA rings: configure reverse ring */ + reverse_ring = &ringacc->rings[ring->ring_id + ringacc->num_rings]; + reverse_ring->size = cfg->size; + reverse_ring->elm_size = cfg->elm_size; + reverse_ring->mode = cfg->mode; + reverse_ring->asel = cfg->asel; + memset(&reverse_ring->state, 0, sizeof(reverse_ring->state)); + reverse_ring->ops = &k3_dmaring_reverse_ops; + + reverse_ring->ring_mem_virt = ring->ring_mem_virt; + reverse_ring->ring_mem_dma = ring->ring_mem_dma; + reverse_ring->flags |= K3_RING_FLAG_BUSY; + k3_ringacc_ring_dump(reverse_ring); + + return 0; + +err_free_mem: + dma_free_coherent(ring->dma_dev, + ring->size * (4 << ring->elm_size), + ring->ring_mem_virt, + ring->ring_mem_dma); +err_free_ops: + ring->ops = NULL; + ring->proxy = NULL; + ring->dma_dev = NULL; + ring->asel = 0; + return ret; +} + int k3_ringacc_ring_cfg(struct k3_ring *ring, struct k3_ring_cfg *cfg) { struct k3_ringacc *ringacc; @@ -597,8 +771,12 @@ int k3_ringacc_ring_cfg(struct k3_ring *ring, struct k3_ring_cfg *cfg) if (!ring || !cfg) return -EINVAL; + ringacc = ring->parent; + if (ringacc->dma_rings) + return k3_dmaring_cfg(ring, cfg); + if (cfg->elm_size > K3_RINGACC_RING_ELSIZE_256 || cfg->mode >= K3_RINGACC_RING_MODE_INVALID || cfg->size & ~K3_RINGACC_CFG_RING_SIZE_ELCNT_MASK || @@ -705,7 +883,7 @@ u32 k3_ringacc_ring_get_free(struct k3_ring *ring) return -EINVAL; if (!ring->state.free) - ring->state.free = ring->size - readl(&ring->rt->occ); + ring->state.free = ring->size - k3_ringacc_ring_read_occ(ring); return ring->state.free; } @@ -716,7 +894,7 @@ u32 k3_ringacc_ring_get_occ(struct k3_ring *ring) if (!ring || !(ring->flags & K3_RING_FLAG_BUSY)) return -EINVAL; - return readl(&ring->rt->occ); + return k3_ringacc_ring_read_occ(ring); } EXPORT_SYMBOL_GPL(k3_ringacc_ring_get_occ); @@ -892,6 +1070,72 @@ static int k3_ringacc_ring_pop_tail_io(struct k3_ring *ring, void *elem) K3_RINGACC_ACCESS_MODE_POP_HEAD); } +/* + * The element is 48 bits of address + ASEL bits in the ring. + * ASEL is used by the DMAs and should be removed for the kernel as it is not + * part of the physical memory address. + */ +static void k3_dmaring_remove_asel_from_elem(u64 *elem) +{ + *elem &= GENMASK_ULL(K3_ADDRESS_ASEL_SHIFT - 1, 0); +} + +static int k3_dmaring_fwd_pop(struct k3_ring *ring, void *elem) +{ + void *elem_ptr; + u32 elem_idx; + + /* + * DMA rings: forward ring is always tied DMA channel and HW does not + * maintain any state data required for POP operation and its unknown + * how much elements were consumed by HW. So, to actually + * do POP, the read pointer has to be recalculated every time. + */ + ring->state.occ = k3_ringacc_ring_read_occ(ring); + if (ring->state.windex >= ring->state.occ) + elem_idx = ring->state.windex - ring->state.occ; + else + elem_idx = ring->size - (ring->state.occ - ring->state.windex); + + elem_ptr = k3_ringacc_get_elm_addr(ring, elem_idx); + memcpy(elem, elem_ptr, (4 << ring->elm_size)); + k3_dmaring_remove_asel_from_elem(elem); + + ring->state.occ--; + writel(-1, &ring->rt->db); + + dev_dbg(ring->parent->dev, "%s: occ%d Windex%d Rindex%d pos_ptr%px\n", + __func__, ring->state.occ, ring->state.windex, elem_idx, + elem_ptr); + return 0; +} + +static int k3_dmaring_reverse_pop(struct k3_ring *ring, void *elem) +{ + void *elem_ptr; + + elem_ptr = k3_ringacc_get_elm_addr(ring, ring->state.rindex); + + if (ring->state.occ) { + memcpy(elem, elem_ptr, (4 << ring->elm_size)); + k3_dmaring_remove_asel_from_elem(elem); + + ring->state.rindex = (ring->state.rindex + 1) % ring->size; + ring->state.occ--; + writel(-1 & K3_DMARING_RT_DB_ENTRY_MASK, &ring->rt->db); + } else if (ring->state.tdown_complete) { + dma_addr_t *value = elem; + + *value = CPPI5_TDCM_MARKER; + writel(K3_DMARING_RT_DB_TDOWN_ACK, &ring->rt->db); + ring->state.tdown_complete = false; + } + + dev_dbg(ring->parent->dev, "%s: occ%d index%d pos_ptr%px\n", + __func__, ring->state.occ, ring->state.rindex, elem_ptr); + return 0; +} + static int k3_ringacc_ring_push_mem(struct k3_ring *ring, void *elem) { void *elem_ptr; @@ -899,6 +1143,11 @@ static int k3_ringacc_ring_push_mem(struct k3_ring *ring, void *elem) elem_ptr = k3_ringacc_get_elm_addr(ring, ring->state.windex); memcpy(elem_ptr, elem, (4 << ring->elm_size)); + if (ring->parent->dma_rings) { + u64 *addr = elem_ptr; + + *addr |= ((u64)ring->asel << K3_ADDRESS_ASEL_SHIFT); + } ring->state.windex = (ring->state.windex + 1) % ring->size; ring->state.free--; @@ -975,12 +1224,12 @@ int k3_ringacc_ring_pop(struct k3_ring *ring, void *elem) return -EINVAL; if (!ring->state.occ) - ring->state.occ = k3_ringacc_ring_get_occ(ring); + k3_ringacc_ring_update_occ(ring); dev_dbg(ring->parent->dev, "ring_pop: occ%d index%d\n", ring->state.occ, ring->state.rindex); - if (!ring->state.occ) + if (!ring->state.occ && !ring->state.tdown_complete) return -ENODATA; if (ring->ops && ring->ops->pop_head) @@ -998,7 +1247,7 @@ int k3_ringacc_ring_pop_tail(struct k3_ring *ring, void *elem) return -EINVAL; if (!ring->state.occ) - ring->state.occ = k3_ringacc_ring_get_occ(ring); + k3_ringacc_ring_update_occ(ring); dev_dbg(ring->parent->dev, "ring_pop_tail: occ%d index%d\n", ring->state.occ, ring->state.rindex); @@ -1203,6 +1452,68 @@ static const struct of_device_id k3_ringacc_of_match[] = { {}, }; +struct k3_ringacc *k3_ringacc_dmarings_init(struct platform_device *pdev, + struct k3_ringacc_init_data *data) +{ + struct device *dev = &pdev->dev; + struct k3_ringacc *ringacc; + void __iomem *base_rt; + struct resource *res; + int i; + + ringacc = devm_kzalloc(dev, sizeof(*ringacc), GFP_KERNEL); + if (!ringacc) + return ERR_PTR(-ENOMEM); + + ringacc->dev = dev; + ringacc->dma_rings = true; + ringacc->num_rings = data->num_rings; + ringacc->tisci = data->tisci; + ringacc->tisci_dev_id = data->tisci_dev_id; + + mutex_init(&ringacc->req_lock); + + res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "ringrt"); + base_rt = devm_ioremap_resource(dev, res); + if (IS_ERR(base_rt)) + return base_rt; + + ringacc->rings = devm_kzalloc(dev, + sizeof(*ringacc->rings) * + ringacc->num_rings * 2, + GFP_KERNEL); + ringacc->rings_inuse = devm_kcalloc(dev, + BITS_TO_LONGS(ringacc->num_rings), + sizeof(unsigned long), GFP_KERNEL); + + if (!ringacc->rings || !ringacc->rings_inuse) + return ERR_PTR(-ENOMEM); + + for (i = 0; i < ringacc->num_rings; i++) { + struct k3_ring *ring = &ringacc->rings[i]; + + ring->rt = base_rt + K3_DMARING_RT_REGS_STEP * i; + ring->parent = ringacc; + ring->ring_id = i; + ring->proxy_id = K3_RINGACC_PROXY_NOT_USED; + + ring = &ringacc->rings[ringacc->num_rings + i]; + ring->rt = base_rt + K3_DMARING_RT_REGS_STEP * i + + K3_DMARING_RT_REGS_REVERSE_OFS; + ring->parent = ringacc; + ring->ring_id = i; + ring->proxy_id = K3_RINGACC_PROXY_NOT_USED; + ring->flags = K3_RING_FLAG_REVERSE; + } + + ringacc->tisci_ring_ops = &ringacc->tisci->ops.rm_ring_ops; + + dev_info(dev, "Number of rings: %u\n", ringacc->num_rings); + + return ringacc; +} +EXPORT_SYMBOL_GPL(k3_ringacc_dmarings_init); + static int k3_ringacc_probe(struct platform_device *pdev) { const struct ringacc_match_data *match_data; diff --git a/include/linux/soc/ti/k3-ringacc.h b/include/linux/soc/ti/k3-ringacc.h index 658dc71d2901..39b022b92598 100644 --- a/include/linux/soc/ti/k3-ringacc.h +++ b/include/linux/soc/ti/k3-ringacc.h @@ -70,6 +70,7 @@ struct k3_ring; * @dma_dev: Master device which is using and accessing to the ring * memory when the mode is K3_RINGACC_RING_MODE_RING. Memory allocations * should be done using this device. + * @asel: Address Space Select value for physical addresses */ struct k3_ring_cfg { u32 size; @@ -79,6 +80,7 @@ struct k3_ring_cfg { u32 flags; struct device *dma_dev; + u32 asel; }; #define K3_RINGACC_RING_ID_ANY (-1) @@ -250,4 +252,19 @@ int k3_ringacc_ring_pop_tail(struct k3_ring *ring, void *elem); u32 k3_ringacc_get_tisci_dev_id(struct k3_ring *ring); +/* DMA ring support */ +struct ti_sci_handle; + +/** + * struct struct k3_ringacc_init_data - Initialization data for DMA rings + */ +struct k3_ringacc_init_data { + const struct ti_sci_handle *tisci; + u32 tisci_dev_id; + u32 num_rings; +}; + +struct k3_ringacc *k3_ringacc_dmarings_init(struct platform_device *pdev, + struct k3_ringacc_init_data *data); + #endif /* __SOC_TI_K3_RINGACC_API_H_ */ -- cgit v1.2.3-59-g8ed1b From 017794739702d444ca48115ff0fcdce19edb5559 Mon Sep 17 00:00:00 2001 From: Peter Ujfalusi Date: Tue, 8 Dec 2020 11:04:37 +0200 Subject: dmaengine: ti: k3-udma: Initial support for K3 BCDMA One of the DMAs introduced with AM64 is the Block Copy DMA (BCDMA). It serves similar purpose as K3 UDMAP channels in TR mode. The rings for the BCDMA is integrated within the DMA itself instead of using rings from the general purpose ringacc. A BCDMA have two different type of channels: - Block Copy Channels (bchan) - Split Channels (tchan and rchan) tchan and rchan can be used to service PSI-L peripherals, similarly to K3 UDMA channels. bchan can be only used for block copy operation (TR type15) like the paired K3 UDMA tchan/rchan configured in block copy mode. bchans can be also used to service peripherals directly if an external trigger is selected for the channel. Most of the driver code can be reused for BCDMA bchan/tchan/rchan support but new setup and allocation functions are needed to handle the differences between the DMAs. Signed-off-by: Peter Ujfalusi Link: https://lore.kernel.org/r/20201208090440.31792-18-peter.ujfalusi@ti.com Signed-off-by: Vinod Koul --- drivers/dma/ti/k3-udma.c | 1384 +++++++++++++++++++++++++++++++++++++++++----- drivers/dma/ti/k3-udma.h | 12 +- 2 files changed, 1245 insertions(+), 151 deletions(-) diff --git a/drivers/dma/ti/k3-udma.c b/drivers/dma/ti/k3-udma.c index b89afa602532..f327d436f8ad 100644 --- a/drivers/dma/ti/k3-udma.c +++ b/drivers/dma/ti/k3-udma.c @@ -26,6 +26,7 @@ #include #include #include +#include #include #include "../virt-dma.h" @@ -55,14 +56,25 @@ struct udma_static_tr { struct udma_chan; +enum k3_dma_type { + DMA_TYPE_UDMA = 0, + DMA_TYPE_BCDMA, +}; + enum udma_mmr { MMR_GCFG = 0, + MMR_BCHANRT, MMR_RCHANRT, MMR_TCHANRT, MMR_LAST, }; -static const char * const mmr_names[] = { "gcfg", "rchanrt", "tchanrt" }; +static const char * const mmr_names[] = { + [MMR_GCFG] = "gcfg", + [MMR_BCHANRT] = "bchanrt", + [MMR_RCHANRT] = "rchanrt", + [MMR_TCHANRT] = "tchanrt", +}; struct udma_tchan { void __iomem *reg_rt; @@ -72,6 +84,8 @@ struct udma_tchan { struct k3_ring *tc_ring; /* Transmit Completion ring */ }; +#define udma_bchan udma_tchan + struct udma_rflow { int id; struct k3_ring *fd_ring; /* Free Descriptor ring */ @@ -84,11 +98,25 @@ struct udma_rchan { int id; }; +struct udma_oes_offsets { + /* K3 UDMA Output Event Offset */ + u32 udma_rchan; + + /* BCDMA Output Event Offsets */ + u32 bcdma_bchan_data; + u32 bcdma_bchan_ring; + u32 bcdma_tchan_data; + u32 bcdma_tchan_ring; + u32 bcdma_rchan_data; + u32 bcdma_rchan_ring; +}; + #define UDMA_FLAG_PDMA_ACC32 BIT(0) #define UDMA_FLAG_PDMA_BURST BIT(1) #define UDMA_FLAG_TDTYPE BIT(2) struct udma_match_data { + enum k3_dma_type type; u32 psil_base; bool enable_memcpy_support; u32 flags; @@ -96,7 +124,8 @@ struct udma_match_data { }; struct udma_soc_data { - u32 rchan_oes_offset; + struct udma_oes_offsets oes; + u32 bcdma_trigger_event_offset; }; struct udma_hwdesc { @@ -139,16 +168,19 @@ struct udma_dev { struct udma_rx_flush rx_flush; + int bchan_cnt; int tchan_cnt; int echan_cnt; int rchan_cnt; int rflow_cnt; + unsigned long *bchan_map; unsigned long *tchan_map; unsigned long *rchan_map; unsigned long *rflow_gp_map; unsigned long *rflow_gp_map_allocated; unsigned long *rflow_in_use; + struct udma_bchan *bchans; struct udma_tchan *tchans; struct udma_rchan *rchans; struct udma_rflow *rflows; @@ -156,6 +188,7 @@ struct udma_dev { struct udma_chan *channels; u32 psil_base; u32 atype; + u32 asel; }; struct udma_desc { @@ -200,6 +233,7 @@ struct udma_chan_config { bool notdpkt; /* Suppress sending TDC packet */ int remote_thread_id; u32 atype; + u32 asel; u32 src_thread; u32 dst_thread; enum psil_endpoint_type ep_type; @@ -207,6 +241,8 @@ struct udma_chan_config { bool enable_burst; enum udma_tp_level channel_tpl; /* Channel Throughput Level */ + u32 tr_trigger_type; + enum dma_transfer_direction dir; }; @@ -214,11 +250,13 @@ struct udma_chan { struct virt_dma_chan vc; struct dma_slave_config cfg; struct udma_dev *ud; + struct device *dma_dev; struct udma_desc *desc; struct udma_desc *terminated_desc; struct udma_static_tr static_tr; char *name; + struct udma_bchan *bchan; struct udma_tchan *tchan; struct udma_rchan *rchan; struct udma_rflow *rflow; @@ -354,6 +392,30 @@ static int navss_psil_unpair(struct udma_dev *ud, u32 src_thread, src_thread, dst_thread); } +static void k3_configure_chan_coherency(struct dma_chan *chan, u32 asel) +{ + struct device *chan_dev = &chan->dev->device; + + if (asel == 0) { + /* No special handling for the channel */ + chan->dev->chan_dma_dev = false; + + chan_dev->dma_coherent = false; + chan_dev->dma_parms = NULL; + } else if (asel == 14 || asel == 15) { + chan->dev->chan_dma_dev = true; + + chan_dev->dma_coherent = true; + dma_coerce_mask_and_coherent(chan_dev, DMA_BIT_MASK(48)); + chan_dev->dma_parms = chan_dev->parent->dma_parms; + } else { + dev_warn(chan->device->dev, "Invalid ASEL value: %u\n", asel); + + chan_dev->dma_coherent = false; + chan_dev->dma_parms = NULL; + } +} + static void udma_reset_uchan(struct udma_chan *uc) { memset(&uc->config, 0, sizeof(uc->config)); @@ -440,9 +502,7 @@ static void udma_free_hwdesc(struct udma_chan *uc, struct udma_desc *d) d->hwdesc[i].cppi5_desc_vaddr = NULL; } } else if (d->hwdesc[0].cppi5_desc_vaddr) { - struct udma_dev *ud = uc->ud; - - dma_free_coherent(ud->dev, d->hwdesc[0].cppi5_desc_size, + dma_free_coherent(uc->dma_dev, d->hwdesc[0].cppi5_desc_size, d->hwdesc[0].cppi5_desc_vaddr, d->hwdesc[0].cppi5_desc_paddr); @@ -671,8 +731,10 @@ static void udma_reset_counters(struct udma_chan *uc) val = udma_tchanrt_read(uc, UDMA_CHAN_RT_PCNT_REG); udma_tchanrt_write(uc, UDMA_CHAN_RT_PCNT_REG, val); - val = udma_tchanrt_read(uc, UDMA_CHAN_RT_PEER_BCNT_REG); - udma_tchanrt_write(uc, UDMA_CHAN_RT_PEER_BCNT_REG, val); + if (!uc->bchan) { + val = udma_tchanrt_read(uc, UDMA_CHAN_RT_PEER_BCNT_REG); + udma_tchanrt_write(uc, UDMA_CHAN_RT_PEER_BCNT_REG, val); + } } if (uc->rchan) { @@ -1233,6 +1295,42 @@ static struct udma_##res *__udma_reserve_##res(struct udma_dev *ud, \ UDMA_RESERVE_RESOURCE(tchan); UDMA_RESERVE_RESOURCE(rchan); +static struct udma_bchan *__bcdma_reserve_bchan(struct udma_dev *ud, int id) +{ + if (id >= 0) { + if (test_bit(id, ud->bchan_map)) { + dev_err(ud->dev, "bchan%d is in use\n", id); + return ERR_PTR(-ENOENT); + } + } else { + id = find_next_zero_bit(ud->bchan_map, ud->bchan_cnt, 0); + if (id == ud->bchan_cnt) + return ERR_PTR(-ENOENT); + } + + set_bit(id, ud->bchan_map); + return &ud->bchans[id]; +} + +static int bcdma_get_bchan(struct udma_chan *uc) +{ + struct udma_dev *ud = uc->ud; + + if (uc->bchan) { + dev_dbg(ud->dev, "chan%d: already have bchan%d allocated\n", + uc->id, uc->bchan->id); + return 0; + } + + uc->bchan = __bcdma_reserve_bchan(ud, -1); + if (IS_ERR(uc->bchan)) + return PTR_ERR(uc->bchan); + + uc->tchan = uc->bchan; + + return 0; +} + static int udma_get_tchan(struct udma_chan *uc) { struct udma_dev *ud = uc->ud; @@ -1325,6 +1423,19 @@ static int udma_get_rflow(struct udma_chan *uc, int flow_id) return PTR_ERR_OR_ZERO(uc->rflow); } +static void bcdma_put_bchan(struct udma_chan *uc) +{ + struct udma_dev *ud = uc->ud; + + if (uc->bchan) { + dev_dbg(ud->dev, "chan%d: put bchan%d\n", uc->id, + uc->bchan->id); + clear_bit(uc->bchan->id, ud->bchan_map); + uc->bchan = NULL; + uc->tchan = NULL; + } +} + static void udma_put_rchan(struct udma_chan *uc) { struct udma_dev *ud = uc->ud; @@ -1361,6 +1472,65 @@ static void udma_put_rflow(struct udma_chan *uc) } } +static void bcdma_free_bchan_resources(struct udma_chan *uc) +{ + if (!uc->bchan) + return; + + k3_ringacc_ring_free(uc->bchan->tc_ring); + k3_ringacc_ring_free(uc->bchan->t_ring); + uc->bchan->tc_ring = NULL; + uc->bchan->t_ring = NULL; + k3_configure_chan_coherency(&uc->vc.chan, 0); + + bcdma_put_bchan(uc); +} + +static int bcdma_alloc_bchan_resources(struct udma_chan *uc) +{ + struct k3_ring_cfg ring_cfg; + struct udma_dev *ud = uc->ud; + int ret; + + ret = bcdma_get_bchan(uc); + if (ret) + return ret; + + ret = k3_ringacc_request_rings_pair(ud->ringacc, uc->bchan->id, -1, + &uc->bchan->t_ring, + &uc->bchan->tc_ring); + if (ret) { + ret = -EBUSY; + goto err_ring; + } + + memset(&ring_cfg, 0, sizeof(ring_cfg)); + ring_cfg.size = K3_UDMA_DEFAULT_RING_SIZE; + ring_cfg.elm_size = K3_RINGACC_RING_ELSIZE_8; + ring_cfg.mode = K3_RINGACC_RING_MODE_RING; + + k3_configure_chan_coherency(&uc->vc.chan, ud->asel); + ring_cfg.asel = ud->asel; + ring_cfg.dma_dev = dmaengine_get_dma_device(&uc->vc.chan); + + ret = k3_ringacc_ring_cfg(uc->bchan->t_ring, &ring_cfg); + if (ret) + goto err_ringcfg; + + return 0; + +err_ringcfg: + k3_ringacc_ring_free(uc->bchan->tc_ring); + uc->bchan->tc_ring = NULL; + k3_ringacc_ring_free(uc->bchan->t_ring); + uc->bchan->t_ring = NULL; + k3_configure_chan_coherency(&uc->vc.chan, 0); +err_ring: + bcdma_put_bchan(uc); + + return ret; +} + static void udma_free_tx_resources(struct udma_chan *uc) { if (!uc->tchan) @@ -1378,15 +1548,19 @@ static int udma_alloc_tx_resources(struct udma_chan *uc) { struct k3_ring_cfg ring_cfg; struct udma_dev *ud = uc->ud; - int ret; + struct udma_tchan *tchan; + int ring_idx, ret; ret = udma_get_tchan(uc); if (ret) return ret; - ret = k3_ringacc_request_rings_pair(ud->ringacc, uc->tchan->id, -1, - &uc->tchan->t_ring, - &uc->tchan->tc_ring); + tchan = uc->tchan; + ring_idx = ud->bchan_cnt + tchan->id; + + ret = k3_ringacc_request_rings_pair(ud->ringacc, ring_idx, -1, + &tchan->t_ring, + &tchan->tc_ring); if (ret) { ret = -EBUSY; goto err_ring; @@ -1395,10 +1569,18 @@ static int udma_alloc_tx_resources(struct udma_chan *uc) memset(&ring_cfg, 0, sizeof(ring_cfg)); ring_cfg.size = K3_UDMA_DEFAULT_RING_SIZE; ring_cfg.elm_size = K3_RINGACC_RING_ELSIZE_8; - ring_cfg.mode = K3_RINGACC_RING_MODE_MESSAGE; + if (ud->match_data->type == DMA_TYPE_UDMA) { + ring_cfg.mode = K3_RINGACC_RING_MODE_MESSAGE; + } else { + ring_cfg.mode = K3_RINGACC_RING_MODE_RING; + + k3_configure_chan_coherency(&uc->vc.chan, uc->config.asel); + ring_cfg.asel = uc->config.asel; + ring_cfg.dma_dev = dmaengine_get_dma_device(&uc->vc.chan); + } - ret = k3_ringacc_ring_cfg(uc->tchan->t_ring, &ring_cfg); - ret |= k3_ringacc_ring_cfg(uc->tchan->tc_ring, &ring_cfg); + ret = k3_ringacc_ring_cfg(tchan->t_ring, &ring_cfg); + ret |= k3_ringacc_ring_cfg(tchan->tc_ring, &ring_cfg); if (ret) goto err_ringcfg; @@ -1458,7 +1640,8 @@ static int udma_alloc_rx_resources(struct udma_chan *uc) } rflow = uc->rflow; - fd_ring_id = ud->tchan_cnt + ud->echan_cnt + uc->rchan->id; + fd_ring_id = ud->bchan_cnt + ud->tchan_cnt + ud->echan_cnt + + uc->rchan->id; ret = k3_ringacc_request_rings_pair(ud->ringacc, fd_ring_id, -1, &rflow->fd_ring, &rflow->r_ring); if (ret) { @@ -1468,15 +1651,25 @@ static int udma_alloc_rx_resources(struct udma_chan *uc) memset(&ring_cfg, 0, sizeof(ring_cfg)); - if (uc->config.pkt_mode) - ring_cfg.size = SG_MAX_SEGMENTS; - else + ring_cfg.elm_size = K3_RINGACC_RING_ELSIZE_8; + if (ud->match_data->type == DMA_TYPE_UDMA) { + if (uc->config.pkt_mode) + ring_cfg.size = SG_MAX_SEGMENTS; + else + ring_cfg.size = K3_UDMA_DEFAULT_RING_SIZE; + + ring_cfg.mode = K3_RINGACC_RING_MODE_MESSAGE; + } else { ring_cfg.size = K3_UDMA_DEFAULT_RING_SIZE; + ring_cfg.mode = K3_RINGACC_RING_MODE_RING; - ring_cfg.elm_size = K3_RINGACC_RING_ELSIZE_8; - ring_cfg.mode = K3_RINGACC_RING_MODE_MESSAGE; + k3_configure_chan_coherency(&uc->vc.chan, uc->config.asel); + ring_cfg.asel = uc->config.asel; + ring_cfg.dma_dev = dmaengine_get_dma_device(&uc->vc.chan); + } ret = k3_ringacc_ring_cfg(rflow->fd_ring, &ring_cfg); + ring_cfg.size = K3_UDMA_DEFAULT_RING_SIZE; ret |= k3_ringacc_ring_cfg(rflow->r_ring, &ring_cfg); @@ -1498,7 +1691,18 @@ err_rflow: return ret; } -#define TISCI_TCHAN_VALID_PARAMS ( \ +#define TISCI_BCDMA_BCHAN_VALID_PARAMS ( \ + TI_SCI_MSG_VALUE_RM_UDMAP_CH_PAUSE_ON_ERR_VALID | \ + TI_SCI_MSG_VALUE_RM_UDMAP_CH_EXTENDED_CH_TYPE_VALID) + +#define TISCI_BCDMA_TCHAN_VALID_PARAMS ( \ + TI_SCI_MSG_VALUE_RM_UDMAP_CH_PAUSE_ON_ERR_VALID | \ + TI_SCI_MSG_VALUE_RM_UDMAP_CH_TX_SUPR_TDPKT_VALID) + +#define TISCI_BCDMA_RCHAN_VALID_PARAMS ( \ + TI_SCI_MSG_VALUE_RM_UDMAP_CH_PAUSE_ON_ERR_VALID) + +#define TISCI_UDMA_TCHAN_VALID_PARAMS ( \ TI_SCI_MSG_VALUE_RM_UDMAP_CH_PAUSE_ON_ERR_VALID | \ TI_SCI_MSG_VALUE_RM_UDMAP_CH_TX_FILT_EINFO_VALID | \ TI_SCI_MSG_VALUE_RM_UDMAP_CH_TX_FILT_PSWORDS_VALID | \ @@ -1508,7 +1712,7 @@ err_rflow: TI_SCI_MSG_VALUE_RM_UDMAP_CH_CQ_QNUM_VALID | \ TI_SCI_MSG_VALUE_RM_UDMAP_CH_ATYPE_VALID) -#define TISCI_RCHAN_VALID_PARAMS ( \ +#define TISCI_UDMA_RCHAN_VALID_PARAMS ( \ TI_SCI_MSG_VALUE_RM_UDMAP_CH_PAUSE_ON_ERR_VALID | \ TI_SCI_MSG_VALUE_RM_UDMAP_CH_FETCH_SIZE_VALID | \ TI_SCI_MSG_VALUE_RM_UDMAP_CH_CQ_QNUM_VALID | \ @@ -1533,7 +1737,7 @@ static int udma_tisci_m2m_channel_config(struct udma_chan *uc) struct ti_sci_msg_rm_udmap_tx_ch_cfg req_tx = { 0 }; struct ti_sci_msg_rm_udmap_rx_ch_cfg req_rx = { 0 }; - req_tx.valid_params = TISCI_TCHAN_VALID_PARAMS; + req_tx.valid_params = TISCI_UDMA_TCHAN_VALID_PARAMS; req_tx.nav_id = tisci_rm->tisci_dev_id; req_tx.index = tchan->id; req_tx.tx_chan_type = TI_SCI_RM_UDMAP_CHAN_TYPE_3RDP_BCOPY_PBRR; @@ -1547,7 +1751,7 @@ static int udma_tisci_m2m_channel_config(struct udma_chan *uc) return ret; } - req_rx.valid_params = TISCI_RCHAN_VALID_PARAMS; + req_rx.valid_params = TISCI_UDMA_RCHAN_VALID_PARAMS; req_rx.nav_id = tisci_rm->tisci_dev_id; req_rx.index = rchan->id; req_rx.rx_fetch_size = sizeof(struct cppi5_desc_hdr_t) >> 2; @@ -1562,6 +1766,27 @@ static int udma_tisci_m2m_channel_config(struct udma_chan *uc) return ret; } +static int bcdma_tisci_m2m_channel_config(struct udma_chan *uc) +{ + struct udma_dev *ud = uc->ud; + struct udma_tisci_rm *tisci_rm = &ud->tisci_rm; + const struct ti_sci_rm_udmap_ops *tisci_ops = tisci_rm->tisci_udmap_ops; + struct ti_sci_msg_rm_udmap_tx_ch_cfg req_tx = { 0 }; + struct udma_bchan *bchan = uc->bchan; + int ret = 0; + + req_tx.valid_params = TISCI_BCDMA_BCHAN_VALID_PARAMS; + req_tx.nav_id = tisci_rm->tisci_dev_id; + req_tx.extended_ch_type = TI_SCI_RM_BCDMA_EXTENDED_CH_TYPE_BCHAN; + req_tx.index = bchan->id; + + ret = tisci_ops->tx_ch_cfg(tisci_rm->tisci, &req_tx); + if (ret) + dev_err(ud->dev, "bchan%d cfg failed %d\n", bchan->id, ret); + + return ret; +} + static int udma_tisci_tx_channel_config(struct udma_chan *uc) { struct udma_dev *ud = uc->ud; @@ -1582,7 +1807,7 @@ static int udma_tisci_tx_channel_config(struct udma_chan *uc) fetch_size = sizeof(struct cppi5_desc_hdr_t); } - req_tx.valid_params = TISCI_TCHAN_VALID_PARAMS; + req_tx.valid_params = TISCI_UDMA_TCHAN_VALID_PARAMS; req_tx.nav_id = tisci_rm->tisci_dev_id; req_tx.index = tchan->id; req_tx.tx_chan_type = mode; @@ -1605,6 +1830,33 @@ static int udma_tisci_tx_channel_config(struct udma_chan *uc) return ret; } +static int bcdma_tisci_tx_channel_config(struct udma_chan *uc) +{ + struct udma_dev *ud = uc->ud; + struct udma_tisci_rm *tisci_rm = &ud->tisci_rm; + const struct ti_sci_rm_udmap_ops *tisci_ops = tisci_rm->tisci_udmap_ops; + struct udma_tchan *tchan = uc->tchan; + struct ti_sci_msg_rm_udmap_tx_ch_cfg req_tx = { 0 }; + int ret = 0; + + req_tx.valid_params = TISCI_BCDMA_TCHAN_VALID_PARAMS; + req_tx.nav_id = tisci_rm->tisci_dev_id; + req_tx.index = tchan->id; + req_tx.tx_supr_tdpkt = uc->config.notdpkt; + if (ud->match_data->flags & UDMA_FLAG_TDTYPE) { + /* wait for peer to complete the teardown for PDMAs */ + req_tx.valid_params |= + TI_SCI_MSG_VALUE_RM_UDMAP_CH_TX_TDTYPE_VALID; + req_tx.tx_tdtype = 1; + } + + ret = tisci_ops->tx_ch_cfg(tisci_rm->tisci, &req_tx); + if (ret) + dev_err(ud->dev, "tchan%d cfg failed %d\n", tchan->id, ret); + + return ret; +} + static int udma_tisci_rx_channel_config(struct udma_chan *uc) { struct udma_dev *ud = uc->ud; @@ -1627,7 +1879,7 @@ static int udma_tisci_rx_channel_config(struct udma_chan *uc) fetch_size = sizeof(struct cppi5_desc_hdr_t); } - req_rx.valid_params = TISCI_RCHAN_VALID_PARAMS; + req_rx.valid_params = TISCI_UDMA_RCHAN_VALID_PARAMS; req_rx.nav_id = tisci_rm->tisci_dev_id; req_rx.index = rchan->id; req_rx.rx_fetch_size = fetch_size >> 2; @@ -1686,6 +1938,26 @@ static int udma_tisci_rx_channel_config(struct udma_chan *uc) return 0; } +static int bcdma_tisci_rx_channel_config(struct udma_chan *uc) +{ + struct udma_dev *ud = uc->ud; + struct udma_tisci_rm *tisci_rm = &ud->tisci_rm; + const struct ti_sci_rm_udmap_ops *tisci_ops = tisci_rm->tisci_udmap_ops; + struct udma_rchan *rchan = uc->rchan; + struct ti_sci_msg_rm_udmap_rx_ch_cfg req_rx = { 0 }; + int ret = 0; + + req_rx.valid_params = TISCI_BCDMA_RCHAN_VALID_PARAMS; + req_rx.nav_id = tisci_rm->tisci_dev_id; + req_rx.index = rchan->id; + + ret = tisci_ops->rx_ch_cfg(tisci_rm->tisci, &req_rx); + if (ret) + dev_err(ud->dev, "rchan%d cfg failed %d\n", rchan->id, ret); + + return ret; +} + static int udma_alloc_chan_resources(struct dma_chan *chan) { struct udma_chan *uc = to_udma_chan(chan); @@ -1695,6 +1967,8 @@ static int udma_alloc_chan_resources(struct dma_chan *chan) u32 irq_udma_idx; int ret; + uc->dma_dev = ud->dev; + if (uc->config.pkt_mode || uc->config.dir == DMA_MEM_TO_MEM) { uc->use_dma_pool = true; /* in case of MEM_TO_MEM we have maximum of two TRs */ @@ -1790,7 +2064,7 @@ static int udma_alloc_chan_resources(struct dma_chan *chan) K3_PSIL_DST_THREAD_ID_OFFSET; irq_ring = uc->rflow->r_ring; - irq_udma_idx = soc_data->rchan_oes_offset + uc->rchan->id; + irq_udma_idx = soc_data->oes.udma_rchan + uc->rchan->id; ret = udma_tisci_rx_channel_config(uc); break; @@ -1890,81 +2164,293 @@ err_cleanup: return ret; } -static int udma_slave_config(struct dma_chan *chan, - struct dma_slave_config *cfg) +static int bcdma_alloc_chan_resources(struct dma_chan *chan) { struct udma_chan *uc = to_udma_chan(chan); + struct udma_dev *ud = to_udma_dev(chan->device); + const struct udma_oes_offsets *oes = &ud->soc_data->oes; + u32 irq_udma_idx, irq_ring_idx; + int ret; - memcpy(&uc->cfg, cfg, sizeof(uc->cfg)); + /* Only TR mode is supported */ + uc->config.pkt_mode = false; - return 0; -} + /* + * Make sure that the completion is in a known state: + * No teardown, the channel is idle + */ + reinit_completion(&uc->teardown_completed); + complete_all(&uc->teardown_completed); + uc->state = UDMA_CHAN_IS_IDLE; -static struct udma_desc *udma_alloc_tr_desc(struct udma_chan *uc, - size_t tr_size, int tr_count, - enum dma_transfer_direction dir) -{ - struct udma_hwdesc *hwdesc; - struct cppi5_desc_hdr_t *tr_desc; - struct udma_desc *d; - u32 reload_count = 0; - u32 ring_id; + switch (uc->config.dir) { + case DMA_MEM_TO_MEM: + /* Non synchronized - mem to mem type of transfer */ + dev_dbg(uc->ud->dev, "%s: chan%d as MEM-to-MEM\n", __func__, + uc->id); - switch (tr_size) { - case 16: - case 32: - case 64: - case 128: - break; - default: - dev_err(uc->ud->dev, "Unsupported TR size of %zu\n", tr_size); - return NULL; - } + ret = bcdma_alloc_bchan_resources(uc); + if (ret) + return ret; - /* We have only one descriptor containing multiple TRs */ - d = kzalloc(sizeof(*d) + sizeof(d->hwdesc[0]), GFP_NOWAIT); - if (!d) - return NULL; + irq_ring_idx = uc->bchan->id + oes->bcdma_bchan_ring; + irq_udma_idx = uc->bchan->id + oes->bcdma_bchan_data; - d->sglen = tr_count; + ret = bcdma_tisci_m2m_channel_config(uc); + break; + case DMA_MEM_TO_DEV: + /* Slave transfer synchronized - mem to dev (TX) trasnfer */ + dev_dbg(uc->ud->dev, "%s: chan%d as MEM-to-DEV\n", __func__, + uc->id); - d->hwdesc_count = 1; - hwdesc = &d->hwdesc[0]; + ret = udma_alloc_tx_resources(uc); + if (ret) { + uc->config.remote_thread_id = -1; + return ret; + } - /* Allocate memory for DMA ring descriptor */ - if (uc->use_dma_pool) { - hwdesc->cppi5_desc_size = uc->config.hdesc_size; - hwdesc->cppi5_desc_vaddr = dma_pool_zalloc(uc->hdesc_pool, - GFP_NOWAIT, - &hwdesc->cppi5_desc_paddr); - } else { - hwdesc->cppi5_desc_size = cppi5_trdesc_calc_size(tr_size, - tr_count); - hwdesc->cppi5_desc_size = ALIGN(hwdesc->cppi5_desc_size, - uc->ud->desc_align); - hwdesc->cppi5_desc_vaddr = dma_alloc_coherent(uc->ud->dev, - hwdesc->cppi5_desc_size, - &hwdesc->cppi5_desc_paddr, - GFP_NOWAIT); - } + uc->config.src_thread = ud->psil_base + uc->tchan->id; + uc->config.dst_thread = uc->config.remote_thread_id; + uc->config.dst_thread |= K3_PSIL_DST_THREAD_ID_OFFSET; - if (!hwdesc->cppi5_desc_vaddr) { - kfree(d); - return NULL; - } + irq_ring_idx = uc->tchan->id + oes->bcdma_tchan_ring; + irq_udma_idx = uc->tchan->id + oes->bcdma_tchan_data; - /* Start of the TR req records */ - hwdesc->tr_req_base = hwdesc->cppi5_desc_vaddr + tr_size; - /* Start address of the TR response array */ - hwdesc->tr_resp_base = hwdesc->tr_req_base + tr_size * tr_count; + ret = bcdma_tisci_tx_channel_config(uc); + break; + case DMA_DEV_TO_MEM: + /* Slave transfer synchronized - dev to mem (RX) trasnfer */ + dev_dbg(uc->ud->dev, "%s: chan%d as DEV-to-MEM\n", __func__, + uc->id); - tr_desc = hwdesc->cppi5_desc_vaddr; + ret = udma_alloc_rx_resources(uc); + if (ret) { + uc->config.remote_thread_id = -1; + return ret; + } - if (uc->cyclic) - reload_count = CPPI5_INFO0_TRDESC_RLDCNT_INFINITE; + uc->config.src_thread = uc->config.remote_thread_id; + uc->config.dst_thread = (ud->psil_base + uc->rchan->id) | + K3_PSIL_DST_THREAD_ID_OFFSET; - if (dir == DMA_DEV_TO_MEM) - ring_id = k3_ringacc_get_ring_id(uc->rflow->r_ring); + irq_ring_idx = uc->rchan->id + oes->bcdma_rchan_ring; + irq_udma_idx = uc->rchan->id + oes->bcdma_rchan_data; + + ret = bcdma_tisci_rx_channel_config(uc); + break; + default: + /* Can not happen */ + dev_err(uc->ud->dev, "%s: chan%d invalid direction (%u)\n", + __func__, uc->id, uc->config.dir); + return -EINVAL; + } + + /* check if the channel configuration was successful */ + if (ret) + goto err_res_free; + + if (udma_is_chan_running(uc)) { + dev_warn(ud->dev, "chan%d: is running!\n", uc->id); + udma_reset_chan(uc, false); + if (udma_is_chan_running(uc)) { + dev_err(ud->dev, "chan%d: won't stop!\n", uc->id); + ret = -EBUSY; + goto err_res_free; + } + } + + uc->dma_dev = dmaengine_get_dma_device(chan); + if (uc->config.dir == DMA_MEM_TO_MEM && !uc->config.tr_trigger_type) { + uc->config.hdesc_size = cppi5_trdesc_calc_size( + sizeof(struct cppi5_tr_type15_t), 2); + + uc->hdesc_pool = dma_pool_create(uc->name, ud->ddev.dev, + uc->config.hdesc_size, + ud->desc_align, + 0); + if (!uc->hdesc_pool) { + dev_err(ud->ddev.dev, + "Descriptor pool allocation failed\n"); + uc->use_dma_pool = false; + return -ENOMEM; + } + + uc->use_dma_pool = true; + } else if (uc->config.dir != DMA_MEM_TO_MEM) { + /* PSI-L pairing */ + ret = navss_psil_pair(ud, uc->config.src_thread, + uc->config.dst_thread); + if (ret) { + dev_err(ud->dev, + "PSI-L pairing failed: 0x%04x -> 0x%04x\n", + uc->config.src_thread, uc->config.dst_thread); + goto err_res_free; + } + + uc->psil_paired = true; + } + + uc->irq_num_ring = ti_sci_inta_msi_get_virq(ud->dev, irq_ring_idx); + if (uc->irq_num_ring <= 0) { + dev_err(ud->dev, "Failed to get ring irq (index: %u)\n", + irq_ring_idx); + ret = -EINVAL; + goto err_psi_free; + } + + ret = request_irq(uc->irq_num_ring, udma_ring_irq_handler, + IRQF_TRIGGER_HIGH, uc->name, uc); + if (ret) { + dev_err(ud->dev, "chan%d: ring irq request failed\n", uc->id); + goto err_irq_free; + } + + /* Event from BCDMA (TR events) only needed for slave channels */ + if (is_slave_direction(uc->config.dir)) { + uc->irq_num_udma = ti_sci_inta_msi_get_virq(ud->dev, + irq_udma_idx); + if (uc->irq_num_udma <= 0) { + dev_err(ud->dev, "Failed to get bcdma irq (index: %u)\n", + irq_udma_idx); + free_irq(uc->irq_num_ring, uc); + ret = -EINVAL; + goto err_irq_free; + } + + ret = request_irq(uc->irq_num_udma, udma_udma_irq_handler, 0, + uc->name, uc); + if (ret) { + dev_err(ud->dev, "chan%d: BCDMA irq request failed\n", + uc->id); + free_irq(uc->irq_num_ring, uc); + goto err_irq_free; + } + } else { + uc->irq_num_udma = 0; + } + + udma_reset_rings(uc); + + INIT_DELAYED_WORK_ONSTACK(&uc->tx_drain.work, + udma_check_tx_completion); + return 0; + +err_irq_free: + uc->irq_num_ring = 0; + uc->irq_num_udma = 0; +err_psi_free: + if (uc->psil_paired) + navss_psil_unpair(ud, uc->config.src_thread, + uc->config.dst_thread); + uc->psil_paired = false; +err_res_free: + bcdma_free_bchan_resources(uc); + udma_free_tx_resources(uc); + udma_free_rx_resources(uc); + + udma_reset_uchan(uc); + + if (uc->use_dma_pool) { + dma_pool_destroy(uc->hdesc_pool); + uc->use_dma_pool = false; + } + + return ret; +} + +static int bcdma_router_config(struct dma_chan *chan) +{ + struct k3_event_route_data *router_data = chan->route_data; + struct udma_chan *uc = to_udma_chan(chan); + u32 trigger_event; + + if (!uc->bchan) + return -EINVAL; + + if (uc->config.tr_trigger_type != 1 && uc->config.tr_trigger_type != 2) + return -EINVAL; + + trigger_event = uc->ud->soc_data->bcdma_trigger_event_offset; + trigger_event += (uc->bchan->id * 2) + uc->config.tr_trigger_type - 1; + + return router_data->set_event(router_data->priv, trigger_event); +} + +static int udma_slave_config(struct dma_chan *chan, + struct dma_slave_config *cfg) +{ + struct udma_chan *uc = to_udma_chan(chan); + + memcpy(&uc->cfg, cfg, sizeof(uc->cfg)); + + return 0; +} + +static struct udma_desc *udma_alloc_tr_desc(struct udma_chan *uc, + size_t tr_size, int tr_count, + enum dma_transfer_direction dir) +{ + struct udma_hwdesc *hwdesc; + struct cppi5_desc_hdr_t *tr_desc; + struct udma_desc *d; + u32 reload_count = 0; + u32 ring_id; + + switch (tr_size) { + case 16: + case 32: + case 64: + case 128: + break; + default: + dev_err(uc->ud->dev, "Unsupported TR size of %zu\n", tr_size); + return NULL; + } + + /* We have only one descriptor containing multiple TRs */ + d = kzalloc(sizeof(*d) + sizeof(d->hwdesc[0]), GFP_NOWAIT); + if (!d) + return NULL; + + d->sglen = tr_count; + + d->hwdesc_count = 1; + hwdesc = &d->hwdesc[0]; + + /* Allocate memory for DMA ring descriptor */ + if (uc->use_dma_pool) { + hwdesc->cppi5_desc_size = uc->config.hdesc_size; + hwdesc->cppi5_desc_vaddr = dma_pool_zalloc(uc->hdesc_pool, + GFP_NOWAIT, + &hwdesc->cppi5_desc_paddr); + } else { + hwdesc->cppi5_desc_size = cppi5_trdesc_calc_size(tr_size, + tr_count); + hwdesc->cppi5_desc_size = ALIGN(hwdesc->cppi5_desc_size, + uc->ud->desc_align); + hwdesc->cppi5_desc_vaddr = dma_alloc_coherent(uc->ud->dev, + hwdesc->cppi5_desc_size, + &hwdesc->cppi5_desc_paddr, + GFP_NOWAIT); + } + + if (!hwdesc->cppi5_desc_vaddr) { + kfree(d); + return NULL; + } + + /* Start of the TR req records */ + hwdesc->tr_req_base = hwdesc->cppi5_desc_vaddr + tr_size; + /* Start address of the TR response array */ + hwdesc->tr_resp_base = hwdesc->tr_req_base + tr_size * tr_count; + + tr_desc = hwdesc->cppi5_desc_vaddr; + + if (uc->cyclic) + reload_count = CPPI5_INFO0_TRDESC_RLDCNT_INFINITE; + + if (dir == DMA_DEV_TO_MEM) + ring_id = k3_ringacc_get_ring_id(uc->rflow->r_ring); else ring_id = k3_ringacc_get_ring_id(uc->tchan->tc_ring); @@ -2034,6 +2520,7 @@ udma_prep_slave_sg_tr(struct udma_chan *uc, struct scatterlist *sgl, size_t tr_size; int num_tr = 0; int tr_idx = 0; + u64 asel; /* estimate the number of TRs we will need */ for_each_sg(sgl, sgent, sglen, i) { @@ -2051,6 +2538,11 @@ udma_prep_slave_sg_tr(struct udma_chan *uc, struct scatterlist *sgl, d->sglen = sglen; + if (uc->ud->match_data->type == DMA_TYPE_UDMA) + asel = 0; + else + asel = (u64)uc->config.asel << K3_ADDRESS_ASEL_SHIFT; + tr_req = d->hwdesc[0].tr_req_base; for_each_sg(sgl, sgent, sglen, i) { dma_addr_t sg_addr = sg_dma_address(sgent); @@ -2069,6 +2561,7 @@ udma_prep_slave_sg_tr(struct udma_chan *uc, struct scatterlist *sgl, false, CPPI5_TR_EVENT_SIZE_COMPLETION, 0); cppi5_tr_csf_set(&tr_req[tr_idx].flags, CPPI5_TR_CSF_SUPR_EVT); + sg_addr |= asel; tr_req[tr_idx].addr = sg_addr; tr_req[tr_idx].icnt0 = tr0_cnt0; tr_req[tr_idx].icnt1 = tr0_cnt1; @@ -2098,6 +2591,205 @@ udma_prep_slave_sg_tr(struct udma_chan *uc, struct scatterlist *sgl, return d; } +static struct udma_desc * +udma_prep_slave_sg_triggered_tr(struct udma_chan *uc, struct scatterlist *sgl, + unsigned int sglen, + enum dma_transfer_direction dir, + unsigned long tx_flags, void *context) +{ + struct scatterlist *sgent; + struct cppi5_tr_type15_t *tr_req = NULL; + enum dma_slave_buswidth dev_width; + u16 tr_cnt0, tr_cnt1; + dma_addr_t dev_addr; + struct udma_desc *d; + unsigned int i; + size_t tr_size, sg_len; + int num_tr = 0; + int tr_idx = 0; + u32 burst, trigger_size, port_window; + u64 asel; + + if (dir == DMA_DEV_TO_MEM) { + dev_addr = uc->cfg.src_addr; + dev_width = uc->cfg.src_addr_width; + burst = uc->cfg.src_maxburst; + port_window = uc->cfg.src_port_window_size; + } else if (dir == DMA_MEM_TO_DEV) { + dev_addr = uc->cfg.dst_addr; + dev_width = uc->cfg.dst_addr_width; + burst = uc->cfg.dst_maxburst; + port_window = uc->cfg.dst_port_window_size; + } else { + dev_err(uc->ud->dev, "%s: bad direction?\n", __func__); + return NULL; + } + + if (!burst) + burst = 1; + + if (port_window) { + if (port_window != burst) { + dev_err(uc->ud->dev, + "The burst must be equal to port_window\n"); + return NULL; + } + + tr_cnt0 = dev_width * port_window; + tr_cnt1 = 1; + } else { + tr_cnt0 = dev_width; + tr_cnt1 = burst; + } + trigger_size = tr_cnt0 * tr_cnt1; + + /* estimate the number of TRs we will need */ + for_each_sg(sgl, sgent, sglen, i) { + sg_len = sg_dma_len(sgent); + + if (sg_len % trigger_size) { + dev_err(uc->ud->dev, + "Not aligned SG entry (%zu for %u)\n", sg_len, + trigger_size); + return NULL; + } + + if (sg_len / trigger_size < SZ_64K) + num_tr++; + else + num_tr += 2; + } + + /* Now allocate and setup the descriptor. */ + tr_size = sizeof(struct cppi5_tr_type15_t); + d = udma_alloc_tr_desc(uc, tr_size, num_tr, dir); + if (!d) + return NULL; + + d->sglen = sglen; + + if (uc->ud->match_data->type == DMA_TYPE_UDMA) { + asel = 0; + } else { + asel = (u64)uc->config.asel << K3_ADDRESS_ASEL_SHIFT; + dev_addr |= asel; + } + + tr_req = d->hwdesc[0].tr_req_base; + for_each_sg(sgl, sgent, sglen, i) { + u16 tr0_cnt2, tr0_cnt3, tr1_cnt2; + dma_addr_t sg_addr = sg_dma_address(sgent); + + sg_len = sg_dma_len(sgent); + num_tr = udma_get_tr_counters(sg_len / trigger_size, 0, + &tr0_cnt2, &tr0_cnt3, &tr1_cnt2); + if (num_tr < 0) { + dev_err(uc->ud->dev, "size %zu is not supported\n", + sg_len); + udma_free_hwdesc(uc, d); + kfree(d); + return NULL; + } + + cppi5_tr_init(&tr_req[tr_idx].flags, CPPI5_TR_TYPE15, false, + true, CPPI5_TR_EVENT_SIZE_COMPLETION, 0); + cppi5_tr_csf_set(&tr_req[tr_idx].flags, CPPI5_TR_CSF_SUPR_EVT); + cppi5_tr_set_trigger(&tr_req[tr_idx].flags, + uc->config.tr_trigger_type, + CPPI5_TR_TRIGGER_TYPE_ICNT2_DEC, 0, 0); + + sg_addr |= asel; + if (dir == DMA_DEV_TO_MEM) { + tr_req[tr_idx].addr = dev_addr; + tr_req[tr_idx].icnt0 = tr_cnt0; + tr_req[tr_idx].icnt1 = tr_cnt1; + tr_req[tr_idx].icnt2 = tr0_cnt2; + tr_req[tr_idx].icnt3 = tr0_cnt3; + tr_req[tr_idx].dim1 = (-1) * tr_cnt0; + + tr_req[tr_idx].daddr = sg_addr; + tr_req[tr_idx].dicnt0 = tr_cnt0; + tr_req[tr_idx].dicnt1 = tr_cnt1; + tr_req[tr_idx].dicnt2 = tr0_cnt2; + tr_req[tr_idx].dicnt3 = tr0_cnt3; + tr_req[tr_idx].ddim1 = tr_cnt0; + tr_req[tr_idx].ddim2 = trigger_size; + tr_req[tr_idx].ddim3 = trigger_size * tr0_cnt2; + } else { + tr_req[tr_idx].addr = sg_addr; + tr_req[tr_idx].icnt0 = tr_cnt0; + tr_req[tr_idx].icnt1 = tr_cnt1; + tr_req[tr_idx].icnt2 = tr0_cnt2; + tr_req[tr_idx].icnt3 = tr0_cnt3; + tr_req[tr_idx].dim1 = tr_cnt0; + tr_req[tr_idx].dim2 = trigger_size; + tr_req[tr_idx].dim3 = trigger_size * tr0_cnt2; + + tr_req[tr_idx].daddr = dev_addr; + tr_req[tr_idx].dicnt0 = tr_cnt0; + tr_req[tr_idx].dicnt1 = tr_cnt1; + tr_req[tr_idx].dicnt2 = tr0_cnt2; + tr_req[tr_idx].dicnt3 = tr0_cnt3; + tr_req[tr_idx].ddim1 = (-1) * tr_cnt0; + } + + tr_idx++; + + if (num_tr == 2) { + cppi5_tr_init(&tr_req[tr_idx].flags, CPPI5_TR_TYPE15, + false, true, + CPPI5_TR_EVENT_SIZE_COMPLETION, 0); + cppi5_tr_csf_set(&tr_req[tr_idx].flags, + CPPI5_TR_CSF_SUPR_EVT); + cppi5_tr_set_trigger(&tr_req[tr_idx].flags, + uc->config.tr_trigger_type, + CPPI5_TR_TRIGGER_TYPE_ICNT2_DEC, + 0, 0); + + sg_addr += trigger_size * tr0_cnt2 * tr0_cnt3; + if (dir == DMA_DEV_TO_MEM) { + tr_req[tr_idx].addr = dev_addr; + tr_req[tr_idx].icnt0 = tr_cnt0; + tr_req[tr_idx].icnt1 = tr_cnt1; + tr_req[tr_idx].icnt2 = tr1_cnt2; + tr_req[tr_idx].icnt3 = 1; + tr_req[tr_idx].dim1 = (-1) * tr_cnt0; + + tr_req[tr_idx].daddr = sg_addr; + tr_req[tr_idx].dicnt0 = tr_cnt0; + tr_req[tr_idx].dicnt1 = tr_cnt1; + tr_req[tr_idx].dicnt2 = tr1_cnt2; + tr_req[tr_idx].dicnt3 = 1; + tr_req[tr_idx].ddim1 = tr_cnt0; + tr_req[tr_idx].ddim2 = trigger_size; + } else { + tr_req[tr_idx].addr = sg_addr; + tr_req[tr_idx].icnt0 = tr_cnt0; + tr_req[tr_idx].icnt1 = tr_cnt1; + tr_req[tr_idx].icnt2 = tr1_cnt2; + tr_req[tr_idx].icnt3 = 1; + tr_req[tr_idx].dim1 = tr_cnt0; + tr_req[tr_idx].dim2 = trigger_size; + + tr_req[tr_idx].daddr = dev_addr; + tr_req[tr_idx].dicnt0 = tr_cnt0; + tr_req[tr_idx].dicnt1 = tr_cnt1; + tr_req[tr_idx].dicnt2 = tr1_cnt2; + tr_req[tr_idx].dicnt3 = 1; + tr_req[tr_idx].ddim1 = (-1) * tr_cnt0; + } + tr_idx++; + } + + d->residue += sg_len; + } + + cppi5_tr_csf_set(&tr_req[tr_idx - 1].flags, + CPPI5_TR_CSF_SUPR_EVT | CPPI5_TR_CSF_EOP); + + return d; +} + static int udma_configure_statictr(struct udma_chan *uc, struct udma_desc *d, enum dma_slave_buswidth dev_width, u16 elcnt) @@ -2339,7 +3031,8 @@ udma_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl, struct udma_desc *d; u32 burst; - if (dir != uc->config.dir) { + if (dir != uc->config.dir && + (uc->config.dir == DMA_MEM_TO_MEM && !uc->config.tr_trigger_type)) { dev_err(chan->device->dev, "%s: chan%d is for %s, not supporting %s\n", __func__, uc->id, @@ -2365,9 +3058,12 @@ udma_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl, if (uc->config.pkt_mode) d = udma_prep_slave_sg_pkt(uc, sgl, sglen, dir, tx_flags, context); - else + else if (is_slave_direction(uc->config.dir)) d = udma_prep_slave_sg_tr(uc, sgl, sglen, dir, tx_flags, context); + else + d = udma_prep_slave_sg_triggered_tr(uc, sgl, sglen, dir, + tx_flags, context); if (!d) return NULL; @@ -2421,7 +3117,12 @@ udma_prep_dma_cyclic_tr(struct udma_chan *uc, dma_addr_t buf_addr, return NULL; tr_req = d->hwdesc[0].tr_req_base; - period_addr = buf_addr; + if (uc->ud->match_data->type == DMA_TYPE_UDMA) + period_addr = buf_addr; + else + period_addr = buf_addr | + ((u64)uc->config.asel << K3_ADDRESS_ASEL_SHIFT); + for (i = 0; i < periods; i++) { int tr_idx = i * num_tr; @@ -2627,6 +3328,11 @@ udma_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src, d->tr_idx = 0; d->residue = len; + if (uc->ud->match_data->type != DMA_TYPE_UDMA) { + src |= (u64)uc->ud->asel << K3_ADDRESS_ASEL_SHIFT; + dest |= (u64)uc->ud->asel << K3_ADDRESS_ASEL_SHIFT; + } + tr_req = d->hwdesc[0].tr_req_base; cppi5_tr_init(&tr_req[0].flags, CPPI5_TR_TYPE15, false, true, @@ -2984,6 +3690,7 @@ static void udma_free_chan_resources(struct dma_chan *chan) vchan_free_chan_resources(&uc->vc); tasklet_kill(&uc->vc.task); + bcdma_free_bchan_resources(uc); udma_free_tx_resources(uc); udma_free_rx_resources(uc); udma_reset_uchan(uc); @@ -2995,10 +3702,13 @@ static void udma_free_chan_resources(struct dma_chan *chan) } static struct platform_driver udma_driver; +static struct platform_driver bcdma_driver; struct udma_filter_param { int remote_thread_id; u32 atype; + u32 asel; + u32 tr_trigger_type; }; static bool udma_dma_filter_fn(struct dma_chan *chan, void *param) @@ -3009,7 +3719,8 @@ static bool udma_dma_filter_fn(struct dma_chan *chan, void *param) struct udma_chan *uc; struct udma_dev *ud; - if (chan->device->dev->driver != &udma_driver.driver) + if (chan->device->dev->driver != &udma_driver.driver && + chan->device->dev->driver != &bcdma_driver.driver) return false; uc = to_udma_chan(chan); @@ -3023,13 +3734,25 @@ static bool udma_dma_filter_fn(struct dma_chan *chan, void *param) return false; } + if (filter_param->asel > 15) { + dev_err(ud->dev, "Invalid channel asel: %u\n", + filter_param->asel); + return false; + } + ucc->remote_thread_id = filter_param->remote_thread_id; ucc->atype = filter_param->atype; + ucc->asel = filter_param->asel; + ucc->tr_trigger_type = filter_param->tr_trigger_type; - if (ucc->remote_thread_id & K3_PSIL_DST_THREAD_ID_OFFSET) + if (ucc->tr_trigger_type) { + ucc->dir = DMA_MEM_TO_MEM; + goto triggered_bchan; + } else if (ucc->remote_thread_id & K3_PSIL_DST_THREAD_ID_OFFSET) { ucc->dir = DMA_MEM_TO_DEV; - else + } else { ucc->dir = DMA_DEV_TO_MEM; + } ep_config = psil_get_ep_config(ucc->remote_thread_id); if (IS_ERR(ep_config)) { @@ -3038,6 +3761,19 @@ static bool udma_dma_filter_fn(struct dma_chan *chan, void *param) ucc->dir = DMA_MEM_TO_MEM; ucc->remote_thread_id = -1; ucc->atype = 0; + ucc->asel = 0; + return false; + } + + if (ud->match_data->type == DMA_TYPE_BCDMA && + ep_config->pkt_mode) { + dev_err(ud->dev, + "Only TR mode is supported (psi-l thread 0x%04x)\n", + ucc->remote_thread_id); + ucc->dir = DMA_MEM_TO_MEM; + ucc->remote_thread_id = -1; + ucc->atype = 0; + ucc->asel = 0; return false; } @@ -3069,6 +3805,13 @@ static bool udma_dma_filter_fn(struct dma_chan *chan, void *param) ucc->remote_thread_id, dmaengine_get_direction_text(ucc->dir)); return true; + +triggered_bchan: + dev_dbg(ud->dev, "chan%d: triggered channel (type: %u)\n", uc->id, + ucc->tr_trigger_type); + + return true; + } static struct dma_chan *udma_of_xlate(struct of_phandle_args *dma_spec, @@ -3079,14 +3822,33 @@ static struct dma_chan *udma_of_xlate(struct of_phandle_args *dma_spec, struct udma_filter_param filter_param; struct dma_chan *chan; - if (dma_spec->args_count != 1 && dma_spec->args_count != 2) - return NULL; + if (ud->match_data->type == DMA_TYPE_BCDMA) { + if (dma_spec->args_count != 3) + return NULL; - filter_param.remote_thread_id = dma_spec->args[0]; - if (dma_spec->args_count == 2) - filter_param.atype = dma_spec->args[1]; - else + filter_param.tr_trigger_type = dma_spec->args[0]; + filter_param.remote_thread_id = dma_spec->args[1]; + filter_param.asel = dma_spec->args[2]; filter_param.atype = 0; + } else { + if (dma_spec->args_count != 1 && dma_spec->args_count != 2) + return NULL; + + filter_param.remote_thread_id = dma_spec->args[0]; + filter_param.tr_trigger_type = 0; + if (dma_spec->args_count == 2) { + if (ud->match_data->type == DMA_TYPE_UDMA) { + filter_param.atype = dma_spec->args[1]; + filter_param.asel = 0; + } else { + filter_param.atype = 0; + filter_param.asel = dma_spec->args[1]; + } + } else { + filter_param.atype = 0; + filter_param.asel = 0; + } + } chan = __dma_request_channel(&mask, udma_dma_filter_fn, &filter_param, ofdma->of_node); @@ -3099,18 +3861,21 @@ static struct dma_chan *udma_of_xlate(struct of_phandle_args *dma_spec, } static struct udma_match_data am654_main_data = { + .type = DMA_TYPE_UDMA, .psil_base = 0x1000, .enable_memcpy_support = true, .statictr_z_mask = GENMASK(11, 0), }; static struct udma_match_data am654_mcu_data = { + .type = DMA_TYPE_UDMA, .psil_base = 0x6000, .enable_memcpy_support = false, .statictr_z_mask = GENMASK(11, 0), }; static struct udma_match_data j721e_main_data = { + .type = DMA_TYPE_UDMA, .psil_base = 0x1000, .enable_memcpy_support = true, .flags = UDMA_FLAG_PDMA_ACC32 | UDMA_FLAG_PDMA_BURST | UDMA_FLAG_TDTYPE, @@ -3118,12 +3883,21 @@ static struct udma_match_data j721e_main_data = { }; static struct udma_match_data j721e_mcu_data = { + .type = DMA_TYPE_UDMA, .psil_base = 0x6000, .enable_memcpy_support = false, /* MEM_TO_MEM is slow via MCU UDMA */ .flags = UDMA_FLAG_PDMA_ACC32 | UDMA_FLAG_PDMA_BURST | UDMA_FLAG_TDTYPE, .statictr_z_mask = GENMASK(23, 0), }; +static struct udma_match_data am64_bcdma_data = { + .type = DMA_TYPE_BCDMA, + .psil_base = 0x2000, /* for tchan and rchan, not applicable to bchan */ + .enable_memcpy_support = true, /* Supported via bchan */ + .flags = UDMA_FLAG_PDMA_ACC32 | UDMA_FLAG_PDMA_BURST | UDMA_FLAG_TDTYPE, + .statictr_z_mask = GENMASK(23, 0), +}; + static const struct of_device_id udma_of_match[] = { { .compatible = "ti,am654-navss-main-udmap", @@ -3142,30 +3916,88 @@ static const struct of_device_id udma_of_match[] = { { /* Sentinel */ }, }; +static const struct of_device_id bcdma_of_match[] = { + { + .compatible = "ti,am64-dmss-bcdma", + .data = &am64_bcdma_data, + }, + { /* Sentinel */ }, +}; + static struct udma_soc_data am654_soc_data = { - .rchan_oes_offset = 0x200, + .oes = { + .udma_rchan = 0x200, + }, }; static struct udma_soc_data j721e_soc_data = { - .rchan_oes_offset = 0x400, + .oes = { + .udma_rchan = 0x400, + }, }; static struct udma_soc_data j7200_soc_data = { - .rchan_oes_offset = 0x80, + .oes = { + .udma_rchan = 0x80, + }, +}; + +static struct udma_soc_data am64_soc_data = { + .oes = { + .bcdma_bchan_data = 0x2200, + .bcdma_bchan_ring = 0x2400, + .bcdma_tchan_data = 0x2800, + .bcdma_tchan_ring = 0x2a00, + .bcdma_rchan_data = 0x2e00, + .bcdma_rchan_ring = 0x3000, + }, + .bcdma_trigger_event_offset = 0xc400, }; static const struct soc_device_attribute k3_soc_devices[] = { { .family = "AM65X", .data = &am654_soc_data }, { .family = "J721E", .data = &j721e_soc_data }, { .family = "J7200", .data = &j7200_soc_data }, + { .family = "AM64X", .data = &am64_soc_data }, { /* sentinel */ } }; static int udma_get_mmrs(struct platform_device *pdev, struct udma_dev *ud) { + u32 cap2, cap3; int i; - for (i = 0; i < MMR_LAST; i++) { + ud->mmrs[MMR_GCFG] = devm_platform_ioremap_resource_byname(pdev, mmr_names[MMR_GCFG]); + if (IS_ERR(ud->mmrs[MMR_GCFG])) + return PTR_ERR(ud->mmrs[MMR_GCFG]); + + cap2 = udma_read(ud->mmrs[MMR_GCFG], 0x28); + cap3 = udma_read(ud->mmrs[MMR_GCFG], 0x2c); + + switch (ud->match_data->type) { + case DMA_TYPE_UDMA: + ud->rflow_cnt = UDMA_CAP3_RFLOW_CNT(cap3); + ud->tchan_cnt = UDMA_CAP2_TCHAN_CNT(cap2); + ud->echan_cnt = UDMA_CAP2_ECHAN_CNT(cap2); + ud->rchan_cnt = UDMA_CAP2_RCHAN_CNT(cap2); + break; + case DMA_TYPE_BCDMA: + ud->bchan_cnt = BCDMA_CAP2_BCHAN_CNT(cap2); + ud->tchan_cnt = BCDMA_CAP2_TCHAN_CNT(cap2); + ud->rchan_cnt = BCDMA_CAP2_RCHAN_CNT(cap2); + break; + default: + return -EINVAL; + } + + for (i = 1; i < MMR_LAST; i++) { + if (i == MMR_BCHANRT && ud->bchan_cnt == 0) + continue; + if (i == MMR_TCHANRT && ud->tchan_cnt == 0) + continue; + if (i == MMR_RCHANRT && ud->rchan_cnt == 0) + continue; + ud->mmrs[i] = devm_platform_ioremap_resource_byname(pdev, mmr_names[i]); if (IS_ERR(ud->mmrs[i])) return PTR_ERR(ud->mmrs[i]); @@ -3185,27 +4017,23 @@ static void udma_mark_resource_ranges(struct udma_dev *ud, unsigned long *map, rm_desc->num_sec); } +static const char * const range_names[] = { + [RM_RANGE_BCHAN] = "ti,sci-rm-range-bchan", + [RM_RANGE_TCHAN] = "ti,sci-rm-range-tchan", + [RM_RANGE_RCHAN] = "ti,sci-rm-range-rchan", + [RM_RANGE_RFLOW] = "ti,sci-rm-range-rflow" +}; + static int udma_setup_resources(struct udma_dev *ud) { + int ret, i, j; struct device *dev = ud->dev; - int ch_count, ret, i, j; - u32 cap2, cap3; struct ti_sci_resource *rm_res, irq_res; struct udma_tisci_rm *tisci_rm = &ud->tisci_rm; - static const char * const range_names[] = { "ti,sci-rm-range-tchan", - "ti,sci-rm-range-rchan", - "ti,sci-rm-range-rflow" }; - - cap2 = udma_read(ud->mmrs[MMR_GCFG], UDMA_CAP_REG(2)); - cap3 = udma_read(ud->mmrs[MMR_GCFG], UDMA_CAP_REG(3)); - - ud->rflow_cnt = UDMA_CAP3_RFLOW_CNT(cap3); - ud->tchan_cnt = UDMA_CAP2_TCHAN_CNT(cap2); - ud->echan_cnt = UDMA_CAP2_ECHAN_CNT(cap2); - ud->rchan_cnt = UDMA_CAP2_RCHAN_CNT(cap2); - ch_count = ud->tchan_cnt + ud->rchan_cnt; + u32 cap3; /* Set up the throughput level start indexes */ + cap3 = udma_read(ud->mmrs[MMR_GCFG], 0x2c); if (of_device_is_compatible(dev->of_node, "ti,am654-navss-main-udmap")) { ud->tpl_levels = 2; @@ -3262,11 +4090,15 @@ static int udma_setup_resources(struct udma_dev *ud) bitmap_set(ud->rflow_gp_map, 0, ud->rflow_cnt); /* Get resource ranges from tisci */ - for (i = 0; i < RM_RANGE_LAST; i++) + for (i = 0; i < RM_RANGE_LAST; i++) { + if (i == RM_RANGE_BCHAN) + continue; + tisci_rm->rm_ranges[i] = devm_ti_sci_get_of_resource(tisci_rm->tisci, dev, tisci_rm->tisci_dev_id, (char *)range_names[i]); + } /* tchan ranges */ rm_res = tisci_rm->rm_ranges[RM_RANGE_TCHAN]; @@ -3304,12 +4136,12 @@ static int udma_setup_resources(struct udma_dev *ud) for (j = 0; j < rm_res->sets; j++, i++) { if (rm_res->desc[j].num) { irq_res.desc[i].start = rm_res->desc[j].start + - ud->soc_data->rchan_oes_offset; + ud->soc_data->oes.udma_rchan; irq_res.desc[i].num = rm_res->desc[j].num; } if (rm_res->desc[j].num_sec) { irq_res.desc[i].start_sec = rm_res->desc[j].start_sec + - ud->soc_data->rchan_oes_offset; + ud->soc_data->oes.udma_rchan; irq_res.desc[i].num_sec = rm_res->desc[j].num_sec; } } @@ -3332,6 +4164,174 @@ static int udma_setup_resources(struct udma_dev *ud) &rm_res->desc[i], "gp-rflow"); } + return 0; +} + +static int bcdma_setup_resources(struct udma_dev *ud) +{ + int ret, i, j; + struct device *dev = ud->dev; + struct ti_sci_resource *rm_res, irq_res; + struct udma_tisci_rm *tisci_rm = &ud->tisci_rm; + const struct udma_oes_offsets *oes = &ud->soc_data->oes; + + ud->bchan_map = devm_kmalloc_array(dev, BITS_TO_LONGS(ud->bchan_cnt), + sizeof(unsigned long), GFP_KERNEL); + ud->bchans = devm_kcalloc(dev, ud->bchan_cnt, sizeof(*ud->bchans), + GFP_KERNEL); + ud->tchan_map = devm_kmalloc_array(dev, BITS_TO_LONGS(ud->tchan_cnt), + sizeof(unsigned long), GFP_KERNEL); + ud->tchans = devm_kcalloc(dev, ud->tchan_cnt, sizeof(*ud->tchans), + GFP_KERNEL); + ud->rchan_map = devm_kmalloc_array(dev, BITS_TO_LONGS(ud->rchan_cnt), + sizeof(unsigned long), GFP_KERNEL); + ud->rchans = devm_kcalloc(dev, ud->rchan_cnt, sizeof(*ud->rchans), + GFP_KERNEL); + /* BCDMA do not really have flows, but the driver expect it */ + ud->rflow_in_use = devm_kcalloc(dev, BITS_TO_LONGS(ud->rchan_cnt), + sizeof(unsigned long), + GFP_KERNEL); + ud->rflows = devm_kcalloc(dev, ud->rchan_cnt, sizeof(*ud->rflows), + GFP_KERNEL); + + if (!ud->bchan_map || !ud->tchan_map || !ud->rchan_map || + !ud->rflow_in_use || !ud->bchans || !ud->tchans || !ud->rchans || + !ud->rflows) + return -ENOMEM; + + /* TPL is not yet supported for BCDMA */ + ud->tpl_levels = 1; + + /* Get resource ranges from tisci */ + for (i = 0; i < RM_RANGE_LAST; i++) { + if (i == RM_RANGE_RFLOW) + continue; + if (i == RM_RANGE_BCHAN && ud->bchan_cnt == 0) + continue; + if (i == RM_RANGE_TCHAN && ud->tchan_cnt == 0) + continue; + if (i == RM_RANGE_RCHAN && ud->rchan_cnt == 0) + continue; + + tisci_rm->rm_ranges[i] = + devm_ti_sci_get_of_resource(tisci_rm->tisci, dev, + tisci_rm->tisci_dev_id, + (char *)range_names[i]); + } + + irq_res.sets = 0; + + /* bchan ranges */ + if (ud->bchan_cnt) { + rm_res = tisci_rm->rm_ranges[RM_RANGE_BCHAN]; + if (IS_ERR(rm_res)) { + bitmap_zero(ud->bchan_map, ud->bchan_cnt); + } else { + bitmap_fill(ud->bchan_map, ud->bchan_cnt); + for (i = 0; i < rm_res->sets; i++) + udma_mark_resource_ranges(ud, ud->bchan_map, + &rm_res->desc[i], + "bchan"); + } + irq_res.sets += rm_res->sets; + } + + /* tchan ranges */ + if (ud->tchan_cnt) { + rm_res = tisci_rm->rm_ranges[RM_RANGE_TCHAN]; + if (IS_ERR(rm_res)) { + bitmap_zero(ud->tchan_map, ud->tchan_cnt); + } else { + bitmap_fill(ud->tchan_map, ud->tchan_cnt); + for (i = 0; i < rm_res->sets; i++) + udma_mark_resource_ranges(ud, ud->tchan_map, + &rm_res->desc[i], + "tchan"); + } + irq_res.sets += rm_res->sets * 2; + } + + /* rchan ranges */ + if (ud->rchan_cnt) { + rm_res = tisci_rm->rm_ranges[RM_RANGE_RCHAN]; + if (IS_ERR(rm_res)) { + bitmap_zero(ud->rchan_map, ud->rchan_cnt); + } else { + bitmap_fill(ud->rchan_map, ud->rchan_cnt); + for (i = 0; i < rm_res->sets; i++) + udma_mark_resource_ranges(ud, ud->rchan_map, + &rm_res->desc[i], + "rchan"); + } + irq_res.sets += rm_res->sets * 2; + } + + irq_res.desc = kcalloc(irq_res.sets, sizeof(*irq_res.desc), GFP_KERNEL); + if (ud->bchan_cnt) { + rm_res = tisci_rm->rm_ranges[RM_RANGE_BCHAN]; + for (i = 0; i < rm_res->sets; i++) { + irq_res.desc[i].start = rm_res->desc[i].start + + oes->bcdma_bchan_ring; + irq_res.desc[i].num = rm_res->desc[i].num; + } + } + if (ud->tchan_cnt) { + rm_res = tisci_rm->rm_ranges[RM_RANGE_TCHAN]; + for (j = 0; j < rm_res->sets; j++, i += 2) { + irq_res.desc[i].start = rm_res->desc[j].start + + oes->bcdma_tchan_data; + irq_res.desc[i].num = rm_res->desc[j].num; + + irq_res.desc[i + 1].start = rm_res->desc[j].start + + oes->bcdma_tchan_ring; + irq_res.desc[i + 1].num = rm_res->desc[j].num; + } + } + if (ud->rchan_cnt) { + rm_res = tisci_rm->rm_ranges[RM_RANGE_RCHAN]; + for (j = 0; j < rm_res->sets; j++, i += 2) { + irq_res.desc[i].start = rm_res->desc[j].start + + oes->bcdma_rchan_data; + irq_res.desc[i].num = rm_res->desc[j].num; + + irq_res.desc[i + 1].start = rm_res->desc[j].start + + oes->bcdma_rchan_ring; + irq_res.desc[i + 1].num = rm_res->desc[j].num; + } + } + + ret = ti_sci_inta_msi_domain_alloc_irqs(ud->dev, &irq_res); + kfree(irq_res.desc); + if (ret) { + dev_err(ud->dev, "Failed to allocate MSI interrupts\n"); + return ret; + } + + return 0; +} + +static int setup_resources(struct udma_dev *ud) +{ + struct device *dev = ud->dev; + int ch_count, ret; + + switch (ud->match_data->type) { + case DMA_TYPE_UDMA: + ret = udma_setup_resources(ud); + break; + case DMA_TYPE_BCDMA: + ret = bcdma_setup_resources(ud); + break; + default: + return -EINVAL; + } + + if (ret) + return ret; + + ch_count = ud->bchan_cnt + ud->tchan_cnt + ud->rchan_cnt; + if (ud->bchan_cnt) + ch_count -= bitmap_weight(ud->bchan_map, ud->bchan_cnt); ch_count -= bitmap_weight(ud->tchan_map, ud->tchan_cnt); ch_count -= bitmap_weight(ud->rchan_map, ud->rchan_cnt); if (!ch_count) @@ -3342,12 +4342,32 @@ static int udma_setup_resources(struct udma_dev *ud) if (!ud->channels) return -ENOMEM; - dev_info(dev, "Channels: %d (tchan: %u, rchan: %u, gp-rflow: %u)\n", - ch_count, - ud->tchan_cnt - bitmap_weight(ud->tchan_map, ud->tchan_cnt), - ud->rchan_cnt - bitmap_weight(ud->rchan_map, ud->rchan_cnt), - ud->rflow_cnt - bitmap_weight(ud->rflow_gp_map, - ud->rflow_cnt)); + switch (ud->match_data->type) { + case DMA_TYPE_UDMA: + dev_info(dev, + "Channels: %d (tchan: %u, rchan: %u, gp-rflow: %u)\n", + ch_count, + ud->tchan_cnt - bitmap_weight(ud->tchan_map, + ud->tchan_cnt), + ud->rchan_cnt - bitmap_weight(ud->rchan_map, + ud->rchan_cnt), + ud->rflow_cnt - bitmap_weight(ud->rflow_gp_map, + ud->rflow_cnt)); + break; + case DMA_TYPE_BCDMA: + dev_info(dev, + "Channels: %d (bchan: %u, tchan: %u, rchan: %u)\n", + ch_count, + ud->bchan_cnt - bitmap_weight(ud->bchan_map, + ud->bchan_cnt), + ud->tchan_cnt - bitmap_weight(ud->tchan_map, + ud->tchan_cnt), + ud->rchan_cnt - bitmap_weight(ud->rchan_map, + ud->rchan_cnt)); + break; + default: + break; + } return ch_count; } @@ -3456,10 +4476,19 @@ static void udma_dbg_summary_show_chan(struct seq_file *s, seq_printf(s, " %-13s| %s", dma_chan_name(chan), chan->dbg_client_name ?: "in-use"); - seq_printf(s, " (%s, ", dmaengine_get_direction_text(uc->config.dir)); + if (ucc->tr_trigger_type) + seq_puts(s, " (triggered, "); + else + seq_printf(s, " (%s, ", + dmaengine_get_direction_text(uc->config.dir)); switch (uc->config.dir) { case DMA_MEM_TO_MEM: + if (uc->ud->match_data->type == DMA_TYPE_BCDMA) { + seq_printf(s, "bchan%d)\n", uc->bchan->id); + return; + } + seq_printf(s, "chan%d pair [0x%04x -> 0x%04x], ", uc->tchan->id, ucc->src_thread, ucc->dst_thread); break; @@ -3531,6 +4560,23 @@ static int udma_probe(struct platform_device *pdev) if (!ud) return -ENOMEM; + match = of_match_node(udma_of_match, dev->of_node); + if (!match) { + match = of_match_node(bcdma_of_match, dev->of_node); + if (!match) { + dev_err(dev, "No compatible match found\n"); + return -ENODEV; + } + } + ud->match_data = match->data; + + soc = soc_device_match(k3_soc_devices); + if (!soc) { + dev_err(dev, "No compatible SoC found\n"); + return -ENODEV; + } + ud->soc_data = soc->data; + ret = udma_get_mmrs(pdev, ud); if (ret) return ret; @@ -3554,16 +4600,38 @@ static int udma_probe(struct platform_device *pdev) return ret; } - ret = of_property_read_u32(dev->of_node, "ti,udma-atype", &ud->atype); - if (!ret && ud->atype > 2) { - dev_err(dev, "Invalid atype: %u\n", ud->atype); - return -EINVAL; + if (ud->match_data->type == DMA_TYPE_UDMA) { + ret = of_property_read_u32(dev->of_node, "ti,udma-atype", + &ud->atype); + if (!ret && ud->atype > 2) { + dev_err(dev, "Invalid atype: %u\n", ud->atype); + return -EINVAL; + } + } else { + ret = of_property_read_u32(dev->of_node, "ti,asel", + &ud->asel); + if (!ret && ud->asel > 15) { + dev_err(dev, "Invalid asel: %u\n", ud->asel); + return -EINVAL; + } } ud->tisci_rm.tisci_udmap_ops = &ud->tisci_rm.tisci->ops.rm_udmap_ops; ud->tisci_rm.tisci_psil_ops = &ud->tisci_rm.tisci->ops.rm_psil_ops; - ud->ringacc = of_k3_ringacc_get_by_phandle(dev->of_node, "ti,ringacc"); + if (ud->match_data->type == DMA_TYPE_UDMA) { + ud->ringacc = of_k3_ringacc_get_by_phandle(dev->of_node, "ti,ringacc"); + } else { + struct k3_ringacc_init_data ring_init_data; + + ring_init_data.tisci = ud->tisci_rm.tisci; + ring_init_data.tisci_dev_id = ud->tisci_rm.tisci_dev_id; + ring_init_data.num_rings = ud->bchan_cnt + ud->tchan_cnt + + ud->rchan_cnt; + + ud->ringacc = k3_ringacc_dmarings_init(pdev, &ring_init_data); + } + if (IS_ERR(ud->ringacc)) return PTR_ERR(ud->ringacc); @@ -3574,24 +4642,9 @@ static int udma_probe(struct platform_device *pdev) return -EPROBE_DEFER; } - match = of_match_node(udma_of_match, dev->of_node); - if (!match) { - dev_err(dev, "No compatible match found\n"); - return -ENODEV; - } - ud->match_data = match->data; - - soc = soc_device_match(k3_soc_devices); - if (!soc) { - dev_err(dev, "No compatible SoC found\n"); - return -ENODEV; - } - ud->soc_data = soc->data; - dma_cap_set(DMA_SLAVE, ud->ddev.cap_mask); dma_cap_set(DMA_CYCLIC, ud->ddev.cap_mask); - ud->ddev.device_alloc_chan_resources = udma_alloc_chan_resources; ud->ddev.device_config = udma_slave_config; ud->ddev.device_prep_slave_sg = udma_prep_slave_sg; ud->ddev.device_prep_dma_cyclic = udma_prep_dma_cyclic; @@ -3605,7 +4658,21 @@ static int udma_probe(struct platform_device *pdev) ud->ddev.dbg_summary_show = udma_dbg_summary_show; #endif + switch (ud->match_data->type) { + case DMA_TYPE_UDMA: + ud->ddev.device_alloc_chan_resources = + udma_alloc_chan_resources; + break; + case DMA_TYPE_BCDMA: + ud->ddev.device_alloc_chan_resources = + bcdma_alloc_chan_resources; + ud->ddev.device_router_config = bcdma_router_config; + break; + default: + return -EINVAL; + } ud->ddev.device_free_chan_resources = udma_free_chan_resources; + ud->ddev.src_addr_widths = TI_UDMAC_BUSWIDTHS; ud->ddev.dst_addr_widths = TI_UDMAC_BUSWIDTHS; ud->ddev.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV); @@ -3613,7 +4680,8 @@ static int udma_probe(struct platform_device *pdev) ud->ddev.copy_align = DMAENGINE_ALIGN_8_BYTES; ud->ddev.desc_metadata_modes = DESC_METADATA_CLIENT | DESC_METADATA_ENGINE; - if (ud->match_data->enable_memcpy_support) { + if (ud->match_data->enable_memcpy_support && + !(ud->match_data->type == DMA_TYPE_BCDMA && ud->bchan_cnt == 0)) { dma_cap_set(DMA_MEMCPY, ud->ddev.cap_mask); ud->ddev.device_prep_dma_memcpy = udma_prep_dma_memcpy; ud->ddev.directions |= BIT(DMA_MEM_TO_MEM); @@ -3626,7 +4694,7 @@ static int udma_probe(struct platform_device *pdev) INIT_LIST_HEAD(&ud->ddev.channels); INIT_LIST_HEAD(&ud->desc_to_purge); - ch_count = udma_setup_resources(ud); + ch_count = setup_resources(ud); if (ch_count <= 0) return ch_count; @@ -3641,6 +4709,13 @@ static int udma_probe(struct platform_device *pdev) if (ret) return ret; + for (i = 0; i < ud->bchan_cnt; i++) { + struct udma_bchan *bchan = &ud->bchans[i]; + + bchan->id = i; + bchan->reg_rt = ud->mmrs[MMR_BCHANRT] + i * 0x1000; + } + for (i = 0; i < ud->tchan_cnt; i++) { struct udma_tchan *tchan = &ud->tchans[i]; @@ -3667,6 +4742,7 @@ static int udma_probe(struct platform_device *pdev) uc->ud = ud; uc->vc.desc_free = udma_desc_free; uc->id = i; + uc->bchan = NULL; uc->tchan = NULL; uc->rchan = NULL; uc->config.remote_thread_id = -1; @@ -3708,5 +4784,15 @@ static struct platform_driver udma_driver = { }; builtin_platform_driver(udma_driver); +static struct platform_driver bcdma_driver = { + .driver = { + .name = "ti-bcdma", + .of_match_table = bcdma_of_match, + .suppress_bind_attrs = true, + }, + .probe = udma_probe, +}; +builtin_platform_driver(bcdma_driver); + /* Private interfaces to UDMA */ #include "k3-udma-private.c" diff --git a/drivers/dma/ti/k3-udma.h b/drivers/dma/ti/k3-udma.h index b4334b1b7b14..c97ed5271fd9 100644 --- a/drivers/dma/ti/k3-udma.h +++ b/drivers/dma/ti/k3-udma.h @@ -18,7 +18,7 @@ #define UDMA_RX_FLOW_ID_FW_OES_REG 0x80 #define UDMA_RX_FLOW_ID_FW_STATUS_REG 0x88 -/* TCHANRT/RCHANRT registers */ +/* BCHANRT/TCHANRT/RCHANRT registers */ #define UDMA_CHAN_RT_CTL_REG 0x0 #define UDMA_CHAN_RT_SWTRIG_REG 0x8 #define UDMA_CHAN_RT_STDATA_REG 0x80 @@ -45,6 +45,10 @@ #define UDMA_CAP3_HCHAN_CNT(val) (((val) >> 14) & 0x1ff) #define UDMA_CAP3_UCHAN_CNT(val) (((val) >> 23) & 0x1ff) +#define BCDMA_CAP2_BCHAN_CNT(val) ((val) & 0x1ff) +#define BCDMA_CAP2_TCHAN_CNT(val) (((val) >> 9) & 0x1ff) +#define BCDMA_CAP2_RCHAN_CNT(val) (((val) >> 18) & 0x1ff) + /* UDMA_CHAN_RT_CTL_REG */ #define UDMA_CHAN_RT_CTL_EN BIT(31) #define UDMA_CHAN_RT_CTL_TDOWN BIT(30) @@ -82,13 +86,17 @@ */ #define PDMA_STATIC_TR_Z(x, mask) ((x) & (mask)) +/* Address Space Select */ +#define K3_ADDRESS_ASEL_SHIFT 48 + struct udma_dev; struct udma_tchan; struct udma_rchan; struct udma_rflow; enum udma_rm_range { - RM_RANGE_TCHAN = 0, + RM_RANGE_BCHAN = 0, + RM_RANGE_TCHAN, RM_RANGE_RCHAN, RM_RANGE_RFLOW, RM_RANGE_LAST, -- cgit v1.2.3-59-g8ed1b From 8844898028d4127b0071ecb5a72e1894bd8b21d6 Mon Sep 17 00:00:00 2001 From: Peter Ujfalusi Date: Tue, 8 Dec 2020 11:04:38 +0200 Subject: dmaengine: ti: k3-udma: Add support for BCDMA channel TPL handling Unlike UDMAP the BCDMA defines the channel TPL levels per channel type. In UDMAP the number of high and ultra-high channels applies to both tchan and rchan. BCDMA defines the TPL per channel types: bchan, tchan and rchan can have different number of high and ultra-high channels. In order to support BCDMA channel TPL we need to move the tpl information as per channel type property for the DMAs. Signed-off-by: Peter Ujfalusi Link: https://lore.kernel.org/r/20201208090440.31792-19-peter.ujfalusi@ti.com Signed-off-by: Vinod Koul --- drivers/dma/ti/k3-udma.c | 117 ++++++++++++++++++++++++++++++++--------------- drivers/dma/ti/k3-udma.h | 6 +++ 2 files changed, 85 insertions(+), 38 deletions(-) diff --git a/drivers/dma/ti/k3-udma.c b/drivers/dma/ti/k3-udma.c index f327d436f8ad..0dc3430fea40 100644 --- a/drivers/dma/ti/k3-udma.c +++ b/drivers/dma/ti/k3-udma.c @@ -146,6 +146,11 @@ struct udma_rx_flush { dma_addr_t buffer_paddr; }; +struct udma_tpl { + u8 levels; + u32 start_idx[3]; +}; + struct udma_dev { struct dma_device ddev; struct device *dev; @@ -153,8 +158,9 @@ struct udma_dev { const struct udma_match_data *match_data; const struct udma_soc_data *soc_data; - u8 tpl_levels; - u32 tpl_start_idx[3]; + struct udma_tpl bchan_tpl; + struct udma_tpl tchan_tpl; + struct udma_tpl rchan_tpl; size_t desc_align; /* alignment to use for descriptors */ @@ -1276,10 +1282,10 @@ static struct udma_##res *__udma_reserve_##res(struct udma_dev *ud, \ } else { \ int start; \ \ - if (tpl >= ud->tpl_levels) \ - tpl = ud->tpl_levels - 1; \ + if (tpl >= ud->res##_tpl.levels) \ + tpl = ud->res##_tpl.levels - 1; \ \ - start = ud->tpl_start_idx[tpl]; \ + start = ud->res##_tpl.start_idx[tpl]; \ \ id = find_next_zero_bit(ud->res##_map, ud->res##_cnt, \ start); \ @@ -1292,29 +1298,14 @@ static struct udma_##res *__udma_reserve_##res(struct udma_dev *ud, \ return &ud->res##s[id]; \ } +UDMA_RESERVE_RESOURCE(bchan); UDMA_RESERVE_RESOURCE(tchan); UDMA_RESERVE_RESOURCE(rchan); -static struct udma_bchan *__bcdma_reserve_bchan(struct udma_dev *ud, int id) -{ - if (id >= 0) { - if (test_bit(id, ud->bchan_map)) { - dev_err(ud->dev, "bchan%d is in use\n", id); - return ERR_PTR(-ENOENT); - } - } else { - id = find_next_zero_bit(ud->bchan_map, ud->bchan_cnt, 0); - if (id == ud->bchan_cnt) - return ERR_PTR(-ENOENT); - } - - set_bit(id, ud->bchan_map); - return &ud->bchans[id]; -} - static int bcdma_get_bchan(struct udma_chan *uc) { struct udma_dev *ud = uc->ud; + enum udma_tp_level tpl; if (uc->bchan) { dev_dbg(ud->dev, "chan%d: already have bchan%d allocated\n", @@ -1322,7 +1313,16 @@ static int bcdma_get_bchan(struct udma_chan *uc) return 0; } - uc->bchan = __bcdma_reserve_bchan(ud, -1); + /* + * Use normal channels for peripherals, and highest TPL channel for + * mem2mem + */ + if (uc->config.tr_trigger_type) + tpl = 0; + else + tpl = ud->bchan_tpl.levels - 1; + + uc->bchan = __udma_reserve_bchan(ud, tpl, -1); if (IS_ERR(uc->bchan)) return PTR_ERR(uc->bchan); @@ -1384,8 +1384,11 @@ static int udma_get_chan_pair(struct udma_chan *uc) /* Can be optimized, but let's have it like this for now */ end = min(ud->tchan_cnt, ud->rchan_cnt); - /* Try to use the highest TPL channel pair for MEM_TO_MEM channels */ - chan_id = ud->tpl_start_idx[ud->tpl_levels - 1]; + /* + * Try to use the highest TPL channel pair for MEM_TO_MEM channels + * Note: in UDMAP the channel TPL is symmetric between tchan and rchan + */ + chan_id = ud->tchan_tpl.start_idx[ud->tchan_tpl.levels - 1]; for (; chan_id < end; chan_id++) { if (!test_bit(chan_id, ud->tchan_map) && !test_bit(chan_id, ud->rchan_map)) @@ -4036,23 +4039,27 @@ static int udma_setup_resources(struct udma_dev *ud) cap3 = udma_read(ud->mmrs[MMR_GCFG], 0x2c); if (of_device_is_compatible(dev->of_node, "ti,am654-navss-main-udmap")) { - ud->tpl_levels = 2; - ud->tpl_start_idx[0] = 8; + ud->tchan_tpl.levels = 2; + ud->tchan_tpl.start_idx[0] = 8; } else if (of_device_is_compatible(dev->of_node, "ti,am654-navss-mcu-udmap")) { - ud->tpl_levels = 2; - ud->tpl_start_idx[0] = 2; + ud->tchan_tpl.levels = 2; + ud->tchan_tpl.start_idx[0] = 2; } else if (UDMA_CAP3_UCHAN_CNT(cap3)) { - ud->tpl_levels = 3; - ud->tpl_start_idx[1] = UDMA_CAP3_UCHAN_CNT(cap3); - ud->tpl_start_idx[0] = UDMA_CAP3_HCHAN_CNT(cap3); + ud->tchan_tpl.levels = 3; + ud->tchan_tpl.start_idx[1] = UDMA_CAP3_UCHAN_CNT(cap3); + ud->tchan_tpl.start_idx[0] = UDMA_CAP3_HCHAN_CNT(cap3); } else if (UDMA_CAP3_HCHAN_CNT(cap3)) { - ud->tpl_levels = 2; - ud->tpl_start_idx[0] = UDMA_CAP3_HCHAN_CNT(cap3); + ud->tchan_tpl.levels = 2; + ud->tchan_tpl.start_idx[0] = UDMA_CAP3_HCHAN_CNT(cap3); } else { - ud->tpl_levels = 1; + ud->tchan_tpl.levels = 1; } + ud->rchan_tpl.levels = ud->tchan_tpl.levels; + ud->rchan_tpl.start_idx[0] = ud->tchan_tpl.start_idx[0]; + ud->rchan_tpl.start_idx[1] = ud->tchan_tpl.start_idx[1]; + ud->tchan_map = devm_kmalloc_array(dev, BITS_TO_LONGS(ud->tchan_cnt), sizeof(unsigned long), GFP_KERNEL); ud->tchans = devm_kcalloc(dev, ud->tchan_cnt, sizeof(*ud->tchans), @@ -4174,6 +4181,43 @@ static int bcdma_setup_resources(struct udma_dev *ud) struct ti_sci_resource *rm_res, irq_res; struct udma_tisci_rm *tisci_rm = &ud->tisci_rm; const struct udma_oes_offsets *oes = &ud->soc_data->oes; + u32 cap; + + /* Set up the throughput level start indexes */ + cap = udma_read(ud->mmrs[MMR_GCFG], 0x2c); + if (BCDMA_CAP3_UBCHAN_CNT(cap)) { + ud->bchan_tpl.levels = 3; + ud->bchan_tpl.start_idx[1] = BCDMA_CAP3_UBCHAN_CNT(cap); + ud->bchan_tpl.start_idx[0] = BCDMA_CAP3_HBCHAN_CNT(cap); + } else if (BCDMA_CAP3_HBCHAN_CNT(cap)) { + ud->bchan_tpl.levels = 2; + ud->bchan_tpl.start_idx[0] = BCDMA_CAP3_HBCHAN_CNT(cap); + } else { + ud->bchan_tpl.levels = 1; + } + + cap = udma_read(ud->mmrs[MMR_GCFG], 0x30); + if (BCDMA_CAP4_URCHAN_CNT(cap)) { + ud->rchan_tpl.levels = 3; + ud->rchan_tpl.start_idx[1] = BCDMA_CAP4_URCHAN_CNT(cap); + ud->rchan_tpl.start_idx[0] = BCDMA_CAP4_HRCHAN_CNT(cap); + } else if (BCDMA_CAP4_HRCHAN_CNT(cap)) { + ud->rchan_tpl.levels = 2; + ud->rchan_tpl.start_idx[0] = BCDMA_CAP4_HRCHAN_CNT(cap); + } else { + ud->rchan_tpl.levels = 1; + } + + if (BCDMA_CAP4_UTCHAN_CNT(cap)) { + ud->tchan_tpl.levels = 3; + ud->tchan_tpl.start_idx[1] = BCDMA_CAP4_UTCHAN_CNT(cap); + ud->tchan_tpl.start_idx[0] = BCDMA_CAP4_HTCHAN_CNT(cap); + } else if (BCDMA_CAP4_HTCHAN_CNT(cap)) { + ud->tchan_tpl.levels = 2; + ud->tchan_tpl.start_idx[0] = BCDMA_CAP4_HTCHAN_CNT(cap); + } else { + ud->tchan_tpl.levels = 1; + } ud->bchan_map = devm_kmalloc_array(dev, BITS_TO_LONGS(ud->bchan_cnt), sizeof(unsigned long), GFP_KERNEL); @@ -4199,9 +4243,6 @@ static int bcdma_setup_resources(struct udma_dev *ud) !ud->rflows) return -ENOMEM; - /* TPL is not yet supported for BCDMA */ - ud->tpl_levels = 1; - /* Get resource ranges from tisci */ for (i = 0; i < RM_RANGE_LAST; i++) { if (i == RM_RANGE_RFLOW) diff --git a/drivers/dma/ti/k3-udma.h b/drivers/dma/ti/k3-udma.h index c97ed5271fd9..3ec38b383b76 100644 --- a/drivers/dma/ti/k3-udma.h +++ b/drivers/dma/ti/k3-udma.h @@ -48,6 +48,12 @@ #define BCDMA_CAP2_BCHAN_CNT(val) ((val) & 0x1ff) #define BCDMA_CAP2_TCHAN_CNT(val) (((val) >> 9) & 0x1ff) #define BCDMA_CAP2_RCHAN_CNT(val) (((val) >> 18) & 0x1ff) +#define BCDMA_CAP3_HBCHAN_CNT(val) (((val) >> 14) & 0x1ff) +#define BCDMA_CAP3_UBCHAN_CNT(val) (((val) >> 23) & 0x1ff) +#define BCDMA_CAP4_HRCHAN_CNT(val) ((val) & 0xff) +#define BCDMA_CAP4_URCHAN_CNT(val) (((val) >> 8) & 0xff) +#define BCDMA_CAP4_HTCHAN_CNT(val) (((val) >> 16) & 0xff) +#define BCDMA_CAP4_UTCHAN_CNT(val) (((val) >> 24) & 0xff) /* UDMA_CHAN_RT_CTL_REG */ #define UDMA_CHAN_RT_CTL_EN BIT(31) -- cgit v1.2.3-59-g8ed1b From d2abc982333c02f9e1ff1c6b3782174f5b7662d7 Mon Sep 17 00:00:00 2001 From: Peter Ujfalusi Date: Tue, 8 Dec 2020 11:04:39 +0200 Subject: dmaengine: ti: k3-udma: Initial support for K3 PKTDMA One of the DMAs introduced with AM64 is the Packet DMA (PKTDMA). It serves similar purpose as K3 UDMAP channels in packet mode, but with notable differences, like tflow support and channels being allocated to service specific peripherals. The rings for the PKTDMA is integrated within the DMA itself instead of using rings from the general purpose ringacc. PKTDMA can be used to service PSI-L peripherals, similarly to K3 UDMA channels. Most of the driver code can be reused for PKTDMA tchan/rchan support but new setup and allocation functions are needed to handle the differences between the DMAs. Signed-off-by: Peter Ujfalusi Link: https://lore.kernel.org/r/20201208090440.31792-20-peter.ujfalusi@ti.com Signed-off-by: Vinod Koul --- drivers/dma/ti/k3-udma-private.c | 9 + drivers/dma/ti/k3-udma.c | 545 +++++++++++++++++++++++++++++++++++++-- drivers/dma/ti/k3-udma.h | 4 + 3 files changed, 533 insertions(+), 25 deletions(-) diff --git a/drivers/dma/ti/k3-udma-private.c b/drivers/dma/ti/k3-udma-private.c index 346a4dd9640a..5436b19d656e 100644 --- a/drivers/dma/ti/k3-udma-private.c +++ b/drivers/dma/ti/k3-udma-private.c @@ -88,6 +88,9 @@ EXPORT_SYMBOL(xudma_free_gp_rflow_range); bool xudma_rflow_is_gp(struct udma_dev *ud, int id) { + if (!ud->rflow_gp_map) + return false; + return !test_bit(id, ud->rflow_gp_map); } EXPORT_SYMBOL(xudma_rflow_is_gp); @@ -119,6 +122,12 @@ void xudma_rflow_put(struct udma_dev *ud, struct udma_rflow *p) } EXPORT_SYMBOL(xudma_rflow_put); +int xudma_get_rflow_ring_offset(struct udma_dev *ud) +{ + return ud->tflow_cnt; +} +EXPORT_SYMBOL(xudma_get_rflow_ring_offset); + #define XUDMA_GET_RESOURCE_ID(res) \ int xudma_##res##_get_id(struct udma_##res *p) \ { \ diff --git a/drivers/dma/ti/k3-udma.c b/drivers/dma/ti/k3-udma.c index 0dc3430fea40..87157cbae1b8 100644 --- a/drivers/dma/ti/k3-udma.c +++ b/drivers/dma/ti/k3-udma.c @@ -59,6 +59,7 @@ struct udma_chan; enum k3_dma_type { DMA_TYPE_UDMA = 0, DMA_TYPE_BCDMA, + DMA_TYPE_PKTDMA, }; enum udma_mmr { @@ -82,6 +83,8 @@ struct udma_tchan { int id; struct k3_ring *t_ring; /* Transmit ring */ struct k3_ring *tc_ring; /* Transmit Completion ring */ + int tflow_id; /* applicable only for PKTDMA */ + }; #define udma_bchan udma_tchan @@ -109,6 +112,10 @@ struct udma_oes_offsets { u32 bcdma_tchan_ring; u32 bcdma_rchan_data; u32 bcdma_rchan_ring; + + /* PKTDMA Output Event Offsets */ + u32 pktdma_tchan_flow; + u32 pktdma_rchan_flow; }; #define UDMA_FLAG_PDMA_ACC32 BIT(0) @@ -179,12 +186,14 @@ struct udma_dev { int echan_cnt; int rchan_cnt; int rflow_cnt; + int tflow_cnt; unsigned long *bchan_map; unsigned long *tchan_map; unsigned long *rchan_map; unsigned long *rflow_gp_map; unsigned long *rflow_gp_map_allocated; unsigned long *rflow_in_use; + unsigned long *tflow_map; struct udma_bchan *bchans; struct udma_tchan *tchans; @@ -249,6 +258,11 @@ struct udma_chan_config { u32 tr_trigger_type; + /* PKDMA mapped channel */ + int mapped_channel_id; + /* PKTDMA default tflow or rflow for mapped channel */ + int default_flow_id; + enum dma_transfer_direction dir; }; @@ -426,6 +440,8 @@ static void udma_reset_uchan(struct udma_chan *uc) { memset(&uc->config, 0, sizeof(uc->config)); uc->config.remote_thread_id = -1; + uc->config.mapped_channel_id = -1; + uc->config.default_flow_id = -1; uc->state = UDMA_CHAN_IS_IDLE; } @@ -815,10 +831,16 @@ static void udma_start_desc(struct udma_chan *uc) { struct udma_chan_config *ucc = &uc->config; - if (ucc->pkt_mode && (uc->cyclic || ucc->dir == DMA_DEV_TO_MEM)) { + if (uc->ud->match_data->type == DMA_TYPE_UDMA && ucc->pkt_mode && + (uc->cyclic || ucc->dir == DMA_DEV_TO_MEM)) { int i; - /* Push all descriptors to ring for packet mode cyclic or RX */ + /* + * UDMA only: Push all descriptors to ring for packet mode + * cyclic or RX + * PKTDMA supports pre-linked descriptor and cyclic is not + * supported + */ for (i = 0; i < uc->desc->sglen; i++) udma_push_to_ring(uc, i); } else { @@ -1248,10 +1270,12 @@ static struct udma_rflow *__udma_get_rflow(struct udma_dev *ud, int id) if (test_bit(id, ud->rflow_in_use)) return ERR_PTR(-ENOENT); - /* GP rflow has to be allocated first */ - if (!test_bit(id, ud->rflow_gp_map) && - !test_bit(id, ud->rflow_gp_map_allocated)) - return ERR_PTR(-EINVAL); + if (ud->rflow_gp_map) { + /* GP rflow has to be allocated first */ + if (!test_bit(id, ud->rflow_gp_map) && + !test_bit(id, ud->rflow_gp_map_allocated)) + return ERR_PTR(-EINVAL); + } dev_dbg(ud->dev, "get rflow%d\n", id); set_bit(id, ud->rflow_in_use); @@ -1341,9 +1365,39 @@ static int udma_get_tchan(struct udma_chan *uc) return 0; } - uc->tchan = __udma_reserve_tchan(ud, uc->config.channel_tpl, -1); + /* + * mapped_channel_id is -1 for UDMA, BCDMA and PKTDMA unmapped channels. + * For PKTDMA mapped channels it is configured to a channel which must + * be used to service the peripheral. + */ + uc->tchan = __udma_reserve_tchan(ud, uc->config.channel_tpl, + uc->config.mapped_channel_id); + if (IS_ERR(uc->tchan)) + return PTR_ERR(uc->tchan); + + if (ud->tflow_cnt) { + int tflow_id; + + /* Only PKTDMA have support for tx flows */ + if (uc->config.default_flow_id >= 0) + tflow_id = uc->config.default_flow_id; + else + tflow_id = uc->tchan->id; + + if (test_bit(tflow_id, ud->tflow_map)) { + dev_err(ud->dev, "tflow%d is in use\n", tflow_id); + clear_bit(uc->tchan->id, ud->tchan_map); + uc->tchan = NULL; + return -ENOENT; + } + + uc->tchan->tflow_id = tflow_id; + set_bit(tflow_id, ud->tflow_map); + } else { + uc->tchan->tflow_id = -1; + } - return PTR_ERR_OR_ZERO(uc->tchan); + return 0; } static int udma_get_rchan(struct udma_chan *uc) @@ -1356,7 +1410,13 @@ static int udma_get_rchan(struct udma_chan *uc) return 0; } - uc->rchan = __udma_reserve_rchan(ud, uc->config.channel_tpl, -1); + /* + * mapped_channel_id is -1 for UDMA, BCDMA and PKTDMA unmapped channels. + * For PKTDMA mapped channels it is configured to a channel which must + * be used to service the peripheral. + */ + uc->rchan = __udma_reserve_rchan(ud, uc->config.channel_tpl, + uc->config.mapped_channel_id); return PTR_ERR_OR_ZERO(uc->rchan); } @@ -1403,6 +1463,9 @@ static int udma_get_chan_pair(struct udma_chan *uc) uc->tchan = &ud->tchans[chan_id]; uc->rchan = &ud->rchans[chan_id]; + /* UDMA does not use tx flows */ + uc->tchan->tflow_id = -1; + return 0; } @@ -1459,6 +1522,10 @@ static void udma_put_tchan(struct udma_chan *uc) dev_dbg(ud->dev, "chan%d: put tchan%d\n", uc->id, uc->tchan->id); clear_bit(uc->tchan->id, ud->tchan_map); + + if (uc->tchan->tflow_id >= 0) + clear_bit(uc->tchan->tflow_id, ud->tflow_map); + uc->tchan = NULL; } } @@ -1559,7 +1626,10 @@ static int udma_alloc_tx_resources(struct udma_chan *uc) return ret; tchan = uc->tchan; - ring_idx = ud->bchan_cnt + tchan->id; + if (tchan->tflow_id >= 0) + ring_idx = tchan->tflow_id; + else + ring_idx = ud->bchan_cnt + tchan->id; ret = k3_ringacc_request_rings_pair(ud->ringacc, ring_idx, -1, &tchan->t_ring, @@ -1636,15 +1706,23 @@ static int udma_alloc_rx_resources(struct udma_chan *uc) if (uc->config.dir == DMA_MEM_TO_MEM) return 0; - ret = udma_get_rflow(uc, uc->rchan->id); + if (uc->config.default_flow_id >= 0) + ret = udma_get_rflow(uc, uc->config.default_flow_id); + else + ret = udma_get_rflow(uc, uc->rchan->id); + if (ret) { ret = -EBUSY; goto err_rflow; } rflow = uc->rflow; - fd_ring_id = ud->bchan_cnt + ud->tchan_cnt + ud->echan_cnt + - uc->rchan->id; + if (ud->tflow_cnt) + fd_ring_id = ud->tflow_cnt + rflow->id; + else + fd_ring_id = ud->bchan_cnt + ud->tchan_cnt + ud->echan_cnt + + uc->rchan->id; + ret = k3_ringacc_request_rings_pair(ud->ringacc, fd_ring_id, -1, &rflow->fd_ring, &rflow->r_ring); if (ret) { @@ -1860,6 +1938,8 @@ static int bcdma_tisci_tx_channel_config(struct udma_chan *uc) return ret; } +#define pktdma_tisci_tx_channel_config bcdma_tisci_tx_channel_config + static int udma_tisci_rx_channel_config(struct udma_chan *uc) { struct udma_dev *ud = uc->ud; @@ -1961,6 +2041,52 @@ static int bcdma_tisci_rx_channel_config(struct udma_chan *uc) return ret; } +static int pktdma_tisci_rx_channel_config(struct udma_chan *uc) +{ + struct udma_dev *ud = uc->ud; + struct udma_tisci_rm *tisci_rm = &ud->tisci_rm; + const struct ti_sci_rm_udmap_ops *tisci_ops = tisci_rm->tisci_udmap_ops; + struct ti_sci_msg_rm_udmap_rx_ch_cfg req_rx = { 0 }; + struct ti_sci_msg_rm_udmap_flow_cfg flow_req = { 0 }; + int ret = 0; + + req_rx.valid_params = TISCI_BCDMA_RCHAN_VALID_PARAMS; + req_rx.nav_id = tisci_rm->tisci_dev_id; + req_rx.index = uc->rchan->id; + + ret = tisci_ops->rx_ch_cfg(tisci_rm->tisci, &req_rx); + if (ret) { + dev_err(ud->dev, "rchan%d cfg failed %d\n", uc->rchan->id, ret); + return ret; + } + + flow_req.valid_params = + TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_EINFO_PRESENT_VALID | + TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_PSINFO_PRESENT_VALID | + TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_ERROR_HANDLING_VALID; + + flow_req.nav_id = tisci_rm->tisci_dev_id; + flow_req.flow_index = uc->rflow->id; + + if (uc->config.needs_epib) + flow_req.rx_einfo_present = 1; + else + flow_req.rx_einfo_present = 0; + if (uc->config.psd_size) + flow_req.rx_psinfo_present = 1; + else + flow_req.rx_psinfo_present = 0; + flow_req.rx_error_handling = 1; + + ret = tisci_ops->rx_flow_cfg(tisci_rm->tisci, &flow_req); + + if (ret) + dev_err(ud->dev, "flow%d config failed: %d\n", uc->rflow->id, + ret); + + return ret; +} + static int udma_alloc_chan_resources(struct dma_chan *chan) { struct udma_chan *uc = to_udma_chan(chan); @@ -2379,6 +2505,157 @@ static int bcdma_router_config(struct dma_chan *chan) return router_data->set_event(router_data->priv, trigger_event); } +static int pktdma_alloc_chan_resources(struct dma_chan *chan) +{ + struct udma_chan *uc = to_udma_chan(chan); + struct udma_dev *ud = to_udma_dev(chan->device); + const struct udma_oes_offsets *oes = &ud->soc_data->oes; + u32 irq_ring_idx; + int ret; + + /* + * Make sure that the completion is in a known state: + * No teardown, the channel is idle + */ + reinit_completion(&uc->teardown_completed); + complete_all(&uc->teardown_completed); + uc->state = UDMA_CHAN_IS_IDLE; + + switch (uc->config.dir) { + case DMA_MEM_TO_DEV: + /* Slave transfer synchronized - mem to dev (TX) trasnfer */ + dev_dbg(uc->ud->dev, "%s: chan%d as MEM-to-DEV\n", __func__, + uc->id); + + ret = udma_alloc_tx_resources(uc); + if (ret) { + uc->config.remote_thread_id = -1; + return ret; + } + + uc->config.src_thread = ud->psil_base + uc->tchan->id; + uc->config.dst_thread = uc->config.remote_thread_id; + uc->config.dst_thread |= K3_PSIL_DST_THREAD_ID_OFFSET; + + irq_ring_idx = uc->tchan->tflow_id + oes->pktdma_tchan_flow; + + ret = pktdma_tisci_tx_channel_config(uc); + break; + case DMA_DEV_TO_MEM: + /* Slave transfer synchronized - dev to mem (RX) trasnfer */ + dev_dbg(uc->ud->dev, "%s: chan%d as DEV-to-MEM\n", __func__, + uc->id); + + ret = udma_alloc_rx_resources(uc); + if (ret) { + uc->config.remote_thread_id = -1; + return ret; + } + + uc->config.src_thread = uc->config.remote_thread_id; + uc->config.dst_thread = (ud->psil_base + uc->rchan->id) | + K3_PSIL_DST_THREAD_ID_OFFSET; + + irq_ring_idx = uc->rflow->id + oes->pktdma_rchan_flow; + + ret = pktdma_tisci_rx_channel_config(uc); + break; + default: + /* Can not happen */ + dev_err(uc->ud->dev, "%s: chan%d invalid direction (%u)\n", + __func__, uc->id, uc->config.dir); + return -EINVAL; + } + + /* check if the channel configuration was successful */ + if (ret) + goto err_res_free; + + if (udma_is_chan_running(uc)) { + dev_warn(ud->dev, "chan%d: is running!\n", uc->id); + udma_reset_chan(uc, false); + if (udma_is_chan_running(uc)) { + dev_err(ud->dev, "chan%d: won't stop!\n", uc->id); + ret = -EBUSY; + goto err_res_free; + } + } + + uc->dma_dev = dmaengine_get_dma_device(chan); + uc->hdesc_pool = dma_pool_create(uc->name, uc->dma_dev, + uc->config.hdesc_size, ud->desc_align, + 0); + if (!uc->hdesc_pool) { + dev_err(ud->ddev.dev, + "Descriptor pool allocation failed\n"); + uc->use_dma_pool = false; + ret = -ENOMEM; + goto err_res_free; + } + + uc->use_dma_pool = true; + + /* PSI-L pairing */ + ret = navss_psil_pair(ud, uc->config.src_thread, uc->config.dst_thread); + if (ret) { + dev_err(ud->dev, "PSI-L pairing failed: 0x%04x -> 0x%04x\n", + uc->config.src_thread, uc->config.dst_thread); + goto err_res_free; + } + + uc->psil_paired = true; + + uc->irq_num_ring = ti_sci_inta_msi_get_virq(ud->dev, irq_ring_idx); + if (uc->irq_num_ring <= 0) { + dev_err(ud->dev, "Failed to get ring irq (index: %u)\n", + irq_ring_idx); + ret = -EINVAL; + goto err_psi_free; + } + + ret = request_irq(uc->irq_num_ring, udma_ring_irq_handler, + IRQF_TRIGGER_HIGH, uc->name, uc); + if (ret) { + dev_err(ud->dev, "chan%d: ring irq request failed\n", uc->id); + goto err_irq_free; + } + + uc->irq_num_udma = 0; + + udma_reset_rings(uc); + + INIT_DELAYED_WORK_ONSTACK(&uc->tx_drain.work, + udma_check_tx_completion); + + if (uc->tchan) + dev_dbg(ud->dev, + "chan%d: tchan%d, tflow%d, Remote thread: 0x%04x\n", + uc->id, uc->tchan->id, uc->tchan->tflow_id, + uc->config.remote_thread_id); + else if (uc->rchan) + dev_dbg(ud->dev, + "chan%d: rchan%d, rflow%d, Remote thread: 0x%04x\n", + uc->id, uc->rchan->id, uc->rflow->id, + uc->config.remote_thread_id); + return 0; + +err_irq_free: + uc->irq_num_ring = 0; +err_psi_free: + navss_psil_unpair(ud, uc->config.src_thread, uc->config.dst_thread); + uc->psil_paired = false; +err_res_free: + udma_free_tx_resources(uc); + udma_free_rx_resources(uc); + + udma_reset_uchan(uc); + + dma_pool_destroy(uc->hdesc_pool); + uc->use_dma_pool = false; + + return ret; +} + static int udma_slave_config(struct dma_chan *chan, struct dma_slave_config *cfg) { @@ -2857,6 +3134,7 @@ udma_prep_slave_sg_pkt(struct udma_chan *uc, struct scatterlist *sgl, struct udma_desc *d; u32 ring_id; unsigned int i; + u64 asel; d = kzalloc(struct_size(d, hwdesc, sglen), GFP_NOWAIT); if (!d) @@ -2870,6 +3148,11 @@ udma_prep_slave_sg_pkt(struct udma_chan *uc, struct scatterlist *sgl, else ring_id = k3_ringacc_get_ring_id(uc->tchan->tc_ring); + if (uc->ud->match_data->type == DMA_TYPE_UDMA) + asel = 0; + else + asel = (u64)uc->config.asel << K3_ADDRESS_ASEL_SHIFT; + for_each_sg(sgl, sgent, sglen, i) { struct udma_hwdesc *hwdesc = &d->hwdesc[i]; dma_addr_t sg_addr = sg_dma_address(sgent); @@ -2904,14 +3187,16 @@ udma_prep_slave_sg_pkt(struct udma_chan *uc, struct scatterlist *sgl, } /* attach the sg buffer to the descriptor */ + sg_addr |= asel; cppi5_hdesc_attach_buf(desc, sg_addr, sg_len, sg_addr, sg_len); /* Attach link as host buffer descriptor */ if (h_desc) cppi5_hdesc_link_hbdesc(h_desc, - hwdesc->cppi5_desc_paddr); + hwdesc->cppi5_desc_paddr | asel); - if (dir == DMA_MEM_TO_DEV) + if (uc->ud->match_data->type == DMA_TYPE_PKTDMA || + dir == DMA_MEM_TO_DEV) h_desc = desc; } @@ -3190,6 +3475,9 @@ udma_prep_dma_cyclic_pkt(struct udma_chan *uc, dma_addr_t buf_addr, else ring_id = k3_ringacc_get_ring_id(uc->tchan->tc_ring); + if (uc->ud->match_data->type != DMA_TYPE_UDMA) + buf_addr |= (u64)uc->config.asel << K3_ADDRESS_ASEL_SHIFT; + for (i = 0; i < periods; i++) { struct udma_hwdesc *hwdesc = &d->hwdesc[i]; dma_addr_t period_addr = buf_addr + (period_len * i); @@ -3706,6 +3994,7 @@ static void udma_free_chan_resources(struct dma_chan *chan) static struct platform_driver udma_driver; static struct platform_driver bcdma_driver; +static struct platform_driver pktdma_driver; struct udma_filter_param { int remote_thread_id; @@ -3723,7 +4012,8 @@ static bool udma_dma_filter_fn(struct dma_chan *chan, void *param) struct udma_dev *ud; if (chan->device->dev->driver != &udma_driver.driver && - chan->device->dev->driver != &bcdma_driver.driver) + chan->device->dev->driver != &bcdma_driver.driver && + chan->device->dev->driver != &pktdma_driver.driver) return false; uc = to_udma_chan(chan); @@ -3785,6 +4075,15 @@ static bool udma_dma_filter_fn(struct dma_chan *chan, void *param) ucc->notdpkt = ep_config->notdpkt; ucc->ep_type = ep_config->ep_type; + if (ud->match_data->type == DMA_TYPE_PKTDMA && + ep_config->mapped_channel_id >= 0) { + ucc->mapped_channel_id = ep_config->mapped_channel_id; + ucc->default_flow_id = ep_config->default_flow_id; + } else { + ucc->mapped_channel_id = -1; + ucc->default_flow_id = -1; + } + if (ucc->ep_type != PSIL_EP_NATIVE) { const struct udma_match_data *match_data = ud->match_data; @@ -3901,6 +4200,14 @@ static struct udma_match_data am64_bcdma_data = { .statictr_z_mask = GENMASK(23, 0), }; +static struct udma_match_data am64_pktdma_data = { + .type = DMA_TYPE_PKTDMA, + .psil_base = 0x1000, + .enable_memcpy_support = false, /* PKTDMA does not support MEM_TO_MEM */ + .flags = UDMA_FLAG_PDMA_ACC32 | UDMA_FLAG_PDMA_BURST | UDMA_FLAG_TDTYPE, + .statictr_z_mask = GENMASK(23, 0), +}; + static const struct of_device_id udma_of_match[] = { { .compatible = "ti,am654-navss-main-udmap", @@ -3927,6 +4234,14 @@ static const struct of_device_id bcdma_of_match[] = { { /* Sentinel */ }, }; +static const struct of_device_id pktdma_of_match[] = { + { + .compatible = "ti,am64-dmss-pktdma", + .data = &am64_pktdma_data, + }, + { /* Sentinel */ }, +}; + static struct udma_soc_data am654_soc_data = { .oes = { .udma_rchan = 0x200, @@ -3953,6 +4268,8 @@ static struct udma_soc_data am64_soc_data = { .bcdma_tchan_ring = 0x2a00, .bcdma_rchan_data = 0x2e00, .bcdma_rchan_ring = 0x3000, + .pktdma_tchan_flow = 0x1200, + .pktdma_rchan_flow = 0x1600, }, .bcdma_trigger_event_offset = 0xc400, }; @@ -3967,7 +4284,7 @@ static const struct soc_device_attribute k3_soc_devices[] = { static int udma_get_mmrs(struct platform_device *pdev, struct udma_dev *ud) { - u32 cap2, cap3; + u32 cap2, cap3, cap4; int i; ud->mmrs[MMR_GCFG] = devm_platform_ioremap_resource_byname(pdev, mmr_names[MMR_GCFG]); @@ -3989,6 +4306,13 @@ static int udma_get_mmrs(struct platform_device *pdev, struct udma_dev *ud) ud->tchan_cnt = BCDMA_CAP2_TCHAN_CNT(cap2); ud->rchan_cnt = BCDMA_CAP2_RCHAN_CNT(cap2); break; + case DMA_TYPE_PKTDMA: + cap4 = udma_read(ud->mmrs[MMR_GCFG], 0x30); + ud->tchan_cnt = UDMA_CAP2_TCHAN_CNT(cap2); + ud->rchan_cnt = UDMA_CAP2_RCHAN_CNT(cap2); + ud->rflow_cnt = UDMA_CAP3_RFLOW_CNT(cap3); + ud->tflow_cnt = PKTDMA_CAP4_TFLOW_CNT(cap4); + break; default: return -EINVAL; } @@ -4024,7 +4348,8 @@ static const char * const range_names[] = { [RM_RANGE_BCHAN] = "ti,sci-rm-range-bchan", [RM_RANGE_TCHAN] = "ti,sci-rm-range-tchan", [RM_RANGE_RCHAN] = "ti,sci-rm-range-rchan", - [RM_RANGE_RFLOW] = "ti,sci-rm-range-rflow" + [RM_RANGE_RFLOW] = "ti,sci-rm-range-rflow", + [RM_RANGE_TFLOW] = "ti,sci-rm-range-tflow", }; static int udma_setup_resources(struct udma_dev *ud) @@ -4098,7 +4423,7 @@ static int udma_setup_resources(struct udma_dev *ud) /* Get resource ranges from tisci */ for (i = 0; i < RM_RANGE_LAST; i++) { - if (i == RM_RANGE_BCHAN) + if (i == RM_RANGE_BCHAN || i == RM_RANGE_TFLOW) continue; tisci_rm->rm_ranges[i] = @@ -4245,7 +4570,7 @@ static int bcdma_setup_resources(struct udma_dev *ud) /* Get resource ranges from tisci */ for (i = 0; i < RM_RANGE_LAST; i++) { - if (i == RM_RANGE_RFLOW) + if (i == RM_RANGE_RFLOW || i == RM_RANGE_TFLOW) continue; if (i == RM_RANGE_BCHAN && ud->bchan_cnt == 0) continue; @@ -4351,6 +4676,134 @@ static int bcdma_setup_resources(struct udma_dev *ud) return 0; } +static int pktdma_setup_resources(struct udma_dev *ud) +{ + int ret, i, j; + struct device *dev = ud->dev; + struct ti_sci_resource *rm_res, irq_res; + struct udma_tisci_rm *tisci_rm = &ud->tisci_rm; + const struct udma_oes_offsets *oes = &ud->soc_data->oes; + u32 cap3; + + /* Set up the throughput level start indexes */ + cap3 = udma_read(ud->mmrs[MMR_GCFG], 0x2c); + if (UDMA_CAP3_UCHAN_CNT(cap3)) { + ud->tchan_tpl.levels = 3; + ud->tchan_tpl.start_idx[1] = UDMA_CAP3_UCHAN_CNT(cap3); + ud->tchan_tpl.start_idx[0] = UDMA_CAP3_HCHAN_CNT(cap3); + } else if (UDMA_CAP3_HCHAN_CNT(cap3)) { + ud->tchan_tpl.levels = 2; + ud->tchan_tpl.start_idx[0] = UDMA_CAP3_HCHAN_CNT(cap3); + } else { + ud->tchan_tpl.levels = 1; + } + + ud->tchan_tpl.levels = ud->tchan_tpl.levels; + ud->tchan_tpl.start_idx[0] = ud->tchan_tpl.start_idx[0]; + ud->tchan_tpl.start_idx[1] = ud->tchan_tpl.start_idx[1]; + + ud->tchan_map = devm_kmalloc_array(dev, BITS_TO_LONGS(ud->tchan_cnt), + sizeof(unsigned long), GFP_KERNEL); + ud->tchans = devm_kcalloc(dev, ud->tchan_cnt, sizeof(*ud->tchans), + GFP_KERNEL); + ud->rchan_map = devm_kmalloc_array(dev, BITS_TO_LONGS(ud->rchan_cnt), + sizeof(unsigned long), GFP_KERNEL); + ud->rchans = devm_kcalloc(dev, ud->rchan_cnt, sizeof(*ud->rchans), + GFP_KERNEL); + ud->rflow_in_use = devm_kcalloc(dev, BITS_TO_LONGS(ud->rflow_cnt), + sizeof(unsigned long), + GFP_KERNEL); + ud->rflows = devm_kcalloc(dev, ud->rflow_cnt, sizeof(*ud->rflows), + GFP_KERNEL); + ud->tflow_map = devm_kmalloc_array(dev, BITS_TO_LONGS(ud->tflow_cnt), + sizeof(unsigned long), GFP_KERNEL); + + if (!ud->tchan_map || !ud->rchan_map || !ud->tflow_map || !ud->tchans || + !ud->rchans || !ud->rflows || !ud->rflow_in_use) + return -ENOMEM; + + /* Get resource ranges from tisci */ + for (i = 0; i < RM_RANGE_LAST; i++) { + if (i == RM_RANGE_BCHAN) + continue; + + tisci_rm->rm_ranges[i] = + devm_ti_sci_get_of_resource(tisci_rm->tisci, dev, + tisci_rm->tisci_dev_id, + (char *)range_names[i]); + } + + /* tchan ranges */ + rm_res = tisci_rm->rm_ranges[RM_RANGE_TCHAN]; + if (IS_ERR(rm_res)) { + bitmap_zero(ud->tchan_map, ud->tchan_cnt); + } else { + bitmap_fill(ud->tchan_map, ud->tchan_cnt); + for (i = 0; i < rm_res->sets; i++) + udma_mark_resource_ranges(ud, ud->tchan_map, + &rm_res->desc[i], "tchan"); + } + + /* rchan ranges */ + rm_res = tisci_rm->rm_ranges[RM_RANGE_RCHAN]; + if (IS_ERR(rm_res)) { + bitmap_zero(ud->rchan_map, ud->rchan_cnt); + } else { + bitmap_fill(ud->rchan_map, ud->rchan_cnt); + for (i = 0; i < rm_res->sets; i++) + udma_mark_resource_ranges(ud, ud->rchan_map, + &rm_res->desc[i], "rchan"); + } + + /* rflow ranges */ + rm_res = tisci_rm->rm_ranges[RM_RANGE_RFLOW]; + if (IS_ERR(rm_res)) { + /* all rflows are assigned exclusively to Linux */ + bitmap_zero(ud->rflow_in_use, ud->rflow_cnt); + } else { + bitmap_fill(ud->rflow_in_use, ud->rflow_cnt); + for (i = 0; i < rm_res->sets; i++) + udma_mark_resource_ranges(ud, ud->rflow_in_use, + &rm_res->desc[i], "rflow"); + } + irq_res.sets = rm_res->sets; + + /* tflow ranges */ + rm_res = tisci_rm->rm_ranges[RM_RANGE_TFLOW]; + if (IS_ERR(rm_res)) { + /* all tflows are assigned exclusively to Linux */ + bitmap_zero(ud->tflow_map, ud->tflow_cnt); + } else { + bitmap_fill(ud->tflow_map, ud->tflow_cnt); + for (i = 0; i < rm_res->sets; i++) + udma_mark_resource_ranges(ud, ud->tflow_map, + &rm_res->desc[i], "tflow"); + } + irq_res.sets += rm_res->sets; + + irq_res.desc = kcalloc(irq_res.sets, sizeof(*irq_res.desc), GFP_KERNEL); + rm_res = tisci_rm->rm_ranges[RM_RANGE_TFLOW]; + for (i = 0; i < rm_res->sets; i++) { + irq_res.desc[i].start = rm_res->desc[i].start + + oes->pktdma_tchan_flow; + irq_res.desc[i].num = rm_res->desc[i].num; + } + rm_res = tisci_rm->rm_ranges[RM_RANGE_RFLOW]; + for (j = 0; j < rm_res->sets; j++, i++) { + irq_res.desc[i].start = rm_res->desc[j].start + + oes->pktdma_rchan_flow; + irq_res.desc[i].num = rm_res->desc[j].num; + } + ret = ti_sci_inta_msi_domain_alloc_irqs(ud->dev, &irq_res); + kfree(irq_res.desc); + if (ret) { + dev_err(ud->dev, "Failed to allocate MSI interrupts\n"); + return ret; + } + + return 0; +} + static int setup_resources(struct udma_dev *ud) { struct device *dev = ud->dev; @@ -4363,6 +4816,9 @@ static int setup_resources(struct udma_dev *ud) case DMA_TYPE_BCDMA: ret = bcdma_setup_resources(ud); break; + case DMA_TYPE_PKTDMA: + ret = pktdma_setup_resources(ud); + break; default: return -EINVAL; } @@ -4406,6 +4862,14 @@ static int setup_resources(struct udma_dev *ud) ud->rchan_cnt - bitmap_weight(ud->rchan_map, ud->rchan_cnt)); break; + case DMA_TYPE_PKTDMA: + dev_info(dev, + "Channels: %d (tchan: %u, rchan: %u)\n", + ch_count, + ud->tchan_cnt - bitmap_weight(ud->tchan_map, + ud->tchan_cnt), + ud->rchan_cnt - bitmap_weight(ud->rchan_map, + ud->rchan_cnt)); default: break; } @@ -4536,10 +5000,14 @@ static void udma_dbg_summary_show_chan(struct seq_file *s, case DMA_DEV_TO_MEM: seq_printf(s, "rchan%d [0x%04x -> 0x%04x], ", uc->rchan->id, ucc->src_thread, ucc->dst_thread); + if (uc->ud->match_data->type == DMA_TYPE_PKTDMA) + seq_printf(s, "rflow%d, ", uc->rflow->id); break; case DMA_MEM_TO_DEV: seq_printf(s, "tchan%d [0x%04x -> 0x%04x], ", uc->tchan->id, ucc->src_thread, ucc->dst_thread); + if (uc->ud->match_data->type == DMA_TYPE_PKTDMA) + seq_printf(s, "tflow%d, ", uc->tchan->tflow_id); break; default: seq_printf(s, ")\n"); @@ -4602,8 +5070,10 @@ static int udma_probe(struct platform_device *pdev) return -ENOMEM; match = of_match_node(udma_of_match, dev->of_node); - if (!match) { + if (!match) match = of_match_node(bcdma_of_match, dev->of_node); + if (!match) { + match = of_match_node(pktdma_of_match, dev->of_node); if (!match) { dev_err(dev, "No compatible match found\n"); return -ENODEV; @@ -4667,8 +5137,14 @@ static int udma_probe(struct platform_device *pdev) ring_init_data.tisci = ud->tisci_rm.tisci; ring_init_data.tisci_dev_id = ud->tisci_rm.tisci_dev_id; - ring_init_data.num_rings = ud->bchan_cnt + ud->tchan_cnt + - ud->rchan_cnt; + if (ud->match_data->type == DMA_TYPE_BCDMA) { + ring_init_data.num_rings = ud->bchan_cnt + + ud->tchan_cnt + + ud->rchan_cnt; + } else { + ring_init_data.num_rings = ud->rflow_cnt + + ud->tflow_cnt; + } ud->ringacc = k3_ringacc_dmarings_init(pdev, &ring_init_data); } @@ -4684,11 +5160,14 @@ static int udma_probe(struct platform_device *pdev) } dma_cap_set(DMA_SLAVE, ud->ddev.cap_mask); - dma_cap_set(DMA_CYCLIC, ud->ddev.cap_mask); + /* cyclic operation is not supported via PKTDMA */ + if (ud->match_data->type != DMA_TYPE_PKTDMA) { + dma_cap_set(DMA_CYCLIC, ud->ddev.cap_mask); + ud->ddev.device_prep_dma_cyclic = udma_prep_dma_cyclic; + } ud->ddev.device_config = udma_slave_config; ud->ddev.device_prep_slave_sg = udma_prep_slave_sg; - ud->ddev.device_prep_dma_cyclic = udma_prep_dma_cyclic; ud->ddev.device_issue_pending = udma_issue_pending; ud->ddev.device_tx_status = udma_tx_status; ud->ddev.device_pause = udma_pause; @@ -4709,6 +5188,10 @@ static int udma_probe(struct platform_device *pdev) bcdma_alloc_chan_resources; ud->ddev.device_router_config = bcdma_router_config; break; + case DMA_TYPE_PKTDMA: + ud->ddev.device_alloc_chan_resources = + pktdma_alloc_chan_resources; + break; default: return -EINVAL; } @@ -4787,6 +5270,8 @@ static int udma_probe(struct platform_device *pdev) uc->tchan = NULL; uc->rchan = NULL; uc->config.remote_thread_id = -1; + uc->config.mapped_channel_id = -1; + uc->config.default_flow_id = -1; uc->config.dir = DMA_MEM_TO_MEM; uc->name = devm_kasprintf(dev, GFP_KERNEL, "%s chan%d", dev_name(dev), i); @@ -4835,5 +5320,15 @@ static struct platform_driver bcdma_driver = { }; builtin_platform_driver(bcdma_driver); +static struct platform_driver pktdma_driver = { + .driver = { + .name = "ti-pktdma", + .of_match_table = pktdma_of_match, + .suppress_bind_attrs = true, + }, + .probe = udma_probe, +}; +builtin_platform_driver(pktdma_driver); + /* Private interfaces to UDMA */ #include "k3-udma-private.c" diff --git a/drivers/dma/ti/k3-udma.h b/drivers/dma/ti/k3-udma.h index 3ec38b383b76..ccb19f286daf 100644 --- a/drivers/dma/ti/k3-udma.h +++ b/drivers/dma/ti/k3-udma.h @@ -55,6 +55,8 @@ #define BCDMA_CAP4_HTCHAN_CNT(val) (((val) >> 16) & 0xff) #define BCDMA_CAP4_UTCHAN_CNT(val) (((val) >> 24) & 0xff) +#define PKTDMA_CAP4_TFLOW_CNT(val) ((val) & 0x3fff) + /* UDMA_CHAN_RT_CTL_REG */ #define UDMA_CHAN_RT_CTL_EN BIT(31) #define UDMA_CHAN_RT_CTL_TDOWN BIT(30) @@ -105,6 +107,7 @@ enum udma_rm_range { RM_RANGE_TCHAN, RM_RANGE_RCHAN, RM_RANGE_RFLOW, + RM_RANGE_TFLOW, RM_RANGE_LAST, }; @@ -152,5 +155,6 @@ void xudma_tchanrt_write(struct udma_tchan *tchan, int reg, u32 val); u32 xudma_rchanrt_read(struct udma_rchan *rchan, int reg); void xudma_rchanrt_write(struct udma_rchan *rchan, int reg, u32 val); bool xudma_rflow_is_gp(struct udma_dev *ud, int id); +int xudma_get_rflow_ring_offset(struct udma_dev *ud); #endif /* K3_UDMA_H_ */ -- cgit v1.2.3-59-g8ed1b From 5b65781d06ea90ef2f8e51a13352c43c3daa8cdc Mon Sep 17 00:00:00 2001 From: Vignesh Raghavendra Date: Tue, 8 Dec 2020 11:04:40 +0200 Subject: dmaengine: ti: k3-udma-glue: Add support for K3 PKTDMA This commit adds support for PKTDMA in k3-udma glue driver. Use new psil_endpoint_config struct to get static data for a given channel or a flow during setup. Make sure that the RX flows being mapped to a RX channel is within the range of flows that is been allocated to that RX channel. Signed-off-by: Vignesh Raghavendra Signed-off-by: Peter Ujfalusi Link: https://lore.kernel.org/r/20201208090440.31792-21-peter.ujfalusi@ti.com Signed-off-by: Vinod Koul --- drivers/dma/ti/k3-udma-glue.c | 291 ++++++++++++++++++++++++++++++++++----- drivers/dma/ti/k3-udma-private.c | 24 ++++ drivers/dma/ti/k3-udma.h | 4 + include/linux/dma/k3-udma-glue.h | 8 ++ 4 files changed, 289 insertions(+), 38 deletions(-) diff --git a/drivers/dma/ti/k3-udma-glue.c b/drivers/dma/ti/k3-udma-glue.c index e6ebcd98c02a..4fdd9f06b723 100644 --- a/drivers/dma/ti/k3-udma-glue.c +++ b/drivers/dma/ti/k3-udma-glue.c @@ -22,6 +22,7 @@ struct k3_udma_glue_common { struct device *dev; + struct device chan_dev; struct udma_dev *udmax; const struct udma_tisci_rm *tisci_rm; struct k3_ringacc *ringacc; @@ -32,7 +33,8 @@ struct k3_udma_glue_common { bool epib; u32 psdata_size; u32 swdata_size; - u32 atype; + u32 atype_asel; + struct psil_endpoint_config *ep_config; }; struct k3_udma_glue_tx_channel { @@ -53,6 +55,8 @@ struct k3_udma_glue_tx_channel { bool tx_filt_einfo; bool tx_filt_pswords; bool tx_supr_tdpkt; + + int udma_tflow_id; }; struct k3_udma_glue_rx_flow { @@ -81,6 +85,16 @@ struct k3_udma_glue_rx_channel { u32 flows_ready; }; +static void k3_udma_chan_dev_release(struct device *dev) +{ + /* The struct containing the device is devm managed */ +} + +static struct class k3_udma_glue_devclass = { + .name = "k3_udma_glue_chan", + .dev_release = k3_udma_chan_dev_release, +}; + #define K3_UDMAX_TDOWN_TIMEOUT_US 1000 static int of_k3_udma_glue_parse(struct device_node *udmax_np, @@ -100,7 +114,6 @@ static int of_k3_udma_glue_parse_chn(struct device_node *chn_np, const char *name, struct k3_udma_glue_common *common, bool tx_chn) { - struct psil_endpoint_config *ep_config; struct of_phandle_args dma_spec; u32 thread_id; int ret = 0; @@ -117,15 +130,26 @@ static int of_k3_udma_glue_parse_chn(struct device_node *chn_np, &dma_spec)) return -ENOENT; + ret = of_k3_udma_glue_parse(dma_spec.np, common); + if (ret) + goto out_put_spec; + thread_id = dma_spec.args[0]; if (dma_spec.args_count == 2) { - if (dma_spec.args[1] > 2) { + if (dma_spec.args[1] > 2 && !xudma_is_pktdma(common->udmax)) { dev_err(common->dev, "Invalid channel atype: %u\n", dma_spec.args[1]); ret = -EINVAL; goto out_put_spec; } - common->atype = dma_spec.args[1]; + if (dma_spec.args[1] > 15 && xudma_is_pktdma(common->udmax)) { + dev_err(common->dev, "Invalid channel asel: %u\n", + dma_spec.args[1]); + ret = -EINVAL; + goto out_put_spec; + } + + common->atype_asel = dma_spec.args[1]; } if (tx_chn && !(thread_id & K3_PSIL_DST_THREAD_ID_OFFSET)) { @@ -139,25 +163,23 @@ static int of_k3_udma_glue_parse_chn(struct device_node *chn_np, } /* get psil endpoint config */ - ep_config = psil_get_ep_config(thread_id); - if (IS_ERR(ep_config)) { + common->ep_config = psil_get_ep_config(thread_id); + if (IS_ERR(common->ep_config)) { dev_err(common->dev, "No configuration for psi-l thread 0x%04x\n", thread_id); - ret = PTR_ERR(ep_config); + ret = PTR_ERR(common->ep_config); goto out_put_spec; } - common->epib = ep_config->needs_epib; - common->psdata_size = ep_config->psd_size; + common->epib = common->ep_config->needs_epib; + common->psdata_size = common->ep_config->psd_size; if (tx_chn) common->dst_thread = thread_id; else common->src_thread = thread_id; - ret = of_k3_udma_glue_parse(dma_spec.np, common); - out_put_spec: of_node_put(dma_spec.np); return ret; @@ -223,7 +245,7 @@ static int k3_udma_glue_cfg_tx_chn(struct k3_udma_glue_tx_channel *tx_chn) req.tx_supr_tdpkt = 1; req.tx_fetch_size = tx_chn->common.hdesc_size >> 2; req.txcq_qnum = k3_ringacc_get_ring_id(tx_chn->ringtxcq); - req.tx_atype = tx_chn->common.atype; + req.tx_atype = tx_chn->common.atype_asel; return tisci_rm->tisci_udmap_ops->tx_ch_cfg(tisci_rm->tisci, &req); } @@ -255,8 +277,14 @@ struct k3_udma_glue_tx_channel *k3_udma_glue_request_tx_chn(struct device *dev, tx_chn->common.psdata_size, tx_chn->common.swdata_size); + if (xudma_is_pktdma(tx_chn->common.udmax)) + tx_chn->udma_tchan_id = tx_chn->common.ep_config->mapped_channel_id; + else + tx_chn->udma_tchan_id = -1; + /* request and cfg UDMAP TX channel */ - tx_chn->udma_tchanx = xudma_tchan_get(tx_chn->common.udmax, -1); + tx_chn->udma_tchanx = xudma_tchan_get(tx_chn->common.udmax, + tx_chn->udma_tchan_id); if (IS_ERR(tx_chn->udma_tchanx)) { ret = PTR_ERR(tx_chn->udma_tchanx); dev_err(dev, "UDMAX tchanx get err %d\n", ret); @@ -264,11 +292,34 @@ struct k3_udma_glue_tx_channel *k3_udma_glue_request_tx_chn(struct device *dev, } tx_chn->udma_tchan_id = xudma_tchan_get_id(tx_chn->udma_tchanx); + tx_chn->common.chan_dev.class = &k3_udma_glue_devclass; + tx_chn->common.chan_dev.parent = xudma_get_device(tx_chn->common.udmax); + dev_set_name(&tx_chn->common.chan_dev, "tchan%d-0x%04x", + tx_chn->udma_tchan_id, tx_chn->common.dst_thread); + ret = device_register(&tx_chn->common.chan_dev); + if (ret) { + dev_err(dev, "Channel Device registration failed %d\n", ret); + tx_chn->common.chan_dev.parent = NULL; + goto err; + } + + if (xudma_is_pktdma(tx_chn->common.udmax)) { + /* prepare the channel device as coherent */ + tx_chn->common.chan_dev.dma_coherent = true; + dma_coerce_mask_and_coherent(&tx_chn->common.chan_dev, + DMA_BIT_MASK(48)); + } + atomic_set(&tx_chn->free_pkts, cfg->txcq_cfg.size); + if (xudma_is_pktdma(tx_chn->common.udmax)) + tx_chn->udma_tflow_id = tx_chn->common.ep_config->default_flow_id; + else + tx_chn->udma_tflow_id = tx_chn->udma_tchan_id; + /* request and cfg rings */ ret = k3_ringacc_request_rings_pair(tx_chn->common.ringacc, - tx_chn->udma_tchan_id, -1, + tx_chn->udma_tflow_id, -1, &tx_chn->ringtx, &tx_chn->ringtxcq); if (ret) { @@ -280,6 +331,12 @@ struct k3_udma_glue_tx_channel *k3_udma_glue_request_tx_chn(struct device *dev, cfg->tx_cfg.dma_dev = k3_udma_glue_tx_get_dma_device(tx_chn); cfg->txcq_cfg.dma_dev = cfg->tx_cfg.dma_dev; + /* Set the ASEL value for DMA rings of PKTDMA */ + if (xudma_is_pktdma(tx_chn->common.udmax)) { + cfg->tx_cfg.asel = tx_chn->common.atype_asel; + cfg->txcq_cfg.asel = tx_chn->common.atype_asel; + } + ret = k3_ringacc_ring_cfg(tx_chn->ringtx, &cfg->tx_cfg); if (ret) { dev_err(dev, "Failed to cfg ringtx %d\n", ret); @@ -331,6 +388,11 @@ void k3_udma_glue_release_tx_chn(struct k3_udma_glue_tx_channel *tx_chn) if (tx_chn->ringtx) k3_ringacc_ring_free(tx_chn->ringtx); + + if (tx_chn->common.chan_dev.parent) { + device_unregister(&tx_chn->common.chan_dev); + tx_chn->common.chan_dev.parent = NULL; + } } EXPORT_SYMBOL_GPL(k3_udma_glue_release_tx_chn); @@ -443,13 +505,10 @@ void k3_udma_glue_reset_tx_chn(struct k3_udma_glue_tx_channel *tx_chn, void *data, void (*cleanup)(void *data, dma_addr_t desc_dma)) { + struct device *dev = tx_chn->common.dev; dma_addr_t desc_dma; int occ_tx, i, ret; - /* reset TXCQ as it is not input for udma - expected to be empty */ - if (tx_chn->ringtxcq) - k3_ringacc_ring_reset(tx_chn->ringtxcq); - /* * TXQ reset need to be special way as it is input for udma and its * state cached by udma, so: @@ -458,17 +517,20 @@ void k3_udma_glue_reset_tx_chn(struct k3_udma_glue_tx_channel *tx_chn, * 3) reset TXQ in a special way */ occ_tx = k3_ringacc_ring_get_occ(tx_chn->ringtx); - dev_dbg(tx_chn->common.dev, "TX reset occ_tx %u\n", occ_tx); + dev_dbg(dev, "TX reset occ_tx %u\n", occ_tx); for (i = 0; i < occ_tx; i++) { ret = k3_ringacc_ring_pop(tx_chn->ringtx, &desc_dma); if (ret) { - dev_err(tx_chn->common.dev, "TX reset pop %d\n", ret); + if (ret != -ENODATA) + dev_err(dev, "TX reset pop %d\n", ret); break; } cleanup(data, desc_dma); } + /* reset TXCQ as it is not input for udma - expected to be empty */ + k3_ringacc_ring_reset(tx_chn->ringtxcq); k3_ringacc_ring_reset_dma(tx_chn->ringtx, occ_tx); } EXPORT_SYMBOL_GPL(k3_udma_glue_reset_tx_chn); @@ -487,7 +549,12 @@ EXPORT_SYMBOL_GPL(k3_udma_glue_tx_get_txcq_id); int k3_udma_glue_tx_get_irq(struct k3_udma_glue_tx_channel *tx_chn) { - tx_chn->virq = k3_ringacc_get_ring_irq_num(tx_chn->ringtxcq); + if (xudma_is_pktdma(tx_chn->common.udmax)) { + tx_chn->virq = xudma_pktdma_tflow_get_irq(tx_chn->common.udmax, + tx_chn->udma_tflow_id); + } else { + tx_chn->virq = k3_ringacc_get_ring_irq_num(tx_chn->ringtxcq); + } return tx_chn->virq; } @@ -496,10 +563,36 @@ EXPORT_SYMBOL_GPL(k3_udma_glue_tx_get_irq); struct device * k3_udma_glue_tx_get_dma_device(struct k3_udma_glue_tx_channel *tx_chn) { + if (xudma_is_pktdma(tx_chn->common.udmax) && + (tx_chn->common.atype_asel == 14 || tx_chn->common.atype_asel == 15)) + return &tx_chn->common.chan_dev; + return xudma_get_device(tx_chn->common.udmax); } EXPORT_SYMBOL_GPL(k3_udma_glue_tx_get_dma_device); +void k3_udma_glue_tx_dma_to_cppi5_addr(struct k3_udma_glue_tx_channel *tx_chn, + dma_addr_t *addr) +{ + if (!xudma_is_pktdma(tx_chn->common.udmax) || + !tx_chn->common.atype_asel) + return; + + *addr |= (u64)tx_chn->common.atype_asel << K3_ADDRESS_ASEL_SHIFT; +} +EXPORT_SYMBOL_GPL(k3_udma_glue_tx_dma_to_cppi5_addr); + +void k3_udma_glue_tx_cppi5_to_dma_addr(struct k3_udma_glue_tx_channel *tx_chn, + dma_addr_t *addr) +{ + if (!xudma_is_pktdma(tx_chn->common.udmax) || + !tx_chn->common.atype_asel) + return; + + *addr &= (u64)GENMASK(K3_ADDRESS_ASEL_SHIFT - 1, 0); +} +EXPORT_SYMBOL_GPL(k3_udma_glue_tx_cppi5_to_dma_addr); + static int k3_udma_glue_cfg_rx_chn(struct k3_udma_glue_rx_channel *rx_chn) { const struct udma_tisci_rm *tisci_rm = rx_chn->common.tisci_rm; @@ -511,8 +604,6 @@ static int k3_udma_glue_cfg_rx_chn(struct k3_udma_glue_rx_channel *rx_chn) req.valid_params = TI_SCI_MSG_VALUE_RM_UDMAP_CH_FETCH_SIZE_VALID | TI_SCI_MSG_VALUE_RM_UDMAP_CH_CQ_QNUM_VALID | TI_SCI_MSG_VALUE_RM_UDMAP_CH_CHAN_TYPE_VALID | - TI_SCI_MSG_VALUE_RM_UDMAP_CH_RX_FLOWID_START_VALID | - TI_SCI_MSG_VALUE_RM_UDMAP_CH_RX_FLOWID_CNT_VALID | TI_SCI_MSG_VALUE_RM_UDMAP_CH_ATYPE_VALID; req.nav_id = tisci_rm->tisci_dev_id; @@ -524,13 +615,16 @@ static int k3_udma_glue_cfg_rx_chn(struct k3_udma_glue_rx_channel *rx_chn) * req.rxcq_qnum = k3_ringacc_get_ring_id(rx_chn->flows[0].ringrx); */ req.rxcq_qnum = 0xFFFF; - if (rx_chn->flow_num && rx_chn->flow_id_base != rx_chn->udma_rchan_id) { + if (!xudma_is_pktdma(rx_chn->common.udmax) && rx_chn->flow_num && + rx_chn->flow_id_base != rx_chn->udma_rchan_id) { /* Default flow + extra ones */ + req.valid_params |= TI_SCI_MSG_VALUE_RM_UDMAP_CH_RX_FLOWID_START_VALID | + TI_SCI_MSG_VALUE_RM_UDMAP_CH_RX_FLOWID_CNT_VALID; req.flowid_start = rx_chn->flow_id_base; req.flowid_cnt = rx_chn->flow_num; } req.rx_chan_type = TI_SCI_RM_UDMAP_CHAN_TYPE_PKT_PBRR; - req.rx_atype = rx_chn->common.atype; + req.rx_atype = rx_chn->common.atype_asel; ret = tisci_rm->tisci_udmap_ops->rx_ch_cfg(tisci_rm->tisci, &req); if (ret) @@ -584,10 +678,18 @@ static int k3_udma_glue_cfg_rx_flow(struct k3_udma_glue_rx_channel *rx_chn, goto err_rflow_put; } + if (xudma_is_pktdma(rx_chn->common.udmax)) { + rx_ringfdq_id = flow->udma_rflow_id + + xudma_get_rflow_ring_offset(rx_chn->common.udmax); + rx_ring_id = 0; + } else { + rx_ring_id = flow_cfg->ring_rxq_id; + rx_ringfdq_id = flow_cfg->ring_rxfdq0_id; + } + /* request and cfg rings */ ret = k3_ringacc_request_rings_pair(rx_chn->common.ringacc, - flow_cfg->ring_rxfdq0_id, - flow_cfg->ring_rxq_id, + rx_ringfdq_id, rx_ring_id, &flow->ringrxfdq, &flow->ringrx); if (ret) { @@ -599,6 +701,12 @@ static int k3_udma_glue_cfg_rx_flow(struct k3_udma_glue_rx_channel *rx_chn, flow_cfg->rx_cfg.dma_dev = k3_udma_glue_rx_get_dma_device(rx_chn); flow_cfg->rxfdq_cfg.dma_dev = flow_cfg->rx_cfg.dma_dev; + /* Set the ASEL value for DMA rings of PKTDMA */ + if (xudma_is_pktdma(rx_chn->common.udmax)) { + flow_cfg->rx_cfg.asel = rx_chn->common.atype_asel; + flow_cfg->rxfdq_cfg.asel = rx_chn->common.atype_asel; + } + ret = k3_ringacc_ring_cfg(flow->ringrx, &flow_cfg->rx_cfg); if (ret) { dev_err(dev, "Failed to cfg ringrx %d\n", ret); @@ -757,6 +865,7 @@ k3_udma_glue_request_rx_chn_priv(struct device *dev, const char *name, struct k3_udma_glue_rx_channel_cfg *cfg) { struct k3_udma_glue_rx_channel *rx_chn; + struct psil_endpoint_config *ep_cfg; int ret, i; if (cfg->flow_id_num <= 0) @@ -784,8 +893,16 @@ k3_udma_glue_request_rx_chn_priv(struct device *dev, const char *name, rx_chn->common.psdata_size, rx_chn->common.swdata_size); + ep_cfg = rx_chn->common.ep_config; + + if (xudma_is_pktdma(rx_chn->common.udmax)) + rx_chn->udma_rchan_id = ep_cfg->mapped_channel_id; + else + rx_chn->udma_rchan_id = -1; + /* request and cfg UDMAP RX channel */ - rx_chn->udma_rchanx = xudma_rchan_get(rx_chn->common.udmax, -1); + rx_chn->udma_rchanx = xudma_rchan_get(rx_chn->common.udmax, + rx_chn->udma_rchan_id); if (IS_ERR(rx_chn->udma_rchanx)) { ret = PTR_ERR(rx_chn->udma_rchanx); dev_err(dev, "UDMAX rchanx get err %d\n", ret); @@ -793,12 +910,48 @@ k3_udma_glue_request_rx_chn_priv(struct device *dev, const char *name, } rx_chn->udma_rchan_id = xudma_rchan_get_id(rx_chn->udma_rchanx); - rx_chn->flow_num = cfg->flow_id_num; - rx_chn->flow_id_base = cfg->flow_id_base; + rx_chn->common.chan_dev.class = &k3_udma_glue_devclass; + rx_chn->common.chan_dev.parent = xudma_get_device(rx_chn->common.udmax); + dev_set_name(&rx_chn->common.chan_dev, "rchan%d-0x%04x", + rx_chn->udma_rchan_id, rx_chn->common.src_thread); + ret = device_register(&rx_chn->common.chan_dev); + if (ret) { + dev_err(dev, "Channel Device registration failed %d\n", ret); + rx_chn->common.chan_dev.parent = NULL; + goto err; + } - /* Use RX channel id as flow id: target dev can't generate flow_id */ - if (cfg->flow_id_use_rxchan_id) - rx_chn->flow_id_base = rx_chn->udma_rchan_id; + if (xudma_is_pktdma(rx_chn->common.udmax)) { + /* prepare the channel device as coherent */ + rx_chn->common.chan_dev.dma_coherent = true; + dma_coerce_mask_and_coherent(&rx_chn->common.chan_dev, + DMA_BIT_MASK(48)); + } + + if (xudma_is_pktdma(rx_chn->common.udmax)) { + int flow_start = cfg->flow_id_base; + int flow_end; + + if (flow_start == -1) + flow_start = ep_cfg->flow_start; + + flow_end = flow_start + cfg->flow_id_num - 1; + if (flow_start < ep_cfg->flow_start || + flow_end > (ep_cfg->flow_start + ep_cfg->flow_num - 1)) { + dev_err(dev, "Invalid flow range requested\n"); + ret = -EINVAL; + goto err; + } + rx_chn->flow_id_base = flow_start; + } else { + rx_chn->flow_id_base = cfg->flow_id_base; + + /* Use RX channel id as flow id: target dev can't generate flow_id */ + if (cfg->flow_id_use_rxchan_id) + rx_chn->flow_id_base = rx_chn->udma_rchan_id; + } + + rx_chn->flow_num = cfg->flow_id_num; rx_chn->flows = devm_kcalloc(dev, rx_chn->flow_num, sizeof(*rx_chn->flows), GFP_KERNEL); @@ -888,6 +1041,24 @@ k3_udma_glue_request_remote_rx_chn(struct device *dev, const char *name, goto err; } + rx_chn->common.chan_dev.class = &k3_udma_glue_devclass; + rx_chn->common.chan_dev.parent = xudma_get_device(rx_chn->common.udmax); + dev_set_name(&rx_chn->common.chan_dev, "rchan_remote-0x%04x", + rx_chn->common.src_thread); + ret = device_register(&rx_chn->common.chan_dev); + if (ret) { + dev_err(dev, "Channel Device registration failed %d\n", ret); + rx_chn->common.chan_dev.parent = NULL; + goto err; + } + + if (xudma_is_pktdma(rx_chn->common.udmax)) { + /* prepare the channel device as coherent */ + rx_chn->common.chan_dev.dma_coherent = true; + dma_coerce_mask_and_coherent(&rx_chn->common.chan_dev, + DMA_BIT_MASK(48)); + } + ret = k3_udma_glue_allocate_rx_flows(rx_chn, cfg); if (ret) goto err; @@ -940,6 +1111,11 @@ void k3_udma_glue_release_rx_chn(struct k3_udma_glue_rx_channel *rx_chn) if (!IS_ERR_OR_NULL(rx_chn->udma_rchanx)) xudma_rchan_put(rx_chn->common.udmax, rx_chn->udma_rchanx); + + if (rx_chn->common.chan_dev.parent) { + device_unregister(&rx_chn->common.chan_dev); + rx_chn->common.chan_dev.parent = NULL; + } } EXPORT_SYMBOL_GPL(k3_udma_glue_release_rx_chn); @@ -1151,12 +1327,10 @@ void k3_udma_glue_reset_rx_chn(struct k3_udma_glue_rx_channel *rx_chn, /* reset RXCQ as it is not input for udma - expected to be empty */ occ_rx = k3_ringacc_ring_get_occ(flow->ringrx); dev_dbg(dev, "RX reset flow %u occ_rx %u\n", flow_num, occ_rx); - if (flow->ringrx) - k3_ringacc_ring_reset(flow->ringrx); /* Skip RX FDQ in case one FDQ is used for the set of flows */ if (skip_fdq) - return; + goto do_reset; /* * RX FDQ reset need to be special way as it is input for udma and its @@ -1171,13 +1345,17 @@ void k3_udma_glue_reset_rx_chn(struct k3_udma_glue_rx_channel *rx_chn, for (i = 0; i < occ_rx; i++) { ret = k3_ringacc_ring_pop(flow->ringrxfdq, &desc_dma); if (ret) { - dev_err(dev, "RX reset pop %d\n", ret); + if (ret != -ENODATA) + dev_err(dev, "RX reset pop %d\n", ret); break; } cleanup(data, desc_dma); } k3_ringacc_ring_reset_dma(flow->ringrxfdq, occ_rx); + +do_reset: + k3_ringacc_ring_reset(flow->ringrx); } EXPORT_SYMBOL_GPL(k3_udma_glue_reset_rx_chn); @@ -1207,7 +1385,12 @@ int k3_udma_glue_rx_get_irq(struct k3_udma_glue_rx_channel *rx_chn, flow = &rx_chn->flows[flow_num]; - flow->virq = k3_ringacc_get_ring_irq_num(flow->ringrx); + if (xudma_is_pktdma(rx_chn->common.udmax)) { + flow->virq = xudma_pktdma_rflow_get_irq(rx_chn->common.udmax, + flow->udma_rflow_id); + } else { + flow->virq = k3_ringacc_get_ring_irq_num(flow->ringrx); + } return flow->virq; } @@ -1216,6 +1399,38 @@ EXPORT_SYMBOL_GPL(k3_udma_glue_rx_get_irq); struct device * k3_udma_glue_rx_get_dma_device(struct k3_udma_glue_rx_channel *rx_chn) { + if (xudma_is_pktdma(rx_chn->common.udmax) && + (rx_chn->common.atype_asel == 14 || rx_chn->common.atype_asel == 15)) + return &rx_chn->common.chan_dev; + return xudma_get_device(rx_chn->common.udmax); } EXPORT_SYMBOL_GPL(k3_udma_glue_rx_get_dma_device); + +void k3_udma_glue_rx_dma_to_cppi5_addr(struct k3_udma_glue_rx_channel *rx_chn, + dma_addr_t *addr) +{ + if (!xudma_is_pktdma(rx_chn->common.udmax) || + !rx_chn->common.atype_asel) + return; + + *addr |= (u64)rx_chn->common.atype_asel << K3_ADDRESS_ASEL_SHIFT; +} +EXPORT_SYMBOL_GPL(k3_udma_glue_rx_dma_to_cppi5_addr); + +void k3_udma_glue_rx_cppi5_to_dma_addr(struct k3_udma_glue_rx_channel *rx_chn, + dma_addr_t *addr) +{ + if (!xudma_is_pktdma(rx_chn->common.udmax) || + !rx_chn->common.atype_asel) + return; + + *addr &= (u64)GENMASK(K3_ADDRESS_ASEL_SHIFT - 1, 0); +} +EXPORT_SYMBOL_GPL(k3_udma_glue_rx_cppi5_to_dma_addr); + +static int __init k3_udma_glue_class_init(void) +{ + return class_register(&k3_udma_glue_devclass); +} +arch_initcall(k3_udma_glue_class_init); diff --git a/drivers/dma/ti/k3-udma-private.c b/drivers/dma/ti/k3-udma-private.c index 5436b19d656e..eb4795c089bd 100644 --- a/drivers/dma/ti/k3-udma-private.c +++ b/drivers/dma/ti/k3-udma-private.c @@ -157,3 +157,27 @@ void xudma_##res##rt_write(struct udma_##res *p, int reg, u32 val) \ EXPORT_SYMBOL(xudma_##res##rt_write) XUDMA_RT_IO_FUNCTIONS(tchan); XUDMA_RT_IO_FUNCTIONS(rchan); + +int xudma_is_pktdma(struct udma_dev *ud) +{ + return ud->match_data->type == DMA_TYPE_PKTDMA; +} +EXPORT_SYMBOL(xudma_is_pktdma); + +int xudma_pktdma_tflow_get_irq(struct udma_dev *ud, int udma_tflow_id) +{ + const struct udma_oes_offsets *oes = &ud->soc_data->oes; + + return ti_sci_inta_msi_get_virq(ud->dev, udma_tflow_id + + oes->pktdma_tchan_flow); +} +EXPORT_SYMBOL(xudma_pktdma_tflow_get_irq); + +int xudma_pktdma_rflow_get_irq(struct udma_dev *ud, int udma_rflow_id) +{ + const struct udma_oes_offsets *oes = &ud->soc_data->oes; + + return ti_sci_inta_msi_get_virq(ud->dev, udma_rflow_id + + oes->pktdma_rchan_flow); +} +EXPORT_SYMBOL(xudma_pktdma_rflow_get_irq); diff --git a/drivers/dma/ti/k3-udma.h b/drivers/dma/ti/k3-udma.h index ccb19f286daf..d349c6d482ae 100644 --- a/drivers/dma/ti/k3-udma.h +++ b/drivers/dma/ti/k3-udma.h @@ -157,4 +157,8 @@ void xudma_rchanrt_write(struct udma_rchan *rchan, int reg, u32 val); bool xudma_rflow_is_gp(struct udma_dev *ud, int id); int xudma_get_rflow_ring_offset(struct udma_dev *ud); +int xudma_is_pktdma(struct udma_dev *ud); + +int xudma_pktdma_tflow_get_irq(struct udma_dev *ud, int udma_tflow_id); +int xudma_pktdma_rflow_get_irq(struct udma_dev *ud, int udma_rflow_id); #endif /* K3_UDMA_H_ */ diff --git a/include/linux/dma/k3-udma-glue.h b/include/linux/dma/k3-udma-glue.h index d7c12f31377c..e443be4d3b4b 100644 --- a/include/linux/dma/k3-udma-glue.h +++ b/include/linux/dma/k3-udma-glue.h @@ -43,6 +43,10 @@ u32 k3_udma_glue_tx_get_txcq_id(struct k3_udma_glue_tx_channel *tx_chn); int k3_udma_glue_tx_get_irq(struct k3_udma_glue_tx_channel *tx_chn); struct device * k3_udma_glue_tx_get_dma_device(struct k3_udma_glue_tx_channel *tx_chn); +void k3_udma_glue_tx_dma_to_cppi5_addr(struct k3_udma_glue_tx_channel *tx_chn, + dma_addr_t *addr); +void k3_udma_glue_tx_cppi5_to_dma_addr(struct k3_udma_glue_tx_channel *tx_chn, + dma_addr_t *addr); enum { K3_UDMA_GLUE_SRC_TAG_LO_KEEP = 0, @@ -134,5 +138,9 @@ int k3_udma_glue_rx_flow_disable(struct k3_udma_glue_rx_channel *rx_chn, u32 flow_idx); struct device * k3_udma_glue_rx_get_dma_device(struct k3_udma_glue_rx_channel *rx_chn); +void k3_udma_glue_rx_dma_to_cppi5_addr(struct k3_udma_glue_rx_channel *rx_chn, + dma_addr_t *addr); +void k3_udma_glue_rx_cppi5_to_dma_addr(struct k3_udma_glue_rx_channel *rx_chn, + dma_addr_t *addr); #endif /* K3_UDMA_GLUE_H_ */ -- cgit v1.2.3-59-g8ed1b From 115ff12aecfd55376d704fa2c0a2d117e5827f9f Mon Sep 17 00:00:00 2001 From: Peter Ujfalusi Date: Mon, 14 Dec 2020 08:54:21 +0200 Subject: soc: ti: k3-ringacc: Use correct error casting in k3_ringacc_dmarings_init Use ERR_CAST() when devm_ioremap_resource() fails. Reported-by: kernel test robot Signed-off-by: Peter Ujfalusi Link: https://lore.kernel.org/r/20201214065421.5138-1-peter.ujfalusi@ti.com Signed-off-by: Vinod Koul --- drivers/soc/ti/k3-ringacc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/soc/ti/k3-ringacc.c b/drivers/soc/ti/k3-ringacc.c index c88c305ba367..b495b0d5d0fa 100644 --- a/drivers/soc/ti/k3-ringacc.c +++ b/drivers/soc/ti/k3-ringacc.c @@ -1476,7 +1476,7 @@ struct k3_ringacc *k3_ringacc_dmarings_init(struct platform_device *pdev, res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "ringrt"); base_rt = devm_ioremap_resource(dev, res); if (IS_ERR(base_rt)) - return base_rt; + return ERR_CAST(base_rt); ringacc->rings = devm_kzalloc(dev, sizeof(*ringacc->rings) * -- cgit v1.2.3-59-g8ed1b