From c841f9909b0d947ae7593040cc59646dc9f14455 Mon Sep 17 00:00:00 2001 From: Viken Dadhaniya Date: Fri, 15 Nov 2024 14:58:54 +0530 Subject: dt-bindings: dma: qcom,gpi: Add QCS615 compatible Document compatible for GPI DMA controller on QCS615 platform. Acked-by: Krzysztof Kozlowski Signed-off-by: Viken Dadhaniya Link: https://lore.kernel.org/r/20241115092854.1877369-1-quic_vdadhani@quicinc.com Signed-off-by: Vinod Koul --- Documentation/devicetree/bindings/dma/qcom,gpi.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/Documentation/devicetree/bindings/dma/qcom,gpi.yaml b/Documentation/devicetree/bindings/dma/qcom,gpi.yaml index 4ad56a409b9c..58c7863c5f41 100644 --- a/Documentation/devicetree/bindings/dma/qcom,gpi.yaml +++ b/Documentation/devicetree/bindings/dma/qcom,gpi.yaml @@ -39,6 +39,7 @@ properties: - const: qcom,sm6350-gpi-dma - items: - enum: + - qcom,qcs615-gpi-dma - qcom,sdm670-gpi-dma - qcom,sm6125-gpi-dma - qcom,sm8150-gpi-dma -- cgit v1.2.3-59-g8ed1b From 794dae89874c6be76b46ae90e0c78c76a80175c3 Mon Sep 17 00:00:00 2001 From: Viken Dadhaniya Date: Tue, 12 Nov 2024 09:42:51 +0530 Subject: dt-bindings: dma: qcom,gpi: Add QCS8300 compatible Document compatible for GPI DMA controller on QCS8300 platform. Signed-off-by: Viken Dadhaniya Acked-by: Rob Herring (Arm) Link: https://lore.kernel.org/r/20241112041252.351266-1-quic_vdadhani@quicinc.com Signed-off-by: Vinod Koul --- Documentation/devicetree/bindings/dma/qcom,gpi.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/Documentation/devicetree/bindings/dma/qcom,gpi.yaml b/Documentation/devicetree/bindings/dma/qcom,gpi.yaml index 58c7863c5f41..9d0f32ee181e 100644 --- a/Documentation/devicetree/bindings/dma/qcom,gpi.yaml +++ b/Documentation/devicetree/bindings/dma/qcom,gpi.yaml @@ -25,6 +25,7 @@ properties: - items: - enum: - qcom,qcm2290-gpi-dma + - qcom,qcs8300-gpi-dma - qcom,qdu1000-gpi-dma - qcom,sar2130p-gpi-dma - qcom,sc7280-gpi-dma -- cgit v1.2.3-59-g8ed1b From bf9b0834552e615b1dbd3015c2f0ed2a3bdf62a4 Mon Sep 17 00:00:00 2001 From: Konrad Dybcio Date: Fri, 8 Nov 2024 22:41:17 +0100 Subject: dt-bindings: dma: qcom,gpi: Add SA8775P compatible Add a compatible for the GPI DMA controller on SA8775P. Signed-off-by: Konrad Dybcio Acked-by: Krzysztof Kozlowski Link: https://lore.kernel.org/r/20241108-topic-sa8775_dma2-v1-1-1d3b0d08d153@oss.qualcomm.com Signed-off-by: Vinod Koul --- Documentation/devicetree/bindings/dma/qcom,gpi.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/Documentation/devicetree/bindings/dma/qcom,gpi.yaml b/Documentation/devicetree/bindings/dma/qcom,gpi.yaml index 9d0f32ee181e..2b95e86e40d5 100644 --- a/Documentation/devicetree/bindings/dma/qcom,gpi.yaml +++ b/Documentation/devicetree/bindings/dma/qcom,gpi.yaml @@ -27,6 +27,7 @@ properties: - qcom,qcm2290-gpi-dma - qcom,qcs8300-gpi-dma - qcom,qdu1000-gpi-dma + - qcom,sa8775p-gpi-dma - qcom,sar2130p-gpi-dma - qcom,sc7280-gpi-dma - qcom,sdx75-gpi-dma -- cgit v1.2.3-59-g8ed1b From 0f31c0912286f84b34b15e39b286db8f4765ced8 Mon Sep 17 00:00:00 2001 From: Colin Ian King Date: Thu, 7 Nov 2024 11:46:56 +0000 Subject: dmaengine: xilinx: xdma: remove redundant check on ret The variable ret is being checked for an error and returning ret and the following statement returns ret too. The if check is redundant, and remove it. Just return the value returned from the call to regmap_write. Signed-off-by: Colin Ian King Link: https://lore.kernel.org/r/20241107114656.17611-1-colin.i.king@gmail.com Signed-off-by: Vinod Koul --- drivers/dma/xilinx/xdma.c | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/drivers/dma/xilinx/xdma.c b/drivers/dma/xilinx/xdma.c index 93772abc3b49..0d88b1a670e1 100644 --- a/drivers/dma/xilinx/xdma.c +++ b/drivers/dma/xilinx/xdma.c @@ -390,15 +390,11 @@ static int xdma_xfer_start(struct xdma_chan *xchan) */ static int xdma_xfer_stop(struct xdma_chan *xchan) { - int ret; struct xdma_device *xdev = xchan->xdev_hdl; /* clear run stop bit to prevent any further auto-triggering */ - ret = regmap_write(xdev->rmap, xchan->base + XDMA_CHAN_CONTROL_W1C, - CHAN_CTRL_RUN_STOP); - if (ret) - return ret; - return ret; + return regmap_write(xdev->rmap, xchan->base + XDMA_CHAN_CONTROL_W1C, + CHAN_CTRL_RUN_STOP); } /** -- cgit v1.2.3-59-g8ed1b From 8a20040f9de3a4ea90ab900cdd7745d57bc2da82 Mon Sep 17 00:00:00 2001 From: Marek Vasut Date: Thu, 31 Oct 2024 18:11:04 +0100 Subject: dmaengine: xilinx_dma: Configure parking registers only if parking enabled The VDMA can work in two modes, parking or circular. Do not program the parking mode registers in case circular mode is used, it is useless. Signed-off-by: Marek Vasut Reviewed-by: Radhey Shyam Pandey Link: https://lore.kernel.org/r/20241031171132.56861-1-marex@denx.de Signed-off-by: Vinod Koul --- drivers/dma/xilinx/xilinx_dma.c | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/drivers/dma/xilinx/xilinx_dma.c b/drivers/dma/xilinx/xilinx_dma.c index 1bdd57de87a6..108a7287f4cd 100644 --- a/drivers/dma/xilinx/xilinx_dma.c +++ b/drivers/dma/xilinx/xilinx_dma.c @@ -1404,16 +1404,18 @@ static void xilinx_vdma_start_transfer(struct xilinx_dma_chan *chan) dma_ctrl_write(chan, XILINX_DMA_REG_DMACR, reg); - j = chan->desc_submitcount; - reg = dma_read(chan, XILINX_DMA_REG_PARK_PTR); - if (chan->direction == DMA_MEM_TO_DEV) { - reg &= ~XILINX_DMA_PARK_PTR_RD_REF_MASK; - reg |= j << XILINX_DMA_PARK_PTR_RD_REF_SHIFT; - } else { - reg &= ~XILINX_DMA_PARK_PTR_WR_REF_MASK; - reg |= j << XILINX_DMA_PARK_PTR_WR_REF_SHIFT; + if (config->park) { + j = chan->desc_submitcount; + reg = dma_read(chan, XILINX_DMA_REG_PARK_PTR); + if (chan->direction == DMA_MEM_TO_DEV) { + reg &= ~XILINX_DMA_PARK_PTR_RD_REF_MASK; + reg |= j << XILINX_DMA_PARK_PTR_RD_REF_SHIFT; + } else { + reg &= ~XILINX_DMA_PARK_PTR_WR_REF_MASK; + reg |= j << XILINX_DMA_PARK_PTR_WR_REF_SHIFT; + } + dma_write(chan, XILINX_DMA_REG_PARK_PTR, reg); } - dma_write(chan, XILINX_DMA_REG_PARK_PTR, reg); /* Start the hardware */ xilinx_dma_start(chan); -- cgit v1.2.3-59-g8ed1b From 8d0191a6020e325a1c9730539dd2f0c03d71d9b4 Mon Sep 17 00:00:00 2001 From: Christophe JAILLET Date: Sat, 2 Nov 2024 12:46:04 +0100 Subject: dmaengine: idxd: Remove a useless mutex ida_alloc()/ida_free() don't need any mutex, so remove this one. It was introduced by commit e6fd6d7e5f0f ("dmaengine: idxd: add a device to represent the file opened"). Signed-off-by: Christophe JAILLET Reviewed-by: Fenghua Yu Reviewed-by: Dave Jiang Link: https://lore.kernel.org/r/e08df764e7046178ada4ec066852c0ce65410373.1730547933.git.christophe.jaillet@wanadoo.fr Signed-off-by: Vinod Koul --- drivers/dma/idxd/cdev.c | 5 ----- 1 file changed, 5 deletions(-) diff --git a/drivers/dma/idxd/cdev.c b/drivers/dma/idxd/cdev.c index 57f1bf2ab20b..ff94ee892339 100644 --- a/drivers/dma/idxd/cdev.c +++ b/drivers/dma/idxd/cdev.c @@ -28,7 +28,6 @@ struct idxd_cdev_context { * global to avoid conflict file names. */ static DEFINE_IDA(file_ida); -static DEFINE_MUTEX(ida_lock); /* * ictx is an array based off of accelerator types. enum idxd_type @@ -123,9 +122,7 @@ static void idxd_file_dev_release(struct device *dev) struct idxd_device *idxd = wq->idxd; int rc; - mutex_lock(&ida_lock); ida_free(&file_ida, ctx->id); - mutex_unlock(&ida_lock); /* Wait for in-flight operations to complete. */ if (wq_shared(wq)) { @@ -284,9 +281,7 @@ static int idxd_cdev_open(struct inode *inode, struct file *filp) } idxd_cdev = wq->idxd_cdev; - mutex_lock(&ida_lock); ctx->id = ida_alloc(&file_ida, GFP_KERNEL); - mutex_unlock(&ida_lock); if (ctx->id < 0) { dev_warn(dev, "ida alloc failure\n"); goto failed_ida; -- cgit v1.2.3-59-g8ed1b From e01ee7c660752923b2d0881fb20045ce79677258 Mon Sep 17 00:00:00 2001 From: Basavaraj Natikar Date: Fri, 25 Oct 2024 15:29:26 +0530 Subject: dmaengine: Move AMD PTDMA driver to amd directory PTDMA driver is the AMD DMA driver, and newer AMD platforms support newer DMA engines. Hence, move the current drivers to the AMD directory. This would also mean that future driver submissions to the AMD DMA driver will also land in the AMD-specific directory. Reviewed-by: Raju Rangoju Signed-off-by: Basavaraj Natikar Link: https://lore.kernel.org/r/20241025095931.726018-2-Basavaraj.Natikar@amd.com Signed-off-by: Vinod Koul --- MAINTAINERS | 4 +- drivers/dma/Kconfig | 2 - drivers/dma/Makefile | 1 - drivers/dma/amd/Kconfig | 13 + drivers/dma/amd/Makefile | 1 + drivers/dma/amd/ptdma/Makefile | 10 + drivers/dma/amd/ptdma/ptdma-debugfs.c | 106 ++++++++ drivers/dma/amd/ptdma/ptdma-dev.c | 309 ++++++++++++++++++++++++ drivers/dma/amd/ptdma/ptdma-dmaengine.c | 410 +++++++++++++++++++++++++++++++ drivers/dma/amd/ptdma/ptdma-pci.c | 243 +++++++++++++++++++ drivers/dma/amd/ptdma/ptdma.h | 337 ++++++++++++++++++++++++++ drivers/dma/ptdma/Kconfig | 13 - drivers/dma/ptdma/Makefile | 10 - drivers/dma/ptdma/ptdma-debugfs.c | 106 -------- drivers/dma/ptdma/ptdma-dev.c | 309 ------------------------ drivers/dma/ptdma/ptdma-dmaengine.c | 411 -------------------------------- drivers/dma/ptdma/ptdma-pci.c | 243 ------------------- drivers/dma/ptdma/ptdma.h | 337 -------------------------- 18 files changed, 1431 insertions(+), 1434 deletions(-) create mode 100644 drivers/dma/amd/ptdma/Makefile create mode 100644 drivers/dma/amd/ptdma/ptdma-debugfs.c create mode 100644 drivers/dma/amd/ptdma/ptdma-dev.c create mode 100644 drivers/dma/amd/ptdma/ptdma-dmaengine.c create mode 100644 drivers/dma/amd/ptdma/ptdma-pci.c create mode 100644 drivers/dma/amd/ptdma/ptdma.h delete mode 100644 drivers/dma/ptdma/Kconfig delete mode 100644 drivers/dma/ptdma/Makefile delete mode 100644 drivers/dma/ptdma/ptdma-debugfs.c delete mode 100644 drivers/dma/ptdma/ptdma-dev.c delete mode 100644 drivers/dma/ptdma/ptdma-dmaengine.c delete mode 100644 drivers/dma/ptdma/ptdma-pci.c delete mode 100644 drivers/dma/ptdma/ptdma.h diff --git a/MAINTAINERS b/MAINTAINERS index 1e930c7a58b1..b01420aa576d 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -1164,8 +1164,8 @@ F: tools/power/x86/amd_pstate_tracer/amd_pstate_trace.py AMD PTDMA DRIVER M: Basavaraj Natikar L: dmaengine@vger.kernel.org -S: Maintained -F: drivers/dma/ptdma/ +S: Supported +F: drivers/dma/amd/ptdma/ AMD QDMA DRIVER M: Nishad Saraf diff --git a/drivers/dma/Kconfig b/drivers/dma/Kconfig index e994d6e0779e..0cacfefaa2d9 100644 --- a/drivers/dma/Kconfig +++ b/drivers/dma/Kconfig @@ -740,8 +740,6 @@ source "drivers/dma/bestcomm/Kconfig" source "drivers/dma/mediatek/Kconfig" -source "drivers/dma/ptdma/Kconfig" - source "drivers/dma/qcom/Kconfig" source "drivers/dma/dw/Kconfig" diff --git a/drivers/dma/Makefile b/drivers/dma/Makefile index 5b2a52f4f2ee..19ba465011a6 100644 --- a/drivers/dma/Makefile +++ b/drivers/dma/Makefile @@ -16,7 +16,6 @@ obj-$(CONFIG_DMATEST) += dmatest.o obj-$(CONFIG_ALTERA_MSGDMA) += altera-msgdma.o obj-$(CONFIG_AMBA_PL08X) += amba-pl08x.o obj-$(CONFIG_AMCC_PPC440SPE_ADMA) += ppc4xx/ -obj-$(CONFIG_AMD_PTDMA) += ptdma/ obj-$(CONFIG_APPLE_ADMAC) += apple-admac.o obj-$(CONFIG_AT_HDMAC) += at_hdmac.o obj-$(CONFIG_AT_XDMAC) += at_xdmac.o diff --git a/drivers/dma/amd/Kconfig b/drivers/dma/amd/Kconfig index 7d1f51d69675..a09517d51449 100644 --- a/drivers/dma/amd/Kconfig +++ b/drivers/dma/amd/Kconfig @@ -1,4 +1,17 @@ # SPDX-License-Identifier: GPL-2.0-only +# +config AMD_PTDMA + tristate "AMD PassThru DMA Engine" + depends on X86_64 && PCI + select DMA_ENGINE + select DMA_VIRTUAL_CHANNELS + help + Enable support for the AMD PTDMA controller. This controller + provides DMA capabilities to perform high bandwidth memory to + memory and IO copy operations. It performs DMA transfer through + queue-based descriptor management. This DMA controller is intended + to be used with AMD Non-Transparent Bridge devices and not for + general purpose peripheral DMA. config AMD_QDMA tristate "AMD Queue-based DMA" diff --git a/drivers/dma/amd/Makefile b/drivers/dma/amd/Makefile index 37212be9364f..fb12f2f9e7b7 100644 --- a/drivers/dma/amd/Makefile +++ b/drivers/dma/amd/Makefile @@ -1,3 +1,4 @@ # SPDX-License-Identifier: GPL-2.0 +obj-$(CONFIG_AMD_PTDMA) += ptdma/ obj-$(CONFIG_AMD_QDMA) += qdma/ diff --git a/drivers/dma/amd/ptdma/Makefile b/drivers/dma/amd/ptdma/Makefile new file mode 100644 index 000000000000..ce5410268a9a --- /dev/null +++ b/drivers/dma/amd/ptdma/Makefile @@ -0,0 +1,10 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# AMD Passthru DMA driver +# + +obj-$(CONFIG_AMD_PTDMA) += ptdma.o + +ptdma-objs := ptdma-dev.o ptdma-dmaengine.o ptdma-debugfs.o + +ptdma-$(CONFIG_PCI) += ptdma-pci.o diff --git a/drivers/dma/amd/ptdma/ptdma-debugfs.c b/drivers/dma/amd/ptdma/ptdma-debugfs.c new file mode 100644 index 000000000000..c8307d3044a3 --- /dev/null +++ b/drivers/dma/amd/ptdma/ptdma-debugfs.c @@ -0,0 +1,106 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * AMD Passthrough DMA device driver + * -- Based on the CCP driver + * + * Copyright (C) 2016,2021 Advanced Micro Devices, Inc. + * + * Author: Sanjay R Mehta + * Author: Gary R Hook + */ + +#include +#include + +#include "ptdma.h" + +/* DebugFS helpers */ +#define RI_VERSION_NUM 0x0000003F + +#define RI_NUM_VQM 0x00078000 +#define RI_NVQM_SHIFT 15 + +static int pt_debugfs_info_show(struct seq_file *s, void *p) +{ + struct pt_device *pt = s->private; + unsigned int regval; + + seq_printf(s, "Device name: %s\n", dev_name(pt->dev)); + seq_printf(s, " # Queues: %d\n", 1); + seq_printf(s, " # Cmds: %d\n", pt->cmd_count); + + regval = ioread32(pt->io_regs + CMD_PT_VERSION); + + seq_printf(s, " Version: %d\n", regval & RI_VERSION_NUM); + seq_puts(s, " Engines:"); + seq_puts(s, "\n"); + seq_printf(s, " Queues: %d\n", (regval & RI_NUM_VQM) >> RI_NVQM_SHIFT); + + return 0; +} + +/* + * Return a formatted buffer containing the current + * statistics of queue for PTDMA + */ +static int pt_debugfs_stats_show(struct seq_file *s, void *p) +{ + struct pt_device *pt = s->private; + + seq_printf(s, "Total Interrupts Handled: %ld\n", pt->total_interrupts); + + return 0; +} + +static int pt_debugfs_queue_show(struct seq_file *s, void *p) +{ + struct pt_cmd_queue *cmd_q = s->private; + unsigned int regval; + + if (!cmd_q) + return 0; + + seq_printf(s, " Pass-Thru: %ld\n", cmd_q->total_pt_ops); + + regval = ioread32(cmd_q->reg_control + 0x000C); + + seq_puts(s, " Enabled Interrupts:"); + if (regval & INT_EMPTY_QUEUE) + seq_puts(s, " EMPTY"); + if (regval & INT_QUEUE_STOPPED) + seq_puts(s, " STOPPED"); + if (regval & INT_ERROR) + seq_puts(s, " ERROR"); + if (regval & INT_COMPLETION) + seq_puts(s, " COMPLETION"); + seq_puts(s, "\n"); + + return 0; +} + +DEFINE_SHOW_ATTRIBUTE(pt_debugfs_info); +DEFINE_SHOW_ATTRIBUTE(pt_debugfs_queue); +DEFINE_SHOW_ATTRIBUTE(pt_debugfs_stats); + +void ptdma_debugfs_setup(struct pt_device *pt) +{ + struct pt_cmd_queue *cmd_q; + struct dentry *debugfs_q_instance; + + if (!debugfs_initialized()) + return; + + debugfs_create_file("info", 0400, pt->dma_dev.dbg_dev_root, pt, + &pt_debugfs_info_fops); + + debugfs_create_file("stats", 0400, pt->dma_dev.dbg_dev_root, pt, + &pt_debugfs_stats_fops); + + cmd_q = &pt->cmd_q; + + debugfs_q_instance = + debugfs_create_dir("q", pt->dma_dev.dbg_dev_root); + + debugfs_create_file("stats", 0400, debugfs_q_instance, cmd_q, + &pt_debugfs_queue_fops); +} diff --git a/drivers/dma/amd/ptdma/ptdma-dev.c b/drivers/dma/amd/ptdma/ptdma-dev.c new file mode 100644 index 000000000000..a2bf13ff18b6 --- /dev/null +++ b/drivers/dma/amd/ptdma/ptdma-dev.c @@ -0,0 +1,309 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * AMD Passthru DMA device driver + * -- Based on the CCP driver + * + * Copyright (C) 2016,2021 Advanced Micro Devices, Inc. + * + * Author: Sanjay R Mehta + * Author: Gary R Hook + */ + +#include +#include +#include +#include +#include +#include +#include + +#include "ptdma.h" + +/* Human-readable error strings */ +static char *pt_error_codes[] = { + "", + "ERR 01: ILLEGAL_ENGINE", + "ERR 03: ILLEGAL_FUNCTION_TYPE", + "ERR 04: ILLEGAL_FUNCTION_MODE", + "ERR 06: ILLEGAL_FUNCTION_SIZE", + "ERR 08: ILLEGAL_FUNCTION_RSVD", + "ERR 09: ILLEGAL_BUFFER_LENGTH", + "ERR 10: VLSB_FAULT", + "ERR 11: ILLEGAL_MEM_ADDR", + "ERR 12: ILLEGAL_MEM_SEL", + "ERR 13: ILLEGAL_CONTEXT_ID", + "ERR 15: 0xF Reserved", + "ERR 18: CMD_TIMEOUT", + "ERR 19: IDMA0_AXI_SLVERR", + "ERR 20: IDMA0_AXI_DECERR", + "ERR 21: 0x15 Reserved", + "ERR 22: IDMA1_AXI_SLAVE_FAULT", + "ERR 23: IDMA1_AIXI_DECERR", + "ERR 24: 0x18 Reserved", + "ERR 27: 0x1B Reserved", + "ERR 38: ODMA0_AXI_SLVERR", + "ERR 39: ODMA0_AXI_DECERR", + "ERR 40: 0x28 Reserved", + "ERR 41: ODMA1_AXI_SLVERR", + "ERR 42: ODMA1_AXI_DECERR", + "ERR 43: LSB_PARITY_ERR", +}; + +static void pt_log_error(struct pt_device *d, int e) +{ + dev_err(d->dev, "PTDMA error: %s (0x%x)\n", pt_error_codes[e], e); +} + +void pt_start_queue(struct pt_cmd_queue *cmd_q) +{ + /* Turn on the run bit */ + iowrite32(cmd_q->qcontrol | CMD_Q_RUN, cmd_q->reg_control); +} + +void pt_stop_queue(struct pt_cmd_queue *cmd_q) +{ + /* Turn off the run bit */ + iowrite32(cmd_q->qcontrol & ~CMD_Q_RUN, cmd_q->reg_control); +} + +static int pt_core_execute_cmd(struct ptdma_desc *desc, struct pt_cmd_queue *cmd_q) +{ + bool soc = FIELD_GET(DWORD0_SOC, desc->dw0); + u8 *q_desc = (u8 *)&cmd_q->qbase[cmd_q->qidx]; + u32 tail; + unsigned long flags; + + if (soc) { + desc->dw0 |= FIELD_PREP(DWORD0_IOC, desc->dw0); + desc->dw0 &= ~DWORD0_SOC; + } + spin_lock_irqsave(&cmd_q->q_lock, flags); + + /* Copy 32-byte command descriptor to hw queue. */ + memcpy(q_desc, desc, 32); + cmd_q->qidx = (cmd_q->qidx + 1) % CMD_Q_LEN; + + /* The data used by this command must be flushed to memory */ + wmb(); + + /* Write the new tail address back to the queue register */ + tail = lower_32_bits(cmd_q->qdma_tail + cmd_q->qidx * Q_DESC_SIZE); + iowrite32(tail, cmd_q->reg_control + 0x0004); + + /* Turn the queue back on using our cached control register */ + pt_start_queue(cmd_q); + spin_unlock_irqrestore(&cmd_q->q_lock, flags); + + return 0; +} + +int pt_core_perform_passthru(struct pt_cmd_queue *cmd_q, + struct pt_passthru_engine *pt_engine) +{ + struct ptdma_desc desc; + struct pt_device *pt = container_of(cmd_q, struct pt_device, cmd_q); + + cmd_q->cmd_error = 0; + cmd_q->total_pt_ops++; + memset(&desc, 0, sizeof(desc)); + desc.dw0 = CMD_DESC_DW0_VAL; + desc.length = pt_engine->src_len; + desc.src_lo = lower_32_bits(pt_engine->src_dma); + desc.dw3.src_hi = upper_32_bits(pt_engine->src_dma); + desc.dst_lo = lower_32_bits(pt_engine->dst_dma); + desc.dw5.dst_hi = upper_32_bits(pt_engine->dst_dma); + + if (cmd_q->int_en) + pt_core_enable_queue_interrupts(pt); + else + pt_core_disable_queue_interrupts(pt); + + return pt_core_execute_cmd(&desc, cmd_q); +} + +static void pt_do_cmd_complete(unsigned long data) +{ + struct pt_tasklet_data *tdata = (struct pt_tasklet_data *)data; + struct pt_cmd *cmd = tdata->cmd; + struct pt_cmd_queue *cmd_q = &cmd->pt->cmd_q; + u32 tail; + + if (cmd_q->cmd_error) { + /* + * Log the error and flush the queue by + * moving the head pointer + */ + tail = lower_32_bits(cmd_q->qdma_tail + cmd_q->qidx * Q_DESC_SIZE); + pt_log_error(cmd_q->pt, cmd_q->cmd_error); + iowrite32(tail, cmd_q->reg_control + 0x0008); + } + + cmd->pt_cmd_callback(cmd->data, cmd->ret); +} + +void pt_check_status_trans(struct pt_device *pt, struct pt_cmd_queue *cmd_q) +{ + u32 status; + + status = ioread32(cmd_q->reg_control + 0x0010); + if (status) { + cmd_q->int_status = status; + cmd_q->q_status = ioread32(cmd_q->reg_control + 0x0100); + cmd_q->q_int_status = ioread32(cmd_q->reg_control + 0x0104); + + /* On error, only save the first error value */ + if ((status & INT_ERROR) && !cmd_q->cmd_error) + cmd_q->cmd_error = CMD_Q_ERROR(cmd_q->q_status); + + /* Acknowledge the completion */ + iowrite32(status, cmd_q->reg_control + 0x0010); + pt_do_cmd_complete((ulong)&pt->tdata); + } +} + +static irqreturn_t pt_core_irq_handler(int irq, void *data) +{ + struct pt_device *pt = data; + struct pt_cmd_queue *cmd_q = &pt->cmd_q; + + pt_core_disable_queue_interrupts(pt); + pt->total_interrupts++; + pt_check_status_trans(pt, cmd_q); + pt_core_enable_queue_interrupts(pt); + return IRQ_HANDLED; +} + +int pt_core_init(struct pt_device *pt) +{ + char dma_pool_name[MAX_DMAPOOL_NAME_LEN]; + struct pt_cmd_queue *cmd_q = &pt->cmd_q; + u32 dma_addr_lo, dma_addr_hi; + struct device *dev = pt->dev; + struct dma_pool *dma_pool; + int ret; + + /* Allocate a dma pool for the queue */ + snprintf(dma_pool_name, sizeof(dma_pool_name), "%s_q", dev_name(pt->dev)); + + dma_pool = dma_pool_create(dma_pool_name, dev, + PT_DMAPOOL_MAX_SIZE, + PT_DMAPOOL_ALIGN, 0); + if (!dma_pool) + return -ENOMEM; + + /* ptdma core initialisation */ + iowrite32(CMD_CONFIG_VHB_EN, pt->io_regs + CMD_CONFIG_OFFSET); + iowrite32(CMD_QUEUE_PRIO, pt->io_regs + CMD_QUEUE_PRIO_OFFSET); + iowrite32(CMD_TIMEOUT_DISABLE, pt->io_regs + CMD_TIMEOUT_OFFSET); + iowrite32(CMD_CLK_GATE_CONFIG, pt->io_regs + CMD_CLK_GATE_CTL_OFFSET); + iowrite32(CMD_CONFIG_REQID, pt->io_regs + CMD_REQID_CONFIG_OFFSET); + + cmd_q->pt = pt; + cmd_q->dma_pool = dma_pool; + spin_lock_init(&cmd_q->q_lock); + + /* Page alignment satisfies our needs for N <= 128 */ + cmd_q->qsize = Q_SIZE(Q_DESC_SIZE); + cmd_q->qbase = dma_alloc_coherent(dev, cmd_q->qsize, + &cmd_q->qbase_dma, + GFP_KERNEL); + if (!cmd_q->qbase) { + dev_err(dev, "unable to allocate command queue\n"); + ret = -ENOMEM; + goto e_destroy_pool; + } + + cmd_q->qidx = 0; + + /* Preset some register values */ + cmd_q->reg_control = pt->io_regs + CMD_Q_STATUS_INCR; + + /* Turn off the queues and disable interrupts until ready */ + pt_core_disable_queue_interrupts(pt); + + cmd_q->qcontrol = 0; /* Start with nothing */ + iowrite32(cmd_q->qcontrol, cmd_q->reg_control); + + ioread32(cmd_q->reg_control + 0x0104); + ioread32(cmd_q->reg_control + 0x0100); + + /* Clear the interrupt status */ + iowrite32(SUPPORTED_INTERRUPTS, cmd_q->reg_control + 0x0010); + + /* Request an irq */ + ret = request_irq(pt->pt_irq, pt_core_irq_handler, 0, dev_name(pt->dev), pt); + if (ret) { + dev_err(dev, "unable to allocate an IRQ\n"); + goto e_free_dma; + } + + /* Update the device registers with queue information. */ + cmd_q->qcontrol &= ~CMD_Q_SIZE; + cmd_q->qcontrol |= FIELD_PREP(CMD_Q_SIZE, QUEUE_SIZE_VAL); + + cmd_q->qdma_tail = cmd_q->qbase_dma; + dma_addr_lo = lower_32_bits(cmd_q->qdma_tail); + iowrite32((u32)dma_addr_lo, cmd_q->reg_control + 0x0004); + iowrite32((u32)dma_addr_lo, cmd_q->reg_control + 0x0008); + + dma_addr_hi = upper_32_bits(cmd_q->qdma_tail); + cmd_q->qcontrol |= (dma_addr_hi << 16); + iowrite32(cmd_q->qcontrol, cmd_q->reg_control); + + pt_core_enable_queue_interrupts(pt); + + /* Register the DMA engine support */ + ret = pt_dmaengine_register(pt); + if (ret) + goto e_free_irq; + + /* Set up debugfs entries */ + ptdma_debugfs_setup(pt); + + return 0; + +e_free_irq: + free_irq(pt->pt_irq, pt); + +e_free_dma: + dma_free_coherent(dev, cmd_q->qsize, cmd_q->qbase, cmd_q->qbase_dma); + +e_destroy_pool: + dma_pool_destroy(pt->cmd_q.dma_pool); + + return ret; +} + +void pt_core_destroy(struct pt_device *pt) +{ + struct device *dev = pt->dev; + struct pt_cmd_queue *cmd_q = &pt->cmd_q; + struct pt_cmd *cmd; + + /* Unregister the DMA engine */ + pt_dmaengine_unregister(pt); + + /* Disable and clear interrupts */ + pt_core_disable_queue_interrupts(pt); + + /* Turn off the run bit */ + pt_stop_queue(cmd_q); + + /* Clear the interrupt status */ + iowrite32(SUPPORTED_INTERRUPTS, cmd_q->reg_control + 0x0010); + ioread32(cmd_q->reg_control + 0x0104); + ioread32(cmd_q->reg_control + 0x0100); + + free_irq(pt->pt_irq, pt); + + dma_free_coherent(dev, cmd_q->qsize, cmd_q->qbase, + cmd_q->qbase_dma); + + /* Flush the cmd queue */ + while (!list_empty(&pt->cmd)) { + /* Invoke the callback directly with an error code */ + cmd = list_first_entry(&pt->cmd, struct pt_cmd, entry); + list_del(&cmd->entry); + cmd->pt_cmd_callback(cmd->data, -ENODEV); + } +} diff --git a/drivers/dma/amd/ptdma/ptdma-dmaengine.c b/drivers/dma/amd/ptdma/ptdma-dmaengine.c new file mode 100644 index 000000000000..a2e7c2cec15e --- /dev/null +++ b/drivers/dma/amd/ptdma/ptdma-dmaengine.c @@ -0,0 +1,410 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * AMD Passthrough DMA device driver + * -- Based on the CCP driver + * + * Copyright (C) 2016,2021 Advanced Micro Devices, Inc. + * + * Author: Sanjay R Mehta + * Author: Gary R Hook + */ + +#include "ptdma.h" +#include "../../dmaengine.h" + +static inline struct pt_dma_chan *to_pt_chan(struct dma_chan *dma_chan) +{ + return container_of(dma_chan, struct pt_dma_chan, vc.chan); +} + +static inline struct pt_dma_desc *to_pt_desc(struct virt_dma_desc *vd) +{ + return container_of(vd, struct pt_dma_desc, vd); +} + +static void pt_free_chan_resources(struct dma_chan *dma_chan) +{ + struct pt_dma_chan *chan = to_pt_chan(dma_chan); + + vchan_free_chan_resources(&chan->vc); +} + +static void pt_synchronize(struct dma_chan *dma_chan) +{ + struct pt_dma_chan *chan = to_pt_chan(dma_chan); + + vchan_synchronize(&chan->vc); +} + +static void pt_do_cleanup(struct virt_dma_desc *vd) +{ + struct pt_dma_desc *desc = to_pt_desc(vd); + struct pt_device *pt = desc->pt; + + kmem_cache_free(pt->dma_desc_cache, desc); +} + +static int pt_dma_start_desc(struct pt_dma_desc *desc) +{ + struct pt_passthru_engine *pt_engine; + struct pt_device *pt; + struct pt_cmd *pt_cmd; + struct pt_cmd_queue *cmd_q; + + desc->issued_to_hw = 1; + + pt_cmd = &desc->pt_cmd; + pt = pt_cmd->pt; + cmd_q = &pt->cmd_q; + pt_engine = &pt_cmd->passthru; + + pt->tdata.cmd = pt_cmd; + + /* Execute the command */ + pt_cmd->ret = pt_core_perform_passthru(cmd_q, pt_engine); + + return 0; +} + +static struct pt_dma_desc *pt_next_dma_desc(struct pt_dma_chan *chan) +{ + /* Get the next DMA descriptor on the active list */ + struct virt_dma_desc *vd = vchan_next_desc(&chan->vc); + + return vd ? to_pt_desc(vd) : NULL; +} + +static struct pt_dma_desc *pt_handle_active_desc(struct pt_dma_chan *chan, + struct pt_dma_desc *desc) +{ + struct dma_async_tx_descriptor *tx_desc; + struct virt_dma_desc *vd; + unsigned long flags; + + /* Loop over descriptors until one is found with commands */ + do { + if (desc) { + if (!desc->issued_to_hw) { + /* No errors, keep going */ + if (desc->status != DMA_ERROR) + return desc; + } + + tx_desc = &desc->vd.tx; + vd = &desc->vd; + } else { + tx_desc = NULL; + } + + spin_lock_irqsave(&chan->vc.lock, flags); + + if (desc) { + if (desc->status != DMA_COMPLETE) { + if (desc->status != DMA_ERROR) + desc->status = DMA_COMPLETE; + + dma_cookie_complete(tx_desc); + dma_descriptor_unmap(tx_desc); + list_del(&desc->vd.node); + } else { + /* Don't handle it twice */ + tx_desc = NULL; + } + } + + desc = pt_next_dma_desc(chan); + + spin_unlock_irqrestore(&chan->vc.lock, flags); + + if (tx_desc) { + dmaengine_desc_get_callback_invoke(tx_desc, NULL); + dma_run_dependencies(tx_desc); + vchan_vdesc_fini(vd); + } + } while (desc); + + return NULL; +} + +static void pt_cmd_callback(void *data, int err) +{ + struct pt_dma_desc *desc = data; + struct dma_chan *dma_chan; + struct pt_dma_chan *chan; + int ret; + + if (err == -EINPROGRESS) + return; + + dma_chan = desc->vd.tx.chan; + chan = to_pt_chan(dma_chan); + + if (err) + desc->status = DMA_ERROR; + + while (true) { + /* Check for DMA descriptor completion */ + desc = pt_handle_active_desc(chan, desc); + + /* Don't submit cmd if no descriptor or DMA is paused */ + if (!desc) + break; + + ret = pt_dma_start_desc(desc); + if (!ret) + break; + + desc->status = DMA_ERROR; + } +} + +static struct pt_dma_desc *pt_alloc_dma_desc(struct pt_dma_chan *chan, + unsigned long flags) +{ + struct pt_dma_desc *desc; + + desc = kmem_cache_zalloc(chan->pt->dma_desc_cache, GFP_NOWAIT); + if (!desc) + return NULL; + + vchan_tx_prep(&chan->vc, &desc->vd, flags); + + desc->pt = chan->pt; + desc->pt->cmd_q.int_en = !!(flags & DMA_PREP_INTERRUPT); + desc->issued_to_hw = 0; + desc->status = DMA_IN_PROGRESS; + + return desc; +} + +static struct pt_dma_desc *pt_create_desc(struct dma_chan *dma_chan, + dma_addr_t dst, + dma_addr_t src, + unsigned int len, + unsigned long flags) +{ + struct pt_dma_chan *chan = to_pt_chan(dma_chan); + struct pt_passthru_engine *pt_engine; + struct pt_dma_desc *desc; + struct pt_cmd *pt_cmd; + + desc = pt_alloc_dma_desc(chan, flags); + if (!desc) + return NULL; + + pt_cmd = &desc->pt_cmd; + pt_cmd->pt = chan->pt; + pt_engine = &pt_cmd->passthru; + pt_cmd->engine = PT_ENGINE_PASSTHRU; + pt_engine->src_dma = src; + pt_engine->dst_dma = dst; + pt_engine->src_len = len; + pt_cmd->pt_cmd_callback = pt_cmd_callback; + pt_cmd->data = desc; + + desc->len = len; + + return desc; +} + +static struct dma_async_tx_descriptor * +pt_prep_dma_memcpy(struct dma_chan *dma_chan, dma_addr_t dst, + dma_addr_t src, size_t len, unsigned long flags) +{ + struct pt_dma_desc *desc; + + desc = pt_create_desc(dma_chan, dst, src, len, flags); + if (!desc) + return NULL; + + return &desc->vd.tx; +} + +static struct dma_async_tx_descriptor * +pt_prep_dma_interrupt(struct dma_chan *dma_chan, unsigned long flags) +{ + struct pt_dma_chan *chan = to_pt_chan(dma_chan); + struct pt_dma_desc *desc; + + desc = pt_alloc_dma_desc(chan, flags); + if (!desc) + return NULL; + + return &desc->vd.tx; +} + +static void pt_issue_pending(struct dma_chan *dma_chan) +{ + struct pt_dma_chan *chan = to_pt_chan(dma_chan); + struct pt_dma_desc *desc; + unsigned long flags; + bool engine_is_idle = true; + + spin_lock_irqsave(&chan->vc.lock, flags); + + desc = pt_next_dma_desc(chan); + if (desc) + engine_is_idle = false; + + vchan_issue_pending(&chan->vc); + + desc = pt_next_dma_desc(chan); + + spin_unlock_irqrestore(&chan->vc.lock, flags); + + /* If there was nothing active, start processing */ + if (engine_is_idle && desc) + pt_cmd_callback(desc, 0); +} + +static enum dma_status +pt_tx_status(struct dma_chan *c, dma_cookie_t cookie, + struct dma_tx_state *txstate) +{ + struct pt_device *pt = to_pt_chan(c)->pt; + struct pt_cmd_queue *cmd_q = &pt->cmd_q; + + pt_check_status_trans(pt, cmd_q); + return dma_cookie_status(c, cookie, txstate); +} + +static int pt_pause(struct dma_chan *dma_chan) +{ + struct pt_dma_chan *chan = to_pt_chan(dma_chan); + unsigned long flags; + + spin_lock_irqsave(&chan->vc.lock, flags); + pt_stop_queue(&chan->pt->cmd_q); + spin_unlock_irqrestore(&chan->vc.lock, flags); + + return 0; +} + +static int pt_resume(struct dma_chan *dma_chan) +{ + struct pt_dma_chan *chan = to_pt_chan(dma_chan); + struct pt_dma_desc *desc = NULL; + unsigned long flags; + + spin_lock_irqsave(&chan->vc.lock, flags); + pt_start_queue(&chan->pt->cmd_q); + desc = pt_next_dma_desc(chan); + spin_unlock_irqrestore(&chan->vc.lock, flags); + + /* If there was something active, re-start */ + if (desc) + pt_cmd_callback(desc, 0); + + return 0; +} + +static int pt_terminate_all(struct dma_chan *dma_chan) +{ + struct pt_dma_chan *chan = to_pt_chan(dma_chan); + unsigned long flags; + struct pt_cmd_queue *cmd_q = &chan->pt->cmd_q; + LIST_HEAD(head); + + iowrite32(SUPPORTED_INTERRUPTS, cmd_q->reg_control + 0x0010); + spin_lock_irqsave(&chan->vc.lock, flags); + vchan_get_all_descriptors(&chan->vc, &head); + spin_unlock_irqrestore(&chan->vc.lock, flags); + + vchan_dma_desc_free_list(&chan->vc, &head); + vchan_free_chan_resources(&chan->vc); + + return 0; +} + +int pt_dmaengine_register(struct pt_device *pt) +{ + struct pt_dma_chan *chan; + struct dma_device *dma_dev = &pt->dma_dev; + char *cmd_cache_name; + char *desc_cache_name; + int ret; + + pt->pt_dma_chan = devm_kzalloc(pt->dev, sizeof(*pt->pt_dma_chan), + GFP_KERNEL); + if (!pt->pt_dma_chan) + return -ENOMEM; + + cmd_cache_name = devm_kasprintf(pt->dev, GFP_KERNEL, + "%s-dmaengine-cmd-cache", + dev_name(pt->dev)); + if (!cmd_cache_name) + return -ENOMEM; + + desc_cache_name = devm_kasprintf(pt->dev, GFP_KERNEL, + "%s-dmaengine-desc-cache", + dev_name(pt->dev)); + if (!desc_cache_name) { + ret = -ENOMEM; + goto err_cache; + } + + pt->dma_desc_cache = kmem_cache_create(desc_cache_name, + sizeof(struct pt_dma_desc), 0, + SLAB_HWCACHE_ALIGN, NULL); + if (!pt->dma_desc_cache) { + ret = -ENOMEM; + goto err_cache; + } + + dma_dev->dev = pt->dev; + dma_dev->src_addr_widths = DMA_SLAVE_BUSWIDTH_64_BYTES; + dma_dev->dst_addr_widths = DMA_SLAVE_BUSWIDTH_64_BYTES; + dma_dev->directions = DMA_MEM_TO_MEM; + dma_dev->residue_granularity = DMA_RESIDUE_GRANULARITY_DESCRIPTOR; + dma_cap_set(DMA_MEMCPY, dma_dev->cap_mask); + dma_cap_set(DMA_INTERRUPT, dma_dev->cap_mask); + + /* + * PTDMA is intended to be used with the AMD NTB devices, hence + * marking it as DMA_PRIVATE. + */ + dma_cap_set(DMA_PRIVATE, dma_dev->cap_mask); + + INIT_LIST_HEAD(&dma_dev->channels); + + chan = pt->pt_dma_chan; + chan->pt = pt; + + /* Set base and prep routines */ + dma_dev->device_free_chan_resources = pt_free_chan_resources; + dma_dev->device_prep_dma_memcpy = pt_prep_dma_memcpy; + dma_dev->device_prep_dma_interrupt = pt_prep_dma_interrupt; + dma_dev->device_issue_pending = pt_issue_pending; + dma_dev->device_tx_status = pt_tx_status; + dma_dev->device_pause = pt_pause; + dma_dev->device_resume = pt_resume; + dma_dev->device_terminate_all = pt_terminate_all; + dma_dev->device_synchronize = pt_synchronize; + + chan->vc.desc_free = pt_do_cleanup; + vchan_init(&chan->vc, dma_dev); + + ret = dma_async_device_register(dma_dev); + if (ret) + goto err_reg; + + return 0; + +err_reg: + kmem_cache_destroy(pt->dma_desc_cache); + +err_cache: + kmem_cache_destroy(pt->dma_cmd_cache); + + return ret; +} + +void pt_dmaengine_unregister(struct pt_device *pt) +{ + struct dma_device *dma_dev = &pt->dma_dev; + + dma_async_device_unregister(dma_dev); + + kmem_cache_destroy(pt->dma_desc_cache); + kmem_cache_destroy(pt->dma_cmd_cache); +} diff --git a/drivers/dma/amd/ptdma/ptdma-pci.c b/drivers/dma/amd/ptdma/ptdma-pci.c new file mode 100644 index 000000000000..22739ff0c3c5 --- /dev/null +++ b/drivers/dma/amd/ptdma/ptdma-pci.c @@ -0,0 +1,243 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * AMD Passthru DMA device driver + * -- Based on the CCP driver + * + * Copyright (C) 2016,2021 Advanced Micro Devices, Inc. + * + * Author: Sanjay R Mehta + * Author: Tom Lendacky + * Author: Gary R Hook + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "ptdma.h" + +struct pt_msix { + int msix_count; + struct msix_entry msix_entry; +}; + +/* + * pt_alloc_struct - allocate and initialize the pt_device struct + * + * @dev: device struct of the PTDMA + */ +static struct pt_device *pt_alloc_struct(struct device *dev) +{ + struct pt_device *pt; + + pt = devm_kzalloc(dev, sizeof(*pt), GFP_KERNEL); + + if (!pt) + return NULL; + pt->dev = dev; + + INIT_LIST_HEAD(&pt->cmd); + + return pt; +} + +static int pt_get_msix_irqs(struct pt_device *pt) +{ + struct pt_msix *pt_msix = pt->pt_msix; + struct device *dev = pt->dev; + struct pci_dev *pdev = to_pci_dev(dev); + int ret; + + pt_msix->msix_entry.entry = 0; + + ret = pci_enable_msix_range(pdev, &pt_msix->msix_entry, 1, 1); + if (ret < 0) + return ret; + + pt_msix->msix_count = ret; + + pt->pt_irq = pt_msix->msix_entry.vector; + + return 0; +} + +static int pt_get_msi_irq(struct pt_device *pt) +{ + struct device *dev = pt->dev; + struct pci_dev *pdev = to_pci_dev(dev); + int ret; + + ret = pci_enable_msi(pdev); + if (ret) + return ret; + + pt->pt_irq = pdev->irq; + + return 0; +} + +static int pt_get_irqs(struct pt_device *pt) +{ + struct device *dev = pt->dev; + int ret; + + ret = pt_get_msix_irqs(pt); + if (!ret) + return 0; + + /* Couldn't get MSI-X vectors, try MSI */ + dev_err(dev, "could not enable MSI-X (%d), trying MSI\n", ret); + ret = pt_get_msi_irq(pt); + if (!ret) + return 0; + + /* Couldn't get MSI interrupt */ + dev_err(dev, "could not enable MSI (%d)\n", ret); + + return ret; +} + +static void pt_free_irqs(struct pt_device *pt) +{ + struct pt_msix *pt_msix = pt->pt_msix; + struct device *dev = pt->dev; + struct pci_dev *pdev = to_pci_dev(dev); + + if (pt_msix->msix_count) + pci_disable_msix(pdev); + else if (pt->pt_irq) + pci_disable_msi(pdev); + + pt->pt_irq = 0; +} + +static int pt_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) +{ + struct pt_device *pt; + struct pt_msix *pt_msix; + struct device *dev = &pdev->dev; + void __iomem * const *iomap_table; + int bar_mask; + int ret = -ENOMEM; + + pt = pt_alloc_struct(dev); + if (!pt) + goto e_err; + + pt_msix = devm_kzalloc(dev, sizeof(*pt_msix), GFP_KERNEL); + if (!pt_msix) + goto e_err; + + pt->pt_msix = pt_msix; + pt->dev_vdata = (struct pt_dev_vdata *)id->driver_data; + if (!pt->dev_vdata) { + ret = -ENODEV; + dev_err(dev, "missing driver data\n"); + goto e_err; + } + + ret = pcim_enable_device(pdev); + if (ret) { + dev_err(dev, "pcim_enable_device failed (%d)\n", ret); + goto e_err; + } + + bar_mask = pci_select_bars(pdev, IORESOURCE_MEM); + ret = pcim_iomap_regions(pdev, bar_mask, "ptdma"); + if (ret) { + dev_err(dev, "pcim_iomap_regions failed (%d)\n", ret); + goto e_err; + } + + iomap_table = pcim_iomap_table(pdev); + if (!iomap_table) { + dev_err(dev, "pcim_iomap_table failed\n"); + ret = -ENOMEM; + goto e_err; + } + + pt->io_regs = iomap_table[pt->dev_vdata->bar]; + if (!pt->io_regs) { + dev_err(dev, "ioremap failed\n"); + ret = -ENOMEM; + goto e_err; + } + + ret = pt_get_irqs(pt); + if (ret) + goto e_err; + + pci_set_master(pdev); + + ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(48)); + if (ret) { + ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(32)); + if (ret) { + dev_err(dev, "dma_set_mask_and_coherent failed (%d)\n", + ret); + goto e_err; + } + } + + dev_set_drvdata(dev, pt); + + if (pt->dev_vdata) + ret = pt_core_init(pt); + + if (ret) + goto e_err; + + return 0; + +e_err: + dev_err(dev, "initialization failed ret = %d\n", ret); + + return ret; +} + +static void pt_pci_remove(struct pci_dev *pdev) +{ + struct device *dev = &pdev->dev; + struct pt_device *pt = dev_get_drvdata(dev); + + if (!pt) + return; + + if (pt->dev_vdata) + pt_core_destroy(pt); + + pt_free_irqs(pt); +} + +static const struct pt_dev_vdata dev_vdata[] = { + { + .bar = 2, + }, +}; + +static const struct pci_device_id pt_pci_table[] = { + { PCI_VDEVICE(AMD, 0x1498), (kernel_ulong_t)&dev_vdata[0] }, + /* Last entry must be zero */ + { 0, } +}; +MODULE_DEVICE_TABLE(pci, pt_pci_table); + +static struct pci_driver pt_pci_driver = { + .name = "ptdma", + .id_table = pt_pci_table, + .probe = pt_pci_probe, + .remove = pt_pci_remove, +}; + +module_pci_driver(pt_pci_driver); + +MODULE_AUTHOR("Sanjay R Mehta "); +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("AMD PassThru DMA driver"); diff --git a/drivers/dma/amd/ptdma/ptdma.h b/drivers/dma/amd/ptdma/ptdma.h new file mode 100644 index 000000000000..7a8ca8e239e0 --- /dev/null +++ b/drivers/dma/amd/ptdma/ptdma.h @@ -0,0 +1,337 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * AMD Passthru DMA device driver + * -- Based on the CCP driver + * + * Copyright (C) 2016,2021 Advanced Micro Devices, Inc. + * + * Author: Sanjay R Mehta + * Author: Tom Lendacky + * Author: Gary R Hook + */ + +#ifndef __PT_DEV_H__ +#define __PT_DEV_H__ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "../../virt-dma.h" + +#define MAX_PT_NAME_LEN 16 +#define MAX_DMAPOOL_NAME_LEN 32 + +#define MAX_HW_QUEUES 1 +#define MAX_CMD_QLEN 100 + +#define PT_ENGINE_PASSTHRU 5 + +/* Register Mappings */ +#define IRQ_MASK_REG 0x040 +#define IRQ_STATUS_REG 0x200 + +#define CMD_Q_ERROR(__qs) ((__qs) & 0x0000003f) + +#define CMD_QUEUE_PRIO_OFFSET 0x00 +#define CMD_REQID_CONFIG_OFFSET 0x04 +#define CMD_TIMEOUT_OFFSET 0x08 +#define CMD_PT_VERSION 0x10 + +#define CMD_Q_CONTROL_BASE 0x0000 +#define CMD_Q_TAIL_LO_BASE 0x0004 +#define CMD_Q_HEAD_LO_BASE 0x0008 +#define CMD_Q_INT_ENABLE_BASE 0x000C +#define CMD_Q_INTERRUPT_STATUS_BASE 0x0010 + +#define CMD_Q_STATUS_BASE 0x0100 +#define CMD_Q_INT_STATUS_BASE 0x0104 +#define CMD_Q_DMA_STATUS_BASE 0x0108 +#define CMD_Q_DMA_READ_STATUS_BASE 0x010C +#define CMD_Q_DMA_WRITE_STATUS_BASE 0x0110 +#define CMD_Q_ABORT_BASE 0x0114 +#define CMD_Q_AX_CACHE_BASE 0x0118 + +#define CMD_CONFIG_OFFSET 0x1120 +#define CMD_CLK_GATE_CTL_OFFSET 0x6004 + +#define CMD_DESC_DW0_VAL 0x500012 + +/* Address offset for virtual queue registers */ +#define CMD_Q_STATUS_INCR 0x1000 + +/* Bit masks */ +#define CMD_CONFIG_REQID 0 +#define CMD_TIMEOUT_DISABLE 0 +#define CMD_CLK_DYN_GATING_DIS 0 +#define CMD_CLK_SW_GATE_MODE 0 +#define CMD_CLK_GATE_CTL 0 +#define CMD_QUEUE_PRIO GENMASK(2, 1) +#define CMD_CONFIG_VHB_EN BIT(0) +#define CMD_CLK_DYN_GATING_EN BIT(0) +#define CMD_CLK_HW_GATE_MODE BIT(0) +#define CMD_CLK_GATE_ON_DELAY BIT(12) +#define CMD_CLK_GATE_OFF_DELAY BIT(12) + +#define CMD_CLK_GATE_CONFIG (CMD_CLK_GATE_CTL | \ + CMD_CLK_HW_GATE_MODE | \ + CMD_CLK_GATE_ON_DELAY | \ + CMD_CLK_DYN_GATING_EN | \ + CMD_CLK_GATE_OFF_DELAY) + +#define CMD_Q_LEN 32 +#define CMD_Q_RUN BIT(0) +#define CMD_Q_HALT BIT(1) +#define CMD_Q_MEM_LOCATION BIT(2) +#define CMD_Q_SIZE_MASK GENMASK(4, 0) +#define CMD_Q_SIZE GENMASK(7, 3) +#define CMD_Q_SHIFT GENMASK(1, 0) +#define QUEUE_SIZE_VAL ((ffs(CMD_Q_LEN) - 2) & \ + CMD_Q_SIZE_MASK) +#define Q_PTR_MASK (2 << (QUEUE_SIZE_VAL + 5) - 1) +#define Q_DESC_SIZE sizeof(struct ptdma_desc) +#define Q_SIZE(n) (CMD_Q_LEN * (n)) + +#define INT_COMPLETION BIT(0) +#define INT_ERROR BIT(1) +#define INT_QUEUE_STOPPED BIT(2) +#define INT_EMPTY_QUEUE BIT(3) +#define SUPPORTED_INTERRUPTS (INT_COMPLETION | INT_ERROR) + +/****** Local Storage Block ******/ +#define LSB_START 0 +#define LSB_END 127 +#define LSB_COUNT (LSB_END - LSB_START + 1) + +#define PT_DMAPOOL_MAX_SIZE 64 +#define PT_DMAPOOL_ALIGN BIT(5) + +#define PT_PASSTHRU_BLOCKSIZE 512 + +struct pt_device; + +struct pt_tasklet_data { + struct completion completion; + struct pt_cmd *cmd; +}; + +/* + * struct pt_passthru_engine - pass-through operation + * without performing DMA mapping + * @mask: mask to be applied to data + * @mask_len: length in bytes of mask + * @src_dma: data to be used for this operation + * @dst_dma: data produced by this operation + * @src_len: length in bytes of data used for this operation + * + * Variables required to be set when calling pt_enqueue_cmd(): + * - bit_mod, byte_swap, src, dst, src_len + * - mask, mask_len if bit_mod is not PT_PASSTHRU_BITWISE_NOOP + */ +struct pt_passthru_engine { + dma_addr_t mask; + u32 mask_len; /* In bytes */ + + dma_addr_t src_dma, dst_dma; + u64 src_len; /* In bytes */ +}; + +/* + * struct pt_cmd - PTDMA operation request + * @entry: list element + * @work: work element used for callbacks + * @pt: PT device to be run on + * @ret: operation return code + * @flags: cmd processing flags + * @engine: PTDMA operation to perform (passthru) + * @engine_error: PT engine return code + * @passthru: engine specific structures, refer to specific engine struct below + * @callback: operation completion callback function + * @data: parameter value to be supplied to the callback function + * + * Variables required to be set when calling pt_enqueue_cmd(): + * - engine, callback + * - See the operation structures below for what is required for each + * operation. + */ +struct pt_cmd { + struct list_head entry; + struct work_struct work; + struct pt_device *pt; + int ret; + u32 engine; + u32 engine_error; + struct pt_passthru_engine passthru; + /* Completion callback support */ + void (*pt_cmd_callback)(void *data, int err); + void *data; +}; + +struct pt_dma_desc { + struct virt_dma_desc vd; + struct pt_device *pt; + enum dma_status status; + size_t len; + bool issued_to_hw; + struct pt_cmd pt_cmd; +}; + +struct pt_dma_chan { + struct virt_dma_chan vc; + struct pt_device *pt; +}; + +struct pt_cmd_queue { + struct pt_device *pt; + + /* Queue dma pool */ + struct dma_pool *dma_pool; + + /* Queue base address (not necessarily aligned)*/ + struct ptdma_desc *qbase; + + /* Aligned queue start address (per requirement) */ + spinlock_t q_lock ____cacheline_aligned; + unsigned int qidx; + + unsigned int qsize; + dma_addr_t qbase_dma; + dma_addr_t qdma_tail; + + unsigned int active; + unsigned int suspended; + + /* Interrupt flag */ + bool int_en; + + /* Register addresses for queue */ + void __iomem *reg_control; + u32 qcontrol; /* Cached control register */ + + /* Status values from job */ + u32 int_status; + u32 q_status; + u32 q_int_status; + u32 cmd_error; + /* Queue Statistics */ + unsigned long total_pt_ops; +} ____cacheline_aligned; + +struct pt_device { + struct list_head entry; + + unsigned int ord; + char name[MAX_PT_NAME_LEN]; + + struct device *dev; + + /* Bus specific device information */ + struct pt_msix *pt_msix; + + struct pt_dev_vdata *dev_vdata; + + unsigned int pt_irq; + + /* I/O area used for device communication */ + void __iomem *io_regs; + + spinlock_t cmd_lock ____cacheline_aligned; + unsigned int cmd_count; + struct list_head cmd; + + /* + * The command queue. This represent the queue available on the + * PTDMA that are available for processing cmds + */ + struct pt_cmd_queue cmd_q; + + /* Support for the DMA Engine capabilities */ + struct dma_device dma_dev; + struct pt_dma_chan *pt_dma_chan; + struct kmem_cache *dma_cmd_cache; + struct kmem_cache *dma_desc_cache; + + wait_queue_head_t lsb_queue; + + /* Device Statistics */ + unsigned long total_interrupts; + + struct pt_tasklet_data tdata; +}; + +/* + * descriptor for PTDMA commands + * 8 32-bit words: + * word 0: function; engine; control bits + * word 1: length of source data + * word 2: low 32 bits of source pointer + * word 3: upper 16 bits of source pointer; source memory type + * word 4: low 32 bits of destination pointer + * word 5: upper 16 bits of destination pointer; destination memory type + * word 6: reserved 32 bits + * word 7: reserved 32 bits + */ + +#define DWORD0_SOC BIT(0) +#define DWORD0_IOC BIT(1) + +struct dword3 { + unsigned int src_hi:16; + unsigned int src_mem:2; + unsigned int lsb_cxt_id:8; + unsigned int rsvd1:5; + unsigned int fixed:1; +}; + +struct dword5 { + unsigned int dst_hi:16; + unsigned int dst_mem:2; + unsigned int rsvd1:13; + unsigned int fixed:1; +}; + +struct ptdma_desc { + u32 dw0; + u32 length; + u32 src_lo; + struct dword3 dw3; + u32 dst_lo; + struct dword5 dw5; + __le32 rsvd1; + __le32 rsvd2; +}; + +/* Structure to hold PT device data */ +struct pt_dev_vdata { + const unsigned int bar; +}; + +int pt_dmaengine_register(struct pt_device *pt); +void pt_dmaengine_unregister(struct pt_device *pt); + +void ptdma_debugfs_setup(struct pt_device *pt); +int pt_core_init(struct pt_device *pt); +void pt_core_destroy(struct pt_device *pt); + +int pt_core_perform_passthru(struct pt_cmd_queue *cmd_q, + struct pt_passthru_engine *pt_engine); + +void pt_check_status_trans(struct pt_device *pt, struct pt_cmd_queue *cmd_q); +void pt_start_queue(struct pt_cmd_queue *cmd_q); +void pt_stop_queue(struct pt_cmd_queue *cmd_q); + +static inline void pt_core_disable_queue_interrupts(struct pt_device *pt) +{ + iowrite32(0, pt->cmd_q.reg_control + 0x000C); +} + +static inline void pt_core_enable_queue_interrupts(struct pt_device *pt) +{ + iowrite32(SUPPORTED_INTERRUPTS, pt->cmd_q.reg_control + 0x000C); +} +#endif diff --git a/drivers/dma/ptdma/Kconfig b/drivers/dma/ptdma/Kconfig deleted file mode 100644 index b430edd709f9..000000000000 --- a/drivers/dma/ptdma/Kconfig +++ /dev/null @@ -1,13 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0-only -config AMD_PTDMA - tristate "AMD PassThru DMA Engine" - depends on X86_64 && PCI - select DMA_ENGINE - select DMA_VIRTUAL_CHANNELS - help - Enable support for the AMD PTDMA controller. This controller - provides DMA capabilities to perform high bandwidth memory to - memory and IO copy operations. It performs DMA transfer through - queue-based descriptor management. This DMA controller is intended - to be used with AMD Non-Transparent Bridge devices and not for - general purpose peripheral DMA. diff --git a/drivers/dma/ptdma/Makefile b/drivers/dma/ptdma/Makefile deleted file mode 100644 index ce5410268a9a..000000000000 --- a/drivers/dma/ptdma/Makefile +++ /dev/null @@ -1,10 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0-only -# -# AMD Passthru DMA driver -# - -obj-$(CONFIG_AMD_PTDMA) += ptdma.o - -ptdma-objs := ptdma-dev.o ptdma-dmaengine.o ptdma-debugfs.o - -ptdma-$(CONFIG_PCI) += ptdma-pci.o diff --git a/drivers/dma/ptdma/ptdma-debugfs.c b/drivers/dma/ptdma/ptdma-debugfs.c deleted file mode 100644 index c8307d3044a3..000000000000 --- a/drivers/dma/ptdma/ptdma-debugfs.c +++ /dev/null @@ -1,106 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * AMD Passthrough DMA device driver - * -- Based on the CCP driver - * - * Copyright (C) 2016,2021 Advanced Micro Devices, Inc. - * - * Author: Sanjay R Mehta - * Author: Gary R Hook - */ - -#include -#include - -#include "ptdma.h" - -/* DebugFS helpers */ -#define RI_VERSION_NUM 0x0000003F - -#define RI_NUM_VQM 0x00078000 -#define RI_NVQM_SHIFT 15 - -static int pt_debugfs_info_show(struct seq_file *s, void *p) -{ - struct pt_device *pt = s->private; - unsigned int regval; - - seq_printf(s, "Device name: %s\n", dev_name(pt->dev)); - seq_printf(s, " # Queues: %d\n", 1); - seq_printf(s, " # Cmds: %d\n", pt->cmd_count); - - regval = ioread32(pt->io_regs + CMD_PT_VERSION); - - seq_printf(s, " Version: %d\n", regval & RI_VERSION_NUM); - seq_puts(s, " Engines:"); - seq_puts(s, "\n"); - seq_printf(s, " Queues: %d\n", (regval & RI_NUM_VQM) >> RI_NVQM_SHIFT); - - return 0; -} - -/* - * Return a formatted buffer containing the current - * statistics of queue for PTDMA - */ -static int pt_debugfs_stats_show(struct seq_file *s, void *p) -{ - struct pt_device *pt = s->private; - - seq_printf(s, "Total Interrupts Handled: %ld\n", pt->total_interrupts); - - return 0; -} - -static int pt_debugfs_queue_show(struct seq_file *s, void *p) -{ - struct pt_cmd_queue *cmd_q = s->private; - unsigned int regval; - - if (!cmd_q) - return 0; - - seq_printf(s, " Pass-Thru: %ld\n", cmd_q->total_pt_ops); - - regval = ioread32(cmd_q->reg_control + 0x000C); - - seq_puts(s, " Enabled Interrupts:"); - if (regval & INT_EMPTY_QUEUE) - seq_puts(s, " EMPTY"); - if (regval & INT_QUEUE_STOPPED) - seq_puts(s, " STOPPED"); - if (regval & INT_ERROR) - seq_puts(s, " ERROR"); - if (regval & INT_COMPLETION) - seq_puts(s, " COMPLETION"); - seq_puts(s, "\n"); - - return 0; -} - -DEFINE_SHOW_ATTRIBUTE(pt_debugfs_info); -DEFINE_SHOW_ATTRIBUTE(pt_debugfs_queue); -DEFINE_SHOW_ATTRIBUTE(pt_debugfs_stats); - -void ptdma_debugfs_setup(struct pt_device *pt) -{ - struct pt_cmd_queue *cmd_q; - struct dentry *debugfs_q_instance; - - if (!debugfs_initialized()) - return; - - debugfs_create_file("info", 0400, pt->dma_dev.dbg_dev_root, pt, - &pt_debugfs_info_fops); - - debugfs_create_file("stats", 0400, pt->dma_dev.dbg_dev_root, pt, - &pt_debugfs_stats_fops); - - cmd_q = &pt->cmd_q; - - debugfs_q_instance = - debugfs_create_dir("q", pt->dma_dev.dbg_dev_root); - - debugfs_create_file("stats", 0400, debugfs_q_instance, cmd_q, - &pt_debugfs_queue_fops); -} diff --git a/drivers/dma/ptdma/ptdma-dev.c b/drivers/dma/ptdma/ptdma-dev.c deleted file mode 100644 index a2bf13ff18b6..000000000000 --- a/drivers/dma/ptdma/ptdma-dev.c +++ /dev/null @@ -1,309 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * AMD Passthru DMA device driver - * -- Based on the CCP driver - * - * Copyright (C) 2016,2021 Advanced Micro Devices, Inc. - * - * Author: Sanjay R Mehta - * Author: Gary R Hook - */ - -#include -#include -#include -#include -#include -#include -#include - -#include "ptdma.h" - -/* Human-readable error strings */ -static char *pt_error_codes[] = { - "", - "ERR 01: ILLEGAL_ENGINE", - "ERR 03: ILLEGAL_FUNCTION_TYPE", - "ERR 04: ILLEGAL_FUNCTION_MODE", - "ERR 06: ILLEGAL_FUNCTION_SIZE", - "ERR 08: ILLEGAL_FUNCTION_RSVD", - "ERR 09: ILLEGAL_BUFFER_LENGTH", - "ERR 10: VLSB_FAULT", - "ERR 11: ILLEGAL_MEM_ADDR", - "ERR 12: ILLEGAL_MEM_SEL", - "ERR 13: ILLEGAL_CONTEXT_ID", - "ERR 15: 0xF Reserved", - "ERR 18: CMD_TIMEOUT", - "ERR 19: IDMA0_AXI_SLVERR", - "ERR 20: IDMA0_AXI_DECERR", - "ERR 21: 0x15 Reserved", - "ERR 22: IDMA1_AXI_SLAVE_FAULT", - "ERR 23: IDMA1_AIXI_DECERR", - "ERR 24: 0x18 Reserved", - "ERR 27: 0x1B Reserved", - "ERR 38: ODMA0_AXI_SLVERR", - "ERR 39: ODMA0_AXI_DECERR", - "ERR 40: 0x28 Reserved", - "ERR 41: ODMA1_AXI_SLVERR", - "ERR 42: ODMA1_AXI_DECERR", - "ERR 43: LSB_PARITY_ERR", -}; - -static void pt_log_error(struct pt_device *d, int e) -{ - dev_err(d->dev, "PTDMA error: %s (0x%x)\n", pt_error_codes[e], e); -} - -void pt_start_queue(struct pt_cmd_queue *cmd_q) -{ - /* Turn on the run bit */ - iowrite32(cmd_q->qcontrol | CMD_Q_RUN, cmd_q->reg_control); -} - -void pt_stop_queue(struct pt_cmd_queue *cmd_q) -{ - /* Turn off the run bit */ - iowrite32(cmd_q->qcontrol & ~CMD_Q_RUN, cmd_q->reg_control); -} - -static int pt_core_execute_cmd(struct ptdma_desc *desc, struct pt_cmd_queue *cmd_q) -{ - bool soc = FIELD_GET(DWORD0_SOC, desc->dw0); - u8 *q_desc = (u8 *)&cmd_q->qbase[cmd_q->qidx]; - u32 tail; - unsigned long flags; - - if (soc) { - desc->dw0 |= FIELD_PREP(DWORD0_IOC, desc->dw0); - desc->dw0 &= ~DWORD0_SOC; - } - spin_lock_irqsave(&cmd_q->q_lock, flags); - - /* Copy 32-byte command descriptor to hw queue. */ - memcpy(q_desc, desc, 32); - cmd_q->qidx = (cmd_q->qidx + 1) % CMD_Q_LEN; - - /* The data used by this command must be flushed to memory */ - wmb(); - - /* Write the new tail address back to the queue register */ - tail = lower_32_bits(cmd_q->qdma_tail + cmd_q->qidx * Q_DESC_SIZE); - iowrite32(tail, cmd_q->reg_control + 0x0004); - - /* Turn the queue back on using our cached control register */ - pt_start_queue(cmd_q); - spin_unlock_irqrestore(&cmd_q->q_lock, flags); - - return 0; -} - -int pt_core_perform_passthru(struct pt_cmd_queue *cmd_q, - struct pt_passthru_engine *pt_engine) -{ - struct ptdma_desc desc; - struct pt_device *pt = container_of(cmd_q, struct pt_device, cmd_q); - - cmd_q->cmd_error = 0; - cmd_q->total_pt_ops++; - memset(&desc, 0, sizeof(desc)); - desc.dw0 = CMD_DESC_DW0_VAL; - desc.length = pt_engine->src_len; - desc.src_lo = lower_32_bits(pt_engine->src_dma); - desc.dw3.src_hi = upper_32_bits(pt_engine->src_dma); - desc.dst_lo = lower_32_bits(pt_engine->dst_dma); - desc.dw5.dst_hi = upper_32_bits(pt_engine->dst_dma); - - if (cmd_q->int_en) - pt_core_enable_queue_interrupts(pt); - else - pt_core_disable_queue_interrupts(pt); - - return pt_core_execute_cmd(&desc, cmd_q); -} - -static void pt_do_cmd_complete(unsigned long data) -{ - struct pt_tasklet_data *tdata = (struct pt_tasklet_data *)data; - struct pt_cmd *cmd = tdata->cmd; - struct pt_cmd_queue *cmd_q = &cmd->pt->cmd_q; - u32 tail; - - if (cmd_q->cmd_error) { - /* - * Log the error and flush the queue by - * moving the head pointer - */ - tail = lower_32_bits(cmd_q->qdma_tail + cmd_q->qidx * Q_DESC_SIZE); - pt_log_error(cmd_q->pt, cmd_q->cmd_error); - iowrite32(tail, cmd_q->reg_control + 0x0008); - } - - cmd->pt_cmd_callback(cmd->data, cmd->ret); -} - -void pt_check_status_trans(struct pt_device *pt, struct pt_cmd_queue *cmd_q) -{ - u32 status; - - status = ioread32(cmd_q->reg_control + 0x0010); - if (status) { - cmd_q->int_status = status; - cmd_q->q_status = ioread32(cmd_q->reg_control + 0x0100); - cmd_q->q_int_status = ioread32(cmd_q->reg_control + 0x0104); - - /* On error, only save the first error value */ - if ((status & INT_ERROR) && !cmd_q->cmd_error) - cmd_q->cmd_error = CMD_Q_ERROR(cmd_q->q_status); - - /* Acknowledge the completion */ - iowrite32(status, cmd_q->reg_control + 0x0010); - pt_do_cmd_complete((ulong)&pt->tdata); - } -} - -static irqreturn_t pt_core_irq_handler(int irq, void *data) -{ - struct pt_device *pt = data; - struct pt_cmd_queue *cmd_q = &pt->cmd_q; - - pt_core_disable_queue_interrupts(pt); - pt->total_interrupts++; - pt_check_status_trans(pt, cmd_q); - pt_core_enable_queue_interrupts(pt); - return IRQ_HANDLED; -} - -int pt_core_init(struct pt_device *pt) -{ - char dma_pool_name[MAX_DMAPOOL_NAME_LEN]; - struct pt_cmd_queue *cmd_q = &pt->cmd_q; - u32 dma_addr_lo, dma_addr_hi; - struct device *dev = pt->dev; - struct dma_pool *dma_pool; - int ret; - - /* Allocate a dma pool for the queue */ - snprintf(dma_pool_name, sizeof(dma_pool_name), "%s_q", dev_name(pt->dev)); - - dma_pool = dma_pool_create(dma_pool_name, dev, - PT_DMAPOOL_MAX_SIZE, - PT_DMAPOOL_ALIGN, 0); - if (!dma_pool) - return -ENOMEM; - - /* ptdma core initialisation */ - iowrite32(CMD_CONFIG_VHB_EN, pt->io_regs + CMD_CONFIG_OFFSET); - iowrite32(CMD_QUEUE_PRIO, pt->io_regs + CMD_QUEUE_PRIO_OFFSET); - iowrite32(CMD_TIMEOUT_DISABLE, pt->io_regs + CMD_TIMEOUT_OFFSET); - iowrite32(CMD_CLK_GATE_CONFIG, pt->io_regs + CMD_CLK_GATE_CTL_OFFSET); - iowrite32(CMD_CONFIG_REQID, pt->io_regs + CMD_REQID_CONFIG_OFFSET); - - cmd_q->pt = pt; - cmd_q->dma_pool = dma_pool; - spin_lock_init(&cmd_q->q_lock); - - /* Page alignment satisfies our needs for N <= 128 */ - cmd_q->qsize = Q_SIZE(Q_DESC_SIZE); - cmd_q->qbase = dma_alloc_coherent(dev, cmd_q->qsize, - &cmd_q->qbase_dma, - GFP_KERNEL); - if (!cmd_q->qbase) { - dev_err(dev, "unable to allocate command queue\n"); - ret = -ENOMEM; - goto e_destroy_pool; - } - - cmd_q->qidx = 0; - - /* Preset some register values */ - cmd_q->reg_control = pt->io_regs + CMD_Q_STATUS_INCR; - - /* Turn off the queues and disable interrupts until ready */ - pt_core_disable_queue_interrupts(pt); - - cmd_q->qcontrol = 0; /* Start with nothing */ - iowrite32(cmd_q->qcontrol, cmd_q->reg_control); - - ioread32(cmd_q->reg_control + 0x0104); - ioread32(cmd_q->reg_control + 0x0100); - - /* Clear the interrupt status */ - iowrite32(SUPPORTED_INTERRUPTS, cmd_q->reg_control + 0x0010); - - /* Request an irq */ - ret = request_irq(pt->pt_irq, pt_core_irq_handler, 0, dev_name(pt->dev), pt); - if (ret) { - dev_err(dev, "unable to allocate an IRQ\n"); - goto e_free_dma; - } - - /* Update the device registers with queue information. */ - cmd_q->qcontrol &= ~CMD_Q_SIZE; - cmd_q->qcontrol |= FIELD_PREP(CMD_Q_SIZE, QUEUE_SIZE_VAL); - - cmd_q->qdma_tail = cmd_q->qbase_dma; - dma_addr_lo = lower_32_bits(cmd_q->qdma_tail); - iowrite32((u32)dma_addr_lo, cmd_q->reg_control + 0x0004); - iowrite32((u32)dma_addr_lo, cmd_q->reg_control + 0x0008); - - dma_addr_hi = upper_32_bits(cmd_q->qdma_tail); - cmd_q->qcontrol |= (dma_addr_hi << 16); - iowrite32(cmd_q->qcontrol, cmd_q->reg_control); - - pt_core_enable_queue_interrupts(pt); - - /* Register the DMA engine support */ - ret = pt_dmaengine_register(pt); - if (ret) - goto e_free_irq; - - /* Set up debugfs entries */ - ptdma_debugfs_setup(pt); - - return 0; - -e_free_irq: - free_irq(pt->pt_irq, pt); - -e_free_dma: - dma_free_coherent(dev, cmd_q->qsize, cmd_q->qbase, cmd_q->qbase_dma); - -e_destroy_pool: - dma_pool_destroy(pt->cmd_q.dma_pool); - - return ret; -} - -void pt_core_destroy(struct pt_device *pt) -{ - struct device *dev = pt->dev; - struct pt_cmd_queue *cmd_q = &pt->cmd_q; - struct pt_cmd *cmd; - - /* Unregister the DMA engine */ - pt_dmaengine_unregister(pt); - - /* Disable and clear interrupts */ - pt_core_disable_queue_interrupts(pt); - - /* Turn off the run bit */ - pt_stop_queue(cmd_q); - - /* Clear the interrupt status */ - iowrite32(SUPPORTED_INTERRUPTS, cmd_q->reg_control + 0x0010); - ioread32(cmd_q->reg_control + 0x0104); - ioread32(cmd_q->reg_control + 0x0100); - - free_irq(pt->pt_irq, pt); - - dma_free_coherent(dev, cmd_q->qsize, cmd_q->qbase, - cmd_q->qbase_dma); - - /* Flush the cmd queue */ - while (!list_empty(&pt->cmd)) { - /* Invoke the callback directly with an error code */ - cmd = list_first_entry(&pt->cmd, struct pt_cmd, entry); - list_del(&cmd->entry); - cmd->pt_cmd_callback(cmd->data, -ENODEV); - } -} diff --git a/drivers/dma/ptdma/ptdma-dmaengine.c b/drivers/dma/ptdma/ptdma-dmaengine.c deleted file mode 100644 index f79240734807..000000000000 --- a/drivers/dma/ptdma/ptdma-dmaengine.c +++ /dev/null @@ -1,411 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * AMD Passthrough DMA device driver - * -- Based on the CCP driver - * - * Copyright (C) 2016,2021 Advanced Micro Devices, Inc. - * - * Author: Sanjay R Mehta - * Author: Gary R Hook - */ - -#include "ptdma.h" -#include "../dmaengine.h" -#include "../virt-dma.h" - -static inline struct pt_dma_chan *to_pt_chan(struct dma_chan *dma_chan) -{ - return container_of(dma_chan, struct pt_dma_chan, vc.chan); -} - -static inline struct pt_dma_desc *to_pt_desc(struct virt_dma_desc *vd) -{ - return container_of(vd, struct pt_dma_desc, vd); -} - -static void pt_free_chan_resources(struct dma_chan *dma_chan) -{ - struct pt_dma_chan *chan = to_pt_chan(dma_chan); - - vchan_free_chan_resources(&chan->vc); -} - -static void pt_synchronize(struct dma_chan *dma_chan) -{ - struct pt_dma_chan *chan = to_pt_chan(dma_chan); - - vchan_synchronize(&chan->vc); -} - -static void pt_do_cleanup(struct virt_dma_desc *vd) -{ - struct pt_dma_desc *desc = to_pt_desc(vd); - struct pt_device *pt = desc->pt; - - kmem_cache_free(pt->dma_desc_cache, desc); -} - -static int pt_dma_start_desc(struct pt_dma_desc *desc) -{ - struct pt_passthru_engine *pt_engine; - struct pt_device *pt; - struct pt_cmd *pt_cmd; - struct pt_cmd_queue *cmd_q; - - desc->issued_to_hw = 1; - - pt_cmd = &desc->pt_cmd; - pt = pt_cmd->pt; - cmd_q = &pt->cmd_q; - pt_engine = &pt_cmd->passthru; - - pt->tdata.cmd = pt_cmd; - - /* Execute the command */ - pt_cmd->ret = pt_core_perform_passthru(cmd_q, pt_engine); - - return 0; -} - -static struct pt_dma_desc *pt_next_dma_desc(struct pt_dma_chan *chan) -{ - /* Get the next DMA descriptor on the active list */ - struct virt_dma_desc *vd = vchan_next_desc(&chan->vc); - - return vd ? to_pt_desc(vd) : NULL; -} - -static struct pt_dma_desc *pt_handle_active_desc(struct pt_dma_chan *chan, - struct pt_dma_desc *desc) -{ - struct dma_async_tx_descriptor *tx_desc; - struct virt_dma_desc *vd; - unsigned long flags; - - /* Loop over descriptors until one is found with commands */ - do { - if (desc) { - if (!desc->issued_to_hw) { - /* No errors, keep going */ - if (desc->status != DMA_ERROR) - return desc; - } - - tx_desc = &desc->vd.tx; - vd = &desc->vd; - } else { - tx_desc = NULL; - } - - spin_lock_irqsave(&chan->vc.lock, flags); - - if (desc) { - if (desc->status != DMA_COMPLETE) { - if (desc->status != DMA_ERROR) - desc->status = DMA_COMPLETE; - - dma_cookie_complete(tx_desc); - dma_descriptor_unmap(tx_desc); - list_del(&desc->vd.node); - } else { - /* Don't handle it twice */ - tx_desc = NULL; - } - } - - desc = pt_next_dma_desc(chan); - - spin_unlock_irqrestore(&chan->vc.lock, flags); - - if (tx_desc) { - dmaengine_desc_get_callback_invoke(tx_desc, NULL); - dma_run_dependencies(tx_desc); - vchan_vdesc_fini(vd); - } - } while (desc); - - return NULL; -} - -static void pt_cmd_callback(void *data, int err) -{ - struct pt_dma_desc *desc = data; - struct dma_chan *dma_chan; - struct pt_dma_chan *chan; - int ret; - - if (err == -EINPROGRESS) - return; - - dma_chan = desc->vd.tx.chan; - chan = to_pt_chan(dma_chan); - - if (err) - desc->status = DMA_ERROR; - - while (true) { - /* Check for DMA descriptor completion */ - desc = pt_handle_active_desc(chan, desc); - - /* Don't submit cmd if no descriptor or DMA is paused */ - if (!desc) - break; - - ret = pt_dma_start_desc(desc); - if (!ret) - break; - - desc->status = DMA_ERROR; - } -} - -static struct pt_dma_desc *pt_alloc_dma_desc(struct pt_dma_chan *chan, - unsigned long flags) -{ - struct pt_dma_desc *desc; - - desc = kmem_cache_zalloc(chan->pt->dma_desc_cache, GFP_NOWAIT); - if (!desc) - return NULL; - - vchan_tx_prep(&chan->vc, &desc->vd, flags); - - desc->pt = chan->pt; - desc->pt->cmd_q.int_en = !!(flags & DMA_PREP_INTERRUPT); - desc->issued_to_hw = 0; - desc->status = DMA_IN_PROGRESS; - - return desc; -} - -static struct pt_dma_desc *pt_create_desc(struct dma_chan *dma_chan, - dma_addr_t dst, - dma_addr_t src, - unsigned int len, - unsigned long flags) -{ - struct pt_dma_chan *chan = to_pt_chan(dma_chan); - struct pt_passthru_engine *pt_engine; - struct pt_dma_desc *desc; - struct pt_cmd *pt_cmd; - - desc = pt_alloc_dma_desc(chan, flags); - if (!desc) - return NULL; - - pt_cmd = &desc->pt_cmd; - pt_cmd->pt = chan->pt; - pt_engine = &pt_cmd->passthru; - pt_cmd->engine = PT_ENGINE_PASSTHRU; - pt_engine->src_dma = src; - pt_engine->dst_dma = dst; - pt_engine->src_len = len; - pt_cmd->pt_cmd_callback = pt_cmd_callback; - pt_cmd->data = desc; - - desc->len = len; - - return desc; -} - -static struct dma_async_tx_descriptor * -pt_prep_dma_memcpy(struct dma_chan *dma_chan, dma_addr_t dst, - dma_addr_t src, size_t len, unsigned long flags) -{ - struct pt_dma_desc *desc; - - desc = pt_create_desc(dma_chan, dst, src, len, flags); - if (!desc) - return NULL; - - return &desc->vd.tx; -} - -static struct dma_async_tx_descriptor * -pt_prep_dma_interrupt(struct dma_chan *dma_chan, unsigned long flags) -{ - struct pt_dma_chan *chan = to_pt_chan(dma_chan); - struct pt_dma_desc *desc; - - desc = pt_alloc_dma_desc(chan, flags); - if (!desc) - return NULL; - - return &desc->vd.tx; -} - -static void pt_issue_pending(struct dma_chan *dma_chan) -{ - struct pt_dma_chan *chan = to_pt_chan(dma_chan); - struct pt_dma_desc *desc; - unsigned long flags; - bool engine_is_idle = true; - - spin_lock_irqsave(&chan->vc.lock, flags); - - desc = pt_next_dma_desc(chan); - if (desc) - engine_is_idle = false; - - vchan_issue_pending(&chan->vc); - - desc = pt_next_dma_desc(chan); - - spin_unlock_irqrestore(&chan->vc.lock, flags); - - /* If there was nothing active, start processing */ - if (engine_is_idle && desc) - pt_cmd_callback(desc, 0); -} - -static enum dma_status -pt_tx_status(struct dma_chan *c, dma_cookie_t cookie, - struct dma_tx_state *txstate) -{ - struct pt_device *pt = to_pt_chan(c)->pt; - struct pt_cmd_queue *cmd_q = &pt->cmd_q; - - pt_check_status_trans(pt, cmd_q); - return dma_cookie_status(c, cookie, txstate); -} - -static int pt_pause(struct dma_chan *dma_chan) -{ - struct pt_dma_chan *chan = to_pt_chan(dma_chan); - unsigned long flags; - - spin_lock_irqsave(&chan->vc.lock, flags); - pt_stop_queue(&chan->pt->cmd_q); - spin_unlock_irqrestore(&chan->vc.lock, flags); - - return 0; -} - -static int pt_resume(struct dma_chan *dma_chan) -{ - struct pt_dma_chan *chan = to_pt_chan(dma_chan); - struct pt_dma_desc *desc = NULL; - unsigned long flags; - - spin_lock_irqsave(&chan->vc.lock, flags); - pt_start_queue(&chan->pt->cmd_q); - desc = pt_next_dma_desc(chan); - spin_unlock_irqrestore(&chan->vc.lock, flags); - - /* If there was something active, re-start */ - if (desc) - pt_cmd_callback(desc, 0); - - return 0; -} - -static int pt_terminate_all(struct dma_chan *dma_chan) -{ - struct pt_dma_chan *chan = to_pt_chan(dma_chan); - unsigned long flags; - struct pt_cmd_queue *cmd_q = &chan->pt->cmd_q; - LIST_HEAD(head); - - iowrite32(SUPPORTED_INTERRUPTS, cmd_q->reg_control + 0x0010); - spin_lock_irqsave(&chan->vc.lock, flags); - vchan_get_all_descriptors(&chan->vc, &head); - spin_unlock_irqrestore(&chan->vc.lock, flags); - - vchan_dma_desc_free_list(&chan->vc, &head); - vchan_free_chan_resources(&chan->vc); - - return 0; -} - -int pt_dmaengine_register(struct pt_device *pt) -{ - struct pt_dma_chan *chan; - struct dma_device *dma_dev = &pt->dma_dev; - char *cmd_cache_name; - char *desc_cache_name; - int ret; - - pt->pt_dma_chan = devm_kzalloc(pt->dev, sizeof(*pt->pt_dma_chan), - GFP_KERNEL); - if (!pt->pt_dma_chan) - return -ENOMEM; - - cmd_cache_name = devm_kasprintf(pt->dev, GFP_KERNEL, - "%s-dmaengine-cmd-cache", - dev_name(pt->dev)); - if (!cmd_cache_name) - return -ENOMEM; - - desc_cache_name = devm_kasprintf(pt->dev, GFP_KERNEL, - "%s-dmaengine-desc-cache", - dev_name(pt->dev)); - if (!desc_cache_name) { - ret = -ENOMEM; - goto err_cache; - } - - pt->dma_desc_cache = kmem_cache_create(desc_cache_name, - sizeof(struct pt_dma_desc), 0, - SLAB_HWCACHE_ALIGN, NULL); - if (!pt->dma_desc_cache) { - ret = -ENOMEM; - goto err_cache; - } - - dma_dev->dev = pt->dev; - dma_dev->src_addr_widths = DMA_SLAVE_BUSWIDTH_64_BYTES; - dma_dev->dst_addr_widths = DMA_SLAVE_BUSWIDTH_64_BYTES; - dma_dev->directions = DMA_MEM_TO_MEM; - dma_dev->residue_granularity = DMA_RESIDUE_GRANULARITY_DESCRIPTOR; - dma_cap_set(DMA_MEMCPY, dma_dev->cap_mask); - dma_cap_set(DMA_INTERRUPT, dma_dev->cap_mask); - - /* - * PTDMA is intended to be used with the AMD NTB devices, hence - * marking it as DMA_PRIVATE. - */ - dma_cap_set(DMA_PRIVATE, dma_dev->cap_mask); - - INIT_LIST_HEAD(&dma_dev->channels); - - chan = pt->pt_dma_chan; - chan->pt = pt; - - /* Set base and prep routines */ - dma_dev->device_free_chan_resources = pt_free_chan_resources; - dma_dev->device_prep_dma_memcpy = pt_prep_dma_memcpy; - dma_dev->device_prep_dma_interrupt = pt_prep_dma_interrupt; - dma_dev->device_issue_pending = pt_issue_pending; - dma_dev->device_tx_status = pt_tx_status; - dma_dev->device_pause = pt_pause; - dma_dev->device_resume = pt_resume; - dma_dev->device_terminate_all = pt_terminate_all; - dma_dev->device_synchronize = pt_synchronize; - - chan->vc.desc_free = pt_do_cleanup; - vchan_init(&chan->vc, dma_dev); - - ret = dma_async_device_register(dma_dev); - if (ret) - goto err_reg; - - return 0; - -err_reg: - kmem_cache_destroy(pt->dma_desc_cache); - -err_cache: - kmem_cache_destroy(pt->dma_cmd_cache); - - return ret; -} - -void pt_dmaengine_unregister(struct pt_device *pt) -{ - struct dma_device *dma_dev = &pt->dma_dev; - - dma_async_device_unregister(dma_dev); - - kmem_cache_destroy(pt->dma_desc_cache); - kmem_cache_destroy(pt->dma_cmd_cache); -} diff --git a/drivers/dma/ptdma/ptdma-pci.c b/drivers/dma/ptdma/ptdma-pci.c deleted file mode 100644 index 22739ff0c3c5..000000000000 --- a/drivers/dma/ptdma/ptdma-pci.c +++ /dev/null @@ -1,243 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * AMD Passthru DMA device driver - * -- Based on the CCP driver - * - * Copyright (C) 2016,2021 Advanced Micro Devices, Inc. - * - * Author: Sanjay R Mehta - * Author: Tom Lendacky - * Author: Gary R Hook - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "ptdma.h" - -struct pt_msix { - int msix_count; - struct msix_entry msix_entry; -}; - -/* - * pt_alloc_struct - allocate and initialize the pt_device struct - * - * @dev: device struct of the PTDMA - */ -static struct pt_device *pt_alloc_struct(struct device *dev) -{ - struct pt_device *pt; - - pt = devm_kzalloc(dev, sizeof(*pt), GFP_KERNEL); - - if (!pt) - return NULL; - pt->dev = dev; - - INIT_LIST_HEAD(&pt->cmd); - - return pt; -} - -static int pt_get_msix_irqs(struct pt_device *pt) -{ - struct pt_msix *pt_msix = pt->pt_msix; - struct device *dev = pt->dev; - struct pci_dev *pdev = to_pci_dev(dev); - int ret; - - pt_msix->msix_entry.entry = 0; - - ret = pci_enable_msix_range(pdev, &pt_msix->msix_entry, 1, 1); - if (ret < 0) - return ret; - - pt_msix->msix_count = ret; - - pt->pt_irq = pt_msix->msix_entry.vector; - - return 0; -} - -static int pt_get_msi_irq(struct pt_device *pt) -{ - struct device *dev = pt->dev; - struct pci_dev *pdev = to_pci_dev(dev); - int ret; - - ret = pci_enable_msi(pdev); - if (ret) - return ret; - - pt->pt_irq = pdev->irq; - - return 0; -} - -static int pt_get_irqs(struct pt_device *pt) -{ - struct device *dev = pt->dev; - int ret; - - ret = pt_get_msix_irqs(pt); - if (!ret) - return 0; - - /* Couldn't get MSI-X vectors, try MSI */ - dev_err(dev, "could not enable MSI-X (%d), trying MSI\n", ret); - ret = pt_get_msi_irq(pt); - if (!ret) - return 0; - - /* Couldn't get MSI interrupt */ - dev_err(dev, "could not enable MSI (%d)\n", ret); - - return ret; -} - -static void pt_free_irqs(struct pt_device *pt) -{ - struct pt_msix *pt_msix = pt->pt_msix; - struct device *dev = pt->dev; - struct pci_dev *pdev = to_pci_dev(dev); - - if (pt_msix->msix_count) - pci_disable_msix(pdev); - else if (pt->pt_irq) - pci_disable_msi(pdev); - - pt->pt_irq = 0; -} - -static int pt_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) -{ - struct pt_device *pt; - struct pt_msix *pt_msix; - struct device *dev = &pdev->dev; - void __iomem * const *iomap_table; - int bar_mask; - int ret = -ENOMEM; - - pt = pt_alloc_struct(dev); - if (!pt) - goto e_err; - - pt_msix = devm_kzalloc(dev, sizeof(*pt_msix), GFP_KERNEL); - if (!pt_msix) - goto e_err; - - pt->pt_msix = pt_msix; - pt->dev_vdata = (struct pt_dev_vdata *)id->driver_data; - if (!pt->dev_vdata) { - ret = -ENODEV; - dev_err(dev, "missing driver data\n"); - goto e_err; - } - - ret = pcim_enable_device(pdev); - if (ret) { - dev_err(dev, "pcim_enable_device failed (%d)\n", ret); - goto e_err; - } - - bar_mask = pci_select_bars(pdev, IORESOURCE_MEM); - ret = pcim_iomap_regions(pdev, bar_mask, "ptdma"); - if (ret) { - dev_err(dev, "pcim_iomap_regions failed (%d)\n", ret); - goto e_err; - } - - iomap_table = pcim_iomap_table(pdev); - if (!iomap_table) { - dev_err(dev, "pcim_iomap_table failed\n"); - ret = -ENOMEM; - goto e_err; - } - - pt->io_regs = iomap_table[pt->dev_vdata->bar]; - if (!pt->io_regs) { - dev_err(dev, "ioremap failed\n"); - ret = -ENOMEM; - goto e_err; - } - - ret = pt_get_irqs(pt); - if (ret) - goto e_err; - - pci_set_master(pdev); - - ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(48)); - if (ret) { - ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(32)); - if (ret) { - dev_err(dev, "dma_set_mask_and_coherent failed (%d)\n", - ret); - goto e_err; - } - } - - dev_set_drvdata(dev, pt); - - if (pt->dev_vdata) - ret = pt_core_init(pt); - - if (ret) - goto e_err; - - return 0; - -e_err: - dev_err(dev, "initialization failed ret = %d\n", ret); - - return ret; -} - -static void pt_pci_remove(struct pci_dev *pdev) -{ - struct device *dev = &pdev->dev; - struct pt_device *pt = dev_get_drvdata(dev); - - if (!pt) - return; - - if (pt->dev_vdata) - pt_core_destroy(pt); - - pt_free_irqs(pt); -} - -static const struct pt_dev_vdata dev_vdata[] = { - { - .bar = 2, - }, -}; - -static const struct pci_device_id pt_pci_table[] = { - { PCI_VDEVICE(AMD, 0x1498), (kernel_ulong_t)&dev_vdata[0] }, - /* Last entry must be zero */ - { 0, } -}; -MODULE_DEVICE_TABLE(pci, pt_pci_table); - -static struct pci_driver pt_pci_driver = { - .name = "ptdma", - .id_table = pt_pci_table, - .probe = pt_pci_probe, - .remove = pt_pci_remove, -}; - -module_pci_driver(pt_pci_driver); - -MODULE_AUTHOR("Sanjay R Mehta "); -MODULE_LICENSE("GPL"); -MODULE_DESCRIPTION("AMD PassThru DMA driver"); diff --git a/drivers/dma/ptdma/ptdma.h b/drivers/dma/ptdma/ptdma.h deleted file mode 100644 index 39bc37268235..000000000000 --- a/drivers/dma/ptdma/ptdma.h +++ /dev/null @@ -1,337 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * AMD Passthru DMA device driver - * -- Based on the CCP driver - * - * Copyright (C) 2016,2021 Advanced Micro Devices, Inc. - * - * Author: Sanjay R Mehta - * Author: Tom Lendacky - * Author: Gary R Hook - */ - -#ifndef __PT_DEV_H__ -#define __PT_DEV_H__ - -#include -#include -#include -#include -#include -#include -#include -#include - -#include "../virt-dma.h" - -#define MAX_PT_NAME_LEN 16 -#define MAX_DMAPOOL_NAME_LEN 32 - -#define MAX_HW_QUEUES 1 -#define MAX_CMD_QLEN 100 - -#define PT_ENGINE_PASSTHRU 5 - -/* Register Mappings */ -#define IRQ_MASK_REG 0x040 -#define IRQ_STATUS_REG 0x200 - -#define CMD_Q_ERROR(__qs) ((__qs) & 0x0000003f) - -#define CMD_QUEUE_PRIO_OFFSET 0x00 -#define CMD_REQID_CONFIG_OFFSET 0x04 -#define CMD_TIMEOUT_OFFSET 0x08 -#define CMD_PT_VERSION 0x10 - -#define CMD_Q_CONTROL_BASE 0x0000 -#define CMD_Q_TAIL_LO_BASE 0x0004 -#define CMD_Q_HEAD_LO_BASE 0x0008 -#define CMD_Q_INT_ENABLE_BASE 0x000C -#define CMD_Q_INTERRUPT_STATUS_BASE 0x0010 - -#define CMD_Q_STATUS_BASE 0x0100 -#define CMD_Q_INT_STATUS_BASE 0x0104 -#define CMD_Q_DMA_STATUS_BASE 0x0108 -#define CMD_Q_DMA_READ_STATUS_BASE 0x010C -#define CMD_Q_DMA_WRITE_STATUS_BASE 0x0110 -#define CMD_Q_ABORT_BASE 0x0114 -#define CMD_Q_AX_CACHE_BASE 0x0118 - -#define CMD_CONFIG_OFFSET 0x1120 -#define CMD_CLK_GATE_CTL_OFFSET 0x6004 - -#define CMD_DESC_DW0_VAL 0x500012 - -/* Address offset for virtual queue registers */ -#define CMD_Q_STATUS_INCR 0x1000 - -/* Bit masks */ -#define CMD_CONFIG_REQID 0 -#define CMD_TIMEOUT_DISABLE 0 -#define CMD_CLK_DYN_GATING_DIS 0 -#define CMD_CLK_SW_GATE_MODE 0 -#define CMD_CLK_GATE_CTL 0 -#define CMD_QUEUE_PRIO GENMASK(2, 1) -#define CMD_CONFIG_VHB_EN BIT(0) -#define CMD_CLK_DYN_GATING_EN BIT(0) -#define CMD_CLK_HW_GATE_MODE BIT(0) -#define CMD_CLK_GATE_ON_DELAY BIT(12) -#define CMD_CLK_GATE_OFF_DELAY BIT(12) - -#define CMD_CLK_GATE_CONFIG (CMD_CLK_GATE_CTL | \ - CMD_CLK_HW_GATE_MODE | \ - CMD_CLK_GATE_ON_DELAY | \ - CMD_CLK_DYN_GATING_EN | \ - CMD_CLK_GATE_OFF_DELAY) - -#define CMD_Q_LEN 32 -#define CMD_Q_RUN BIT(0) -#define CMD_Q_HALT BIT(1) -#define CMD_Q_MEM_LOCATION BIT(2) -#define CMD_Q_SIZE_MASK GENMASK(4, 0) -#define CMD_Q_SIZE GENMASK(7, 3) -#define CMD_Q_SHIFT GENMASK(1, 0) -#define QUEUE_SIZE_VAL ((ffs(CMD_Q_LEN) - 2) & \ - CMD_Q_SIZE_MASK) -#define Q_PTR_MASK (2 << (QUEUE_SIZE_VAL + 5) - 1) -#define Q_DESC_SIZE sizeof(struct ptdma_desc) -#define Q_SIZE(n) (CMD_Q_LEN * (n)) - -#define INT_COMPLETION BIT(0) -#define INT_ERROR BIT(1) -#define INT_QUEUE_STOPPED BIT(2) -#define INT_EMPTY_QUEUE BIT(3) -#define SUPPORTED_INTERRUPTS (INT_COMPLETION | INT_ERROR) - -/****** Local Storage Block ******/ -#define LSB_START 0 -#define LSB_END 127 -#define LSB_COUNT (LSB_END - LSB_START + 1) - -#define PT_DMAPOOL_MAX_SIZE 64 -#define PT_DMAPOOL_ALIGN BIT(5) - -#define PT_PASSTHRU_BLOCKSIZE 512 - -struct pt_device; - -struct pt_tasklet_data { - struct completion completion; - struct pt_cmd *cmd; -}; - -/* - * struct pt_passthru_engine - pass-through operation - * without performing DMA mapping - * @mask: mask to be applied to data - * @mask_len: length in bytes of mask - * @src_dma: data to be used for this operation - * @dst_dma: data produced by this operation - * @src_len: length in bytes of data used for this operation - * - * Variables required to be set when calling pt_enqueue_cmd(): - * - bit_mod, byte_swap, src, dst, src_len - * - mask, mask_len if bit_mod is not PT_PASSTHRU_BITWISE_NOOP - */ -struct pt_passthru_engine { - dma_addr_t mask; - u32 mask_len; /* In bytes */ - - dma_addr_t src_dma, dst_dma; - u64 src_len; /* In bytes */ -}; - -/* - * struct pt_cmd - PTDMA operation request - * @entry: list element - * @work: work element used for callbacks - * @pt: PT device to be run on - * @ret: operation return code - * @flags: cmd processing flags - * @engine: PTDMA operation to perform (passthru) - * @engine_error: PT engine return code - * @passthru: engine specific structures, refer to specific engine struct below - * @callback: operation completion callback function - * @data: parameter value to be supplied to the callback function - * - * Variables required to be set when calling pt_enqueue_cmd(): - * - engine, callback - * - See the operation structures below for what is required for each - * operation. - */ -struct pt_cmd { - struct list_head entry; - struct work_struct work; - struct pt_device *pt; - int ret; - u32 engine; - u32 engine_error; - struct pt_passthru_engine passthru; - /* Completion callback support */ - void (*pt_cmd_callback)(void *data, int err); - void *data; -}; - -struct pt_dma_desc { - struct virt_dma_desc vd; - struct pt_device *pt; - enum dma_status status; - size_t len; - bool issued_to_hw; - struct pt_cmd pt_cmd; -}; - -struct pt_dma_chan { - struct virt_dma_chan vc; - struct pt_device *pt; -}; - -struct pt_cmd_queue { - struct pt_device *pt; - - /* Queue dma pool */ - struct dma_pool *dma_pool; - - /* Queue base address (not necessarily aligned)*/ - struct ptdma_desc *qbase; - - /* Aligned queue start address (per requirement) */ - spinlock_t q_lock ____cacheline_aligned; - unsigned int qidx; - - unsigned int qsize; - dma_addr_t qbase_dma; - dma_addr_t qdma_tail; - - unsigned int active; - unsigned int suspended; - - /* Interrupt flag */ - bool int_en; - - /* Register addresses for queue */ - void __iomem *reg_control; - u32 qcontrol; /* Cached control register */ - - /* Status values from job */ - u32 int_status; - u32 q_status; - u32 q_int_status; - u32 cmd_error; - /* Queue Statistics */ - unsigned long total_pt_ops; -} ____cacheline_aligned; - -struct pt_device { - struct list_head entry; - - unsigned int ord; - char name[MAX_PT_NAME_LEN]; - - struct device *dev; - - /* Bus specific device information */ - struct pt_msix *pt_msix; - - struct pt_dev_vdata *dev_vdata; - - unsigned int pt_irq; - - /* I/O area used for device communication */ - void __iomem *io_regs; - - spinlock_t cmd_lock ____cacheline_aligned; - unsigned int cmd_count; - struct list_head cmd; - - /* - * The command queue. This represent the queue available on the - * PTDMA that are available for processing cmds - */ - struct pt_cmd_queue cmd_q; - - /* Support for the DMA Engine capabilities */ - struct dma_device dma_dev; - struct pt_dma_chan *pt_dma_chan; - struct kmem_cache *dma_cmd_cache; - struct kmem_cache *dma_desc_cache; - - wait_queue_head_t lsb_queue; - - /* Device Statistics */ - unsigned long total_interrupts; - - struct pt_tasklet_data tdata; -}; - -/* - * descriptor for PTDMA commands - * 8 32-bit words: - * word 0: function; engine; control bits - * word 1: length of source data - * word 2: low 32 bits of source pointer - * word 3: upper 16 bits of source pointer; source memory type - * word 4: low 32 bits of destination pointer - * word 5: upper 16 bits of destination pointer; destination memory type - * word 6: reserved 32 bits - * word 7: reserved 32 bits - */ - -#define DWORD0_SOC BIT(0) -#define DWORD0_IOC BIT(1) - -struct dword3 { - unsigned int src_hi:16; - unsigned int src_mem:2; - unsigned int lsb_cxt_id:8; - unsigned int rsvd1:5; - unsigned int fixed:1; -}; - -struct dword5 { - unsigned int dst_hi:16; - unsigned int dst_mem:2; - unsigned int rsvd1:13; - unsigned int fixed:1; -}; - -struct ptdma_desc { - u32 dw0; - u32 length; - u32 src_lo; - struct dword3 dw3; - u32 dst_lo; - struct dword5 dw5; - __le32 rsvd1; - __le32 rsvd2; -}; - -/* Structure to hold PT device data */ -struct pt_dev_vdata { - const unsigned int bar; -}; - -int pt_dmaengine_register(struct pt_device *pt); -void pt_dmaengine_unregister(struct pt_device *pt); - -void ptdma_debugfs_setup(struct pt_device *pt); -int pt_core_init(struct pt_device *pt); -void pt_core_destroy(struct pt_device *pt); - -int pt_core_perform_passthru(struct pt_cmd_queue *cmd_q, - struct pt_passthru_engine *pt_engine); - -void pt_check_status_trans(struct pt_device *pt, struct pt_cmd_queue *cmd_q); -void pt_start_queue(struct pt_cmd_queue *cmd_q); -void pt_stop_queue(struct pt_cmd_queue *cmd_q); - -static inline void pt_core_disable_queue_interrupts(struct pt_device *pt) -{ - iowrite32(0, pt->cmd_q.reg_control + 0x000C); -} - -static inline void pt_core_enable_queue_interrupts(struct pt_device *pt) -{ - iowrite32(SUPPORTED_INTERRUPTS, pt->cmd_q.reg_control + 0x000C); -} -#endif -- cgit v1.2.3-59-g8ed1b From 90a30e268d9bd29dac63fece0509f9f8d4835d57 Mon Sep 17 00:00:00 2001 From: Basavaraj Natikar Date: Fri, 25 Oct 2024 15:29:27 +0530 Subject: dmaengine: ae4dma: Add AMD ae4dma controller driver Add support for AMD AE4DMA controller. It performs high-bandwidth memory to memory and IO copy operation. Device commands are managed via a circular queue of 'descriptors', each of which specifies source and destination addresses for copying a single buffer of data. Reviewed-by: Raju Rangoju Reviewed-by: Philipp Stanner Signed-off-by: Basavaraj Natikar Link: https://lore.kernel.org/r/20241025095931.726018-3-Basavaraj.Natikar@amd.com Signed-off-by: Vinod Koul --- MAINTAINERS | 6 ++ drivers/dma/amd/Kconfig | 15 +++ drivers/dma/amd/Makefile | 1 + drivers/dma/amd/ae4dma/Makefile | 10 ++ drivers/dma/amd/ae4dma/ae4dma-dev.c | 151 ++++++++++++++++++++++++++++++ drivers/dma/amd/ae4dma/ae4dma-pci.c | 157 ++++++++++++++++++++++++++++++++ drivers/dma/amd/ae4dma/ae4dma.h | 96 +++++++++++++++++++ drivers/dma/amd/ptdma/ptdma-dmaengine.c | 49 ++++++++++ 8 files changed, 485 insertions(+) create mode 100644 drivers/dma/amd/ae4dma/Makefile create mode 100644 drivers/dma/amd/ae4dma/ae4dma-dev.c create mode 100644 drivers/dma/amd/ae4dma/ae4dma-pci.c create mode 100644 drivers/dma/amd/ae4dma/ae4dma.h diff --git a/MAINTAINERS b/MAINTAINERS index b01420aa576d..ae633a697a4e 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -980,6 +980,12 @@ L: linux-edac@vger.kernel.org S: Supported F: drivers/ras/amd/atl/* +AMD AE4DMA DRIVER +M: Basavaraj Natikar +L: dmaengine@vger.kernel.org +S: Supported +F: drivers/dma/amd/ae4dma/ + AMD AXI W1 DRIVER M: Kris Chaplin R: Thomas Delev diff --git a/drivers/dma/amd/Kconfig b/drivers/dma/amd/Kconfig index a09517d51449..00d874872a8f 100644 --- a/drivers/dma/amd/Kconfig +++ b/drivers/dma/amd/Kconfig @@ -1,5 +1,20 @@ # SPDX-License-Identifier: GPL-2.0-only # + +config AMD_AE4DMA + tristate "AMD AE4DMA Engine" + depends on (X86_64 || COMPILE_TEST) && PCI + depends on AMD_PTDMA + select DMA_ENGINE + select DMA_VIRTUAL_CHANNELS + help + Enable support for the AMD AE4DMA controller. This controller + provides DMA capabilities to perform high bandwidth memory to + memory and IO copy operations. It performs DMA transfer through + queue-based descriptor management. This DMA controller is intended + to be used with AMD Non-Transparent Bridge devices and not for + general purpose peripheral DMA. + config AMD_PTDMA tristate "AMD PassThru DMA Engine" depends on X86_64 && PCI diff --git a/drivers/dma/amd/Makefile b/drivers/dma/amd/Makefile index fb12f2f9e7b7..11278c06374d 100644 --- a/drivers/dma/amd/Makefile +++ b/drivers/dma/amd/Makefile @@ -1,4 +1,5 @@ # SPDX-License-Identifier: GPL-2.0 +obj-$(CONFIG_AMD_AE4DMA) += ae4dma/ obj-$(CONFIG_AMD_PTDMA) += ptdma/ obj-$(CONFIG_AMD_QDMA) += qdma/ diff --git a/drivers/dma/amd/ae4dma/Makefile b/drivers/dma/amd/ae4dma/Makefile new file mode 100644 index 000000000000..e918f85a80ec --- /dev/null +++ b/drivers/dma/amd/ae4dma/Makefile @@ -0,0 +1,10 @@ +# SPDX-License-Identifier: GPL-2.0 +# +# AMD AE4DMA driver +# + +obj-$(CONFIG_AMD_AE4DMA) += ae4dma.o + +ae4dma-objs := ae4dma-dev.o + +ae4dma-$(CONFIG_PCI) += ae4dma-pci.o diff --git a/drivers/dma/amd/ae4dma/ae4dma-dev.c b/drivers/dma/amd/ae4dma/ae4dma-dev.c new file mode 100644 index 000000000000..7cbef9e79f38 --- /dev/null +++ b/drivers/dma/amd/ae4dma/ae4dma-dev.c @@ -0,0 +1,151 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * AMD AE4DMA driver + * + * Copyright (c) 2024, Advanced Micro Devices, Inc. + * All Rights Reserved. + * + * Author: Basavaraj Natikar + */ + +#include "ae4dma.h" + +static unsigned int max_hw_q = 1; +module_param(max_hw_q, uint, 0444); +MODULE_PARM_DESC(max_hw_q, "max hw queues supported by engine (any non-zero value, default: 1)"); + +static void ae4_pending_work(struct work_struct *work) +{ + struct ae4_cmd_queue *ae4cmd_q = container_of(work, struct ae4_cmd_queue, p_work.work); + struct pt_cmd_queue *cmd_q = &ae4cmd_q->cmd_q; + struct pt_cmd *cmd; + u32 cridx; + + for (;;) { + wait_event_interruptible(ae4cmd_q->q_w, + ((atomic64_read(&ae4cmd_q->done_cnt)) < + atomic64_read(&ae4cmd_q->intr_cnt))); + + atomic64_inc(&ae4cmd_q->done_cnt); + + mutex_lock(&ae4cmd_q->cmd_lock); + cridx = readl(cmd_q->reg_control + AE4_RD_IDX_OFF); + while ((ae4cmd_q->dridx != cridx) && !list_empty(&ae4cmd_q->cmd)) { + cmd = list_first_entry(&ae4cmd_q->cmd, struct pt_cmd, entry); + list_del(&cmd->entry); + + ae4_check_status_error(ae4cmd_q, ae4cmd_q->dridx); + cmd->pt_cmd_callback(cmd->data, cmd->ret); + + ae4cmd_q->q_cmd_count--; + ae4cmd_q->dridx = (ae4cmd_q->dridx + 1) % CMD_Q_LEN; + + complete_all(&ae4cmd_q->cmp); + } + mutex_unlock(&ae4cmd_q->cmd_lock); + } +} + +static irqreturn_t ae4_core_irq_handler(int irq, void *data) +{ + struct ae4_cmd_queue *ae4cmd_q = data; + struct pt_cmd_queue *cmd_q; + struct pt_device *pt; + u32 status; + + cmd_q = &ae4cmd_q->cmd_q; + pt = cmd_q->pt; + + pt->total_interrupts++; + atomic64_inc(&ae4cmd_q->intr_cnt); + + status = readl(cmd_q->reg_control + AE4_INTR_STS_OFF); + if (status & BIT(0)) { + status &= GENMASK(31, 1); + writel(status, cmd_q->reg_control + AE4_INTR_STS_OFF); + } + + wake_up(&ae4cmd_q->q_w); + + return IRQ_HANDLED; +} + +void ae4_destroy_work(struct ae4_device *ae4) +{ + struct ae4_cmd_queue *ae4cmd_q; + int i; + + for (i = 0; i < ae4->cmd_q_count; i++) { + ae4cmd_q = &ae4->ae4cmd_q[i]; + + if (!ae4cmd_q->pws) + break; + + cancel_delayed_work_sync(&ae4cmd_q->p_work); + destroy_workqueue(ae4cmd_q->pws); + } +} + +int ae4_core_init(struct ae4_device *ae4) +{ + struct pt_device *pt = &ae4->pt; + struct ae4_cmd_queue *ae4cmd_q; + struct device *dev = pt->dev; + struct pt_cmd_queue *cmd_q; + int i, ret = 0; + + writel(max_hw_q, pt->io_regs); + + for (i = 0; i < max_hw_q; i++) { + ae4cmd_q = &ae4->ae4cmd_q[i]; + ae4cmd_q->id = ae4->cmd_q_count; + ae4->cmd_q_count++; + + cmd_q = &ae4cmd_q->cmd_q; + cmd_q->pt = pt; + + cmd_q->reg_control = pt->io_regs + ((i + 1) * AE4_Q_SZ); + + ret = devm_request_irq(dev, ae4->ae4_irq[i], ae4_core_irq_handler, 0, + dev_name(pt->dev), ae4cmd_q); + if (ret) + return ret; + + cmd_q->qsize = Q_SIZE(sizeof(struct ae4dma_desc)); + + cmd_q->qbase = dmam_alloc_coherent(dev, cmd_q->qsize, &cmd_q->qbase_dma, + GFP_KERNEL); + if (!cmd_q->qbase) + return -ENOMEM; + } + + for (i = 0; i < ae4->cmd_q_count; i++) { + ae4cmd_q = &ae4->ae4cmd_q[i]; + + cmd_q = &ae4cmd_q->cmd_q; + + cmd_q->reg_control = pt->io_regs + ((i + 1) * AE4_Q_SZ); + + /* Update the device registers with queue information. */ + writel(CMD_Q_LEN, cmd_q->reg_control + AE4_MAX_IDX_OFF); + + cmd_q->qdma_tail = cmd_q->qbase_dma; + writel(lower_32_bits(cmd_q->qdma_tail), cmd_q->reg_control + AE4_Q_BASE_L_OFF); + writel(upper_32_bits(cmd_q->qdma_tail), cmd_q->reg_control + AE4_Q_BASE_H_OFF); + + INIT_LIST_HEAD(&ae4cmd_q->cmd); + init_waitqueue_head(&ae4cmd_q->q_w); + + ae4cmd_q->pws = alloc_ordered_workqueue("ae4dma_%d", WQ_MEM_RECLAIM, ae4cmd_q->id); + if (!ae4cmd_q->pws) { + ae4_destroy_work(ae4); + return -ENOMEM; + } + INIT_DELAYED_WORK(&ae4cmd_q->p_work, ae4_pending_work); + queue_delayed_work(ae4cmd_q->pws, &ae4cmd_q->p_work, usecs_to_jiffies(100)); + + init_completion(&ae4cmd_q->cmp); + } + + return ret; +} diff --git a/drivers/dma/amd/ae4dma/ae4dma-pci.c b/drivers/dma/amd/ae4dma/ae4dma-pci.c new file mode 100644 index 000000000000..43d36e9d1efb --- /dev/null +++ b/drivers/dma/amd/ae4dma/ae4dma-pci.c @@ -0,0 +1,157 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * AMD AE4DMA driver + * + * Copyright (c) 2024, Advanced Micro Devices, Inc. + * All Rights Reserved. + * + * Author: Basavaraj Natikar + */ + +#include "ae4dma.h" + +static int ae4_get_irqs(struct ae4_device *ae4) +{ + struct ae4_msix *ae4_msix = ae4->ae4_msix; + struct pt_device *pt = &ae4->pt; + struct device *dev = pt->dev; + struct pci_dev *pdev; + int i, v, ret; + + pdev = to_pci_dev(dev); + + for (v = 0; v < ARRAY_SIZE(ae4_msix->msix_entry); v++) + ae4_msix->msix_entry[v].entry = v; + + ret = pci_alloc_irq_vectors(pdev, v, v, PCI_IRQ_MSIX); + if (ret != v) { + if (ret > 0) + pci_free_irq_vectors(pdev); + + dev_err(dev, "could not enable MSI-X (%d), trying MSI\n", ret); + ret = pci_alloc_irq_vectors(pdev, 1, 1, PCI_IRQ_MSI); + if (ret < 0) { + dev_err(dev, "could not enable MSI (%d)\n", ret); + return ret; + } + + ret = pci_irq_vector(pdev, 0); + if (ret < 0) { + pci_free_irq_vectors(pdev); + return ret; + } + + for (i = 0; i < MAX_AE4_HW_QUEUES; i++) + ae4->ae4_irq[i] = ret; + + } else { + ae4_msix->msix_count = ret; + for (i = 0; i < MAX_AE4_HW_QUEUES; i++) + ae4->ae4_irq[i] = ae4_msix->msix_entry[i].vector; + } + + return ret; +} + +static void ae4_free_irqs(struct ae4_device *ae4) +{ + struct ae4_msix *ae4_msix = ae4->ae4_msix; + struct pt_device *pt = &ae4->pt; + struct device *dev = pt->dev; + struct pci_dev *pdev; + + pdev = to_pci_dev(dev); + + if (ae4_msix && (ae4_msix->msix_count || ae4->ae4_irq[MAX_AE4_HW_QUEUES - 1])) + pci_free_irq_vectors(pdev); +} + +static void ae4_deinit(struct ae4_device *ae4) +{ + ae4_free_irqs(ae4); +} + +static int ae4_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) +{ + struct device *dev = &pdev->dev; + struct ae4_device *ae4; + struct pt_device *pt; + int bar_mask; + int ret = 0; + + ae4 = devm_kzalloc(dev, sizeof(*ae4), GFP_KERNEL); + if (!ae4) + return -ENOMEM; + + ae4->ae4_msix = devm_kzalloc(dev, sizeof(struct ae4_msix), GFP_KERNEL); + if (!ae4->ae4_msix) + return -ENOMEM; + + ret = pcim_enable_device(pdev); + if (ret) + goto ae4_error; + + bar_mask = pci_select_bars(pdev, IORESOURCE_MEM); + ret = pcim_iomap_regions(pdev, bar_mask, "ae4dma"); + if (ret) + goto ae4_error; + + pt = &ae4->pt; + pt->dev = dev; + + pt->io_regs = pcim_iomap_table(pdev)[0]; + if (!pt->io_regs) { + ret = -ENOMEM; + goto ae4_error; + } + + ret = ae4_get_irqs(ae4); + if (ret < 0) + goto ae4_error; + + pci_set_master(pdev); + + dma_set_mask_and_coherent(dev, DMA_BIT_MASK(48)); + + dev_set_drvdata(dev, ae4); + + ret = ae4_core_init(ae4); + if (ret) + goto ae4_error; + + return 0; + +ae4_error: + ae4_deinit(ae4); + + return ret; +} + +static void ae4_pci_remove(struct pci_dev *pdev) +{ + struct ae4_device *ae4 = dev_get_drvdata(&pdev->dev); + + ae4_destroy_work(ae4); + ae4_deinit(ae4); +} + +static const struct pci_device_id ae4_pci_table[] = { + { PCI_VDEVICE(AMD, 0x14C8), }, + { PCI_VDEVICE(AMD, 0x14DC), }, + { PCI_VDEVICE(AMD, 0x149B), }, + /* Last entry must be zero */ + { 0, } +}; +MODULE_DEVICE_TABLE(pci, ae4_pci_table); + +static struct pci_driver ae4_pci_driver = { + .name = "ae4dma", + .id_table = ae4_pci_table, + .probe = ae4_pci_probe, + .remove = ae4_pci_remove, +}; + +module_pci_driver(ae4_pci_driver); + +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("AMD AE4DMA driver"); diff --git a/drivers/dma/amd/ae4dma/ae4dma.h b/drivers/dma/amd/ae4dma/ae4dma.h new file mode 100644 index 000000000000..4a1dfcf620c1 --- /dev/null +++ b/drivers/dma/amd/ae4dma/ae4dma.h @@ -0,0 +1,96 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * AMD AE4DMA driver + * + * Copyright (c) 2024, Advanced Micro Devices, Inc. + * All Rights Reserved. + * + * Author: Basavaraj Natikar + */ +#ifndef __AE4DMA_H__ +#define __AE4DMA_H__ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "../ptdma/ptdma.h" +#include "../../virt-dma.h" + +#define MAX_AE4_HW_QUEUES 16 + +#define AE4_DESC_COMPLETED 0x03 + +#define AE4_MAX_IDX_OFF 0x08 +#define AE4_RD_IDX_OFF 0x0c +#define AE4_WR_IDX_OFF 0x10 +#define AE4_INTR_STS_OFF 0x14 +#define AE4_Q_BASE_L_OFF 0x18 +#define AE4_Q_BASE_H_OFF 0x1c +#define AE4_Q_SZ 0x20 + +struct ae4_msix { + int msix_count; + struct msix_entry msix_entry[MAX_AE4_HW_QUEUES]; +}; + +struct ae4_cmd_queue { + struct ae4_device *ae4; + struct pt_cmd_queue cmd_q; + struct list_head cmd; + /* protect command operations */ + struct mutex cmd_lock; + struct delayed_work p_work; + struct workqueue_struct *pws; + struct completion cmp; + wait_queue_head_t q_w; + atomic64_t intr_cnt; + atomic64_t done_cnt; + u64 q_cmd_count; + u32 dridx; + u32 id; +}; + +union dwou { + u32 dw0; + struct dword0 { + u8 byte0; + u8 byte1; + u16 timestamp; + } dws; +}; + +struct dword1 { + u8 status; + u8 err_code; + u16 desc_id; +}; + +struct ae4dma_desc { + union dwou dwouv; + struct dword1 dw1; + u32 length; + u32 rsvd; + u32 src_hi; + u32 src_lo; + u32 dst_hi; + u32 dst_lo; +}; + +struct ae4_device { + struct pt_device pt; + struct ae4_msix *ae4_msix; + struct ae4_cmd_queue ae4cmd_q[MAX_AE4_HW_QUEUES]; + unsigned int ae4_irq[MAX_AE4_HW_QUEUES]; + unsigned int cmd_q_count; +}; + +int ae4_core_init(struct ae4_device *ae4); +void ae4_destroy_work(struct ae4_device *ae4); +void ae4_check_status_error(struct ae4_cmd_queue *ae4cmd_q, int idx); +#endif diff --git a/drivers/dma/amd/ptdma/ptdma-dmaengine.c b/drivers/dma/amd/ptdma/ptdma-dmaengine.c index a2e7c2cec15e..77fe709fb327 100644 --- a/drivers/dma/amd/ptdma/ptdma-dmaengine.c +++ b/drivers/dma/amd/ptdma/ptdma-dmaengine.c @@ -10,8 +10,57 @@ */ #include "ptdma.h" +#include "../ae4dma/ae4dma.h" #include "../../dmaengine.h" +static char *ae4_error_codes[] = { + "", + "ERR 01: INVALID HEADER DW0", + "ERR 02: INVALID STATUS", + "ERR 03: INVALID LENGTH - 4 BYTE ALIGNMENT", + "ERR 04: INVALID SRC ADDR - 4 BYTE ALIGNMENT", + "ERR 05: INVALID DST ADDR - 4 BYTE ALIGNMENT", + "ERR 06: INVALID ALIGNMENT", + "ERR 07: INVALID DESCRIPTOR", +}; + +static void ae4_log_error(struct pt_device *d, int e) +{ + /* ERR 01 - 07 represents Invalid AE4 errors */ + if (e <= 7) + dev_info(d->dev, "AE4DMA error: %s (0x%x)\n", ae4_error_codes[e], e); + /* ERR 08 - 15 represents Invalid Descriptor errors */ + else if (e > 7 && e <= 15) + dev_info(d->dev, "AE4DMA error: %s (0x%x)\n", "INVALID DESCRIPTOR", e); + /* ERR 16 - 31 represents Firmware errors */ + else if (e > 15 && e <= 31) + dev_info(d->dev, "AE4DMA error: %s (0x%x)\n", "FIRMWARE ERROR", e); + /* ERR 32 - 63 represents Fatal errors */ + else if (e > 31 && e <= 63) + dev_info(d->dev, "AE4DMA error: %s (0x%x)\n", "FATAL ERROR", e); + /* ERR 64 - 255 represents PTE errors */ + else if (e > 63 && e <= 255) + dev_info(d->dev, "AE4DMA error: %s (0x%x)\n", "PTE ERROR", e); + else + dev_info(d->dev, "Unknown AE4DMA error"); +} + +void ae4_check_status_error(struct ae4_cmd_queue *ae4cmd_q, int idx) +{ + struct pt_cmd_queue *cmd_q = &ae4cmd_q->cmd_q; + struct ae4dma_desc desc; + u8 status; + + memcpy(&desc, &cmd_q->qbase[idx], sizeof(struct ae4dma_desc)); + status = desc.dw1.status; + if (status && status != AE4_DESC_COMPLETED) { + cmd_q->cmd_error = desc.dw1.err_code; + if (cmd_q->cmd_error) + ae4_log_error(cmd_q->pt, cmd_q->cmd_error); + } +} +EXPORT_SYMBOL_GPL(ae4_check_status_error); + static inline struct pt_dma_chan *to_pt_chan(struct dma_chan *dma_chan) { return container_of(dma_chan, struct pt_dma_chan, vc.chan); -- cgit v1.2.3-59-g8ed1b From 69a47b16a51b3a52b808823f667248715ecef8c9 Mon Sep 17 00:00:00 2001 From: Basavaraj Natikar Date: Fri, 25 Oct 2024 15:29:28 +0530 Subject: dmaengine: ptdma: Extend ptdma to support multi-channel and version To support multi-channel functionality with AE4DMA engine, extend the PTDMA code with reusable components. Reviewed-by: Raju Rangoju Signed-off-by: Basavaraj Natikar Link: https://lore.kernel.org/r/20241025095931.726018-4-Basavaraj.Natikar@amd.com Signed-off-by: Vinod Koul --- drivers/dma/amd/ae4dma/ae4dma.h | 2 + drivers/dma/amd/ptdma/ptdma-dmaengine.c | 105 ++++++++++++++++++++++++++------ drivers/dma/amd/ptdma/ptdma.h | 2 + 3 files changed, 89 insertions(+), 20 deletions(-) diff --git a/drivers/dma/amd/ae4dma/ae4dma.h b/drivers/dma/amd/ae4dma/ae4dma.h index 4a1dfcf620c1..92cb8c379c18 100644 --- a/drivers/dma/amd/ae4dma/ae4dma.h +++ b/drivers/dma/amd/ae4dma/ae4dma.h @@ -34,6 +34,8 @@ #define AE4_Q_BASE_H_OFF 0x1c #define AE4_Q_SZ 0x20 +#define AE4_DMA_VERSION 4 + struct ae4_msix { int msix_count; struct msix_entry msix_entry[MAX_AE4_HW_QUEUES]; diff --git a/drivers/dma/amd/ptdma/ptdma-dmaengine.c b/drivers/dma/amd/ptdma/ptdma-dmaengine.c index 77fe709fb327..e2d4bc8aa1de 100644 --- a/drivers/dma/amd/ptdma/ptdma-dmaengine.c +++ b/drivers/dma/amd/ptdma/ptdma-dmaengine.c @@ -93,7 +93,24 @@ static void pt_do_cleanup(struct virt_dma_desc *vd) kmem_cache_free(pt->dma_desc_cache, desc); } -static int pt_dma_start_desc(struct pt_dma_desc *desc) +static struct pt_cmd_queue *pt_get_cmd_queue(struct pt_device *pt, struct pt_dma_chan *chan) +{ + struct ae4_cmd_queue *ae4cmd_q; + struct pt_cmd_queue *cmd_q; + struct ae4_device *ae4; + + if (pt->ver == AE4_DMA_VERSION) { + ae4 = container_of(pt, struct ae4_device, pt); + ae4cmd_q = &ae4->ae4cmd_q[chan->id]; + cmd_q = &ae4cmd_q->cmd_q; + } else { + cmd_q = &pt->cmd_q; + } + + return cmd_q; +} + +static int pt_dma_start_desc(struct pt_dma_desc *desc, struct pt_dma_chan *chan) { struct pt_passthru_engine *pt_engine; struct pt_device *pt; @@ -104,7 +121,9 @@ static int pt_dma_start_desc(struct pt_dma_desc *desc) pt_cmd = &desc->pt_cmd; pt = pt_cmd->pt; - cmd_q = &pt->cmd_q; + + cmd_q = pt_get_cmd_queue(pt, chan); + pt_engine = &pt_cmd->passthru; pt->tdata.cmd = pt_cmd; @@ -199,7 +218,7 @@ static void pt_cmd_callback(void *data, int err) if (!desc) break; - ret = pt_dma_start_desc(desc); + ret = pt_dma_start_desc(desc, chan); if (!ret) break; @@ -234,7 +253,10 @@ static struct pt_dma_desc *pt_create_desc(struct dma_chan *dma_chan, { struct pt_dma_chan *chan = to_pt_chan(dma_chan); struct pt_passthru_engine *pt_engine; + struct pt_device *pt = chan->pt; + struct ae4_cmd_queue *ae4cmd_q; struct pt_dma_desc *desc; + struct ae4_device *ae4; struct pt_cmd *pt_cmd; desc = pt_alloc_dma_desc(chan, flags); @@ -242,7 +264,7 @@ static struct pt_dma_desc *pt_create_desc(struct dma_chan *dma_chan, return NULL; pt_cmd = &desc->pt_cmd; - pt_cmd->pt = chan->pt; + pt_cmd->pt = pt; pt_engine = &pt_cmd->passthru; pt_cmd->engine = PT_ENGINE_PASSTHRU; pt_engine->src_dma = src; @@ -253,6 +275,14 @@ static struct pt_dma_desc *pt_create_desc(struct dma_chan *dma_chan, desc->len = len; + if (pt->ver == AE4_DMA_VERSION) { + ae4 = container_of(pt, struct ae4_device, pt); + ae4cmd_q = &ae4->ae4cmd_q[chan->id]; + mutex_lock(&ae4cmd_q->cmd_lock); + list_add_tail(&pt_cmd->entry, &ae4cmd_q->cmd); + mutex_unlock(&ae4cmd_q->cmd_lock); + } + return desc; } @@ -310,8 +340,11 @@ static enum dma_status pt_tx_status(struct dma_chan *c, dma_cookie_t cookie, struct dma_tx_state *txstate) { - struct pt_device *pt = to_pt_chan(c)->pt; - struct pt_cmd_queue *cmd_q = &pt->cmd_q; + struct pt_dma_chan *chan = to_pt_chan(c); + struct pt_device *pt = chan->pt; + struct pt_cmd_queue *cmd_q; + + cmd_q = pt_get_cmd_queue(pt, chan); pt_check_status_trans(pt, cmd_q); return dma_cookie_status(c, cookie, txstate); @@ -320,10 +353,13 @@ pt_tx_status(struct dma_chan *c, dma_cookie_t cookie, static int pt_pause(struct dma_chan *dma_chan) { struct pt_dma_chan *chan = to_pt_chan(dma_chan); + struct pt_device *pt = chan->pt; + struct pt_cmd_queue *cmd_q; unsigned long flags; spin_lock_irqsave(&chan->vc.lock, flags); - pt_stop_queue(&chan->pt->cmd_q); + cmd_q = pt_get_cmd_queue(pt, chan); + pt_stop_queue(cmd_q); spin_unlock_irqrestore(&chan->vc.lock, flags); return 0; @@ -333,10 +369,13 @@ static int pt_resume(struct dma_chan *dma_chan) { struct pt_dma_chan *chan = to_pt_chan(dma_chan); struct pt_dma_desc *desc = NULL; + struct pt_device *pt = chan->pt; + struct pt_cmd_queue *cmd_q; unsigned long flags; spin_lock_irqsave(&chan->vc.lock, flags); - pt_start_queue(&chan->pt->cmd_q); + cmd_q = pt_get_cmd_queue(pt, chan); + pt_start_queue(cmd_q); desc = pt_next_dma_desc(chan); spin_unlock_irqrestore(&chan->vc.lock, flags); @@ -350,11 +389,17 @@ static int pt_resume(struct dma_chan *dma_chan) static int pt_terminate_all(struct dma_chan *dma_chan) { struct pt_dma_chan *chan = to_pt_chan(dma_chan); + struct pt_device *pt = chan->pt; + struct pt_cmd_queue *cmd_q; unsigned long flags; - struct pt_cmd_queue *cmd_q = &chan->pt->cmd_q; LIST_HEAD(head); - iowrite32(SUPPORTED_INTERRUPTS, cmd_q->reg_control + 0x0010); + cmd_q = pt_get_cmd_queue(pt, chan); + if (pt->ver == AE4_DMA_VERSION) + pt_stop_queue(cmd_q); + else + iowrite32(SUPPORTED_INTERRUPTS, cmd_q->reg_control + 0x0010); + spin_lock_irqsave(&chan->vc.lock, flags); vchan_get_all_descriptors(&chan->vc, &head); spin_unlock_irqrestore(&chan->vc.lock, flags); @@ -367,14 +412,24 @@ static int pt_terminate_all(struct dma_chan *dma_chan) int pt_dmaengine_register(struct pt_device *pt) { - struct pt_dma_chan *chan; struct dma_device *dma_dev = &pt->dma_dev; - char *cmd_cache_name; + struct ae4_cmd_queue *ae4cmd_q = NULL; + struct ae4_device *ae4 = NULL; + struct pt_dma_chan *chan; char *desc_cache_name; - int ret; + char *cmd_cache_name; + int ret, i; + + if (pt->ver == AE4_DMA_VERSION) + ae4 = container_of(pt, struct ae4_device, pt); + + if (ae4) + pt->pt_dma_chan = devm_kcalloc(pt->dev, ae4->cmd_q_count, + sizeof(*pt->pt_dma_chan), GFP_KERNEL); + else + pt->pt_dma_chan = devm_kzalloc(pt->dev, sizeof(*pt->pt_dma_chan), + GFP_KERNEL); - pt->pt_dma_chan = devm_kzalloc(pt->dev, sizeof(*pt->pt_dma_chan), - GFP_KERNEL); if (!pt->pt_dma_chan) return -ENOMEM; @@ -416,9 +471,6 @@ int pt_dmaengine_register(struct pt_device *pt) INIT_LIST_HEAD(&dma_dev->channels); - chan = pt->pt_dma_chan; - chan->pt = pt; - /* Set base and prep routines */ dma_dev->device_free_chan_resources = pt_free_chan_resources; dma_dev->device_prep_dma_memcpy = pt_prep_dma_memcpy; @@ -430,8 +482,21 @@ int pt_dmaengine_register(struct pt_device *pt) dma_dev->device_terminate_all = pt_terminate_all; dma_dev->device_synchronize = pt_synchronize; - chan->vc.desc_free = pt_do_cleanup; - vchan_init(&chan->vc, dma_dev); + if (ae4) { + for (i = 0; i < ae4->cmd_q_count; i++) { + chan = pt->pt_dma_chan + i; + ae4cmd_q = &ae4->ae4cmd_q[i]; + chan->id = ae4cmd_q->id; + chan->pt = pt; + chan->vc.desc_free = pt_do_cleanup; + vchan_init(&chan->vc, dma_dev); + } + } else { + chan = pt->pt_dma_chan; + chan->pt = pt; + chan->vc.desc_free = pt_do_cleanup; + vchan_init(&chan->vc, dma_dev); + } ret = dma_async_device_register(dma_dev); if (ret) diff --git a/drivers/dma/amd/ptdma/ptdma.h b/drivers/dma/amd/ptdma/ptdma.h index 7a8ca8e239e0..0a7939105e51 100644 --- a/drivers/dma/amd/ptdma/ptdma.h +++ b/drivers/dma/amd/ptdma/ptdma.h @@ -184,6 +184,7 @@ struct pt_dma_desc { struct pt_dma_chan { struct virt_dma_chan vc; struct pt_device *pt; + u32 id; }; struct pt_cmd_queue { @@ -262,6 +263,7 @@ struct pt_device { unsigned long total_interrupts; struct pt_tasklet_data tdata; + int ver; }; /* -- cgit v1.2.3-59-g8ed1b From 98f5a44326229d3fa33db0adb3d15bdbccb59bf5 Mon Sep 17 00:00:00 2001 From: Basavaraj Natikar Date: Fri, 25 Oct 2024 15:29:29 +0530 Subject: dmaengine: ae4dma: Register AE4DMA using pt_dmaengine_register Use the pt_dmaengine_register function to register a AE4DMA DMA engine. Reviewed-by: Raju Rangoju Reviewed-by: Philipp Stanner Signed-off-by: Basavaraj Natikar Link: https://lore.kernel.org/r/20241025095931.726018-5-Basavaraj.Natikar@amd.com Signed-off-by: Vinod Koul --- drivers/dma/amd/ae4dma/ae4dma-dev.c | 4 ++ drivers/dma/amd/ae4dma/ae4dma-pci.c | 1 + drivers/dma/amd/ae4dma/ae4dma.h | 2 + drivers/dma/amd/ptdma/ptdma-dmaengine.c | 69 ++++++++++++++++++++++++++++++++- 4 files changed, 74 insertions(+), 2 deletions(-) diff --git a/drivers/dma/amd/ae4dma/ae4dma-dev.c b/drivers/dma/amd/ae4dma/ae4dma-dev.c index 7cbef9e79f38..cd84b502265e 100644 --- a/drivers/dma/amd/ae4dma/ae4dma-dev.c +++ b/drivers/dma/amd/ae4dma/ae4dma-dev.c @@ -147,5 +147,9 @@ int ae4_core_init(struct ae4_device *ae4) init_completion(&ae4cmd_q->cmp); } + ret = pt_dmaengine_register(pt); + if (ret) + ae4_destroy_work(ae4); + return ret; } diff --git a/drivers/dma/amd/ae4dma/ae4dma-pci.c b/drivers/dma/amd/ae4dma/ae4dma-pci.c index 43d36e9d1efb..aad0dc4294a3 100644 --- a/drivers/dma/amd/ae4dma/ae4dma-pci.c +++ b/drivers/dma/amd/ae4dma/ae4dma-pci.c @@ -98,6 +98,7 @@ static int ae4_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) pt = &ae4->pt; pt->dev = dev; + pt->ver = AE4_DMA_VERSION; pt->io_regs = pcim_iomap_table(pdev)[0]; if (!pt->io_regs) { diff --git a/drivers/dma/amd/ae4dma/ae4dma.h b/drivers/dma/amd/ae4dma/ae4dma.h index 92cb8c379c18..265c5d436008 100644 --- a/drivers/dma/amd/ae4dma/ae4dma.h +++ b/drivers/dma/amd/ae4dma/ae4dma.h @@ -35,6 +35,7 @@ #define AE4_Q_SZ 0x20 #define AE4_DMA_VERSION 4 +#define CMD_AE4_DESC_DW0_VAL 2 struct ae4_msix { int msix_count; @@ -55,6 +56,7 @@ struct ae4_cmd_queue { atomic64_t done_cnt; u64 q_cmd_count; u32 dridx; + u32 tail_wi; u32 id; }; diff --git a/drivers/dma/amd/ptdma/ptdma-dmaengine.c b/drivers/dma/amd/ptdma/ptdma-dmaengine.c index e2d4bc8aa1de..35c84ec9608b 100644 --- a/drivers/dma/amd/ptdma/ptdma-dmaengine.c +++ b/drivers/dma/amd/ptdma/ptdma-dmaengine.c @@ -9,6 +9,7 @@ * Author: Gary R Hook */ +#include #include "ptdma.h" #include "../ae4dma/ae4dma.h" #include "../../dmaengine.h" @@ -110,6 +111,53 @@ static struct pt_cmd_queue *pt_get_cmd_queue(struct pt_device *pt, struct pt_dma return cmd_q; } +static int ae4_core_execute_cmd(struct ae4dma_desc *desc, struct ae4_cmd_queue *ae4cmd_q) +{ + bool soc = FIELD_GET(DWORD0_SOC, desc->dwouv.dw0); + struct pt_cmd_queue *cmd_q = &ae4cmd_q->cmd_q; + + if (soc) { + desc->dwouv.dw0 |= FIELD_PREP(DWORD0_IOC, desc->dwouv.dw0); + desc->dwouv.dw0 &= ~DWORD0_SOC; + } + + mutex_lock(&ae4cmd_q->cmd_lock); + memcpy(&cmd_q->qbase[ae4cmd_q->tail_wi], desc, sizeof(struct ae4dma_desc)); + ae4cmd_q->q_cmd_count++; + ae4cmd_q->tail_wi = (ae4cmd_q->tail_wi + 1) % CMD_Q_LEN; + writel(ae4cmd_q->tail_wi, cmd_q->reg_control + AE4_WR_IDX_OFF); + mutex_unlock(&ae4cmd_q->cmd_lock); + + wake_up(&ae4cmd_q->q_w); + + return 0; +} + +static int pt_core_perform_passthru_ae4(struct pt_cmd_queue *cmd_q, + struct pt_passthru_engine *pt_engine) +{ + struct ae4_cmd_queue *ae4cmd_q = container_of(cmd_q, struct ae4_cmd_queue, cmd_q); + struct ae4dma_desc desc; + + cmd_q->cmd_error = 0; + cmd_q->total_pt_ops++; + memset(&desc, 0, sizeof(desc)); + desc.dwouv.dws.byte0 = CMD_AE4_DESC_DW0_VAL; + + desc.dw1.status = 0; + desc.dw1.err_code = 0; + desc.dw1.desc_id = 0; + + desc.length = pt_engine->src_len; + + desc.src_lo = upper_32_bits(pt_engine->src_dma); + desc.src_hi = lower_32_bits(pt_engine->src_dma); + desc.dst_lo = upper_32_bits(pt_engine->dst_dma); + desc.dst_hi = lower_32_bits(pt_engine->dst_dma); + + return ae4_core_execute_cmd(&desc, ae4cmd_q); +} + static int pt_dma_start_desc(struct pt_dma_desc *desc, struct pt_dma_chan *chan) { struct pt_passthru_engine *pt_engine; @@ -129,7 +177,10 @@ static int pt_dma_start_desc(struct pt_dma_desc *desc, struct pt_dma_chan *chan) pt->tdata.cmd = pt_cmd; /* Execute the command */ - pt_cmd->ret = pt_core_perform_passthru(cmd_q, pt_engine); + if (pt->ver == AE4_DMA_VERSION) + pt_cmd->ret = pt_core_perform_passthru_ae4(cmd_q, pt_engine); + else + pt_cmd->ret = pt_core_perform_passthru(cmd_q, pt_engine); return 0; } @@ -336,6 +387,15 @@ static void pt_issue_pending(struct dma_chan *dma_chan) pt_cmd_callback(desc, 0); } +static void pt_check_status_trans_ae4(struct pt_device *pt, struct pt_cmd_queue *cmd_q) +{ + struct ae4_cmd_queue *ae4cmd_q = container_of(cmd_q, struct ae4_cmd_queue, cmd_q); + int i; + + for (i = 0; i < CMD_Q_LEN; i++) + ae4_check_status_error(ae4cmd_q, i); +} + static enum dma_status pt_tx_status(struct dma_chan *c, dma_cookie_t cookie, struct dma_tx_state *txstate) @@ -346,7 +406,11 @@ pt_tx_status(struct dma_chan *c, dma_cookie_t cookie, cmd_q = pt_get_cmd_queue(pt, chan); - pt_check_status_trans(pt, cmd_q); + if (pt->ver == AE4_DMA_VERSION) + pt_check_status_trans_ae4(pt, cmd_q); + else + pt_check_status_trans(pt, cmd_q); + return dma_cookie_status(c, cookie, txstate); } @@ -512,6 +576,7 @@ err_cache: return ret; } +EXPORT_SYMBOL_GPL(pt_dmaengine_register); void pt_dmaengine_unregister(struct pt_device *pt) { -- cgit v1.2.3-59-g8ed1b From b10b278ea003f4ca0ac81950c109ba154b84745e Mon Sep 17 00:00:00 2001 From: Basavaraj Natikar Date: Fri, 25 Oct 2024 15:29:30 +0530 Subject: dmaengine: ptdma: Extend ptdma-debugfs to support multi-queue To support multi-channel functionality with AE4DMA engine, extend the ptdma-debugfs with reusable components. Reviewed-by: Raju Rangoju Signed-off-by: Basavaraj Natikar Link: https://lore.kernel.org/r/20241025095931.726018-6-Basavaraj.Natikar@amd.com Signed-off-by: Vinod Koul --- drivers/dma/amd/ptdma/ptdma-debugfs.c | 78 +++++++++++++++++++++++++---------- 1 file changed, 57 insertions(+), 21 deletions(-) diff --git a/drivers/dma/amd/ptdma/ptdma-debugfs.c b/drivers/dma/amd/ptdma/ptdma-debugfs.c index c8307d3044a3..0e54060d6c34 100644 --- a/drivers/dma/amd/ptdma/ptdma-debugfs.c +++ b/drivers/dma/amd/ptdma/ptdma-debugfs.c @@ -13,6 +13,7 @@ #include #include "ptdma.h" +#include "../ae4dma/ae4dma.h" /* DebugFS helpers */ #define RI_VERSION_NUM 0x0000003F @@ -23,11 +24,19 @@ static int pt_debugfs_info_show(struct seq_file *s, void *p) { struct pt_device *pt = s->private; + struct ae4_device *ae4; unsigned int regval; seq_printf(s, "Device name: %s\n", dev_name(pt->dev)); - seq_printf(s, " # Queues: %d\n", 1); - seq_printf(s, " # Cmds: %d\n", pt->cmd_count); + + if (pt->ver == AE4_DMA_VERSION) { + ae4 = container_of(pt, struct ae4_device, pt); + seq_printf(s, " # Queues: %d\n", ae4->cmd_q_count); + seq_printf(s, " # Cmds per queue: %d\n", CMD_Q_LEN); + } else { + seq_printf(s, " # Queues: %d\n", 1); + seq_printf(s, " # Cmds: %d\n", pt->cmd_count); + } regval = ioread32(pt->io_regs + CMD_PT_VERSION); @@ -55,6 +64,7 @@ static int pt_debugfs_stats_show(struct seq_file *s, void *p) static int pt_debugfs_queue_show(struct seq_file *s, void *p) { struct pt_cmd_queue *cmd_q = s->private; + struct pt_device *pt; unsigned int regval; if (!cmd_q) @@ -62,18 +72,24 @@ static int pt_debugfs_queue_show(struct seq_file *s, void *p) seq_printf(s, " Pass-Thru: %ld\n", cmd_q->total_pt_ops); - regval = ioread32(cmd_q->reg_control + 0x000C); - - seq_puts(s, " Enabled Interrupts:"); - if (regval & INT_EMPTY_QUEUE) - seq_puts(s, " EMPTY"); - if (regval & INT_QUEUE_STOPPED) - seq_puts(s, " STOPPED"); - if (regval & INT_ERROR) - seq_puts(s, " ERROR"); - if (regval & INT_COMPLETION) - seq_puts(s, " COMPLETION"); - seq_puts(s, "\n"); + pt = cmd_q->pt; + if (pt->ver == AE4_DMA_VERSION) { + regval = readl(cmd_q->reg_control + 0x4); + seq_printf(s, " Enabled Interrupts:: status 0x%x\n", regval); + } else { + regval = ioread32(cmd_q->reg_control + 0x000C); + + seq_puts(s, " Enabled Interrupts:"); + if (regval & INT_EMPTY_QUEUE) + seq_puts(s, " EMPTY"); + if (regval & INT_QUEUE_STOPPED) + seq_puts(s, " STOPPED"); + if (regval & INT_ERROR) + seq_puts(s, " ERROR"); + if (regval & INT_COMPLETION) + seq_puts(s, " COMPLETION"); + seq_puts(s, "\n"); + } return 0; } @@ -84,8 +100,12 @@ DEFINE_SHOW_ATTRIBUTE(pt_debugfs_stats); void ptdma_debugfs_setup(struct pt_device *pt) { - struct pt_cmd_queue *cmd_q; struct dentry *debugfs_q_instance; + struct ae4_cmd_queue *ae4cmd_q; + struct pt_cmd_queue *cmd_q; + struct ae4_device *ae4; + char name[30]; + int i; if (!debugfs_initialized()) return; @@ -96,11 +116,27 @@ void ptdma_debugfs_setup(struct pt_device *pt) debugfs_create_file("stats", 0400, pt->dma_dev.dbg_dev_root, pt, &pt_debugfs_stats_fops); - cmd_q = &pt->cmd_q; - - debugfs_q_instance = - debugfs_create_dir("q", pt->dma_dev.dbg_dev_root); - debugfs_create_file("stats", 0400, debugfs_q_instance, cmd_q, - &pt_debugfs_queue_fops); + if (pt->ver == AE4_DMA_VERSION) { + ae4 = container_of(pt, struct ae4_device, pt); + for (i = 0; i < ae4->cmd_q_count; i++) { + ae4cmd_q = &ae4->ae4cmd_q[i]; + cmd_q = &ae4cmd_q->cmd_q; + + memset(name, 0, sizeof(name)); + snprintf(name, 29, "q%d", ae4cmd_q->id); + + debugfs_q_instance = + debugfs_create_dir(name, pt->dma_dev.dbg_dev_root); + + debugfs_create_file("stats", 0400, debugfs_q_instance, cmd_q, + &pt_debugfs_queue_fops); + } + } else { + debugfs_q_instance = + debugfs_create_dir("q", pt->dma_dev.dbg_dev_root); + cmd_q = &pt->cmd_q; + debugfs_create_file("stats", 0400, debugfs_q_instance, cmd_q, + &pt_debugfs_queue_fops); + } } -- cgit v1.2.3-59-g8ed1b From a2d09455b27b64ff6afe409fc87f0418adf2d3fe Mon Sep 17 00:00:00 2001 From: Basavaraj Natikar Date: Fri, 25 Oct 2024 15:29:31 +0530 Subject: dmaengine: ae4dma: Register debugfs using ptdma_debugfs_setup Use the ptdma_debugfs_setup function to register debugfs for AE4DMA DMA engine. Reviewed-by: Raju Rangoju Signed-off-by: Basavaraj Natikar Link: https://lore.kernel.org/r/20241025095931.726018-7-Basavaraj.Natikar@amd.com Signed-off-by: Vinod Koul --- drivers/dma/amd/ae4dma/ae4dma-dev.c | 2 ++ drivers/dma/amd/ptdma/ptdma-debugfs.c | 1 + 2 files changed, 3 insertions(+) diff --git a/drivers/dma/amd/ae4dma/ae4dma-dev.c b/drivers/dma/amd/ae4dma/ae4dma-dev.c index cd84b502265e..8de3bef41b58 100644 --- a/drivers/dma/amd/ae4dma/ae4dma-dev.c +++ b/drivers/dma/amd/ae4dma/ae4dma-dev.c @@ -150,6 +150,8 @@ int ae4_core_init(struct ae4_device *ae4) ret = pt_dmaengine_register(pt); if (ret) ae4_destroy_work(ae4); + else + ptdma_debugfs_setup(pt); return ret; } diff --git a/drivers/dma/amd/ptdma/ptdma-debugfs.c b/drivers/dma/amd/ptdma/ptdma-debugfs.c index 0e54060d6c34..c7c90bbf6fd8 100644 --- a/drivers/dma/amd/ptdma/ptdma-debugfs.c +++ b/drivers/dma/amd/ptdma/ptdma-debugfs.c @@ -140,3 +140,4 @@ void ptdma_debugfs_setup(struct pt_device *pt) &pt_debugfs_queue_fops); } } +EXPORT_SYMBOL_GPL(ptdma_debugfs_setup); -- cgit v1.2.3-59-g8ed1b From 23417899110410f2fc1bf7dd8df381312d60c933 Mon Sep 17 00:00:00 2001 From: Kuninori Morimoto Date: Wed, 27 Nov 2024 00:42:12 +0000 Subject: dmaengine: sh: rcar-dmac: add comment for r8a779a0 compatible Add the reason why we need r8a779a0 compatible. Signed-off-by: Kuninori Morimoto Link: https://lore.kernel.org/r/87a5dlwlr0.wl-kuninori.morimoto.gx@renesas.com Signed-off-by: Vinod Koul --- drivers/dma/sh/rcar-dmac.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/drivers/dma/sh/rcar-dmac.c b/drivers/dma/sh/rcar-dmac.c index 2679c1f09faf..0c45ce8c74aa 100644 --- a/drivers/dma/sh/rcar-dmac.c +++ b/drivers/dma/sh/rcar-dmac.c @@ -2023,6 +2023,10 @@ static const struct of_device_id rcar_dmac_of_ids[] = { .compatible = "renesas,rcar-gen4-dmac", .data = &rcar_gen4_dmac_data, }, { + /* + * Backward compatibility for between v5.12 - v5.19 + * which didn't combined with "renesas,rcar-gen4-dmac" + */ .compatible = "renesas,dmac-r8a779a0", .data = &rcar_gen4_dmac_data, }, -- cgit v1.2.3-59-g8ed1b From a2186c2cd3a700df0467d2c3b23fab4e50dde373 Mon Sep 17 00:00:00 2001 From: Mesih Kilinc Date: Fri, 22 Nov 2024 17:11:25 +0100 Subject: dma-engine: sun4i: Add a quirk to support different chips MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Allwinner suniv F1C100s has similar DMA engine to sun4i. Several registers has different addresses. Total dma channels, endpoint counts and max burst counts are also different. In order to support F1C100s add a quirk structure to hold IC specific data. Signed-off-by: Mesih Kilinc [ csokas.bence: Resolve conflict in `sun4i_dma_prep_dma_cyclic()`, fix whitespace ] Signed-off-by: Csókás Bence Link: https://lore.kernel.org/r/20241122161128.2619172-2-csokas.bence@prolan.hu Signed-off-by: Vinod Koul --- drivers/dma/sun4i-dma.c | 137 +++++++++++++++++++++++++++++++++++++----------- 1 file changed, 105 insertions(+), 32 deletions(-) diff --git a/drivers/dma/sun4i-dma.c b/drivers/dma/sun4i-dma.c index f37cdf6f2179..e1c8eeb18dc4 100644 --- a/drivers/dma/sun4i-dma.c +++ b/drivers/dma/sun4i-dma.c @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -31,6 +32,8 @@ #define SUN4I_DMA_CFG_SRC_ADDR_MODE(mode) ((mode) << 5) #define SUN4I_DMA_CFG_SRC_DRQ_TYPE(type) (type) +#define SUN4I_MAX_BURST 8 + /** Normal DMA register values **/ /* Normal DMA source/destination data request type values */ @@ -132,6 +135,32 @@ #define SUN4I_DDMA_MAX_SEG_SIZE SZ_16M #define SUN4I_DMA_MAX_SEG_SIZE SUN4I_NDMA_MAX_SEG_SIZE +/* + * Hardware channels / ports representation + * + * The hardware is used in several SoCs, with differing numbers + * of channels and endpoints. This structure ties those numbers + * to a certain compatible string. + */ +struct sun4i_dma_config { + u32 ndma_nr_max_channels; + u32 ndma_nr_max_vchans; + + u32 ddma_nr_max_channels; + u32 ddma_nr_max_vchans; + + u32 dma_nr_max_channels; + + void (*set_dst_data_width)(u32 *p_cfg, s8 data_width); + void (*set_src_data_width)(u32 *p_cfg, s8 data_width); + int (*convert_burst)(u32 maxburst); + + u8 ndma_drq_sdram; + u8 ddma_drq_sdram; + + u8 max_burst; +}; + struct sun4i_dma_pchan { /* Register base of channel */ void __iomem *base; @@ -170,7 +199,7 @@ struct sun4i_dma_contract { }; struct sun4i_dma_dev { - DECLARE_BITMAP(pchans_used, SUN4I_DMA_NR_MAX_CHANNELS); + unsigned long *pchans_used; struct dma_device slave; struct sun4i_dma_pchan *pchans; struct sun4i_dma_vchan *vchans; @@ -178,6 +207,7 @@ struct sun4i_dma_dev { struct clk *clk; int irq; spinlock_t lock; + const struct sun4i_dma_config *cfg; }; static struct sun4i_dma_dev *to_sun4i_dma_dev(struct dma_device *dev) @@ -200,7 +230,17 @@ static struct device *chan2dev(struct dma_chan *chan) return &chan->dev->device; } -static int convert_burst(u32 maxburst) +static void set_dst_data_width_a10(u32 *p_cfg, s8 data_width) +{ + *p_cfg |= SUN4I_DMA_CFG_DST_DATA_WIDTH(data_width); +} + +static void set_src_data_width_a10(u32 *p_cfg, s8 data_width) +{ + *p_cfg |= SUN4I_DMA_CFG_SRC_DATA_WIDTH(data_width); +} + +static int convert_burst_a10(u32 maxburst) { if (maxburst > 8) return -EINVAL; @@ -233,15 +273,15 @@ static struct sun4i_dma_pchan *find_and_use_pchan(struct sun4i_dma_dev *priv, int i, max; /* - * pchans 0-SUN4I_NDMA_NR_MAX_CHANNELS are normal, and - * SUN4I_NDMA_NR_MAX_CHANNELS+ are dedicated ones + * pchans 0-priv->cfg->ndma_nr_max_channels are normal, and + * priv->cfg->ndma_nr_max_channels+ are dedicated ones */ if (vchan->is_dedicated) { - i = SUN4I_NDMA_NR_MAX_CHANNELS; - max = SUN4I_DMA_NR_MAX_CHANNELS; + i = priv->cfg->ndma_nr_max_channels; + max = priv->cfg->dma_nr_max_channels; } else { i = 0; - max = SUN4I_NDMA_NR_MAX_CHANNELS; + max = priv->cfg->ndma_nr_max_channels; } spin_lock_irqsave(&priv->lock, flags); @@ -444,6 +484,7 @@ generate_ndma_promise(struct dma_chan *chan, dma_addr_t src, dma_addr_t dest, size_t len, struct dma_slave_config *sconfig, enum dma_transfer_direction direction) { + struct sun4i_dma_dev *priv = to_sun4i_dma_dev(chan->device); struct sun4i_dma_promise *promise; int ret; @@ -467,13 +508,13 @@ generate_ndma_promise(struct dma_chan *chan, dma_addr_t src, dma_addr_t dest, sconfig->src_addr_width, sconfig->dst_addr_width); /* Source burst */ - ret = convert_burst(sconfig->src_maxburst); + ret = priv->cfg->convert_burst(sconfig->src_maxburst); if (ret < 0) goto fail; promise->cfg |= SUN4I_DMA_CFG_SRC_BURST_LENGTH(ret); /* Destination burst */ - ret = convert_burst(sconfig->dst_maxburst); + ret = priv->cfg->convert_burst(sconfig->dst_maxburst); if (ret < 0) goto fail; promise->cfg |= SUN4I_DMA_CFG_DST_BURST_LENGTH(ret); @@ -482,13 +523,13 @@ generate_ndma_promise(struct dma_chan *chan, dma_addr_t src, dma_addr_t dest, ret = convert_buswidth(sconfig->src_addr_width); if (ret < 0) goto fail; - promise->cfg |= SUN4I_DMA_CFG_SRC_DATA_WIDTH(ret); + priv->cfg->set_src_data_width(&promise->cfg, ret); /* Destination bus width */ ret = convert_buswidth(sconfig->dst_addr_width); if (ret < 0) goto fail; - promise->cfg |= SUN4I_DMA_CFG_DST_DATA_WIDTH(ret); + priv->cfg->set_dst_data_width(&promise->cfg, ret); return promise; @@ -510,6 +551,7 @@ static struct sun4i_dma_promise * generate_ddma_promise(struct dma_chan *chan, dma_addr_t src, dma_addr_t dest, size_t len, struct dma_slave_config *sconfig) { + struct sun4i_dma_dev *priv = to_sun4i_dma_dev(chan->device); struct sun4i_dma_promise *promise; int ret; @@ -524,13 +566,13 @@ generate_ddma_promise(struct dma_chan *chan, dma_addr_t src, dma_addr_t dest, SUN4I_DDMA_CFG_BYTE_COUNT_MODE_REMAIN; /* Source burst */ - ret = convert_burst(sconfig->src_maxburst); + ret = priv->cfg->convert_burst(sconfig->src_maxburst); if (ret < 0) goto fail; promise->cfg |= SUN4I_DMA_CFG_SRC_BURST_LENGTH(ret); /* Destination burst */ - ret = convert_burst(sconfig->dst_maxburst); + ret = priv->cfg->convert_burst(sconfig->dst_maxburst); if (ret < 0) goto fail; promise->cfg |= SUN4I_DMA_CFG_DST_BURST_LENGTH(ret); @@ -539,13 +581,13 @@ generate_ddma_promise(struct dma_chan *chan, dma_addr_t src, dma_addr_t dest, ret = convert_buswidth(sconfig->src_addr_width); if (ret < 0) goto fail; - promise->cfg |= SUN4I_DMA_CFG_SRC_DATA_WIDTH(ret); + priv->cfg->set_src_data_width(&promise->cfg, ret); /* Destination bus width */ ret = convert_buswidth(sconfig->dst_addr_width); if (ret < 0) goto fail; - promise->cfg |= SUN4I_DMA_CFG_DST_DATA_WIDTH(ret); + priv->cfg->set_dst_data_width(&promise->cfg, ret); return promise; @@ -622,6 +664,7 @@ static struct dma_async_tx_descriptor * sun4i_dma_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src, size_t len, unsigned long flags) { + struct sun4i_dma_dev *priv = to_sun4i_dma_dev(chan->device); struct sun4i_dma_vchan *vchan = to_sun4i_dma_vchan(chan); struct dma_slave_config *sconfig = &vchan->cfg; struct sun4i_dma_promise *promise; @@ -638,8 +681,8 @@ sun4i_dma_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, */ sconfig->src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES; sconfig->dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES; - sconfig->src_maxburst = 8; - sconfig->dst_maxburst = 8; + sconfig->src_maxburst = priv->cfg->max_burst; + sconfig->dst_maxburst = priv->cfg->max_burst; if (vchan->is_dedicated) promise = generate_ddma_promise(chan, src, dest, len, sconfig); @@ -654,11 +697,13 @@ sun4i_dma_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, /* Configure memcpy mode */ if (vchan->is_dedicated) { - promise->cfg |= SUN4I_DMA_CFG_SRC_DRQ_TYPE(SUN4I_DDMA_DRQ_TYPE_SDRAM) | - SUN4I_DMA_CFG_DST_DRQ_TYPE(SUN4I_DDMA_DRQ_TYPE_SDRAM); + promise->cfg |= + SUN4I_DMA_CFG_SRC_DRQ_TYPE(priv->cfg->ddma_drq_sdram) | + SUN4I_DMA_CFG_DST_DRQ_TYPE(priv->cfg->ddma_drq_sdram); } else { - promise->cfg |= SUN4I_DMA_CFG_SRC_DRQ_TYPE(SUN4I_NDMA_DRQ_TYPE_SDRAM) | - SUN4I_DMA_CFG_DST_DRQ_TYPE(SUN4I_NDMA_DRQ_TYPE_SDRAM); + promise->cfg |= + SUN4I_DMA_CFG_SRC_DRQ_TYPE(priv->cfg->ndma_drq_sdram) | + SUN4I_DMA_CFG_DST_DRQ_TYPE(priv->cfg->ndma_drq_sdram); } /* Fill the contract with our only promise */ @@ -673,6 +718,7 @@ sun4i_dma_prep_dma_cyclic(struct dma_chan *chan, dma_addr_t buf, size_t len, size_t period_len, enum dma_transfer_direction dir, unsigned long flags) { + struct sun4i_dma_dev *priv = to_sun4i_dma_dev(chan->device); struct sun4i_dma_vchan *vchan = to_sun4i_dma_vchan(chan); struct dma_slave_config *sconfig = &vchan->cfg; struct sun4i_dma_promise *promise; @@ -696,11 +742,11 @@ sun4i_dma_prep_dma_cyclic(struct dma_chan *chan, dma_addr_t buf, size_t len, if (vchan->is_dedicated) { io_mode = SUN4I_DDMA_ADDR_MODE_IO; linear_mode = SUN4I_DDMA_ADDR_MODE_LINEAR; - ram_type = SUN4I_DDMA_DRQ_TYPE_SDRAM; + ram_type = priv->cfg->ddma_drq_sdram; } else { io_mode = SUN4I_NDMA_ADDR_MODE_IO; linear_mode = SUN4I_NDMA_ADDR_MODE_LINEAR; - ram_type = SUN4I_NDMA_DRQ_TYPE_SDRAM; + ram_type = priv->cfg->ndma_drq_sdram; } if (dir == DMA_MEM_TO_DEV) { @@ -793,6 +839,7 @@ sun4i_dma_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl, unsigned int sg_len, enum dma_transfer_direction dir, unsigned long flags, void *context) { + struct sun4i_dma_dev *priv = to_sun4i_dma_dev(chan->device); struct sun4i_dma_vchan *vchan = to_sun4i_dma_vchan(chan); struct dma_slave_config *sconfig = &vchan->cfg; struct sun4i_dma_promise *promise; @@ -818,11 +865,11 @@ sun4i_dma_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl, if (vchan->is_dedicated) { io_mode = SUN4I_DDMA_ADDR_MODE_IO; linear_mode = SUN4I_DDMA_ADDR_MODE_LINEAR; - ram_type = SUN4I_DDMA_DRQ_TYPE_SDRAM; + ram_type = priv->cfg->ddma_drq_sdram; } else { io_mode = SUN4I_NDMA_ADDR_MODE_IO; linear_mode = SUN4I_NDMA_ADDR_MODE_LINEAR; - ram_type = SUN4I_NDMA_DRQ_TYPE_SDRAM; + ram_type = priv->cfg->ndma_drq_sdram; } if (dir == DMA_MEM_TO_DEV) @@ -1150,6 +1197,10 @@ static int sun4i_dma_probe(struct platform_device *pdev) if (!priv) return -ENOMEM; + priv->cfg = of_device_get_match_data(&pdev->dev); + if (!priv->cfg) + return -ENODEV; + priv->base = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(priv->base)) return PTR_ERR(priv->base); @@ -1197,23 +1248,26 @@ static int sun4i_dma_probe(struct platform_device *pdev) priv->slave.dev = &pdev->dev; - priv->pchans = devm_kcalloc(&pdev->dev, SUN4I_DMA_NR_MAX_CHANNELS, + priv->pchans = devm_kcalloc(&pdev->dev, priv->cfg->dma_nr_max_channels, sizeof(struct sun4i_dma_pchan), GFP_KERNEL); priv->vchans = devm_kcalloc(&pdev->dev, SUN4I_DMA_NR_MAX_VCHANS, sizeof(struct sun4i_dma_vchan), GFP_KERNEL); - if (!priv->vchans || !priv->pchans) + priv->pchans_used = devm_kcalloc(&pdev->dev, + BITS_TO_LONGS(priv->cfg->dma_nr_max_channels), + sizeof(unsigned long), GFP_KERNEL); + if (!priv->vchans || !priv->pchans || !priv->pchans_used) return -ENOMEM; /* - * [0..SUN4I_NDMA_NR_MAX_CHANNELS) are normal pchans, and - * [SUN4I_NDMA_NR_MAX_CHANNELS..SUN4I_DMA_NR_MAX_CHANNELS) are + * [0..priv->cfg->ndma_nr_max_channels) are normal pchans, and + * [priv->cfg->ndma_nr_max_channels..priv->cfg->dma_nr_max_channels) are * dedicated ones */ - for (i = 0; i < SUN4I_NDMA_NR_MAX_CHANNELS; i++) + for (i = 0; i < priv->cfg->ndma_nr_max_channels; i++) priv->pchans[i].base = priv->base + SUN4I_NDMA_CHANNEL_REG_BASE(i); - for (j = 0; i < SUN4I_DMA_NR_MAX_CHANNELS; i++, j++) { + for (j = 0; i < priv->cfg->dma_nr_max_channels; i++, j++) { priv->pchans[i].base = priv->base + SUN4I_DDMA_CHANNEL_REG_BASE(j); priv->pchans[i].is_dedicated = 1; @@ -1284,8 +1338,27 @@ static void sun4i_dma_remove(struct platform_device *pdev) clk_disable_unprepare(priv->clk); } +static struct sun4i_dma_config sun4i_a10_dma_cfg = { + .ndma_nr_max_channels = SUN4I_NDMA_NR_MAX_CHANNELS, + .ndma_nr_max_vchans = SUN4I_NDMA_NR_MAX_VCHANS, + + .ddma_nr_max_channels = SUN4I_DDMA_NR_MAX_CHANNELS, + .ddma_nr_max_vchans = SUN4I_DDMA_NR_MAX_VCHANS, + + .dma_nr_max_channels = SUN4I_DMA_NR_MAX_CHANNELS, + + .set_dst_data_width = set_dst_data_width_a10, + .set_src_data_width = set_src_data_width_a10, + .convert_burst = convert_burst_a10, + + .ndma_drq_sdram = SUN4I_NDMA_DRQ_TYPE_SDRAM, + .ddma_drq_sdram = SUN4I_DDMA_DRQ_TYPE_SDRAM, + + .max_burst = SUN4I_MAX_BURST, +}; + static const struct of_device_id sun4i_dma_match[] = { - { .compatible = "allwinner,sun4i-a10-dma" }, + { .compatible = "allwinner,sun4i-a10-dma", .data = &sun4i_a10_dma_cfg }, { /* sentinel */ }, }; MODULE_DEVICE_TABLE(of, sun4i_dma_match); -- cgit v1.2.3-59-g8ed1b From e17ca0effaed4616a0e07fa77d7ee1acdad9c85e Mon Sep 17 00:00:00 2001 From: Mesih Kilinc Date: Fri, 22 Nov 2024 17:11:29 +0100 Subject: dma-engine: sun4i: Add has_reset option to quirk MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Allwinner suniv F1C100s has a reset bit for DMA in CCU. Sun4i do not has this bit but in order to support suniv we need to add it. So add support for reset bit. Signed-off-by: Mesih Kilinc [ csokas.bence: Rebased and addressed comments ] Signed-off-by: Csókás Bence Link: https://lore.kernel.org/r/20241122161128.2619172-3-csokas.bence@prolan.hu Signed-off-by: Vinod Koul --- drivers/dma/sun4i-dma.c | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/drivers/dma/sun4i-dma.c b/drivers/dma/sun4i-dma.c index e1c8eeb18dc4..999ccd47a971 100644 --- a/drivers/dma/sun4i-dma.c +++ b/drivers/dma/sun4i-dma.c @@ -15,6 +15,7 @@ #include #include #include +#include #include #include @@ -159,6 +160,7 @@ struct sun4i_dma_config { u8 ddma_drq_sdram; u8 max_burst; + bool has_reset; }; struct sun4i_dma_pchan { @@ -208,6 +210,7 @@ struct sun4i_dma_dev { int irq; spinlock_t lock; const struct sun4i_dma_config *cfg; + struct reset_control *rst; }; static struct sun4i_dma_dev *to_sun4i_dma_dev(struct dma_device *dev) @@ -1215,6 +1218,13 @@ static int sun4i_dma_probe(struct platform_device *pdev) return PTR_ERR(priv->clk); } + if (priv->cfg->has_reset) { + priv->rst = devm_reset_control_get_exclusive_deasserted(&pdev->dev, NULL); + if (IS_ERR(priv->rst)) + return dev_err_probe(&pdev->dev, PTR_ERR(priv->rst), + "Failed to get reset control\n"); + } + platform_set_drvdata(pdev, priv); spin_lock_init(&priv->lock); @@ -1355,6 +1365,7 @@ static struct sun4i_dma_config sun4i_a10_dma_cfg = { .ddma_drq_sdram = SUN4I_DDMA_DRQ_TYPE_SDRAM, .max_burst = SUN4I_MAX_BURST, + .has_reset = false, }; static const struct of_device_id sun4i_dma_match[] = { -- cgit v1.2.3-59-g8ed1b From fdcdcc57152a5f4250dab5a1ec17d2e4b8311c02 Mon Sep 17 00:00:00 2001 From: "Csókás, Bence" Date: Fri, 22 Nov 2024 17:11:30 +0100 Subject: dt-bindings: dmaengine: Add Allwinner suniv F1C100s DMA MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add compatible string for Allwinner suniv F1C100s DMA. Acked-by: Conor Dooley [ csokas.bence: Reimplemented Mesih Kilinc's binding in YAML ] Signed-off-by: Csókás Bence Link: https://lore.kernel.org/r/20241122161128.2619172-4-csokas.bence@prolan.hu Signed-off-by: Vinod Koul --- Documentation/devicetree/bindings/dma/allwinner,sun4i-a10-dma.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Documentation/devicetree/bindings/dma/allwinner,sun4i-a10-dma.yaml b/Documentation/devicetree/bindings/dma/allwinner,sun4i-a10-dma.yaml index 02d5bd035409..9b5180c0a7c4 100644 --- a/Documentation/devicetree/bindings/dma/allwinner,sun4i-a10-dma.yaml +++ b/Documentation/devicetree/bindings/dma/allwinner,sun4i-a10-dma.yaml @@ -22,7 +22,9 @@ properties: number. compatible: - const: allwinner,sun4i-a10-dma + enum: + - allwinner,sun4i-a10-dma + - allwinner,suniv-f1c100s-dma reg: maxItems: 1 -- cgit v1.2.3-59-g8ed1b From 6faf1ccadad6718957753d6cda2281416b288bde Mon Sep 17 00:00:00 2001 From: Mesih Kilinc Date: Fri, 22 Nov 2024 17:11:31 +0100 Subject: dma-engine: sun4i: Add support for Allwinner suniv F1C100s MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit DMA of Allwinner suniv F1C100s is similar to sun4i. It has 4 NDMA, 4 DDMA channels and endpoints are different. Also F1C100s has reset bit for DMA in CCU. Add support for it. Signed-off-by: Mesih Kilinc [ csokas.bence: Rebased on current master ] Signed-off-by: Csókás Bence Link: https://lore.kernel.org/r/20241122161128.2619172-5-csokas.bence@prolan.hu Signed-off-by: Vinod Koul --- drivers/dma/Kconfig | 4 ++-- drivers/dma/sun4i-dma.c | 60 +++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 62 insertions(+), 2 deletions(-) diff --git a/drivers/dma/Kconfig b/drivers/dma/Kconfig index 0cacfefaa2d9..8afea2e23360 100644 --- a/drivers/dma/Kconfig +++ b/drivers/dma/Kconfig @@ -162,8 +162,8 @@ config DMA_SA11X0 config DMA_SUN4I tristate "Allwinner A10 DMA SoCs support" - depends on MACH_SUN4I || MACH_SUN5I || MACH_SUN7I - default (MACH_SUN4I || MACH_SUN5I || MACH_SUN7I) + depends on MACH_SUN4I || MACH_SUN5I || MACH_SUN7I || MACH_SUNIV + default (MACH_SUN4I || MACH_SUN5I || MACH_SUN7I || MACH_SUNIV) select DMA_ENGINE select DMA_VIRTUAL_CHANNELS help diff --git a/drivers/dma/sun4i-dma.c b/drivers/dma/sun4i-dma.c index 999ccd47a971..24796aaaddfa 100644 --- a/drivers/dma/sun4i-dma.c +++ b/drivers/dma/sun4i-dma.c @@ -33,7 +33,11 @@ #define SUN4I_DMA_CFG_SRC_ADDR_MODE(mode) ((mode) << 5) #define SUN4I_DMA_CFG_SRC_DRQ_TYPE(type) (type) +#define SUNIV_DMA_CFG_DST_DATA_WIDTH(width) ((width) << 24) +#define SUNIV_DMA_CFG_SRC_DATA_WIDTH(width) ((width) << 8) + #define SUN4I_MAX_BURST 8 +#define SUNIV_MAX_BURST 4 /** Normal DMA register values **/ @@ -41,6 +45,9 @@ #define SUN4I_NDMA_DRQ_TYPE_SDRAM 0x16 #define SUN4I_NDMA_DRQ_TYPE_LIMIT (0x1F + 1) +#define SUNIV_NDMA_DRQ_TYPE_SDRAM 0x11 +#define SUNIV_NDMA_DRQ_TYPE_LIMIT (0x17 + 1) + /** Normal DMA register layout **/ /* Dedicated DMA source/destination address mode values */ @@ -54,6 +61,9 @@ #define SUN4I_NDMA_CFG_BYTE_COUNT_MODE_REMAIN BIT(15) #define SUN4I_NDMA_CFG_SRC_NON_SECURE BIT(6) +#define SUNIV_NDMA_CFG_CONT_MODE BIT(29) +#define SUNIV_NDMA_CFG_WAIT_STATE(n) ((n) << 26) + /** Dedicated DMA register values **/ /* Dedicated DMA source/destination address mode values */ @@ -66,6 +76,9 @@ #define SUN4I_DDMA_DRQ_TYPE_SDRAM 0x1 #define SUN4I_DDMA_DRQ_TYPE_LIMIT (0x1F + 1) +#define SUNIV_DDMA_DRQ_TYPE_SDRAM 0x1 +#define SUNIV_DDMA_DRQ_TYPE_LIMIT (0x9 + 1) + /** Dedicated DMA register layout **/ /* Dedicated DMA configuration register layout */ @@ -119,6 +132,11 @@ #define SUN4I_DMA_NR_MAX_VCHANS \ (SUN4I_NDMA_NR_MAX_VCHANS + SUN4I_DDMA_NR_MAX_VCHANS) +#define SUNIV_NDMA_NR_MAX_CHANNELS 4 +#define SUNIV_DDMA_NR_MAX_CHANNELS 4 +#define SUNIV_NDMA_NR_MAX_VCHANS (24 * 2 - 1) +#define SUNIV_DDMA_NR_MAX_VCHANS 10 + /* This set of SUN4I_DDMA timing parameters were found experimentally while * working with the SPI driver and seem to make it behave correctly */ #define SUN4I_DDMA_MAGIC_SPI_PARAMETERS \ @@ -243,6 +261,16 @@ static void set_src_data_width_a10(u32 *p_cfg, s8 data_width) *p_cfg |= SUN4I_DMA_CFG_SRC_DATA_WIDTH(data_width); } +static void set_dst_data_width_f1c100s(u32 *p_cfg, s8 data_width) +{ + *p_cfg |= SUNIV_DMA_CFG_DST_DATA_WIDTH(data_width); +} + +static void set_src_data_width_f1c100s(u32 *p_cfg, s8 data_width) +{ + *p_cfg |= SUNIV_DMA_CFG_SRC_DATA_WIDTH(data_width); +} + static int convert_burst_a10(u32 maxburst) { if (maxburst > 8) @@ -252,6 +280,15 @@ static int convert_burst_a10(u32 maxburst) return (maxburst >> 2); } +static int convert_burst_f1c100s(u32 maxburst) +{ + if (maxburst > 4) + return -EINVAL; + + /* 1 -> 0, 4 -> 1 */ + return (maxburst >> 2); +} + static int convert_buswidth(enum dma_slave_buswidth addr_width) { if (addr_width > DMA_SLAVE_BUSWIDTH_4_BYTES) @@ -1368,8 +1405,31 @@ static struct sun4i_dma_config sun4i_a10_dma_cfg = { .has_reset = false, }; +static struct sun4i_dma_config suniv_f1c100s_dma_cfg = { + .ndma_nr_max_channels = SUNIV_NDMA_NR_MAX_CHANNELS, + .ndma_nr_max_vchans = SUNIV_NDMA_NR_MAX_VCHANS, + + .ddma_nr_max_channels = SUNIV_DDMA_NR_MAX_CHANNELS, + .ddma_nr_max_vchans = SUNIV_DDMA_NR_MAX_VCHANS, + + .dma_nr_max_channels = SUNIV_NDMA_NR_MAX_CHANNELS + + SUNIV_DDMA_NR_MAX_CHANNELS, + + .set_dst_data_width = set_dst_data_width_f1c100s, + .set_src_data_width = set_src_data_width_f1c100s, + .convert_burst = convert_burst_f1c100s, + + .ndma_drq_sdram = SUNIV_NDMA_DRQ_TYPE_SDRAM, + .ddma_drq_sdram = SUNIV_DDMA_DRQ_TYPE_SDRAM, + + .max_burst = SUNIV_MAX_BURST, + .has_reset = true, +}; + static const struct of_device_id sun4i_dma_match[] = { { .compatible = "allwinner,sun4i-a10-dma", .data = &sun4i_a10_dma_cfg }, + { .compatible = "allwinner,suniv-f1c100s-dma", + .data = &suniv_f1c100s_dma_cfg }, { /* sentinel */ }, }; MODULE_DEVICE_TABLE(of, sun4i_dma_match); -- cgit v1.2.3-59-g8ed1b From 80a9b50b1333fce036154615c31be981044108a1 Mon Sep 17 00:00:00 2001 From: Fenghua Yu Date: Thu, 24 Oct 2024 11:35:00 -0700 Subject: dmaengine: idxd: Add a new IAA device ID on Panther Lake family platforms A new IAA device ID, 0xb02d, is introduced across all Panther Lake family platforms. Add the device ID to the IDXD driver. Signed-off-by: Fenghua Yu Link: https://lore.kernel.org/r/20241024183500.281268-1-fenghua.yu@intel.com Signed-off-by: Vinod Koul --- drivers/dma/idxd/init.c | 2 ++ drivers/dma/idxd/registers.h | 1 + 2 files changed, 3 insertions(+) diff --git a/drivers/dma/idxd/init.c b/drivers/dma/idxd/init.c index 234c1c658ec7..a96f49567313 100644 --- a/drivers/dma/idxd/init.c +++ b/drivers/dma/idxd/init.c @@ -78,6 +78,8 @@ static struct pci_device_id idxd_pci_tbl[] = { { PCI_DEVICE_DATA(INTEL, IAX_SPR0, &idxd_driver_data[IDXD_TYPE_IAX]) }, /* IAA on DMR platforms */ { PCI_DEVICE_DATA(INTEL, IAA_DMR, &idxd_driver_data[IDXD_TYPE_IAX]) }, + /* IAA PTL platforms */ + { PCI_DEVICE_DATA(INTEL, IAA_PTL, &idxd_driver_data[IDXD_TYPE_IAX]) }, { 0, } }; MODULE_DEVICE_TABLE(pci, idxd_pci_tbl); diff --git a/drivers/dma/idxd/registers.h b/drivers/dma/idxd/registers.h index c426511f2104..006ba206ab1b 100644 --- a/drivers/dma/idxd/registers.h +++ b/drivers/dma/idxd/registers.h @@ -9,6 +9,7 @@ #define PCI_DEVICE_ID_INTEL_DSA_GNRD 0x11fb #define PCI_DEVICE_ID_INTEL_DSA_DMR 0x1212 #define PCI_DEVICE_ID_INTEL_IAA_DMR 0x1216 +#define PCI_DEVICE_ID_INTEL_IAA_PTL 0xb02d #define DEVICE_VERSION_1 0x100 #define DEVICE_VERSION_2 0x200 -- cgit v1.2.3-59-g8ed1b From 24947be1c7da99036e8336de160457599bc124d7 Mon Sep 17 00:00:00 2001 From: Melody Olvera Date: Mon, 21 Oct 2024 16:05:00 -0700 Subject: dt-bindings: dma: qcom,gpi: Document the sm8750 GPI DMA engine Document the GPI DMA engine on the sm8750 platform. Signed-off-by: Melody Olvera Reviewed-by: Krzysztof Kozlowski Link: https://lore.kernel.org/r/20241021230500.2632527-1-quic_molvera@quicinc.com Signed-off-by: Vinod Koul --- Documentation/devicetree/bindings/dma/qcom,gpi.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/Documentation/devicetree/bindings/dma/qcom,gpi.yaml b/Documentation/devicetree/bindings/dma/qcom,gpi.yaml index 2b95e86e40d5..7052468b15c8 100644 --- a/Documentation/devicetree/bindings/dma/qcom,gpi.yaml +++ b/Documentation/devicetree/bindings/dma/qcom,gpi.yaml @@ -37,6 +37,7 @@ properties: - qcom,sm8450-gpi-dma - qcom,sm8550-gpi-dma - qcom,sm8650-gpi-dma + - qcom,sm8750-gpi-dma - qcom,x1e80100-gpi-dma - const: qcom,sm6350-gpi-dma - items: -- cgit v1.2.3-59-g8ed1b From 9d880452fb3edc4645e28264381ce35606fb1b19 Mon Sep 17 00:00:00 2001 From: Colin Ian King Date: Thu, 12 Sep 2024 14:10:17 +0100 Subject: dmaengine: amd: qdma: make read-only arrays h2c_types and c2h_types static const Don't populate the read-only arrays h2c_types and c2h_types on the stack at run time, instead make them static const. Signed-off-by: Colin Ian King Link: https://lore.kernel.org/r/20240912131017.588141-1-colin.i.king@gmail.com Signed-off-by: Vinod Koul --- drivers/dma/amd/qdma/qdma.c | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/drivers/dma/amd/qdma/qdma.c b/drivers/dma/amd/qdma/qdma.c index 6d9079458fe9..a2c6d9ca68cd 100644 --- a/drivers/dma/amd/qdma/qdma.c +++ b/drivers/dma/amd/qdma/qdma.c @@ -283,16 +283,20 @@ static int qdma_check_queue_status(struct qdma_device *qdev, static int qdma_clear_queue_context(const struct qdma_queue *queue) { - enum qdma_ctxt_type h2c_types[] = { QDMA_CTXT_DESC_SW_H2C, - QDMA_CTXT_DESC_HW_H2C, - QDMA_CTXT_DESC_CR_H2C, - QDMA_CTXT_PFTCH, }; - enum qdma_ctxt_type c2h_types[] = { QDMA_CTXT_DESC_SW_C2H, - QDMA_CTXT_DESC_HW_C2H, - QDMA_CTXT_DESC_CR_C2H, - QDMA_CTXT_PFTCH, }; + static const enum qdma_ctxt_type h2c_types[] = { + QDMA_CTXT_DESC_SW_H2C, + QDMA_CTXT_DESC_HW_H2C, + QDMA_CTXT_DESC_CR_H2C, + QDMA_CTXT_PFTCH, + }; + static const enum qdma_ctxt_type c2h_types[] = { + QDMA_CTXT_DESC_SW_C2H, + QDMA_CTXT_DESC_HW_C2H, + QDMA_CTXT_DESC_CR_C2H, + QDMA_CTXT_PFTCH, + }; struct qdma_device *qdev = queue->qdev; - enum qdma_ctxt_type *type; + const enum qdma_ctxt_type *type; int ret, num, i; if (queue->dir == DMA_MEM_TO_DEV) { -- cgit v1.2.3-59-g8ed1b From 308213731f8cc772245adb729ae2364fb1d20e98 Mon Sep 17 00:00:00 2001 From: "Dr. David Alan Gilbert" Date: Sat, 21 Dec 2024 14:16:35 +0000 Subject: dmaengine: idxd: Remove unused idxd_(un)register_bus_type idxd_register_bus_type() and idxd_unregister_bus_type() have been unused since 2021's commit d9e5481fca74 ("dmaengine: dsa: move dsa_bus_type out of idxd driver to standalone") Remove them. Signed-off-by: Dr. David Alan Gilbert Reviewed-by: Fenghua Yu Link: https://lore.kernel.org/r/20241221141635.69412-1-linux@treblig.org Signed-off-by: Vinod Koul --- drivers/dma/idxd/idxd.h | 2 -- drivers/dma/idxd/sysfs.c | 10 ---------- 2 files changed, 12 deletions(-) diff --git a/drivers/dma/idxd/idxd.h b/drivers/dma/idxd/idxd.h index d84e21daa991..ff33ebf08ee7 100644 --- a/drivers/dma/idxd/idxd.h +++ b/drivers/dma/idxd/idxd.h @@ -725,8 +725,6 @@ static inline void idxd_desc_complete(struct idxd_desc *desc, &desc->txd, &status); } -int idxd_register_bus_type(void); -void idxd_unregister_bus_type(void); int idxd_register_devices(struct idxd_device *idxd); void idxd_unregister_devices(struct idxd_device *idxd); void idxd_wqs_quiesce(struct idxd_device *idxd); diff --git a/drivers/dma/idxd/sysfs.c b/drivers/dma/idxd/sysfs.c index f706eae0e76b..6af493f6ba77 100644 --- a/drivers/dma/idxd/sysfs.c +++ b/drivers/dma/idxd/sysfs.c @@ -1979,13 +1979,3 @@ void idxd_unregister_devices(struct idxd_device *idxd) device_unregister(group_confdev(group)); } } - -int idxd_register_bus_type(void) -{ - return bus_register(&dsa_bus_type); -} - -void idxd_unregister_bus_type(void) -{ - bus_unregister(&dsa_bus_type); -} -- cgit v1.2.3-59-g8ed1b From 57a7138d0627309d469719f1845d2778c251f358 Mon Sep 17 00:00:00 2001 From: Md Sadre Alam Date: Fri, 20 Dec 2024 15:12:03 +0530 Subject: dmaengine: qcom: bam_dma: Avoid writing unavailable register Avoid writing unavailable register in BAM-Lite mode. BAM_DESC_CNT_TRSHLD register is unavailable in BAM-Lite mode. Its only available in BAM-NDP mode. So only write this register for clients who is using BAM-NDP. Signed-off-by: Md Sadre Alam Link: https://lore.kernel.org/r/20241220094203.3510335-1-quic_mdalam@quicinc.com Signed-off-by: Vinod Koul --- drivers/dma/qcom/bam_dma.c | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/drivers/dma/qcom/bam_dma.c b/drivers/dma/qcom/bam_dma.c index bbc3276992bb..c14557efd577 100644 --- a/drivers/dma/qcom/bam_dma.c +++ b/drivers/dma/qcom/bam_dma.c @@ -59,6 +59,9 @@ struct bam_desc_hw { #define DESC_FLAG_NWD BIT(12) #define DESC_FLAG_CMD BIT(11) +#define BAM_NDP_REVISION_START 0x20 +#define BAM_NDP_REVISION_END 0x27 + struct bam_async_desc { struct virt_dma_desc vd; @@ -398,6 +401,7 @@ struct bam_device { /* dma start transaction tasklet */ struct tasklet_struct task; + u32 bam_revision; }; /** @@ -441,8 +445,10 @@ static void bam_reset(struct bam_device *bdev) writel_relaxed(val, bam_addr(bdev, 0, BAM_CTRL)); /* set descriptor threshold, start with 4 bytes */ - writel_relaxed(DEFAULT_CNT_THRSHLD, - bam_addr(bdev, 0, BAM_DESC_CNT_TRSHLD)); + if (in_range(bdev->bam_revision, BAM_NDP_REVISION_START, + BAM_NDP_REVISION_END)) + writel_relaxed(DEFAULT_CNT_THRSHLD, + bam_addr(bdev, 0, BAM_DESC_CNT_TRSHLD)); /* Enable default set of h/w workarounds, ie all except BAM_FULL_PIPE */ writel_relaxed(BAM_CNFG_BITS_DEFAULT, bam_addr(bdev, 0, BAM_CNFG_BITS)); @@ -1000,9 +1006,10 @@ static void bam_apply_new_config(struct bam_chan *bchan, maxburst = bchan->slave.src_maxburst; else maxburst = bchan->slave.dst_maxburst; - - writel_relaxed(maxburst, - bam_addr(bdev, 0, BAM_DESC_CNT_TRSHLD)); + if (in_range(bdev->bam_revision, BAM_NDP_REVISION_START, + BAM_NDP_REVISION_END)) + writel_relaxed(maxburst, + bam_addr(bdev, 0, BAM_DESC_CNT_TRSHLD)); } bchan->reconfigure = 0; @@ -1192,10 +1199,11 @@ static int bam_init(struct bam_device *bdev) u32 val; /* read revision and configuration information */ - if (!bdev->num_ees) { - val = readl_relaxed(bam_addr(bdev, 0, BAM_REVISION)); + val = readl_relaxed(bam_addr(bdev, 0, BAM_REVISION)); + if (!bdev->num_ees) bdev->num_ees = (val >> NUM_EES_SHIFT) & NUM_EES_MASK; - } + + bdev->bam_revision = val & REVISION_MASK; /* check that configured EE is within range */ if (bdev->ee >= bdev->num_ees) -- cgit v1.2.3-59-g8ed1b From a4b00f54a20bba0bbfc952a8cb4c3cbe29e408b0 Mon Sep 17 00:00:00 2001 From: Larisa Grigore Date: Thu, 19 Dec 2024 12:24:10 +0200 Subject: dmaengine: fsl-edma: select of_dma_xlate based on the dmamuxs presence Select the of_dma_xlate function based on the dmamuxs definition rather than the FSL_EDMA_DRV_SPLIT_REG flag, which pertains to the eDMA3 layout. This change is a prerequisite for the S32G platforms, which integrate both eDMAv3 and DMAMUX. Existing platforms with FSL_EDMA_DRV_SPLIT_REG will not be impacted, as they all have dmamuxs set to zero. Signed-off-by: Larisa Grigore Reviewed-by: Frank Li Link: https://lore.kernel.org/r/20241219102415.1208328-2-larisa.grigore@oss.nxp.com Signed-off-by: Vinod Koul --- drivers/dma/fsl-edma-main.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/dma/fsl-edma-main.c b/drivers/dma/fsl-edma-main.c index 60de1003193a..2a7d19f51287 100644 --- a/drivers/dma/fsl-edma-main.c +++ b/drivers/dma/fsl-edma-main.c @@ -646,7 +646,7 @@ static int fsl_edma_probe(struct platform_device *pdev) } ret = of_dma_controller_register(np, - drvdata->flags & FSL_EDMA_DRV_SPLIT_REG ? fsl_edma3_xlate : fsl_edma_xlate, + drvdata->dmamuxs ? fsl_edma_xlate : fsl_edma3_xlate, fsl_edma); if (ret) { dev_err(&pdev->dev, -- cgit v1.2.3-59-g8ed1b From e7732945db1d4612072e26e5b459d74e9d790b7c Mon Sep 17 00:00:00 2001 From: Larisa Grigore Date: Thu, 19 Dec 2024 12:24:11 +0200 Subject: dmaengine: fsl-edma: remove FSL_EDMA_DRV_SPLIT_REG check when parsing muxbase Clean up dead code. dmamuxs is always 0 when FSL_EDMA_DRV_SPLIT_REG set. So it is redundant to check FSL_EDMA_DRV_SPLIT_REG again in the for loop because it will never enter for loop. Signed-off-by: Larisa Grigore Reviewed-by: Frank Li Link: https://lore.kernel.org/r/20241219102415.1208328-3-larisa.grigore@oss.nxp.com Signed-off-by: Vinod Koul --- drivers/dma/fsl-edma-main.c | 4 ---- 1 file changed, 4 deletions(-) diff --git a/drivers/dma/fsl-edma-main.c b/drivers/dma/fsl-edma-main.c index 2a7d19f51287..9873cce00c68 100644 --- a/drivers/dma/fsl-edma-main.c +++ b/drivers/dma/fsl-edma-main.c @@ -517,10 +517,6 @@ static int fsl_edma_probe(struct platform_device *pdev) for (i = 0; i < fsl_edma->drvdata->dmamuxs; i++) { char clkname[32]; - /* eDMAv3 mux register move to TCD area if ch_mux exist */ - if (drvdata->flags & FSL_EDMA_DRV_SPLIT_REG) - break; - fsl_edma->muxbase[i] = devm_platform_ioremap_resource(pdev, 1 + i); if (IS_ERR(fsl_edma->muxbase[i])) { -- cgit v1.2.3-59-g8ed1b From 57eeb0a566a82621ab731b0372a5a2894b0d572e Mon Sep 17 00:00:00 2001 From: Larisa Grigore Date: Thu, 19 Dec 2024 12:24:12 +0200 Subject: dt-bindings: dma: fsl-edma: add nxp,s32g2-edma compatible string Introduce the compatible strings 'nxp,s32g2-edma' and 'nxp,s32g3-edma' to enable the support for the eDMAv3 present on S32G2/S32G3 platforms. The S32G2/S32G3 eDMA architecture features 32 DMA channels. Each of the two eDMA instances is integrated with two DMAMUX blocks. Another particularity of these SoCs is that the interrupts are shared between channels in the following way: - DMA Channels 0-15 share the 'tx-0-15' interrupt - DMA Channels 16-31 share the 'tx-16-31' interrupt - all channels share the 'err' interrupt Signed-off-by: Larisa Grigore Reviewed-by: Frank Li Acked-by: Conor Dooley Link: https://lore.kernel.org/r/20241219102415.1208328-4-larisa.grigore@oss.nxp.com Signed-off-by: Vinod Koul --- .../devicetree/bindings/dma/fsl,edma.yaml | 34 ++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/Documentation/devicetree/bindings/dma/fsl,edma.yaml b/Documentation/devicetree/bindings/dma/fsl,edma.yaml index d54140f18d34..4f925469533e 100644 --- a/Documentation/devicetree/bindings/dma/fsl,edma.yaml +++ b/Documentation/devicetree/bindings/dma/fsl,edma.yaml @@ -26,9 +26,13 @@ properties: - fsl,imx93-edma3 - fsl,imx93-edma4 - fsl,imx95-edma5 + - nxp,s32g2-edma - items: - const: fsl,ls1028a-edma - const: fsl,vf610-edma + - items: + - const: nxp,s32g3-edma + - const: nxp,s32g2-edma reg: minItems: 1 @@ -221,6 +225,36 @@ allOf: properties: power-domains: false + - if: + properties: + compatible: + contains: + const: nxp,s32g2-edma + then: + properties: + clocks: + minItems: 2 + maxItems: 2 + clock-names: + items: + - const: dmamux0 + - const: dmamux1 + interrupts: + minItems: 3 + maxItems: 3 + interrupt-names: + items: + - const: tx-0-15 + - const: tx-16-31 + - const: err + reg: + minItems: 3 + maxItems: 3 + "#dma-cells": + const: 2 + dma-channels: + const: 32 + unevaluatedProperties: false examples: -- cgit v1.2.3-59-g8ed1b From 2500243e5cc2e45e6fae826cbc64e9986a9b8194 Mon Sep 17 00:00:00 2001 From: Larisa Grigore Date: Thu, 19 Dec 2024 12:24:13 +0200 Subject: dmaengine: fsl-edma: add support for S32G based platforms S32G2/S32G3 includes two system eDMA instances based on v3 version, each of them integrated with two DMAMUX blocks. Another particularity of these SoCs is that the interrupts are shared between channels as follows: - DMA Channels 0-15 share the 'tx-0-15' interrupt - DMA Channels 16-31 share the 'tx-16-31' interrupt - all channels share the 'err' interrupt Signed-off-by: Larisa Grigore Co-developed-by: Ciprian Marian Costea Signed-off-by: Ciprian Marian Costea Reviewed-by: Frank Li Link: https://lore.kernel.org/r/20241219102415.1208328-5-larisa.grigore@oss.nxp.com Signed-off-by: Vinod Koul --- drivers/dma/fsl-edma-common.h | 3 ++ drivers/dma/fsl-edma-main.c | 109 +++++++++++++++++++++++++++++++++++++++++- 2 files changed, 111 insertions(+), 1 deletion(-) diff --git a/drivers/dma/fsl-edma-common.h b/drivers/dma/fsl-edma-common.h index ce37e1ee9c46..707fea4139b6 100644 --- a/drivers/dma/fsl-edma-common.h +++ b/drivers/dma/fsl-edma-common.h @@ -68,6 +68,8 @@ #define EDMA_V3_CH_CSR_EEI BIT(2) #define EDMA_V3_CH_CSR_DONE BIT(30) #define EDMA_V3_CH_CSR_ACTIVE BIT(31) +#define EDMA_V3_CH_ES_ERR BIT(31) +#define EDMA_V3_MP_ES_VLD BIT(31) enum fsl_edma_pm_state { RUNNING = 0, @@ -240,6 +242,7 @@ struct fsl_edma_engine { const struct fsl_edma_drvdata *drvdata; u32 n_chans; int txirq; + int txirq_16_31; int errirq; bool big_endian; struct edma_regs regs; diff --git a/drivers/dma/fsl-edma-main.c b/drivers/dma/fsl-edma-main.c index 9873cce00c68..c9e3252d0da0 100644 --- a/drivers/dma/fsl-edma-main.c +++ b/drivers/dma/fsl-edma-main.c @@ -3,10 +3,11 @@ * drivers/dma/fsl-edma.c * * Copyright 2013-2014 Freescale Semiconductor, Inc. + * Copyright 2024 NXP * * Driver for the Freescale eDMA engine with flexible channel multiplexing * capability for DMA request sources. The eDMA block can be found on some - * Vybrid and Layerscape SoCs. + * Vybrid, Layerscape and S32G SoCs. */ #include @@ -72,6 +73,60 @@ static irqreturn_t fsl_edma2_tx_handler(int irq, void *devi_id) return fsl_edma_tx_handler(irq, fsl_chan->edma); } +static irqreturn_t fsl_edma3_or_tx_handler(int irq, void *dev_id, + u8 start, u8 end) +{ + struct fsl_edma_engine *fsl_edma = dev_id; + struct fsl_edma_chan *chan; + int i; + + end = min(end, fsl_edma->n_chans); + + for (i = start; i < end; i++) { + chan = &fsl_edma->chans[i]; + + fsl_edma3_tx_handler(irq, chan); + } + + return IRQ_HANDLED; +} + +static irqreturn_t fsl_edma3_tx_0_15_handler(int irq, void *dev_id) +{ + return fsl_edma3_or_tx_handler(irq, dev_id, 0, 16); +} + +static irqreturn_t fsl_edma3_tx_16_31_handler(int irq, void *dev_id) +{ + return fsl_edma3_or_tx_handler(irq, dev_id, 16, 32); +} + +static irqreturn_t fsl_edma3_or_err_handler(int irq, void *dev_id) +{ + struct fsl_edma_engine *fsl_edma = dev_id; + struct edma_regs *regs = &fsl_edma->regs; + unsigned int err, ch, ch_es; + struct fsl_edma_chan *chan; + + err = edma_readl(fsl_edma, regs->es); + if (!(err & EDMA_V3_MP_ES_VLD)) + return IRQ_NONE; + + for (ch = 0; ch < fsl_edma->n_chans; ch++) { + chan = &fsl_edma->chans[ch]; + + ch_es = edma_readl_chreg(chan, ch_es); + if (!(ch_es & EDMA_V3_CH_ES_ERR)) + continue; + + edma_writel_chreg(chan, EDMA_V3_CH_ES_ERR, ch_es); + fsl_edma_disable_request(chan); + fsl_edma->chans[ch].status = DMA_ERROR; + } + + return IRQ_HANDLED; +} + static irqreturn_t fsl_edma_err_handler(int irq, void *dev_id) { struct fsl_edma_engine *fsl_edma = dev_id; @@ -274,6 +329,49 @@ static int fsl_edma3_irq_init(struct platform_device *pdev, struct fsl_edma_engi return 0; } +static int fsl_edma3_or_irq_init(struct platform_device *pdev, + struct fsl_edma_engine *fsl_edma) +{ + int ret; + + fsl_edma->txirq = platform_get_irq_byname(pdev, "tx-0-15"); + if (fsl_edma->txirq < 0) + return fsl_edma->txirq; + + fsl_edma->txirq_16_31 = platform_get_irq_byname(pdev, "tx-16-31"); + if (fsl_edma->txirq_16_31 < 0) + return fsl_edma->txirq_16_31; + + fsl_edma->errirq = platform_get_irq_byname(pdev, "err"); + if (fsl_edma->errirq < 0) + return fsl_edma->errirq; + + ret = devm_request_irq(&pdev->dev, fsl_edma->txirq, + fsl_edma3_tx_0_15_handler, 0, "eDMA tx0_15", + fsl_edma); + if (ret) + return dev_err_probe(&pdev->dev, ret, + "Can't register eDMA tx0_15 IRQ.\n"); + + if (fsl_edma->n_chans > 16) { + ret = devm_request_irq(&pdev->dev, fsl_edma->txirq_16_31, + fsl_edma3_tx_16_31_handler, 0, + "eDMA tx16_31", fsl_edma); + if (ret) + return dev_err_probe(&pdev->dev, ret, + "Can't register eDMA tx16_31 IRQ.\n"); + } + + ret = devm_request_irq(&pdev->dev, fsl_edma->errirq, + fsl_edma3_or_err_handler, 0, "eDMA err", + fsl_edma); + if (ret) + return dev_err_probe(&pdev->dev, ret, + "Can't register eDMA err IRQ.\n"); + + return 0; +} + static int fsl_edma2_irq_init(struct platform_device *pdev, struct fsl_edma_engine *fsl_edma) @@ -404,6 +502,14 @@ static struct fsl_edma_drvdata imx95_data5 = { .setup_irq = fsl_edma3_irq_init, }; +static const struct fsl_edma_drvdata s32g2_data = { + .dmamuxs = DMAMUX_NR, + .chreg_space_sz = EDMA_TCD, + .chreg_off = 0x4000, + .flags = FSL_EDMA_DRV_EDMA3 | FSL_EDMA_DRV_MUX_SWAP, + .setup_irq = fsl_edma3_or_irq_init, +}; + static const struct of_device_id fsl_edma_dt_ids[] = { { .compatible = "fsl,vf610-edma", .data = &vf610_data}, { .compatible = "fsl,ls1028a-edma", .data = &ls1028a_data}, @@ -413,6 +519,7 @@ static const struct of_device_id fsl_edma_dt_ids[] = { { .compatible = "fsl,imx93-edma3", .data = &imx93_data3}, { .compatible = "fsl,imx93-edma4", .data = &imx93_data4}, { .compatible = "fsl,imx95-edma5", .data = &imx95_data5}, + { .compatible = "nxp,s32g2-edma", .data = &s32g2_data}, { /* sentinel */ } }; MODULE_DEVICE_TABLE(of, fsl_edma_dt_ids); -- cgit v1.2.3-59-g8ed1b From 66d88e16f2044400fe6cc75cd51e1e74c4f9d96d Mon Sep 17 00:00:00 2001 From: Larisa Grigore Date: Thu, 19 Dec 2024 12:24:14 +0200 Subject: dmaengine: fsl-edma: read/write multiple registers in cyclic transactions Add support for reading multiple registers in DEV_TO_MEM transactions and for writing multiple registers in MEM_TO_DEV transactions. Signed-off-by: Frank Li Co-developed-by: Alexandru-Catalin Ionita Signed-off-by: Alexandru-Catalin Ionita Signed-off-by: Larisa Grigore Link: https://lore.kernel.org/r/20241219102415.1208328-6-larisa.grigore@oss.nxp.com Signed-off-by: Vinod Koul --- drivers/dma/fsl-edma-common.c | 36 +++++++++++++++++++++++++++--------- 1 file changed, 27 insertions(+), 9 deletions(-) diff --git a/drivers/dma/fsl-edma-common.c b/drivers/dma/fsl-edma-common.c index b7f15ab96855..443b2430466c 100644 --- a/drivers/dma/fsl-edma-common.c +++ b/drivers/dma/fsl-edma-common.c @@ -480,8 +480,8 @@ void fsl_edma_fill_tcd(struct fsl_edma_chan *fsl_chan, bool disable_req, bool enable_sg) { struct dma_slave_config *cfg = &fsl_chan->cfg; + u32 burst = 0; u16 csr = 0; - u32 burst; /* * eDMA hardware SGs require the TCDs to be stored in little @@ -496,16 +496,30 @@ void fsl_edma_fill_tcd(struct fsl_edma_chan *fsl_chan, fsl_edma_set_tcd_to_le(fsl_chan, tcd, soff, soff); - if (fsl_chan->is_multi_fifo) { - /* set mloff to support multiple fifo */ - burst = cfg->direction == DMA_DEV_TO_MEM ? - cfg->src_maxburst : cfg->dst_maxburst; - nbytes |= EDMA_V3_TCD_NBYTES_MLOFF(-(burst * 4)); - /* enable DMLOE/SMLOE */ - if (cfg->direction == DMA_MEM_TO_DEV) { + /* If we expect to have either multi_fifo or a port window size, + * we will use minor loop offset, meaning bits 29-10 will be used for + * address offset, while bits 9-0 will be used to tell DMA how much + * data to read from addr. + * If we don't have either of those, will use a major loop reading from addr + * nbytes (29bits). + */ + if (cfg->direction == DMA_MEM_TO_DEV) { + if (fsl_chan->is_multi_fifo) + burst = cfg->dst_maxburst * 4; + if (cfg->dst_port_window_size) + burst = cfg->dst_port_window_size * cfg->dst_addr_width; + if (burst) { + nbytes |= EDMA_V3_TCD_NBYTES_MLOFF(-burst); nbytes |= EDMA_V3_TCD_NBYTES_DMLOE; nbytes &= ~EDMA_V3_TCD_NBYTES_SMLOE; - } else { + } + } else { + if (fsl_chan->is_multi_fifo) + burst = cfg->src_maxburst * 4; + if (cfg->src_port_window_size) + burst = cfg->src_port_window_size * cfg->src_addr_width; + if (burst) { + nbytes |= EDMA_V3_TCD_NBYTES_MLOFF(-burst); nbytes |= EDMA_V3_TCD_NBYTES_SMLOE; nbytes &= ~EDMA_V3_TCD_NBYTES_DMLOE; } @@ -623,11 +637,15 @@ struct dma_async_tx_descriptor *fsl_edma_prep_dma_cyclic( dst_addr = fsl_chan->dma_dev_addr; soff = fsl_chan->cfg.dst_addr_width; doff = fsl_chan->is_multi_fifo ? 4 : 0; + if (fsl_chan->cfg.dst_port_window_size) + doff = fsl_chan->cfg.dst_addr_width; } else if (direction == DMA_DEV_TO_MEM) { src_addr = fsl_chan->dma_dev_addr; dst_addr = dma_buf_next; soff = fsl_chan->is_multi_fifo ? 4 : 0; doff = fsl_chan->cfg.src_addr_width; + if (fsl_chan->cfg.src_port_window_size) + soff = fsl_chan->cfg.src_addr_width; } else { /* DMA_DEV_TO_DEV */ src_addr = fsl_chan->cfg.src_addr; -- cgit v1.2.3-59-g8ed1b From 0ab433180eb29bc69f9327e84028d878fb4670c5 Mon Sep 17 00:00:00 2001 From: Joe Hattori Date: Thu, 19 Dec 2024 11:05:06 +0900 Subject: dmaengine: ti: edma: make the loop condition simpler in edma_probe() When i == ecc->num_tc, the edma_probe() calls of_parse_phandle_with_fixed_args() and breaks from the loop regardless of the return value. Since neither the returned value nor the output argument tc_args is used, set i < ecc->num_tc as the loop condition. Signed-off-by: Joe Hattori Link: https://lore.kernel.org/r/20241219020507.1983124-2-joe@pf.is.s.u-tokyo.ac.jp Signed-off-by: Vinod Koul --- drivers/dma/ti/edma.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/dma/ti/edma.c b/drivers/dma/ti/edma.c index 343e986e66e7..08f6c67d381e 100644 --- a/drivers/dma/ti/edma.c +++ b/drivers/dma/ti/edma.c @@ -2460,10 +2460,10 @@ static int edma_probe(struct platform_device *pdev) goto err_reg1; } - for (i = 0;; i++) { + for (i = 0; i < ecc->num_tc; i++) { ret = of_parse_phandle_with_fixed_args(node, "ti,tptcs", 1, i, &tc_args); - if (ret || i == ecc->num_tc) + if (ret) break; ecc->tc_list[i].node = tc_args.np; -- cgit v1.2.3-59-g8ed1b From e883c64778e5a9905fce955681f8ee38c7197e0f Mon Sep 17 00:00:00 2001 From: Joe Hattori Date: Thu, 19 Dec 2024 11:05:07 +0900 Subject: dmaengine: ti: edma: fix OF node reference leaks in edma_driver The .probe() of edma_driver calls of_parse_phandle_with_fixed_args() but does not release the obtained OF nodes. Thus add a of_node_put() call. This bug was found by an experimental verification tool that I am developing. Fixes: 1be5336bc7ba ("dmaengine: edma: New device tree binding") Signed-off-by: Joe Hattori Reviewed-by: Dan Carpenter Link: https://lore.kernel.org/r/20241219020507.1983124-3-joe@pf.is.s.u-tokyo.ac.jp Signed-off-by: Vinod Koul --- drivers/dma/ti/edma.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/drivers/dma/ti/edma.c b/drivers/dma/ti/edma.c index 08f6c67d381e..4ece125b2ae7 100644 --- a/drivers/dma/ti/edma.c +++ b/drivers/dma/ti/edma.c @@ -208,7 +208,6 @@ struct edma_desc { struct edma_cc; struct edma_tc { - struct device_node *node; u16 id; }; @@ -2466,13 +2465,13 @@ static int edma_probe(struct platform_device *pdev) if (ret) break; - ecc->tc_list[i].node = tc_args.np; ecc->tc_list[i].id = i; queue_priority_mapping[i][1] = tc_args.args[0]; if (queue_priority_mapping[i][1] > lowest_priority) { lowest_priority = queue_priority_mapping[i][1]; info->default_queue = i; } + of_node_put(tc_args.np); } /* See if we have optional dma-channel-mask array */ -- cgit v1.2.3-59-g8ed1b From 775363772f5e72b984a883e22d510fec5357477a Mon Sep 17 00:00:00 2001 From: Vaishnav Achath Date: Wed, 27 Nov 2024 15:46:26 +0530 Subject: dt-bindings: dma: ti: k3-bcdma: Add J722S CSI BCDMA J722S CSI BCDMA is similar to J721S2 CSI BCDMA and supports both RX and TX channels but has a different PSIL thread base ID which is currently handled in k3-udma driver. Add an entry for J722S CSIRX BCDMA. Signed-off-by: Vaishnav Achath Reviewed-by: Rob Herring (Arm) Link: https://lore.kernel.org/r/20241127101627.617537-2-vaishnav.a@ti.com Signed-off-by: Vinod Koul --- Documentation/devicetree/bindings/dma/ti/k3-bcdma.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Documentation/devicetree/bindings/dma/ti/k3-bcdma.yaml b/Documentation/devicetree/bindings/dma/ti/k3-bcdma.yaml index 27b8e1636560..b5bc842c5a0e 100644 --- a/Documentation/devicetree/bindings/dma/ti/k3-bcdma.yaml +++ b/Documentation/devicetree/bindings/dma/ti/k3-bcdma.yaml @@ -34,6 +34,7 @@ properties: - ti,am62a-dmss-bcdma-csirx - ti,am64-dmss-bcdma - ti,j721s2-dmss-bcdma-csi + - ti,j722s-dmss-bcdma-csi reg: minItems: 3 @@ -196,7 +197,9 @@ allOf: properties: compatible: contains: - const: ti,j721s2-dmss-bcdma-csi + enum: + - ti,j721s2-dmss-bcdma-csi + - ti,j722s-dmss-bcdma-csi then: properties: ti,sci-rm-range-bchan: false -- cgit v1.2.3-59-g8ed1b From d0301fdbb50dfc99215b0f999d4ff7ab0a7675d9 Mon Sep 17 00:00:00 2001 From: Vaishnav Achath Date: Wed, 27 Nov 2024 15:46:27 +0530 Subject: dmaengine: ti: k3-udma: Add support for J722S CSI BCDMA J722S CSI BCDMA is similar to J721S2 CSI BCDMA but there are slight integration differences like different PSIL thread base ID which is currently handled in the driver based on udma_of_match data. Add an entry to support J722S CSIRX. Signed-off-by: Vaishnav Achath Acked-by: Peter Ujfalusi Link: https://lore.kernel.org/r/20241127101627.617537-3-vaishnav.a@ti.com Signed-off-by: Vinod Koul --- drivers/dma/ti/k3-udma.c | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/drivers/dma/ti/k3-udma.c b/drivers/dma/ti/k3-udma.c index b3f27b3f9209..7ed1956b4642 100644 --- a/drivers/dma/ti/k3-udma.c +++ b/drivers/dma/ti/k3-udma.c @@ -4404,6 +4404,18 @@ static struct udma_match_data j721s2_bcdma_csi_data = { .soc_data = &j721s2_bcdma_csi_soc_data, }; +static struct udma_match_data j722s_bcdma_csi_data = { + .type = DMA_TYPE_BCDMA, + .psil_base = 0x3100, + .enable_memcpy_support = false, + .burst_size = { + TI_SCI_RM_UDMAP_CHAN_BURST_SIZE_64_BYTES, /* Normal Channels */ + 0, /* No H Channels */ + 0, /* No UH Channels */ + }, + .soc_data = &j721s2_bcdma_csi_soc_data, +}; + static const struct of_device_id udma_of_match[] = { { .compatible = "ti,am654-navss-main-udmap", @@ -4435,6 +4447,10 @@ static const struct of_device_id udma_of_match[] = { .compatible = "ti,j721s2-dmss-bcdma-csi", .data = &j721s2_bcdma_csi_data, }, + { + .compatible = "ti,j722s-dmss-bcdma-csi", + .data = &j722s_bcdma_csi_data, + }, { /* Sentinel */ }, }; MODULE_DEVICE_TABLE(of, udma_of_match); -- cgit v1.2.3-59-g8ed1b From 762b37fc6ae2af0c7ddf36556fe7427575e9c759 Mon Sep 17 00:00:00 2001 From: Mohan Kumar D Date: Tue, 17 Dec 2024 13:13:57 +0530 Subject: dt-bindings: dma: Support channel page to nvidia,tegra210-adma Multiple ADMA Channel page hardware support has been added from TEGRA186 and onwards. Update the DT binding to use any of the ADMA channel page address space region. Signed-off-by: Mohan Kumar D Acked-by: Conor Dooley Link: https://lore.kernel.org/r/20241217074358.340180-2-mkumard@nvidia.com Signed-off-by: Vinod Koul --- .../bindings/dma/nvidia,tegra210-adma.yaml | 60 ++++++++++++++++++++-- 1 file changed, 56 insertions(+), 4 deletions(-) diff --git a/Documentation/devicetree/bindings/dma/nvidia,tegra210-adma.yaml b/Documentation/devicetree/bindings/dma/nvidia,tegra210-adma.yaml index 877147e95ecc..d3f8c269916c 100644 --- a/Documentation/devicetree/bindings/dma/nvidia,tegra210-adma.yaml +++ b/Documentation/devicetree/bindings/dma/nvidia,tegra210-adma.yaml @@ -13,9 +13,6 @@ description: | maintainers: - Jon Hunter -allOf: - - $ref: dma-controller.yaml# - properties: compatible: oneOf: @@ -29,7 +26,19 @@ properties: - const: nvidia,tegra186-adma reg: - maxItems: 1 + description: + The 'page' region describes the address space of the page + used for accessing the DMA channel registers. The 'global' + region describes the address space of the global DMA registers. + In the absence of the 'reg-names' property, there must be a + single entry that covers the address space of the global DMA + registers and the DMA channel registers. + minItems: 1 + maxItems: 2 + + reg-names: + minItems: 1 + maxItems: 2 interrupts: description: | @@ -63,6 +72,49 @@ required: - clocks - clock-names +allOf: + - $ref: dma-controller.yaml# + - if: + properties: + compatible: + contains: + enum: + - nvidia,tegra210-adma + then: + properties: + reg: + items: + - description: Full address space range of DMA registers. + + - if: + properties: + compatible: + contains: + enum: + - nvidia,tegra186-adma + then: + anyOf: + - properties: + reg: + items: + - description: Full address space range of DMA registers. + - properties: + reg: + items: + - description: Channel Page address space range of DMA registers. + reg-names: + items: + - const: page + - properties: + reg: + items: + - description: Channel Page address space range of DMA registers. + - description: Global Page address space range of DMA registers. + reg-names: + items: + - const: page + - const: global + additionalProperties: false examples: -- cgit v1.2.3-59-g8ed1b From 68811c928f88828f188656dd3c9c184eeec2ce86 Mon Sep 17 00:00:00 2001 From: Mohan Kumar D Date: Tue, 17 Dec 2024 13:13:58 +0530 Subject: dmaengine: tegra210-adma: Support channel page Multiple ADMA Channel page hardware support has been added from TEGRA186 and onwards. - Add support in the tegra adma driver to handle selective channel page usage - Make global register programming optional Signed-off-by: Mohan Kumar D Link: https://lore.kernel.org/r/20241217074358.340180-3-mkumard@nvidia.com Signed-off-by: Vinod Koul --- drivers/dma/tegra210-adma.c | 86 +++++++++++++++++++++++++++++++++++++++------ 1 file changed, 76 insertions(+), 10 deletions(-) diff --git a/drivers/dma/tegra210-adma.c b/drivers/dma/tegra210-adma.c index 2953008d42ef..6896da8ac7ef 100644 --- a/drivers/dma/tegra210-adma.c +++ b/drivers/dma/tegra210-adma.c @@ -43,6 +43,10 @@ #define ADMA_CH_CONFIG_MAX_BUFS 8 #define TEGRA186_ADMA_CH_CONFIG_OUTSTANDING_REQS(reqs) (reqs << 4) +#define TEGRA186_ADMA_GLOBAL_PAGE_CHGRP 0x30 +#define TEGRA186_ADMA_GLOBAL_PAGE_RX_REQ 0x70 +#define TEGRA186_ADMA_GLOBAL_PAGE_TX_REQ 0x84 + #define ADMA_CH_FIFO_CTRL 0x2c #define ADMA_CH_TX_FIFO_SIZE_SHIFT 8 #define ADMA_CH_RX_FIFO_SIZE_SHIFT 0 @@ -96,6 +100,7 @@ struct tegra_adma_chip_data { unsigned int ch_fifo_size_mask; unsigned int sreq_index_offset; bool has_outstanding_reqs; + void (*set_global_pg_config)(struct tegra_adma *tdma); }; /* @@ -151,6 +156,7 @@ struct tegra_adma { struct dma_device dma_dev; struct device *dev; void __iomem *base_addr; + void __iomem *ch_base_addr; struct clk *ahub_clk; unsigned int nr_channels; unsigned long *dma_chan_mask; @@ -159,6 +165,7 @@ struct tegra_adma { /* Used to store global command register state when suspending */ unsigned int global_cmd; + unsigned int ch_page_no; const struct tegra_adma_chip_data *cdata; @@ -176,6 +183,11 @@ static inline u32 tdma_read(struct tegra_adma *tdma, u32 reg) return readl(tdma->base_addr + tdma->cdata->global_reg_offset + reg); } +static inline void tdma_ch_global_write(struct tegra_adma *tdma, u32 reg, u32 val) +{ + writel(val, tdma->ch_base_addr + tdma->cdata->global_reg_offset + reg); +} + static inline void tdma_ch_write(struct tegra_adma_chan *tdc, u32 reg, u32 val) { writel(val, tdc->chan_addr + reg); @@ -217,13 +229,30 @@ static int tegra_adma_slave_config(struct dma_chan *dc, return 0; } +static void tegra186_adma_global_page_config(struct tegra_adma *tdma) +{ + /* + * Clear the default page1 channel group configs and program + * the global registers based on the actual page usage + */ + tdma_write(tdma, TEGRA186_ADMA_GLOBAL_PAGE_CHGRP, 0); + tdma_write(tdma, TEGRA186_ADMA_GLOBAL_PAGE_RX_REQ, 0); + tdma_write(tdma, TEGRA186_ADMA_GLOBAL_PAGE_TX_REQ, 0); + tdma_write(tdma, TEGRA186_ADMA_GLOBAL_PAGE_CHGRP + (tdma->ch_page_no * 0x4), 0xff); + tdma_write(tdma, TEGRA186_ADMA_GLOBAL_PAGE_RX_REQ + (tdma->ch_page_no * 0x4), 0x1ffffff); + tdma_write(tdma, TEGRA186_ADMA_GLOBAL_PAGE_TX_REQ + (tdma->ch_page_no * 0x4), 0xffffff); +} + static int tegra_adma_init(struct tegra_adma *tdma) { u32 status; int ret; - /* Clear any interrupts */ - tdma_write(tdma, tdma->cdata->ch_base_offset + tdma->cdata->global_int_clear, 0x1); + /* Clear any channels group global interrupts */ + tdma_ch_global_write(tdma, tdma->cdata->global_int_clear, 0x1); + + if (!tdma->base_addr) + return 0; /* Assert soft reset */ tdma_write(tdma, ADMA_GLOBAL_SOFT_RESET, 0x1); @@ -237,6 +266,9 @@ static int tegra_adma_init(struct tegra_adma *tdma) if (ret) return ret; + if (tdma->cdata->set_global_pg_config) + tdma->cdata->set_global_pg_config(tdma); + /* Enable global ADMA registers */ tdma_write(tdma, ADMA_GLOBAL_CMD, 1); @@ -736,7 +768,9 @@ static int __maybe_unused tegra_adma_runtime_suspend(struct device *dev) struct tegra_adma_chan *tdc; int i; - tdma->global_cmd = tdma_read(tdma, ADMA_GLOBAL_CMD); + if (tdma->base_addr) + tdma->global_cmd = tdma_read(tdma, ADMA_GLOBAL_CMD); + if (!tdma->global_cmd) goto clk_disable; @@ -777,7 +811,11 @@ static int __maybe_unused tegra_adma_runtime_resume(struct device *dev) dev_err(dev, "ahub clk_enable failed: %d\n", ret); return ret; } - tdma_write(tdma, ADMA_GLOBAL_CMD, tdma->global_cmd); + if (tdma->base_addr) { + tdma_write(tdma, ADMA_GLOBAL_CMD, tdma->global_cmd); + if (tdma->cdata->set_global_pg_config) + tdma->cdata->set_global_pg_config(tdma); + } if (!tdma->global_cmd) return 0; @@ -817,6 +855,7 @@ static const struct tegra_adma_chip_data tegra210_chip_data = { .ch_fifo_size_mask = 0xf, .sreq_index_offset = 2, .has_outstanding_reqs = false, + .set_global_pg_config = NULL, }; static const struct tegra_adma_chip_data tegra186_chip_data = { @@ -833,6 +872,7 @@ static const struct tegra_adma_chip_data tegra186_chip_data = { .ch_fifo_size_mask = 0x1f, .sreq_index_offset = 4, .has_outstanding_reqs = true, + .set_global_pg_config = tegra186_adma_global_page_config, }; static const struct of_device_id tegra_adma_of_match[] = { @@ -846,7 +886,8 @@ static int tegra_adma_probe(struct platform_device *pdev) { const struct tegra_adma_chip_data *cdata; struct tegra_adma *tdma; - int ret, i; + struct resource *res_page, *res_base; + int ret, i, page_no; cdata = of_device_get_match_data(&pdev->dev); if (!cdata) { @@ -865,9 +906,35 @@ static int tegra_adma_probe(struct platform_device *pdev) tdma->nr_channels = cdata->nr_channels; platform_set_drvdata(pdev, tdma); - tdma->base_addr = devm_platform_ioremap_resource(pdev, 0); - if (IS_ERR(tdma->base_addr)) - return PTR_ERR(tdma->base_addr); + res_page = platform_get_resource_byname(pdev, IORESOURCE_MEM, "page"); + if (res_page) { + tdma->ch_base_addr = devm_ioremap_resource(&pdev->dev, res_page); + if (IS_ERR(tdma->ch_base_addr)) + return PTR_ERR(tdma->ch_base_addr); + + res_base = platform_get_resource_byname(pdev, IORESOURCE_MEM, "global"); + if (res_base) { + page_no = (res_page->start - res_base->start) / cdata->ch_base_offset; + if (page_no <= 0) + return -EINVAL; + tdma->ch_page_no = page_no - 1; + tdma->base_addr = devm_ioremap_resource(&pdev->dev, res_base); + if (IS_ERR(tdma->base_addr)) + return PTR_ERR(tdma->base_addr); + } + } else { + /* If no 'page' property found, then reg DT binding would be legacy */ + res_base = platform_get_resource(pdev, IORESOURCE_MEM, 0); + if (res_base) { + tdma->base_addr = devm_ioremap_resource(&pdev->dev, res_base); + if (IS_ERR(tdma->base_addr)) + return PTR_ERR(tdma->base_addr); + } else { + return -ENODEV; + } + + tdma->ch_base_addr = tdma->base_addr + cdata->ch_base_offset; + } tdma->ahub_clk = devm_clk_get(&pdev->dev, "d_audio"); if (IS_ERR(tdma->ahub_clk)) { @@ -900,8 +967,7 @@ static int tegra_adma_probe(struct platform_device *pdev) if (!test_bit(i, tdma->dma_chan_mask)) continue; - tdc->chan_addr = tdma->base_addr + cdata->ch_base_offset - + (cdata->ch_reg_size * i); + tdc->chan_addr = tdma->ch_base_addr + (cdata->ch_reg_size * i); tdc->irq = of_irq_get(pdev->dev.of_node, i); if (tdc->irq <= 0) { -- cgit v1.2.3-59-g8ed1b From 9602a843cb3a16df8930eb9b046aa7aeb769521b Mon Sep 17 00:00:00 2001 From: Stefan Wahren Date: Wed, 4 Dec 2024 17:55:46 +0100 Subject: dmaengine: bcm2835-dma: Prevent suspend if DMA channel is busy bcm2835-dma provides the service to others, so it should suspend late and resume early. Suspend should be prevented in case a DMA channel is still busy. Signed-off-by: Stefan Wahren Link: https://lore.kernel.org/r/20241204165546.77941-1-wahrenst@gmx.net Signed-off-by: Vinod Koul --- drivers/dma/bcm2835-dma.c | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/drivers/dma/bcm2835-dma.c b/drivers/dma/bcm2835-dma.c index 7ba52dee40a9..20b10c15c696 100644 --- a/drivers/dma/bcm2835-dma.c +++ b/drivers/dma/bcm2835-dma.c @@ -875,6 +875,27 @@ static struct dma_chan *bcm2835_dma_xlate(struct of_phandle_args *spec, return chan; } +static int bcm2835_dma_suspend_late(struct device *dev) +{ + struct bcm2835_dmadev *od = dev_get_drvdata(dev); + struct bcm2835_chan *c, *next; + + list_for_each_entry_safe(c, next, &od->ddev.channels, + vc.chan.device_node) { + void __iomem *chan_base = c->chan_base; + + /* Check if DMA channel is busy */ + if (readl(chan_base + BCM2835_DMA_ADDR)) + return -EBUSY; + } + + return 0; +} + +static const struct dev_pm_ops bcm2835_dma_pm_ops = { + SET_LATE_SYSTEM_SLEEP_PM_OPS(bcm2835_dma_suspend_late, NULL) +}; + static int bcm2835_dma_probe(struct platform_device *pdev) { struct bcm2835_dmadev *od; @@ -1033,6 +1054,7 @@ static struct platform_driver bcm2835_dma_driver = { .driver = { .name = "bcm2835-dma", .of_match_table = of_match_ptr(bcm2835_dma_of_match), + .pm = pm_ptr(&bcm2835_dma_pm_ops), }, }; -- cgit v1.2.3-59-g8ed1b From 9f6caa3978b0e859da39e4ace7973b877222dfd4 Mon Sep 17 00:00:00 2001 From: Javier Carrasco Date: Fri, 11 Oct 2024 22:58:00 +0200 Subject: dmaengine: mv_xor: switch to for_each_child_of_node_scoped() Introduce the scoped variant of the loop to automatically release the child node when it goes out of scope, which is more robust than the non-scoped variant, and accounts for new early exits that could be added in the future. Signed-off-by: Javier Carrasco Link: https://lore.kernel.org/r/20241011-dma_mv_xor_of_node_put-v1-2-3c2de819f463@gmail.com Signed-off-by: Vinod Koul --- drivers/dma/mv_xor.c | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/drivers/dma/mv_xor.c b/drivers/dma/mv_xor.c index 40b76b40bc30..fa6e4646fdc2 100644 --- a/drivers/dma/mv_xor.c +++ b/drivers/dma/mv_xor.c @@ -1369,10 +1369,9 @@ static int mv_xor_probe(struct platform_device *pdev) return 0; if (pdev->dev.of_node) { - struct device_node *np; int i = 0; - for_each_child_of_node(pdev->dev.of_node, np) { + for_each_child_of_node_scoped(pdev->dev.of_node, np) { struct mv_xor_chan *chan; dma_cap_mask_t cap_mask; int irq; @@ -1388,7 +1387,6 @@ static int mv_xor_probe(struct platform_device *pdev) irq = irq_of_parse_and_map(np, 0); if (!irq) { ret = -ENODEV; - of_node_put(np); goto err_channel_add; } @@ -1397,7 +1395,6 @@ static int mv_xor_probe(struct platform_device *pdev) if (IS_ERR(chan)) { ret = PTR_ERR(chan); irq_dispose_mapping(irq); - of_node_put(np); goto err_channel_add; } -- cgit v1.2.3-59-g8ed1b From 788726fcea1fd79ca238403c56c012ca9c159798 Mon Sep 17 00:00:00 2001 From: David Lechner Date: Mon, 16 Dec 2024 14:51:01 -0600 Subject: dt-bindings: dma: adi,axi-dmac: convert to yaml schema Convert the AXI DMAC bindings from .txt to .yaml. Acked-by: Nuno Sa Reviewed-by: Rob Herring (Arm) Signed-off-by: David Lechner Link: https://lore.kernel.org/r/20241216-axi-dma-dt-yaml-v3-1-7b994710c43f@baylibre.com Signed-off-by: Vinod Koul --- .../devicetree/bindings/dma/adi,axi-dmac.txt | 61 --------- .../devicetree/bindings/dma/adi,axi-dmac.yaml | 139 +++++++++++++++++++++ 2 files changed, 139 insertions(+), 61 deletions(-) delete mode 100644 Documentation/devicetree/bindings/dma/adi,axi-dmac.txt create mode 100644 Documentation/devicetree/bindings/dma/adi,axi-dmac.yaml diff --git a/Documentation/devicetree/bindings/dma/adi,axi-dmac.txt b/Documentation/devicetree/bindings/dma/adi,axi-dmac.txt deleted file mode 100644 index cd17684aaab5..000000000000 --- a/Documentation/devicetree/bindings/dma/adi,axi-dmac.txt +++ /dev/null @@ -1,61 +0,0 @@ -Analog Devices AXI-DMAC DMA controller - -Required properties: - - compatible: Must be "adi,axi-dmac-1.00.a". - - reg: Specification for the controllers memory mapped register map. - - interrupts: Specification for the controllers interrupt. - - clocks: Phandle and specifier to the controllers AXI interface clock - - #dma-cells: Must be 1. - -Required sub-nodes: - - adi,channels: This sub-node must contain a sub-node for each DMA channel. For - the channel sub-nodes the following bindings apply. They must match the - configuration options of the peripheral as it was instantiated. - -Required properties for adi,channels sub-node: - - #size-cells: Must be 0 - - #address-cells: Must be 1 - -Required channel sub-node properties: - - reg: Which channel this node refers to. - - adi,source-bus-width, - adi,destination-bus-width: Width of the source or destination bus in bits. - - adi,source-bus-type, - adi,destination-bus-type: Type of the source or destination bus. Must be one - of the following: - 0 (AXI_DMAC_TYPE_AXI_MM): Memory mapped AXI interface - 1 (AXI_DMAC_TYPE_AXI_STREAM): Streaming AXI interface - 2 (AXI_DMAC_TYPE_AXI_FIFO): FIFO interface - -Deprecated optional channel properties: - - adi,length-width: Width of the DMA transfer length register. - - adi,cyclic: Must be set if the channel supports hardware cyclic DMA - transfers. - - adi,2d: Must be set if the channel supports hardware 2D DMA transfers. - -DMA clients connected to the AXI-DMAC DMA controller must use the format -described in the dma.txt file using a one-cell specifier. The value of the -specifier refers to the DMA channel index. - -Example: - -dma: dma@7c420000 { - compatible = "adi,axi-dmac-1.00.a"; - reg = <0x7c420000 0x10000>; - interrupts = <0 57 0>; - clocks = <&clkc 16>; - #dma-cells = <1>; - - adi,channels { - #size-cells = <0>; - #address-cells = <1>; - - dma-channel@0 { - reg = <0>; - adi,source-bus-width = <32>; - adi,source-bus-type = ; - adi,destination-bus-width = <64>; - adi,destination-bus-type = ; - }; - }; -}; diff --git a/Documentation/devicetree/bindings/dma/adi,axi-dmac.yaml b/Documentation/devicetree/bindings/dma/adi,axi-dmac.yaml new file mode 100644 index 000000000000..b1f4bdcab4fd --- /dev/null +++ b/Documentation/devicetree/bindings/dma/adi,axi-dmac.yaml @@ -0,0 +1,139 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/dma/adi,axi-dmac.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Analog Devices AXI-DMAC DMA controller + +description: | + FPGA-based DMA controller designed for use with high-speed converter hardware. + + http://analogdevicesinc.github.io/hdl/library/axi_dmac/index.html + +maintainers: + - Nuno Sa + +additionalProperties: false + +properties: + compatible: + const: adi,axi-dmac-1.00.a + + reg: + maxItems: 1 + + interrupts: + maxItems: 1 + + clocks: + maxItems: 1 + + "#dma-cells": + const: 1 + + adi,channels: + type: object + description: This sub-node must contain a sub-node for each DMA channel. + additionalProperties: false + + properties: + "#size-cells": + const: 0 + "#address-cells": + const: 1 + + patternProperties: + "^dma-channel@[0-9a-f]+$": + type: object + description: + DMA channel properties based on HDL compile-time configuration. + additionalProperties: false + + properties: + reg: + maxItems: 1 + + adi,source-bus-width: + $ref: /schemas/types.yaml#/definitions/uint32 + description: Width of the source bus in bits. + enum: [8, 16, 32, 64, 128] + + adi,destination-bus-width: + $ref: /schemas/types.yaml#/definitions/uint32 + description: Width of the destination bus in bits. + enum: [8, 16, 32, 64, 128] + + adi,source-bus-type: + $ref: /schemas/types.yaml#/definitions/uint32 + description: | + Type of the source bus. + + 0: Memory mapped AXI interface + 1: Streaming AXI interface + 2: FIFO interface + enum: [0, 1, 2] + + adi,destination-bus-type: + $ref: /schemas/types.yaml#/definitions/uint32 + description: Type of the destination bus (see adi,source-bus-type). + enum: [0, 1, 2] + + adi,length-width: + deprecated: true + $ref: /schemas/types.yaml#/definitions/uint32 + description: Width of the DMA transfer length register. + + adi,cyclic: + deprecated: true + type: boolean + description: + Must be set if the channel supports hardware cyclic DMA transfers. + + adi,2d: + deprecated: true + type: boolean + description: + Must be set if the channel supports hardware 2D DMA transfers. + + required: + - reg + - adi,source-bus-width + - adi,destination-bus-width + - adi,source-bus-type + - adi,destination-bus-type + + required: + - "#size-cells" + - "#address-cells" + +required: + - compatible + - reg + - interrupts + - clocks + - "#dma-cells" + - adi,channels + +examples: + - | + dma-controller@7c420000 { + compatible = "adi,axi-dmac-1.00.a"; + reg = <0x7c420000 0x10000>; + interrupts = <0 57 0>; + clocks = <&clkc 16>; + #dma-cells = <1>; + + adi,channels { + #size-cells = <0>; + #address-cells = <1>; + + dma-channel@0 { + reg = <0>; + adi,source-bus-width = <32>; + adi,source-bus-type = <0>; /* Memory mapped */ + adi,destination-bus-width = <64>; + adi,destination-bus-type = <2>; /* FIFO */ + }; + }; + }; -- cgit v1.2.3-59-g8ed1b From 06d5363296dbcffb9e52ca4c9cec097105eb81e9 Mon Sep 17 00:00:00 2001 From: David Lechner Date: Mon, 16 Dec 2024 14:51:02 -0600 Subject: dt-bindings: dma: adi,axi-dmac: deprecate adi,channels node Deprecate the adi,channels node in the adi,axi-dmac binding. Prior to IP version 4.3.a, this information was required. Since then, there are memory-mapped registers that can be read to get the same information. Acked-by: Nuno Sa Reviewed-by: Rob Herring (Arm) Signed-off-by: David Lechner Link: https://lore.kernel.org/r/20241216-axi-dma-dt-yaml-v3-2-7b994710c43f@baylibre.com Signed-off-by: Vinod Koul --- .../devicetree/bindings/dma/adi,axi-dmac.yaml | 20 +++++--------------- 1 file changed, 5 insertions(+), 15 deletions(-) diff --git a/Documentation/devicetree/bindings/dma/adi,axi-dmac.yaml b/Documentation/devicetree/bindings/dma/adi,axi-dmac.yaml index b1f4bdcab4fd..63b6fb0423c2 100644 --- a/Documentation/devicetree/bindings/dma/adi,axi-dmac.yaml +++ b/Documentation/devicetree/bindings/dma/adi,axi-dmac.yaml @@ -33,8 +33,12 @@ properties: const: 1 adi,channels: + deprecated: true type: object - description: This sub-node must contain a sub-node for each DMA channel. + description: + This sub-node must contain a sub-node for each DMA channel. This node is + only required for IP versions older than 4.3.a and should otherwise be + omitted. additionalProperties: false properties: @@ -113,7 +117,6 @@ required: - interrupts - clocks - "#dma-cells" - - adi,channels examples: - | @@ -123,17 +126,4 @@ examples: interrupts = <0 57 0>; clocks = <&clkc 16>; #dma-cells = <1>; - - adi,channels { - #size-cells = <0>; - #address-cells = <1>; - - dma-channel@0 { - reg = <0>; - adi,source-bus-width = <32>; - adi,source-bus-type = <0>; /* Memory mapped */ - adi,destination-bus-width = <64>; - adi,destination-bus-type = <2>; /* FIFO */ - }; - }; }; -- cgit v1.2.3-59-g8ed1b From a131169dfa48d6d40da45bca67d1e4f54fa130a6 Mon Sep 17 00:00:00 2001 From: Jyothi Kumar Seerapu Date: Mon, 9 Dec 2024 13:20:33 +0530 Subject: dmaengine: qcom: gpi: Add GPI immediate DMA support for SPI protocol The DMA TRE(Transfer ring element) buffer contains the DMA buffer address. Accessing data from this address can cause significant delays in SPI transfers, which can be mitigated to some extent by utilizing immediate DMA support. QCOM GPI DMA hardware supports an immediate DMA feature for data up to 8 bytes, storing the data directly in the DMA TRE buffer instead of the DMA buffer address. This enhancement enables faster SPI data transfers. This optimization reduces the average transfer time from 25 us to 16 us for a single SPI transfer of 8 bytes length, with a clock frequency of 50 MHz. Signed-off-by: Jyothi Kumar Seerapu Reviewed-by: Neil Armstrong Tested-by: Neil Armstrong # on SM8550-QRD Reviewed-by: Dmitry Baryshkov Link: https://lore.kernel.org/r/20241209075033.16860-1-quic_jseerapu@quicinc.com Signed-off-by: Vinod Koul --- drivers/dma/qcom/gpi.c | 31 +++++++++++++++++++++++++------ 1 file changed, 25 insertions(+), 6 deletions(-) diff --git a/drivers/dma/qcom/gpi.c b/drivers/dma/qcom/gpi.c index 52a7c8f2498f..b1f0001cc99c 100644 --- a/drivers/dma/qcom/gpi.c +++ b/drivers/dma/qcom/gpi.c @@ -18,6 +18,7 @@ #include "../virt-dma.h" #define TRE_TYPE_DMA 0x10 +#define TRE_TYPE_IMMEDIATE_DMA 0x11 #define TRE_TYPE_GO 0x20 #define TRE_TYPE_CONFIG0 0x22 @@ -64,6 +65,7 @@ /* DMA TRE */ #define TRE_DMA_LEN GENMASK(23, 0) +#define TRE_DMA_IMMEDIATE_LEN GENMASK(3, 0) /* Register offsets from gpi-top */ #define GPII_n_CH_k_CNTXT_0_OFFS(n, k) (0x20000 + (0x4000 * (n)) + (0x80 * (k))) @@ -1711,6 +1713,7 @@ static int gpi_create_spi_tre(struct gchan *chan, struct gpi_desc *desc, dma_addr_t address; struct gpi_tre *tre; unsigned int i; + int len; /* first create config tre if applicable */ if (direction == DMA_MEM_TO_DEV && spi->set_config) { @@ -1763,14 +1766,30 @@ static int gpi_create_spi_tre(struct gchan *chan, struct gpi_desc *desc, tre_idx++; address = sg_dma_address(sgl); - tre->dword[0] = lower_32_bits(address); - tre->dword[1] = upper_32_bits(address); + len = sg_dma_len(sgl); - tre->dword[2] = u32_encode_bits(sg_dma_len(sgl), TRE_DMA_LEN); + /* Support Immediate dma for write transfers for data length up to 8 bytes */ + if (direction == DMA_MEM_TO_DEV && len <= 2 * sizeof(tre->dword[0])) { + /* + * For Immediate dma, data length may not always be length of 8 bytes, + * it can be length less than 8, hence initialize both dword's with 0 + */ + tre->dword[0] = 0; + tre->dword[1] = 0; + memcpy(&tre->dword[0], sg_virt(sgl), len); - tre->dword[3] = u32_encode_bits(TRE_TYPE_DMA, TRE_FLAGS_TYPE); - if (direction == DMA_MEM_TO_DEV) - tre->dword[3] |= u32_encode_bits(1, TRE_FLAGS_IEOT); + tre->dword[2] = u32_encode_bits(len, TRE_DMA_IMMEDIATE_LEN); + tre->dword[3] = u32_encode_bits(TRE_TYPE_IMMEDIATE_DMA, TRE_FLAGS_TYPE); + } else { + tre->dword[0] = lower_32_bits(address); + tre->dword[1] = upper_32_bits(address); + + tre->dword[2] = u32_encode_bits(len, TRE_DMA_LEN); + tre->dword[3] = u32_encode_bits(TRE_TYPE_DMA, TRE_FLAGS_TYPE); + } + + tre->dword[3] |= u32_encode_bits(direction == DMA_MEM_TO_DEV, + TRE_FLAGS_IEOT); for (i = 0; i < tre_idx; i++) dev_dbg(dev, "TRE:%d %x:%x:%x:%x\n", i, desc->tre[i].dword[0], -- cgit v1.2.3-59-g8ed1b From 54e09c8e2d3b0b7d603a64368fa49fe2a8031dd1 Mon Sep 17 00:00:00 2001 From: Ken Sloat Date: Fri, 6 Dec 2024 06:50:18 -0500 Subject: dt-bindings: dma: st-stm32-dmamux: Add description for dma-cell values The dma-cell values for the stm32-dmamux are used to craft the DMA spec for the actual controller. These values are currently undocumented leaving the user to reverse engineer the driver in order to determine their meaning. Add a basic description, while avoiding duplicating information by pointing the user to the associated DMA docs that describe the fields in depth. Signed-off-by: Ken Sloat Acked-by: Conor Dooley Reviewed-by: Amelie Delaunay Link: https://lore.kernel.org/r/20241206115018.1155149-1-ksloat@cornersoftsolutions.com Signed-off-by: Vinod Koul --- .../devicetree/bindings/dma/stm32/st,stm32-dmamux.yaml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/Documentation/devicetree/bindings/dma/stm32/st,stm32-dmamux.yaml b/Documentation/devicetree/bindings/dma/stm32/st,stm32-dmamux.yaml index f26c914a3a9a..b7bca1a83769 100644 --- a/Documentation/devicetree/bindings/dma/stm32/st,stm32-dmamux.yaml +++ b/Documentation/devicetree/bindings/dma/stm32/st,stm32-dmamux.yaml @@ -15,6 +15,16 @@ allOf: properties: "#dma-cells": const: 3 + description: | + Each cell represents the following: + 1. The mux input number/line for the request + 2. Bitfield representing DMA channel configuration that is passed + to the real DMA controller + 3. Bitfield representing device dependent DMA features passed to + the real DMA controller + + For bitfield definitions of cells 2 and 3, see the associated + bindings doc for the actual DMA controller in st,stm32-dma.yaml. compatible: const: st,stm32h7-dmamux -- cgit v1.2.3-59-g8ed1b From 5d6670033a67b288ffbaa0774966f356b9c4ba5d Mon Sep 17 00:00:00 2001 From: Charan Pedumuru Date: Thu, 5 Dec 2024 15:26:18 +0530 Subject: dt-bindings: dma: atmel: Convert to json schema Convert old text based binding to json schema. Changes during conversion: - Add the required properties `clock` and `clock-names`, which were missing in the original binding. - Add a fallback for `microchip,sam9x7-dma` and `microchip,sam9x60-dma` as they are compatible with the dma IP core on `atmel,sama5d4-dma`. - Update examples and include appropriate file directives to resolve errors identified by `dt_binding_check` and `dtbs_check`. Signed-off-by: Charan Pedumuru Acked-by: Conor Dooley Link: https://lore.kernel.org/r/20241205-xdma-v1-1-76a4a44670b5@microchip.com Signed-off-by: Vinod Koul --- .../devicetree/bindings/dma/atmel,sama5d4-dma.yaml | 79 ++++++++++++++++++++++ .../devicetree/bindings/dma/atmel-xdma.txt | 54 --------------- 2 files changed, 79 insertions(+), 54 deletions(-) create mode 100644 Documentation/devicetree/bindings/dma/atmel,sama5d4-dma.yaml delete mode 100644 Documentation/devicetree/bindings/dma/atmel-xdma.txt diff --git a/Documentation/devicetree/bindings/dma/atmel,sama5d4-dma.yaml b/Documentation/devicetree/bindings/dma/atmel,sama5d4-dma.yaml new file mode 100644 index 000000000000..9ca1c5d1f00f --- /dev/null +++ b/Documentation/devicetree/bindings/dma/atmel,sama5d4-dma.yaml @@ -0,0 +1,79 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/dma/atmel,sama5d4-dma.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Microchip AT91 Extensible Direct Memory Access Controller + +maintainers: + - Nicolas Ferre + - Charan Pedumuru + +description: + The DMA Controller (XDMAC) is a AHB-protocol central direct memory access + controller. It performs peripheral data transfer and memory move operations + over one or two bus ports through the unidirectional communication + channel. Each channel is fully programmable and provides both peripheral + or memory-to-memory transfers. The channel features are configurable at + implementation. + +allOf: + - $ref: dma-controller.yaml# + +properties: + compatible: + oneOf: + - enum: + - atmel,sama5d4-dma + - microchip,sama7g5-dma + - items: + - enum: + - microchip,sam9x60-dma + - microchip,sam9x7-dma + - const: atmel,sama5d4-dma + + "#dma-cells": + description: | + Represents the number of integer cells in the `dmas` property of client + devices. The single cell specifies the channel configuration register: + - bit 13: SIF (Source Interface Identifier) for memory interface. + - bit 14: DIF (Destination Interface Identifier) for peripheral interface. + - bit 30-24: PERID (Peripheral Identifier). + const: 1 + + reg: + maxItems: 1 + + interrupts: + maxItems: 1 + + clocks: + maxItems: 1 + + clock-names: + const: dma_clk + +required: + - compatible + - reg + - interrupts + - clocks + - clock-names + - "#dma-cells" + +unevaluatedProperties: false + +examples: + - | + #include + #include + #include + dma-controller@f0008000 { + compatible = "atmel,sama5d4-dma"; + reg = <0xf0008000 0x1000>; + interrupts = <20 IRQ_TYPE_LEVEL_HIGH 0>; + #dma-cells = <1>; + clocks = <&pmc PMC_TYPE_PERIPHERAL 20>; + clock-names = "dma_clk"; + }; diff --git a/Documentation/devicetree/bindings/dma/atmel-xdma.txt b/Documentation/devicetree/bindings/dma/atmel-xdma.txt deleted file mode 100644 index 76d649b3a25d..000000000000 --- a/Documentation/devicetree/bindings/dma/atmel-xdma.txt +++ /dev/null @@ -1,54 +0,0 @@ -* Atmel Extensible Direct Memory Access Controller (XDMAC) - -* XDMA Controller -Required properties: -- compatible: Should be "atmel,sama5d4-dma", "microchip,sam9x60-dma" or - "microchip,sama7g5-dma" or - "microchip,sam9x7-dma", "atmel,sama5d4-dma". -- reg: Should contain DMA registers location and length. -- interrupts: Should contain DMA interrupt. -- #dma-cells: Must be <1>, used to represent the number of integer cells in -the dmas property of client devices. - - The 1st cell specifies the channel configuration register: - - bit 13: SIF, source interface identifier, used to get the memory - interface identifier, - - bit 14: DIF, destination interface identifier, used to get the peripheral - interface identifier, - - bit 30-24: PERID, peripheral identifier. - -Example: - -dma1: dma-controller@f0004000 { - compatible = "atmel,sama5d4-dma"; - reg = <0xf0004000 0x200>; - interrupts = <50 4 0>; - #dma-cells = <1>; -}; - - -* DMA clients -DMA clients connected to the Atmel XDMA controller must use the format -described in the dma.txt file, using a one-cell specifier for each channel. -The two cells in order are: -1. A phandle pointing to the DMA controller. -2. Channel configuration register. Configurable fields are: - - bit 13: SIF, source interface identifier, used to get the memory - interface identifier, - - bit 14: DIF, destination interface identifier, used to get the peripheral - interface identifier, - - bit 30-24: PERID, peripheral identifier. - -Example: - -i2c2: i2c@f8024000 { - compatible = "atmel,at91sam9x5-i2c"; - reg = <0xf8024000 0x4000>; - interrupts = <34 4 6>; - dmas = <&dma1 - (AT91_XDMAC_DT_MEM_IF(0) | AT91_XDMAC_DT_PER_IF(1) - | AT91_XDMAC_DT_PERID(6))>, - <&dma1 - (AT91_XDMAC_DT_MEM_IF(0) | AT91_XDMAC_DT_PER_IF(1) - | AT91_XDMAC_DT_PERID(7))>; - dma-names = "tx", "rx"; -}; -- cgit v1.2.3-59-g8ed1b From 087e89b69b5fe5529a8809a06b4b4680e54f87e2 Mon Sep 17 00:00:00 2001 From: Fenghua Yu Date: Fri, 22 Nov 2024 15:30:24 -0800 Subject: dmaengine: idxd: Add idxd_pci_probe_alloc() helper Add the idxd_pci_probe_alloc() helper to probe IDXD PCI device with or without allocating and setting idxd software values. The idxd_pci_probe() function is refactored to call this helper and always probe the IDXD device with allocating and setting the software values. This helper will be called later in the Function Level Reset (FLR) process without modifying the idxd software data. Signed-off-by: Fenghua Yu Reviewed-by: Dave Jiang Link: https://lore.kernel.org/r/20241122233028.2762809-2-fenghua.yu@intel.com Signed-off-by: Vinod Koul --- drivers/dma/idxd/idxd.h | 2 + drivers/dma/idxd/init.c | 102 +++++++++++++++++++++++++++++------------------- 2 files changed, 64 insertions(+), 40 deletions(-) diff --git a/drivers/dma/idxd/idxd.h b/drivers/dma/idxd/idxd.h index ff33ebf08ee7..0da282b84b59 100644 --- a/drivers/dma/idxd/idxd.h +++ b/drivers/dma/idxd/idxd.h @@ -740,6 +740,8 @@ void idxd_unmask_error_interrupts(struct idxd_device *idxd); /* device control */ int idxd_device_drv_probe(struct idxd_dev *idxd_dev); +int idxd_pci_probe_alloc(struct idxd_device *idxd, struct pci_dev *pdev, + const struct pci_device_id *id); void idxd_device_drv_remove(struct idxd_dev *idxd_dev); int idxd_drv_enable_wq(struct idxd_wq *wq); void idxd_drv_disable_wq(struct idxd_wq *wq); diff --git a/drivers/dma/idxd/init.c b/drivers/dma/idxd/init.c index a96f49567313..02c285499c5a 100644 --- a/drivers/dma/idxd/init.c +++ b/drivers/dma/idxd/init.c @@ -725,67 +725,84 @@ static void idxd_cleanup(struct idxd_device *idxd) idxd_disable_sva(idxd->pdev); } -static int idxd_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) +/* + * Probe idxd PCI device. + * If idxd is not given, need to allocate idxd and set up its data. + * + * If idxd is given, idxd was allocated and setup already. Just need to + * configure device without re-allocating and re-configuring idxd data. + * This is useful for recovering from FLR. + */ +int idxd_pci_probe_alloc(struct idxd_device *idxd, struct pci_dev *pdev, + const struct pci_device_id *id) { - struct device *dev = &pdev->dev; - struct idxd_device *idxd; - struct idxd_driver_data *data = (struct idxd_driver_data *)id->driver_data; + bool alloc_idxd = idxd ? false : true; + struct idxd_driver_data *data; + struct device *dev; int rc; + pdev = idxd ? idxd->pdev : pdev; + dev = &pdev->dev; + data = id ? (struct idxd_driver_data *)id->driver_data : NULL; rc = pci_enable_device(pdev); if (rc) return rc; - dev_dbg(dev, "Alloc IDXD context\n"); - idxd = idxd_alloc(pdev, data); - if (!idxd) { - rc = -ENOMEM; - goto err_idxd_alloc; - } + if (alloc_idxd) { + dev_dbg(dev, "Alloc IDXD context\n"); + idxd = idxd_alloc(pdev, data); + if (!idxd) { + rc = -ENOMEM; + goto err_idxd_alloc; + } - dev_dbg(dev, "Mapping BARs\n"); - idxd->reg_base = pci_iomap(pdev, IDXD_MMIO_BAR, 0); - if (!idxd->reg_base) { - rc = -ENOMEM; - goto err_iomap; - } + dev_dbg(dev, "Mapping BARs\n"); + idxd->reg_base = pci_iomap(pdev, IDXD_MMIO_BAR, 0); + if (!idxd->reg_base) { + rc = -ENOMEM; + goto err_iomap; + } - dev_dbg(dev, "Set DMA masks\n"); - rc = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); - if (rc) - goto err; + dev_dbg(dev, "Set DMA masks\n"); + rc = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); + if (rc) + goto err; + } dev_dbg(dev, "Set PCI master\n"); pci_set_master(pdev); pci_set_drvdata(pdev, idxd); - idxd->hw.version = ioread32(idxd->reg_base + IDXD_VER_OFFSET); - rc = idxd_probe(idxd); - if (rc) { - dev_err(dev, "Intel(R) IDXD DMA Engine init failed\n"); - goto err; - } + if (alloc_idxd) { + idxd->hw.version = ioread32(idxd->reg_base + IDXD_VER_OFFSET); + rc = idxd_probe(idxd); + if (rc) { + dev_err(dev, "Intel(R) IDXD DMA Engine init failed\n"); + goto err; + } - if (data->load_device_defaults) { - rc = data->load_device_defaults(idxd); - if (rc) - dev_warn(dev, "IDXD loading device defaults failed\n"); - } + if (data->load_device_defaults) { + rc = data->load_device_defaults(idxd); + if (rc) + dev_warn(dev, "IDXD loading device defaults failed\n"); + } - rc = idxd_register_devices(idxd); - if (rc) { - dev_err(dev, "IDXD sysfs setup failed\n"); - goto err_dev_register; - } + rc = idxd_register_devices(idxd); + if (rc) { + dev_err(dev, "IDXD sysfs setup failed\n"); + goto err_dev_register; + } - rc = idxd_device_init_debugfs(idxd); - if (rc) - dev_warn(dev, "IDXD debugfs failed to setup\n"); + rc = idxd_device_init_debugfs(idxd); + if (rc) + dev_warn(dev, "IDXD debugfs failed to setup\n"); + } dev_info(&pdev->dev, "Intel(R) Accelerator Device (v%x)\n", idxd->hw.version); - idxd->user_submission_safe = data->user_submission_safe; + if (data) + idxd->user_submission_safe = data->user_submission_safe; return 0; @@ -800,6 +817,11 @@ static int idxd_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) return rc; } +static int idxd_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) +{ + return idxd_pci_probe_alloc(NULL, pdev, id); +} + void idxd_wqs_quiesce(struct idxd_device *idxd) { struct idxd_wq *wq; -- cgit v1.2.3-59-g8ed1b From 3ab45516772b813315324dc63a900703144e80c4 Mon Sep 17 00:00:00 2001 From: Fenghua Yu Date: Fri, 22 Nov 2024 15:30:25 -0800 Subject: dmaengine: idxd: Binding and unbinding IDXD device and driver Add idxd_bind() and idxd_unbind() helpers to bind and unbind the IDXD device and driver. These helpers will be called during Function Level Reset (FLR) processing. Signed-off-by: Fenghua Yu Reviewed-by: Dave Jiang Link: https://lore.kernel.org/r/20241122233028.2762809-3-fenghua.yu@intel.com Signed-off-by: Vinod Koul --- drivers/dma/idxd/init.c | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/drivers/dma/idxd/init.c b/drivers/dma/idxd/init.c index 02c285499c5a..eb64a38444c1 100644 --- a/drivers/dma/idxd/init.c +++ b/drivers/dma/idxd/init.c @@ -725,6 +725,39 @@ static void idxd_cleanup(struct idxd_device *idxd) idxd_disable_sva(idxd->pdev); } +/* + * Attach IDXD device to IDXD driver. + */ +static int idxd_bind(struct device_driver *drv, const char *buf) +{ + const struct bus_type *bus = drv->bus; + struct device *dev; + int err = -ENODEV; + + dev = bus_find_device_by_name(bus, NULL, buf); + if (dev) + err = device_driver_attach(drv, dev); + + put_device(dev); + + return err; +} + +/* + * Detach IDXD device from driver. + */ +static void idxd_unbind(struct device_driver *drv, const char *buf) +{ + const struct bus_type *bus = drv->bus; + struct device *dev; + + dev = bus_find_device_by_name(bus, NULL, buf); + if (dev && dev->driver == drv) + device_release_driver(dev); + + put_device(dev); +} + /* * Probe idxd PCI device. * If idxd is not given, need to allocate idxd and set up its data. -- cgit v1.2.3-59-g8ed1b From 6078a315aec15e0776fa90347cf4eba7478cdbd7 Mon Sep 17 00:00:00 2001 From: Fenghua Yu Date: Fri, 22 Nov 2024 15:30:26 -0800 Subject: dmaengine: idxd: Add idxd_device_config_save() and idxd_device_config_restore() helpers Add the helpers to save and restore IDXD device configurations. These helpers will be called during Function Level Reset (FLR) processing. Signed-off-by: Fenghua Yu Reviewed-by: Dave Jiang Link: https://lore.kernel.org/r/20241122233028.2762809-4-fenghua.yu@intel.com Signed-off-by: Vinod Koul --- drivers/dma/idxd/idxd.h | 11 +++ drivers/dma/idxd/init.c | 225 ++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 236 insertions(+) diff --git a/drivers/dma/idxd/idxd.h b/drivers/dma/idxd/idxd.h index 0da282b84b59..214b8039439f 100644 --- a/drivers/dma/idxd/idxd.h +++ b/drivers/dma/idxd/idxd.h @@ -374,6 +374,17 @@ struct idxd_device { struct dentry *dbgfs_evl_file; bool user_submission_safe; + + struct idxd_saved_states *idxd_saved; +}; + +struct idxd_saved_states { + struct idxd_device saved_idxd; + struct idxd_evl saved_evl; + struct idxd_engine **saved_engines; + struct idxd_wq **saved_wqs; + struct idxd_group **saved_groups; + unsigned long *saved_wq_enable_map; }; static inline unsigned int evl_ent_size(struct idxd_device *idxd) diff --git a/drivers/dma/idxd/init.c b/drivers/dma/idxd/init.c index eb64a38444c1..ade76a5f2760 100644 --- a/drivers/dma/idxd/init.c +++ b/drivers/dma/idxd/init.c @@ -758,6 +758,231 @@ static void idxd_unbind(struct device_driver *drv, const char *buf) put_device(dev); } +#define idxd_free_saved_configs(saved_configs, count) \ + do { \ + int i; \ + \ + for (i = 0; i < (count); i++) \ + kfree(saved_configs[i]); \ + } while (0) + +static void idxd_free_saved(struct idxd_group **saved_groups, + struct idxd_engine **saved_engines, + struct idxd_wq **saved_wqs, + struct idxd_device *idxd) +{ + if (saved_groups) + idxd_free_saved_configs(saved_groups, idxd->max_groups); + if (saved_engines) + idxd_free_saved_configs(saved_engines, idxd->max_engines); + if (saved_wqs) + idxd_free_saved_configs(saved_wqs, idxd->max_wqs); +} + +/* + * Save IDXD device configurations including engines, groups, wqs etc. + * The saved configurations can be restored when needed. + */ +static int idxd_device_config_save(struct idxd_device *idxd, + struct idxd_saved_states *idxd_saved) +{ + struct device *dev = &idxd->pdev->dev; + int i; + + memcpy(&idxd_saved->saved_idxd, idxd, sizeof(*idxd)); + + if (idxd->evl) { + memcpy(&idxd_saved->saved_evl, idxd->evl, + sizeof(struct idxd_evl)); + } + + struct idxd_group **saved_groups __free(kfree) = + kcalloc_node(idxd->max_groups, + sizeof(struct idxd_group *), + GFP_KERNEL, dev_to_node(dev)); + if (!saved_groups) + return -ENOMEM; + + for (i = 0; i < idxd->max_groups; i++) { + struct idxd_group *saved_group __free(kfree) = + kzalloc_node(sizeof(*saved_group), GFP_KERNEL, + dev_to_node(dev)); + + if (!saved_group) { + /* Free saved groups */ + idxd_free_saved(saved_groups, NULL, NULL, idxd); + + return -ENOMEM; + } + + memcpy(saved_group, idxd->groups[i], sizeof(*saved_group)); + saved_groups[i] = no_free_ptr(saved_group); + } + + struct idxd_engine **saved_engines = + kcalloc_node(idxd->max_engines, + sizeof(struct idxd_engine *), + GFP_KERNEL, dev_to_node(dev)); + if (!saved_engines) { + /* Free saved groups */ + idxd_free_saved(saved_groups, NULL, NULL, idxd); + + return -ENOMEM; + } + for (i = 0; i < idxd->max_engines; i++) { + struct idxd_engine *saved_engine __free(kfree) = + kzalloc_node(sizeof(*saved_engine), GFP_KERNEL, + dev_to_node(dev)); + if (!saved_engine) { + /* Free saved groups and engines */ + idxd_free_saved(saved_groups, saved_engines, NULL, + idxd); + + return -ENOMEM; + } + + memcpy(saved_engine, idxd->engines[i], sizeof(*saved_engine)); + saved_engines[i] = no_free_ptr(saved_engine); + } + + unsigned long *saved_wq_enable_map __free(bitmap) = + bitmap_zalloc_node(idxd->max_wqs, GFP_KERNEL, + dev_to_node(dev)); + if (!saved_wq_enable_map) { + /* Free saved groups and engines */ + idxd_free_saved(saved_groups, saved_engines, NULL, idxd); + + return -ENOMEM; + } + + bitmap_copy(saved_wq_enable_map, idxd->wq_enable_map, idxd->max_wqs); + + struct idxd_wq **saved_wqs __free(kfree) = + kcalloc_node(idxd->max_wqs, sizeof(struct idxd_wq *), + GFP_KERNEL, dev_to_node(dev)); + if (!saved_wqs) { + /* Free saved groups and engines */ + idxd_free_saved(saved_groups, saved_engines, NULL, idxd); + + return -ENOMEM; + } + + for (i = 0; i < idxd->max_wqs; i++) { + struct idxd_wq *saved_wq __free(kfree) = + kzalloc_node(sizeof(*saved_wq), GFP_KERNEL, + dev_to_node(dev)); + struct idxd_wq *wq; + + if (!saved_wq) { + /* Free saved groups, engines, and wqs */ + idxd_free_saved(saved_groups, saved_engines, saved_wqs, + idxd); + + return -ENOMEM; + } + + if (!test_bit(i, saved_wq_enable_map)) + continue; + + wq = idxd->wqs[i]; + mutex_lock(&wq->wq_lock); + memcpy(saved_wq, wq, sizeof(*saved_wq)); + saved_wqs[i] = no_free_ptr(saved_wq); + mutex_unlock(&wq->wq_lock); + } + + /* Save configurations */ + idxd_saved->saved_groups = no_free_ptr(saved_groups); + idxd_saved->saved_engines = no_free_ptr(saved_engines); + idxd_saved->saved_wq_enable_map = no_free_ptr(saved_wq_enable_map); + idxd_saved->saved_wqs = no_free_ptr(saved_wqs); + + return 0; +} + +/* + * Restore IDXD device configurations including engines, groups, wqs etc + * that were saved before. + */ +static void idxd_device_config_restore(struct idxd_device *idxd, + struct idxd_saved_states *idxd_saved) +{ + struct idxd_evl *saved_evl = &idxd_saved->saved_evl; + int i; + + idxd->rdbuf_limit = idxd_saved->saved_idxd.rdbuf_limit; + + if (saved_evl) + idxd->evl->size = saved_evl->size; + + for (i = 0; i < idxd->max_groups; i++) { + struct idxd_group *saved_group, *group; + + saved_group = idxd_saved->saved_groups[i]; + group = idxd->groups[i]; + + group->rdbufs_allowed = saved_group->rdbufs_allowed; + group->rdbufs_reserved = saved_group->rdbufs_reserved; + group->tc_a = saved_group->tc_a; + group->tc_b = saved_group->tc_b; + group->use_rdbuf_limit = saved_group->use_rdbuf_limit; + + kfree(saved_group); + } + kfree(idxd_saved->saved_groups); + + for (i = 0; i < idxd->max_engines; i++) { + struct idxd_engine *saved_engine, *engine; + + saved_engine = idxd_saved->saved_engines[i]; + engine = idxd->engines[i]; + + engine->group = saved_engine->group; + + kfree(saved_engine); + } + kfree(idxd_saved->saved_engines); + + bitmap_copy(idxd->wq_enable_map, idxd_saved->saved_wq_enable_map, + idxd->max_wqs); + bitmap_free(idxd_saved->saved_wq_enable_map); + + for (i = 0; i < idxd->max_wqs; i++) { + struct idxd_wq *saved_wq, *wq; + size_t len; + + if (!test_bit(i, idxd->wq_enable_map)) + continue; + + saved_wq = idxd_saved->saved_wqs[i]; + wq = idxd->wqs[i]; + + mutex_lock(&wq->wq_lock); + + wq->group = saved_wq->group; + wq->flags = saved_wq->flags; + wq->threshold = saved_wq->threshold; + wq->size = saved_wq->size; + wq->priority = saved_wq->priority; + wq->type = saved_wq->type; + len = strlen(saved_wq->name) + 1; + strscpy(wq->name, saved_wq->name, len); + wq->max_xfer_bytes = saved_wq->max_xfer_bytes; + wq->max_batch_size = saved_wq->max_batch_size; + wq->enqcmds_retries = saved_wq->enqcmds_retries; + wq->descs = saved_wq->descs; + wq->idxd_chan = saved_wq->idxd_chan; + len = strlen(saved_wq->driver_name) + 1; + strscpy(wq->driver_name, saved_wq->driver_name, len); + + mutex_unlock(&wq->wq_lock); + + kfree(saved_wq); + } + + kfree(idxd_saved->saved_wqs); +} + /* * Probe idxd PCI device. * If idxd is not given, need to allocate idxd and set up its data. -- cgit v1.2.3-59-g8ed1b From 3e114fa0fd1506c9e75aa0e2eb6a9050eb16b2f8 Mon Sep 17 00:00:00 2001 From: Fenghua Yu Date: Fri, 22 Nov 2024 15:30:27 -0800 Subject: dmaengine: idxd: Refactor halt handler Define a halt handling helper idxd_halt(). Refactor the halt interrupt handler to call the helper. This will simplify the Function Level Reset (FLR) code. No functional change. Signed-off-by: Fenghua Yu Reviewed-by: Dave Jiang Link: https://lore.kernel.org/r/20241122233028.2762809-5-fenghua.yu@intel.com Signed-off-by: Vinod Koul --- drivers/dma/idxd/irq.c | 63 +++++++++++++++++++++++++------------------------- 1 file changed, 31 insertions(+), 32 deletions(-) diff --git a/drivers/dma/idxd/irq.c b/drivers/dma/idxd/irq.c index fc049c9c9892..a46e58b756a5 100644 --- a/drivers/dma/idxd/irq.c +++ b/drivers/dma/idxd/irq.c @@ -383,15 +383,43 @@ static void process_evl_entries(struct idxd_device *idxd) mutex_unlock(&evl->lock); } +static irqreturn_t idxd_halt(struct idxd_device *idxd) +{ + union gensts_reg gensts; + + gensts.bits = ioread32(idxd->reg_base + IDXD_GENSTATS_OFFSET); + if (gensts.state == IDXD_DEVICE_STATE_HALT) { + idxd->state = IDXD_DEV_HALTED; + if (gensts.reset_type == IDXD_DEVICE_RESET_SOFTWARE) { + /* + * If we need a software reset, we will throw the work + * on a system workqueue in order to allow interrupts + * for the device command completions. + */ + INIT_WORK(&idxd->work, idxd_device_reinit); + queue_work(idxd->wq, &idxd->work); + } else { + idxd->state = IDXD_DEV_HALTED; + idxd_wqs_quiesce(idxd); + idxd_wqs_unmap_portal(idxd); + idxd_device_clear_state(idxd); + dev_err(&idxd->pdev->dev, + "idxd halted, need %s.\n", + gensts.reset_type == IDXD_DEVICE_RESET_FLR ? + "FLR" : "system reset"); + } + } + + return IRQ_HANDLED; +} + irqreturn_t idxd_misc_thread(int vec, void *data) { struct idxd_irq_entry *irq_entry = data; struct idxd_device *idxd = ie_to_idxd(irq_entry); struct device *dev = &idxd->pdev->dev; - union gensts_reg gensts; u32 val = 0; int i; - bool err = false; u32 cause; cause = ioread32(idxd->reg_base + IDXD_INTCAUSE_OFFSET); @@ -401,7 +429,7 @@ irqreturn_t idxd_misc_thread(int vec, void *data) iowrite32(cause, idxd->reg_base + IDXD_INTCAUSE_OFFSET); if (cause & IDXD_INTC_HALT_STATE) - goto halt; + return idxd_halt(idxd); if (cause & IDXD_INTC_ERR) { spin_lock(&idxd->dev_lock); @@ -435,7 +463,6 @@ irqreturn_t idxd_misc_thread(int vec, void *data) for (i = 0; i < 4; i++) dev_warn_ratelimited(dev, "err[%d]: %#16.16llx\n", i, idxd->sw_err.bits[i]); - err = true; } if (cause & IDXD_INTC_INT_HANDLE_REVOKED) { @@ -480,34 +507,6 @@ irqreturn_t idxd_misc_thread(int vec, void *data) dev_warn_once(dev, "Unexpected interrupt cause bits set: %#x\n", val); - if (!err) - goto out; - -halt: - gensts.bits = ioread32(idxd->reg_base + IDXD_GENSTATS_OFFSET); - if (gensts.state == IDXD_DEVICE_STATE_HALT) { - idxd->state = IDXD_DEV_HALTED; - if (gensts.reset_type == IDXD_DEVICE_RESET_SOFTWARE) { - /* - * If we need a software reset, we will throw the work - * on a system workqueue in order to allow interrupts - * for the device command completions. - */ - INIT_WORK(&idxd->work, idxd_device_reinit); - queue_work(idxd->wq, &idxd->work); - } else { - idxd->state = IDXD_DEV_HALTED; - idxd_wqs_quiesce(idxd); - idxd_wqs_unmap_portal(idxd); - idxd_device_clear_state(idxd); - dev_err(&idxd->pdev->dev, - "idxd halted, need %s.\n", - gensts.reset_type == IDXD_DEVICE_RESET_FLR ? - "FLR" : "system reset"); - } - } - -out: return IRQ_HANDLED; } -- cgit v1.2.3-59-g8ed1b From 98d187a989036096feaa2fef1ec3b2240ecdeacf Mon Sep 17 00:00:00 2001 From: Fenghua Yu Date: Fri, 22 Nov 2024 15:30:28 -0800 Subject: dmaengine: idxd: Enable Function Level Reset (FLR) for halt When DSA/IAA device hits a fatal error, the device enters a halt state. The driver can reset the device depending on Reset Type required by hardware to recover the device. Supported Reset Types are: 0: Reset Device command 1: Function Level Reset (FLR) 2: Warm reset 3: Cold reset Currently, the driver only supports Reset Type 0. This patch adds support for FLR recovery Type 1. Before issuing a PCIe FLR command, IDXD device and WQ states are saved. After the FLR command execution, the device is recovered to its previous states, allowing the user can continue using the device. Signed-off-by: Fenghua Yu Reviewed-by: Dave Jiang Link: https://lore.kernel.org/r/20241122233028.2762809-6-fenghua.yu@intel.com Signed-off-by: Vinod Koul --- drivers/dma/idxd/init.c | 123 ++++++++++++++++++++++++++++++++++++++++++++++++ drivers/dma/idxd/irq.c | 28 +++++++++-- 2 files changed, 148 insertions(+), 3 deletions(-) diff --git a/drivers/dma/idxd/init.c b/drivers/dma/idxd/init.c index ade76a5f2760..fdf03e253d53 100644 --- a/drivers/dma/idxd/init.c +++ b/drivers/dma/idxd/init.c @@ -983,6 +983,118 @@ static void idxd_device_config_restore(struct idxd_device *idxd, kfree(idxd_saved->saved_wqs); } +static void idxd_reset_prepare(struct pci_dev *pdev) +{ + struct idxd_device *idxd = pci_get_drvdata(pdev); + struct device *dev = &idxd->pdev->dev; + const char *idxd_name; + int rc; + + dev = &idxd->pdev->dev; + idxd_name = dev_name(idxd_confdev(idxd)); + + struct idxd_saved_states *idxd_saved __free(kfree) = + kzalloc_node(sizeof(*idxd_saved), GFP_KERNEL, + dev_to_node(&pdev->dev)); + if (!idxd_saved) { + dev_err(dev, "HALT: no memory\n"); + + return; + } + + /* Save IDXD configurations. */ + rc = idxd_device_config_save(idxd, idxd_saved); + if (rc < 0) { + dev_err(dev, "HALT: cannot save %s configs\n", idxd_name); + + return; + } + + idxd->idxd_saved = no_free_ptr(idxd_saved); + + /* Save PCI device state. */ + pci_save_state(idxd->pdev); +} + +static void idxd_reset_done(struct pci_dev *pdev) +{ + struct idxd_device *idxd = pci_get_drvdata(pdev); + const char *idxd_name; + struct device *dev; + int rc, i; + + if (!idxd->idxd_saved) + return; + + dev = &idxd->pdev->dev; + idxd_name = dev_name(idxd_confdev(idxd)); + + /* Restore PCI device state. */ + pci_restore_state(idxd->pdev); + + /* Unbind idxd device from driver. */ + idxd_unbind(&idxd_drv.drv, idxd_name); + + /* + * Probe PCI device without allocating or changing + * idxd software data which keeps the same as before FLR. + */ + idxd_pci_probe_alloc(idxd, NULL, NULL); + + /* Restore IDXD configurations. */ + idxd_device_config_restore(idxd, idxd->idxd_saved); + + /* Re-configure IDXD device if allowed. */ + if (test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags)) { + rc = idxd_device_config(idxd); + if (rc < 0) { + dev_err(dev, "HALT: %s config fails\n", idxd_name); + goto out; + } + } + + /* Bind IDXD device to driver. */ + rc = idxd_bind(&idxd_drv.drv, idxd_name); + if (rc < 0) { + dev_err(dev, "HALT: binding %s to driver fails\n", idxd_name); + goto out; + } + + /* Bind enabled wq in the IDXD device to driver. */ + for (i = 0; i < idxd->max_wqs; i++) { + if (test_bit(i, idxd->wq_enable_map)) { + struct idxd_wq *wq = idxd->wqs[i]; + char wq_name[32]; + + wq->state = IDXD_WQ_DISABLED; + sprintf(wq_name, "wq%d.%d", idxd->id, wq->id); + /* + * Bind to user driver depending on wq type. + * + * Currently only support user type WQ. Will support + * kernel type WQ in the future. + */ + if (wq->type == IDXD_WQT_USER) + rc = idxd_bind(&idxd_user_drv.drv, wq_name); + else + rc = -EINVAL; + if (rc < 0) { + clear_bit(i, idxd->wq_enable_map); + dev_err(dev, + "HALT: unable to re-enable wq %s\n", + dev_name(wq_confdev(wq))); + } + } + } +out: + kfree(idxd->idxd_saved); +} + +static const struct pci_error_handlers idxd_error_handler = { + .reset_prepare = idxd_reset_prepare, + .reset_done = idxd_reset_done, +}; + /* * Probe idxd PCI device. * If idxd is not given, need to allocate idxd and set up its data. @@ -1056,6 +1168,16 @@ int idxd_pci_probe_alloc(struct idxd_device *idxd, struct pci_dev *pdev, dev_warn(dev, "IDXD debugfs failed to setup\n"); } + if (!alloc_idxd) { + /* Release interrupts in the IDXD device. */ + idxd_cleanup_interrupts(idxd); + + /* Re-enable interrupts in the IDXD device. */ + rc = idxd_setup_interrupts(idxd); + if (rc) + dev_warn(dev, "IDXD interrupts failed to setup\n"); + } + dev_info(&pdev->dev, "Intel(R) Accelerator Device (v%x)\n", idxd->hw.version); @@ -1146,6 +1268,7 @@ static struct pci_driver idxd_pci_driver = { .probe = idxd_pci_probe, .remove = idxd_remove, .shutdown = idxd_shutdown, + .err_handler = &idxd_error_handler, }; static int __init idxd_init_module(void) diff --git a/drivers/dma/idxd/irq.c b/drivers/dma/idxd/irq.c index a46e58b756a5..1107db3ce0a3 100644 --- a/drivers/dma/idxd/irq.c +++ b/drivers/dma/idxd/irq.c @@ -383,6 +383,20 @@ static void process_evl_entries(struct idxd_device *idxd) mutex_unlock(&evl->lock); } +static void idxd_device_flr(struct work_struct *work) +{ + struct idxd_device *idxd = container_of(work, struct idxd_device, work); + int rc; + + /* + * IDXD device requires a Function Level Reset (FLR). + * pci_reset_function() will reset the device with FLR. + */ + rc = pci_reset_function(idxd->pdev); + if (rc) + dev_err(&idxd->pdev->dev, "FLR failed\n"); +} + static irqreturn_t idxd_halt(struct idxd_device *idxd) { union gensts_reg gensts; @@ -398,15 +412,23 @@ static irqreturn_t idxd_halt(struct idxd_device *idxd) */ INIT_WORK(&idxd->work, idxd_device_reinit); queue_work(idxd->wq, &idxd->work); + } else if (gensts.reset_type == IDXD_DEVICE_RESET_FLR) { + idxd->state = IDXD_DEV_HALTED; + idxd_mask_error_interrupts(idxd); + dev_dbg(&idxd->pdev->dev, + "idxd halted, doing FLR. After FLR, configs are restored\n"); + INIT_WORK(&idxd->work, idxd_device_flr); + queue_work(idxd->wq, &idxd->work); + } else { idxd->state = IDXD_DEV_HALTED; idxd_wqs_quiesce(idxd); idxd_wqs_unmap_portal(idxd); idxd_device_clear_state(idxd); dev_err(&idxd->pdev->dev, - "idxd halted, need %s.\n", - gensts.reset_type == IDXD_DEVICE_RESET_FLR ? - "FLR" : "system reset"); + "idxd halted, need system reset"); + + return -ENXIO; } } -- cgit v1.2.3-59-g8ed1b