aboutsummaryrefslogtreecommitdiffstats
path: root/drivers/dma
diff options
context:
space:
mode:
Diffstat (limited to 'drivers/dma')
-rw-r--r--drivers/dma/Kconfig8
-rw-r--r--drivers/dma/Makefile2
-rw-r--r--drivers/dma/cppi41.c97
-rw-r--r--drivers/dma/dmaengine.c21
-rw-r--r--drivers/dma/dw/Kconfig2
-rw-r--r--drivers/dma/dw/core.c211
-rw-r--r--drivers/dma/dw/pci.c19
-rw-r--r--drivers/dma/dw/platform.c1
-rw-r--r--drivers/dma/dw/regs.h59
-rw-r--r--drivers/dma/hsu/pci.c17
-rw-r--r--drivers/dma/ioat/hw.h2
-rw-r--r--drivers/dma/ioat/init.c15
-rw-r--r--drivers/dma/ipu/ipu_irq.c2
-rw-r--r--drivers/dma/omap-dma.c61
-rw-r--r--drivers/dma/pl330.c35
-rw-r--r--drivers/dma/sh/rcar-dmac.c9
-rw-r--r--drivers/dma/ste_dma40.c7
-rw-r--r--drivers/dma/stm32-dma.c103
-rw-r--r--drivers/dma/ti-dma-crossbar.c2
-rw-r--r--drivers/dma/zx_dma.c (renamed from drivers/dma/zx296702_dma.c)6
20 files changed, 463 insertions, 216 deletions
diff --git a/drivers/dma/Kconfig b/drivers/dma/Kconfig
index 263495d0adbd..d01d59812cf3 100644
--- a/drivers/dma/Kconfig
+++ b/drivers/dma/Kconfig
@@ -157,7 +157,7 @@ config DMA_SUN4I
config DMA_SUN6I
tristate "Allwinner A31 SoCs DMA support"
- depends on MACH_SUN6I || MACH_SUN8I || COMPILE_TEST
+ depends on MACH_SUN6I || MACH_SUN8I || (ARM64 && ARCH_SUNXI) || COMPILE_TEST
depends on RESET_CONTROLLER
select DMA_ENGINE
select DMA_VIRTUAL_CHANNELS
@@ -458,7 +458,7 @@ config STM32_DMA
help
Enable support for the on-chip DMA controller on STMicroelectronics
STM32 MCUs.
- If you have a board based on such a MCU and wish to use DMA say Y or M
+ If you have a board based on such a MCU and wish to use DMA say Y
here.
config S3C24XX_DMAC
@@ -571,12 +571,12 @@ config XILINX_ZYNQMP_DMA
Enable support for Xilinx ZynqMP DMA controller.
config ZX_DMA
- tristate "ZTE ZX296702 DMA support"
+ tristate "ZTE ZX DMA support"
depends on ARCH_ZX || COMPILE_TEST
select DMA_ENGINE
select DMA_VIRTUAL_CHANNELS
help
- Support the DMA engine for ZTE ZX296702 platform devices.
+ Support the DMA engine for ZTE ZX family platform devices.
# driver files
diff --git a/drivers/dma/Makefile b/drivers/dma/Makefile
index a4fa3360e609..0b723e94d9e6 100644
--- a/drivers/dma/Makefile
+++ b/drivers/dma/Makefile
@@ -66,7 +66,7 @@ obj-$(CONFIG_TI_CPPI41) += cppi41.o
obj-$(CONFIG_TI_DMA_CROSSBAR) += ti-dma-crossbar.o
obj-$(CONFIG_TI_EDMA) += edma.o
obj-$(CONFIG_XGENE_DMA) += xgene-dma.o
-obj-$(CONFIG_ZX_DMA) += zx296702_dma.o
+obj-$(CONFIG_ZX_DMA) += zx_dma.o
obj-$(CONFIG_ST_FDMA) += st_fdma.o
obj-y += qcom/
diff --git a/drivers/dma/cppi41.c b/drivers/dma/cppi41.c
index d5ba43a87a68..d74cee077842 100644
--- a/drivers/dma/cppi41.c
+++ b/drivers/dma/cppi41.c
@@ -79,14 +79,6 @@
#define QMGR_QUEUE_C(n) (0x2008 + (n) * 0x10)
#define QMGR_QUEUE_D(n) (0x200c + (n) * 0x10)
-/* Glue layer specific */
-/* USBSS / USB AM335x */
-#define USBSS_IRQ_STATUS 0x28
-#define USBSS_IRQ_ENABLER 0x2c
-#define USBSS_IRQ_CLEARR 0x30
-
-#define USBSS_IRQ_PD_COMP (1 << 2)
-
/* Packet Descriptor */
#define PD2_ZERO_LENGTH (1 << 19)
@@ -153,6 +145,8 @@ struct cppi41_dd {
/* context for suspend/resume */
unsigned int dma_tdfdq;
+
+ bool is_suspended;
};
#define FIST_COMPLETION_QUEUE 93
@@ -257,6 +251,10 @@ static struct cppi41_channel *desc_to_chan(struct cppi41_dd *cdd, u32 desc)
BUG_ON(desc_num >= ALLOC_DECS_NUM);
c = cdd->chan_busy[desc_num];
cdd->chan_busy[desc_num] = NULL;
+
+ /* Usecount for chan_busy[], paired with push_desc_queue() */
+ pm_runtime_put(cdd->ddev.dev);
+
return c;
}
@@ -288,14 +286,8 @@ static irqreturn_t cppi41_irq(int irq, void *data)
{
struct cppi41_dd *cdd = data;
struct cppi41_channel *c;
- u32 status;
int i;
- status = cppi_readl(cdd->usbss_mem + USBSS_IRQ_STATUS);
- if (!(status & USBSS_IRQ_PD_COMP))
- return IRQ_NONE;
- cppi_writel(status, cdd->usbss_mem + USBSS_IRQ_STATUS);
-
for (i = QMGR_PENDING_SLOT_Q(FIST_COMPLETION_QUEUE); i < QMGR_NUM_PEND;
i++) {
u32 val;
@@ -317,12 +309,12 @@ static irqreturn_t cppi41_irq(int irq, void *data)
while (val) {
u32 desc, len;
- int error;
- error = pm_runtime_get(cdd->ddev.dev);
- if (error < 0)
- dev_err(cdd->ddev.dev, "%s pm runtime get: %i\n",
- __func__, error);
+ /*
+ * This should never trigger, see the comments in
+ * push_desc_queue()
+ */
+ WARN_ON(cdd->is_suspended);
q_num = __fls(val);
val &= ~(1 << q_num);
@@ -343,9 +335,6 @@ static irqreturn_t cppi41_irq(int irq, void *data)
c->residue = pd_trans_len(c->desc->pd6) - len;
dma_cookie_complete(&c->txd);
dmaengine_desc_get_callback_invoke(&c->txd, NULL);
-
- pm_runtime_mark_last_busy(cdd->ddev.dev);
- pm_runtime_put_autosuspend(cdd->ddev.dev);
}
}
return IRQ_HANDLED;
@@ -447,6 +436,15 @@ static void push_desc_queue(struct cppi41_channel *c)
*/
__iowmb();
+ /*
+ * DMA transfers can take at least 200ms to complete with USB mass
+ * storage connected. To prevent autosuspend timeouts, we must use
+ * pm_runtime_get/put() when chan_busy[] is modified. This will get
+ * cleared in desc_to_chan() or cppi41_stop_chan() depending on the
+ * outcome of the transfer.
+ */
+ pm_runtime_get(cdd->ddev.dev);
+
desc_phys = lower_32_bits(c->desc_phys);
desc_num = (desc_phys - cdd->descs_phys) / sizeof(struct cppi41_desc);
WARN_ON(cdd->chan_busy[desc_num]);
@@ -457,20 +455,26 @@ static void push_desc_queue(struct cppi41_channel *c)
cppi_writel(reg, cdd->qmgr_mem + QMGR_QUEUE_D(c->q_num));
}
-static void pending_desc(struct cppi41_channel *c)
+/*
+ * Caller must hold cdd->lock to prevent push_desc_queue()
+ * getting called out of order. We have both cppi41_dma_issue_pending()
+ * and cppi41_runtime_resume() call this function.
+ */
+static void cppi41_run_queue(struct cppi41_dd *cdd)
{
- struct cppi41_dd *cdd = c->cdd;
- unsigned long flags;
+ struct cppi41_channel *c, *_c;
- spin_lock_irqsave(&cdd->lock, flags);
- list_add_tail(&c->node, &cdd->pending);
- spin_unlock_irqrestore(&cdd->lock, flags);
+ list_for_each_entry_safe(c, _c, &cdd->pending, node) {
+ push_desc_queue(c);
+ list_del(&c->node);
+ }
}
static void cppi41_dma_issue_pending(struct dma_chan *chan)
{
struct cppi41_channel *c = to_cpp41_chan(chan);
struct cppi41_dd *cdd = c->cdd;
+ unsigned long flags;
int error;
error = pm_runtime_get(cdd->ddev.dev);
@@ -482,10 +486,11 @@ static void cppi41_dma_issue_pending(struct dma_chan *chan)
return;
}
- if (likely(pm_runtime_active(cdd->ddev.dev)))
- push_desc_queue(c);
- else
- pending_desc(c);
+ spin_lock_irqsave(&cdd->lock, flags);
+ list_add_tail(&c->node, &cdd->pending);
+ if (!cdd->is_suspended)
+ cppi41_run_queue(cdd);
+ spin_unlock_irqrestore(&cdd->lock, flags);
pm_runtime_mark_last_busy(cdd->ddev.dev);
pm_runtime_put_autosuspend(cdd->ddev.dev);
@@ -599,6 +604,7 @@ static void cppi41_compute_td_desc(struct cppi41_desc *d)
static int cppi41_tear_down_chan(struct cppi41_channel *c)
{
+ struct dmaengine_result abort_result;
struct cppi41_dd *cdd = c->cdd;
struct cppi41_desc *td;
u32 reg;
@@ -682,6 +688,12 @@ static int cppi41_tear_down_chan(struct cppi41_channel *c)
c->td_seen = 0;
c->td_desc_seen = 0;
cppi_writel(0, c->gcr_reg);
+
+ /* Invoke the callback to do the necessary clean-up */
+ abort_result.result = DMA_TRANS_ABORTED;
+ dma_cookie_complete(&c->txd);
+ dmaengine_desc_get_callback_invoke(&c->txd, &abort_result);
+
return 0;
}
@@ -705,6 +717,9 @@ static int cppi41_stop_chan(struct dma_chan *chan)
WARN_ON(!cdd->chan_busy[desc_num]);
cdd->chan_busy[desc_num] = NULL;
+ /* Usecount for chan_busy[], paired with push_desc_queue() */
+ pm_runtime_put(cdd->ddev.dev);
+
return 0;
}
@@ -1044,8 +1059,6 @@ static int cppi41_dma_probe(struct platform_device *pdev)
goto err_irq;
}
- cppi_writel(USBSS_IRQ_PD_COMP, cdd->usbss_mem + USBSS_IRQ_ENABLER);
-
ret = devm_request_irq(&pdev->dev, irq, glue_info->isr, IRQF_SHARED,
dev_name(dev), cdd);
if (ret)
@@ -1069,7 +1082,6 @@ err_of:
dma_async_device_unregister(&cdd->ddev);
err_dma_reg:
err_irq:
- cppi_writel(0, cdd->usbss_mem + USBSS_IRQ_CLEARR);
cleanup_chans(cdd);
err_chans:
deinit_cppi41(dev, cdd);
@@ -1097,7 +1109,6 @@ static int cppi41_dma_remove(struct platform_device *pdev)
of_dma_controller_free(pdev->dev.of_node);
dma_async_device_unregister(&cdd->ddev);
- cppi_writel(0, cdd->usbss_mem + USBSS_IRQ_CLEARR);
devm_free_irq(&pdev->dev, cdd->irq, cdd);
cleanup_chans(cdd);
deinit_cppi41(&pdev->dev, cdd);
@@ -1116,7 +1127,6 @@ static int __maybe_unused cppi41_suspend(struct device *dev)
struct cppi41_dd *cdd = dev_get_drvdata(dev);
cdd->dma_tdfdq = cppi_readl(cdd->ctrl_mem + DMA_TDFDQ);
- cppi_writel(0, cdd->usbss_mem + USBSS_IRQ_CLEARR);
disable_sched(cdd);
return 0;
@@ -1142,16 +1152,18 @@ static int __maybe_unused cppi41_resume(struct device *dev)
cppi_writel(QMGR_SCRATCH_SIZE, cdd->qmgr_mem + QMGR_LRAM_SIZE);
cppi_writel(0, cdd->qmgr_mem + QMGR_LRAM1_BASE);
- cppi_writel(USBSS_IRQ_PD_COMP, cdd->usbss_mem + USBSS_IRQ_ENABLER);
-
return 0;
}
static int __maybe_unused cppi41_runtime_suspend(struct device *dev)
{
struct cppi41_dd *cdd = dev_get_drvdata(dev);
+ unsigned long flags;
+ spin_lock_irqsave(&cdd->lock, flags);
+ cdd->is_suspended = true;
WARN_ON(!list_empty(&cdd->pending));
+ spin_unlock_irqrestore(&cdd->lock, flags);
return 0;
}
@@ -1159,14 +1171,11 @@ static int __maybe_unused cppi41_runtime_suspend(struct device *dev)
static int __maybe_unused cppi41_runtime_resume(struct device *dev)
{
struct cppi41_dd *cdd = dev_get_drvdata(dev);
- struct cppi41_channel *c, *_c;
unsigned long flags;
spin_lock_irqsave(&cdd->lock, flags);
- list_for_each_entry_safe(c, _c, &cdd->pending, node) {
- push_desc_queue(c);
- list_del(&c->node);
- }
+ cdd->is_suspended = false;
+ cppi41_run_queue(cdd);
spin_unlock_irqrestore(&cdd->lock, flags);
return 0;
diff --git a/drivers/dma/dmaengine.c b/drivers/dma/dmaengine.c
index 6b535262ac5d..24e0221fd66d 100644
--- a/drivers/dma/dmaengine.c
+++ b/drivers/dma/dmaengine.c
@@ -65,7 +65,7 @@
#include <linux/mempool.h>
static DEFINE_MUTEX(dma_list_mutex);
-static DEFINE_IDR(dma_idr);
+static DEFINE_IDA(dma_ida);
static LIST_HEAD(dma_device_list);
static long dmaengine_ref_count;
@@ -162,7 +162,7 @@ static void chan_dev_release(struct device *dev)
chan_dev = container_of(dev, typeof(*chan_dev), device);
if (atomic_dec_and_test(chan_dev->idr_ref)) {
mutex_lock(&dma_list_mutex);
- idr_remove(&dma_idr, chan_dev->dev_id);
+ ida_remove(&dma_ida, chan_dev->dev_id);
mutex_unlock(&dma_list_mutex);
kfree(chan_dev->idr_ref);
}
@@ -898,14 +898,15 @@ static int get_dma_id(struct dma_device *device)
{
int rc;
- mutex_lock(&dma_list_mutex);
-
- rc = idr_alloc(&dma_idr, NULL, 0, 0, GFP_KERNEL);
- if (rc >= 0)
- device->dev_id = rc;
+ do {
+ if (!ida_pre_get(&dma_ida, GFP_KERNEL))
+ return -ENOMEM;
+ mutex_lock(&dma_list_mutex);
+ rc = ida_get_new(&dma_ida, &device->dev_id);
+ mutex_unlock(&dma_list_mutex);
+ } while (rc == -EAGAIN);
- mutex_unlock(&dma_list_mutex);
- return rc < 0 ? rc : 0;
+ return rc;
}
/**
@@ -1035,7 +1036,7 @@ err_out:
/* if we never registered a channel just release the idr */
if (atomic_read(idr_ref) == 0) {
mutex_lock(&dma_list_mutex);
- idr_remove(&dma_idr, device->dev_id);
+ ida_remove(&dma_ida, device->dev_id);
mutex_unlock(&dma_list_mutex);
kfree(idr_ref);
return rc;
diff --git a/drivers/dma/dw/Kconfig b/drivers/dma/dw/Kconfig
index e00c9b022964..5a37b9fcf40d 100644
--- a/drivers/dma/dw/Kconfig
+++ b/drivers/dma/dw/Kconfig
@@ -24,5 +24,5 @@ config DW_DMAC_PCI
select DW_DMAC_CORE
help
Support the Synopsys DesignWare AHB DMA controller on the
- platfroms that enumerate it as a PCI device. For example,
+ platforms that enumerate it as a PCI device. For example,
Intel Medfield has integrated this GPDMA controller.
diff --git a/drivers/dma/dw/core.c b/drivers/dma/dw/core.c
index e5adf5d1c34f..e500950dad82 100644
--- a/drivers/dma/dw/core.c
+++ b/drivers/dma/dw/core.c
@@ -138,16 +138,32 @@ static void dwc_desc_put(struct dw_dma_chan *dwc, struct dw_desc *desc)
dwc->descs_allocated--;
}
-static void dwc_initialize(struct dw_dma_chan *dwc)
+static void dwc_initialize_chan_idma32(struct dw_dma_chan *dwc)
+{
+ u32 cfghi = 0;
+ u32 cfglo = 0;
+
+ /* Set default burst alignment */
+ cfglo |= IDMA32C_CFGL_DST_BURST_ALIGN | IDMA32C_CFGL_SRC_BURST_ALIGN;
+
+ /* Low 4 bits of the request lines */
+ cfghi |= IDMA32C_CFGH_DST_PER(dwc->dws.dst_id & 0xf);
+ cfghi |= IDMA32C_CFGH_SRC_PER(dwc->dws.src_id & 0xf);
+
+ /* Request line extension (2 bits) */
+ cfghi |= IDMA32C_CFGH_DST_PER_EXT(dwc->dws.dst_id >> 4 & 0x3);
+ cfghi |= IDMA32C_CFGH_SRC_PER_EXT(dwc->dws.src_id >> 4 & 0x3);
+
+ channel_writel(dwc, CFG_LO, cfglo);
+ channel_writel(dwc, CFG_HI, cfghi);
+}
+
+static void dwc_initialize_chan_dw(struct dw_dma_chan *dwc)
{
- struct dw_dma *dw = to_dw_dma(dwc->chan.device);
u32 cfghi = DWC_CFGH_FIFO_MODE;
u32 cfglo = DWC_CFGL_CH_PRIOR(dwc->priority);
bool hs_polarity = dwc->dws.hs_polarity;
- if (test_bit(DW_DMA_IS_INITIALIZED, &dwc->flags))
- return;
-
cfghi |= DWC_CFGH_DST_PER(dwc->dws.dst_id);
cfghi |= DWC_CFGH_SRC_PER(dwc->dws.src_id);
@@ -156,6 +172,19 @@ static void dwc_initialize(struct dw_dma_chan *dwc)
channel_writel(dwc, CFG_LO, cfglo);
channel_writel(dwc, CFG_HI, cfghi);
+}
+
+static void dwc_initialize(struct dw_dma_chan *dwc)
+{
+ struct dw_dma *dw = to_dw_dma(dwc->chan.device);
+
+ if (test_bit(DW_DMA_IS_INITIALIZED, &dwc->flags))
+ return;
+
+ if (dw->pdata->is_idma32)
+ dwc_initialize_chan_idma32(dwc);
+ else
+ dwc_initialize_chan_dw(dwc);
/* Enable interrupts */
channel_set_bit(dw, MASK.XFER, dwc->mask);
@@ -184,6 +213,37 @@ static inline void dwc_chan_disable(struct dw_dma *dw, struct dw_dma_chan *dwc)
cpu_relax();
}
+static u32 bytes2block(struct dw_dma_chan *dwc, size_t bytes,
+ unsigned int width, size_t *len)
+{
+ struct dw_dma *dw = to_dw_dma(dwc->chan.device);
+ u32 block;
+
+ /* Always in bytes for iDMA 32-bit */
+ if (dw->pdata->is_idma32)
+ width = 0;
+
+ if ((bytes >> width) > dwc->block_size) {
+ block = dwc->block_size;
+ *len = block << width;
+ } else {
+ block = bytes >> width;
+ *len = bytes;
+ }
+
+ return block;
+}
+
+static size_t block2bytes(struct dw_dma_chan *dwc, u32 block, u32 width)
+{
+ struct dw_dma *dw = to_dw_dma(dwc->chan.device);
+
+ if (dw->pdata->is_idma32)
+ return IDMA32C_CTLH_BLOCK_TS(block);
+
+ return DWC_CTLH_BLOCK_TS(block) << width;
+}
+
/*----------------------------------------------------------------------*/
/* Perform single block transfer */
@@ -332,7 +392,7 @@ static inline u32 dwc_get_sent(struct dw_dma_chan *dwc)
u32 ctlhi = channel_readl(dwc, CTL_HI);
u32 ctllo = channel_readl(dwc, CTL_LO);
- return (ctlhi & DWC_CTLH_BLOCK_TS_MASK) * (1 << (ctllo >> 4 & 7));
+ return block2bytes(dwc, ctlhi, ctllo >> 4 & 7);
}
static void dwc_scan_descriptors(struct dw_dma *dw, struct dw_dma_chan *dwc)
@@ -692,10 +752,7 @@ dwc_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src,
| DWC_CTLL_FC_M2M;
prev = first = NULL;
- for (offset = 0; offset < len; offset += xfer_count << src_width) {
- xfer_count = min_t(size_t, (len - offset) >> src_width,
- dwc->block_size);
-
+ for (offset = 0; offset < len; offset += xfer_count) {
desc = dwc_desc_get(dwc);
if (!desc)
goto err_desc_get;
@@ -703,8 +760,8 @@ dwc_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src,
lli_write(desc, sar, src + offset);
lli_write(desc, dar, dest + offset);
lli_write(desc, ctllo, ctllo);
- lli_write(desc, ctlhi, xfer_count);
- desc->len = xfer_count << src_width;
+ lli_write(desc, ctlhi, bytes2block(dwc, len - offset, src_width, &xfer_count));
+ desc->len = xfer_count;
if (!first) {
first = desc;
@@ -775,7 +832,8 @@ dwc_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,
for_each_sg(sgl, sg, sg_len, i) {
struct dw_desc *desc;
- u32 len, dlen, mem;
+ u32 len, mem;
+ size_t dlen;
mem = sg_dma_address(sg);
len = sg_dma_len(sg);
@@ -789,17 +847,8 @@ slave_sg_todev_fill_desc:
lli_write(desc, sar, mem);
lli_write(desc, dar, reg);
+ lli_write(desc, ctlhi, bytes2block(dwc, len, mem_width, &dlen));
lli_write(desc, ctllo, ctllo | DWC_CTLL_SRC_WIDTH(mem_width));
- if ((len >> mem_width) > dwc->block_size) {
- dlen = dwc->block_size << mem_width;
- mem += dlen;
- len -= dlen;
- } else {
- dlen = len;
- len = 0;
- }
-
- lli_write(desc, ctlhi, dlen >> mem_width);
desc->len = dlen;
if (!first) {
@@ -809,6 +858,9 @@ slave_sg_todev_fill_desc:
list_add_tail(&desc->desc_node, &first->tx_list);
}
prev = desc;
+
+ mem += dlen;
+ len -= dlen;
total_len += dlen;
if (len)
@@ -828,13 +880,12 @@ slave_sg_todev_fill_desc:
for_each_sg(sgl, sg, sg_len, i) {
struct dw_desc *desc;
- u32 len, dlen, mem;
+ u32 len, mem;
+ size_t dlen;
mem = sg_dma_address(sg);
len = sg_dma_len(sg);
- mem_width = __ffs(data_width | mem | len);
-
slave_sg_fromdev_fill_desc:
desc = dwc_desc_get(dwc);
if (!desc)
@@ -842,16 +893,9 @@ slave_sg_fromdev_fill_desc:
lli_write(desc, sar, reg);
lli_write(desc, dar, mem);
+ lli_write(desc, ctlhi, bytes2block(dwc, len, reg_width, &dlen));
+ mem_width = __ffs(data_width | mem | dlen);
lli_write(desc, ctllo, ctllo | DWC_CTLL_DST_WIDTH(mem_width));
- if ((len >> reg_width) > dwc->block_size) {
- dlen = dwc->block_size << reg_width;
- mem += dlen;
- len -= dlen;
- } else {
- dlen = len;
- len = 0;
- }
- lli_write(desc, ctlhi, dlen >> reg_width);
desc->len = dlen;
if (!first) {
@@ -861,6 +905,9 @@ slave_sg_fromdev_fill_desc:
list_add_tail(&desc->desc_node, &first->tx_list);
}
prev = desc;
+
+ mem += dlen;
+ len -= dlen;
total_len += dlen;
if (len)
@@ -903,25 +950,20 @@ bool dw_dma_filter(struct dma_chan *chan, void *param)
}
EXPORT_SYMBOL_GPL(dw_dma_filter);
-/*
- * Fix sconfig's burst size according to dw_dmac. We need to convert them as:
- * 1 -> 0, 4 -> 1, 8 -> 2, 16 -> 3.
- *
- * NOTE: burst size 2 is not supported by controller.
- *
- * This can be done by finding least significant bit set: n & (n - 1)
- */
-static inline void convert_burst(u32 *maxburst)
-{
- if (*maxburst > 1)
- *maxburst = fls(*maxburst) - 2;
- else
- *maxburst = 0;
-}
-
static int dwc_config(struct dma_chan *chan, struct dma_slave_config *sconfig)
{
struct dw_dma_chan *dwc = to_dw_dma_chan(chan);
+ struct dma_slave_config *sc = &dwc->dma_sconfig;
+ struct dw_dma *dw = to_dw_dma(chan->device);
+ /*
+ * Fix sconfig's burst size according to dw_dmac. We need to convert
+ * them as:
+ * 1 -> 0, 4 -> 1, 8 -> 2, 16 -> 3.
+ *
+ * NOTE: burst size 2 is not supported by DesignWare controller.
+ * iDMA 32-bit supports it.
+ */
+ u32 s = dw->pdata->is_idma32 ? 1 : 2;
/* Check if chan will be configured for slave transfers */
if (!is_slave_direction(sconfig->direction))
@@ -930,28 +972,39 @@ static int dwc_config(struct dma_chan *chan, struct dma_slave_config *sconfig)
memcpy(&dwc->dma_sconfig, sconfig, sizeof(*sconfig));
dwc->direction = sconfig->direction;
- convert_burst(&dwc->dma_sconfig.src_maxburst);
- convert_burst(&dwc->dma_sconfig.dst_maxburst);
+ sc->src_maxburst = sc->src_maxburst > 1 ? fls(sc->src_maxburst) - s : 0;
+ sc->dst_maxburst = sc->dst_maxburst > 1 ? fls(sc->dst_maxburst) - s : 0;
return 0;
}
-static int dwc_pause(struct dma_chan *chan)
+static void dwc_chan_pause(struct dw_dma_chan *dwc, bool drain)
{
- struct dw_dma_chan *dwc = to_dw_dma_chan(chan);
- unsigned long flags;
+ struct dw_dma *dw = to_dw_dma(dwc->chan.device);
unsigned int count = 20; /* timeout iterations */
u32 cfglo;
- spin_lock_irqsave(&dwc->lock, flags);
-
cfglo = channel_readl(dwc, CFG_LO);
+ if (dw->pdata->is_idma32) {
+ if (drain)
+ cfglo |= IDMA32C_CFGL_CH_DRAIN;
+ else
+ cfglo &= ~IDMA32C_CFGL_CH_DRAIN;
+ }
channel_writel(dwc, CFG_LO, cfglo | DWC_CFGL_CH_SUSP);
while (!(channel_readl(dwc, CFG_LO) & DWC_CFGL_FIFO_EMPTY) && count--)
udelay(2);
set_bit(DW_DMA_IS_PAUSED, &dwc->flags);
+}
+static int dwc_pause(struct dma_chan *chan)
+{
+ struct dw_dma_chan *dwc = to_dw_dma_chan(chan);
+ unsigned long flags;
+
+ spin_lock_irqsave(&dwc->lock, flags);
+ dwc_chan_pause(dwc, false);
spin_unlock_irqrestore(&dwc->lock, flags);
return 0;
@@ -993,6 +1046,8 @@ static int dwc_terminate_all(struct dma_chan *chan)
clear_bit(DW_DMA_IS_SOFT_LLP, &dwc->flags);
+ dwc_chan_pause(dwc, true);
+
dwc_chan_disable(dw, dwc);
dwc_chan_resume(dwc);
@@ -1085,6 +1140,32 @@ static void dwc_issue_pending(struct dma_chan *chan)
/*----------------------------------------------------------------------*/
+/*
+ * Program FIFO size of channels.
+ *
+ * By default full FIFO (1024 bytes) is assigned to channel 0. Here we
+ * slice FIFO on equal parts between channels.
+ */
+static void idma32_fifo_partition(struct dw_dma *dw)
+{
+ u64 value = IDMA32C_FP_PSIZE_CH0(128) | IDMA32C_FP_PSIZE_CH1(128) |
+ IDMA32C_FP_UPDATE;
+ u64 fifo_partition = 0;
+
+ if (!dw->pdata->is_idma32)
+ return;
+
+ /* Fill FIFO_PARTITION low bits (Channels 0..1, 4..5) */
+ fifo_partition |= value << 0;
+
+ /* Fill FIFO_PARTITION high bits (Channels 2..3, 6..7) */
+ fifo_partition |= value << 32;
+
+ /* Program FIFO Partition registers - 128 bytes for each channel */
+ idma32_writeq(dw, FIFO_PARTITION1, fifo_partition);
+ idma32_writeq(dw, FIFO_PARTITION0, fifo_partition);
+}
+
static void dw_dma_off(struct dw_dma *dw)
{
unsigned int i;
@@ -1504,8 +1585,16 @@ int dw_dma_probe(struct dw_dma_chip *chip)
/* Force dma off, just in case */
dw_dma_off(dw);
+ idma32_fifo_partition(dw);
+
+ /* Device and instance ID for IRQ and DMA pool */
+ if (pdata->is_idma32)
+ snprintf(dw->name, sizeof(dw->name), "idma32:dmac%d", chip->id);
+ else
+ snprintf(dw->name, sizeof(dw->name), "dw:dmac%d", chip->id);
+
/* Create a pool of consistent memory blocks for hardware descriptors */
- dw->desc_pool = dmam_pool_create("dw_dmac_desc_pool", chip->dev,
+ dw->desc_pool = dmam_pool_create(dw->name, chip->dev,
sizeof(struct dw_desc), 4, 0);
if (!dw->desc_pool) {
dev_err(chip->dev, "No memory for descriptors dma pool\n");
@@ -1516,7 +1605,7 @@ int dw_dma_probe(struct dw_dma_chip *chip)
tasklet_init(&dw->tasklet, dw_dma_tasklet, (unsigned long)dw);
err = request_irq(chip->irq, dw_dma_interrupt, IRQF_SHARED,
- "dw_dmac", dw);
+ dw->name, dw);
if (err)
goto err_pdata;
@@ -1665,6 +1754,8 @@ int dw_dma_enable(struct dw_dma_chip *chip)
{
struct dw_dma *dw = chip->dw;
+ idma32_fifo_partition(dw);
+
dw_dma_on(dw);
return 0;
}
diff --git a/drivers/dma/dw/pci.c b/drivers/dma/dw/pci.c
index 0ae6c3b1d34e..7778ed705a1a 100644
--- a/drivers/dma/dw/pci.c
+++ b/drivers/dma/dw/pci.c
@@ -15,6 +15,18 @@
#include "internal.h"
+static struct dw_dma_platform_data mrfld_pdata = {
+ .nr_channels = 8,
+ .is_private = true,
+ .is_memcpy = true,
+ .is_idma32 = true,
+ .chan_allocation_order = CHAN_ALLOCATION_ASCENDING,
+ .chan_priority = CHAN_PRIORITY_ASCENDING,
+ .block_size = 131071,
+ .nr_masters = 1,
+ .data_width = {4},
+};
+
static int dw_pci_probe(struct pci_dev *pdev, const struct pci_device_id *pid)
{
const struct dw_dma_platform_data *pdata = (void *)pid->driver_data;
@@ -47,6 +59,7 @@ static int dw_pci_probe(struct pci_dev *pdev, const struct pci_device_id *pid)
return -ENOMEM;
chip->dev = &pdev->dev;
+ chip->id = pdev->devfn;
chip->regs = pcim_iomap_table(pdev)[0];
chip->irq = pdev->irq;
chip->pdata = pdata;
@@ -95,14 +108,16 @@ static const struct dev_pm_ops dw_pci_dev_pm_ops = {
};
static const struct pci_device_id dw_pci_id_table[] = {
- /* Medfield */
+ /* Medfield (GPDMA) */
{ PCI_VDEVICE(INTEL, 0x0827) },
- { PCI_VDEVICE(INTEL, 0x0830) },
/* BayTrail */
{ PCI_VDEVICE(INTEL, 0x0f06) },
{ PCI_VDEVICE(INTEL, 0x0f40) },
+ /* Merrifield iDMA 32-bit (GPDMA) */
+ { PCI_VDEVICE(INTEL, 0x11a2), (kernel_ulong_t)&mrfld_pdata },
+
/* Braswell */
{ PCI_VDEVICE(INTEL, 0x2286) },
{ PCI_VDEVICE(INTEL, 0x22c0) },
diff --git a/drivers/dma/dw/platform.c b/drivers/dma/dw/platform.c
index b1655e40cfa2..c639c60b825a 100644
--- a/drivers/dma/dw/platform.c
+++ b/drivers/dma/dw/platform.c
@@ -202,6 +202,7 @@ static int dw_probe(struct platform_device *pdev)
pdata = dw_dma_parse_dt(pdev);
chip->dev = dev;
+ chip->id = pdev->id;
chip->pdata = pdata;
chip->clk = devm_clk_get(chip->dev, "hclk");
diff --git a/drivers/dma/dw/regs.h b/drivers/dma/dw/regs.h
index 4e0128c62704..32a328721c88 100644
--- a/drivers/dma/dw/regs.h
+++ b/drivers/dma/dw/regs.h
@@ -3,15 +3,19 @@
*
* Copyright (C) 2005-2007 Atmel Corporation
* Copyright (C) 2010-2011 ST Microelectronics
+ * Copyright (C) 2016 Intel Corporation
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License version 2 as
* published by the Free Software Foundation.
*/
+#include <linux/bitops.h>
#include <linux/interrupt.h>
#include <linux/dmaengine.h>
+#include <linux/io-64-nonatomic-hi-lo.h>
+
#include "internal.h"
#define DW_DMA_MAX_NR_REQUESTS 16
@@ -85,9 +89,9 @@ struct dw_dma_regs {
DW_REG(ID);
DW_REG(TEST);
- /* reserved */
- DW_REG(__reserved0);
- DW_REG(__reserved1);
+ /* iDMA 32-bit support */
+ DW_REG(CLASS_PRIORITY0);
+ DW_REG(CLASS_PRIORITY1);
/* optional encoded params, 0x3c8..0x3f7 */
u32 __reserved;
@@ -99,6 +103,17 @@ struct dw_dma_regs {
/* top-level parameters */
u32 DW_PARAMS;
+
+ /* component ID */
+ u32 COMP_TYPE;
+ u32 COMP_VERSION;
+
+ /* iDMA 32-bit support */
+ DW_REG(FIFO_PARTITION0);
+ DW_REG(FIFO_PARTITION1);
+
+ DW_REG(SAI_ERR);
+ DW_REG(GLOBAL_CFG);
};
/*
@@ -170,8 +185,9 @@ enum dw_dma_msize {
#define DWC_CTLL_LLP_S_EN (1 << 28) /* src block chain */
/* Bitfields in CTL_HI */
-#define DWC_CTLH_DONE 0x00001000
-#define DWC_CTLH_BLOCK_TS_MASK 0x00000fff
+#define DWC_CTLH_BLOCK_TS_MASK GENMASK(11, 0)
+#define DWC_CTLH_BLOCK_TS(x) ((x) & DWC_CTLH_BLOCK_TS_MASK)
+#define DWC_CTLH_DONE (1 << 12)
/* Bitfields in CFG_LO */
#define DWC_CFGL_CH_PRIOR_MASK (0x7 << 5) /* priority mask */
@@ -214,6 +230,33 @@ enum dw_dma_msize {
/* Bitfields in CFG */
#define DW_CFG_DMA_EN (1 << 0)
+/* iDMA 32-bit support */
+
+/* Bitfields in CTL_HI */
+#define IDMA32C_CTLH_BLOCK_TS_MASK GENMASK(16, 0)
+#define IDMA32C_CTLH_BLOCK_TS(x) ((x) & IDMA32C_CTLH_BLOCK_TS_MASK)
+#define IDMA32C_CTLH_DONE (1 << 17)
+
+/* Bitfields in CFG_LO */
+#define IDMA32C_CFGL_DST_BURST_ALIGN (1 << 0) /* dst burst align */
+#define IDMA32C_CFGL_SRC_BURST_ALIGN (1 << 1) /* src burst align */
+#define IDMA32C_CFGL_CH_DRAIN (1 << 10) /* drain FIFO */
+#define IDMA32C_CFGL_DST_OPT_BL (1 << 20) /* optimize dst burst length */
+#define IDMA32C_CFGL_SRC_OPT_BL (1 << 21) /* optimize src burst length */
+
+/* Bitfields in CFG_HI */
+#define IDMA32C_CFGH_SRC_PER(x) ((x) << 0)
+#define IDMA32C_CFGH_DST_PER(x) ((x) << 4)
+#define IDMA32C_CFGH_RD_ISSUE_THD(x) ((x) << 8)
+#define IDMA32C_CFGH_RW_ISSUE_THD(x) ((x) << 18)
+#define IDMA32C_CFGH_SRC_PER_EXT(x) ((x) << 28) /* src peripheral extension */
+#define IDMA32C_CFGH_DST_PER_EXT(x) ((x) << 30) /* dst peripheral extension */
+
+/* Bitfields in FIFO_PARTITION */
+#define IDMA32C_FP_PSIZE_CH0(x) ((x) << 0)
+#define IDMA32C_FP_PSIZE_CH1(x) ((x) << 13)
+#define IDMA32C_FP_UPDATE (1 << 26)
+
enum dw_dmac_flags {
DW_DMA_IS_CYCLIC = 0,
DW_DMA_IS_SOFT_LLP = 1,
@@ -270,6 +313,7 @@ static inline struct dw_dma_chan *to_dw_dma_chan(struct dma_chan *chan)
struct dw_dma {
struct dma_device dma;
+ char name[20];
void __iomem *regs;
struct dma_pool *desc_pool;
struct tasklet_struct tasklet;
@@ -293,6 +337,11 @@ static inline struct dw_dma_regs __iomem *__dw_regs(struct dw_dma *dw)
#define dma_writel(dw, name, val) \
dma_writel_native((val), &(__dw_regs(dw)->name))
+#define idma32_readq(dw, name) \
+ hi_lo_readq(&(__dw_regs(dw)->name))
+#define idma32_writeq(dw, name, val) \
+ hi_lo_writeq((val), &(__dw_regs(dw)->name))
+
#define channel_set_bit(dw, reg, mask) \
dma_writel(dw, reg, ((mask) << 8) | (mask))
#define channel_clear_bit(dw, reg, mask) \
diff --git a/drivers/dma/hsu/pci.c b/drivers/dma/hsu/pci.c
index 4875fa428e81..ad45cd344bba 100644
--- a/drivers/dma/hsu/pci.c
+++ b/drivers/dma/hsu/pci.c
@@ -23,15 +23,28 @@
#define HSU_PCI_CHAN_OFFSET 0x100
+#define PCI_DEVICE_ID_INTEL_MFLD_HSU_DMA 0x081e
+#define PCI_DEVICE_ID_INTEL_MRFLD_HSU_DMA 0x1192
+
static irqreturn_t hsu_pci_irq(int irq, void *dev)
{
struct hsu_dma_chip *chip = dev;
+ struct pci_dev *pdev = to_pci_dev(chip->dev);
u32 dmaisr;
u32 status;
unsigned short i;
int ret = 0;
int err;
+ /*
+ * On Intel Tangier B0 and Anniedale the interrupt line, disregarding
+ * to have different numbers, is shared between HSU DMA and UART IPs.
+ * Thus on such SoCs we are expecting that IRQ handler is called in
+ * UART driver only.
+ */
+ if (pdev->device == PCI_DEVICE_ID_INTEL_MRFLD_HSU_DMA)
+ return IRQ_HANDLED;
+
dmaisr = readl(chip->regs + HSU_PCI_DMAISR);
for (i = 0; i < chip->hsu->nr_channels; i++) {
if (dmaisr & 0x1) {
@@ -113,8 +126,8 @@ static void hsu_pci_remove(struct pci_dev *pdev)
}
static const struct pci_device_id hsu_pci_id_table[] = {
- { PCI_VDEVICE(INTEL, 0x081e), 0 },
- { PCI_VDEVICE(INTEL, 0x1192), 0 },
+ { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_MFLD_HSU_DMA), 0 },
+ { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_MRFLD_HSU_DMA), 0 },
{ }
};
MODULE_DEVICE_TABLE(pci, hsu_pci_id_table);
diff --git a/drivers/dma/ioat/hw.h b/drivers/dma/ioat/hw.h
index 8e67895bcca3..abcc51b343ce 100644
--- a/drivers/dma/ioat/hw.h
+++ b/drivers/dma/ioat/hw.h
@@ -64,6 +64,8 @@
#define PCI_DEVICE_ID_INTEL_IOAT_BDX8 0x6f2e
#define PCI_DEVICE_ID_INTEL_IOAT_BDX9 0x6f2f
+#define PCI_DEVICE_ID_INTEL_IOAT_SKX 0x2021
+
#define IOAT_VER_1_2 0x12 /* Version 1.2 */
#define IOAT_VER_2_0 0x20 /* Version 2.0 */
#define IOAT_VER_3_0 0x30 /* Version 3.0 */
diff --git a/drivers/dma/ioat/init.c b/drivers/dma/ioat/init.c
index 90eddd9f07e4..cc5259b881d4 100644
--- a/drivers/dma/ioat/init.c
+++ b/drivers/dma/ioat/init.c
@@ -106,6 +106,8 @@ static struct pci_device_id ioat_pci_tbl[] = {
{ PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_BDX8) },
{ PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_BDX9) },
+ { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_SKX) },
+
/* I/OAT v3.3 platforms */
{ PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_BWD0) },
{ PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_BWD1) },
@@ -243,10 +245,15 @@ static bool is_bdx_ioat(struct pci_dev *pdev)
}
}
+static inline bool is_skx_ioat(struct pci_dev *pdev)
+{
+ return (pdev->device == PCI_DEVICE_ID_INTEL_IOAT_SKX) ? true : false;
+}
+
static bool is_xeon_cb32(struct pci_dev *pdev)
{
return is_jf_ioat(pdev) || is_snb_ioat(pdev) || is_ivb_ioat(pdev) ||
- is_hsw_ioat(pdev) || is_bdx_ioat(pdev);
+ is_hsw_ioat(pdev) || is_bdx_ioat(pdev) || is_skx_ioat(pdev);
}
bool is_bwd_ioat(struct pci_dev *pdev)
@@ -693,7 +700,7 @@ static int ioat_alloc_chan_resources(struct dma_chan *c)
/* doing 2 32bit writes to mmio since 1 64b write doesn't work */
ioat_chan->completion =
dma_pool_zalloc(ioat_chan->ioat_dma->completion_pool,
- GFP_KERNEL, &ioat_chan->completion_dma);
+ GFP_NOWAIT, &ioat_chan->completion_dma);
if (!ioat_chan->completion)
return -ENOMEM;
@@ -703,7 +710,7 @@ static int ioat_alloc_chan_resources(struct dma_chan *c)
ioat_chan->reg_base + IOAT_CHANCMP_OFFSET_HIGH);
order = IOAT_MAX_ORDER;
- ring = ioat_alloc_ring(c, order, GFP_KERNEL);
+ ring = ioat_alloc_ring(c, order, GFP_NOWAIT);
if (!ring)
return -ENOMEM;
@@ -1357,6 +1364,8 @@ static int ioat_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
device->version = readb(device->reg_base + IOAT_VER_OFFSET);
if (device->version >= IOAT_VER_3_0) {
+ if (is_skx_ioat(pdev))
+ device->version = IOAT_VER_3_2;
err = ioat3_dma_probe(device, ioat_dca_enabled);
if (device->version >= IOAT_VER_3_3)
diff --git a/drivers/dma/ipu/ipu_irq.c b/drivers/dma/ipu/ipu_irq.c
index dd184b50e5b4..284627806b88 100644
--- a/drivers/dma/ipu/ipu_irq.c
+++ b/drivers/dma/ipu/ipu_irq.c
@@ -272,7 +272,7 @@ static void ipu_irq_handler(struct irq_desc *desc)
u32 status;
int i, line;
- for (i = IPU_IRQ_NR_FN_BANKS; i < IPU_IRQ_NR_BANKS; i++) {
+ for (i = 0; i < IPU_IRQ_NR_BANKS; i++) {
struct ipu_irq_bank *bank = irq_bank + i;
raw_spin_lock(&bank_lock);
diff --git a/drivers/dma/omap-dma.c b/drivers/dma/omap-dma.c
index ac68666cd3f4..daf479cce691 100644
--- a/drivers/dma/omap-dma.c
+++ b/drivers/dma/omap-dma.c
@@ -938,21 +938,14 @@ static struct dma_async_tx_descriptor *omap_dma_prep_slave_sg(
d->ccr |= CCR_DST_AMODE_POSTINC;
if (port_window) {
d->ccr |= CCR_SRC_AMODE_DBLIDX;
- d->ei = 1;
- /*
- * One frame covers the port_window and by configure
- * the source frame index to be -1 * (port_window - 1)
- * we instruct the sDMA that after a frame is processed
- * it should move back to the start of the window.
- */
- d->fi = -(port_window_bytes - 1);
if (port_window_bytes >= 64)
- d->csdp = CSDP_SRC_BURST_64 | CSDP_SRC_PACKED;
+ d->csdp |= CSDP_SRC_BURST_64;
else if (port_window_bytes >= 32)
- d->csdp = CSDP_SRC_BURST_32 | CSDP_SRC_PACKED;
+ d->csdp |= CSDP_SRC_BURST_32;
else if (port_window_bytes >= 16)
- d->csdp = CSDP_SRC_BURST_16 | CSDP_SRC_PACKED;
+ d->csdp |= CSDP_SRC_BURST_16;
+
} else {
d->ccr |= CCR_SRC_AMODE_CONSTANT;
}
@@ -962,13 +955,21 @@ static struct dma_async_tx_descriptor *omap_dma_prep_slave_sg(
d->ccr |= CCR_SRC_AMODE_POSTINC;
if (port_window) {
d->ccr |= CCR_DST_AMODE_DBLIDX;
+ d->ei = 1;
+ /*
+ * One frame covers the port_window and by configure
+ * the source frame index to be -1 * (port_window - 1)
+ * we instruct the sDMA that after a frame is processed
+ * it should move back to the start of the window.
+ */
+ d->fi = -(port_window_bytes - 1);
if (port_window_bytes >= 64)
- d->csdp = CSDP_DST_BURST_64 | CSDP_DST_PACKED;
+ d->csdp |= CSDP_DST_BURST_64;
else if (port_window_bytes >= 32)
- d->csdp = CSDP_DST_BURST_32 | CSDP_DST_PACKED;
+ d->csdp |= CSDP_DST_BURST_32;
else if (port_window_bytes >= 16)
- d->csdp = CSDP_DST_BURST_16 | CSDP_DST_PACKED;
+ d->csdp |= CSDP_DST_BURST_16;
} else {
d->ccr |= CCR_DST_AMODE_CONSTANT;
}
@@ -1017,7 +1018,7 @@ static struct dma_async_tx_descriptor *omap_dma_prep_slave_sg(
osg->addr = sg_dma_address(sgent);
osg->en = en;
osg->fn = sg_dma_len(sgent) / frame_bytes;
- if (port_window && dir == DMA_MEM_TO_DEV) {
+ if (port_window && dir == DMA_DEV_TO_MEM) {
osg->ei = 1;
/*
* One frame covers the port_window and by configure
@@ -1452,6 +1453,7 @@ static int omap_dma_probe(struct platform_device *pdev)
struct omap_dmadev *od;
struct resource *res;
int rc, i, irq;
+ u32 lch_count;
od = devm_kzalloc(&pdev->dev, sizeof(*od), GFP_KERNEL);
if (!od)
@@ -1494,20 +1496,31 @@ static int omap_dma_probe(struct platform_device *pdev)
spin_lock_init(&od->lock);
spin_lock_init(&od->irq_lock);
- if (!pdev->dev.of_node) {
- od->dma_requests = od->plat->dma_attr->lch_count;
- if (unlikely(!od->dma_requests))
- od->dma_requests = OMAP_SDMA_REQUESTS;
- } else if (of_property_read_u32(pdev->dev.of_node, "dma-requests",
- &od->dma_requests)) {
+ /* Number of DMA requests */
+ od->dma_requests = OMAP_SDMA_REQUESTS;
+ if (pdev->dev.of_node && of_property_read_u32(pdev->dev.of_node,
+ "dma-requests",
+ &od->dma_requests)) {
dev_info(&pdev->dev,
"Missing dma-requests property, using %u.\n",
OMAP_SDMA_REQUESTS);
- od->dma_requests = OMAP_SDMA_REQUESTS;
}
- od->lch_map = devm_kcalloc(&pdev->dev, od->dma_requests,
- sizeof(*od->lch_map), GFP_KERNEL);
+ /* Number of available logical channels */
+ if (!pdev->dev.of_node) {
+ lch_count = od->plat->dma_attr->lch_count;
+ if (unlikely(!lch_count))
+ lch_count = OMAP_SDMA_CHANNELS;
+ } else if (of_property_read_u32(pdev->dev.of_node, "dma-channels",
+ &lch_count)) {
+ dev_info(&pdev->dev,
+ "Missing dma-channels property, using %u.\n",
+ OMAP_SDMA_CHANNELS);
+ lch_count = OMAP_SDMA_CHANNELS;
+ }
+
+ od->lch_map = devm_kcalloc(&pdev->dev, lch_count, sizeof(*od->lch_map),
+ GFP_KERNEL);
if (!od->lch_map)
return -ENOMEM;
diff --git a/drivers/dma/pl330.c b/drivers/dma/pl330.c
index 87fd01539fcb..f37f4978dabb 100644
--- a/drivers/dma/pl330.c
+++ b/drivers/dma/pl330.c
@@ -448,6 +448,9 @@ struct dma_pl330_chan {
/* for cyclic capability */
bool cyclic;
+
+ /* for runtime pm tracking */
+ bool active;
};
struct pl330_dmac {
@@ -1696,7 +1699,6 @@ static bool _chan_ns(const struct pl330_dmac *pl330, int i)
static struct pl330_thread *pl330_request_channel(struct pl330_dmac *pl330)
{
struct pl330_thread *thrd = NULL;
- unsigned long flags;
int chans, i;
if (pl330->state == DYING)
@@ -1704,8 +1706,6 @@ static struct pl330_thread *pl330_request_channel(struct pl330_dmac *pl330)
chans = pl330->pcfg.num_chan;
- spin_lock_irqsave(&pl330->lock, flags);
-
for (i = 0; i < chans; i++) {
thrd = &pl330->channels[i];
if ((thrd->free) && (!_manager_ns(thrd) ||
@@ -1723,8 +1723,6 @@ static struct pl330_thread *pl330_request_channel(struct pl330_dmac *pl330)
thrd = NULL;
}
- spin_unlock_irqrestore(&pl330->lock, flags);
-
return thrd;
}
@@ -1742,7 +1740,6 @@ static inline void _free_event(struct pl330_thread *thrd, int ev)
static void pl330_release_channel(struct pl330_thread *thrd)
{
struct pl330_dmac *pl330;
- unsigned long flags;
if (!thrd || thrd->free)
return;
@@ -1754,10 +1751,8 @@ static void pl330_release_channel(struct pl330_thread *thrd)
pl330 = thrd->dmac;
- spin_lock_irqsave(&pl330->lock, flags);
_free_event(thrd, thrd->ev);
thrd->free = true;
- spin_unlock_irqrestore(&pl330->lock, flags);
}
/* Initialize the structure for PL330 configuration, that can be used
@@ -1864,9 +1859,10 @@ static int dmac_alloc_resources(struct pl330_dmac *pl330)
* Alloc MicroCode buffer for 'chans' Channel threads.
* A channel's buffer offset is (Channel_Id * MCODE_BUFF_PERCHAN)
*/
- pl330->mcode_cpu = dma_alloc_coherent(pl330->ddma.dev,
+ pl330->mcode_cpu = dma_alloc_attrs(pl330->ddma.dev,
chans * pl330->mcbufsz,
- &pl330->mcode_bus, GFP_KERNEL);
+ &pl330->mcode_bus, GFP_KERNEL,
+ DMA_ATTR_PRIVILEGED);
if (!pl330->mcode_cpu) {
dev_err(pl330->ddma.dev, "%s:%d Can't allocate memory!\n",
__func__, __LINE__);
@@ -2033,6 +2029,7 @@ static void pl330_tasklet(unsigned long data)
_stop(pch->thread);
spin_unlock(&pch->thread->dmac->lock);
power_down = true;
+ pch->active = false;
} else {
/* Make sure the PL330 Channel thread is active */
spin_lock(&pch->thread->dmac->lock);
@@ -2052,6 +2049,7 @@ static void pl330_tasklet(unsigned long data)
desc->status = PREP;
list_move_tail(&desc->node, &pch->work_list);
if (power_down) {
+ pch->active = true;
spin_lock(&pch->thread->dmac->lock);
_start(pch->thread);
spin_unlock(&pch->thread->dmac->lock);
@@ -2117,20 +2115,20 @@ static int pl330_alloc_chan_resources(struct dma_chan *chan)
struct pl330_dmac *pl330 = pch->dmac;
unsigned long flags;
- spin_lock_irqsave(&pch->lock, flags);
+ spin_lock_irqsave(&pl330->lock, flags);
dma_cookie_init(chan);
pch->cyclic = false;
pch->thread = pl330_request_channel(pl330);
if (!pch->thread) {
- spin_unlock_irqrestore(&pch->lock, flags);
+ spin_unlock_irqrestore(&pl330->lock, flags);
return -ENOMEM;
}
tasklet_init(&pch->task, pl330_tasklet, (unsigned long) pch);
- spin_unlock_irqrestore(&pch->lock, flags);
+ spin_unlock_irqrestore(&pl330->lock, flags);
return 1;
}
@@ -2166,6 +2164,7 @@ static int pl330_terminate_all(struct dma_chan *chan)
unsigned long flags;
struct pl330_dmac *pl330 = pch->dmac;
LIST_HEAD(list);
+ bool power_down = false;
pm_runtime_get_sync(pl330->ddma.dev);
spin_lock_irqsave(&pch->lock, flags);
@@ -2176,6 +2175,8 @@ static int pl330_terminate_all(struct dma_chan *chan)
pch->thread->req[0].desc = NULL;
pch->thread->req[1].desc = NULL;
pch->thread->req_running = -1;
+ power_down = pch->active;
+ pch->active = false;
/* Mark all desc done */
list_for_each_entry(desc, &pch->submitted_list, node) {
@@ -2193,6 +2194,8 @@ static int pl330_terminate_all(struct dma_chan *chan)
list_splice_tail_init(&pch->completed_list, &pl330->desc_pool);
spin_unlock_irqrestore(&pch->lock, flags);
pm_runtime_mark_last_busy(pl330->ddma.dev);
+ if (power_down)
+ pm_runtime_put_autosuspend(pl330->ddma.dev);
pm_runtime_put_autosuspend(pl330->ddma.dev);
return 0;
@@ -2228,12 +2231,13 @@ static int pl330_pause(struct dma_chan *chan)
static void pl330_free_chan_resources(struct dma_chan *chan)
{
struct dma_pl330_chan *pch = to_pchan(chan);
+ struct pl330_dmac *pl330 = pch->dmac;
unsigned long flags;
tasklet_kill(&pch->task);
pm_runtime_get_sync(pch->dmac->ddma.dev);
- spin_lock_irqsave(&pch->lock, flags);
+ spin_lock_irqsave(&pl330->lock, flags);
pl330_release_channel(pch->thread);
pch->thread = NULL;
@@ -2241,7 +2245,7 @@ static void pl330_free_chan_resources(struct dma_chan *chan)
if (pch->cyclic)
list_splice_tail_init(&pch->work_list, &pch->dmac->desc_pool);
- spin_unlock_irqrestore(&pch->lock, flags);
+ spin_unlock_irqrestore(&pl330->lock, flags);
pm_runtime_mark_last_busy(pch->dmac->ddma.dev);
pm_runtime_put_autosuspend(pch->dmac->ddma.dev);
}
@@ -2357,6 +2361,7 @@ static void pl330_issue_pending(struct dma_chan *chan)
* updated on work_list emptiness status.
*/
WARN_ON(list_empty(&pch->submitted_list));
+ pch->active = true;
pm_runtime_get_sync(pch->dmac->ddma.dev);
}
list_splice_tail_init(&pch->submitted_list, &pch->work_list);
diff --git a/drivers/dma/sh/rcar-dmac.c b/drivers/dma/sh/rcar-dmac.c
index 2e441d0ccd79..48b22d5c8602 100644
--- a/drivers/dma/sh/rcar-dmac.c
+++ b/drivers/dma/sh/rcar-dmac.c
@@ -986,6 +986,7 @@ static void rcar_dmac_free_chan_resources(struct dma_chan *chan)
{
struct rcar_dmac_chan *rchan = to_rcar_dmac_chan(chan);
struct rcar_dmac *dmac = to_rcar_dmac(chan->device);
+ struct rcar_dmac_chan_map *map = &rchan->map;
struct rcar_dmac_desc_page *page, *_page;
struct rcar_dmac_desc *desc;
LIST_HEAD(list);
@@ -1019,6 +1020,13 @@ static void rcar_dmac_free_chan_resources(struct dma_chan *chan)
free_page((unsigned long)page);
}
+ /* Remove slave mapping if present. */
+ if (map->slave.xfer_size) {
+ dma_unmap_resource(chan->device->dev, map->addr,
+ map->slave.xfer_size, map->dir, 0);
+ map->slave.xfer_size = 0;
+ }
+
pm_runtime_put(chan->device->dev);
}
@@ -1716,6 +1724,7 @@ static int rcar_dmac_probe(struct platform_device *pdev)
dmac->dev = &pdev->dev;
platform_set_drvdata(pdev, dmac);
+ dma_set_mask_and_coherent(dmac->dev, DMA_BIT_MASK(40));
ret = rcar_dmac_parse_of(&pdev->dev, dmac);
if (ret < 0)
diff --git a/drivers/dma/ste_dma40.c b/drivers/dma/ste_dma40.c
index 8684d11b29bb..a6620b671d1d 100644
--- a/drivers/dma/ste_dma40.c
+++ b/drivers/dma/ste_dma40.c
@@ -2809,12 +2809,14 @@ static void __init d40_chan_init(struct d40_base *base, struct dma_device *dma,
static void d40_ops_init(struct d40_base *base, struct dma_device *dev)
{
- if (dma_has_cap(DMA_SLAVE, dev->cap_mask))
+ if (dma_has_cap(DMA_SLAVE, dev->cap_mask)) {
dev->device_prep_slave_sg = d40_prep_slave_sg;
+ dev->directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV);
+ }
if (dma_has_cap(DMA_MEMCPY, dev->cap_mask)) {
dev->device_prep_dma_memcpy = d40_prep_memcpy;
-
+ dev->directions = BIT(DMA_MEM_TO_MEM);
/*
* This controller can only access address at even
* 32bit boundaries, i.e. 2^2
@@ -2836,6 +2838,7 @@ static void d40_ops_init(struct d40_base *base, struct dma_device *dev)
dev->device_pause = d40_pause;
dev->device_resume = d40_resume;
dev->device_terminate_all = d40_terminate_all;
+ dev->residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;
dev->dev = base->dev;
}
diff --git a/drivers/dma/stm32-dma.c b/drivers/dma/stm32-dma.c
index 3688d0873a3e..49f86cabcfec 100644
--- a/drivers/dma/stm32-dma.c
+++ b/drivers/dma/stm32-dma.c
@@ -114,6 +114,7 @@
#define STM32_DMA_MAX_CHANNELS 0x08
#define STM32_DMA_MAX_REQUEST_ID 0x08
#define STM32_DMA_MAX_DATA_PARAM 0x03
+#define STM32_DMA_MAX_BURST 16
enum stm32_dma_width {
STM32_DMA_BYTE,
@@ -403,6 +404,13 @@ static int stm32_dma_terminate_all(struct dma_chan *c)
return 0;
}
+static void stm32_dma_synchronize(struct dma_chan *c)
+{
+ struct stm32_dma_chan *chan = to_stm32_dma_chan(c);
+
+ vchan_synchronize(&chan->vchan);
+}
+
static void stm32_dma_dump_reg(struct stm32_dma_chan *chan)
{
struct stm32_dma_device *dmadev = stm32_dma_get_dev(chan);
@@ -421,7 +429,7 @@ static void stm32_dma_dump_reg(struct stm32_dma_chan *chan)
dev_dbg(chan2dev(chan), "SFCR: 0x%08x\n", sfcr);
}
-static int stm32_dma_start_transfer(struct stm32_dma_chan *chan)
+static void stm32_dma_start_transfer(struct stm32_dma_chan *chan)
{
struct stm32_dma_device *dmadev = stm32_dma_get_dev(chan);
struct virt_dma_desc *vdesc;
@@ -432,12 +440,12 @@ static int stm32_dma_start_transfer(struct stm32_dma_chan *chan)
ret = stm32_dma_disable_chan(chan);
if (ret < 0)
- return ret;
+ return;
if (!chan->desc) {
vdesc = vchan_next_desc(&chan->vchan);
if (!vdesc)
- return -EPERM;
+ return;
chan->desc = to_stm32_dma_desc(vdesc);
chan->next_sg = 0;
@@ -471,7 +479,7 @@ static int stm32_dma_start_transfer(struct stm32_dma_chan *chan)
chan->busy = true;
- return 0;
+ dev_dbg(chan2dev(chan), "vchan %p: started\n", &chan->vchan);
}
static void stm32_dma_configure_next_sg(struct stm32_dma_chan *chan)
@@ -500,8 +508,6 @@ static void stm32_dma_configure_next_sg(struct stm32_dma_chan *chan)
dev_dbg(chan2dev(chan), "CT=0 <=> SM1AR: 0x%08x\n",
stm32_dma_read(dmadev, STM32_DMA_SM1AR(id)));
}
-
- chan->next_sg++;
}
}
@@ -510,6 +516,7 @@ static void stm32_dma_handle_chan_done(struct stm32_dma_chan *chan)
if (chan->desc) {
if (chan->desc->cyclic) {
vchan_cyclic_callback(&chan->desc->vdesc);
+ chan->next_sg++;
stm32_dma_configure_next_sg(chan);
} else {
chan->busy = false;
@@ -552,15 +559,13 @@ static void stm32_dma_issue_pending(struct dma_chan *c)
{
struct stm32_dma_chan *chan = to_stm32_dma_chan(c);
unsigned long flags;
- int ret;
spin_lock_irqsave(&chan->vchan.lock, flags);
- if (!chan->busy) {
- if (vchan_issue_pending(&chan->vchan) && !chan->desc) {
- ret = stm32_dma_start_transfer(chan);
- if ((!ret) && (chan->desc->cyclic))
- stm32_dma_configure_next_sg(chan);
- }
+ if (vchan_issue_pending(&chan->vchan) && !chan->desc && !chan->busy) {
+ dev_dbg(chan2dev(chan), "vchan %p: issued\n", &chan->vchan);
+ stm32_dma_start_transfer(chan);
+ if (chan->desc->cyclic)
+ stm32_dma_configure_next_sg(chan);
}
spin_unlock_irqrestore(&chan->vchan.lock, flags);
}
@@ -848,26 +853,40 @@ static struct dma_async_tx_descriptor *stm32_dma_prep_dma_memcpy(
return vchan_tx_prep(&chan->vchan, &desc->vdesc, flags);
}
+static u32 stm32_dma_get_remaining_bytes(struct stm32_dma_chan *chan)
+{
+ u32 dma_scr, width, ndtr;
+ struct stm32_dma_device *dmadev = stm32_dma_get_dev(chan);
+
+ dma_scr = stm32_dma_read(dmadev, STM32_DMA_SCR(chan->id));
+ width = STM32_DMA_SCR_PSIZE_GET(dma_scr);
+ ndtr = stm32_dma_read(dmadev, STM32_DMA_SNDTR(chan->id));
+
+ return ndtr << width;
+}
+
static size_t stm32_dma_desc_residue(struct stm32_dma_chan *chan,
struct stm32_dma_desc *desc,
u32 next_sg)
{
- struct stm32_dma_device *dmadev = stm32_dma_get_dev(chan);
- u32 dma_scr, width, residue, count;
+ u32 residue = 0;
int i;
- residue = 0;
+ /*
+ * In cyclic mode, for the last period, residue = remaining bytes from
+ * NDTR
+ */
+ if (chan->desc->cyclic && next_sg == 0)
+ return stm32_dma_get_remaining_bytes(chan);
+ /*
+ * For all other periods in cyclic mode, and in sg mode,
+ * residue = remaining bytes from NDTR + remaining periods/sg to be
+ * transferred
+ */
for (i = next_sg; i < desc->num_sgs; i++)
residue += desc->sg_req[i].len;
-
- if (next_sg != 0) {
- dma_scr = stm32_dma_read(dmadev, STM32_DMA_SCR(chan->id));
- width = STM32_DMA_SCR_PSIZE_GET(dma_scr);
- count = stm32_dma_read(dmadev, STM32_DMA_SNDTR(chan->id));
-
- residue += count << width;
- }
+ residue += stm32_dma_get_remaining_bytes(chan);
return residue;
}
@@ -880,7 +899,7 @@ static enum dma_status stm32_dma_tx_status(struct dma_chan *c,
struct virt_dma_desc *vdesc;
enum dma_status status;
unsigned long flags;
- u32 residue;
+ u32 residue = 0;
status = dma_cookie_status(c, cookie, state);
if ((status == DMA_COMPLETE) || (!state))
@@ -888,16 +907,12 @@ static enum dma_status stm32_dma_tx_status(struct dma_chan *c,
spin_lock_irqsave(&chan->vchan.lock, flags);
vdesc = vchan_find_desc(&chan->vchan, cookie);
- if (cookie == chan->desc->vdesc.tx.cookie) {
+ if (chan->desc && cookie == chan->desc->vdesc.tx.cookie)
residue = stm32_dma_desc_residue(chan, chan->desc,
chan->next_sg);
- } else if (vdesc) {
+ else if (vdesc)
residue = stm32_dma_desc_residue(chan,
to_stm32_dma_desc(vdesc), 0);
- } else {
- residue = 0;
- }
-
dma_set_residue(state, residue);
spin_unlock_irqrestore(&chan->vchan.lock, flags);
@@ -968,30 +983,36 @@ static struct dma_chan *stm32_dma_of_xlate(struct of_phandle_args *dma_spec,
struct of_dma *ofdma)
{
struct stm32_dma_device *dmadev = ofdma->of_dma_data;
+ struct device *dev = dmadev->ddev.dev;
struct stm32_dma_cfg cfg;
struct stm32_dma_chan *chan;
struct dma_chan *c;
- if (dma_spec->args_count < 3)
+ if (dma_spec->args_count < 4) {
+ dev_err(dev, "Bad number of cells\n");
return NULL;
+ }
cfg.channel_id = dma_spec->args[0];
cfg.request_line = dma_spec->args[1];
cfg.stream_config = dma_spec->args[2];
- cfg.threshold = 0;
+ cfg.threshold = dma_spec->args[3];
- if ((cfg.channel_id >= STM32_DMA_MAX_CHANNELS) || (cfg.request_line >=
- STM32_DMA_MAX_REQUEST_ID))
+ if ((cfg.channel_id >= STM32_DMA_MAX_CHANNELS) ||
+ (cfg.request_line >= STM32_DMA_MAX_REQUEST_ID)) {
+ dev_err(dev, "Bad channel and/or request id\n");
return NULL;
-
- if (dma_spec->args_count > 3)
- cfg.threshold = dma_spec->args[3];
+ }
chan = &dmadev->chan[cfg.channel_id];
c = dma_get_slave_channel(&chan->vchan.chan);
- if (c)
- stm32_dma_set_config(chan, &cfg);
+ if (!c) {
+ dev_err(dev, "No more channel avalaible\n");
+ return NULL;
+ }
+
+ stm32_dma_set_config(chan, &cfg);
return c;
}
@@ -1055,6 +1076,7 @@ static int stm32_dma_probe(struct platform_device *pdev)
dd->device_prep_dma_cyclic = stm32_dma_prep_dma_cyclic;
dd->device_config = stm32_dma_slave_config;
dd->device_terminate_all = stm32_dma_terminate_all;
+ dd->device_synchronize = stm32_dma_synchronize;
dd->src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) |
BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) |
BIT(DMA_SLAVE_BUSWIDTH_4_BYTES);
@@ -1063,6 +1085,7 @@ static int stm32_dma_probe(struct platform_device *pdev)
BIT(DMA_SLAVE_BUSWIDTH_4_BYTES);
dd->directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV);
dd->residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;
+ dd->max_burst = STM32_DMA_MAX_BURST;
dd->dev = &pdev->dev;
INIT_LIST_HEAD(&dd->channels);
diff --git a/drivers/dma/ti-dma-crossbar.c b/drivers/dma/ti-dma-crossbar.c
index 3f24aeb48c0e..2403475a37cf 100644
--- a/drivers/dma/ti-dma-crossbar.c
+++ b/drivers/dma/ti-dma-crossbar.c
@@ -149,6 +149,7 @@ static int ti_am335x_xbar_probe(struct platform_device *pdev)
match = of_match_node(ti_am335x_master_match, dma_node);
if (!match) {
dev_err(&pdev->dev, "DMA master is not supported\n");
+ of_node_put(dma_node);
return -EINVAL;
}
@@ -339,6 +340,7 @@ static int ti_dra7_xbar_probe(struct platform_device *pdev)
match = of_match_node(ti_dra7_master_match, dma_node);
if (!match) {
dev_err(&pdev->dev, "DMA master is not supported\n");
+ of_node_put(dma_node);
return -EINVAL;
}
diff --git a/drivers/dma/zx296702_dma.c b/drivers/dma/zx_dma.c
index 380276d078b2..2bb695315300 100644
--- a/drivers/dma/zx296702_dma.c
+++ b/drivers/dma/zx_dma.c
@@ -26,7 +26,7 @@
#define DRIVER_NAME "zx-dma"
#define DMA_ALIGN 4
-#define DMA_MAX_SIZE (0x10000 - PAGE_SIZE)
+#define DMA_MAX_SIZE (0x10000 - 512)
#define LLI_BLOCK_SIZE (4 * PAGE_SIZE)
#define REG_ZX_SRC_ADDR 0x00
@@ -365,7 +365,8 @@ static enum dma_status zx_dma_tx_status(struct dma_chan *chan,
bytes = 0;
clli = zx_dma_get_curr_lli(p);
- index = (clli - ds->desc_hw_lli) / sizeof(struct zx_desc_hw);
+ index = (clli - ds->desc_hw_lli) /
+ sizeof(struct zx_desc_hw) + 1;
for (; index < ds->desc_num; index++) {
bytes += ds->desc_hw[index].src_x;
/* end of lli */
@@ -812,6 +813,7 @@ static int zx_dma_probe(struct platform_device *op)
INIT_LIST_HEAD(&d->slave.channels);
dma_cap_set(DMA_SLAVE, d->slave.cap_mask);
dma_cap_set(DMA_MEMCPY, d->slave.cap_mask);
+ dma_cap_set(DMA_CYCLIC, d->slave.cap_mask);
dma_cap_set(DMA_PRIVATE, d->slave.cap_mask);
d->slave.dev = &op->dev;
d->slave.device_free_chan_resources = zx_dma_free_chan_resources;
Copyright © 1996 – 2023 Jason A. Donenfeld. All Rights Reverse Engineered.