aboutsummaryrefslogtreecommitdiffstats
path: root/arch/arm/mach-tegra/dma.c
diff options
context:
space:
mode:
Diffstat (limited to '')
-rw-r--r--arch/arm/mach-tegra/dma.c247
1 files changed, 144 insertions, 103 deletions
diff --git a/arch/arm/mach-tegra/dma.c b/arch/arm/mach-tegra/dma.c
index edda6ec5e925..f4ef5eb317bd 100644
--- a/arch/arm/mach-tegra/dma.c
+++ b/arch/arm/mach-tegra/dma.c
@@ -27,9 +27,11 @@
#include <linux/err.h>
#include <linux/irq.h>
#include <linux/delay.h>
+#include <linux/clk.h>
#include <mach/dma.h>
#include <mach/irqs.h>
#include <mach/iomap.h>
+#include <mach/suspend.h>
#define APB_DMA_GEN 0x000
#define GEN_ENABLE (1<<31)
@@ -120,17 +122,14 @@ struct tegra_dma_channel {
void __iomem *addr;
int mode;
int irq;
-
- /* Register shadow */
- u32 csr;
- u32 ahb_seq;
- u32 ahb_ptr;
- u32 apb_seq;
- u32 apb_ptr;
+ int req_transfer_count;
};
#define NV_DMA_MAX_CHANNELS 32
+static bool tegra_dma_initialized;
+static DEFINE_MUTEX(tegra_dma_lock);
+
static DECLARE_BITMAP(channel_usage, NV_DMA_MAX_CHANNELS);
static struct tegra_dma_channel dma_channels[NV_DMA_MAX_CHANNELS];
@@ -138,7 +137,6 @@ static void tegra_dma_update_hw(struct tegra_dma_channel *ch,
struct tegra_dma_req *req);
static void tegra_dma_update_hw_partial(struct tegra_dma_channel *ch,
struct tegra_dma_req *req);
-static void tegra_dma_init_hw(struct tegra_dma_channel *ch);
static void tegra_dma_stop(struct tegra_dma_channel *ch);
void tegra_dma_flush(struct tegra_dma_channel *ch)
@@ -150,6 +148,9 @@ void tegra_dma_dequeue(struct tegra_dma_channel *ch)
{
struct tegra_dma_req *req;
+ if (tegra_dma_is_empty(ch))
+ return;
+
req = list_entry(ch->list.next, typeof(*req), node);
tegra_dma_dequeue_req(ch, req);
@@ -158,10 +159,10 @@ void tegra_dma_dequeue(struct tegra_dma_channel *ch)
void tegra_dma_stop(struct tegra_dma_channel *ch)
{
- unsigned int csr;
- unsigned int status;
+ u32 csr;
+ u32 status;
- csr = ch->csr;
+ csr = readl(ch->addr + APB_DMA_CHAN_CSR);
csr &= ~CSR_IE_EOC;
writel(csr, ch->addr + APB_DMA_CHAN_CSR);
@@ -175,19 +176,16 @@ void tegra_dma_stop(struct tegra_dma_channel *ch)
int tegra_dma_cancel(struct tegra_dma_channel *ch)
{
- unsigned int csr;
+ u32 csr;
unsigned long irq_flags;
spin_lock_irqsave(&ch->lock, irq_flags);
while (!list_empty(&ch->list))
list_del(ch->list.next);
- csr = ch->csr;
+ csr = readl(ch->addr + APB_DMA_CHAN_CSR);
csr &= ~CSR_REQ_SEL_MASK;
csr |= CSR_REQ_SEL_INVALID;
-
- /* Set the enable as that is not shadowed */
- csr |= CSR_ENB;
writel(csr, ch->addr + APB_DMA_CHAN_CSR);
tegra_dma_stop(ch);
@@ -225,22 +223,19 @@ int tegra_dma_dequeue_req(struct tegra_dma_channel *ch,
* - Change the source selector to invalid to stop the DMA from
* FIFO to memory.
* - Read the status register to know the number of pending
- * bytes to be transfered.
+ * bytes to be transferred.
* - Finally stop or program the DMA to the next buffer in the
* list.
*/
- csr = ch->csr;
+ csr = readl(ch->addr + APB_DMA_CHAN_CSR);
csr &= ~CSR_REQ_SEL_MASK;
csr |= CSR_REQ_SEL_INVALID;
-
- /* Set the enable as that is not shadowed */
- csr |= CSR_ENB;
writel(csr, ch->addr + APB_DMA_CHAN_CSR);
/* Get the transfer count */
status = readl(ch->addr + APB_DMA_CHAN_STA);
to_transfer = (status & STA_COUNT_MASK) >> STA_COUNT_SHIFT;
- req_transfer_count = (ch->csr & CSR_WCOUNT_MASK) >> CSR_WCOUNT_SHIFT;
+ req_transfer_count = ch->req_transfer_count;
req_transfer_count += 1;
to_transfer += 1;
@@ -249,7 +244,7 @@ int tegra_dma_dequeue_req(struct tegra_dma_channel *ch,
if (status & STA_BUSY)
req->bytes_transferred -= to_transfer;
- /* In continous transfer mode, DMA only tracks the count of the
+ /* In continuous transfer mode, DMA only tracks the count of the
* half DMA buffer. So, if the DMA already finished half the DMA
* then add the half buffer to the completed count.
*
@@ -318,6 +313,7 @@ int tegra_dma_enqueue_req(struct tegra_dma_channel *ch,
struct tegra_dma_req *req)
{
unsigned long irq_flags;
+ struct tegra_dma_req *_req;
int start_dma = 0;
if (req->size > NV_DMA_MAX_TRASFER_SIZE ||
@@ -328,6 +324,13 @@ int tegra_dma_enqueue_req(struct tegra_dma_channel *ch,
spin_lock_irqsave(&ch->lock, irq_flags);
+ list_for_each_entry(_req, &ch->list, node) {
+ if (req == _req) {
+ spin_unlock_irqrestore(&ch->lock, irq_flags);
+ return -EEXIST;
+ }
+ }
+
req->bytes_transferred = 0;
req->status = 0;
req->buffer_status = 0;
@@ -348,7 +351,12 @@ EXPORT_SYMBOL(tegra_dma_enqueue_req);
struct tegra_dma_channel *tegra_dma_allocate_channel(int mode)
{
int channel;
- struct tegra_dma_channel *ch;
+ struct tegra_dma_channel *ch = NULL;
+
+ if (WARN_ON(!tegra_dma_initialized))
+ return NULL;
+
+ mutex_lock(&tegra_dma_lock);
/* first channel is the shared channel */
if (mode & TEGRA_DMA_SHARED) {
@@ -357,11 +365,14 @@ struct tegra_dma_channel *tegra_dma_allocate_channel(int mode)
channel = find_first_zero_bit(channel_usage,
ARRAY_SIZE(dma_channels));
if (channel >= ARRAY_SIZE(dma_channels))
- return NULL;
+ goto out;
}
__set_bit(channel, channel_usage);
ch = &dma_channels[channel];
ch->mode = mode;
+
+out:
+ mutex_unlock(&tegra_dma_lock);
return ch;
}
EXPORT_SYMBOL(tegra_dma_allocate_channel);
@@ -371,22 +382,27 @@ void tegra_dma_free_channel(struct tegra_dma_channel *ch)
if (ch->mode & TEGRA_DMA_SHARED)
return;
tegra_dma_cancel(ch);
+ mutex_lock(&tegra_dma_lock);
__clear_bit(ch->id, channel_usage);
+ mutex_unlock(&tegra_dma_lock);
}
EXPORT_SYMBOL(tegra_dma_free_channel);
static void tegra_dma_update_hw_partial(struct tegra_dma_channel *ch,
struct tegra_dma_req *req)
{
+ u32 apb_ptr;
+ u32 ahb_ptr;
+
if (req->to_memory) {
- ch->apb_ptr = req->source_addr;
- ch->ahb_ptr = req->dest_addr;
+ apb_ptr = req->source_addr;
+ ahb_ptr = req->dest_addr;
} else {
- ch->apb_ptr = req->dest_addr;
- ch->ahb_ptr = req->source_addr;
+ apb_ptr = req->dest_addr;
+ ahb_ptr = req->source_addr;
}
- writel(ch->apb_ptr, ch->addr + APB_DMA_CHAN_APB_PTR);
- writel(ch->ahb_ptr, ch->addr + APB_DMA_CHAN_AHB_PTR);
+ writel(apb_ptr, ch->addr + APB_DMA_CHAN_APB_PTR);
+ writel(ahb_ptr, ch->addr + APB_DMA_CHAN_AHB_PTR);
req->status = TEGRA_DMA_REQ_INFLIGHT;
return;
@@ -400,38 +416,39 @@ static void tegra_dma_update_hw(struct tegra_dma_channel *ch,
int ahb_bus_width;
int apb_bus_width;
int index;
- unsigned long csr;
+ u32 ahb_seq;
+ u32 apb_seq;
+ u32 ahb_ptr;
+ u32 apb_ptr;
+ u32 csr;
+
+ csr = CSR_IE_EOC | CSR_FLOW;
+ ahb_seq = AHB_SEQ_INTR_ENB | AHB_SEQ_BURST_1;
+ apb_seq = 0;
- ch->csr |= CSR_FLOW;
- ch->csr &= ~CSR_REQ_SEL_MASK;
- ch->csr |= req->req_sel << CSR_REQ_SEL_SHIFT;
- ch->ahb_seq &= ~AHB_SEQ_BURST_MASK;
- ch->ahb_seq |= AHB_SEQ_BURST_1;
+ csr |= req->req_sel << CSR_REQ_SEL_SHIFT;
/* One shot mode is always single buffered,
* continuous mode is always double buffered
* */
if (ch->mode & TEGRA_DMA_MODE_ONESHOT) {
- ch->csr |= CSR_ONCE;
- ch->ahb_seq &= ~AHB_SEQ_DBL_BUF;
- ch->csr &= ~CSR_WCOUNT_MASK;
- ch->csr |= ((req->size>>2) - 1) << CSR_WCOUNT_SHIFT;
+ csr |= CSR_ONCE;
+ ch->req_transfer_count = (req->size >> 2) - 1;
} else {
- ch->csr &= ~CSR_ONCE;
- ch->ahb_seq |= AHB_SEQ_DBL_BUF;
+ ahb_seq |= AHB_SEQ_DBL_BUF;
/* In double buffered mode, we set the size to half the
* requested size and interrupt when half the buffer
* is full */
- ch->csr &= ~CSR_WCOUNT_MASK;
- ch->csr |= ((req->size>>3) - 1) << CSR_WCOUNT_SHIFT;
+ ch->req_transfer_count = (req->size >> 3) - 1;
}
+ csr |= ch->req_transfer_count << CSR_WCOUNT_SHIFT;
+
if (req->to_memory) {
- ch->csr &= ~CSR_DIR;
- ch->apb_ptr = req->source_addr;
- ch->ahb_ptr = req->dest_addr;
+ apb_ptr = req->source_addr;
+ ahb_ptr = req->dest_addr;
apb_addr_wrap = req->source_wrap;
ahb_addr_wrap = req->dest_wrap;
@@ -439,9 +456,9 @@ static void tegra_dma_update_hw(struct tegra_dma_channel *ch,
ahb_bus_width = req->dest_bus_width;
} else {
- ch->csr |= CSR_DIR;
- ch->apb_ptr = req->dest_addr;
- ch->ahb_ptr = req->source_addr;
+ csr |= CSR_DIR;
+ apb_ptr = req->dest_addr;
+ ahb_ptr = req->source_addr;
apb_addr_wrap = req->dest_wrap;
ahb_addr_wrap = req->source_wrap;
@@ -460,8 +477,7 @@ static void tegra_dma_update_hw(struct tegra_dma_channel *ch,
index++;
} while (index < ARRAY_SIZE(apb_addr_wrap_table));
BUG_ON(index == ARRAY_SIZE(apb_addr_wrap_table));
- ch->apb_seq &= ~APB_SEQ_WRAP_MASK;
- ch->apb_seq |= index << APB_SEQ_WRAP_SHIFT;
+ apb_seq |= index << APB_SEQ_WRAP_SHIFT;
/* set address wrap for AHB size */
index = 0;
@@ -471,55 +487,42 @@ static void tegra_dma_update_hw(struct tegra_dma_channel *ch,
index++;
} while (index < ARRAY_SIZE(ahb_addr_wrap_table));
BUG_ON(index == ARRAY_SIZE(ahb_addr_wrap_table));
- ch->ahb_seq &= ~AHB_SEQ_WRAP_MASK;
- ch->ahb_seq |= index << AHB_SEQ_WRAP_SHIFT;
+ ahb_seq |= index << AHB_SEQ_WRAP_SHIFT;
for (index = 0; index < ARRAY_SIZE(bus_width_table); index++) {
if (bus_width_table[index] == ahb_bus_width)
break;
}
BUG_ON(index == ARRAY_SIZE(bus_width_table));
- ch->ahb_seq &= ~AHB_SEQ_BUS_WIDTH_MASK;
- ch->ahb_seq |= index << AHB_SEQ_BUS_WIDTH_SHIFT;
+ ahb_seq |= index << AHB_SEQ_BUS_WIDTH_SHIFT;
for (index = 0; index < ARRAY_SIZE(bus_width_table); index++) {
if (bus_width_table[index] == apb_bus_width)
break;
}
BUG_ON(index == ARRAY_SIZE(bus_width_table));
- ch->apb_seq &= ~APB_SEQ_BUS_WIDTH_MASK;
- ch->apb_seq |= index << APB_SEQ_BUS_WIDTH_SHIFT;
-
- ch->csr |= CSR_IE_EOC;
+ apb_seq |= index << APB_SEQ_BUS_WIDTH_SHIFT;
- /* update hw registers with the shadow */
- writel(ch->csr, ch->addr + APB_DMA_CHAN_CSR);
- writel(ch->apb_seq, ch->addr + APB_DMA_CHAN_APB_SEQ);
- writel(ch->apb_ptr, ch->addr + APB_DMA_CHAN_APB_PTR);
- writel(ch->ahb_seq, ch->addr + APB_DMA_CHAN_AHB_SEQ);
- writel(ch->ahb_ptr, ch->addr + APB_DMA_CHAN_AHB_PTR);
+ writel(csr, ch->addr + APB_DMA_CHAN_CSR);
+ writel(apb_seq, ch->addr + APB_DMA_CHAN_APB_SEQ);
+ writel(apb_ptr, ch->addr + APB_DMA_CHAN_APB_PTR);
+ writel(ahb_seq, ch->addr + APB_DMA_CHAN_AHB_SEQ);
+ writel(ahb_ptr, ch->addr + APB_DMA_CHAN_AHB_PTR);
- csr = ch->csr | CSR_ENB;
+ csr |= CSR_ENB;
writel(csr, ch->addr + APB_DMA_CHAN_CSR);
req->status = TEGRA_DMA_REQ_INFLIGHT;
}
-static void tegra_dma_init_hw(struct tegra_dma_channel *ch)
-{
- /* One shot with an interrupt to CPU after transfer */
- ch->csr = CSR_ONCE | CSR_IE_EOC;
- ch->ahb_seq = AHB_SEQ_BUS_WIDTH_32 | AHB_SEQ_INTR_ENB;
- ch->apb_seq = APB_SEQ_BUS_WIDTH_32 | 1 << APB_SEQ_WRAP_SHIFT;
-}
-
static void handle_oneshot_dma(struct tegra_dma_channel *ch)
{
struct tegra_dma_req *req;
+ unsigned long irq_flags;
- spin_lock(&ch->lock);
+ spin_lock_irqsave(&ch->lock, irq_flags);
if (list_empty(&ch->list)) {
- spin_unlock(&ch->lock);
+ spin_unlock_irqrestore(&ch->lock, irq_flags);
return;
}
@@ -527,8 +530,7 @@ static void handle_oneshot_dma(struct tegra_dma_channel *ch)
if (req) {
int bytes_transferred;
- bytes_transferred =
- (ch->csr & CSR_WCOUNT_MASK) >> CSR_WCOUNT_SHIFT;
+ bytes_transferred = ch->req_transfer_count;
bytes_transferred += 1;
bytes_transferred <<= 2;
@@ -536,12 +538,12 @@ static void handle_oneshot_dma(struct tegra_dma_channel *ch)
req->bytes_transferred = bytes_transferred;
req->status = TEGRA_DMA_REQ_SUCCESS;
- spin_unlock(&ch->lock);
+ spin_unlock_irqrestore(&ch->lock, irq_flags);
/* Callback should be called without any lock */
pr_debug("%s: transferred %d bytes\n", __func__,
req->bytes_transferred);
req->complete(req);
- spin_lock(&ch->lock);
+ spin_lock_irqsave(&ch->lock, irq_flags);
}
if (!list_empty(&ch->list)) {
@@ -551,22 +553,55 @@ static void handle_oneshot_dma(struct tegra_dma_channel *ch)
if (req->status != TEGRA_DMA_REQ_INFLIGHT)
tegra_dma_update_hw(ch, req);
}
- spin_unlock(&ch->lock);
+ spin_unlock_irqrestore(&ch->lock, irq_flags);
}
static void handle_continuous_dma(struct tegra_dma_channel *ch)
{
struct tegra_dma_req *req;
+ unsigned long irq_flags;
- spin_lock(&ch->lock);
+ spin_lock_irqsave(&ch->lock, irq_flags);
if (list_empty(&ch->list)) {
- spin_unlock(&ch->lock);
+ spin_unlock_irqrestore(&ch->lock, irq_flags);
return;
}
req = list_entry(ch->list.next, typeof(*req), node);
if (req) {
if (req->buffer_status == TEGRA_DMA_REQ_BUF_STATUS_EMPTY) {
+ bool is_dma_ping_complete;
+ is_dma_ping_complete = (readl(ch->addr + APB_DMA_CHAN_STA)
+ & STA_PING_PONG) ? true : false;
+ if (req->to_memory)
+ is_dma_ping_complete = !is_dma_ping_complete;
+ /* Out of sync - Release current buffer */
+ if (!is_dma_ping_complete) {
+ int bytes_transferred;
+
+ bytes_transferred = ch->req_transfer_count;
+ bytes_transferred += 1;
+ bytes_transferred <<= 3;
+ req->buffer_status = TEGRA_DMA_REQ_BUF_STATUS_FULL;
+ req->bytes_transferred = bytes_transferred;
+ req->status = TEGRA_DMA_REQ_SUCCESS;
+ tegra_dma_stop(ch);
+
+ if (!list_is_last(&req->node, &ch->list)) {
+ struct tegra_dma_req *next_req;
+
+ next_req = list_entry(req->node.next,
+ typeof(*next_req), node);
+ tegra_dma_update_hw(ch, next_req);
+ }
+
+ list_del(&req->node);
+
+ /* DMA lock is NOT held when callbak is called */
+ spin_unlock_irqrestore(&ch->lock, irq_flags);
+ req->complete(req);
+ return;
+ }
/* Load the next request into the hardware, if available
* */
if (!list_is_last(&req->node, &ch->list)) {
@@ -579,7 +614,7 @@ static void handle_continuous_dma(struct tegra_dma_channel *ch)
req->buffer_status = TEGRA_DMA_REQ_BUF_STATUS_HALF_FULL;
req->status = TEGRA_DMA_REQ_SUCCESS;
/* DMA lock is NOT held when callback is called */
- spin_unlock(&ch->lock);
+ spin_unlock_irqrestore(&ch->lock, irq_flags);
if (likely(req->threshold))
req->threshold(req);
return;
@@ -590,8 +625,7 @@ static void handle_continuous_dma(struct tegra_dma_channel *ch)
* the second interrupt */
int bytes_transferred;
- bytes_transferred =
- (ch->csr & CSR_WCOUNT_MASK) >> CSR_WCOUNT_SHIFT;
+ bytes_transferred = ch->req_transfer_count;
bytes_transferred += 1;
bytes_transferred <<= 3;
@@ -601,7 +635,7 @@ static void handle_continuous_dma(struct tegra_dma_channel *ch)
list_del(&req->node);
/* DMA lock is NOT held when callbak is called */
- spin_unlock(&ch->lock);
+ spin_unlock_irqrestore(&ch->lock, irq_flags);
req->complete(req);
return;
@@ -609,7 +643,7 @@ static void handle_continuous_dma(struct tegra_dma_channel *ch)
BUG();
}
}
- spin_unlock(&ch->lock);
+ spin_unlock_irqrestore(&ch->lock, irq_flags);
}
static irqreturn_t dma_isr(int irq, void *data)
@@ -646,6 +680,21 @@ int __init tegra_dma_init(void)
int i;
unsigned int irq;
void __iomem *addr;
+ struct clk *c;
+
+ bitmap_fill(channel_usage, NV_DMA_MAX_CHANNELS);
+
+ c = clk_get_sys("tegra-dma", NULL);
+ if (IS_ERR(c)) {
+ pr_err("Unable to get clock for APB DMA\n");
+ ret = PTR_ERR(c);
+ goto fail;
+ }
+ ret = clk_enable(c);
+ if (ret != 0) {
+ pr_err("Unable to enable clock for APB DMA\n");
+ goto fail;
+ }
addr = IO_ADDRESS(TEGRA_APB_DMA_BASE);
writel(GEN_ENABLE, addr + APB_DMA_GEN);
@@ -653,18 +702,9 @@ int __init tegra_dma_init(void)
writel(0xFFFFFFFFul >> (31 - TEGRA_SYSTEM_DMA_CH_MAX),
addr + APB_DMA_IRQ_MASK_SET);
- memset(channel_usage, 0, sizeof(channel_usage));
- memset(dma_channels, 0, sizeof(dma_channels));
-
- /* Reserve all the channels we are not supposed to touch */
- for (i = 0; i < TEGRA_SYSTEM_DMA_CH_MIN; i++)
- __set_bit(i, channel_usage);
-
for (i = TEGRA_SYSTEM_DMA_CH_MIN; i <= TEGRA_SYSTEM_DMA_CH_MAX; i++) {
struct tegra_dma_channel *ch = &dma_channels[i];
- __clear_bit(i, channel_usage);
-
ch->id = i;
snprintf(ch->name, TEGRA_DMA_NAME_SIZE, "dma_channel_%d", i);
@@ -673,7 +713,6 @@ int __init tegra_dma_init(void)
spin_lock_init(&ch->lock);
INIT_LIST_HEAD(&ch->list);
- tegra_dma_init_hw(ch);
irq = INT_APB_DMA_CH0 + i;
ret = request_threaded_irq(irq, dma_isr, dma_thread_fn, 0,
@@ -684,14 +723,15 @@ int __init tegra_dma_init(void)
goto fail;
}
ch->irq = irq;
+
+ __clear_bit(i, channel_usage);
}
/* mark the shared channel allocated */
__set_bit(TEGRA_SYSTEM_DMA_CH_MIN, channel_usage);
- for (i = TEGRA_SYSTEM_DMA_CH_MAX+1; i < NV_DMA_MAX_CHANNELS; i++)
- __set_bit(i, channel_usage);
+ tegra_dma_initialized = true;
- return ret;
+ return 0;
fail:
writel(0, addr + APB_DMA_GEN);
for (i = TEGRA_SYSTEM_DMA_CH_MIN; i <= TEGRA_SYSTEM_DMA_CH_MAX; i++) {
@@ -701,6 +741,7 @@ fail:
}
return ret;
}
+postcore_initcall(tegra_dma_init);
#ifdef CONFIG_PM
static u32 apb_dma[5*TEGRA_SYSTEM_DMA_CH_NR + 3];
Copyright © 1996 – 2023 Jason A. Donenfeld. All Rights Reverse Engineered.