summaryrefslogtreecommitdiff
path: root/drivers/dma
diff options
context:
space:
mode:
authorDan Williams <dan.j.williams@intel.com>2009-09-08 17:55:21 -0700
committerDan Williams <dan.j.williams@intel.com>2009-09-08 17:55:21 -0700
commitbbb20089a3275a19e475dbc21320c3742e3ca423 (patch)
tree216fdc1cbef450ca688135c5b8969169482d9a48 /drivers/dma
parent3e48e656903e9fd8bc805c6a2c4264d7808d315b (diff)
parent657a77fa7284d8ae28dfa48f1dc5d919bf5b2843 (diff)
Merge branch 'dmaengine' into async-tx-next
Conflicts: crypto/async_tx/async_xor.c drivers/dma/ioat/dma_v2.h drivers/dma/ioat/pci.c drivers/md/raid5.c
Diffstat (limited to 'drivers/dma')
-rw-r--r--drivers/dma/Kconfig18
-rw-r--r--drivers/dma/Makefile2
-rw-r--r--drivers/dma/at_hdmac.c1215
-rw-r--r--drivers/dma/at_hdmac_regs.h354
-rw-r--r--drivers/dma/dmaengine.c1
-rw-r--r--drivers/dma/dmatest.c21
-rw-r--r--drivers/dma/dw_dmac.c50
-rw-r--r--drivers/dma/dw_dmac_regs.h1
-rw-r--r--drivers/dma/fsldma.c305
-rw-r--r--drivers/dma/fsldma.h5
-rw-r--r--drivers/dma/ioat/dma.c7
-rw-r--r--drivers/dma/ioat/dma.h3
-rw-r--r--drivers/dma/ioat/dma_v2.c5
-rw-r--r--drivers/dma/ioat/dma_v2.h3
-rw-r--r--drivers/dma/ioat/pci.c16
-rw-r--r--drivers/dma/iop-adma.c9
-rw-r--r--drivers/dma/mv_xor.c9
-rw-r--r--drivers/dma/mv_xor.h4
-rw-r--r--drivers/dma/txx9dmac.c1356
-rw-r--r--drivers/dma/txx9dmac.h308
20 files changed, 3618 insertions, 74 deletions
diff --git a/drivers/dma/Kconfig b/drivers/dma/Kconfig
index ddcd9793b25c..fe1f3717b1ff 100644
--- a/drivers/dma/Kconfig
+++ b/drivers/dma/Kconfig
@@ -50,6 +50,14 @@ config DW_DMAC
Support the Synopsys DesignWare AHB DMA controller. This
can be integrated in chips such as the Atmel AT32ap7000.
+config AT_HDMAC
+ tristate "Atmel AHB DMA support"
+ depends on ARCH_AT91SAM9RL
+ select DMA_ENGINE
+ help
+ Support the Atmel AHB DMA controller. This can be integrated in
+ chips such as the Atmel AT91SAM9RL.
+
config FSL_DMA
tristate "Freescale Elo and Elo Plus DMA support"
depends on FSL_SOC
@@ -85,6 +93,14 @@ config MX3_IPU_IRQS
To avoid bloating the irq_desc[] array we allocate a sufficient
number of IRQ slots and map them dynamically to specific sources.
+config TXX9_DMAC
+ tristate "Toshiba TXx9 SoC DMA support"
+ depends on MACH_TX49XX || MACH_TX39XX
+ select DMA_ENGINE
+ help
+ Support the TXx9 SoC internal DMA controller. This can be
+ integrated in chips such as the Toshiba TX4927/38/39.
+
config DMA_ENGINE
bool
@@ -104,7 +120,7 @@ config NET_DMA
config ASYNC_TX_DMA
bool "Async_tx: Offload support for the async_tx api"
- depends on DMA_ENGINE
+ depends on DMA_ENGINE && !HIGHMEM64G
help
This allows the async_tx api to take advantage of offload engines for
memcpy, memset, xor, and raid6 p+q operations. If your platform has
diff --git a/drivers/dma/Makefile b/drivers/dma/Makefile
index a1cb2857bba6..8f115e93b4a1 100644
--- a/drivers/dma/Makefile
+++ b/drivers/dma/Makefile
@@ -6,4 +6,6 @@ obj-$(CONFIG_INTEL_IOP_ADMA) += iop-adma.o
obj-$(CONFIG_FSL_DMA) += fsldma.o
obj-$(CONFIG_MV_XOR) += mv_xor.o
obj-$(CONFIG_DW_DMAC) += dw_dmac.o
+obj-$(CONFIG_AT_HDMAC) += at_hdmac.o
obj-$(CONFIG_MX3_IPU) += ipu/
+obj-$(CONFIG_TXX9_DMAC) += txx9dmac.o
diff --git a/drivers/dma/at_hdmac.c b/drivers/dma/at_hdmac.c
new file mode 100644
index 000000000000..0aeb578a24e3
--- /dev/null
+++ b/drivers/dma/at_hdmac.c
@@ -0,0 +1,1215 @@
+/*
+ * Driver for the Atmel AHB DMA Controller (aka HDMA or DMAC on AT91 systems)
+ *
+ * Copyright (C) 2008 Atmel Corporation
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ *
+ * This supports the Atmel AHB DMA Controller,
+ *
+ * The driver has currently been tested with the Atmel AT91SAM9RL
+ * and AT91SAM9G45 series.
+ */
+
+#include <linux/clk.h>
+#include <linux/dmaengine.h>
+#include <linux/dma-mapping.h>
+#include <linux/dmapool.h>
+#include <linux/interrupt.h>
+#include <linux/module.h>
+#include <linux/platform_device.h>
+
+#include "at_hdmac_regs.h"
+
+/*
+ * Glossary
+ * --------
+ *
+ * at_hdmac : Name of the ATmel AHB DMA Controller
+ * at_dma_ / atdma : ATmel DMA controller entity related
+ * atc_ / atchan : ATmel DMA Channel entity related
+ */
+
+#define ATC_DEFAULT_CFG (ATC_FIFOCFG_HALFFIFO)
+#define ATC_DEFAULT_CTRLA (0)
+#define ATC_DEFAULT_CTRLB (ATC_SIF(0) \
+ |ATC_DIF(1))
+
+/*
+ * Initial number of descriptors to allocate for each channel. This could
+ * be increased during dma usage.
+ */
+static unsigned int init_nr_desc_per_channel = 64;
+module_param(init_nr_desc_per_channel, uint, 0644);
+MODULE_PARM_DESC(init_nr_desc_per_channel,
+ "initial descriptors per channel (default: 64)");
+
+
+/* prototypes */
+static dma_cookie_t atc_tx_submit(struct dma_async_tx_descriptor *tx);
+
+
+/*----------------------------------------------------------------------*/
+
+static struct at_desc *atc_first_active(struct at_dma_chan *atchan)
+{
+ return list_first_entry(&atchan->active_list,
+ struct at_desc, desc_node);
+}
+
+static struct at_desc *atc_first_queued(struct at_dma_chan *atchan)
+{
+ return list_first_entry(&atchan->queue,
+ struct at_desc, desc_node);
+}
+
+/**
+ * atc_alloc_descriptor - allocate and return an initilized descriptor
+ * @chan: the channel to allocate descriptors for
+ * @gfp_flags: GFP allocation flags
+ *
+ * Note: The ack-bit is positioned in the descriptor flag at creation time
+ * to make initial allocation more convenient. This bit will be cleared
+ * and control will be given to client at usage time (during
+ * preparation functions).
+ */
+static struct at_desc *atc_alloc_descriptor(struct dma_chan *chan,
+ gfp_t gfp_flags)
+{
+ struct at_desc *desc = NULL;
+ struct at_dma *atdma = to_at_dma(chan->device);
+ dma_addr_t phys;
+
+ desc = dma_pool_alloc(atdma->dma_desc_pool, gfp_flags, &phys);
+ if (desc) {
+ memset(desc, 0, sizeof(struct at_desc));
+ INIT_LIST_HEAD(&desc->tx_list);
+ dma_async_tx_descriptor_init(&desc->txd, chan);
+ /* txd.flags will be overwritten in prep functions */
+ desc->txd.flags = DMA_CTRL_ACK;
+ desc->txd.tx_submit = atc_tx_submit;
+ desc->txd.phys = phys;
+ }
+
+ return desc;
+}
+
+/**
+ * atc_desc_get - get a unsused descriptor from free_list
+ * @atchan: channel we want a new descriptor for
+ */
+static struct at_desc *atc_desc_get(struct at_dma_chan *atchan)
+{
+ struct at_desc *desc, *_desc;
+ struct at_desc *ret = NULL;
+ unsigned int i = 0;
+ LIST_HEAD(tmp_list);
+
+ spin_lock_bh(&atchan->lock);
+ list_for_each_entry_safe(desc, _desc, &atchan->free_list, desc_node) {
+ i++;
+ if (async_tx_test_ack(&desc->txd)) {
+ list_del(&desc->desc_node);
+ ret = desc;
+ break;
+ }
+ dev_dbg(chan2dev(&atchan->chan_common),
+ "desc %p not ACKed\n", desc);
+ }
+ spin_unlock_bh(&atchan->lock);
+ dev_vdbg(chan2dev(&atchan->chan_common),
+ "scanned %u descriptors on freelist\n", i);
+
+ /* no more descriptor available in initial pool: create one more */
+ if (!ret) {
+ ret = atc_alloc_descriptor(&atchan->chan_common, GFP_ATOMIC);
+ if (ret) {
+ spin_lock_bh(&atchan->lock);
+ atchan->descs_allocated++;
+ spin_unlock_bh(&atchan->lock);
+ } else {
+ dev_err(chan2dev(&atchan->chan_common),
+ "not enough descriptors available\n");
+ }
+ }
+
+ return ret;
+}
+
+/**
+ * atc_desc_put - move a descriptor, including any children, to the free list
+ * @atchan: channel we work on
+ * @desc: descriptor, at the head of a chain, to move to free list
+ */
+static void atc_desc_put(struct at_dma_chan *atchan, struct at_desc *desc)
+{
+ if (desc) {
+ struct at_desc *child;
+
+ spin_lock_bh(&atchan->lock);
+ list_for_each_entry(child, &desc->tx_list, desc_node)
+ dev_vdbg(chan2dev(&atchan->chan_common),
+ "moving child desc %p to freelist\n",
+ child);
+ list_splice_init(&desc->tx_list, &atchan->free_list);
+ dev_vdbg(chan2dev(&atchan->chan_common),
+ "moving desc %p to freelist\n", desc);
+ list_add(&desc->desc_node, &atchan->free_list);
+ spin_unlock_bh(&atchan->lock);
+ }
+}
+
+/**
+ * atc_assign_cookie - compute and assign new cookie
+ * @atchan: channel we work on
+ * @desc: descriptor to asign cookie for
+ *
+ * Called with atchan->lock held and bh disabled
+ */
+static dma_cookie_t
+atc_assign_cookie(struct at_dma_chan *atchan, struct at_desc *desc)
+{
+ dma_cookie_t cookie = atchan->chan_common.cookie;
+
+ if (++cookie < 0)
+ cookie = 1;
+
+ atchan->chan_common.cookie = cookie;
+ desc->txd.cookie = cookie;
+
+ return cookie;
+}
+
+/**
+ * atc_dostart - starts the DMA engine for real
+ * @atchan: the channel we want to start
+ * @first: first descriptor in the list we want to begin with
+ *
+ * Called with atchan->lock held and bh disabled
+ */
+static void atc_dostart(struct at_dma_chan *atchan, struct at_desc *first)
+{
+ struct at_dma *atdma = to_at_dma(atchan->chan_common.device);
+
+ /* ASSERT: channel is idle */
+ if (atc_chan_is_enabled(atchan)) {
+ dev_err(chan2dev(&atchan->chan_common),
+ "BUG: Attempted to start non-idle channel\n");
+ dev_err(chan2dev(&atchan->chan_common),
+ " channel: s0x%x d0x%x ctrl0x%x:0x%x l0x%x\n",
+ channel_readl(atchan, SADDR),
+ channel_readl(atchan, DADDR),
+ channel_readl(atchan, CTRLA),
+ channel_readl(atchan, CTRLB),
+ channel_readl(atchan, DSCR));
+
+ /* The tasklet will hopefully advance the queue... */
+ return;
+ }
+
+ vdbg_dump_regs(atchan);
+
+ /* clear any pending interrupt */
+ while (dma_readl(atdma, EBCISR))
+ cpu_relax();
+
+ channel_writel(atchan, SADDR, 0);
+ channel_writel(atchan, DADDR, 0);
+ channel_writel(atchan, CTRLA, 0);
+ channel_writel(atchan, CTRLB, 0);
+ channel_writel(atchan, DSCR, first->txd.phys);
+ dma_writel(atdma, CHER, atchan->mask);
+
+ vdbg_dump_regs(atchan);
+}
+
+/**
+ * atc_chain_complete - finish work for one transaction chain
+ * @atchan: channel we work on
+ * @desc: descriptor at the head of the chain we want do complete
+ *
+ * Called with atchan->lock held and bh disabled */
+static void
+atc_chain_complete(struct at_dma_chan *atchan, struct at_desc *desc)
+{
+ dma_async_tx_callback callback;
+ void *param;
+ struct dma_async_tx_descriptor *txd = &desc->txd;
+
+ dev_vdbg(chan2dev(&atchan->chan_common),
+ "descriptor %u complete\n", txd->cookie);
+
+ atchan->completed_cookie = txd->cookie;
+ callback = txd->callback;
+ param = txd->callback_param;
+
+ /* move children to free_list */
+ list_splice_init(&desc->tx_list, &atchan->free_list);
+ /* move myself to free_list */
+ list_move(&desc->desc_node, &atchan->free_list);
+
+ /* unmap dma addresses */
+ if (!atchan->chan_common.private) {
+ struct device *parent = chan2parent(&atchan->chan_common);
+ if (!(txd->flags & DMA_COMPL_SKIP_DEST_UNMAP)) {
+ if (txd->flags & DMA_COMPL_DEST_UNMAP_SINGLE)
+ dma_unmap_single(parent,
+ desc->lli.daddr,
+ desc->len, DMA_FROM_DEVICE);
+ else
+ dma_unmap_page(parent,
+ desc->lli.daddr,
+ desc->len, DMA_FROM_DEVICE);
+ }
+ if (!(txd->flags & DMA_COMPL_SKIP_SRC_UNMAP)) {
+ if (txd->flags & DMA_COMPL_SRC_UNMAP_SINGLE)
+ dma_unmap_single(parent,
+ desc->lli.saddr,
+ desc->len, DMA_TO_DEVICE);
+ else
+ dma_unmap_page(parent,
+ desc->lli.saddr,
+ desc->len, DMA_TO_DEVICE);
+ }
+ }
+
+ /*
+ * The API requires that no submissions are done from a
+ * callback, so we don't need to drop the lock here
+ */
+ if (callback)
+ callback(param);
+
+ dma_run_dependencies(txd);
+}
+
+/**
+ * atc_complete_all - finish work for all transactions
+ * @atchan: channel to complete transactions for
+ *
+ * Eventually submit queued descriptors if any
+ *
+ * Assume channel is idle while calling this function
+ * Called with atchan->lock held and bh disabled
+ */
+static void atc_complete_all(struct at_dma_chan *atchan)
+{
+ struct at_desc *desc, *_desc;
+ LIST_HEAD(list);
+
+ dev_vdbg(chan2dev(&atchan->chan_common), "complete all\n");
+
+ BUG_ON(atc_chan_is_enabled(atchan));
+
+ /*
+ * Submit queued descriptors ASAP, i.e. before we go through
+ * the completed ones.
+ */
+ if (!list_empty(&atchan->queue))
+ atc_dostart(atchan, atc_first_queued(atchan));
+ /* empty active_list now it is completed */
+ list_splice_init(&atchan->active_list, &list);
+ /* empty queue list by moving descriptors (if any) to active_list */
+ list_splice_init(&atchan->queue, &atchan->active_list);
+
+ list_for_each_entry_safe(desc, _desc, &list, desc_node)
+ atc_chain_complete(atchan, desc);
+}
+
+/**
+ * atc_cleanup_descriptors - cleanup up finished descriptors in active_list
+ * @atchan: channel to be cleaned up
+ *
+ * Called with atchan->lock held and bh disabled
+ */
+static void atc_cleanup_descriptors(struct at_dma_chan *atchan)
+{
+ struct at_desc *desc, *_desc;
+ struct at_desc *child;
+
+ dev_vdbg(chan2dev(&atchan->chan_common), "cleanup descriptors\n");
+
+ list_for_each_entry_safe(desc, _desc, &atchan->active_list, desc_node) {
+ if (!(desc->lli.ctrla & ATC_DONE))
+ /* This one is currently in progress */
+ return;
+
+ list_for_each_entry(child, &desc->tx_list, desc_node)
+ if (!(child->lli.ctrla & ATC_DONE))
+ /* Currently in progress */
+ return;
+
+ /*
+ * No descriptors so far seem to be in progress, i.e.
+ * this chain must be done.
+ */
+ atc_chain_complete(atchan, desc);
+ }
+}
+
+/**
+ * atc_advance_work - at the end of a transaction, move forward
+ * @atchan: channel where the transaction ended
+ *
+ * Called with atchan->lock held and bh disabled
+ */
+static void atc_advance_work(struct at_dma_chan *atchan)
+{
+ dev_vdbg(chan2dev(&atchan->chan_common), "advance_work\n");
+
+ if (list_empty(&atchan->active_list) ||
+ list_is_singular(&atchan->active_list)) {
+ atc_complete_all(atchan);
+ } else {
+ atc_chain_complete(atchan, atc_first_active(atchan));
+ /* advance work */
+ atc_dostart(atchan, atc_first_active(atchan));
+ }
+}
+
+
+/**
+ * atc_handle_error - handle errors reported by DMA controller
+ * @atchan: channel where error occurs
+ *
+ * Called with atchan->lock held and bh disabled
+ */
+static void atc_handle_error(struct at_dma_chan *atchan)
+{
+ struct at_desc *bad_desc;
+ struct at_desc *child;
+
+ /*
+ * The descriptor currently at the head of the active list is
+ * broked. Since we don't have any way to report errors, we'll
+ * just have to scream loudly and try to carry on.
+ */
+ bad_desc = atc_first_active(atchan);
+ list_del_init(&bad_desc->desc_node);
+
+ /* As we are stopped, take advantage to push queued descriptors
+ * in active_list */
+ list_splice_init(&atchan->queue, atchan->active_list.prev);
+
+ /* Try to restart the controller */
+ if (!list_empty(&atchan->active_list))
+ atc_dostart(atchan, atc_first_active(atchan));
+
+ /*
+ * KERN_CRITICAL may seem harsh, but since this only happens
+ * when someone submits a bad physical address in a
+ * descriptor, we should consider ourselves lucky that the
+ * controller flagged an error instead of scribbling over
+ * random memory locations.
+ */
+ dev_crit(chan2dev(&atchan->chan_common),
+ "Bad descriptor submitted for DMA!\n");
+ dev_crit(chan2dev(&atchan->chan_common),
+ " cookie: %d\n", bad_desc->txd.cookie);
+ atc_dump_lli(atchan, &bad_desc->lli);
+ list_for_each_entry(child, &bad_desc->tx_list, desc_node)
+ atc_dump_lli(atchan, &child->lli);
+
+ /* Pretend the descriptor completed successfully */
+ atc_chain_complete(atchan, bad_desc);
+}
+
+
+/*-- IRQ & Tasklet ---------------------------------------------------*/
+
+static void atc_tasklet(unsigned long data)
+{
+ struct at_dma_chan *atchan = (struct at_dma_chan *)data;
+
+ /* Channel cannot be enabled here */
+ if (atc_chan_is_enabled(atchan)) {
+ dev_err(chan2dev(&atchan->chan_common),
+ "BUG: channel enabled in tasklet\n");
+ return;
+ }
+
+ spin_lock(&atchan->lock);
+ if (test_and_clear_bit(0, &atchan->error_status))
+ atc_handle_error(atchan);
+ else
+ atc_advance_work(atchan);
+
+ spin_unlock(&atchan->lock);
+}
+
+static irqreturn_t at_dma_interrupt(int irq, void *dev_id)
+{
+ struct at_dma *atdma = (struct at_dma *)dev_id;
+ struct at_dma_chan *atchan;
+ int i;
+ u32 status, pending, imr;
+ int ret = IRQ_NONE;
+
+ do {
+ imr = dma_readl(atdma, EBCIMR);
+ status = dma_readl(atdma, EBCISR);
+ pending = status & imr;
+
+ if (!pending)
+ break;
+
+ dev_vdbg(atdma->dma_common.dev,
+ "interrupt: status = 0x%08x, 0x%08x, 0x%08x\n",
+ status, imr, pending);
+
+ for (i = 0; i < atdma->dma_common.chancnt; i++) {
+ atchan = &atdma->chan[i];
+ if (pending & (AT_DMA_CBTC(i) | AT_DMA_ERR(i))) {
+ if (pending & AT_DMA_ERR(i)) {
+ /* Disable channel on AHB error */
+ dma_writel(atdma, CHDR, atchan->mask);
+ /* Give information to tasklet */
+ set_bit(0, &atchan->error_status);
+ }
+ tasklet_schedule(&atchan->tasklet);
+ ret = IRQ_HANDLED;
+ }
+ }
+
+ } while (pending);
+
+ return ret;
+}
+
+
+/*-- DMA Engine API --------------------------------------------------*/
+
+/**
+ * atc_tx_submit - set the prepared descriptor(s) to be executed by the engine
+ * @desc: descriptor at the head of the transaction chain
+ *
+ * Queue chain if DMA engine is working already
+ *
+ * Cookie increment and adding to active_list or queue must be atomic
+ */
+static dma_cookie_t atc_tx_submit(struct dma_async_tx_descriptor *tx)
+{
+ struct at_desc *desc = txd_to_at_desc(tx);
+ struct at_dma_chan *atchan = to_at_dma_chan(tx->chan);
+ dma_cookie_t cookie;
+
+ spin_lock_bh(&atchan->lock);
+ cookie = atc_assign_cookie(atchan, desc);
+
+ if (list_empty(&atchan->active_list)) {
+ dev_vdbg(chan2dev(tx->chan), "tx_submit: started %u\n",
+ desc->txd.cookie);
+ atc_dostart(atchan, desc);
+ list_add_tail(&desc->desc_node, &atchan->active_list);
+ } else {
+ dev_vdbg(chan2dev(tx->chan), "tx_submit: queued %u\n",
+ desc->txd.cookie);
+ list_add_tail(&desc->desc_node, &atchan->queue);
+ }
+
+ spin_unlock_bh(&atchan->lock);
+
+ return cookie;
+}
+
+/**
+ * atc_prep_dma_memcpy - prepare a memcpy operation
+ * @chan: the channel to prepare operation on
+ * @dest: operation virtual destination address
+ * @src: operation virtual source address
+ * @len: operation length
+ * @flags: tx descriptor status flags
+ */
+static struct dma_async_tx_descriptor *
+atc_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src,
+ size_t len, unsigned long flags)
+{
+ struct at_dma_chan *atchan = to_at_dma_chan(chan);
+ struct at_desc *desc = NULL;
+ struct at_desc *first = NULL;
+ struct at_desc *prev = NULL;
+ size_t xfer_count;
+ size_t offset;
+ unsigned int src_width;
+ unsigned int dst_width;
+ u32 ctrla;
+ u32 ctrlb;
+
+ dev_vdbg(chan2dev(chan), "prep_dma_memcpy: d0x%x s0x%x l0x%zx f0x%lx\n",
+ dest, src, len, flags);
+
+ if (unlikely(!len)) {
+ dev_dbg(chan2dev(chan), "prep_dma_memcpy: length is zero!\n");
+ return NULL;
+ }
+
+ ctrla = ATC_DEFAULT_CTRLA;
+ ctrlb = ATC_DEFAULT_CTRLB
+ | ATC_SRC_ADDR_MODE_INCR
+ | ATC_DST_ADDR_MODE_INCR
+ | ATC_FC_MEM2MEM;
+
+ /*
+ * We can be a lot more clever here, but this should take care
+ * of the most common optimization.
+ */
+ if (!((src | dest | len) & 3)) {
+ ctrla |= ATC_SRC_WIDTH_WORD | ATC_DST_WIDTH_WORD;
+ src_width = dst_width = 2;
+ } else if (!((src | dest | len) & 1)) {
+ ctrla |= ATC_SRC_WIDTH_HALFWORD | ATC_DST_WIDTH_HALFWORD;
+ src_width = dst_width = 1;
+ } else {
+ ctrla |= ATC_SRC_WIDTH_BYTE | ATC_DST_WIDTH_BYTE;
+ src_width = dst_width = 0;
+ }
+
+ for (offset = 0; offset < len; offset += xfer_count << src_width) {
+ xfer_count = min_t(size_t, (len - offset) >> src_width,
+ ATC_BTSIZE_MAX);
+
+ desc = atc_desc_get(atchan);
+ if (!desc)
+ goto err_desc_get;
+
+ desc->lli.saddr = src + offset;
+ desc->lli.daddr = dest + offset;
+ desc->lli.ctrla = ctrla | xfer_count;
+ desc->lli.ctrlb = ctrlb;
+
+ desc->txd.cookie = 0;
+ async_tx_ack(&desc->txd);
+
+ if (!first) {
+ first = desc;
+ } else {
+ /* inform the HW lli about chaining */
+ prev->lli.dscr = desc->txd.phys;
+ /* insert the link descriptor to the LD ring */
+ list_add_tail(&desc->desc_node,
+ &first->tx_list);
+ }
+ prev = desc;
+ }
+
+ /* First descriptor of the chain embedds additional information */
+ first->txd.cookie = -EBUSY;
+ first->len = len;
+
+ /* set end-of-link to the last link descriptor of list*/
+ set_desc_eol(desc);
+
+ desc->txd.flags = flags; /* client is in control of this ack */
+
+ return &first->txd;
+
+err_desc_get:
+ atc_desc_put(atchan, first);
+ return NULL;
+}
+
+
+/**
+ * atc_prep_slave_sg - prepare descriptors for a DMA_SLAVE transaction
+ * @chan: DMA channel
+ * @sgl: scatterlist to transfer to/from
+ * @sg_len: number of entries in @scatterlist
+ * @direction: DMA direction
+ * @flags: tx descriptor status flags
+ */
+static struct dma_async_tx_descriptor *
+atc_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,
+ unsigned int sg_len, enum dma_data_direction direction,
+ unsigned long flags)
+{
+ struct at_dma_chan *atchan = to_at_dma_chan(chan);
+ struct at_dma_slave *atslave = chan->private;
+ struct at_desc *first = NULL;
+ struct at_desc *prev = NULL;
+ u32 ctrla;
+ u32 ctrlb;
+ dma_addr_t reg;
+ unsigned int reg_width;
+ unsigned int mem_width;
+ unsigned int i;
+ struct scatterlist *sg;
+ size_t total_len = 0;
+
+ dev_vdbg(chan2dev(chan), "prep_slave_sg: %s f0x%lx\n",
+ direction == DMA_TO_DEVICE ? "TO DEVICE" : "FROM DEVICE",
+ flags);
+
+ if (unlikely(!atslave || !sg_len)) {
+ dev_dbg(chan2dev(chan), "prep_dma_memcpy: length is zero!\n");
+ return NULL;
+ }
+
+ reg_width = atslave->reg_width;
+
+ ctrla = ATC_DEFAULT_CTRLA | atslave->ctrla;
+ ctrlb = ATC_DEFAULT_CTRLB | ATC_IEN;
+
+ switch (direction) {
+ case DMA_TO_DEVICE:
+ ctrla |= ATC_DST_WIDTH(reg_width);
+ ctrlb |= ATC_DST_ADDR_MODE_FIXED
+ | ATC_SRC_ADDR_MODE_INCR
+ | ATC_FC_MEM2PER;
+ reg = atslave->tx_reg;
+ for_each_sg(sgl, sg, sg_len, i) {
+ struct at_desc *desc;
+ u32 len;
+ u32 mem;
+
+ desc = atc_desc_get(atchan);
+ if (!desc)
+ goto err_desc_get;
+
+ mem = sg_phys(sg);
+ len = sg_dma_len(sg);
+ mem_width = 2;
+ if (unlikely(mem & 3 || len & 3))
+ mem_width = 0;
+
+ desc->lli.saddr = mem;
+ desc->lli.daddr = reg;
+ desc->lli.ctrla = ctrla
+ | ATC_SRC_WIDTH(mem_width)
+ | len >> mem_width;
+ desc->lli.ctrlb = ctrlb;
+
+ if (!first) {
+ first = desc;
+ } else {
+ /* inform the HW lli about chaining */
+ prev->lli.dscr = desc->txd.phys;
+ /* insert the link descriptor to the LD ring */
+ list_add_tail(&desc->desc_node,
+ &first->tx_list);
+ }
+ prev = desc;
+ total_len += len;
+ }
+ break;
+ case DMA_FROM_DEVICE:
+ ctrla |= ATC_SRC_WIDTH(reg_width);
+ ctrlb |= ATC_DST_ADDR_MODE_INCR
+ | ATC_SRC_ADDR_MODE_FIXED
+ | ATC_FC_PER2MEM;
+
+ reg = atslave->rx_reg;
+ for_each_sg(sgl, sg, sg_len, i) {
+ struct at_desc *desc;
+ u32 len;
+ u32 mem;
+
+ desc = atc_desc_get(atchan);
+ if (!desc)
+ goto err_desc_get;
+
+ mem = sg_phys(sg);
+ len = sg_dma_len(sg);
+ mem_width = 2;
+ if (unlikely(mem & 3 || len & 3))
+ mem_width = 0;
+
+ desc->lli.saddr = reg;
+ desc->lli.daddr = mem;
+ desc->lli.ctrla = ctrla
+ | ATC_DST_WIDTH(mem_width)
+ | len >> mem_width;
+ desc->lli.ctrlb = ctrlb;
+
+ if (!first) {
+ first = desc;
+ } else {
+ /* inform the HW lli about chaining */
+ prev->lli.dscr = desc->txd.phys;
+ /* insert the link descriptor to the LD ring */
+ list_add_tail(&desc->desc_node,
+ &first->tx_list);
+ }
+ prev = desc;
+ total_len += len;
+ }
+ break;
+ default:
+ return NULL;
+ }
+
+ /* set end-of-link to the last link descriptor of list*/
+ set_desc_eol(prev);
+
+ /* First descriptor of the chain embedds additional information */
+ first->txd.cookie = -EBUSY;
+ first->len = total_len;
+
+ /* last link descriptor of list is responsible of flags */
+ prev->txd.flags = flags; /* client is in control of this ack */
+
+ return &first->txd;
+
+err_desc_get:
+ dev_err(chan2dev(chan), "not enough descriptors available\n");
+ atc_desc_put(atchan, first);
+ return NULL;
+}
+
+static void atc_terminate_all(struct dma_chan *chan)
+{
+ struct at_dma_chan *atchan = to_at_dma_chan(chan);
+ struct at_dma *atdma = to_at_dma(chan->device);
+ struct at_desc *desc, *_desc;
+ LIST_HEAD(list);
+
+ /*
+ * This is only called when something went wrong elsewhere, so
+ * we don't really care about the data. Just disable the
+ * channel. We still have to poll the channel enable bit due
+ * to AHB/HSB limitations.
+ */
+ spin_lock_bh(&atchan->lock);
+
+ dma_writel(atdma, CHDR, atchan->mask);
+
+ /* confirm that this channel is disabled */
+ while (dma_readl(atdma, CHSR) & atchan->mask)
+ cpu_relax();
+
+ /* active_list entries will end up before queued entries */
+ list_splice_init(&atchan->queue, &list);
+ list_splice_init(&atchan->active_list, &list);
+
+ spin_unlock_bh(&atchan->lock);
+
+ /* Flush all pending and queued descriptors */
+ list_for_each_entry_safe(desc, _desc, &list, desc_node)
+ atc_chain_complete(atchan, desc);
+}
+
+/**
+ * atc_is_tx_complete - poll for transaction completion
+ * @chan: DMA channel
+ * @cookie: transaction identifier to check status of
+ * @done: if not %NULL, updated with last completed transaction
+ * @used: if not %NULL, updated with last used transaction
+ *
+ * If @done and @used are passed in, upon return they reflect the driver
+ * internal state and can be used with dma_async_is_complete() to check
+ * the status of multiple cookies without re-checking hardware state.
+ */
+static enum dma_status
+atc_is_tx_complete(struct dma_chan *chan,
+ dma_cookie_t cookie,
+ dma_cookie_t *done, dma_cookie_t *used)
+{
+ struct at_dma_chan *atchan = to_at_dma_chan(chan);
+ dma_cookie_t last_used;
+ dma_cookie_t last_complete;
+ enum dma_status ret;
+
+ dev_vdbg(chan2dev(chan), "is_tx_complete: %d (d%d, u%d)\n",
+ cookie, done ? *done : 0, used ? *used : 0);
+
+ spin_lock_bh(atchan->lock);
+
+ last_complete = atchan->completed_cookie;
+ last_used = chan->cookie;
+
+ ret = dma_async_is_complete(cookie, last_complete, last_used);
+ if (ret != DMA_SUCCESS) {
+ atc_cleanup_descriptors(atchan);
+
+ last_complete = atchan->completed_cookie;
+ last_used = chan->cookie;
+
+ ret = dma_async_is_complete(cookie, last_complete, last_used);
+ }
+
+ spin_unlock_bh(atchan->lock);
+
+ if (done)
+ *done = last_complete;
+ if (used)
+ *used = last_used;
+
+ return ret;
+}
+
+/**
+ * atc_issue_pending - try to finish work
+ * @chan: target DMA channel
+ */
+static void atc_issue_pending(struct dma_chan *chan)
+{
+ struct at_dma_chan *atchan = to_at_dma_chan(chan);
+
+ dev_vdbg(chan2dev(chan), "issue_pending\n");
+
+ if (!atc_chan_is_enabled(atchan)) {
+ spin_lock_bh(&atchan->lock);
+ atc_advance_work(atchan);
+ spin_unlock_bh(&atchan->lock);
+ }
+}
+
+/**
+ * atc_alloc_chan_resources - allocate resources for DMA channel
+ * @chan: allocate descriptor resources for this channel
+ * @client: current client requesting the channel be ready for requests
+ *
+ * return - the number of allocated descriptors
+ */
+static int atc_alloc_chan_resources(struct dma_chan *chan)
+{
+ struct at_dma_chan *atchan = to_at_dma_chan(chan);
+ struct at_dma *atdma = to_at_dma(chan->device);
+ struct at_desc *desc;
+ struct at_dma_slave *atslave;
+ int i;
+ u32 cfg;
+ LIST_HEAD(tmp_list);
+
+ dev_vdbg(chan2dev(chan), "alloc_chan_resources\n");
+
+ /* ASSERT: channel is idle */
+ if (atc_chan_is_enabled(atchan)) {
+ dev_dbg(chan2dev(chan), "DMA channel not idle ?\n");
+ return -EIO;
+ }
+
+ cfg = ATC_DEFAULT_CFG;
+
+ atslave = chan->private;
+ if (atslave) {
+ /*
+ * We need controller-specific data to set up slave
+ * transfers.
+ */
+ BUG_ON(!atslave->dma_dev || atslave->dma_dev != atdma->dma_common.dev);
+
+ /* if cfg configuration specified take it instad of default */
+ if (atslave->cfg)
+ cfg = atslave->cfg;
+ }
+
+ /* have we already been set up?
+ * reconfigure channel but no need to reallocate descriptors */
+ if (!list_empty(&atchan->free_list))
+ return atchan->descs_allocated;
+
+ /* Allocate initial pool of descriptors */
+ for (i = 0; i < init_nr_desc_per_channel; i++) {
+ desc = atc_alloc_descriptor(chan, GFP_KERNEL);
+ if (!desc) {
+ dev_err(atdma->dma_common.dev,
+ "Only %d initial descriptors\n", i);
+ break;
+ }
+ list_add_tail(&desc->desc_node, &tmp_list);
+ }
+
+ spin_lock_bh(&atchan->lock);
+ atchan->descs_allocated = i;
+ list_splice(&tmp_list, &atchan->free_list);
+ atchan->completed_cookie = chan->cookie = 1;
+ spin_unlock_bh(&atchan->lock);
+
+ /* channel parameters */
+ channel_writel(atchan, CFG, cfg);
+
+ dev_dbg(chan2dev(chan),
+ "alloc_chan_resources: allocated %d descriptors\n",
+ atchan->descs_allocated);
+
+ return atchan->descs_allocated;
+}
+
+/**
+ * atc_free_chan_resources - free all channel resources
+ * @chan: DMA channel
+ */
+static void atc_free_chan_resources(struct dma_chan *chan)
+{
+ struct at_dma_chan *atchan = to_at_dma_chan(chan);
+ struct at_dma *atdma = to_at_dma(chan->device);
+ struct at_desc *desc, *_desc;
+ LIST_HEAD(list);
+
+ dev_dbg(chan2dev(chan), "free_chan_resources: (descs allocated=%u)\n",
+ atchan->descs_allocated);
+
+ /* ASSERT: channel is idle */
+ BUG_ON(!list_empty(&atchan->active_list));
+ BUG_ON(!list_empty(&atchan->queue));
+ BUG_ON(atc_chan_is_enabled(atchan));
+
+ list_for_each_entry_safe(desc, _desc, &atchan->free_list, desc_node) {
+ dev_vdbg(chan2dev(chan), " freeing descriptor %p\n", desc);
+ list_del(&desc->desc_node);
+ /* free link descriptor */
+ dma_pool_free(atdma->dma_desc_pool, desc, desc->txd.phys);
+ }
+ list_splice_init(&atchan->free_list, &list);
+ atchan->descs_allocated = 0;
+
+ dev_vdbg(chan2dev(chan), "free_chan_resources: done\n");
+}
+
+
+/*-- Module Management -----------------------------------------------*/
+
+/**
+ * at_dma_off - disable DMA controller
+ * @atdma: the Atmel HDAMC device
+ */
+static void at_dma_off(struct at_dma *atdma)
+{
+ dma_writel(atdma, EN, 0);
+
+ /* disable all interrupts */
+ dma_writel(atdma, EBCIDR, -1L);
+
+ /* confirm that all channels are disabled */
+ while (dma_readl(atdma, CHSR) & atdma->all_chan_mask)
+ cpu_relax();
+}
+
+static int __init at_dma_probe(struct platform_device *pdev)
+{
+ struct at_dma_platform_data *pdata;
+ struct resource *io;
+ struct at_dma *atdma;
+ size_t size;
+ int irq;
+ int err;
+ int i;
+
+ /* get DMA Controller parameters from platform */
+ pdata = pdev->dev.platform_data;
+ if (!pdata || pdata->nr_channels > AT_DMA_MAX_NR_CHANNELS)
+ return -EINVAL;
+
+ io = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+ if (!io)
+ return -EINVAL;
+
+ irq = platform_get_irq(pdev, 0);
+ if (irq < 0)
+ return irq;
+
+ size = sizeof(struct at_dma);
+ size += pdata->nr_channels * sizeof(struct at_dma_chan);
+ atdma = kzalloc(size, GFP_KERNEL);
+ if (!atdma)
+ return -ENOMEM;
+
+ /* discover transaction capabilites from the platform data */
+ atdma->dma_common.cap_mask = pdata->cap_mask;
+ atdma->all_chan_mask = (1 << pdata->nr_channels) - 1;
+
+ size = io->end - io->start + 1;
+ if (!request_mem_region(io->start, size, pdev->dev.driver->name)) {
+ err = -EBUSY;
+ goto err_kfree;
+ }
+
+ atdma->regs = ioremap(io->start, size);
+ if (!atdma->regs) {
+ err = -ENOMEM;
+ goto err_release_r;
+ }
+
+ atdma->clk = clk_get(&pdev->dev, "dma_clk");
+ if (IS_ERR(atdma->clk)) {
+ err = PTR_ERR(atdma->clk);
+ goto err_clk;
+ }
+ clk_enable(atdma->clk);
+
+ /* force dma off, just in case */
+ at_dma_off(atdma);
+
+ err = request_irq(irq, at_dma_interrupt, 0, "at_hdmac", atdma);
+ if (err)
+ goto err_irq;
+
+ platform_set_drvdata(pdev, atdma);
+
+ /* create a pool of consistent memory blocks for hardware descriptors */
+ atdma->dma_desc_pool = dma_pool_create("at_hdmac_desc_pool",
+ &pdev->dev, sizeof(struct at_desc),
+ 4 /* word alignment */, 0);
+ if (!atdma->dma_desc_pool) {
+ dev_err(&pdev->dev, "No memory for descriptors dma pool\n");
+ err = -ENOMEM;
+ goto err_pool_create;
+ }
+
+ /* clear any pending interrupt */
+ while (dma_readl(atdma, EBCISR))
+ cpu_relax();
+
+ /* initialize channels related values */
+ INIT_LIST_HEAD(&atdma->dma_common.channels);
+ for (i = 0; i < pdata->nr_channels; i++, atdma->dma_common.chancnt++) {
+ struct at_dma_chan *atchan = &atdma->chan[i];
+
+ atchan->chan_common.device = &atdma->dma_common;
+ atchan->chan_common.cookie = atchan->completed_cookie = 1;
+ atchan->chan_common.chan_id = i;
+ list_add_tail(&atchan->chan_common.device_node,
+ &atdma->dma_common.channels);
+
+ atchan->ch_regs = atdma->regs + ch_regs(i);
+ spin_lock_init(&atchan->lock);
+ atchan->mask = 1 << i;
+
+ INIT_LIST_HEAD(&atchan->active_list);
+ INIT_LIST_HEAD(&atchan->queue);
+ INIT_LIST_HEAD(&atchan->free_list);
+
+ tasklet_init(&atchan->tasklet, atc_tasklet,
+ (unsigned long)atchan);
+ atc_enable_irq(atchan);
+ }
+
+ /* set base routines */
+ atdma->dma_common.device_alloc_chan_resources = atc_alloc_chan_resources;
+ atdma->dma_common.device_free_chan_resources = atc_free_chan_resources;
+ atdma->dma_common.device_is_tx_complete = atc_is_tx_complete;
+ atdma->dma_common.device_issue_pending = atc_issue_pending;
+ atdma->dma_common.dev = &pdev->dev;
+
+ /* set prep routines based on capability */
+ if (dma_has_cap(DMA_MEMCPY, atdma->dma_common.cap_mask))
+ atdma->dma_common.device_prep_dma_memcpy = atc_prep_dma_memcpy;
+
+ if (dma_has_cap(DMA_SLAVE, atdma->dma_common.cap_mask)) {
+ atdma->dma_common.device_prep_slave_sg = atc_prep_slave_sg;
+ atdma->dma_common.device_terminate_all = atc_terminate_all;
+ }
+
+ dma_writel(atdma, EN, AT_DMA_ENABLE);
+
+ dev_info(&pdev->dev, "Atmel AHB DMA Controller ( %s%s), %d channels\n",
+ dma_has_cap(DMA_MEMCPY, atdma->dma_common.cap_mask) ? "cpy " : "",
+ dma_has_cap(DMA_SLAVE, atdma->dma_common.cap_mask) ? "slave " : "",
+ atdma->dma_common.chancnt);
+
+ dma_async_device_register(&atdma->dma_common);
+
+ return 0;
+
+err_pool_create:
+ platform_set_drvdata(pdev, NULL);
+ free_irq(platform_get_irq(pdev, 0), atdma);
+err_irq:
+ clk_disable(atdma->clk);
+ clk_put(atdma->clk);
+err_clk:
+ iounmap(atdma->regs);
+ atdma->regs = NULL;
+err_release_r:
+ release_mem_region(io->start, size);
+err_kfree:
+ kfree(atdma);
+ return err;
+}
+
+static int __exit at_dma_remove(struct platform_device *pdev)
+{
+ struct at_dma *atdma = platform_get_drvdata(pdev);
+ struct dma_chan *chan, *_chan;
+ struct resource *io;
+
+ at_dma_off(atdma);
+ dma_async_device_unregister(&atdma->dma_common);
+
+ dma_pool_destroy(atdma->dma_desc_pool);
+ platform_set_drvdata(pdev, NULL);
+ free_irq(platform_get_irq(pdev, 0), atdma);
+
+ list_for_each_entry_safe(chan, _chan, &atdma->dma_common.channels,
+ device_node) {
+ struct at_dma_chan *atchan = to_at_dma_chan(chan);
+
+ /* Disable interrupts */
+ atc_disable_irq(atchan);
+ tasklet_disable(&atchan->tasklet);
+
+ tasklet_kill(&atchan->tasklet);
+ list_del(&chan->device_node);
+ }
+
+ clk_disable(atdma->clk);
+ clk_put(atdma->clk);
+
+ iounmap(atdma->regs);
+ atdma->regs = NULL;
+
+ io = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+ release_mem_region(io->start, io->end - io->start + 1);
+
+ kfree(atdma);
+
+ return 0;
+}
+
+static void at_dma_shutdown(struct platform_device *pdev)
+{
+ struct at_dma *atdma = platform_get_drvdata(pdev);
+
+ at_dma_off(platform_get_drvdata(pdev));
+ clk_disable(atdma->clk);
+}
+
+static int at_dma_suspend_late(struct platform_device *pdev, pm_message_t mesg)
+{
+ struct at_dma *atdma = platform_get_drvdata(pdev);
+
+ at_dma_off(platform_get_drvdata(pdev));
+ clk_disable(atdma->clk);
+ return 0;
+}
+
+static int at_dma_resume_early(struct platform_device *pdev)
+{
+ struct at_dma *atdma = platform_get_drvdata(pdev);
+
+ clk_enable(atdma->clk);
+ dma_writel(atdma, EN, AT_DMA_ENABLE);
+ return 0;
+
+}
+
+static struct platform_driver at_dma_driver = {
+ .remove = __exit_p(at_dma_remove),
+ .shutdown = at_dma_shutdown,
+ .suspend_late = at_dma_suspend_late,
+ .resume_early = at_dma_resume_early,
+ .driver = {
+ .name = "at_hdmac",
+ },
+};
+
+static int __init at_dma_init(void)
+{
+ return platform_driver_probe(&at_dma_driver, at_dma_probe);
+}
+module_init(at_dma_init);
+
+static void __exit at_dma_exit(void)
+{
+ platform_driver_unregister(&at_dma_driver);
+}
+module_exit(at_dma_exit);
+
+MODULE_DESCRIPTION("Atmel AHB DMA Controller driver");
+MODULE_AUTHOR("Nicolas Ferre <nicolas.ferre@atmel.com>");
+MODULE_LICENSE("GPL");
+MODULE_ALIAS("platform:at_hdmac");
diff --git a/drivers/dma/at_hdmac_regs.h b/drivers/dma/at_hdmac_regs.h
new file mode 100644
index 000000000000..495457e3dc4b
--- /dev/null
+++ b/drivers/dma/at_hdmac_regs.h
@@ -0,0 +1,354 @@
+/*
+ * Header file for the Atmel AHB DMA Controller driver
+ *
+ * Copyright (C) 2008 Atmel Corporation
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ */
+#ifndef AT_HDMAC_REGS_H
+#define AT_HDMAC_REGS_H
+
+#include <mach/at_hdmac.h>
+
+#define AT_DMA_MAX_NR_CHANNELS 8
+
+
+#define AT_DMA_GCFG 0x00 /* Global Configuration Register */
+#define AT_DMA_IF_BIGEND(i) (0x1 << (i)) /* AHB-Lite Interface i in Big-endian mode */
+#define AT_DMA_ARB_CFG (0x1 << 4) /* Arbiter mode. */
+#define AT_DMA_ARB_CFG_FIXED (0x0 << 4)
+#define AT_DMA_ARB_CFG_ROUND_ROBIN (0x1 << 4)
+
+#define AT_DMA_EN 0x04 /* Controller Enable Register */
+#define AT_DMA_ENABLE (0x1 << 0)
+
+#define AT_DMA_SREQ 0x08 /* Software Single Request Register */
+#define AT_DMA_SSREQ(x) (0x1 << ((x) << 1)) /* Request a source single transfer on channel x */
+#define AT_DMA_DSREQ(x) (0x1 << (1 + ((x) << 1))) /* Request a destination single transfer on channel x */
+
+#define AT_DMA_CREQ 0x0C /* Software Chunk Transfer Request Register */
+#define AT_DMA_SCREQ(x) (0x1 << ((x) << 1)) /* Request a source chunk transfer on channel x */
+#define AT_DMA_DCREQ(x) (0x1 << (1 + ((x) << 1))) /* Request a destination chunk transfer on channel x */
+
+#define AT_DMA_LAST 0x10 /* Software Last Transfer Flag Register */
+#define AT_DMA_SLAST(x) (0x1 << ((x) << 1)) /* This src rq is last tx of buffer on channel x */
+#define AT_DMA_DLAST(x) (0x1 << (1 + ((x) << 1))) /* This dst rq is last tx of buffer on channel x */
+
+#define AT_DMA_SYNC 0x14 /* Request Synchronization Register */
+#define AT_DMA_SYR(h) (0x1 << (h)) /* Synchronize handshake line h */
+
+/* Error, Chained Buffer transfer completed and Buffer transfer completed Interrupt registers */
+#define AT_DMA_EBCIER 0x18 /* Enable register */
+#define AT_DMA_EBCIDR 0x1C /* Disable register */
+#define AT_DMA_EBCIMR 0x20 /* Mask Register */
+#define AT_DMA_EBCISR 0x24 /* Status Register */
+#define AT_DMA_CBTC_OFFSET 8
+#define AT_DMA_ERR_OFFSET 16
+#define AT_DMA_BTC(x) (0x1 << (x))
+#define AT_DMA_CBTC(x) (0x1 << (AT_DMA_CBTC_OFFSET + (x)))
+#define AT_DMA_ERR(x) (0x1 << (AT_DMA_ERR_OFFSET + (x)))
+
+#define AT_DMA_CHER 0x28 /* Channel Handler Enable Register */
+#define AT_DMA_ENA(x) (0x1 << (x))
+#define AT_DMA_SUSP(x) (0x1 << ( 8 + (x)))
+#define AT_DMA_KEEP(x) (0x1 << (24 + (x)))
+
+#define AT_DMA_CHDR 0x2C /* Channel Handler Disable Register */
+#define AT_DMA_DIS(x) (0x1 << (x))
+#define AT_DMA_RES(x) (0x1 << ( 8 + (x)))
+
+#define AT_DMA_CHSR 0x30 /* Channel Handler Status Register */
+#define AT_DMA_EMPT(x) (0x1 << (16 + (x)))
+#define AT_DMA_STAL(x) (0x1 << (24 + (x)))
+
+
+#define AT_DMA_CH_REGS_BASE 0x3C /* Channel registers base address */
+#define ch_regs(x) (AT_DMA_CH_REGS_BASE + (x) * 0x28) /* Channel x base addr */
+
+/* Hardware register offset for each channel */
+#define ATC_SADDR_OFFSET 0x00 /* Source Address Register */
+#define ATC_DADDR_OFFSET 0x04 /* Destination Address Register */
+#define ATC_DSCR_OFFSET 0x08 /* Descriptor Address Register */
+#define ATC_CTRLA_OFFSET 0x0C /* Control A Register */
+#define ATC_CTRLB_OFFSET 0x10 /* Control B Register */
+#define ATC_CFG_OFFSET 0x14 /* Configuration Register */
+#define ATC_SPIP_OFFSET 0x18 /* Src PIP Configuration Register */
+#define ATC_DPIP_OFFSET 0x1C /* Dst PIP Configuration Register */
+
+
+/* Bitfield definitions */
+
+/* Bitfields in DSCR */
+#define ATC_DSCR_IF(i) (0x3 & (i)) /* Dsc feched via AHB-Lite Interface i */
+
+/* Bitfields in CTRLA */
+#define ATC_BTSIZE_MAX 0xFFFFUL /* Maximum Buffer Transfer Size */
+#define ATC_BTSIZE(x) (ATC_BTSIZE_MAX & (x)) /* Buffer Transfer Size */
+/* Chunck Tranfer size definitions are in at_hdmac.h */
+#define ATC_SRC_WIDTH_MASK (0x3 << 24) /* Source Single Transfer Size */
+#define ATC_SRC_WIDTH(x) ((x) << 24)
+#define ATC_SRC_WIDTH_BYTE (0x0 << 24)
+#define ATC_SRC_WIDTH_HALFWORD (0x1 << 24)
+#define ATC_SRC_WIDTH_WORD (0x2 << 24)
+#define ATC_DST_WIDTH_MASK (0x3 << 28) /* Destination Single Transfer Size */
+#define ATC_DST_WIDTH(x) ((x) << 28)
+#define ATC_DST_WIDTH_BYTE (0x0 << 28)
+#define ATC_DST_WIDTH_HALFWORD (0x1 << 28)
+#define ATC_DST_WIDTH_WORD (0x2 << 28)
+#define ATC_DONE (0x1 << 31) /* Tx Done (only written back in descriptor) */
+
+/* Bitfields in CTRLB */
+#define ATC_SIF(i) (0x3 & (i)) /* Src tx done via AHB-Lite Interface i */
+#define ATC_DIF(i) ((0x3 & (i)) << 4) /* Dst tx done via AHB-Lite Interface i */
+#define ATC_SRC_PIP (0x1 << 8) /* Source Picture-in-Picture enabled */
+#define ATC_DST_PIP (0x1 << 12) /* Destination Picture-in-Picture enabled */
+#define ATC_SRC_DSCR_DIS (0x1 << 16) /* Src Descriptor fetch disable */
+#define ATC_DST_DSCR_DIS (0x1 << 20) /* Dst Descriptor fetch disable */
+#define ATC_FC_MASK (0x7 << 21) /* Choose Flow Controller */
+#define ATC_FC_MEM2MEM (0x0 << 21) /* Mem-to-Mem (DMA) */
+#define ATC_FC_MEM2PER (0x1 << 21) /* Mem-to-Periph (DMA) */
+#define ATC_FC_PER2MEM (0x2 << 21) /* Periph-to-Mem (DMA) */
+#define ATC_FC_PER2PER (0x3 << 21) /* Periph-to-Periph (DMA) */
+#define ATC_FC_PER2MEM_PER (0x4 << 21) /* Periph-to-Mem (Peripheral) */
+#define ATC_FC_MEM2PER_PER (0x5 << 21) /* Mem-to-Periph (Peripheral) */
+#define ATC_FC_PER2PER_SRCPER (0x6 << 21) /* Periph-to-Periph (Src Peripheral) */
+#define ATC_FC_PER2PER_DSTPER (0x7 << 21) /* Periph-to-Periph (Dst Peripheral) */
+#define ATC_SRC_ADDR_MODE_MASK (0x3 << 24)
+#define ATC_SRC_ADDR_MODE_INCR (0x0 << 24) /* Incrementing Mode */
+#define ATC_SRC_ADDR_MODE_DECR (0x1 << 24) /* Decrementing Mode */
+#define ATC_SRC_ADDR_MODE_FIXED (0x2 << 24) /* Fixed Mode */
+#define ATC_DST_ADDR_MODE_MASK (0x3 << 28)
+#define ATC_DST_ADDR_MODE_INCR (0x0 << 28) /* Incrementing Mode */
+#define ATC_DST_ADDR_MODE_DECR (0x1 << 28) /* Decrementing Mode */
+#define ATC_DST_ADDR_MODE_FIXED (0x2 << 28) /* Fixed Mode */
+#define ATC_IEN (0x1 << 30) /* BTC interrupt enable (active low) */
+#define ATC_AUTO (0x1 << 31) /* Auto multiple buffer tx enable */
+
+/* Bitfields in CFG */
+/* are in at_hdmac.h */
+
+/* Bitfields in SPIP */
+#define ATC_SPIP_HOLE(x) (0xFFFFU & (x))
+#define ATC_SPIP_BOUNDARY(x) ((0x3FF & (x)) << 16)
+
+/* Bitfields in DPIP */
+#define ATC_DPIP_HOLE(x) (0xFFFFU & (x))
+#define ATC_DPIP_BOUNDARY(x) ((0x3FF & (x)) << 16)
+
+
+/*-- descriptors -----------------------------------------------------*/
+
+/* LLI == Linked List Item; aka DMA buffer descriptor */
+struct at_lli {
+ /* values that are not changed by hardware */
+ dma_addr_t saddr;
+ dma_addr_t daddr;
+ /* value that may get written back: */
+ u32 ctrla;
+ /* more values that are not changed by hardware */
+ u32 ctrlb;
+ dma_addr_t dscr; /* chain to next lli */
+};
+
+/**
+ * struct at_desc - software descriptor
+ * @at_lli: hardware lli structure
+ * @txd: support for the async_tx api
+ * @desc_node: node on the channed descriptors list
+ * @len: total transaction bytecount
+ */
+struct at_desc {
+ /* FIRST values the hardware uses */
+ struct at_lli lli;
+
+ /* THEN values for driver housekeeping */
+ struct list_head tx_list;
+ struct dma_async_tx_descriptor txd;
+ struct list_head desc_node;
+ size_t len;
+};
+
+static inline struct at_desc *
+txd_to_at_desc(struct dma_async_tx_descriptor *txd)
+{
+ return container_of(txd, struct at_desc, txd);
+}
+
+
+/*-- Channels --------------------------------------------------------*/
+
+/**
+ * struct at_dma_chan - internal representation of an Atmel HDMAC channel
+ * @chan_common: common dmaengine channel object members
+ * @device: parent device
+ * @ch_regs: memory mapped register base
+ * @mask: channel index in a mask
+ * @error_status: transmit error status information from irq handler
+ * to tasklet (use atomic operations)
+ * @tasklet: bottom half to finish transaction work
+ * @lock: serializes enqueue/dequeue operations to descriptors lists
+ * @completed_cookie: identifier for the most recently completed operation
+ * @active_list: list of descriptors dmaengine is being running on
+ * @queue: list of descriptors ready to be submitted to engine
+ * @free_list: list of descriptors usable by the channel
+ * @descs_allocated: records the actual size of the descriptor pool
+ */
+struct at_dma_chan {
+ struct dma_chan chan_common;
+ struct at_dma *device;
+ void __iomem *ch_regs;
+ u8 mask;
+ unsigned long error_status;
+ struct tasklet_struct tasklet;
+
+ spinlock_t lock;
+
+ /* these other elements are all protected by lock */
+ dma_cookie_t completed_cookie;
+ struct list_head active_list;
+ struct list_head queue;
+ struct list_head free_list;
+ unsigned int descs_allocated;
+};
+
+#define channel_readl(atchan, name) \
+ __raw_readl((atchan)->ch_regs + ATC_##name##_OFFSET)
+
+#define channel_writel(atchan, name, val) \
+ __raw_writel((val), (atchan)->ch_regs + ATC_##name##_OFFSET)
+
+static inline struct at_dma_chan *to_at_dma_chan(struct dma_chan *dchan)
+{
+ return container_of(dchan, struct at_dma_chan, chan_common);
+}
+
+
+/*-- Controller ------------------------------------------------------*/
+
+/**
+ * struct at_dma - internal representation of an Atmel HDMA Controller
+ * @chan_common: common dmaengine dma_device object members
+ * @ch_regs: memory mapped register base
+ * @clk: dma controller clock
+ * @all_chan_mask: all channels availlable in a mask
+ * @dma_desc_pool: base of DMA descriptor region (DMA address)
+ * @chan: channels table to store at_dma_chan structures
+ */
+struct at_dma {
+ struct dma_device dma_common;
+ void __iomem *regs;
+ struct clk *clk;
+
+ u8 all_chan_mask;
+
+ struct dma_pool *dma_desc_pool;
+ /* AT THE END channels table */
+ struct at_dma_chan chan[0];
+};
+
+#define dma_readl(atdma, name) \
+ __raw_readl((atdma)->regs + AT_DMA_##name)
+#define dma_writel(atdma, name, val) \
+ __raw_writel((val), (atdma)->regs + AT_DMA_##name)
+
+static inline struct at_dma *to_at_dma(struct dma_device *ddev)
+{
+ return container_of(ddev, struct at_dma, dma_common);
+}
+
+
+/*-- Helper functions ------------------------------------------------*/
+
+static struct device *chan2dev(struct dma_chan *chan)
+{
+ return &chan->dev->device;
+}
+static struct device *chan2parent(struct dma_chan *chan)
+{
+ return chan->dev->device.parent;
+}
+
+#if defined(VERBOSE_DEBUG)
+static void vdbg_dump_regs(struct at_dma_chan *atchan)
+{
+ struct at_dma *atdma = to_at_dma(atchan->chan_common.device);
+
+ dev_err(chan2dev(&atchan->chan_common),
+ " channel %d : imr = 0x%x, chsr = 0x%x\n",
+ atchan->chan_common.chan_id,
+ dma_readl(atdma, EBCIMR),
+ dma_readl(atdma, CHSR));
+
+ dev_err(chan2dev(&atchan->chan_common),
+ " channel: s0x%x d0x%x ctrl0x%x:0x%x cfg0x%x l0x%x\n",
+ channel_readl(atchan, SADDR),
+ channel_readl(atchan, DADDR),
+ channel_readl(atchan, CTRLA),
+ channel_readl(atchan, CTRLB),
+ channel_readl(atchan, CFG),
+ channel_readl(atchan, DSCR));
+}
+#else
+static void vdbg_dump_regs(struct at_dma_chan *atchan) {}
+#endif
+
+static void atc_dump_lli(struct at_dma_chan *atchan, struct at_lli *lli)
+{
+ dev_printk(KERN_CRIT, chan2dev(&atchan->chan_common),
+ " desc: s0x%x d0x%x ctrl0x%x:0x%x l0x%x\n",
+ lli->saddr, lli->daddr,
+ lli->ctrla, lli->ctrlb, lli->dscr);
+}
+
+
+static void atc_setup_irq(struct at_dma_chan *atchan, int on)
+{
+ struct at_dma *atdma = to_at_dma(atchan->chan_common.device);
+ u32 ebci;
+
+ /* enable interrupts on buffer chain completion & error */
+ ebci = AT_DMA_CBTC(atchan->chan_common.chan_id)
+ | AT_DMA_ERR(atchan->chan_common.chan_id);
+ if (on)
+ dma_writel(atdma, EBCIER, ebci);
+ else
+ dma_writel(atdma, EBCIDR, ebci);
+}
+
+static inline void atc_enable_irq(struct at_dma_chan *atchan)
+{
+ atc_setup_irq(atchan, 1);
+}
+
+static inline void atc_disable_irq(struct at_dma_chan *atchan)
+{
+ atc_setup_irq(atchan, 0);
+}
+
+
+/**
+ * atc_chan_is_enabled - test if given channel is enabled
+ * @atchan: channel we want to test status
+ */
+static inline int atc_chan_is_enabled(struct at_dma_chan *atchan)
+{
+ struct at_dma *atdma = to_at_dma(atchan->chan_common.device);
+
+ return !!(dma_readl(atdma, CHSR) & atchan->mask);
+}
+
+
+/**
+ * set_desc_eol - set end-of-link to descriptor so it will end transfer
+ * @desc: descriptor, signle or at the end of a chain, to end chain on
+ */
+static void set_desc_eol(struct at_desc *desc)
+{
+ desc->lli.ctrlb |= ATC_SRC_DSCR_DIS | ATC_DST_DSCR_DIS;
+ desc->lli.dscr = 0;
+}
+
+#endif /* AT_HDMAC_REGS_H */
diff --git a/drivers/dma/dmaengine.c b/drivers/dma/dmaengine.c
index d5bc628d207c..bd0b248de2cf 100644
--- a/drivers/dma/dmaengine.c
+++ b/drivers/dma/dmaengine.c
@@ -977,7 +977,6 @@ void dma_async_tx_descriptor_init(struct dma_async_tx_descriptor *tx,
{
tx->chan = chan;
spin_lock_init(&tx->lock);
- INIT_LIST_HEAD(&tx->tx_list);
}
EXPORT_SYMBOL(dma_async_tx_descriptor_init);
diff --git a/drivers/dma/dmatest.c b/drivers/dma/dmatest.c
index a3722a7384b5..a32a4cf7b1e0 100644
--- a/drivers/dma/dmatest.c
+++ b/drivers/dma/dmatest.c
@@ -38,6 +38,11 @@ module_param(max_channels, uint, S_IRUGO);
MODULE_PARM_DESC(max_channels,
"Maximum number of channels to use (default: all)");
+static unsigned int iterations;
+module_param(iterations, uint, S_IRUGO);
+MODULE_PARM_DESC(iterations,
+ "Iterations before stopping test (default: infinite)");
+
static unsigned int xor_sources = 3;
module_param(xor_sources, uint, S_IRUGO);
MODULE_PARM_DESC(xor_sources,
@@ -119,7 +124,7 @@ static void dmatest_init_srcs(u8 **bufs, unsigned int start, unsigned int len)
buf[i] = PATTERN_SRC | (~i & PATTERN_COUNT_MASK);
for ( ; i < start + len; i++)
buf[i] = PATTERN_SRC | PATTERN_COPY
- | (~i & PATTERN_COUNT_MASK);;
+ | (~i & PATTERN_COUNT_MASK);
for ( ; i < test_buf_size; i++)
buf[i] = PATTERN_SRC | (~i & PATTERN_COUNT_MASK);
buf++;
@@ -281,7 +286,8 @@ static int dmatest_func(void *data)
flags = DMA_CTRL_ACK | DMA_COMPL_SKIP_DEST_UNMAP | DMA_PREP_INTERRUPT;
- while (!kthread_should_stop()) {
+ while (!kthread_should_stop()
+ && !(iterations && total_tests >= iterations)) {
struct dma_device *dev = chan->device;
struct dma_async_tx_descriptor *tx = NULL;
dma_addr_t dma_srcs[src_cnt];
@@ -450,6 +456,13 @@ err_srcbuf:
err_srcs:
pr_notice("%s: terminating after %u tests, %u failures (status %d)\n",
thread_name, total_tests, failed_tests, ret);
+
+ if (iterations > 0)
+ while (!kthread_should_stop()) {
+ DECLARE_WAIT_QUEUE_HEAD(wait_dmatest_exit);
+ interruptible_sleep_on(&wait_dmatest_exit);
+ }
+
return ret;
}
@@ -531,11 +544,11 @@ static int dmatest_add_channel(struct dma_chan *chan)
if (dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask)) {
cnt = dmatest_add_threads(dtc, DMA_MEMCPY);
- thread_count += cnt > 0 ?: 0;
+ thread_count += cnt > 0 ? cnt : 0;
}
if (dma_has_cap(DMA_XOR, dma_dev->cap_mask)) {
cnt = dmatest_add_threads(dtc, DMA_XOR);
- thread_count += cnt > 0 ?: 0;
+ thread_count += cnt > 0 ? cnt : 0;
}
if (dma_has_cap(DMA_PQ, dma_dev->cap_mask)) {
cnt = dmatest_add_threads(dtc, DMA_PQ);
diff --git a/drivers/dma/dw_dmac.c b/drivers/dma/dw_dmac.c
index 98c9a847bf51..8fb748280361 100644
--- a/drivers/dma/dw_dmac.c
+++ b/drivers/dma/dw_dmac.c
@@ -116,7 +116,7 @@ static void dwc_sync_desc_for_cpu(struct dw_dma_chan *dwc, struct dw_desc *desc)
{
struct dw_desc *child;
- list_for_each_entry(child, &desc->txd.tx_list, desc_node)
+ list_for_each_entry(child, &desc->tx_list, desc_node)
dma_sync_single_for_cpu(chan2parent(&dwc->chan),
child->txd.phys, sizeof(child->lli),
DMA_TO_DEVICE);
@@ -137,11 +137,11 @@ static void dwc_desc_put(struct dw_dma_chan *dwc, struct dw_desc *desc)
dwc_sync_desc_for_cpu(dwc, desc);
spin_lock_bh(&dwc->lock);
- list_for_each_entry(child, &desc->txd.tx_list, desc_node)
+ list_for_each_entry(child, &desc->tx_list, desc_node)
dev_vdbg(chan2dev(&dwc->chan),
"moving child desc %p to freelist\n",
child);
- list_splice_init(&desc->txd.tx_list, &dwc->free_list);
+ list_splice_init(&desc->tx_list, &dwc->free_list);
dev_vdbg(chan2dev(&dwc->chan), "moving desc %p to freelist\n", desc);
list_add(&desc->desc_node, &dwc->free_list);
spin_unlock_bh(&dwc->lock);
@@ -209,19 +209,28 @@ dwc_descriptor_complete(struct dw_dma_chan *dwc, struct dw_desc *desc)
param = txd->callback_param;
dwc_sync_desc_for_cpu(dwc, desc);
- list_splice_init(&txd->tx_list, &dwc->free_list);
+ list_splice_init(&desc->tx_list, &dwc->free_list);
list_move(&desc->desc_node, &dwc->free_list);
- /*
- * We use dma_unmap_page() regardless of how the buffers were
- * mapped before they were submitted...
- */
- if (!(txd->flags & DMA_COMPL_SKIP_DEST_UNMAP))
- dma_unmap_page(chan2parent(&dwc->chan), desc->lli.dar,
- desc->len, DMA_FROM_DEVICE);
- if (!(txd->flags & DMA_COMPL_SKIP_SRC_UNMAP))
- dma_unmap_page(chan2parent(&dwc->chan), desc->lli.sar,
- desc->len, DMA_TO_DEVICE);
+ if (!dwc->chan.private) {
+ struct device *parent = chan2parent(&dwc->chan);
+ if (!(txd->flags & DMA_COMPL_SKIP_DEST_UNMAP)) {
+ if (txd->flags & DMA_COMPL_DEST_UNMAP_SINGLE)
+ dma_unmap_single(parent, desc->lli.dar,
+ desc->len, DMA_FROM_DEVICE);
+ else
+ dma_unmap_page(parent, desc->lli.dar,
+ desc->len, DMA_FROM_DEVICE);
+ }
+ if (!(txd->flags & DMA_COMPL_SKIP_SRC_UNMAP)) {
+ if (txd->flags & DMA_COMPL_SRC_UNMAP_SINGLE)
+ dma_unmap_single(parent, desc->lli.sar,
+ desc->len, DMA_TO_DEVICE);
+ else
+ dma_unmap_page(parent, desc->lli.sar,
+ desc->len, DMA_TO_DEVICE);
+ }
+ }
/*
* The API requires that no submissions are done from a
@@ -289,7 +298,7 @@ static void dwc_scan_descriptors(struct dw_dma *dw, struct dw_dma_chan *dwc)
/* This one is currently in progress */
return;
- list_for_each_entry(child, &desc->txd.tx_list, desc_node)
+ list_for_each_entry(child, &desc->tx_list, desc_node)
if (child->lli.llp == llp)
/* Currently in progress */
return;
@@ -356,7 +365,7 @@ static void dwc_handle_error(struct dw_dma *dw, struct dw_dma_chan *dwc)
dev_printk(KERN_CRIT, chan2dev(&dwc->chan),
" cookie: %d\n", bad_desc->txd.cookie);
dwc_dump_lli(dwc, &bad_desc->lli);
- list_for_each_entry(child, &bad_desc->txd.tx_list, desc_node)
+ list_for_each_entry(child, &bad_desc->tx_list, desc_node)
dwc_dump_lli(dwc, &child->lli);
/* Pretend the descriptor completed successfully */
@@ -608,7 +617,7 @@ dwc_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src,
prev->txd.phys, sizeof(prev->lli),
DMA_TO_DEVICE);
list_add_tail(&desc->desc_node,
- &first->txd.tx_list);
+ &first->tx_list);
}
prev = desc;
}
@@ -658,8 +667,6 @@ dwc_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,
reg_width = dws->reg_width;
prev = first = NULL;
- sg_len = dma_map_sg(chan2parent(chan), sgl, sg_len, direction);
-
switch (direction) {
case DMA_TO_DEVICE:
ctllo = (DWC_DEFAULT_CTLLO
@@ -700,7 +707,7 @@ dwc_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,
sizeof(prev->lli),
DMA_TO_DEVICE);
list_add_tail(&desc->desc_node,
- &first->txd.tx_list);
+ &first->tx_list);
}
prev = desc;
total_len += len;
@@ -746,7 +753,7 @@ dwc_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,
sizeof(prev->lli),
DMA_TO_DEVICE);
list_add_tail(&desc->desc_node,
- &first->txd.tx_list);
+ &first->tx_list);
}
prev = desc;
total_len += len;
@@ -902,6 +909,7 @@ static int dwc_alloc_chan_resources(struct dma_chan *chan)
break;
}
+ INIT_LIST_HEAD(&desc->tx_list);
dma_async_tx_descriptor_init(&desc->txd, chan);
desc->txd.tx_submit = dwc_tx_submit;
desc->txd.flags = DMA_CTRL_ACK;
diff --git a/drivers/dma/dw_dmac_regs.h b/drivers/dma/dw_dmac_regs.h
index 13a580767031..d9a939f67f46 100644
--- a/drivers/dma/dw_dmac_regs.h
+++ b/drivers/dma/dw_dmac_regs.h
@@ -217,6 +217,7 @@ struct dw_desc {
/* THEN values for driver housekeeping */
struct list_head desc_node;
+ struct list_head tx_list;
struct dma_async_tx_descriptor txd;
size_t len;
};
diff --git a/drivers/dma/fsldma.c b/drivers/dma/fsldma.c
index f18d1bde0439..296f9e747fac 100644
--- a/drivers/dma/fsldma.c
+++ b/drivers/dma/fsldma.c
@@ -12,6 +12,11 @@
* also fit for MPC8560, MPC8555, MPC8548, MPC8641, and etc.
* The support for MPC8349 DMA contorller is also added.
*
+ * This driver instructs the DMA controller to issue the PCI Read Multiple
+ * command for PCI read operations, instead of using the default PCI Read Line
+ * command. Please be aware that this setting may result in read pre-fetching
+ * on some platforms.
+ *
* This is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
@@ -29,6 +34,7 @@
#include <linux/dmapool.h>
#include <linux/of_platform.h>
+#include <asm/fsldma.h>
#include "fsldma.h"
static void dma_init(struct fsl_dma_chan *fsl_chan)
@@ -49,9 +55,10 @@ static void dma_init(struct fsl_dma_chan *fsl_chan)
case FSL_DMA_IP_83XX:
/* Set the channel to below modes:
* EOTIE - End-of-transfer interrupt enable
+ * PRC_RM - PCI read multiple
*/
- DMA_OUT(fsl_chan, &fsl_chan->reg_base->mr, FSL_DMA_MR_EOTIE,
- 32);
+ DMA_OUT(fsl_chan, &fsl_chan->reg_base->mr, FSL_DMA_MR_EOTIE
+ | FSL_DMA_MR_PRC_RM, 32);
break;
}
@@ -136,15 +143,16 @@ static int dma_is_idle(struct fsl_dma_chan *fsl_chan)
static void dma_start(struct fsl_dma_chan *fsl_chan)
{
- u32 mr_set = 0;;
+ u32 mr_set = 0;
if (fsl_chan->feature & FSL_DMA_CHAN_PAUSE_EXT) {
DMA_OUT(fsl_chan, &fsl_chan->reg_base->bcr, 0, 32);
mr_set |= FSL_DMA_MR_EMP_EN;
- } else
+ } else if ((fsl_chan->feature & FSL_DMA_IP_MASK) == FSL_DMA_IP_85XX) {
DMA_OUT(fsl_chan, &fsl_chan->reg_base->mr,
DMA_IN(fsl_chan, &fsl_chan->reg_base->mr, 32)
& ~FSL_DMA_MR_EMP_EN, 32);
+ }
if (fsl_chan->feature & FSL_DMA_CHAN_START_EXT)
mr_set |= FSL_DMA_MR_EMS_EN;
@@ -273,28 +281,40 @@ static void fsl_chan_set_dest_loop_size(struct fsl_dma_chan *fsl_chan, int size)
}
/**
- * fsl_chan_toggle_ext_pause - Toggle channel external pause status
+ * fsl_chan_set_request_count - Set DMA Request Count for external control
* @fsl_chan : Freescale DMA channel
- * @size : Pause control size, 0 for disable external pause control.
- * The maximum is 1024.
+ * @size : Number of bytes to transfer in a single request
+ *
+ * The Freescale DMA channel can be controlled by the external signal DREQ#.
+ * The DMA request count is how many bytes are allowed to transfer before
+ * pausing the channel, after which a new assertion of DREQ# resumes channel
+ * operation.
*
- * The Freescale DMA channel can be controlled by the external
- * signal DREQ#. The pause control size is how many bytes are allowed
- * to transfer before pausing the channel, after which a new assertion
- * of DREQ# resumes channel operation.
+ * A size of 0 disables external pause control. The maximum size is 1024.
*/
-static void fsl_chan_toggle_ext_pause(struct fsl_dma_chan *fsl_chan, int size)
+static void fsl_chan_set_request_count(struct fsl_dma_chan *fsl_chan, int size)
{
- if (size > 1024)
- return;
+ BUG_ON(size > 1024);
+ DMA_OUT(fsl_chan, &fsl_chan->reg_base->mr,
+ DMA_IN(fsl_chan, &fsl_chan->reg_base->mr, 32)
+ | ((__ilog2(size) << 24) & 0x0f000000),
+ 32);
+}
- if (size) {
- DMA_OUT(fsl_chan, &fsl_chan->reg_base->mr,
- DMA_IN(fsl_chan, &fsl_chan->reg_base->mr, 32)
- | ((__ilog2(size) << 24) & 0x0f000000),
- 32);
+/**
+ * fsl_chan_toggle_ext_pause - Toggle channel external pause status
+ * @fsl_chan : Freescale DMA channel
+ * @enable : 0 is disabled, 1 is enabled.
+ *
+ * The Freescale DMA channel can be controlled by the external signal DREQ#.
+ * The DMA Request Count feature should be used in addition to this feature
+ * to set the number of bytes to transfer before pausing the channel.
+ */
+static void fsl_chan_toggle_ext_pause(struct fsl_dma_chan *fsl_chan, int enable)
+{
+ if (enable)
fsl_chan->feature |= FSL_DMA_CHAN_PAUSE_EXT;
- } else
+ else
fsl_chan->feature &= ~FSL_DMA_CHAN_PAUSE_EXT;
}
@@ -319,7 +339,8 @@ static void fsl_chan_toggle_ext_start(struct fsl_dma_chan *fsl_chan, int enable)
static dma_cookie_t fsl_dma_tx_submit(struct dma_async_tx_descriptor *tx)
{
struct fsl_dma_chan *fsl_chan = to_fsl_chan(tx->chan);
- struct fsl_desc_sw *desc;
+ struct fsl_desc_sw *desc = tx_to_fsl_desc(tx);
+ struct fsl_desc_sw *child;
unsigned long flags;
dma_cookie_t cookie;
@@ -327,7 +348,7 @@ static dma_cookie_t fsl_dma_tx_submit(struct dma_async_tx_descriptor *tx)
spin_lock_irqsave(&fsl_chan->desc_lock, flags);
cookie = fsl_chan->common.cookie;
- list_for_each_entry(desc, &tx->tx_list, node) {
+ list_for_each_entry(child, &desc->tx_list, node) {
cookie++;
if (cookie < 0)
cookie = 1;
@@ -336,8 +357,8 @@ static dma_cookie_t fsl_dma_tx_submit(struct dma_async_tx_descriptor *tx)
}
fsl_chan->common.cookie = cookie;
- append_ld_queue(fsl_chan, tx_to_fsl_desc(tx));
- list_splice_init(&tx->tx_list, fsl_chan->ld_queue.prev);
+ append_ld_queue(fsl_chan, desc);
+ list_splice_init(&desc->tx_list, fsl_chan->ld_queue.prev);
spin_unlock_irqrestore(&fsl_chan->desc_lock, flags);
@@ -359,6 +380,7 @@ static struct fsl_desc_sw *fsl_dma_alloc_descriptor(
desc_sw = dma_pool_alloc(fsl_chan->desc_pool, GFP_ATOMIC, &pdesc);
if (desc_sw) {
memset(desc_sw, 0, sizeof(struct fsl_desc_sw));
+ INIT_LIST_HEAD(&desc_sw->tx_list);
dma_async_tx_descriptor_init(&desc_sw->async_tx,
&fsl_chan->common);
desc_sw->async_tx.tx_submit = fsl_dma_tx_submit;
@@ -448,7 +470,7 @@ fsl_dma_prep_interrupt(struct dma_chan *chan, unsigned long flags)
new->async_tx.flags = flags;
/* Insert the link descriptor to the LD ring */
- list_add_tail(&new->node, &new->async_tx.tx_list);
+ list_add_tail(&new->node, &new->tx_list);
/* Set End-of-link to the last link descriptor of new list*/
set_ld_eol(fsl_chan, new);
@@ -506,7 +528,7 @@ static struct dma_async_tx_descriptor *fsl_dma_prep_memcpy(
dma_dest += copy;
/* Insert the link descriptor to the LD ring */
- list_add_tail(&new->node, &first->async_tx.tx_list);
+ list_add_tail(&new->node, &first->tx_list);
} while (len);
new->async_tx.flags = flags; /* client is in control of this ack */
@@ -521,7 +543,7 @@ fail:
if (!first)
return NULL;
- list = &first->async_tx.tx_list;
+ list = &first->tx_list;
list_for_each_entry_safe_reverse(new, prev, list, node) {
list_del(&new->node);
dma_pool_free(fsl_chan->desc_pool, new, new->async_tx.phys);
@@ -531,6 +553,229 @@ fail:
}
/**
+ * fsl_dma_prep_slave_sg - prepare descriptors for a DMA_SLAVE transaction
+ * @chan: DMA channel
+ * @sgl: scatterlist to transfer to/from
+ * @sg_len: number of entries in @scatterlist
+ * @direction: DMA direction
+ * @flags: DMAEngine flags
+ *
+ * Prepare a set of descriptors for a DMA_SLAVE transaction. Following the
+ * DMA_SLAVE API, this gets the device-specific information from the
+ * chan->private variable.
+ */
+static struct dma_async_tx_descriptor *fsl_dma_prep_slave_sg(
+ struct dma_chan *chan, struct scatterlist *sgl, unsigned int sg_len,
+ enum dma_data_direction direction, unsigned long flags)
+{
+ struct fsl_dma_chan *fsl_chan;
+ struct fsl_desc_sw *first = NULL, *prev = NULL, *new = NULL;
+ struct fsl_dma_slave *slave;
+ struct list_head *tx_list;
+ size_t copy;
+
+ int i;
+ struct scatterlist *sg;
+ size_t sg_used;
+ size_t hw_used;
+ struct fsl_dma_hw_addr *hw;
+ dma_addr_t dma_dst, dma_src;
+
+ if (!chan)
+ return NULL;
+
+ if (!chan->private)
+ return NULL;
+
+ fsl_chan = to_fsl_chan(chan);
+ slave = chan->private;
+
+ if (list_empty(&slave->addresses))
+ return NULL;
+
+ hw = list_first_entry(&slave->addresses, struct fsl_dma_hw_addr, entry);
+ hw_used = 0;
+
+ /*
+ * Build the hardware transaction to copy from the scatterlist to
+ * the hardware, or from the hardware to the scatterlist
+ *
+ * If you are copying from the hardware to the scatterlist and it
+ * takes two hardware entries to fill an entire page, then both
+ * hardware entries will be coalesced into the same page
+ *
+ * If you are copying from the scatterlist to the hardware and a
+ * single page can fill two hardware entries, then the data will
+ * be read out of the page into the first hardware entry, and so on
+ */
+ for_each_sg(sgl, sg, sg_len, i) {
+ sg_used = 0;
+
+ /* Loop until the entire scatterlist entry is used */
+ while (sg_used < sg_dma_len(sg)) {
+
+ /*
+ * If we've used up the current hardware address/length
+ * pair, we need to load a new one
+ *
+ * This is done in a while loop so that descriptors with
+ * length == 0 will be skipped
+ */
+ while (hw_used >= hw->length) {
+
+ /*
+ * If the current hardware entry is the last
+ * entry in the list, we're finished
+ */
+ if (list_is_last(&hw->entry, &slave->addresses))
+ goto finished;
+
+ /* Get the next hardware address/length pair */
+ hw = list_entry(hw->entry.next,
+ struct fsl_dma_hw_addr, entry);
+ hw_used = 0;
+ }
+
+ /* Allocate the link descriptor from DMA pool */
+ new = fsl_dma_alloc_descriptor(fsl_chan);
+ if (!new) {
+ dev_err(fsl_chan->dev, "No free memory for "
+ "link descriptor\n");
+ goto fail;
+ }
+#ifdef FSL_DMA_LD_DEBUG
+ dev_dbg(fsl_chan->dev, "new link desc alloc %p\n", new);
+#endif
+
+ /*
+ * Calculate the maximum number of bytes to transfer,
+ * making sure it is less than the DMA controller limit
+ */
+ copy = min_t(size_t, sg_dma_len(sg) - sg_used,
+ hw->length - hw_used);
+ copy = min_t(size_t, copy, FSL_DMA_BCR_MAX_CNT);
+
+ /*
+ * DMA_FROM_DEVICE
+ * from the hardware to the scatterlist
+ *
+ * DMA_TO_DEVICE
+ * from the scatterlist to the hardware
+ */
+ if (direction == DMA_FROM_DEVICE) {
+ dma_src = hw->address + hw_used;
+ dma_dst = sg_dma_address(sg) + sg_used;
+ } else {
+ dma_src = sg_dma_address(sg) + sg_used;
+ dma_dst = hw->address + hw_used;
+ }
+
+ /* Fill in the descriptor */
+ set_desc_cnt(fsl_chan, &new->hw, copy);
+ set_desc_src(fsl_chan, &new->hw, dma_src);
+ set_desc_dest(fsl_chan, &new->hw, dma_dst);
+
+ /*
+ * If this is not the first descriptor, chain the
+ * current descriptor after the previous descriptor
+ */
+ if (!first) {
+ first = new;
+ } else {
+ set_desc_next(fsl_chan, &prev->hw,
+ new->async_tx.phys);
+ }
+
+ new->async_tx.cookie = 0;
+ async_tx_ack(&new->async_tx);
+
+ prev = new;
+ sg_used += copy;
+ hw_used += copy;
+
+ /* Insert the link descriptor into the LD ring */
+ list_add_tail(&new->node, &first->tx_list);
+ }
+ }
+
+finished:
+
+ /* All of the hardware address/length pairs had length == 0 */
+ if (!first || !new)
+ return NULL;
+
+ new->async_tx.flags = flags;
+ new->async_tx.cookie = -EBUSY;
+
+ /* Set End-of-link to the last link descriptor of new list */
+ set_ld_eol(fsl_chan, new);
+
+ /* Enable extra controller features */
+ if (fsl_chan->set_src_loop_size)
+ fsl_chan->set_src_loop_size(fsl_chan, slave->src_loop_size);
+
+ if (fsl_chan->set_dest_loop_size)
+ fsl_chan->set_dest_loop_size(fsl_chan, slave->dst_loop_size);
+
+ if (fsl_chan->toggle_ext_start)
+ fsl_chan->toggle_ext_start(fsl_chan, slave->external_start);
+
+ if (fsl_chan->toggle_ext_pause)
+ fsl_chan->toggle_ext_pause(fsl_chan, slave->external_pause);
+
+ if (fsl_chan->set_request_count)
+ fsl_chan->set_request_count(fsl_chan, slave->request_count);
+
+ return &first->async_tx;
+
+fail:
+ /* If first was not set, then we failed to allocate the very first
+ * descriptor, and we're done */
+ if (!first)
+ return NULL;
+
+ /*
+ * First is set, so all of the descriptors we allocated have been added
+ * to first->tx_list, INCLUDING "first" itself. Therefore we
+ * must traverse the list backwards freeing each descriptor in turn
+ *
+ * We're re-using variables for the loop, oh well
+ */
+ tx_list = &first->tx_list;
+ list_for_each_entry_safe_reverse(new, prev, tx_list, node) {
+ list_del_init(&new->node);
+ dma_pool_free(fsl_chan->desc_pool, new, new->async_tx.phys);
+ }
+
+ return NULL;
+}
+
+static void fsl_dma_device_terminate_all(struct dma_chan *chan)
+{
+ struct fsl_dma_chan *fsl_chan;
+ struct fsl_desc_sw *desc, *tmp;
+ unsigned long flags;
+
+ if (!chan)
+ return;
+
+ fsl_chan = to_fsl_chan(chan);
+
+ /* Halt the DMA engine */
+ dma_halt(fsl_chan);
+
+ spin_lock_irqsave(&fsl_chan->desc_lock, flags);
+
+ /* Remove and free all of the descriptors in the LD queue */
+ list_for_each_entry_safe(desc, tmp, &fsl_chan->ld_queue, node) {
+ list_del(&desc->node);
+ dma_pool_free(fsl_chan->desc_pool, desc, desc->async_tx.phys);
+ }
+
+ spin_unlock_irqrestore(&fsl_chan->desc_lock, flags);
+}
+
+/**
* fsl_dma_update_completed_cookie - Update the completed cookie.
* @fsl_chan : Freescale DMA channel
*/
@@ -871,11 +1116,12 @@ static int __devinit fsl_dma_chan_probe(struct fsl_dma_device *fdev,
switch (new_fsl_chan->feature & FSL_DMA_IP_MASK) {
case FSL_DMA_IP_85XX:
- new_fsl_chan->toggle_ext_start = fsl_chan_toggle_ext_start;
new_fsl_chan->toggle_ext_pause = fsl_chan_toggle_ext_pause;
case FSL_DMA_IP_83XX:
+ new_fsl_chan->toggle_ext_start = fsl_chan_toggle_ext_start;
new_fsl_chan->set_src_loop_size = fsl_chan_set_src_loop_size;
new_fsl_chan->set_dest_loop_size = fsl_chan_set_dest_loop_size;
+ new_fsl_chan->set_request_count = fsl_chan_set_request_count;
}
spin_lock_init(&new_fsl_chan->desc_lock);
@@ -955,12 +1201,15 @@ static int __devinit of_fsl_dma_probe(struct of_device *dev,
dma_cap_set(DMA_MEMCPY, fdev->common.cap_mask);
dma_cap_set(DMA_INTERRUPT, fdev->common.cap_mask);
+ dma_cap_set(DMA_SLAVE, fdev->common.cap_mask);
fdev->common.device_alloc_chan_resources = fsl_dma_alloc_chan_resources;
fdev->common.device_free_chan_resources = fsl_dma_free_chan_resources;
fdev->common.device_prep_dma_interrupt = fsl_dma_prep_interrupt;
fdev->common.device_prep_dma_memcpy = fsl_dma_prep_memcpy;
fdev->common.device_is_tx_complete = fsl_dma_is_complete;
fdev->common.device_issue_pending = fsl_dma_memcpy_issue_pending;
+ fdev->common.device_prep_slave_sg = fsl_dma_prep_slave_sg;
+ fdev->common.device_terminate_all = fsl_dma_device_terminate_all;
fdev->common.dev = &dev->dev;
fdev->irq = irq_of_parse_and_map(dev->node, 0);
diff --git a/drivers/dma/fsldma.h b/drivers/dma/fsldma.h
index 4f21a512d848..0df14cbb8ca3 100644
--- a/drivers/dma/fsldma.h
+++ b/drivers/dma/fsldma.h
@@ -38,6 +38,7 @@
/* Special MR definition for MPC8349 */
#define FSL_DMA_MR_EOTIE 0x00000080
+#define FSL_DMA_MR_PRC_RM 0x00000800
#define FSL_DMA_SR_CH 0x00000020
#define FSL_DMA_SR_PE 0x00000010
@@ -89,6 +90,7 @@ struct fsl_dma_ld_hw {
struct fsl_desc_sw {
struct fsl_dma_ld_hw hw;
struct list_head node;
+ struct list_head tx_list;
struct dma_async_tx_descriptor async_tx;
struct list_head *ld;
void *priv;
@@ -142,10 +144,11 @@ struct fsl_dma_chan {
struct tasklet_struct tasklet;
u32 feature;
- void (*toggle_ext_pause)(struct fsl_dma_chan *fsl_chan, int size);
+ void (*toggle_ext_pause)(struct fsl_dma_chan *fsl_chan, int enable);
void (*toggle_ext_start)(struct fsl_dma_chan *fsl_chan, int enable);
void (*set_src_loop_size)(struct fsl_dma_chan *fsl_chan, int size);
void (*set_dest_loop_size)(struct fsl_dma_chan *fsl_chan, int size);
+ void (*set_request_count)(struct fsl_dma_chan *fsl_chan, int size);
};
#define to_fsl_chan(chan) container_of(chan, struct fsl_dma_chan, common)
diff --git a/drivers/dma/ioat/dma.c b/drivers/dma/ioat/dma.c
index 32a757be75c1..c524d36d3c2e 100644
--- a/drivers/dma/ioat/dma.c
+++ b/drivers/dma/ioat/dma.c
@@ -251,12 +251,12 @@ static dma_cookie_t ioat1_tx_submit(struct dma_async_tx_descriptor *tx)
dev_dbg(to_dev(&ioat->base), "%s: cookie: %d\n", __func__, cookie);
/* write address into NextDescriptor field of last desc in chain */
- first = to_ioat_desc(tx->tx_list.next);
+ first = to_ioat_desc(desc->tx_list.next);
chain_tail = to_ioat_desc(ioat->used_desc.prev);
/* make descriptor updates globally visible before chaining */
wmb();
chain_tail->hw->next = first->txd.phys;
- list_splice_tail_init(&tx->tx_list, &ioat->used_desc);
+ list_splice_tail_init(&desc->tx_list, &ioat->used_desc);
dump_desc_dbg(ioat, chain_tail);
dump_desc_dbg(ioat, first);
@@ -298,6 +298,7 @@ ioat_dma_alloc_descriptor(struct ioat_dma_chan *ioat, gfp_t flags)
memset(desc, 0, sizeof(*desc));
+ INIT_LIST_HEAD(&desc_sw->tx_list);
dma_async_tx_descriptor_init(&desc_sw->txd, &ioat->base.common);
desc_sw->txd.tx_submit = ioat1_tx_submit;
desc_sw->hw = desc;
@@ -522,7 +523,7 @@ ioat1_dma_prep_memcpy(struct dma_chan *c, dma_addr_t dma_dest,
desc->txd.flags = flags;
desc->len = total_len;
- list_splice(&chain, &desc->txd.tx_list);
+ list_splice(&chain, &desc->tx_list);
hw->ctl_f.int_en = !!(flags & DMA_PREP_INTERRUPT);
hw->ctl_f.compl_write = 1;
hw->tx_cnt = tx_cnt;
diff --git a/drivers/dma/ioat/dma.h b/drivers/dma/ioat/dma.h
index 0e37e426c729..6a675a2a2d1c 100644
--- a/drivers/dma/ioat/dma.h
+++ b/drivers/dma/ioat/dma.h
@@ -171,7 +171,7 @@ ioat_is_complete(struct dma_chan *c, dma_cookie_t cookie,
* struct ioat_desc_sw - wrapper around hardware descriptor
* @hw: hardware DMA descriptor (for memcpy)
* @node: this descriptor will either be on the free list,
- * or attached to a transaction list (async_tx.tx_list)
+ * or attached to a transaction list (tx_list)
* @txd: the generic software descriptor for all engines
* @id: identifier for debug
*/
@@ -179,6 +179,7 @@ struct ioat_desc_sw {
struct ioat_dma_descriptor *hw;
struct list_head node;
size_t len;
+ struct list_head tx_list;
struct dma_async_tx_descriptor txd;
#ifdef DEBUG
int id;
diff --git a/drivers/dma/ioat/dma_v2.c b/drivers/dma/ioat/dma_v2.c
index 7bbbd83d12e6..5d6ac49e0d32 100644
--- a/drivers/dma/ioat/dma_v2.c
+++ b/drivers/dma/ioat/dma_v2.c
@@ -397,11 +397,12 @@ static struct ioat_ring_ent *ioat2_alloc_ring_ent(struct dma_chan *chan, gfp_t f
return NULL;
memset(hw, 0, sizeof(*hw));
- desc = kzalloc(sizeof(*desc), flags);
+ desc = kmem_cache_alloc(ioat2_cache, flags);
if (!desc) {
pci_pool_free(dma->dma_pool, hw, phys);
return NULL;
}
+ memset(desc, 0, sizeof(*desc));
dma_async_tx_descriptor_init(&desc->txd, chan);
desc->txd.tx_submit = ioat2_tx_submit_unlock;
@@ -416,7 +417,7 @@ static void ioat2_free_ring_ent(struct ioat_ring_ent *desc, struct dma_chan *cha
dma = to_ioatdma_device(chan->device);
pci_pool_free(dma->dma_pool, desc->hw, desc->txd.phys);
- kfree(desc);
+ kmem_cache_free(ioat2_cache, desc);
}
static struct ioat_ring_ent **ioat2_alloc_ring(struct dma_chan *c, int order, gfp_t flags)
diff --git a/drivers/dma/ioat/dma_v2.h b/drivers/dma/ioat/dma_v2.h
index 246e646b1904..1d849ef74d5f 100644
--- a/drivers/dma/ioat/dma_v2.h
+++ b/drivers/dma/ioat/dma_v2.h
@@ -142,8 +142,8 @@ struct ioat_ring_ent {
struct ioat_pq_update_descriptor *pqu;
struct ioat_raw_descriptor *raw;
};
- struct dma_async_tx_descriptor txd;
size_t len;
+ struct dma_async_tx_descriptor txd;
enum sum_check_flags *result;
#ifdef DEBUG
int id;
@@ -186,4 +186,5 @@ void __ioat2_issue_pending(struct ioat2_dma_chan *ioat);
void ioat2_cleanup_tasklet(unsigned long data);
void ioat2_timer_event(unsigned long data);
extern struct kobj_type ioat2_ktype;
+extern struct kmem_cache *ioat2_cache;
#endif /* IOATDMA_V2_H */
diff --git a/drivers/dma/ioat/pci.c b/drivers/dma/ioat/pci.c
index b77d3a2864ad..c788fa266470 100644
--- a/drivers/dma/ioat/pci.c
+++ b/drivers/dma/ioat/pci.c
@@ -83,6 +83,8 @@ static int ioat_dca_enabled = 1;
module_param(ioat_dca_enabled, int, 0644);
MODULE_PARM_DESC(ioat_dca_enabled, "control support of dca service (default: 1)");
+struct kmem_cache *ioat2_cache;
+
#define DRV_NAME "ioatdma"
static struct pci_driver ioat_pci_driver = {
@@ -182,15 +184,27 @@ static void __devexit ioat_remove(struct pci_dev *pdev)
static int __init ioat_init_module(void)
{
+ int err;
+
pr_info("%s: Intel(R) QuickData Technology Driver %s\n",
DRV_NAME, IOAT_DMA_VERSION);
- return pci_register_driver(&ioat_pci_driver);
+ ioat2_cache = kmem_cache_create("ioat2", sizeof(struct ioat_ring_ent),
+ 0, SLAB_HWCACHE_ALIGN, NULL);
+ if (!ioat2_cache)
+ return -ENOMEM;
+
+ err = pci_register_driver(&ioat_pci_driver);
+ if (err)
+ kmem_cache_destroy(ioat2_cache);
+
+ return err;
}
module_init(ioat_init_module);
static void __exit ioat_exit_module(void)
{
pci_unregister_driver(&ioat_pci_driver);
+ kmem_cache_destroy(ioat2_cache);
}
module_exit(ioat_exit_module);
diff --git a/drivers/dma/iop-adma.c b/drivers/dma/iop-adma.c
index 518f557ef857..645ca8d54ec4 100644
--- a/drivers/dma/iop-adma.c
+++ b/drivers/dma/iop-adma.c
@@ -421,7 +421,7 @@ retry:
}
alloc_tail->group_head = alloc_start;
alloc_tail->async_tx.cookie = -EBUSY;
- list_splice(&chain, &alloc_tail->async_tx.tx_list);
+ list_splice(&chain, &alloc_tail->tx_list);
iop_chan->last_used = last_used;
iop_desc_clear_next_desc(alloc_start);
iop_desc_clear_next_desc(alloc_tail);
@@ -480,7 +480,7 @@ iop_adma_tx_submit(struct dma_async_tx_descriptor *tx)
old_chain_tail = list_entry(iop_chan->chain.prev,
struct iop_adma_desc_slot, chain_node);
- list_splice_init(&sw_desc->async_tx.tx_list,
+ list_splice_init(&sw_desc->tx_list,
&old_chain_tail->chain_node);
/* fix up the hardware chain */
@@ -547,6 +547,7 @@ static int iop_adma_alloc_chan_resources(struct dma_chan *chan)
dma_async_tx_descriptor_init(&slot->async_tx, chan);
slot->async_tx.tx_submit = iop_adma_tx_submit;
+ INIT_LIST_HEAD(&slot->tx_list);
INIT_LIST_HEAD(&slot->chain_node);
INIT_LIST_HEAD(&slot->slot_node);
hw_desc = (char *) iop_chan->device->dma_desc_pool;
@@ -1642,7 +1643,7 @@ static void iop_chan_start_null_memcpy(struct iop_adma_chan *iop_chan)
if (sw_desc) {
grp_start = sw_desc->group_head;
- list_splice_init(&sw_desc->async_tx.tx_list, &iop_chan->chain);
+ list_splice_init(&sw_desc->tx_list, &iop_chan->chain);
async_tx_ack(&sw_desc->async_tx);
iop_desc_init_memcpy(grp_start, 0);
iop_desc_set_byte_count(grp_start, iop_chan, 0);
@@ -1698,7 +1699,7 @@ static void iop_chan_start_null_xor(struct iop_adma_chan *iop_chan)
sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op);
if (sw_desc) {
grp_start = sw_desc->group_head;
- list_splice_init(&sw_desc->async_tx.tx_list, &iop_chan->chain);
+ list_splice_init(&sw_desc->tx_list, &iop_chan->chain);
async_tx_ack(&sw_desc->async_tx);
iop_desc_init_null_xor(grp_start, 2, 0);
iop_desc_set_byte_count(grp_start, iop_chan, 0);
diff --git a/drivers/dma/mv_xor.c b/drivers/dma/mv_xor.c
index ddab94f51224..466ab10c1ff1 100644
--- a/drivers/dma/mv_xor.c
+++ b/drivers/dma/mv_xor.c
@@ -517,7 +517,7 @@ retry:
}
alloc_tail->group_head = alloc_start;
alloc_tail->async_tx.cookie = -EBUSY;
- list_splice(&chain, &alloc_tail->async_tx.tx_list);
+ list_splice(&chain, &alloc_tail->tx_list);
mv_chan->last_used = last_used;
mv_desc_clear_next_desc(alloc_start);
mv_desc_clear_next_desc(alloc_tail);
@@ -565,14 +565,14 @@ mv_xor_tx_submit(struct dma_async_tx_descriptor *tx)
cookie = mv_desc_assign_cookie(mv_chan, sw_desc);
if (list_empty(&mv_chan->chain))
- list_splice_init(&sw_desc->async_tx.tx_list, &mv_chan->chain);
+ list_splice_init(&sw_desc->tx_list, &mv_chan->chain);
else {
new_hw_chain = 0;
old_chain_tail = list_entry(mv_chan->chain.prev,
struct mv_xor_desc_slot,
chain_node);
- list_splice_init(&grp_start->async_tx.tx_list,
+ list_splice_init(&grp_start->tx_list,
&old_chain_tail->chain_node);
if (!mv_can_chain(grp_start))
@@ -632,6 +632,7 @@ static int mv_xor_alloc_chan_resources(struct dma_chan *chan)
slot->async_tx.tx_submit = mv_xor_tx_submit;
INIT_LIST_HEAD(&slot->chain_node);
INIT_LIST_HEAD(&slot->slot_node);
+ INIT_LIST_HEAD(&slot->tx_list);
hw_desc = (char *) mv_chan->device->dma_desc_pool;
slot->async_tx.phys =
(dma_addr_t) &hw_desc[idx * MV_XOR_SLOT_SIZE];
@@ -1176,7 +1177,7 @@ static int __devinit mv_xor_probe(struct platform_device *pdev)
if (dma_has_cap(DMA_MEMSET, dma_dev->cap_mask))
dma_dev->device_prep_dma_memset = mv_xor_prep_dma_memset;
if (dma_has_cap(DMA_XOR, dma_dev->cap_mask)) {
- dma_dev->max_xor = 8; ;
+ dma_dev->max_xor = 8;
dma_dev->device_prep_dma_xor = mv_xor_prep_dma_xor;
}
diff --git a/drivers/dma/mv_xor.h b/drivers/dma/mv_xor.h
index 06cafe1ef521..977b592e976b 100644
--- a/drivers/dma/mv_xor.h
+++ b/drivers/dma/mv_xor.h
@@ -126,9 +126,8 @@ struct mv_xor_chan {
* @idx: pool index
* @unmap_src_cnt: number of xor sources
* @unmap_len: transaction bytecount
+ * @tx_list: list of slots that make up a multi-descriptor transaction
* @async_tx: support for the async_tx api
- * @group_list: list of slots that make up a multi-descriptor transaction
- * for example transfer lengths larger than the supported hw max
* @xor_check_result: result of zero sum
* @crc32_result: result crc calculation
*/
@@ -145,6 +144,7 @@ struct mv_xor_desc_slot {
u16 unmap_src_cnt;
u32 value;
size_t unmap_len;
+ struct list_head tx_list;
struct dma_async_tx_descriptor async_tx;
union {
u32 *xor_check_result;
diff --git a/drivers/dma/txx9dmac.c b/drivers/dma/txx9dmac.c
new file mode 100644
index 000000000000..197c7bc37895
--- /dev/null
+++ b/drivers/dma/txx9dmac.c
@@ -0,0 +1,1356 @@
+/*
+ * Driver for the TXx9 SoC DMA Controller
+ *
+ * Copyright (C) 2009 Atsushi Nemoto
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+#include <linux/dma-mapping.h>
+#include <linux/init.h>
+#include <linux/interrupt.h>
+#include <linux/io.h>
+#include <linux/module.h>
+#include <linux/platform_device.h>
+#include <linux/slab.h>
+#include <linux/scatterlist.h>
+#include "txx9dmac.h"
+
+static struct txx9dmac_chan *to_txx9dmac_chan(struct dma_chan *chan)
+{
+ return container_of(chan, struct txx9dmac_chan, chan);
+}
+
+static struct txx9dmac_cregs __iomem *__dma_regs(const struct txx9dmac_chan *dc)
+{
+ return dc->ch_regs;
+}
+
+static struct txx9dmac_cregs32 __iomem *__dma_regs32(
+ const struct txx9dmac_chan *dc)
+{
+ return dc->ch_regs;
+}
+
+#define channel64_readq(dc, name) \
+ __raw_readq(&(__dma_regs(dc)->name))
+#define channel64_writeq(dc, name, val) \
+ __raw_writeq((val), &(__dma_regs(dc)->name))
+#define channel64_readl(dc, name) \
+ __raw_readl(&(__dma_regs(dc)->name))
+#define channel64_writel(dc, name, val) \
+ __raw_writel((val), &(__dma_regs(dc)->name))
+
+#define channel32_readl(dc, name) \
+ __raw_readl(&(__dma_regs32(dc)->name))
+#define channel32_writel(dc, name, val) \
+ __raw_writel((val), &(__dma_regs32(dc)->name))
+
+#define channel_readq(dc, name) channel64_readq(dc, name)
+#define channel_writeq(dc, name, val) channel64_writeq(dc, name, val)
+#define channel_readl(dc, name) \
+ (is_dmac64(dc) ? \
+ channel64_readl(dc, name) : channel32_readl(dc, name))
+#define channel_writel(dc, name, val) \
+ (is_dmac64(dc) ? \
+ channel64_writel(dc, name, val) : channel32_writel(dc, name, val))
+
+static dma_addr_t channel64_read_CHAR(const struct txx9dmac_chan *dc)
+{
+ if (sizeof(__dma_regs(dc)->CHAR) == sizeof(u64))
+ return channel64_readq(dc, CHAR);
+ else
+ return channel64_readl(dc, CHAR);
+}
+
+static void channel64_write_CHAR(const struct txx9dmac_chan *dc, dma_addr_t val)
+{
+ if (sizeof(__dma_regs(dc)->CHAR) == sizeof(u64))
+ channel64_writeq(dc, CHAR, val);
+ else
+ channel64_writel(dc, CHAR, val);
+}
+
+static void channel64_clear_CHAR(const struct txx9dmac_chan *dc)
+{
+#if defined(CONFIG_32BIT) && !defined(CONFIG_64BIT_PHYS_ADDR)
+ channel64_writel(dc, CHAR, 0);
+ channel64_writel(dc, __pad_CHAR, 0);
+#else
+ channel64_writeq(dc, CHAR, 0);
+#endif
+}
+
+static dma_addr_t channel_read_CHAR(const struct txx9dmac_chan *dc)
+{
+ if (is_dmac64(dc))
+ return channel64_read_CHAR(dc);
+ else
+ return channel32_readl(dc, CHAR);
+}
+
+static void channel_write_CHAR(const struct txx9dmac_chan *dc, dma_addr_t val)
+{
+ if (is_dmac64(dc))
+ channel64_write_CHAR(dc, val);
+ else
+ channel32_writel(dc, CHAR, val);
+}
+
+static struct txx9dmac_regs __iomem *__txx9dmac_regs(
+ const struct txx9dmac_dev *ddev)
+{
+ return ddev->regs;
+}
+
+static struct txx9dmac_regs32 __iomem *__txx9dmac_regs32(
+ const struct txx9dmac_dev *ddev)
+{
+ return ddev->regs;
+}
+
+#define dma64_readl(ddev, name) \
+ __raw_readl(&(__txx9dmac_regs(ddev)->name))
+#define dma64_writel(ddev, name, val) \
+ __raw_writel((val), &(__txx9dmac_regs(ddev)->name))
+
+#define dma32_readl(ddev, name) \
+ __raw_readl(&(__txx9dmac_regs32(ddev)->name))
+#define dma32_writel(ddev, name, val) \
+ __raw_writel((val), &(__txx9dmac_regs32(ddev)->name))
+
+#define dma_readl(ddev, name) \
+ (__is_dmac64(ddev) ? \
+ dma64_readl(ddev, name) : dma32_readl(ddev, name))
+#define dma_writel(ddev, name, val) \
+ (__is_dmac64(ddev) ? \
+ dma64_writel(ddev, name, val) : dma32_writel(ddev, name, val))
+
+static struct device *chan2dev(struct dma_chan *chan)
+{
+ return &chan->dev->device;
+}
+static struct device *chan2parent(struct dma_chan *chan)
+{
+ return chan->dev->device.parent;
+}
+
+static struct txx9dmac_desc *
+txd_to_txx9dmac_desc(struct dma_async_tx_descriptor *txd)
+{
+ return container_of(txd, struct txx9dmac_desc, txd);
+}
+
+static dma_addr_t desc_read_CHAR(const struct txx9dmac_chan *dc,
+ const struct txx9dmac_desc *desc)
+{
+ return is_dmac64(dc) ? desc->hwdesc.CHAR : desc->hwdesc32.CHAR;
+}
+
+static void desc_write_CHAR(const struct txx9dmac_chan *dc,
+ struct txx9dmac_desc *desc, dma_addr_t val)
+{
+ if (is_dmac64(dc))
+ desc->hwdesc.CHAR = val;
+ else
+ desc->hwdesc32.CHAR = val;
+}
+
+#define TXX9_DMA_MAX_COUNT 0x04000000
+
+#define TXX9_DMA_INITIAL_DESC_COUNT 64
+
+static struct txx9dmac_desc *txx9dmac_first_active(struct txx9dmac_chan *dc)
+{
+ return list_entry(dc->active_list.next,
+ struct txx9dmac_desc, desc_node);
+}
+
+static struct txx9dmac_desc *txx9dmac_last_active(struct txx9dmac_chan *dc)
+{
+ return list_entry(dc->active_list.prev,
+ struct txx9dmac_desc, desc_node);
+}
+
+static struct txx9dmac_desc *txx9dmac_first_queued(struct txx9dmac_chan *dc)
+{
+ return list_entry(dc->queue.next, struct txx9dmac_desc, desc_node);
+}
+
+static struct txx9dmac_desc *txx9dmac_last_child(struct txx9dmac_desc *desc)
+{
+ if (!list_empty(&desc->tx_list))
+ desc = list_entry(desc->tx_list.prev, typeof(*desc), desc_node);
+ return desc;
+}
+
+static dma_cookie_t txx9dmac_tx_submit(struct dma_async_tx_descriptor *tx);
+
+static struct txx9dmac_desc *txx9dmac_desc_alloc(struct txx9dmac_chan *dc,
+ gfp_t flags)
+{
+ struct txx9dmac_dev *ddev = dc->ddev;
+ struct txx9dmac_desc *desc;
+
+ desc = kzalloc(sizeof(*desc), flags);
+ if (!desc)
+ return NULL;
+ INIT_LIST_HEAD(&desc->tx_list);
+ dma_async_tx_descriptor_init(&desc->txd, &dc->chan);
+ desc->txd.tx_submit = txx9dmac_tx_submit;
+ /* txd.flags will be overwritten in prep funcs */
+ desc->txd.flags = DMA_CTRL_ACK;
+ desc->txd.phys = dma_map_single(chan2parent(&dc->chan), &desc->hwdesc,
+ ddev->descsize, DMA_TO_DEVICE);
+ return desc;
+}
+
+static struct txx9dmac_desc *txx9dmac_desc_get(struct txx9dmac_chan *dc)
+{
+ struct txx9dmac_desc *desc, *_desc;
+ struct txx9dmac_desc *ret = NULL;
+ unsigned int i = 0;
+
+ spin_lock_bh(&dc->lock);
+ list_for_each_entry_safe(desc, _desc, &dc->free_list, desc_node) {
+ if (async_tx_test_ack(&desc->txd)) {
+ list_del(&desc->desc_node);
+ ret = desc;
+ break;
+ }
+ dev_dbg(chan2dev(&dc->chan), "desc %p not ACKed\n", desc);
+ i++;
+ }
+ spin_unlock_bh(&dc->lock);
+
+ dev_vdbg(chan2dev(&dc->chan), "scanned %u descriptors on freelist\n",
+ i);
+ if (!ret) {
+ ret = txx9dmac_desc_alloc(dc, GFP_ATOMIC);
+ if (ret) {
+ spin_lock_bh(&dc->lock);
+ dc->descs_allocated++;
+ spin_unlock_bh(&dc->lock);
+ } else
+ dev_err(chan2dev(&dc->chan),
+ "not enough descriptors available\n");
+ }
+ return ret;
+}
+
+static void txx9dmac_sync_desc_for_cpu(struct txx9dmac_chan *dc,
+ struct txx9dmac_desc *desc)
+{
+ struct txx9dmac_dev *ddev = dc->ddev;
+ struct txx9dmac_desc *child;
+
+ list_for_each_entry(child, &desc->tx_list, desc_node)
+ dma_sync_single_for_cpu(chan2parent(&dc->chan),
+ child->txd.phys, ddev->descsize,
+ DMA_TO_DEVICE);
+ dma_sync_single_for_cpu(chan2parent(&dc->chan),
+ desc->txd.phys, ddev->descsize,
+ DMA_TO_DEVICE);
+}
+
+/*
+ * Move a descriptor, including any children, to the free list.
+ * `desc' must not be on any lists.
+ */
+static void txx9dmac_desc_put(struct txx9dmac_chan *dc,
+ struct txx9dmac_desc *desc)
+{
+ if (desc) {
+ struct txx9dmac_desc *child;
+
+ txx9dmac_sync_desc_for_cpu(dc, desc);
+
+ spin_lock_bh(&dc->lock);
+ list_for_each_entry(child, &desc->tx_list, desc_node)
+ dev_vdbg(chan2dev(&dc->chan),
+ "moving child desc %p to freelist\n",
+ child);
+ list_splice_init(&desc->tx_list, &dc->free_list);
+ dev_vdbg(chan2dev(&dc->chan), "moving desc %p to freelist\n",
+ desc);
+ list_add(&desc->desc_node, &dc->free_list);
+ spin_unlock_bh(&dc->lock);
+ }
+}
+
+/* Called with dc->lock held and bh disabled */
+static dma_cookie_t
+txx9dmac_assign_cookie(struct txx9dmac_chan *dc, struct txx9dmac_desc *desc)
+{
+ dma_cookie_t cookie = dc->chan.cookie;
+
+ if (++cookie < 0)
+ cookie = 1;
+
+ dc->chan.cookie = cookie;
+ desc->txd.cookie = cookie;
+
+ return cookie;
+}
+
+/*----------------------------------------------------------------------*/
+
+static void txx9dmac_dump_regs(struct txx9dmac_chan *dc)
+{
+ if (is_dmac64(dc))
+ dev_err(chan2dev(&dc->chan),
+ " CHAR: %#llx SAR: %#llx DAR: %#llx CNTR: %#x"
+ " SAIR: %#x DAIR: %#x CCR: %#x CSR: %#x\n",
+ (u64)channel64_read_CHAR(dc),
+ channel64_readq(dc, SAR),
+ channel64_readq(dc, DAR),
+ channel64_readl(dc, CNTR),
+ channel64_readl(dc, SAIR),
+ channel64_readl(dc, DAIR),
+ channel64_readl(dc, CCR),
+ channel64_readl(dc, CSR));
+ else
+ dev_err(chan2dev(&dc->chan),
+ " CHAR: %#x SAR: %#x DAR: %#x CNTR: %#x"
+ " SAIR: %#x DAIR: %#x CCR: %#x CSR: %#x\n",
+ channel32_readl(dc, CHAR),
+ channel32_readl(dc, SAR),
+ channel32_readl(dc, DAR),
+ channel32_readl(dc, CNTR),
+ channel32_readl(dc, SAIR),
+ channel32_readl(dc, DAIR),
+ channel32_readl(dc, CCR),
+ channel32_readl(dc, CSR));
+}
+
+static void txx9dmac_reset_chan(struct txx9dmac_chan *dc)
+{
+ channel_writel(dc, CCR, TXX9_DMA_CCR_CHRST);
+ if (is_dmac64(dc)) {
+ channel64_clear_CHAR(dc);
+ channel_writeq(dc, SAR, 0);
+ channel_writeq(dc, DAR, 0);
+ } else {
+ channel_writel(dc, CHAR, 0);
+ channel_writel(dc, SAR, 0);
+ channel_writel(dc, DAR, 0);
+ }
+ channel_writel(dc, CNTR, 0);
+ channel_writel(dc, SAIR, 0);
+ channel_writel(dc, DAIR, 0);
+ channel_writel(dc, CCR, 0);
+ mmiowb();
+}
+
+/* Called with dc->lock held and bh disabled */
+static void txx9dmac_dostart(struct txx9dmac_chan *dc,
+ struct txx9dmac_desc *first)
+{
+ struct txx9dmac_slave *ds = dc->chan.private;
+ u32 sai, dai;
+
+ dev_vdbg(chan2dev(&dc->chan), "dostart %u %p\n",
+ first->txd.cookie, first);
+ /* ASSERT: channel is idle */
+ if (channel_readl(dc, CSR) & TXX9_DMA_CSR_XFACT) {
+ dev_err(chan2dev(&dc->chan),
+ "BUG: Attempted to start non-idle channel\n");
+ txx9dmac_dump_regs(dc);
+ /* The tasklet will hopefully advance the queue... */
+ return;
+ }
+
+ if (is_dmac64(dc)) {
+ channel64_writel(dc, CNTR, 0);
+ channel64_writel(dc, CSR, 0xffffffff);
+ if (ds) {
+ if (ds->tx_reg) {
+ sai = ds->reg_width;
+ dai = 0;
+ } else {
+ sai = 0;
+ dai = ds->reg_width;
+ }
+ } else {
+ sai = 8;
+ dai = 8;
+ }
+ channel64_writel(dc, SAIR, sai);
+ channel64_writel(dc, DAIR, dai);
+ /* All 64-bit DMAC supports SMPCHN */
+ channel64_writel(dc, CCR, dc->ccr);
+ /* Writing a non zero value to CHAR will assert XFACT */
+ channel64_write_CHAR(dc, first->txd.phys);
+ } else {
+ channel32_writel(dc, CNTR, 0);
+ channel32_writel(dc, CSR, 0xffffffff);
+ if (ds) {
+ if (ds->tx_reg) {
+ sai = ds->reg_width;
+ dai = 0;
+ } else {
+ sai = 0;
+ dai = ds->reg_width;
+ }
+ } else {
+ sai = 4;
+ dai = 4;
+ }
+ channel32_writel(dc, SAIR, sai);
+ channel32_writel(dc, DAIR, dai);
+ if (txx9_dma_have_SMPCHN()) {
+ channel32_writel(dc, CCR, dc->ccr);
+ /* Writing a non zero value to CHAR will assert XFACT */
+ channel32_writel(dc, CHAR, first->txd.phys);
+ } else {
+ channel32_writel(dc, CHAR, first->txd.phys);
+ channel32_writel(dc, CCR, dc->ccr);
+ }
+ }
+}
+
+/*----------------------------------------------------------------------*/
+
+static void
+txx9dmac_descriptor_complete(struct txx9dmac_chan *dc,
+ struct txx9dmac_desc *desc)
+{
+ dma_async_tx_callback callback;
+ void *param;
+ struct dma_async_tx_descriptor *txd = &desc->txd;
+ struct txx9dmac_slave *ds = dc->chan.private;
+
+ dev_vdbg(chan2dev(&dc->chan), "descriptor %u %p complete\n",
+ txd->cookie, desc);
+
+ dc->completed = txd->cookie;
+ callback = txd->callback;
+ param = txd->callback_param;
+
+ txx9dmac_sync_desc_for_cpu(dc, desc);
+ list_splice_init(&desc->tx_list, &dc->free_list);
+ list_move(&desc->desc_node, &dc->free_list);
+
+ if (!ds) {
+ dma_addr_t dmaaddr;
+ if (!(txd->flags & DMA_COMPL_SKIP_DEST_UNMAP)) {
+ dmaaddr = is_dmac64(dc) ?
+ desc->hwdesc.DAR : desc->hwdesc32.DAR;
+ if (txd->flags & DMA_COMPL_DEST_UNMAP_SINGLE)
+ dma_unmap_single(chan2parent(&dc->chan),
+ dmaaddr, desc->len, DMA_FROM_DEVICE);
+ else
+ dma_unmap_page(chan2parent(&dc->chan),
+ dmaaddr, desc->len, DMA_FROM_DEVICE);
+ }
+ if (!(txd->flags & DMA_COMPL_SKIP_SRC_UNMAP)) {
+ dmaaddr = is_dmac64(dc) ?
+ desc->hwdesc.SAR : desc->hwdesc32.SAR;
+ if (txd->flags & DMA_COMPL_SRC_UNMAP_SINGLE)
+ dma_unmap_single(chan2parent(&dc->chan),
+ dmaaddr, desc->len, DMA_TO_DEVICE);
+ else
+ dma_unmap_page(chan2parent(&dc->chan),
+ dmaaddr, desc->len, DMA_TO_DEVICE);
+ }
+ }
+
+ /*
+ * The API requires that no submissions are done from a
+ * callback, so we don't need to drop the lock here
+ */
+ if (callback)
+ callback(param);
+ dma_run_dependencies(txd);
+}
+
+static void txx9dmac_dequeue(struct txx9dmac_chan *dc, struct list_head *list)
+{
+ struct txx9dmac_dev *ddev = dc->ddev;
+ struct txx9dmac_desc *desc;
+ struct txx9dmac_desc *prev = NULL;
+
+ BUG_ON(!list_empty(list));
+ do {
+ desc = txx9dmac_first_queued(dc);
+ if (prev) {
+ desc_write_CHAR(dc, prev, desc->txd.phys);
+ dma_sync_single_for_device(chan2parent(&dc->chan),
+ prev->txd.phys, ddev->descsize,
+ DMA_TO_DEVICE);
+ }
+ prev = txx9dmac_last_child(desc);
+ list_move_tail(&desc->desc_node, list);
+ /* Make chain-completion interrupt happen */
+ if ((desc->txd.flags & DMA_PREP_INTERRUPT) &&
+ !txx9dmac_chan_INTENT(dc))
+ break;
+ } while (!list_empty(&dc->queue));
+}
+
+static void txx9dmac_complete_all(struct txx9dmac_chan *dc)
+{
+ struct txx9dmac_desc *desc, *_desc;
+ LIST_HEAD(list);
+
+ /*
+ * Submit queued descriptors ASAP, i.e. before we go through
+ * the completed ones.
+ */
+ list_splice_init(&dc->active_list, &list);
+ if (!list_empty(&dc->queue)) {
+ txx9dmac_dequeue(dc, &dc->active_list);
+ txx9dmac_dostart(dc, txx9dmac_first_active(dc));
+ }
+
+ list_for_each_entry_safe(desc, _desc, &list, desc_node)
+ txx9dmac_descriptor_complete(dc, desc);
+}
+
+static void txx9dmac_dump_desc(struct txx9dmac_chan *dc,
+ struct txx9dmac_hwdesc *desc)
+{
+ if (is_dmac64(dc)) {
+#ifdef TXX9_DMA_USE_SIMPLE_CHAIN
+ dev_crit(chan2dev(&dc->chan),
+ " desc: ch%#llx s%#llx d%#llx c%#x\n",
+ (u64)desc->CHAR, desc->SAR, desc->DAR, desc->CNTR);
+#else
+ dev_crit(chan2dev(&dc->chan),
+ " desc: ch%#llx s%#llx d%#llx c%#x"
+ " si%#x di%#x cc%#x cs%#x\n",
+ (u64)desc->CHAR, desc->SAR, desc->DAR, desc->CNTR,
+ desc->SAIR, desc->DAIR, desc->CCR, desc->CSR);
+#endif
+ } else {
+ struct txx9dmac_hwdesc32 *d = (struct txx9dmac_hwdesc32 *)desc;
+#ifdef TXX9_DMA_USE_SIMPLE_CHAIN
+ dev_crit(chan2dev(&dc->chan),
+ " desc: ch%#x s%#x d%#x c%#x\n",
+ d->CHAR, d->SAR, d->DAR, d->CNTR);
+#else
+ dev_crit(chan2dev(&dc->chan),
+ " desc: ch%#x s%#x d%#x c%#x"
+ " si%#x di%#x cc%#x cs%#x\n",
+ d->CHAR, d->SAR, d->DAR, d->CNTR,
+ d->SAIR, d->DAIR, d->CCR, d->CSR);
+#endif
+ }
+}
+
+static void txx9dmac_handle_error(struct txx9dmac_chan *dc, u32 csr)
+{
+ struct txx9dmac_desc *bad_desc;
+ struct txx9dmac_desc *child;
+ u32 errors;
+
+ /*
+ * The descriptor currently at the head of the active list is
+ * borked. Since we don't have any way to report errors, we'll
+ * just have to scream loudly and try to carry on.
+ */
+ dev_crit(chan2dev(&dc->chan), "Abnormal Chain Completion\n");
+ txx9dmac_dump_regs(dc);
+
+ bad_desc = txx9dmac_first_active(dc);
+ list_del_init(&bad_desc->desc_node);
+
+ /* Clear all error flags and try to restart the controller */
+ errors = csr & (TXX9_DMA_CSR_ABCHC |
+ TXX9_DMA_CSR_CFERR | TXX9_DMA_CSR_CHERR |
+ TXX9_DMA_CSR_DESERR | TXX9_DMA_CSR_SORERR);
+ channel_writel(dc, CSR, errors);
+
+ if (list_empty(&dc->active_list) && !list_empty(&dc->queue))
+ txx9dmac_dequeue(dc, &dc->active_list);
+ if (!list_empty(&dc->active_list))
+ txx9dmac_dostart(dc, txx9dmac_first_active(dc));
+
+ dev_crit(chan2dev(&dc->chan),
+ "Bad descriptor submitted for DMA! (cookie: %d)\n",
+ bad_desc->txd.cookie);
+ txx9dmac_dump_desc(dc, &bad_desc->hwdesc);
+ list_for_each_entry(child, &bad_desc->tx_list, desc_node)
+ txx9dmac_dump_desc(dc, &child->hwdesc);
+ /* Pretend the descriptor completed successfully */
+ txx9dmac_descriptor_complete(dc, bad_desc);
+}
+
+static void txx9dmac_scan_descriptors(struct txx9dmac_chan *dc)
+{
+ dma_addr_t chain;
+ struct txx9dmac_desc *desc, *_desc;
+ struct txx9dmac_desc *child;
+ u32 csr;
+
+ if (is_dmac64(dc)) {
+ chain = channel64_read_CHAR(dc);
+ csr = channel64_readl(dc, CSR);
+ channel64_writel(dc, CSR, csr);
+ } else {
+ chain = channel32_readl(dc, CHAR);
+ csr = channel32_readl(dc, CSR);
+ channel32_writel(dc, CSR, csr);
+ }
+ /* For dynamic chain, we should look at XFACT instead of NCHNC */
+ if (!(csr & (TXX9_DMA_CSR_XFACT | TXX9_DMA_CSR_ABCHC))) {
+ /* Everything we've submitted is done */
+ txx9dmac_complete_all(dc);
+ return;
+ }
+ if (!(csr & TXX9_DMA_CSR_CHNEN))
+ chain = 0; /* last descriptor of this chain */
+
+ dev_vdbg(chan2dev(&dc->chan), "scan_descriptors: char=%#llx\n",
+ (u64)chain);
+
+ list_for_each_entry_safe(desc, _desc, &dc->active_list, desc_node) {
+ if (desc_read_CHAR(dc, desc) == chain) {
+ /* This one is currently in progress */
+ if (csr & TXX9_DMA_CSR_ABCHC)
+ goto scan_done;
+ return;
+ }
+
+ list_for_each_entry(child, &desc->tx_list, desc_node)
+ if (desc_read_CHAR(dc, child) == chain) {
+ /* Currently in progress */
+ if (csr & TXX9_DMA_CSR_ABCHC)
+ goto scan_done;
+ return;
+ }
+
+ /*
+ * No descriptors so far seem to be in progress, i.e.
+ * this one must be done.
+ */
+ txx9dmac_descriptor_complete(dc, desc);
+ }
+scan_done:
+ if (csr & TXX9_DMA_CSR_ABCHC) {
+ txx9dmac_handle_error(dc, csr);
+ return;
+ }
+
+ dev_err(chan2dev(&dc->chan),
+ "BUG: All descriptors done, but channel not idle!\n");
+
+ /* Try to continue after resetting the channel... */
+ txx9dmac_reset_chan(dc);
+
+ if (!list_empty(&dc->queue)) {
+ txx9dmac_dequeue(dc, &dc->active_list);
+ txx9dmac_dostart(dc, txx9dmac_first_active(dc));
+ }
+}
+
+static void txx9dmac_chan_tasklet(unsigned long data)
+{
+ int irq;
+ u32 csr;
+ struct txx9dmac_chan *dc;
+
+ dc = (struct txx9dmac_chan *)data;
+ csr = channel_readl(dc, CSR);
+ dev_vdbg(chan2dev(&dc->chan), "tasklet: status=%x\n", csr);
+
+ spin_lock(&dc->lock);
+ if (csr & (TXX9_DMA_CSR_ABCHC | TXX9_DMA_CSR_NCHNC |
+ TXX9_DMA_CSR_NTRNFC))
+ txx9dmac_scan_descriptors(dc);
+ spin_unlock(&dc->lock);
+ irq = dc->irq;
+
+ enable_irq(irq);
+}
+
+static irqreturn_t txx9dmac_chan_interrupt(int irq, void *dev_id)
+{
+ struct txx9dmac_chan *dc = dev_id;
+
+ dev_vdbg(chan2dev(&dc->chan), "interrupt: status=%#x\n",
+ channel_readl(dc, CSR));
+
+ tasklet_schedule(&dc->tasklet);
+ /*
+ * Just disable the interrupts. We'll turn them back on in the
+ * softirq handler.
+ */
+ disable_irq_nosync(irq);
+
+ return IRQ_HANDLED;
+}
+
+static void txx9dmac_tasklet(unsigned long data)
+{
+ int irq;
+ u32 csr;
+ struct txx9dmac_chan *dc;
+
+ struct txx9dmac_dev *ddev = (struct txx9dmac_dev *)data;
+ u32 mcr;
+ int i;
+
+ mcr = dma_readl(ddev, MCR);
+ dev_vdbg(ddev->chan[0]->dma.dev, "tasklet: mcr=%x\n", mcr);
+ for (i = 0; i < TXX9_DMA_MAX_NR_CHANNELS; i++) {
+ if ((mcr >> (24 + i)) & 0x11) {
+ dc = ddev->chan[i];
+ csr = channel_readl(dc, CSR);
+ dev_vdbg(chan2dev(&dc->chan), "tasklet: status=%x\n",
+ csr);
+ spin_lock(&dc->lock);
+ if (csr & (TXX9_DMA_CSR_ABCHC | TXX9_DMA_CSR_NCHNC |
+ TXX9_DMA_CSR_NTRNFC))
+ txx9dmac_scan_descriptors(dc);
+ spin_unlock(&dc->lock);
+ }
+ }
+ irq = ddev->irq;
+
+ enable_irq(irq);
+}
+
+static irqreturn_t txx9dmac_interrupt(int irq, void *dev_id)
+{
+ struct txx9dmac_dev *ddev = dev_id;
+
+ dev_vdbg(ddev->chan[0]->dma.dev, "interrupt: status=%#x\n",
+ dma_readl(ddev, MCR));
+
+ tasklet_schedule(&ddev->tasklet);
+ /*
+ * Just disable the interrupts. We'll turn them back on in the
+ * softirq handler.
+ */
+ disable_irq_nosync(irq);
+
+ return IRQ_HANDLED;
+}
+
+/*----------------------------------------------------------------------*/
+
+static dma_cookie_t txx9dmac_tx_submit(struct dma_async_tx_descriptor *tx)
+{
+ struct txx9dmac_desc *desc = txd_to_txx9dmac_desc(tx);
+ struct txx9dmac_chan *dc = to_txx9dmac_chan(tx->chan);
+ dma_cookie_t cookie;
+
+ spin_lock_bh(&dc->lock);
+ cookie = txx9dmac_assign_cookie(dc, desc);
+
+ dev_vdbg(chan2dev(tx->chan), "tx_submit: queued %u %p\n",
+ desc->txd.cookie, desc);
+
+ list_add_tail(&desc->desc_node, &dc->queue);
+ spin_unlock_bh(&dc->lock);
+
+ return cookie;
+}
+
+static struct dma_async_tx_descriptor *
+txx9dmac_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src,
+ size_t len, unsigned long flags)
+{
+ struct txx9dmac_chan *dc = to_txx9dmac_chan(chan);
+ struct txx9dmac_dev *ddev = dc->ddev;
+ struct txx9dmac_desc *desc;
+ struct txx9dmac_desc *first;
+ struct txx9dmac_desc *prev;
+ size_t xfer_count;
+ size_t offset;
+
+ dev_vdbg(chan2dev(chan), "prep_dma_memcpy d%#llx s%#llx l%#zx f%#lx\n",
+ (u64)dest, (u64)src, len, flags);
+
+ if (unlikely(!len)) {
+ dev_dbg(chan2dev(chan), "prep_dma_memcpy: length is zero!\n");
+ return NULL;
+ }
+
+ prev = first = NULL;
+
+ for (offset = 0; offset < len; offset += xfer_count) {
+ xfer_count = min_t(size_t, len - offset, TXX9_DMA_MAX_COUNT);
+ /*
+ * Workaround for ERT-TX49H2-033, ERT-TX49H3-020,
+ * ERT-TX49H4-016 (slightly conservative)
+ */
+ if (__is_dmac64(ddev)) {
+ if (xfer_count > 0x100 &&
+ (xfer_count & 0xff) >= 0xfa &&
+ (xfer_count & 0xff) <= 0xff)
+ xfer_count -= 0x20;
+ } else {
+ if (xfer_count > 0x80 &&
+ (xfer_count & 0x7f) >= 0x7e &&
+ (xfer_count & 0x7f) <= 0x7f)
+ xfer_count -= 0x20;
+ }
+
+ desc = txx9dmac_desc_get(dc);
+ if (!desc) {
+ txx9dmac_desc_put(dc, first);
+ return NULL;
+ }
+
+ if (__is_dmac64(ddev)) {
+ desc->hwdesc.SAR = src + offset;
+ desc->hwdesc.DAR = dest + offset;
+ desc->hwdesc.CNTR = xfer_count;
+ txx9dmac_desc_set_nosimple(ddev, desc, 8, 8,
+ dc->ccr | TXX9_DMA_CCR_XFACT);
+ } else {
+ desc->hwdesc32.SAR = src + offset;
+ desc->hwdesc32.DAR = dest + offset;
+ desc->hwdesc32.CNTR = xfer_count;
+ txx9dmac_desc_set_nosimple(ddev, desc, 4, 4,
+ dc->ccr | TXX9_DMA_CCR_XFACT);
+ }
+
+ /*
+ * The descriptors on tx_list are not reachable from
+ * the dc->queue list or dc->active_list after a
+ * submit. If we put all descriptors on active_list,
+ * calling of callback on the completion will be more
+ * complex.
+ */
+ if (!first) {
+ first = desc;
+ } else {
+ desc_write_CHAR(dc, prev, desc->txd.phys);
+ dma_sync_single_for_device(chan2parent(&dc->chan),
+ prev->txd.phys, ddev->descsize,
+ DMA_TO_DEVICE);
+ list_add_tail(&desc->desc_node, &first->tx_list);
+ }
+ prev = desc;
+ }
+
+ /* Trigger interrupt after last block */
+ if (flags & DMA_PREP_INTERRUPT)
+ txx9dmac_desc_set_INTENT(ddev, prev);
+
+ desc_write_CHAR(dc, prev, 0);
+ dma_sync_single_for_device(chan2parent(&dc->chan),
+ prev->txd.phys, ddev->descsize,
+ DMA_TO_DEVICE);
+
+ first->txd.flags = flags;
+ first->len = len;
+
+ return &first->txd;
+}
+
+static struct dma_async_tx_descriptor *
+txx9dmac_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,
+ unsigned int sg_len, enum dma_data_direction direction,
+ unsigned long flags)
+{
+ struct txx9dmac_chan *dc = to_txx9dmac_chan(chan);
+ struct txx9dmac_dev *ddev = dc->ddev;
+ struct txx9dmac_slave *ds = chan->private;
+ struct txx9dmac_desc *prev;
+ struct txx9dmac_desc *first;
+ unsigned int i;
+ struct scatterlist *sg;
+
+ dev_vdbg(chan2dev(chan), "prep_dma_slave\n");
+
+ BUG_ON(!ds || !ds->reg_width);
+ if (ds->tx_reg)
+ BUG_ON(direction != DMA_TO_DEVICE);
+ else
+ BUG_ON(direction != DMA_FROM_DEVICE);
+ if (unlikely(!sg_len))
+ return NULL;
+
+ prev = first = NULL;
+
+ for_each_sg(sgl, sg, sg_len, i) {
+ struct txx9dmac_desc *desc;
+ dma_addr_t mem;
+ u32 sai, dai;
+
+ desc = txx9dmac_desc_get(dc);
+ if (!desc) {
+ txx9dmac_desc_put(dc, first);
+ return NULL;
+ }
+
+ mem = sg_dma_address(sg);
+
+ if (__is_dmac64(ddev)) {
+ if (direction == DMA_TO_DEVICE) {
+ desc->hwdesc.SAR = mem;
+ desc->hwdesc.DAR = ds->tx_reg;
+ } else {
+ desc->hwdesc.SAR = ds->rx_reg;
+ desc->hwdesc.DAR = mem;
+ }
+ desc->hwdesc.CNTR = sg_dma_len(sg);
+ } else {
+ if (direction == DMA_TO_DEVICE) {
+ desc->hwdesc32.SAR = mem;
+ desc->hwdesc32.DAR = ds->tx_reg;
+ } else {
+ desc->hwdesc32.SAR = ds->rx_reg;
+ desc->hwdesc32.DAR = mem;
+ }
+ desc->hwdesc32.CNTR = sg_dma_len(sg);
+ }
+ if (direction == DMA_TO_DEVICE) {
+ sai = ds->reg_width;
+ dai = 0;
+ } else {
+ sai = 0;
+ dai = ds->reg_width;
+ }
+ txx9dmac_desc_set_nosimple(ddev, desc, sai, dai,
+ dc->ccr | TXX9_DMA_CCR_XFACT);
+
+ if (!first) {
+ first = desc;
+ } else {
+ desc_write_CHAR(dc, prev, desc->txd.phys);
+ dma_sync_single_for_device(chan2parent(&dc->chan),
+ prev->txd.phys,
+ ddev->descsize,
+ DMA_TO_DEVICE);
+ list_add_tail(&desc->desc_node, &first->tx_list);
+ }
+ prev = desc;
+ }
+
+ /* Trigger interrupt after last block */
+ if (flags & DMA_PREP_INTERRUPT)
+ txx9dmac_desc_set_INTENT(ddev, prev);
+
+ desc_write_CHAR(dc, prev, 0);
+ dma_sync_single_for_device(chan2parent(&dc->chan),
+ prev->txd.phys, ddev->descsize,
+ DMA_TO_DEVICE);
+
+ first->txd.flags = flags;
+ first->len = 0;
+
+ return &first->txd;
+}
+
+static void txx9dmac_terminate_all(struct dma_chan *chan)
+{
+ struct txx9dmac_chan *dc = to_txx9dmac_chan(chan);
+ struct txx9dmac_desc *desc, *_desc;
+ LIST_HEAD(list);
+
+ dev_vdbg(chan2dev(chan), "terminate_all\n");
+ spin_lock_bh(&dc->lock);
+
+ txx9dmac_reset_chan(dc);
+
+ /* active_list entries will end up before queued entries */
+ list_splice_init(&dc->queue, &list);
+ list_splice_init(&dc->active_list, &list);
+
+ spin_unlock_bh(&dc->lock);
+
+ /* Flush all pending and queued descriptors */
+ list_for_each_entry_safe(desc, _desc, &list, desc_node)
+ txx9dmac_descriptor_complete(dc, desc);
+}
+
+static enum dma_status
+txx9dmac_is_tx_complete(struct dma_chan *chan,
+ dma_cookie_t cookie,
+ dma_cookie_t *done, dma_cookie_t *used)
+{
+ struct txx9dmac_chan *dc = to_txx9dmac_chan(chan);
+ dma_cookie_t last_used;
+ dma_cookie_t last_complete;
+ int ret;
+
+ last_complete = dc->completed;
+ last_used = chan->cookie;
+
+ ret = dma_async_is_complete(cookie, last_complete, last_used);
+ if (ret != DMA_SUCCESS) {
+ spin_lock_bh(&dc->lock);
+ txx9dmac_scan_descriptors(dc);
+ spin_unlock_bh(&dc->lock);
+
+ last_complete = dc->completed;
+ last_used = chan->cookie;
+
+ ret = dma_async_is_complete(cookie, last_complete, last_used);
+ }
+
+ if (done)
+ *done = last_complete;
+ if (used)
+ *used = last_used;
+
+ return ret;
+}
+
+static void txx9dmac_chain_dynamic(struct txx9dmac_chan *dc,
+ struct txx9dmac_desc *prev)
+{
+ struct txx9dmac_dev *ddev = dc->ddev;
+ struct txx9dmac_desc *desc;
+ LIST_HEAD(list);
+
+ prev = txx9dmac_last_child(prev);
+ txx9dmac_dequeue(dc, &list);
+ desc = list_entry(list.next, struct txx9dmac_desc, desc_node);
+ desc_write_CHAR(dc, prev, desc->txd.phys);
+ dma_sync_single_for_device(chan2parent(&dc->chan),
+ prev->txd.phys, ddev->descsize,
+ DMA_TO_DEVICE);
+ mmiowb();
+ if (!(channel_readl(dc, CSR) & TXX9_DMA_CSR_CHNEN) &&
+ channel_read_CHAR(dc) == prev->txd.phys)
+ /* Restart chain DMA */
+ channel_write_CHAR(dc, desc->txd.phys);
+ list_splice_tail(&list, &dc->active_list);
+}
+
+static void txx9dmac_issue_pending(struct dma_chan *chan)
+{
+ struct txx9dmac_chan *dc = to_txx9dmac_chan(chan);
+
+ spin_lock_bh(&dc->lock);
+
+ if (!list_empty(&dc->active_list))
+ txx9dmac_scan_descriptors(dc);
+ if (!list_empty(&dc->queue)) {
+ if (list_empty(&dc->active_list)) {
+ txx9dmac_dequeue(dc, &dc->active_list);
+ txx9dmac_dostart(dc, txx9dmac_first_active(dc));
+ } else if (txx9_dma_have_SMPCHN()) {
+ struct txx9dmac_desc *prev = txx9dmac_last_active(dc);
+
+ if (!(prev->txd.flags & DMA_PREP_INTERRUPT) ||
+ txx9dmac_chan_INTENT(dc))
+ txx9dmac_chain_dynamic(dc, prev);
+ }
+ }
+
+ spin_unlock_bh(&dc->lock);
+}
+
+static int txx9dmac_alloc_chan_resources(struct dma_chan *chan)
+{
+ struct txx9dmac_chan *dc = to_txx9dmac_chan(chan);
+ struct txx9dmac_slave *ds = chan->private;
+ struct txx9dmac_desc *desc;
+ int i;
+
+ dev_vdbg(chan2dev(chan), "alloc_chan_resources\n");
+
+ /* ASSERT: channel is idle */
+ if (channel_readl(dc, CSR) & TXX9_DMA_CSR_XFACT) {
+ dev_dbg(chan2dev(chan), "DMA channel not idle?\n");
+ return -EIO;
+ }
+
+ dc->completed = chan->cookie = 1;
+
+ dc->ccr = TXX9_DMA_CCR_IMMCHN | TXX9_DMA_CCR_INTENE | CCR_LE;
+ txx9dmac_chan_set_SMPCHN(dc);
+ if (!txx9_dma_have_SMPCHN() || (dc->ccr & TXX9_DMA_CCR_SMPCHN))
+ dc->ccr |= TXX9_DMA_CCR_INTENC;
+ if (chan->device->device_prep_dma_memcpy) {
+ if (ds)
+ return -EINVAL;
+ dc->ccr |= TXX9_DMA_CCR_XFSZ_X8;
+ } else {
+ if (!ds ||
+ (ds->tx_reg && ds->rx_reg) || (!ds->tx_reg && !ds->rx_reg))
+ return -EINVAL;
+ dc->ccr |= TXX9_DMA_CCR_EXTRQ |
+ TXX9_DMA_CCR_XFSZ(__ffs(ds->reg_width));
+ txx9dmac_chan_set_INTENT(dc);
+ }
+
+ spin_lock_bh(&dc->lock);
+ i = dc->descs_allocated;
+ while (dc->descs_allocated < TXX9_DMA_INITIAL_DESC_COUNT) {
+ spin_unlock_bh(&dc->lock);
+
+ desc = txx9dmac_desc_alloc(dc, GFP_KERNEL);
+ if (!desc) {
+ dev_info(chan2dev(chan),
+ "only allocated %d descriptors\n", i);
+ spin_lock_bh(&dc->lock);
+ break;
+ }
+ txx9dmac_desc_put(dc, desc);
+
+ spin_lock_bh(&dc->lock);
+ i = ++dc->descs_allocated;
+ }
+ spin_unlock_bh(&dc->lock);
+
+ dev_dbg(chan2dev(chan),
+ "alloc_chan_resources allocated %d descriptors\n", i);
+
+ return i;
+}
+
+static void txx9dmac_free_chan_resources(struct dma_chan *chan)
+{
+ struct txx9dmac_chan *dc = to_txx9dmac_chan(chan);
+ struct txx9dmac_dev *ddev = dc->ddev;
+ struct txx9dmac_desc *desc, *_desc;
+ LIST_HEAD(list);
+
+ dev_dbg(chan2dev(chan), "free_chan_resources (descs allocated=%u)\n",
+ dc->descs_allocated);
+
+ /* ASSERT: channel is idle */
+ BUG_ON(!list_empty(&dc->active_list));
+ BUG_ON(!list_empty(&dc->queue));
+ BUG_ON(channel_readl(dc, CSR) & TXX9_DMA_CSR_XFACT);
+
+ spin_lock_bh(&dc->lock);
+ list_splice_init(&dc->free_list, &list);
+ dc->descs_allocated = 0;
+ spin_unlock_bh(&dc->lock);
+
+ list_for_each_entry_safe(desc, _desc, &list, desc_node) {
+ dev_vdbg(chan2dev(chan), " freeing descriptor %p\n", desc);
+ dma_unmap_single(chan2parent(chan), desc->txd.phys,
+ ddev->descsize, DMA_TO_DEVICE);
+ kfree(desc);
+ }
+
+ dev_vdbg(chan2dev(chan), "free_chan_resources done\n");
+}
+
+/*----------------------------------------------------------------------*/
+
+static void txx9dmac_off(struct txx9dmac_dev *ddev)
+{
+ dma_writel(ddev, MCR, 0);
+ mmiowb();
+}
+
+static int __init txx9dmac_chan_probe(struct platform_device *pdev)
+{
+ struct txx9dmac_chan_platform_data *cpdata = pdev->dev.platform_data;
+ struct platform_device *dmac_dev = cpdata->dmac_dev;
+ struct txx9dmac_platform_data *pdata = dmac_dev->dev.platform_data;
+ struct txx9dmac_chan *dc;
+ int err;
+ int ch = pdev->id % TXX9_DMA_MAX_NR_CHANNELS;
+ int irq;
+
+ dc = devm_kzalloc(&pdev->dev, sizeof(*dc), GFP_KERNEL);
+ if (!dc)
+ return -ENOMEM;
+
+ dc->dma.dev = &pdev->dev;
+ dc->dma.device_alloc_chan_resources = txx9dmac_alloc_chan_resources;
+ dc->dma.device_free_chan_resources = txx9dmac_free_chan_resources;
+ dc->dma.device_terminate_all = txx9dmac_terminate_all;
+ dc->dma.device_is_tx_complete = txx9dmac_is_tx_complete;
+ dc->dma.device_issue_pending = txx9dmac_issue_pending;
+ if (pdata && pdata->memcpy_chan == ch) {
+ dc->dma.device_prep_dma_memcpy = txx9dmac_prep_dma_memcpy;
+ dma_cap_set(DMA_MEMCPY, dc->dma.cap_mask);
+ } else {
+ dc->dma.device_prep_slave_sg = txx9dmac_prep_slave_sg;
+ dma_cap_set(DMA_SLAVE, dc->dma.cap_mask);
+ dma_cap_set(DMA_PRIVATE, dc->dma.cap_mask);
+ }
+
+ INIT_LIST_HEAD(&dc->dma.channels);
+ dc->ddev = platform_get_drvdata(dmac_dev);
+ if (dc->ddev->irq < 0) {
+ irq = platform_get_irq(pdev, 0);
+ if (irq < 0)
+ return irq;
+ tasklet_init(&dc->tasklet, txx9dmac_chan_tasklet,
+ (unsigned long)dc);
+ dc->irq = irq;
+ err = devm_request_irq(&pdev->dev, dc->irq,
+ txx9dmac_chan_interrupt, 0, dev_name(&pdev->dev), dc);
+ if (err)
+ return err;
+ } else
+ dc->irq = -1;
+ dc->ddev->chan[ch] = dc;
+ dc->chan.device = &dc->dma;
+ list_add_tail(&dc->chan.device_node, &dc->chan.device->channels);
+ dc->chan.cookie = dc->completed = 1;
+
+ if (is_dmac64(dc))
+ dc->ch_regs = &__txx9dmac_regs(dc->ddev)->CHAN[ch];
+ else
+ dc->ch_regs = &__txx9dmac_regs32(dc->ddev)->CHAN[ch];
+ spin_lock_init(&dc->lock);
+
+ INIT_LIST_HEAD(&dc->active_list);
+ INIT_LIST_HEAD(&dc->queue);
+ INIT_LIST_HEAD(&dc->free_list);
+
+ txx9dmac_reset_chan(dc);
+
+ platform_set_drvdata(pdev, dc);
+
+ err = dma_async_device_register(&dc->dma);
+ if (err)
+ return err;
+ dev_dbg(&pdev->dev, "TXx9 DMA Channel (dma%d%s%s)\n",
+ dc->dma.dev_id,
+ dma_has_cap(DMA_MEMCPY, dc->dma.cap_mask) ? " memcpy" : "",
+ dma_has_cap(DMA_SLAVE, dc->dma.cap_mask) ? " slave" : "");
+
+ return 0;
+}
+
+static int __exit txx9dmac_chan_remove(struct platform_device *pdev)
+{
+ struct txx9dmac_chan *dc = platform_get_drvdata(pdev);
+
+ dma_async_device_unregister(&dc->dma);
+ if (dc->irq >= 0)
+ tasklet_kill(&dc->tasklet);
+ dc->ddev->chan[pdev->id % TXX9_DMA_MAX_NR_CHANNELS] = NULL;
+ return 0;
+}
+
+static int __init txx9dmac_probe(struct platform_device *pdev)
+{
+ struct txx9dmac_platform_data *pdata = pdev->dev.platform_data;
+ struct resource *io;
+ struct txx9dmac_dev *ddev;
+ u32 mcr;
+ int err;
+
+ io = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+ if (!io)
+ return -EINVAL;
+
+ ddev = devm_kzalloc(&pdev->dev, sizeof(*ddev), GFP_KERNEL);
+ if (!ddev)
+ return -ENOMEM;
+
+ if (!devm_request_mem_region(&pdev->dev, io->start, resource_size(io),
+ dev_name(&pdev->dev)))
+ return -EBUSY;
+
+ ddev->regs = devm_ioremap(&pdev->dev, io->start, resource_size(io));
+ if (!ddev->regs)
+ return -ENOMEM;
+ ddev->have_64bit_regs = pdata->have_64bit_regs;
+ if (__is_dmac64(ddev))
+ ddev->descsize = sizeof(struct txx9dmac_hwdesc);
+ else
+ ddev->descsize = sizeof(struct txx9dmac_hwdesc32);
+
+ /* force dma off, just in case */
+ txx9dmac_off(ddev);
+
+ ddev->irq = platform_get_irq(pdev, 0);
+ if (ddev->irq >= 0) {
+ tasklet_init(&ddev->tasklet, txx9dmac_tasklet,
+ (unsigned long)ddev);
+ err = devm_request_irq(&pdev->dev, ddev->irq,
+ txx9dmac_interrupt, 0, dev_name(&pdev->dev), ddev);
+ if (err)
+ return err;
+ }
+
+ mcr = TXX9_DMA_MCR_MSTEN | MCR_LE;
+ if (pdata && pdata->memcpy_chan >= 0)
+ mcr |= TXX9_DMA_MCR_FIFUM(pdata->memcpy_chan);
+ dma_writel(ddev, MCR, mcr);
+
+ platform_set_drvdata(pdev, ddev);
+ return 0;
+}
+
+static int __exit txx9dmac_remove(struct platform_device *pdev)
+{
+ struct txx9dmac_dev *ddev = platform_get_drvdata(pdev);
+
+ txx9dmac_off(ddev);
+ if (ddev->irq >= 0)
+ tasklet_kill(&ddev->tasklet);
+ return 0;
+}
+
+static void txx9dmac_shutdown(struct platform_device *pdev)
+{
+ struct txx9dmac_dev *ddev = platform_get_drvdata(pdev);
+
+ txx9dmac_off(ddev);
+}
+
+static int txx9dmac_suspend_late(struct platform_device *pdev,
+ pm_message_t mesg)
+{
+ struct txx9dmac_dev *ddev = platform_get_drvdata(pdev);
+
+ txx9dmac_off(ddev);
+ return 0;
+}
+
+static int txx9dmac_resume_early(struct platform_device *pdev)
+{
+ struct txx9dmac_dev *ddev = platform_get_drvdata(pdev);
+ struct txx9dmac_platform_data *pdata = pdev->dev.platform_data;
+ u32 mcr;
+
+ mcr = TXX9_DMA_MCR_MSTEN | MCR_LE;
+ if (pdata && pdata->memcpy_chan >= 0)
+ mcr |= TXX9_DMA_MCR_FIFUM(pdata->memcpy_chan);
+ dma_writel(ddev, MCR, mcr);
+ return 0;
+
+}
+
+static struct platform_driver txx9dmac_chan_driver = {
+ .remove = __exit_p(txx9dmac_chan_remove),
+ .driver = {
+ .name = "txx9dmac-chan",
+ },
+};
+
+static struct platform_driver txx9dmac_driver = {
+ .remove = __exit_p(txx9dmac_remove),
+ .shutdown = txx9dmac_shutdown,
+ .suspend_late = txx9dmac_suspend_late,
+ .resume_early = txx9dmac_resume_early,
+ .driver = {
+ .name = "txx9dmac",
+ },
+};
+
+static int __init txx9dmac_init(void)
+{
+ int rc;
+
+ rc = platform_driver_probe(&txx9dmac_driver, txx9dmac_probe);
+ if (!rc) {
+ rc = platform_driver_probe(&txx9dmac_chan_driver,
+ txx9dmac_chan_probe);
+ if (rc)
+ platform_driver_unregister(&txx9dmac_driver);
+ }
+ return rc;
+}
+module_init(txx9dmac_init);
+
+static void __exit txx9dmac_exit(void)
+{
+ platform_driver_unregister(&txx9dmac_chan_driver);
+ platform_driver_unregister(&txx9dmac_driver);
+}
+module_exit(txx9dmac_exit);
+
+MODULE_LICENSE("GPL");
+MODULE_DESCRIPTION("TXx9 DMA Controller driver");
+MODULE_AUTHOR("Atsushi Nemoto <anemo@mba.ocn.ne.jp>");
diff --git a/drivers/dma/txx9dmac.h b/drivers/dma/txx9dmac.h
new file mode 100644
index 000000000000..365d42366b9f
--- /dev/null
+++ b/drivers/dma/txx9dmac.h
@@ -0,0 +1,308 @@
+/*
+ * Driver for the TXx9 SoC DMA Controller
+ *
+ * Copyright (C) 2009 Atsushi Nemoto
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+#ifndef TXX9DMAC_H
+#define TXX9DMAC_H
+
+#include <linux/dmaengine.h>
+#include <asm/txx9/dmac.h>
+
+/*
+ * Design Notes:
+ *
+ * This DMAC have four channels and one FIFO buffer. Each channel can
+ * be configured for memory-memory or device-memory transfer, but only
+ * one channel can do alignment-free memory-memory transfer at a time
+ * while the channel should occupy the FIFO buffer for effective
+ * transfers.
+ *
+ * Instead of dynamically assign the FIFO buffer to channels, I chose
+ * make one dedicated channel for memory-memory transfer. The
+ * dedicated channel is public. Other channels are private and used
+ * for slave transfer. Some devices in the SoC are wired to certain
+ * DMA channel.
+ */
+
+#ifdef CONFIG_MACH_TX49XX
+static inline bool txx9_dma_have_SMPCHN(void)
+{
+ return true;
+}
+#define TXX9_DMA_USE_SIMPLE_CHAIN
+#else
+static inline bool txx9_dma_have_SMPCHN(void)
+{
+ return false;
+}
+#endif
+
+#ifdef __LITTLE_ENDIAN
+#ifdef CONFIG_MACH_TX49XX
+#define CCR_LE TXX9_DMA_CCR_LE
+#define MCR_LE 0
+#else
+#define CCR_LE 0
+#define MCR_LE TXX9_DMA_MCR_LE
+#endif
+#else
+#define CCR_LE 0
+#define MCR_LE 0
+#endif
+
+/*
+ * Redefine this macro to handle differences between 32- and 64-bit
+ * addressing, big vs. little endian, etc.
+ */
+#ifdef __BIG_ENDIAN
+#define TXX9_DMA_REG32(name) u32 __pad_##name; u32 name
+#else
+#define TXX9_DMA_REG32(name) u32 name; u32 __pad_##name
+#endif
+
+/* Hardware register definitions. */
+struct txx9dmac_cregs {
+#if defined(CONFIG_32BIT) && !defined(CONFIG_64BIT_PHYS_ADDR)
+ TXX9_DMA_REG32(CHAR); /* Chain Address Register */
+#else
+ u64 CHAR; /* Chain Address Register */
+#endif
+ u64 SAR; /* Source Address Register */
+ u64 DAR; /* Destination Address Register */
+ TXX9_DMA_REG32(CNTR); /* Count Register */
+ TXX9_DMA_REG32(SAIR); /* Source Address Increment Register */
+ TXX9_DMA_REG32(DAIR); /* Destination Address Increment Register */
+ TXX9_DMA_REG32(CCR); /* Channel Control Register */
+ TXX9_DMA_REG32(CSR); /* Channel Status Register */
+};
+struct txx9dmac_cregs32 {
+ u32 CHAR;
+ u32 SAR;
+ u32 DAR;
+ u32 CNTR;
+ u32 SAIR;
+ u32 DAIR;
+ u32 CCR;
+ u32 CSR;
+};
+
+struct txx9dmac_regs {
+ /* per-channel registers */
+ struct txx9dmac_cregs CHAN[TXX9_DMA_MAX_NR_CHANNELS];
+ u64 __pad[9];
+ u64 MFDR; /* Memory Fill Data Register */
+ TXX9_DMA_REG32(MCR); /* Master Control Register */
+};
+struct txx9dmac_regs32 {
+ struct txx9dmac_cregs32 CHAN[TXX9_DMA_MAX_NR_CHANNELS];
+ u32 __pad[9];
+ u32 MFDR;
+ u32 MCR;
+};
+
+/* bits for MCR */
+#define TXX9_DMA_MCR_EIS(ch) (0x10000000<<(ch))
+#define TXX9_DMA_MCR_DIS(ch) (0x01000000<<(ch))
+#define TXX9_DMA_MCR_RSFIF 0x00000080
+#define TXX9_DMA_MCR_FIFUM(ch) (0x00000008<<(ch))
+#define TXX9_DMA_MCR_LE 0x00000004
+#define TXX9_DMA_MCR_RPRT 0x00000002
+#define TXX9_DMA_MCR_MSTEN 0x00000001
+
+/* bits for CCRn */
+#define TXX9_DMA_CCR_IMMCHN 0x20000000
+#define TXX9_DMA_CCR_USEXFSZ 0x10000000
+#define TXX9_DMA_CCR_LE 0x08000000
+#define TXX9_DMA_CCR_DBINH 0x04000000
+#define TXX9_DMA_CCR_SBINH 0x02000000
+#define TXX9_DMA_CCR_CHRST 0x01000000
+#define TXX9_DMA_CCR_RVBYTE 0x00800000
+#define TXX9_DMA_CCR_ACKPOL 0x00400000
+#define TXX9_DMA_CCR_REQPL 0x00200000
+#define TXX9_DMA_CCR_EGREQ 0x00100000
+#define TXX9_DMA_CCR_CHDN 0x00080000
+#define TXX9_DMA_CCR_DNCTL 0x00060000
+#define TXX9_DMA_CCR_EXTRQ 0x00010000
+#define TXX9_DMA_CCR_INTRQD 0x0000e000
+#define TXX9_DMA_CCR_INTENE 0x00001000
+#define TXX9_DMA_CCR_INTENC 0x00000800
+#define TXX9_DMA_CCR_INTENT 0x00000400
+#define TXX9_DMA_CCR_CHNEN 0x00000200
+#define TXX9_DMA_CCR_XFACT 0x00000100
+#define TXX9_DMA_CCR_SMPCHN 0x00000020
+#define TXX9_DMA_CCR_XFSZ(order) (((order) << 2) & 0x0000001c)
+#define TXX9_DMA_CCR_XFSZ_1 TXX9_DMA_CCR_XFSZ(0)
+#define TXX9_DMA_CCR_XFSZ_2 TXX9_DMA_CCR_XFSZ(1)
+#define TXX9_DMA_CCR_XFSZ_4 TXX9_DMA_CCR_XFSZ(2)
+#define TXX9_DMA_CCR_XFSZ_8 TXX9_DMA_CCR_XFSZ(3)
+#define TXX9_DMA_CCR_XFSZ_X4 TXX9_DMA_CCR_XFSZ(4)
+#define TXX9_DMA_CCR_XFSZ_X8 TXX9_DMA_CCR_XFSZ(5)
+#define TXX9_DMA_CCR_XFSZ_X16 TXX9_DMA_CCR_XFSZ(6)
+#define TXX9_DMA_CCR_XFSZ_X32 TXX9_DMA_CCR_XFSZ(7)
+#define TXX9_DMA_CCR_MEMIO 0x00000002
+#define TXX9_DMA_CCR_SNGAD 0x00000001
+
+/* bits for CSRn */
+#define TXX9_DMA_CSR_CHNEN 0x00000400
+#define TXX9_DMA_CSR_STLXFER 0x00000200
+#define TXX9_DMA_CSR_XFACT 0x00000100
+#define TXX9_DMA_CSR_ABCHC 0x00000080
+#define TXX9_DMA_CSR_NCHNC 0x00000040
+#define TXX9_DMA_CSR_NTRNFC 0x00000020
+#define TXX9_DMA_CSR_EXTDN 0x00000010
+#define TXX9_DMA_CSR_CFERR 0x00000008
+#define TXX9_DMA_CSR_CHERR 0x00000004
+#define TXX9_DMA_CSR_DESERR 0x00000002
+#define TXX9_DMA_CSR_SORERR 0x00000001
+
+struct txx9dmac_chan {
+ struct dma_chan chan;
+ struct dma_device dma;
+ struct txx9dmac_dev *ddev;
+ void __iomem *ch_regs;
+ struct tasklet_struct tasklet;
+ int irq;
+ u32 ccr;
+
+ spinlock_t lock;
+
+ /* these other elements are all protected by lock */
+ dma_cookie_t completed;
+ struct list_head active_list;
+ struct list_head queue;
+ struct list_head free_list;
+
+ unsigned int descs_allocated;
+};
+
+struct txx9dmac_dev {
+ void __iomem *regs;
+ struct tasklet_struct tasklet;
+ int irq;
+ struct txx9dmac_chan *chan[TXX9_DMA_MAX_NR_CHANNELS];
+ bool have_64bit_regs;
+ unsigned int descsize;
+};
+
+static inline bool __is_dmac64(const struct txx9dmac_dev *ddev)
+{
+ return ddev->have_64bit_regs;
+}
+
+static inline bool is_dmac64(const struct txx9dmac_chan *dc)
+{
+ return __is_dmac64(dc->ddev);
+}
+
+#ifdef TXX9_DMA_USE_SIMPLE_CHAIN
+/* Hardware descriptor definition. (for simple-chain) */
+struct txx9dmac_hwdesc {
+#if defined(CONFIG_32BIT) && !defined(CONFIG_64BIT_PHYS_ADDR)
+ TXX9_DMA_REG32(CHAR);
+#else
+ u64 CHAR;
+#endif
+ u64 SAR;
+ u64 DAR;
+ TXX9_DMA_REG32(CNTR);
+};
+struct txx9dmac_hwdesc32 {
+ u32 CHAR;
+ u32 SAR;
+ u32 DAR;
+ u32 CNTR;
+};
+#else
+#define txx9dmac_hwdesc txx9dmac_cregs
+#define txx9dmac_hwdesc32 txx9dmac_cregs32
+#endif
+
+struct txx9dmac_desc {
+ /* FIRST values the hardware uses */
+ union {
+ struct txx9dmac_hwdesc hwdesc;
+ struct txx9dmac_hwdesc32 hwdesc32;
+ };
+
+ /* THEN values for driver housekeeping */
+ struct list_head desc_node ____cacheline_aligned;
+ struct list_head tx_list;
+ struct dma_async_tx_descriptor txd;
+ size_t len;
+};
+
+#ifdef TXX9_DMA_USE_SIMPLE_CHAIN
+
+static inline bool txx9dmac_chan_INTENT(struct txx9dmac_chan *dc)
+{
+ return (dc->ccr & TXX9_DMA_CCR_INTENT) != 0;
+}
+
+static inline void txx9dmac_chan_set_INTENT(struct txx9dmac_chan *dc)
+{
+ dc->ccr |= TXX9_DMA_CCR_INTENT;
+}
+
+static inline void txx9dmac_desc_set_INTENT(struct txx9dmac_dev *ddev,
+ struct txx9dmac_desc *desc)
+{
+}
+
+static inline void txx9dmac_chan_set_SMPCHN(struct txx9dmac_chan *dc)
+{
+ dc->ccr |= TXX9_DMA_CCR_SMPCHN;
+}
+
+static inline void txx9dmac_desc_set_nosimple(struct txx9dmac_dev *ddev,
+ struct txx9dmac_desc *desc,
+ u32 sair, u32 dair, u32 ccr)
+{
+}
+
+#else /* TXX9_DMA_USE_SIMPLE_CHAIN */
+
+static inline bool txx9dmac_chan_INTENT(struct txx9dmac_chan *dc)
+{
+ return true;
+}
+
+static void txx9dmac_chan_set_INTENT(struct txx9dmac_chan *dc)
+{
+}
+
+static inline void txx9dmac_desc_set_INTENT(struct txx9dmac_dev *ddev,
+ struct txx9dmac_desc *desc)
+{
+ if (__is_dmac64(ddev))
+ desc->hwdesc.CCR |= TXX9_DMA_CCR_INTENT;
+ else
+ desc->hwdesc32.CCR |= TXX9_DMA_CCR_INTENT;
+}
+
+static inline void txx9dmac_chan_set_SMPCHN(struct txx9dmac_chan *dc)
+{
+}
+
+static inline void txx9dmac_desc_set_nosimple(struct txx9dmac_dev *ddev,
+ struct txx9dmac_desc *desc,
+ u32 sai, u32 dai, u32 ccr)
+{
+ if (__is_dmac64(ddev)) {
+ desc->hwdesc.SAIR = sai;
+ desc->hwdesc.DAIR = dai;
+ desc->hwdesc.CCR = ccr;
+ } else {
+ desc->hwdesc32.SAIR = sai;
+ desc->hwdesc32.DAIR = dai;
+ desc->hwdesc32.CCR = ccr;
+ }
+}
+
+#endif /* TXX9_DMA_USE_SIMPLE_CHAIN */
+
+#endif /* TXX9DMAC_H */