diff options
Diffstat (limited to 'drivers/dma')
32 files changed, 7996 insertions, 1448 deletions
diff --git a/drivers/dma/Kconfig b/drivers/dma/Kconfig index e02d74b1e892..9e01e96fee94 100644 --- a/drivers/dma/Kconfig +++ b/drivers/dma/Kconfig @@ -13,6 +13,22 @@ menuconfig DMADEVICES DMA Device drivers supported by the configured arch, it may be empty in some cases. +config DMADEVICES_DEBUG + bool "DMA Engine debugging" + depends on DMADEVICES != n + help + This is an option for use by developers; most people should + say N here. This enables DMA engine core and driver debugging. + +config DMADEVICES_VDEBUG + bool "DMA Engine verbose debugging" + depends on DMADEVICES_DEBUG != n + help + This is an option for use by developers; most people should + say N here. This enables deeper (more verbose) debugging of + the DMA engine core and drivers. + + if DMADEVICES comment "DMA Devices" @@ -69,6 +85,13 @@ config FSL_DMA The Elo is the DMA controller on some 82xx and 83xx parts, and the Elo Plus is the DMA controller on 85xx and 86xx parts. +config MPC512X_DMA + tristate "Freescale MPC512x built-in DMA engine support" + depends on PPC_MPC512x + select DMA_ENGINE + ---help--- + Enable support for the Freescale MPC512x built-in DMA engine. + config MV_XOR bool "Marvell XOR engine support" depends on PLAT_ORION @@ -118,6 +141,13 @@ config COH901318 help Enable support for ST-Ericsson COH 901 318 DMA. +config STE_DMA40 + bool "ST-Ericsson DMA40 support" + depends on ARCH_U8500 + select DMA_ENGINE + help + Support for ST-Ericsson DMA40 controller + config AMCC_PPC440SPE_ADMA tristate "AMCC PPC440SPe ADMA support" depends on 440SPe || 440SP @@ -126,9 +156,25 @@ config AMCC_PPC440SPE_ADMA help Enable support for the AMCC PPC440SPe RAID engines. +config TIMB_DMA + tristate "Timberdale FPGA DMA support" + depends on MFD_TIMBERDALE || HAS_IOMEM + select DMA_ENGINE + help + Enable support for the Timberdale FPGA DMA engine. + config ARCH_HAS_ASYNC_TX_FIND_CHANNEL bool +config PL330_DMA + tristate "DMA API Driver for PL330" + select DMA_ENGINE + depends on PL330 + help + Select if your platform has one or more PL330 DMACs. + You need to provide platform specific settings via + platform_data for a dma-pl330 device. + config DMA_ENGINE bool diff --git a/drivers/dma/Makefile b/drivers/dma/Makefile index 807053d48232..0fe5ebbfda5d 100644 --- a/drivers/dma/Makefile +++ b/drivers/dma/Makefile @@ -1,9 +1,17 @@ +ifeq ($(CONFIG_DMADEVICES_DEBUG),y) + EXTRA_CFLAGS += -DDEBUG +endif +ifeq ($(CONFIG_DMADEVICES_VDEBUG),y) + EXTRA_CFLAGS += -DVERBOSE_DEBUG +endif + obj-$(CONFIG_DMA_ENGINE) += dmaengine.o obj-$(CONFIG_NET_DMA) += iovlock.o obj-$(CONFIG_DMATEST) += dmatest.o obj-$(CONFIG_INTEL_IOATDMA) += ioat/ obj-$(CONFIG_INTEL_IOP_ADMA) += iop-adma.o obj-$(CONFIG_FSL_DMA) += fsldma.o +obj-$(CONFIG_MPC512X_DMA) += mpc512x_dma.o obj-$(CONFIG_MV_XOR) += mv_xor.o obj-$(CONFIG_DW_DMAC) += dw_dmac.o obj-$(CONFIG_AT_HDMAC) += at_hdmac.o @@ -12,3 +20,6 @@ obj-$(CONFIG_TXX9_DMAC) += txx9dmac.o obj-$(CONFIG_SH_DMAE) += shdma.o obj-$(CONFIG_COH901318) += coh901318.o coh901318_lli.o obj-$(CONFIG_AMCC_PPC440SPE_ADMA) += ppc4xx/ +obj-$(CONFIG_TIMB_DMA) += timb_dma.o +obj-$(CONFIG_STE_DMA40) += ste_dma40.o ste_dma40_ll.o +obj-$(CONFIG_PL330_DMA) += pl330.o diff --git a/drivers/dma/at_hdmac.c b/drivers/dma/at_hdmac.c index efc1a61ca231..bd5250e8c00c 100644 --- a/drivers/dma/at_hdmac.c +++ b/drivers/dma/at_hdmac.c @@ -22,6 +22,7 @@ #include <linux/interrupt.h> #include <linux/module.h> #include <linux/platform_device.h> +#include <linux/slab.h> #include "at_hdmac_regs.h" @@ -759,13 +760,18 @@ err_desc_get: return NULL; } -static void atc_terminate_all(struct dma_chan *chan) +static int atc_control(struct dma_chan *chan, enum dma_ctrl_cmd cmd, + unsigned long arg) { struct at_dma_chan *atchan = to_at_dma_chan(chan); struct at_dma *atdma = to_at_dma(chan->device); struct at_desc *desc, *_desc; LIST_HEAD(list); + /* Only supports DMA_TERMINATE_ALL */ + if (cmd != DMA_TERMINATE_ALL) + return -ENXIO; + /* * This is only called when something went wrong elsewhere, so * we don't really care about the data. Just disable the @@ -789,32 +795,30 @@ static void atc_terminate_all(struct dma_chan *chan) /* Flush all pending and queued descriptors */ list_for_each_entry_safe(desc, _desc, &list, desc_node) atc_chain_complete(atchan, desc); + + return 0; } /** - * atc_is_tx_complete - poll for transaction completion + * atc_tx_status - poll for transaction completion * @chan: DMA channel * @cookie: transaction identifier to check status of - * @done: if not %NULL, updated with last completed transaction - * @used: if not %NULL, updated with last used transaction + * @txstate: if not %NULL updated with transaction state * - * If @done and @used are passed in, upon return they reflect the driver + * If @txstate is passed in, upon return it reflect the driver * internal state and can be used with dma_async_is_complete() to check * the status of multiple cookies without re-checking hardware state. */ static enum dma_status -atc_is_tx_complete(struct dma_chan *chan, +atc_tx_status(struct dma_chan *chan, dma_cookie_t cookie, - dma_cookie_t *done, dma_cookie_t *used) + struct dma_tx_state *txstate) { struct at_dma_chan *atchan = to_at_dma_chan(chan); dma_cookie_t last_used; dma_cookie_t last_complete; enum dma_status ret; - dev_vdbg(chan2dev(chan), "is_tx_complete: %d (d%d, u%d)\n", - cookie, done ? *done : 0, used ? *used : 0); - spin_lock_bh(&atchan->lock); last_complete = atchan->completed_cookie; @@ -832,10 +836,10 @@ atc_is_tx_complete(struct dma_chan *chan, spin_unlock_bh(&atchan->lock); - if (done) - *done = last_complete; - if (used) - *used = last_used; + dma_set_tx_state(txstate, last_complete, last_used, 0); + dev_vdbg(chan2dev(chan), "tx_status: %d (d%d, u%d)\n", + cookie, last_complete ? last_complete : 0, + last_used ? last_used : 0); return ret; } @@ -1081,7 +1085,7 @@ static int __init at_dma_probe(struct platform_device *pdev) /* set base routines */ atdma->dma_common.device_alloc_chan_resources = atc_alloc_chan_resources; atdma->dma_common.device_free_chan_resources = atc_free_chan_resources; - atdma->dma_common.device_is_tx_complete = atc_is_tx_complete; + atdma->dma_common.device_tx_status = atc_tx_status; atdma->dma_common.device_issue_pending = atc_issue_pending; atdma->dma_common.dev = &pdev->dev; @@ -1091,7 +1095,7 @@ static int __init at_dma_probe(struct platform_device *pdev) if (dma_has_cap(DMA_SLAVE, atdma->dma_common.cap_mask)) { atdma->dma_common.device_prep_slave_sg = atc_prep_slave_sg; - atdma->dma_common.device_terminate_all = atc_terminate_all; + atdma->dma_common.device_control = atc_control; } dma_writel(atdma, EN, AT_DMA_ENABLE); diff --git a/drivers/dma/coh901318.c b/drivers/dma/coh901318.c index 64a937262a40..a724e6be1b4d 100644 --- a/drivers/dma/coh901318.c +++ b/drivers/dma/coh901318.c @@ -37,9 +37,8 @@ struct coh901318_desc { struct list_head node; struct scatterlist *sg; unsigned int sg_len; - struct coh901318_lli *data; + struct coh901318_lli *lli; enum dma_data_direction dir; - int pending_irqs; unsigned long flags; }; @@ -72,7 +71,6 @@ struct coh901318_chan { unsigned long nbr_active_done; unsigned long busy; - int pending_irqs; struct coh901318_base *base; }; @@ -80,18 +78,16 @@ struct coh901318_chan { static void coh901318_list_print(struct coh901318_chan *cohc, struct coh901318_lli *lli) { - struct coh901318_lli *l; - dma_addr_t addr = virt_to_phys(lli); + struct coh901318_lli *l = lli; int i = 0; - while (addr) { - l = phys_to_virt(addr); + while (l) { dev_vdbg(COHC_2_DEV(cohc), "i %d, lli %p, ctrl 0x%x, src 0x%x" - ", dst 0x%x, link 0x%x link_virt 0x%p\n", + ", dst 0x%x, link 0x%x virt_link_addr 0x%p\n", i, l, l->control, l->src_addr, l->dst_addr, - l->link_addr, phys_to_virt(l->link_addr)); + l->link_addr, l->virt_link_addr); i++; - addr = l->link_addr; + l = l->virt_link_addr; } } @@ -125,7 +121,7 @@ static int coh901318_debugfs_read(struct file *file, char __user *buf, goto err_kmalloc; tmp = dev_buf; - tmp += sprintf(tmp, "DMA -- enable dma channels\n"); + tmp += sprintf(tmp, "DMA -- enabled dma channels\n"); for (i = 0; i < debugfs_dma_base->platform->max_channels; i++) if (started_channels & (1 << i)) @@ -287,7 +283,7 @@ static int coh901318_start(struct coh901318_chan *cohc) } static int coh901318_prep_linked_list(struct coh901318_chan *cohc, - struct coh901318_lli *data) + struct coh901318_lli *lli) { int channel = cohc->id; void __iomem *virtbase = cohc->base->virtbase; @@ -296,18 +292,18 @@ static int coh901318_prep_linked_list(struct coh901318_chan *cohc, COH901318_CX_STAT_SPACING*channel) & COH901318_CX_STAT_ACTIVE); - writel(data->src_addr, + writel(lli->src_addr, virtbase + COH901318_CX_SRC_ADDR + COH901318_CX_SRC_ADDR_SPACING * channel); - writel(data->dst_addr, virtbase + + writel(lli->dst_addr, virtbase + COH901318_CX_DST_ADDR + COH901318_CX_DST_ADDR_SPACING * channel); - writel(data->link_addr, virtbase + COH901318_CX_LNK_ADDR + + writel(lli->link_addr, virtbase + COH901318_CX_LNK_ADDR + COH901318_CX_LNK_ADDR_SPACING * channel); - writel(data->control, virtbase + COH901318_CX_CTRL + + writel(lli->control, virtbase + COH901318_CX_CTRL + COH901318_CX_CTRL_SPACING * channel); return 0; @@ -337,16 +333,22 @@ coh901318_desc_get(struct coh901318_chan *cohc) * TODO: alloc a pile of descs instead of just one, * avoid many small allocations. */ - desc = kmalloc(sizeof(struct coh901318_desc), GFP_NOWAIT); + desc = kzalloc(sizeof(struct coh901318_desc), GFP_NOWAIT); if (desc == NULL) goto out; INIT_LIST_HEAD(&desc->node); + dma_async_tx_descriptor_init(&desc->desc, &cohc->chan); } else { /* Reuse an old desc. */ desc = list_first_entry(&cohc->free, struct coh901318_desc, node); list_del(&desc->node); + /* Initialize it a bit so it's not insane */ + desc->sg = NULL; + desc->sg_len = 0; + desc->desc.callback = NULL; + desc->desc.callback_param = NULL; } out: @@ -364,10 +366,6 @@ static void coh901318_desc_submit(struct coh901318_chan *cohc, struct coh901318_desc *desc) { list_add_tail(&desc->node, &cohc->active); - - BUG_ON(cohc->pending_irqs != 0); - - cohc->pending_irqs = desc->pending_irqs; } static struct coh901318_desc * @@ -410,33 +408,107 @@ coh901318_first_queued(struct coh901318_chan *cohc) return d; } +static inline u32 coh901318_get_bytes_in_lli(struct coh901318_lli *in_lli) +{ + struct coh901318_lli *lli = in_lli; + u32 bytes = 0; + + while (lli) { + bytes += lli->control & COH901318_CX_CTRL_TC_VALUE_MASK; + lli = lli->virt_link_addr; + } + return bytes; +} + /* - * DMA start/stop controls + * Get the number of bytes left to transfer on this channel, + * it is unwise to call this before stopping the channel for + * absolute measures, but for a rough guess you can still call + * it. */ -u32 coh901318_get_bytes_left(struct dma_chan *chan) +static u32 coh901318_get_bytes_left(struct dma_chan *chan) { - unsigned long flags; - u32 ret; struct coh901318_chan *cohc = to_coh901318_chan(chan); + struct coh901318_desc *cohd; + struct list_head *pos; + unsigned long flags; + u32 left = 0; + int i = 0; spin_lock_irqsave(&cohc->lock, flags); - /* Read transfer count value */ - ret = readl(cohc->base->virtbase + - COH901318_CX_CTRL+COH901318_CX_CTRL_SPACING * - cohc->id) & COH901318_CX_CTRL_TC_VALUE_MASK; + /* + * If there are many queued jobs, we iterate and add the + * size of them all. We take a special look on the first + * job though, since it is probably active. + */ + list_for_each(pos, &cohc->active) { + /* + * The first job in the list will be working on the + * hardware. The job can be stopped but still active, + * so that the transfer counter is somewhere inside + * the buffer. + */ + cohd = list_entry(pos, struct coh901318_desc, node); + + if (i == 0) { + struct coh901318_lli *lli; + dma_addr_t ladd; + + /* Read current transfer count value */ + left = readl(cohc->base->virtbase + + COH901318_CX_CTRL + + COH901318_CX_CTRL_SPACING * cohc->id) & + COH901318_CX_CTRL_TC_VALUE_MASK; + + /* See if the transfer is linked... */ + ladd = readl(cohc->base->virtbase + + COH901318_CX_LNK_ADDR + + COH901318_CX_LNK_ADDR_SPACING * + cohc->id) & + ~COH901318_CX_LNK_LINK_IMMEDIATE; + /* Single transaction */ + if (!ladd) + continue; + + /* + * Linked transaction, follow the lli, find the + * currently processing lli, and proceed to the next + */ + lli = cohd->lli; + while (lli && lli->link_addr != ladd) + lli = lli->virt_link_addr; + + if (lli) + lli = lli->virt_link_addr; + + /* + * Follow remaining lli links around to count the total + * number of bytes left + */ + left += coh901318_get_bytes_in_lli(lli); + } else { + left += coh901318_get_bytes_in_lli(cohd->lli); + } + i++; + } + + /* Also count bytes in the queued jobs */ + list_for_each(pos, &cohc->queue) { + cohd = list_entry(pos, struct coh901318_desc, node); + left += coh901318_get_bytes_in_lli(cohd->lli); + } spin_unlock_irqrestore(&cohc->lock, flags); - return ret; + return left; } -EXPORT_SYMBOL(coh901318_get_bytes_left); - -/* Stops a transfer without losing data. Enables power save. - Use this function in conjunction with coh901318_continue(..) -*/ -void coh901318_stop(struct dma_chan *chan) +/* + * Pauses a transfer without losing data. Enables power save. + * Use this function in conjunction with coh901318_resume. + */ +static void coh901318_pause(struct dma_chan *chan) { u32 val; unsigned long flags; @@ -477,12 +549,11 @@ void coh901318_stop(struct dma_chan *chan) spin_unlock_irqrestore(&cohc->lock, flags); } -EXPORT_SYMBOL(coh901318_stop); -/* Continues a transfer that has been stopped via 300_dma_stop(..). +/* Resumes a transfer that has been stopped via 300_dma_stop(..). Power save is handled. */ -void coh901318_continue(struct dma_chan *chan) +static void coh901318_resume(struct dma_chan *chan) { u32 val; unsigned long flags; @@ -508,7 +579,6 @@ void coh901318_continue(struct dma_chan *chan) spin_unlock_irqrestore(&cohc->lock, flags); } -EXPORT_SYMBOL(coh901318_continue); bool coh901318_filter_id(struct dma_chan *chan, void *chan_id) { @@ -567,31 +637,36 @@ static int coh901318_config(struct coh901318_chan *cohc, */ static struct coh901318_desc *coh901318_queue_start(struct coh901318_chan *cohc) { - struct coh901318_desc *cohd_que; + struct coh901318_desc *cohd; - /* start queued jobs, if any + /* + * start queued jobs, if any * TODO: transmit all queued jobs in one go */ - cohd_que = coh901318_first_queued(cohc); + cohd = coh901318_first_queued(cohc); - if (cohd_que != NULL) { + if (cohd != NULL) { /* Remove from queue */ - coh901318_desc_remove(cohd_que); + coh901318_desc_remove(cohd); /* initiate DMA job */ cohc->busy = 1; - coh901318_desc_submit(cohc, cohd_que); + coh901318_desc_submit(cohc, cohd); - coh901318_prep_linked_list(cohc, cohd_que->data); + coh901318_prep_linked_list(cohc, cohd->lli); - /* start dma job */ + /* start dma job on this channel */ coh901318_start(cohc); } - return cohd_que; + return cohd; } +/* + * This tasklet is called from the interrupt handler to + * handle each descriptor (DMA job) that is sent to a channel. + */ static void dma_tasklet(unsigned long data) { struct coh901318_chan *cohc = (struct coh901318_chan *) data; @@ -600,55 +675,58 @@ static void dma_tasklet(unsigned long data) dma_async_tx_callback callback; void *callback_param; + dev_vdbg(COHC_2_DEV(cohc), "[%s] chan_id %d" + " nbr_active_done %ld\n", __func__, + cohc->id, cohc->nbr_active_done); + spin_lock_irqsave(&cohc->lock, flags); - /* get first active entry from list */ + /* get first active descriptor entry from list */ cohd_fin = coh901318_first_active_get(cohc); - BUG_ON(cohd_fin->pending_irqs == 0); - if (cohd_fin == NULL) goto err; - cohd_fin->pending_irqs--; - cohc->completed = cohd_fin->desc.cookie; - - if (cohc->nbr_active_done == 0) - return; + /* locate callback to client */ + callback = cohd_fin->desc.callback; + callback_param = cohd_fin->desc.callback_param; - if (!cohd_fin->pending_irqs) { - /* release the lli allocation*/ - coh901318_lli_free(&cohc->base->pool, &cohd_fin->data); - } + /* sign this job as completed on the channel */ + cohc->completed = cohd_fin->desc.cookie; - dev_vdbg(COHC_2_DEV(cohc), "[%s] chan_id %d pending_irqs %d" - " nbr_active_done %ld\n", __func__, - cohc->id, cohc->pending_irqs, cohc->nbr_active_done); + /* release the lli allocation and remove the descriptor */ + coh901318_lli_free(&cohc->base->pool, &cohd_fin->lli); - /* callback to client */ - callback = cohd_fin->desc.callback; - callback_param = cohd_fin->desc.callback_param; + /* return desc to free-list */ + coh901318_desc_remove(cohd_fin); + coh901318_desc_free(cohc, cohd_fin); - if (!cohd_fin->pending_irqs) { - coh901318_desc_remove(cohd_fin); + spin_unlock_irqrestore(&cohc->lock, flags); - /* return desc to free-list */ - coh901318_desc_free(cohc, cohd_fin); - } + /* Call the callback when we're done */ + if (callback) + callback(callback_param); - if (cohc->nbr_active_done) - cohc->nbr_active_done--; + spin_lock_irqsave(&cohc->lock, flags); + /* + * If another interrupt fired while the tasklet was scheduling, + * we don't get called twice, so we have this number of active + * counter that keep track of the number of IRQs expected to + * be handled for this channel. If there happen to be more than + * one IRQ to be ack:ed, we simply schedule this tasklet again. + */ + cohc->nbr_active_done--; if (cohc->nbr_active_done) { + dev_dbg(COHC_2_DEV(cohc), "scheduling tasklet again, new IRQs " + "came in while we were scheduling this tasklet\n"); if (cohc_chan_conf(cohc)->priority_high) tasklet_hi_schedule(&cohc->tasklet); else tasklet_schedule(&cohc->tasklet); } - spin_unlock_irqrestore(&cohc->lock, flags); - if (callback) - callback(callback_param); + spin_unlock_irqrestore(&cohc->lock, flags); return; @@ -661,22 +739,44 @@ static void dma_tasklet(unsigned long data) /* called from interrupt context */ static void dma_tc_handle(struct coh901318_chan *cohc) { - BUG_ON(!cohc->allocated && (list_empty(&cohc->active) || - list_empty(&cohc->queue))); - - if (!cohc->allocated) + /* + * If the channel is not allocated, then we shouldn't have + * any TC interrupts on it. + */ + if (!cohc->allocated) { + dev_err(COHC_2_DEV(cohc), "spurious interrupt from " + "unallocated channel\n"); return; + } + + spin_lock(&cohc->lock); - BUG_ON(cohc->pending_irqs == 0); + /* + * When we reach this point, at least one queue item + * should have been moved over from cohc->queue to + * cohc->active and run to completion, that is why we're + * getting a terminal count interrupt is it not? + * If you get this BUG() the most probable cause is that + * the individual nodes in the lli chain have IRQ enabled, + * so check your platform config for lli chain ctrl. + */ + BUG_ON(list_empty(&cohc->active)); - cohc->pending_irqs--; cohc->nbr_active_done++; - if (cohc->pending_irqs == 0 && coh901318_queue_start(cohc) == NULL) + /* + * This attempt to take a job from cohc->queue, put it + * into cohc->active and start it. + */ + if (coh901318_queue_start(cohc) == NULL) cohc->busy = 0; - BUG_ON(list_empty(&cohc->active)); + spin_unlock(&cohc->lock); + /* + * This tasklet will remove items from cohc->active + * and thus terminates them. + */ if (cohc_chan_conf(cohc)->priority_high) tasklet_hi_schedule(&cohc->tasklet); else @@ -803,6 +903,7 @@ static irqreturn_t dma_irq_handler(int irq, void *dev_id) static int coh901318_alloc_chan_resources(struct dma_chan *chan) { struct coh901318_chan *cohc = to_coh901318_chan(chan); + unsigned long flags; dev_vdbg(COHC_2_DEV(cohc), "[%s] DMA channel %d\n", __func__, cohc->id); @@ -810,11 +911,15 @@ static int coh901318_alloc_chan_resources(struct dma_chan *chan) if (chan->client_count > 1) return -EBUSY; + spin_lock_irqsave(&cohc->lock, flags); + coh901318_config(cohc, NULL); cohc->allocated = 1; cohc->completed = chan->cookie = 1; + spin_unlock_irqrestore(&cohc->lock, flags); + return 1; } @@ -837,7 +942,7 @@ coh901318_free_chan_resources(struct dma_chan *chan) spin_unlock_irqrestore(&cohc->lock, flags); - chan->device->device_terminate_all(chan); + chan->device->device_control(chan, DMA_TERMINATE_ALL, 0); } @@ -864,12 +969,13 @@ static struct dma_async_tx_descriptor * coh901318_prep_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src, size_t size, unsigned long flags) { - struct coh901318_lli *data; + struct coh901318_lli *lli; struct coh901318_desc *cohd; unsigned long flg; struct coh901318_chan *cohc = to_coh901318_chan(chan); int lli_len; u32 ctrl_last = cohc_chan_param(cohc)->ctrl_lli_last; + int ret; spin_lock_irqsave(&cohc->lock, flg); @@ -885,27 +991,24 @@ coh901318_prep_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src, if ((lli_len << MAX_DMA_PACKET_SIZE_SHIFT) < size) lli_len++; - data = coh901318_lli_alloc(&cohc->base->pool, lli_len); + lli = coh901318_lli_alloc(&cohc->base->pool, lli_len); - if (data == NULL) + if (lli == NULL) goto err; - cohd = coh901318_desc_get(cohc); - cohd->sg = NULL; - cohd->sg_len = 0; - cohd->data = data; - - cohd->pending_irqs = - coh901318_lli_fill_memcpy( - &cohc->base->pool, data, src, size, dest, - cohc_chan_param(cohc)->ctrl_lli_chained, - ctrl_last); - cohd->flags = flags; - - COH_DBG(coh901318_list_print(cohc, data)); + ret = coh901318_lli_fill_memcpy( + &cohc->base->pool, lli, src, size, dest, + cohc_chan_param(cohc)->ctrl_lli_chained, + ctrl_last); + if (ret) + goto err; - dma_async_tx_descriptor_init(&cohd->desc, chan); + COH_DBG(coh901318_list_print(cohc, lli)); + /* Pick a descriptor to handle this transfer */ + cohd = coh901318_desc_get(cohc); + cohd->lli = lli; + cohd->flags = flags; cohd->desc.tx_submit = coh901318_tx_submit; spin_unlock_irqrestore(&cohc->lock, flg); @@ -922,8 +1025,9 @@ coh901318_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl, unsigned long flags) { struct coh901318_chan *cohc = to_coh901318_chan(chan); - struct coh901318_lli *data; + struct coh901318_lli *lli; struct coh901318_desc *cohd; + const struct coh901318_params *params; struct scatterlist *sg; int len = 0; int size; @@ -931,7 +1035,9 @@ coh901318_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl, u32 ctrl_chained = cohc_chan_param(cohc)->ctrl_lli_chained; u32 ctrl = cohc_chan_param(cohc)->ctrl_lli; u32 ctrl_last = cohc_chan_param(cohc)->ctrl_lli_last; + u32 config; unsigned long flg; + int ret; if (!sgl) goto out; @@ -947,15 +1053,14 @@ coh901318_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl, /* Trigger interrupt after last lli */ ctrl_last |= COH901318_CX_CTRL_TC_IRQ_ENABLE; - cohd = coh901318_desc_get(cohc); - cohd->sg = NULL; - cohd->sg_len = 0; - cohd->dir = direction; + params = cohc_chan_param(cohc); + config = params->config; if (direction == DMA_TO_DEVICE) { u32 tx_flags = COH901318_CX_CTRL_PRDD_SOURCE | COH901318_CX_CTRL_SRC_ADDR_INC_ENABLE; + config |= COH901318_CX_CFG_RM_MEMORY_TO_PRIMARY; ctrl_chained |= tx_flags; ctrl_last |= tx_flags; ctrl |= tx_flags; @@ -963,16 +1068,14 @@ coh901318_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl, u32 rx_flags = COH901318_CX_CTRL_PRDD_DEST | COH901318_CX_CTRL_DST_ADDR_INC_ENABLE; + config |= COH901318_CX_CFG_RM_PRIMARY_TO_MEMORY; ctrl_chained |= rx_flags; ctrl_last |= rx_flags; ctrl |= rx_flags; } else goto err_direction; - dma_async_tx_descriptor_init(&cohd->desc, chan); - - cohd->desc.tx_submit = coh901318_tx_submit; - + coh901318_set_conf(cohc, config); /* The dma only supports transmitting packages up to * MAX_DMA_PACKET_SIZE. Calculate to total number of @@ -994,41 +1097,45 @@ coh901318_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl, len += factor; } - data = coh901318_lli_alloc(&cohc->base->pool, len); + pr_debug("Allocate %d lli:s for this transfer\n", len); + lli = coh901318_lli_alloc(&cohc->base->pool, len); - if (data == NULL) + if (lli == NULL) goto err_dma_alloc; - /* initiate allocated data list */ - cohd->pending_irqs = - coh901318_lli_fill_sg(&cohc->base->pool, data, sgl, sg_len, - cohc_dev_addr(cohc), - ctrl_chained, - ctrl, - ctrl_last, - direction, COH901318_CX_CTRL_TC_IRQ_ENABLE); - cohd->data = data; + /* initiate allocated lli list */ + ret = coh901318_lli_fill_sg(&cohc->base->pool, lli, sgl, sg_len, + cohc_dev_addr(cohc), + ctrl_chained, + ctrl, + ctrl_last, + direction, COH901318_CX_CTRL_TC_IRQ_ENABLE); + if (ret) + goto err_lli_fill; - cohd->flags = flags; + COH_DBG(coh901318_list_print(cohc, lli)); - COH_DBG(coh901318_list_print(cohc, data)); + /* Pick a descriptor to handle this transfer */ + cohd = coh901318_desc_get(cohc); + cohd->dir = direction; + cohd->flags = flags; + cohd->desc.tx_submit = coh901318_tx_submit; + cohd->lli = lli; spin_unlock_irqrestore(&cohc->lock, flg); return &cohd->desc; + err_lli_fill: err_dma_alloc: err_direction: - coh901318_desc_remove(cohd); - coh901318_desc_free(cohc, cohd); spin_unlock_irqrestore(&cohc->lock, flg); out: return NULL; } static enum dma_status -coh901318_is_tx_complete(struct dma_chan *chan, - dma_cookie_t cookie, dma_cookie_t *done, - dma_cookie_t *used) +coh901318_tx_status(struct dma_chan *chan, dma_cookie_t cookie, + struct dma_tx_state *txstate) { struct coh901318_chan *cohc = to_coh901318_chan(chan); dma_cookie_t last_used; @@ -1040,10 +1147,10 @@ coh901318_is_tx_complete(struct dma_chan *chan, ret = dma_async_is_complete(cookie, last_complete, last_used); - if (done) - *done = last_complete; - if (used) - *used = last_used; + dma_set_tx_state(txstate, last_complete, last_used, + coh901318_get_bytes_left(chan)); + if (ret == DMA_IN_PROGRESS && cohc->stopped) + ret = DMA_PAUSED; return ret; } @@ -1056,23 +1163,42 @@ coh901318_issue_pending(struct dma_chan *chan) spin_lock_irqsave(&cohc->lock, flags); - /* Busy means that pending jobs are already being processed */ + /* + * Busy means that pending jobs are already being processed, + * and then there is no point in starting the queue: the + * terminal count interrupt on the channel will take the next + * job on the queue and execute it anyway. + */ if (!cohc->busy) coh901318_queue_start(cohc); spin_unlock_irqrestore(&cohc->lock, flags); } -static void -coh901318_terminate_all(struct dma_chan *chan) +static int +coh901318_control(struct dma_chan *chan, enum dma_ctrl_cmd cmd, + unsigned long arg) { unsigned long flags; struct coh901318_chan *cohc = to_coh901318_chan(chan); struct coh901318_desc *cohd; void __iomem *virtbase = cohc->base->virtbase; - coh901318_stop(chan); + if (cmd == DMA_PAUSE) { + coh901318_pause(chan); + return 0; + } + + if (cmd == DMA_RESUME) { + coh901318_resume(chan); + return 0; + } + + if (cmd != DMA_TERMINATE_ALL) + return -ENXIO; + /* The remainder of this function terminates the transfer */ + coh901318_pause(chan); spin_lock_irqsave(&cohc->lock, flags); /* Clear any pending BE or TC interrupt */ @@ -1090,30 +1216,29 @@ coh901318_terminate_all(struct dma_chan *chan) while ((cohd = coh901318_first_active_get(cohc))) { /* release the lli allocation*/ - coh901318_lli_free(&cohc->base->pool, &cohd->data); - - coh901318_desc_remove(cohd); + coh901318_lli_free(&cohc->base->pool, &cohd->lli); /* return desc to free-list */ + coh901318_desc_remove(cohd); coh901318_desc_free(cohc, cohd); } while ((cohd = coh901318_first_queued(cohc))) { /* release the lli allocation*/ - coh901318_lli_free(&cohc->base->pool, &cohd->data); - - coh901318_desc_remove(cohd); + coh901318_lli_free(&cohc->base->pool, &cohd->lli); /* return desc to free-list */ + coh901318_desc_remove(cohd); coh901318_desc_free(cohc, cohd); } cohc->nbr_active_done = 0; cohc->busy = 0; - cohc->pending_irqs = 0; spin_unlock_irqrestore(&cohc->lock, flags); + + return 0; } void coh901318_base_init(struct dma_device *dma, const int *pick_chans, struct coh901318_base *base) @@ -1138,7 +1263,6 @@ void coh901318_base_init(struct dma_device *dma, const int *pick_chans, spin_lock_init(&cohc->lock); - cohc->pending_irqs = 0; cohc->nbr_active_done = 0; cohc->busy = 0; INIT_LIST_HEAD(&cohc->free); @@ -1230,9 +1354,9 @@ static int __init coh901318_probe(struct platform_device *pdev) base->dma_slave.device_alloc_chan_resources = coh901318_alloc_chan_resources; base->dma_slave.device_free_chan_resources = coh901318_free_chan_resources; base->dma_slave.device_prep_slave_sg = coh901318_prep_slave_sg; - base->dma_slave.device_is_tx_complete = coh901318_is_tx_complete; + base->dma_slave.device_tx_status = coh901318_tx_status; base->dma_slave.device_issue_pending = coh901318_issue_pending; - base->dma_slave.device_terminate_all = coh901318_terminate_all; + base->dma_slave.device_control = coh901318_control; base->dma_slave.dev = &pdev->dev; err = dma_async_device_register(&base->dma_slave); @@ -1250,16 +1374,21 @@ static int __init coh901318_probe(struct platform_device *pdev) base->dma_memcpy.device_alloc_chan_resources = coh901318_alloc_chan_resources; base->dma_memcpy.device_free_chan_resources = coh901318_free_chan_resources; base->dma_memcpy.device_prep_dma_memcpy = coh901318_prep_memcpy; - base->dma_memcpy.device_is_tx_complete = coh901318_is_tx_complete; + base->dma_memcpy.device_tx_status = coh901318_tx_status; base->dma_memcpy.device_issue_pending = coh901318_issue_pending; - base->dma_memcpy.device_terminate_all = coh901318_terminate_all; + base->dma_memcpy.device_control = coh901318_control; base->dma_memcpy.dev = &pdev->dev; + /* + * This controller can only access address at even 32bit boundaries, + * i.e. 2^2 + */ + base->dma_memcpy.copy_align = 2; err = dma_async_device_register(&base->dma_memcpy); if (err) goto err_register_memcpy; - dev_dbg(&pdev->dev, "Initialized COH901318 DMA on virtual base 0x%08x\n", + dev_info(&pdev->dev, "Initialized COH901318 DMA on virtual base 0x%08x\n", (u32) base->virtbase); return err; diff --git a/drivers/dma/coh901318_lli.c b/drivers/dma/coh901318_lli.c index f5120f238a4d..9f7e0e6a7eea 100644 --- a/drivers/dma/coh901318_lli.c +++ b/drivers/dma/coh901318_lli.c @@ -11,6 +11,7 @@ #include <linux/spinlock.h> #include <linux/dmapool.h> #include <linux/memory.h> +#include <linux/gfp.h> #include <mach/coh901318.h> #include "coh901318_lli.h" @@ -74,6 +75,8 @@ coh901318_lli_alloc(struct coh901318_pool *pool, unsigned int len) lli = head; lli->phy_this = phy; + lli->link_addr = 0x00000000; + lli->virt_link_addr = 0x00000000U; for (i = 1; i < len; i++) { lli_prev = lli; @@ -85,13 +88,13 @@ coh901318_lli_alloc(struct coh901318_pool *pool, unsigned int len) DEBUGFS_POOL_COUNTER_ADD(pool, 1); lli->phy_this = phy; + lli->link_addr = 0x00000000; + lli->virt_link_addr = 0x00000000U; lli_prev->link_addr = phy; lli_prev->virt_link_addr = lli; } - lli->link_addr = 0x00000000U; - spin_unlock(&pool->lock); return head; @@ -166,8 +169,7 @@ coh901318_lli_fill_memcpy(struct coh901318_pool *pool, lli->src_addr = src; lli->dst_addr = dst; - /* One irq per single transfer */ - return 1; + return 0; } int @@ -223,8 +225,7 @@ coh901318_lli_fill_single(struct coh901318_pool *pool, lli->src_addr = src; lli->dst_addr = dst; - /* One irq per single transfer */ - return 1; + return 0; } int @@ -240,7 +241,6 @@ coh901318_lli_fill_sg(struct coh901318_pool *pool, u32 ctrl_sg; dma_addr_t src = 0; dma_addr_t dst = 0; - int nbr_of_irq = 0; u32 bytes_to_transfer; u32 elem_size; @@ -269,15 +269,12 @@ coh901318_lli_fill_sg(struct coh901318_pool *pool, ctrl_sg = ctrl ? ctrl : ctrl_last; - if ((ctrl_sg & ctrl_irq_mask)) - nbr_of_irq++; - if (dir == DMA_TO_DEVICE) /* increment source address */ - src = sg_dma_address(sg); + src = sg_phys(sg); else /* increment destination address */ - dst = sg_dma_address(sg); + dst = sg_phys(sg); bytes_to_transfer = sg_dma_len(sg); @@ -310,8 +307,7 @@ coh901318_lli_fill_sg(struct coh901318_pool *pool, } spin_unlock(&pool->lock); - /* There can be many IRQs per sg transfer */ - return nbr_of_irq; + return 0; err: spin_unlock(&pool->lock); return -EINVAL; diff --git a/drivers/dma/coh901318_lli.h b/drivers/dma/coh901318_lli.h index 7bf713b79c6b..7a5c80990e9e 100644 --- a/drivers/dma/coh901318_lli.h +++ b/drivers/dma/coh901318_lli.h @@ -30,7 +30,7 @@ struct device; * @pool: pool handle * @dev: dma device * @lli_nbr: number of lli:s in the pool - * @algin: adress alignemtn of lli:s + * @algin: address alignemtn of lli:s * returns 0 on success otherwise none zero */ int coh901318_pool_create(struct coh901318_pool *pool, diff --git a/drivers/dma/dmaengine.c b/drivers/dma/dmaengine.c index e7a3230fb7d5..9d31d5eb95c1 100644 --- a/drivers/dma/dmaengine.c +++ b/drivers/dma/dmaengine.c @@ -58,6 +58,7 @@ #include <linux/jiffies.h> #include <linux/rculist.h> #include <linux/idr.h> +#include <linux/slab.h> static DEFINE_MUTEX(dma_list_mutex); static LIST_HEAD(dma_device_list); @@ -284,7 +285,7 @@ struct dma_chan_tbl_ent { /** * channel_table - percpu lookup table for memory-to-memory offload providers */ -static struct dma_chan_tbl_ent *channel_table[DMA_TX_TYPE_END]; +static struct dma_chan_tbl_ent __percpu *channel_table[DMA_TX_TYPE_END]; static int __init dma_channel_table_init(void) { @@ -514,7 +515,6 @@ struct dma_chan *__dma_request_channel(dma_cap_mask_t *mask, dma_filter_fn fn, v break; if (--device->privatecnt == 0) dma_cap_clear(DMA_PRIVATE, device->cap_mask); - chan->private = NULL; chan = NULL; } } @@ -536,7 +536,6 @@ void dma_release_channel(struct dma_chan *chan) /* drop PRIVATE cap enabled by __dma_request_channel() */ if (--chan->device->privatecnt == 0) dma_cap_clear(DMA_PRIVATE, chan->device->cap_mask); - chan->private = NULL; mutex_unlock(&dma_list_mutex); } EXPORT_SYMBOL_GPL(dma_release_channel); @@ -694,11 +693,11 @@ int dma_async_device_register(struct dma_device *device) BUG_ON(dma_has_cap(DMA_SLAVE, device->cap_mask) && !device->device_prep_slave_sg); BUG_ON(dma_has_cap(DMA_SLAVE, device->cap_mask) && - !device->device_terminate_all); + !device->device_control); BUG_ON(!device->device_alloc_chan_resources); BUG_ON(!device->device_free_chan_resources); - BUG_ON(!device->device_is_tx_complete); + BUG_ON(!device->device_tx_status); BUG_ON(!device->device_issue_pending); BUG_ON(!device->dev); @@ -977,7 +976,9 @@ void dma_async_tx_descriptor_init(struct dma_async_tx_descriptor *tx, struct dma_chan *chan) { tx->chan = chan; + #ifndef CONFIG_ASYNC_TX_DISABLE_CHANNEL_SWITCH spin_lock_init(&tx->lock); + #endif } EXPORT_SYMBOL(dma_async_tx_descriptor_init); @@ -1010,7 +1011,7 @@ EXPORT_SYMBOL_GPL(dma_wait_for_async_tx); */ void dma_run_dependencies(struct dma_async_tx_descriptor *tx) { - struct dma_async_tx_descriptor *dep = tx->next; + struct dma_async_tx_descriptor *dep = txd_next(tx); struct dma_async_tx_descriptor *dep_next; struct dma_chan *chan; @@ -1018,7 +1019,7 @@ void dma_run_dependencies(struct dma_async_tx_descriptor *tx) return; /* we'll submit tx->next now, so clear the link */ - tx->next = NULL; + txd_clear_next(tx); chan = dep->chan; /* keep submitting up until a channel switch is detected @@ -1026,14 +1027,14 @@ void dma_run_dependencies(struct dma_async_tx_descriptor *tx) * processing the interrupt from async_tx_channel_switch */ for (; dep; dep = dep_next) { - spin_lock_bh(&dep->lock); - dep->parent = NULL; - dep_next = dep->next; + txd_lock(dep); + txd_clear_parent(dep); + dep_next = txd_next(dep); if (dep_next && dep_next->chan == chan) - dep->next = NULL; /* ->next will be submitted */ + txd_clear_next(dep); /* ->next will be submitted */ else dep_next = NULL; /* submit current dep and terminate */ - spin_unlock_bh(&dep->lock); + txd_unlock(dep); dep->tx_submit(dep); } diff --git a/drivers/dma/dmatest.c b/drivers/dma/dmatest.c index 948d563941c9..68d58c414cf0 100644 --- a/drivers/dma/dmatest.c +++ b/drivers/dma/dmatest.c @@ -14,6 +14,7 @@ #include <linux/module.h> #include <linux/moduleparam.h> #include <linux/random.h> +#include <linux/slab.h> #include <linux/wait.h> static unsigned int test_buf_size = 16384; @@ -237,7 +238,7 @@ static int dmatest_func(void *data) dma_cookie_t cookie; enum dma_status status; enum dma_ctrl_flags flags; - u8 pq_coefs[pq_sources]; + u8 pq_coefs[pq_sources + 1]; int ret; int src_cnt; int dst_cnt; @@ -257,7 +258,7 @@ static int dmatest_func(void *data) } else if (thread->type == DMA_PQ) { src_cnt = pq_sources | 1; /* force odd to ensure dst = src */ dst_cnt = 2; - for (i = 0; i < pq_sources; i++) + for (i = 0; i < src_cnt; i++) pq_coefs[i] = 1; } else goto err_srcs; @@ -347,7 +348,7 @@ static int dmatest_func(void *data) else if (thread->type == DMA_XOR) tx = dev->device_prep_dma_xor(chan, dma_dsts[0] + dst_off, - dma_srcs, xor_sources, + dma_srcs, src_cnt, len, flags); else if (thread->type == DMA_PQ) { dma_addr_t dma_pq[dst_cnt]; @@ -355,7 +356,7 @@ static int dmatest_func(void *data) for (i = 0; i < dst_cnt; i++) dma_pq[i] = dma_dsts[i] + dst_off; tx = dev->device_prep_dma_pq(chan, dma_pq, dma_srcs, - pq_sources, pq_coefs, + src_cnt, pq_coefs, len, flags); } diff --git a/drivers/dma/dw_dmac.c b/drivers/dma/dw_dmac.c index d28369f7afd2..a3991ab0d67e 100644 --- a/drivers/dma/dw_dmac.c +++ b/drivers/dma/dw_dmac.c @@ -781,13 +781,18 @@ err_desc_get: return NULL; } -static void dwc_terminate_all(struct dma_chan *chan) +static int dwc_control(struct dma_chan *chan, enum dma_ctrl_cmd cmd, + unsigned long arg) { struct dw_dma_chan *dwc = to_dw_dma_chan(chan); struct dw_dma *dw = to_dw_dma(chan->device); struct dw_desc *desc, *_desc; LIST_HEAD(list); + /* Only supports DMA_TERMINATE_ALL */ + if (cmd != DMA_TERMINATE_ALL) + return -ENXIO; + /* * This is only called when something went wrong elsewhere, so * we don't really care about the data. Just disable the @@ -810,12 +815,14 @@ static void dwc_terminate_all(struct dma_chan *chan) /* Flush all pending and queued descriptors */ list_for_each_entry_safe(desc, _desc, &list, desc_node) dwc_descriptor_complete(dwc, desc); + + return 0; } static enum dma_status -dwc_is_tx_complete(struct dma_chan *chan, - dma_cookie_t cookie, - dma_cookie_t *done, dma_cookie_t *used) +dwc_tx_status(struct dma_chan *chan, + dma_cookie_t cookie, + struct dma_tx_state *txstate) { struct dw_dma_chan *dwc = to_dw_dma_chan(chan); dma_cookie_t last_used; @@ -835,10 +842,7 @@ dwc_is_tx_complete(struct dma_chan *chan, ret = dma_async_is_complete(cookie, last_complete, last_used); } - if (done) - *done = last_complete; - if (used) - *used = last_used; + dma_set_tx_state(txstate, last_complete, last_used, 0); return ret; } @@ -1338,9 +1342,9 @@ static int __init dw_probe(struct platform_device *pdev) dw->dma.device_prep_dma_memcpy = dwc_prep_dma_memcpy; dw->dma.device_prep_slave_sg = dwc_prep_slave_sg; - dw->dma.device_terminate_all = dwc_terminate_all; + dw->dma.device_control = dwc_control; - dw->dma.device_is_tx_complete = dwc_is_tx_complete; + dw->dma.device_tx_status = dwc_tx_status; dw->dma.device_issue_pending = dwc_issue_pending; dma_writel(dw, CFG, DW_CFG_DMA_EN); diff --git a/drivers/dma/fsldma.c b/drivers/dma/fsldma.c index 296f9e747fac..8088b14ba5f7 100644 --- a/drivers/dma/fsldma.c +++ b/drivers/dma/fsldma.c @@ -27,6 +27,7 @@ #include <linux/init.h> #include <linux/module.h> #include <linux/pci.h> +#include <linux/slab.h> #include <linux/interrupt.h> #include <linux/dmaengine.h> #include <linux/delay.h> @@ -37,19 +38,19 @@ #include <asm/fsldma.h> #include "fsldma.h" -static void dma_init(struct fsl_dma_chan *fsl_chan) +static void dma_init(struct fsldma_chan *chan) { /* Reset the channel */ - DMA_OUT(fsl_chan, &fsl_chan->reg_base->mr, 0, 32); + DMA_OUT(chan, &chan->regs->mr, 0, 32); - switch (fsl_chan->feature & FSL_DMA_IP_MASK) { + switch (chan->feature & FSL_DMA_IP_MASK) { case FSL_DMA_IP_85XX: /* Set the channel to below modes: * EIE - Error interrupt enable * EOSIE - End of segments interrupt enable (basic mode) * EOLNIE - End of links interrupt enable */ - DMA_OUT(fsl_chan, &fsl_chan->reg_base->mr, FSL_DMA_MR_EIE + DMA_OUT(chan, &chan->regs->mr, FSL_DMA_MR_EIE | FSL_DMA_MR_EOLNIE | FSL_DMA_MR_EOSIE, 32); break; case FSL_DMA_IP_83XX: @@ -57,170 +58,146 @@ static void dma_init(struct fsl_dma_chan *fsl_chan) * EOTIE - End-of-transfer interrupt enable * PRC_RM - PCI read multiple */ - DMA_OUT(fsl_chan, &fsl_chan->reg_base->mr, FSL_DMA_MR_EOTIE + DMA_OUT(chan, &chan->regs->mr, FSL_DMA_MR_EOTIE | FSL_DMA_MR_PRC_RM, 32); break; } - } -static void set_sr(struct fsl_dma_chan *fsl_chan, u32 val) +static void set_sr(struct fsldma_chan *chan, u32 val) { - DMA_OUT(fsl_chan, &fsl_chan->reg_base->sr, val, 32); + DMA_OUT(chan, &chan->regs->sr, val, 32); } -static u32 get_sr(struct fsl_dma_chan *fsl_chan) +static u32 get_sr(struct fsldma_chan *chan) { - return DMA_IN(fsl_chan, &fsl_chan->reg_base->sr, 32); + return DMA_IN(chan, &chan->regs->sr, 32); } -static void set_desc_cnt(struct fsl_dma_chan *fsl_chan, +static void set_desc_cnt(struct fsldma_chan *chan, struct fsl_dma_ld_hw *hw, u32 count) { - hw->count = CPU_TO_DMA(fsl_chan, count, 32); + hw->count = CPU_TO_DMA(chan, count, 32); } -static void set_desc_src(struct fsl_dma_chan *fsl_chan, +static void set_desc_src(struct fsldma_chan *chan, struct fsl_dma_ld_hw *hw, dma_addr_t src) { u64 snoop_bits; - snoop_bits = ((fsl_chan->feature & FSL_DMA_IP_MASK) == FSL_DMA_IP_85XX) + snoop_bits = ((chan->feature & FSL_DMA_IP_MASK) == FSL_DMA_IP_85XX) ? ((u64)FSL_DMA_SATR_SREADTYPE_SNOOP_READ << 32) : 0; - hw->src_addr = CPU_TO_DMA(fsl_chan, snoop_bits | src, 64); + hw->src_addr = CPU_TO_DMA(chan, snoop_bits | src, 64); } -static void set_desc_dest(struct fsl_dma_chan *fsl_chan, - struct fsl_dma_ld_hw *hw, dma_addr_t dest) +static void set_desc_dst(struct fsldma_chan *chan, + struct fsl_dma_ld_hw *hw, dma_addr_t dst) { u64 snoop_bits; - snoop_bits = ((fsl_chan->feature & FSL_DMA_IP_MASK) == FSL_DMA_IP_85XX) + snoop_bits = ((chan->feature & FSL_DMA_IP_MASK) == FSL_DMA_IP_85XX) ? ((u64)FSL_DMA_DATR_DWRITETYPE_SNOOP_WRITE << 32) : 0; - hw->dst_addr = CPU_TO_DMA(fsl_chan, snoop_bits | dest, 64); + hw->dst_addr = CPU_TO_DMA(chan, snoop_bits | dst, 64); } -static void set_desc_next(struct fsl_dma_chan *fsl_chan, +static void set_desc_next(struct fsldma_chan *chan, struct fsl_dma_ld_hw *hw, dma_addr_t next) { u64 snoop_bits; - snoop_bits = ((fsl_chan->feature & FSL_DMA_IP_MASK) == FSL_DMA_IP_83XX) + snoop_bits = ((chan->feature & FSL_DMA_IP_MASK) == FSL_DMA_IP_83XX) ? FSL_DMA_SNEN : 0; - hw->next_ln_addr = CPU_TO_DMA(fsl_chan, snoop_bits | next, 64); -} - -static void set_cdar(struct fsl_dma_chan *fsl_chan, dma_addr_t addr) -{ - DMA_OUT(fsl_chan, &fsl_chan->reg_base->cdar, addr | FSL_DMA_SNEN, 64); + hw->next_ln_addr = CPU_TO_DMA(chan, snoop_bits | next, 64); } -static dma_addr_t get_cdar(struct fsl_dma_chan *fsl_chan) +static void set_cdar(struct fsldma_chan *chan, dma_addr_t addr) { - return DMA_IN(fsl_chan, &fsl_chan->reg_base->cdar, 64) & ~FSL_DMA_SNEN; + DMA_OUT(chan, &chan->regs->cdar, addr | FSL_DMA_SNEN, 64); } -static void set_ndar(struct fsl_dma_chan *fsl_chan, dma_addr_t addr) +static dma_addr_t get_cdar(struct fsldma_chan *chan) { - DMA_OUT(fsl_chan, &fsl_chan->reg_base->ndar, addr, 64); + return DMA_IN(chan, &chan->regs->cdar, 64) & ~FSL_DMA_SNEN; } -static dma_addr_t get_ndar(struct fsl_dma_chan *fsl_chan) +static dma_addr_t get_ndar(struct fsldma_chan *chan) { - return DMA_IN(fsl_chan, &fsl_chan->reg_base->ndar, 64); + return DMA_IN(chan, &chan->regs->ndar, 64); } -static u32 get_bcr(struct fsl_dma_chan *fsl_chan) +static u32 get_bcr(struct fsldma_chan *chan) { - return DMA_IN(fsl_chan, &fsl_chan->reg_base->bcr, 32); + return DMA_IN(chan, &chan->regs->bcr, 32); } -static int dma_is_idle(struct fsl_dma_chan *fsl_chan) +static int dma_is_idle(struct fsldma_chan *chan) { - u32 sr = get_sr(fsl_chan); + u32 sr = get_sr(chan); return (!(sr & FSL_DMA_SR_CB)) || (sr & FSL_DMA_SR_CH); } -static void dma_start(struct fsl_dma_chan *fsl_chan) +static void dma_start(struct fsldma_chan *chan) { - u32 mr_set = 0; - - if (fsl_chan->feature & FSL_DMA_CHAN_PAUSE_EXT) { - DMA_OUT(fsl_chan, &fsl_chan->reg_base->bcr, 0, 32); - mr_set |= FSL_DMA_MR_EMP_EN; - } else if ((fsl_chan->feature & FSL_DMA_IP_MASK) == FSL_DMA_IP_85XX) { - DMA_OUT(fsl_chan, &fsl_chan->reg_base->mr, - DMA_IN(fsl_chan, &fsl_chan->reg_base->mr, 32) - & ~FSL_DMA_MR_EMP_EN, 32); + u32 mode; + + mode = DMA_IN(chan, &chan->regs->mr, 32); + + if ((chan->feature & FSL_DMA_IP_MASK) == FSL_DMA_IP_85XX) { + if (chan->feature & FSL_DMA_CHAN_PAUSE_EXT) { + DMA_OUT(chan, &chan->regs->bcr, 0, 32); + mode |= FSL_DMA_MR_EMP_EN; + } else { + mode &= ~FSL_DMA_MR_EMP_EN; + } } - if (fsl_chan->feature & FSL_DMA_CHAN_START_EXT) - mr_set |= FSL_DMA_MR_EMS_EN; + if (chan->feature & FSL_DMA_CHAN_START_EXT) + mode |= FSL_DMA_MR_EMS_EN; else - mr_set |= FSL_DMA_MR_CS; + mode |= FSL_DMA_MR_CS; - DMA_OUT(fsl_chan, &fsl_chan->reg_base->mr, - DMA_IN(fsl_chan, &fsl_chan->reg_base->mr, 32) - | mr_set, 32); + DMA_OUT(chan, &chan->regs->mr, mode, 32); } -static void dma_halt(struct fsl_dma_chan *fsl_chan) +static void dma_halt(struct fsldma_chan *chan) { + u32 mode; int i; - DMA_OUT(fsl_chan, &fsl_chan->reg_base->mr, - DMA_IN(fsl_chan, &fsl_chan->reg_base->mr, 32) | FSL_DMA_MR_CA, - 32); - DMA_OUT(fsl_chan, &fsl_chan->reg_base->mr, - DMA_IN(fsl_chan, &fsl_chan->reg_base->mr, 32) & ~(FSL_DMA_MR_CS - | FSL_DMA_MR_EMS_EN | FSL_DMA_MR_CA), 32); + mode = DMA_IN(chan, &chan->regs->mr, 32); + mode |= FSL_DMA_MR_CA; + DMA_OUT(chan, &chan->regs->mr, mode, 32); + + mode &= ~(FSL_DMA_MR_CS | FSL_DMA_MR_EMS_EN | FSL_DMA_MR_CA); + DMA_OUT(chan, &chan->regs->mr, mode, 32); for (i = 0; i < 100; i++) { - if (dma_is_idle(fsl_chan)) - break; + if (dma_is_idle(chan)) + return; + udelay(10); } - if (i >= 100 && !dma_is_idle(fsl_chan)) - dev_err(fsl_chan->dev, "DMA halt timeout!\n"); + + if (!dma_is_idle(chan)) + dev_err(chan->dev, "DMA halt timeout!\n"); } -static void set_ld_eol(struct fsl_dma_chan *fsl_chan, +static void set_ld_eol(struct fsldma_chan *chan, struct fsl_desc_sw *desc) { u64 snoop_bits; - snoop_bits = ((fsl_chan->feature & FSL_DMA_IP_MASK) == FSL_DMA_IP_83XX) + snoop_bits = ((chan->feature & FSL_DMA_IP_MASK) == FSL_DMA_IP_83XX) ? FSL_DMA_SNEN : 0; - desc->hw.next_ln_addr = CPU_TO_DMA(fsl_chan, - DMA_TO_CPU(fsl_chan, desc->hw.next_ln_addr, 64) | FSL_DMA_EOL + desc->hw.next_ln_addr = CPU_TO_DMA(chan, + DMA_TO_CPU(chan, desc->hw.next_ln_addr, 64) | FSL_DMA_EOL | snoop_bits, 64); } -static void append_ld_queue(struct fsl_dma_chan *fsl_chan, - struct fsl_desc_sw *new_desc) -{ - struct fsl_desc_sw *queue_tail = to_fsl_desc(fsl_chan->ld_queue.prev); - - if (list_empty(&fsl_chan->ld_queue)) - return; - - /* Link to the new descriptor physical address and - * Enable End-of-segment interrupt for - * the last link descriptor. - * (the previous node's next link descriptor) - * - * For FSL_DMA_IP_83xx, the snoop enable bit need be set. - */ - queue_tail->hw.next_ln_addr = CPU_TO_DMA(fsl_chan, - new_desc->async_tx.phys | FSL_DMA_EOSIE | - (((fsl_chan->feature & FSL_DMA_IP_MASK) - == FSL_DMA_IP_83XX) ? FSL_DMA_SNEN : 0), 64); -} - /** * fsl_chan_set_src_loop_size - Set source address hold transfer size - * @fsl_chan : Freescale DMA channel + * @chan : Freescale DMA channel * @size : Address loop size, 0 for disable loop * * The set source address hold transfer size. The source @@ -229,29 +206,30 @@ static void append_ld_queue(struct fsl_dma_chan *fsl_chan, * read data from SA, SA + 1, SA + 2, SA + 3, then loop back to SA, * SA + 1 ... and so on. */ -static void fsl_chan_set_src_loop_size(struct fsl_dma_chan *fsl_chan, int size) +static void fsl_chan_set_src_loop_size(struct fsldma_chan *chan, int size) { + u32 mode; + + mode = DMA_IN(chan, &chan->regs->mr, 32); + switch (size) { case 0: - DMA_OUT(fsl_chan, &fsl_chan->reg_base->mr, - DMA_IN(fsl_chan, &fsl_chan->reg_base->mr, 32) & - (~FSL_DMA_MR_SAHE), 32); + mode &= ~FSL_DMA_MR_SAHE; break; case 1: case 2: case 4: case 8: - DMA_OUT(fsl_chan, &fsl_chan->reg_base->mr, - DMA_IN(fsl_chan, &fsl_chan->reg_base->mr, 32) | - FSL_DMA_MR_SAHE | (__ilog2(size) << 14), - 32); + mode |= FSL_DMA_MR_SAHE | (__ilog2(size) << 14); break; } + + DMA_OUT(chan, &chan->regs->mr, mode, 32); } /** - * fsl_chan_set_dest_loop_size - Set destination address hold transfer size - * @fsl_chan : Freescale DMA channel + * fsl_chan_set_dst_loop_size - Set destination address hold transfer size + * @chan : Freescale DMA channel * @size : Address loop size, 0 for disable loop * * The set destination address hold transfer size. The destination @@ -260,29 +238,30 @@ static void fsl_chan_set_src_loop_size(struct fsl_dma_chan *fsl_chan, int size) * write data to TA, TA + 1, TA + 2, TA + 3, then loop back to TA, * TA + 1 ... and so on. */ -static void fsl_chan_set_dest_loop_size(struct fsl_dma_chan *fsl_chan, int size) +static void fsl_chan_set_dst_loop_size(struct fsldma_chan *chan, int size) { + u32 mode; + + mode = DMA_IN(chan, &chan->regs->mr, 32); + switch (size) { case 0: - DMA_OUT(fsl_chan, &fsl_chan->reg_base->mr, - DMA_IN(fsl_chan, &fsl_chan->reg_base->mr, 32) & - (~FSL_DMA_MR_DAHE), 32); + mode &= ~FSL_DMA_MR_DAHE; break; case 1: case 2: case 4: case 8: - DMA_OUT(fsl_chan, &fsl_chan->reg_base->mr, - DMA_IN(fsl_chan, &fsl_chan->reg_base->mr, 32) | - FSL_DMA_MR_DAHE | (__ilog2(size) << 16), - 32); + mode |= FSL_DMA_MR_DAHE | (__ilog2(size) << 16); break; } + + DMA_OUT(chan, &chan->regs->mr, mode, 32); } /** * fsl_chan_set_request_count - Set DMA Request Count for external control - * @fsl_chan : Freescale DMA channel + * @chan : Freescale DMA channel * @size : Number of bytes to transfer in a single request * * The Freescale DMA channel can be controlled by the external signal DREQ#. @@ -292,35 +271,38 @@ static void fsl_chan_set_dest_loop_size(struct fsl_dma_chan *fsl_chan, int size) * * A size of 0 disables external pause control. The maximum size is 1024. */ -static void fsl_chan_set_request_count(struct fsl_dma_chan *fsl_chan, int size) +static void fsl_chan_set_request_count(struct fsldma_chan *chan, int size) { + u32 mode; + BUG_ON(size > 1024); - DMA_OUT(fsl_chan, &fsl_chan->reg_base->mr, - DMA_IN(fsl_chan, &fsl_chan->reg_base->mr, 32) - | ((__ilog2(size) << 24) & 0x0f000000), - 32); + + mode = DMA_IN(chan, &chan->regs->mr, 32); + mode |= (__ilog2(size) << 24) & 0x0f000000; + + DMA_OUT(chan, &chan->regs->mr, mode, 32); } /** * fsl_chan_toggle_ext_pause - Toggle channel external pause status - * @fsl_chan : Freescale DMA channel + * @chan : Freescale DMA channel * @enable : 0 is disabled, 1 is enabled. * * The Freescale DMA channel can be controlled by the external signal DREQ#. * The DMA Request Count feature should be used in addition to this feature * to set the number of bytes to transfer before pausing the channel. */ -static void fsl_chan_toggle_ext_pause(struct fsl_dma_chan *fsl_chan, int enable) +static void fsl_chan_toggle_ext_pause(struct fsldma_chan *chan, int enable) { if (enable) - fsl_chan->feature |= FSL_DMA_CHAN_PAUSE_EXT; + chan->feature |= FSL_DMA_CHAN_PAUSE_EXT; else - fsl_chan->feature &= ~FSL_DMA_CHAN_PAUSE_EXT; + chan->feature &= ~FSL_DMA_CHAN_PAUSE_EXT; } /** * fsl_chan_toggle_ext_start - Toggle channel external start status - * @fsl_chan : Freescale DMA channel + * @chan : Freescale DMA channel * @enable : 0 is disabled, 1 is enabled. * * If enable the external start, the channel can be started by an @@ -328,141 +310,196 @@ static void fsl_chan_toggle_ext_pause(struct fsl_dma_chan *fsl_chan, int enable) * transfer immediately. The DMA channel will wait for the * control pin asserted. */ -static void fsl_chan_toggle_ext_start(struct fsl_dma_chan *fsl_chan, int enable) +static void fsl_chan_toggle_ext_start(struct fsldma_chan *chan, int enable) { if (enable) - fsl_chan->feature |= FSL_DMA_CHAN_START_EXT; + chan->feature |= FSL_DMA_CHAN_START_EXT; else - fsl_chan->feature &= ~FSL_DMA_CHAN_START_EXT; + chan->feature &= ~FSL_DMA_CHAN_START_EXT; +} + +static void append_ld_queue(struct fsldma_chan *chan, + struct fsl_desc_sw *desc) +{ + struct fsl_desc_sw *tail = to_fsl_desc(chan->ld_pending.prev); + + if (list_empty(&chan->ld_pending)) + goto out_splice; + + /* + * Add the hardware descriptor to the chain of hardware descriptors + * that already exists in memory. + * + * This will un-set the EOL bit of the existing transaction, and the + * last link in this transaction will become the EOL descriptor. + */ + set_desc_next(chan, &tail->hw, desc->async_tx.phys); + + /* + * Add the software descriptor and all children to the list + * of pending transactions + */ +out_splice: + list_splice_tail_init(&desc->tx_list, &chan->ld_pending); } static dma_cookie_t fsl_dma_tx_submit(struct dma_async_tx_descriptor *tx) { - struct fsl_dma_chan *fsl_chan = to_fsl_chan(tx->chan); + struct fsldma_chan *chan = to_fsl_chan(tx->chan); struct fsl_desc_sw *desc = tx_to_fsl_desc(tx); struct fsl_desc_sw *child; unsigned long flags; dma_cookie_t cookie; - /* cookie increment and adding to ld_queue must be atomic */ - spin_lock_irqsave(&fsl_chan->desc_lock, flags); + spin_lock_irqsave(&chan->desc_lock, flags); - cookie = fsl_chan->common.cookie; + /* + * assign cookies to all of the software descriptors + * that make up this transaction + */ + cookie = chan->common.cookie; list_for_each_entry(child, &desc->tx_list, node) { cookie++; if (cookie < 0) cookie = 1; - desc->async_tx.cookie = cookie; + child->async_tx.cookie = cookie; } - fsl_chan->common.cookie = cookie; - append_ld_queue(fsl_chan, desc); - list_splice_init(&desc->tx_list, fsl_chan->ld_queue.prev); + chan->common.cookie = cookie; - spin_unlock_irqrestore(&fsl_chan->desc_lock, flags); + /* put this transaction onto the tail of the pending queue */ + append_ld_queue(chan, desc); + + spin_unlock_irqrestore(&chan->desc_lock, flags); return cookie; } /** * fsl_dma_alloc_descriptor - Allocate descriptor from channel's DMA pool. - * @fsl_chan : Freescale DMA channel + * @chan : Freescale DMA channel * * Return - The descriptor allocated. NULL for failed. */ static struct fsl_desc_sw *fsl_dma_alloc_descriptor( - struct fsl_dma_chan *fsl_chan) + struct fsldma_chan *chan) { + struct fsl_desc_sw *desc; dma_addr_t pdesc; - struct fsl_desc_sw *desc_sw; - - desc_sw = dma_pool_alloc(fsl_chan->desc_pool, GFP_ATOMIC, &pdesc); - if (desc_sw) { - memset(desc_sw, 0, sizeof(struct fsl_desc_sw)); - INIT_LIST_HEAD(&desc_sw->tx_list); - dma_async_tx_descriptor_init(&desc_sw->async_tx, - &fsl_chan->common); - desc_sw->async_tx.tx_submit = fsl_dma_tx_submit; - desc_sw->async_tx.phys = pdesc; + + desc = dma_pool_alloc(chan->desc_pool, GFP_ATOMIC, &pdesc); + if (!desc) { + dev_dbg(chan->dev, "out of memory for link desc\n"); + return NULL; } - return desc_sw; + memset(desc, 0, sizeof(*desc)); + INIT_LIST_HEAD(&desc->tx_list); + dma_async_tx_descriptor_init(&desc->async_tx, &chan->common); + desc->async_tx.tx_submit = fsl_dma_tx_submit; + desc->async_tx.phys = pdesc; + + return desc; } /** * fsl_dma_alloc_chan_resources - Allocate resources for DMA channel. - * @fsl_chan : Freescale DMA channel + * @chan : Freescale DMA channel * * This function will create a dma pool for descriptor allocation. * * Return - The number of descriptors allocated. */ -static int fsl_dma_alloc_chan_resources(struct dma_chan *chan) +static int fsl_dma_alloc_chan_resources(struct dma_chan *dchan) { - struct fsl_dma_chan *fsl_chan = to_fsl_chan(chan); + struct fsldma_chan *chan = to_fsl_chan(dchan); /* Has this channel already been allocated? */ - if (fsl_chan->desc_pool) + if (chan->desc_pool) return 1; - /* We need the descriptor to be aligned to 32bytes + /* + * We need the descriptor to be aligned to 32bytes * for meeting FSL DMA specification requirement. */ - fsl_chan->desc_pool = dma_pool_create("fsl_dma_engine_desc_pool", - fsl_chan->dev, sizeof(struct fsl_desc_sw), - 32, 0); - if (!fsl_chan->desc_pool) { - dev_err(fsl_chan->dev, "No memory for channel %d " - "descriptor dma pool.\n", fsl_chan->id); - return 0; + chan->desc_pool = dma_pool_create("fsl_dma_engine_desc_pool", + chan->dev, + sizeof(struct fsl_desc_sw), + __alignof__(struct fsl_desc_sw), 0); + if (!chan->desc_pool) { + dev_err(chan->dev, "unable to allocate channel %d " + "descriptor pool\n", chan->id); + return -ENOMEM; } + /* there is at least one descriptor free to be allocated */ return 1; } /** - * fsl_dma_free_chan_resources - Free all resources of the channel. - * @fsl_chan : Freescale DMA channel + * fsldma_free_desc_list - Free all descriptors in a queue + * @chan: Freescae DMA channel + * @list: the list to free + * + * LOCKING: must hold chan->desc_lock */ -static void fsl_dma_free_chan_resources(struct dma_chan *chan) +static void fsldma_free_desc_list(struct fsldma_chan *chan, + struct list_head *list) { - struct fsl_dma_chan *fsl_chan = to_fsl_chan(chan); struct fsl_desc_sw *desc, *_desc; - unsigned long flags; - dev_dbg(fsl_chan->dev, "Free all channel resources.\n"); - spin_lock_irqsave(&fsl_chan->desc_lock, flags); - list_for_each_entry_safe(desc, _desc, &fsl_chan->ld_queue, node) { -#ifdef FSL_DMA_LD_DEBUG - dev_dbg(fsl_chan->dev, - "LD %p will be released.\n", desc); -#endif + list_for_each_entry_safe(desc, _desc, list, node) { + list_del(&desc->node); + dma_pool_free(chan->desc_pool, desc, desc->async_tx.phys); + } +} + +static void fsldma_free_desc_list_reverse(struct fsldma_chan *chan, + struct list_head *list) +{ + struct fsl_desc_sw *desc, *_desc; + + list_for_each_entry_safe_reverse(desc, _desc, list, node) { list_del(&desc->node); - /* free link descriptor */ - dma_pool_free(fsl_chan->desc_pool, desc, desc->async_tx.phys); + dma_pool_free(chan->desc_pool, desc, desc->async_tx.phys); } - spin_unlock_irqrestore(&fsl_chan->desc_lock, flags); - dma_pool_destroy(fsl_chan->desc_pool); +} + +/** + * fsl_dma_free_chan_resources - Free all resources of the channel. + * @chan : Freescale DMA channel + */ +static void fsl_dma_free_chan_resources(struct dma_chan *dchan) +{ + struct fsldma_chan *chan = to_fsl_chan(dchan); + unsigned long flags; + + dev_dbg(chan->dev, "Free all channel resources.\n"); + spin_lock_irqsave(&chan->desc_lock, flags); + fsldma_free_desc_list(chan, &chan->ld_pending); + fsldma_free_desc_list(chan, &chan->ld_running); + spin_unlock_irqrestore(&chan->desc_lock, flags); - fsl_chan->desc_pool = NULL; + dma_pool_destroy(chan->desc_pool); + chan->desc_pool = NULL; } static struct dma_async_tx_descriptor * -fsl_dma_prep_interrupt(struct dma_chan *chan, unsigned long flags) +fsl_dma_prep_interrupt(struct dma_chan *dchan, unsigned long flags) { - struct fsl_dma_chan *fsl_chan; + struct fsldma_chan *chan; struct fsl_desc_sw *new; - if (!chan) + if (!dchan) return NULL; - fsl_chan = to_fsl_chan(chan); + chan = to_fsl_chan(dchan); - new = fsl_dma_alloc_descriptor(fsl_chan); + new = fsl_dma_alloc_descriptor(chan); if (!new) { - dev_err(fsl_chan->dev, "No free memory for link descriptor\n"); + dev_err(chan->dev, "No free memory for link descriptor\n"); return NULL; } @@ -473,51 +510,50 @@ fsl_dma_prep_interrupt(struct dma_chan *chan, unsigned long flags) list_add_tail(&new->node, &new->tx_list); /* Set End-of-link to the last link descriptor of new list*/ - set_ld_eol(fsl_chan, new); + set_ld_eol(chan, new); return &new->async_tx; } static struct dma_async_tx_descriptor *fsl_dma_prep_memcpy( - struct dma_chan *chan, dma_addr_t dma_dest, dma_addr_t dma_src, + struct dma_chan *dchan, dma_addr_t dma_dst, dma_addr_t dma_src, size_t len, unsigned long flags) { - struct fsl_dma_chan *fsl_chan; + struct fsldma_chan *chan; struct fsl_desc_sw *first = NULL, *prev = NULL, *new; - struct list_head *list; size_t copy; - if (!chan) + if (!dchan) return NULL; if (!len) return NULL; - fsl_chan = to_fsl_chan(chan); + chan = to_fsl_chan(dchan); do { /* Allocate the link descriptor from DMA pool */ - new = fsl_dma_alloc_descriptor(fsl_chan); + new = fsl_dma_alloc_descriptor(chan); if (!new) { - dev_err(fsl_chan->dev, + dev_err(chan->dev, "No free memory for link descriptor\n"); goto fail; } #ifdef FSL_DMA_LD_DEBUG - dev_dbg(fsl_chan->dev, "new link desc alloc %p\n", new); + dev_dbg(chan->dev, "new link desc alloc %p\n", new); #endif copy = min(len, (size_t)FSL_DMA_BCR_MAX_CNT); - set_desc_cnt(fsl_chan, &new->hw, copy); - set_desc_src(fsl_chan, &new->hw, dma_src); - set_desc_dest(fsl_chan, &new->hw, dma_dest); + set_desc_cnt(chan, &new->hw, copy); + set_desc_src(chan, &new->hw, dma_src); + set_desc_dst(chan, &new->hw, dma_dst); if (!first) first = new; else - set_desc_next(fsl_chan, &prev->hw, new->async_tx.phys); + set_desc_next(chan, &prev->hw, new->async_tx.phys); new->async_tx.cookie = 0; async_tx_ack(&new->async_tx); @@ -525,7 +561,7 @@ static struct dma_async_tx_descriptor *fsl_dma_prep_memcpy( prev = new; len -= copy; dma_src += copy; - dma_dest += copy; + dma_dst += copy; /* Insert the link descriptor to the LD ring */ list_add_tail(&new->node, &first->tx_list); @@ -535,7 +571,7 @@ static struct dma_async_tx_descriptor *fsl_dma_prep_memcpy( new->async_tx.cookie = -EBUSY; /* Set End-of-link to the last link descriptor of new list*/ - set_ld_eol(fsl_chan, new); + set_ld_eol(chan, new); return &first->async_tx; @@ -543,12 +579,7 @@ fail: if (!first) return NULL; - list = &first->tx_list; - list_for_each_entry_safe_reverse(new, prev, list, node) { - list_del(&new->node); - dma_pool_free(fsl_chan->desc_pool, new, new->async_tx.phys); - } - + fsldma_free_desc_list_reverse(chan, &first->tx_list); return NULL; } @@ -565,13 +596,12 @@ fail: * chan->private variable. */ static struct dma_async_tx_descriptor *fsl_dma_prep_slave_sg( - struct dma_chan *chan, struct scatterlist *sgl, unsigned int sg_len, + struct dma_chan *dchan, struct scatterlist *sgl, unsigned int sg_len, enum dma_data_direction direction, unsigned long flags) { - struct fsl_dma_chan *fsl_chan; + struct fsldma_chan *chan; struct fsl_desc_sw *first = NULL, *prev = NULL, *new = NULL; struct fsl_dma_slave *slave; - struct list_head *tx_list; size_t copy; int i; @@ -581,14 +611,14 @@ static struct dma_async_tx_descriptor *fsl_dma_prep_slave_sg( struct fsl_dma_hw_addr *hw; dma_addr_t dma_dst, dma_src; - if (!chan) + if (!dchan) return NULL; - if (!chan->private) + if (!dchan->private) return NULL; - fsl_chan = to_fsl_chan(chan); - slave = chan->private; + chan = to_fsl_chan(dchan); + slave = dchan->private; if (list_empty(&slave->addresses)) return NULL; @@ -637,14 +667,14 @@ static struct dma_async_tx_descriptor *fsl_dma_prep_slave_sg( } /* Allocate the link descriptor from DMA pool */ - new = fsl_dma_alloc_descriptor(fsl_chan); + new = fsl_dma_alloc_descriptor(chan); if (!new) { - dev_err(fsl_chan->dev, "No free memory for " + dev_err(chan->dev, "No free memory for " "link descriptor\n"); goto fail; } #ifdef FSL_DMA_LD_DEBUG - dev_dbg(fsl_chan->dev, "new link desc alloc %p\n", new); + dev_dbg(chan->dev, "new link desc alloc %p\n", new); #endif /* @@ -671,9 +701,9 @@ static struct dma_async_tx_descriptor *fsl_dma_prep_slave_sg( } /* Fill in the descriptor */ - set_desc_cnt(fsl_chan, &new->hw, copy); - set_desc_src(fsl_chan, &new->hw, dma_src); - set_desc_dest(fsl_chan, &new->hw, dma_dst); + set_desc_cnt(chan, &new->hw, copy); + set_desc_src(chan, &new->hw, dma_src); + set_desc_dst(chan, &new->hw, dma_dst); /* * If this is not the first descriptor, chain the @@ -682,7 +712,7 @@ static struct dma_async_tx_descriptor *fsl_dma_prep_slave_sg( if (!first) { first = new; } else { - set_desc_next(fsl_chan, &prev->hw, + set_desc_next(chan, &prev->hw, new->async_tx.phys); } @@ -708,23 +738,23 @@ finished: new->async_tx.cookie = -EBUSY; /* Set End-of-link to the last link descriptor of new list */ - set_ld_eol(fsl_chan, new); + set_ld_eol(chan, new); /* Enable extra controller features */ - if (fsl_chan->set_src_loop_size) - fsl_chan->set_src_loop_size(fsl_chan, slave->src_loop_size); + if (chan->set_src_loop_size) + chan->set_src_loop_size(chan, slave->src_loop_size); - if (fsl_chan->set_dest_loop_size) - fsl_chan->set_dest_loop_size(fsl_chan, slave->dst_loop_size); + if (chan->set_dst_loop_size) + chan->set_dst_loop_size(chan, slave->dst_loop_size); - if (fsl_chan->toggle_ext_start) - fsl_chan->toggle_ext_start(fsl_chan, slave->external_start); + if (chan->toggle_ext_start) + chan->toggle_ext_start(chan, slave->external_start); - if (fsl_chan->toggle_ext_pause) - fsl_chan->toggle_ext_pause(fsl_chan, slave->external_pause); + if (chan->toggle_ext_pause) + chan->toggle_ext_pause(chan, slave->external_pause); - if (fsl_chan->set_request_count) - fsl_chan->set_request_count(fsl_chan, slave->request_count); + if (chan->set_request_count) + chan->set_request_count(chan, slave->request_count); return &first->async_tx; @@ -741,251 +771,259 @@ fail: * * We're re-using variables for the loop, oh well */ - tx_list = &first->tx_list; - list_for_each_entry_safe_reverse(new, prev, tx_list, node) { - list_del_init(&new->node); - dma_pool_free(fsl_chan->desc_pool, new, new->async_tx.phys); - } - + fsldma_free_desc_list_reverse(chan, &first->tx_list); return NULL; } -static void fsl_dma_device_terminate_all(struct dma_chan *chan) +static int fsl_dma_device_control(struct dma_chan *dchan, + enum dma_ctrl_cmd cmd, unsigned long arg) { - struct fsl_dma_chan *fsl_chan; - struct fsl_desc_sw *desc, *tmp; + struct fsldma_chan *chan; unsigned long flags; - if (!chan) - return; + /* Only supports DMA_TERMINATE_ALL */ + if (cmd != DMA_TERMINATE_ALL) + return -ENXIO; - fsl_chan = to_fsl_chan(chan); + if (!dchan) + return -EINVAL; + + chan = to_fsl_chan(dchan); /* Halt the DMA engine */ - dma_halt(fsl_chan); + dma_halt(chan); - spin_lock_irqsave(&fsl_chan->desc_lock, flags); + spin_lock_irqsave(&chan->desc_lock, flags); /* Remove and free all of the descriptors in the LD queue */ - list_for_each_entry_safe(desc, tmp, &fsl_chan->ld_queue, node) { - list_del(&desc->node); - dma_pool_free(fsl_chan->desc_pool, desc, desc->async_tx.phys); - } + fsldma_free_desc_list(chan, &chan->ld_pending); + fsldma_free_desc_list(chan, &chan->ld_running); - spin_unlock_irqrestore(&fsl_chan->desc_lock, flags); + spin_unlock_irqrestore(&chan->desc_lock, flags); + + return 0; } /** * fsl_dma_update_completed_cookie - Update the completed cookie. - * @fsl_chan : Freescale DMA channel + * @chan : Freescale DMA channel + * + * CONTEXT: hardirq */ -static void fsl_dma_update_completed_cookie(struct fsl_dma_chan *fsl_chan) +static void fsl_dma_update_completed_cookie(struct fsldma_chan *chan) { - struct fsl_desc_sw *cur_desc, *desc; - dma_addr_t ld_phy; + struct fsl_desc_sw *desc; + unsigned long flags; + dma_cookie_t cookie; - ld_phy = get_cdar(fsl_chan) & FSL_DMA_NLDA_MASK; + spin_lock_irqsave(&chan->desc_lock, flags); - if (ld_phy) { - cur_desc = NULL; - list_for_each_entry(desc, &fsl_chan->ld_queue, node) - if (desc->async_tx.phys == ld_phy) { - cur_desc = desc; - break; - } + if (list_empty(&chan->ld_running)) { + dev_dbg(chan->dev, "no running descriptors\n"); + goto out_unlock; + } - if (cur_desc && cur_desc->async_tx.cookie) { - if (dma_is_idle(fsl_chan)) - fsl_chan->completed_cookie = - cur_desc->async_tx.cookie; - else - fsl_chan->completed_cookie = - cur_desc->async_tx.cookie - 1; - } + /* Get the last descriptor, update the cookie to that */ + desc = to_fsl_desc(chan->ld_running.prev); + if (dma_is_idle(chan)) + cookie = desc->async_tx.cookie; + else { + cookie = desc->async_tx.cookie - 1; + if (unlikely(cookie < DMA_MIN_COOKIE)) + cookie = DMA_MAX_COOKIE; } + + chan->completed_cookie = cookie; + +out_unlock: + spin_unlock_irqrestore(&chan->desc_lock, flags); +} + +/** + * fsldma_desc_status - Check the status of a descriptor + * @chan: Freescale DMA channel + * @desc: DMA SW descriptor + * + * This function will return the status of the given descriptor + */ +static enum dma_status fsldma_desc_status(struct fsldma_chan *chan, + struct fsl_desc_sw *desc) +{ + return dma_async_is_complete(desc->async_tx.cookie, + chan->completed_cookie, + chan->common.cookie); } /** * fsl_chan_ld_cleanup - Clean up link descriptors - * @fsl_chan : Freescale DMA channel + * @chan : Freescale DMA channel * * This function clean up the ld_queue of DMA channel. - * If 'in_intr' is set, the function will move the link descriptor to - * the recycle list. Otherwise, free it directly. */ -static void fsl_chan_ld_cleanup(struct fsl_dma_chan *fsl_chan) +static void fsl_chan_ld_cleanup(struct fsldma_chan *chan) { struct fsl_desc_sw *desc, *_desc; unsigned long flags; - spin_lock_irqsave(&fsl_chan->desc_lock, flags); + spin_lock_irqsave(&chan->desc_lock, flags); - dev_dbg(fsl_chan->dev, "chan completed_cookie = %d\n", - fsl_chan->completed_cookie); - list_for_each_entry_safe(desc, _desc, &fsl_chan->ld_queue, node) { + dev_dbg(chan->dev, "chan completed_cookie = %d\n", chan->completed_cookie); + list_for_each_entry_safe(desc, _desc, &chan->ld_running, node) { dma_async_tx_callback callback; void *callback_param; - if (dma_async_is_complete(desc->async_tx.cookie, - fsl_chan->completed_cookie, fsl_chan->common.cookie) - == DMA_IN_PROGRESS) + if (fsldma_desc_status(chan, desc) == DMA_IN_PROGRESS) break; - callback = desc->async_tx.callback; - callback_param = desc->async_tx.callback_param; - - /* Remove from ld_queue list */ + /* Remove from the list of running transactions */ list_del(&desc->node); - dev_dbg(fsl_chan->dev, "link descriptor %p will be recycle.\n", - desc); - dma_pool_free(fsl_chan->desc_pool, desc, desc->async_tx.phys); - /* Run the link descriptor callback function */ + callback = desc->async_tx.callback; + callback_param = desc->async_tx.callback_param; if (callback) { - spin_unlock_irqrestore(&fsl_chan->desc_lock, flags); - dev_dbg(fsl_chan->dev, "link descriptor %p callback\n", - desc); + spin_unlock_irqrestore(&chan->desc_lock, flags); + dev_dbg(chan->dev, "LD %p callback\n", desc); callback(callback_param); - spin_lock_irqsave(&fsl_chan->desc_lock, flags); + spin_lock_irqsave(&chan->desc_lock, flags); } + + /* Run any dependencies, then free the descriptor */ + dma_run_dependencies(&desc->async_tx); + dma_pool_free(chan->desc_pool, desc, desc->async_tx.phys); } - spin_unlock_irqrestore(&fsl_chan->desc_lock, flags); + + spin_unlock_irqrestore(&chan->desc_lock, flags); } /** - * fsl_chan_xfer_ld_queue - Transfer link descriptors in channel ld_queue. - * @fsl_chan : Freescale DMA channel + * fsl_chan_xfer_ld_queue - transfer any pending transactions + * @chan : Freescale DMA channel + * + * This will make sure that any pending transactions will be run. + * If the DMA controller is idle, it will be started. Otherwise, + * the DMA controller's interrupt handler will start any pending + * transactions when it becomes idle. */ -static void fsl_chan_xfer_ld_queue(struct fsl_dma_chan *fsl_chan) +static void fsl_chan_xfer_ld_queue(struct fsldma_chan *chan) { - struct list_head *ld_node; - dma_addr_t next_dest_addr; + struct fsl_desc_sw *desc; unsigned long flags; - spin_lock_irqsave(&fsl_chan->desc_lock, flags); + spin_lock_irqsave(&chan->desc_lock, flags); - if (!dma_is_idle(fsl_chan)) + /* + * If the list of pending descriptors is empty, then we + * don't need to do any work at all + */ + if (list_empty(&chan->ld_pending)) { + dev_dbg(chan->dev, "no pending LDs\n"); + goto out_unlock; + } + + /* + * The DMA controller is not idle, which means the interrupt + * handler will start any queued transactions when it runs + * at the end of the current transaction + */ + if (!dma_is_idle(chan)) { + dev_dbg(chan->dev, "DMA controller still busy\n"); goto out_unlock; + } - dma_halt(fsl_chan); + /* + * TODO: + * make sure the dma_halt() function really un-wedges the + * controller as much as possible + */ + dma_halt(chan); - /* If there are some link descriptors - * not transfered in queue. We need to start it. + /* + * If there are some link descriptors which have not been + * transferred, we need to start the controller */ - /* Find the first un-transfer desciptor */ - for (ld_node = fsl_chan->ld_queue.next; - (ld_node != &fsl_chan->ld_queue) - && (dma_async_is_complete( - to_fsl_desc(ld_node)->async_tx.cookie, - fsl_chan->completed_cookie, - fsl_chan->common.cookie) == DMA_SUCCESS); - ld_node = ld_node->next); - - if (ld_node != &fsl_chan->ld_queue) { - /* Get the ld start address from ld_queue */ - next_dest_addr = to_fsl_desc(ld_node)->async_tx.phys; - dev_dbg(fsl_chan->dev, "xfer LDs staring from 0x%llx\n", - (unsigned long long)next_dest_addr); - set_cdar(fsl_chan, next_dest_addr); - dma_start(fsl_chan); - } else { - set_cdar(fsl_chan, 0); - set_ndar(fsl_chan, 0); - } + /* + * Move all elements from the queue of pending transactions + * onto the list of running transactions + */ + desc = list_first_entry(&chan->ld_pending, struct fsl_desc_sw, node); + list_splice_tail_init(&chan->ld_pending, &chan->ld_running); + + /* + * Program the descriptor's address into the DMA controller, + * then start the DMA transaction + */ + set_cdar(chan, desc->async_tx.phys); + dma_start(chan); out_unlock: - spin_unlock_irqrestore(&fsl_chan->desc_lock, flags); + spin_unlock_irqrestore(&chan->desc_lock, flags); } /** * fsl_dma_memcpy_issue_pending - Issue the DMA start command - * @fsl_chan : Freescale DMA channel + * @chan : Freescale DMA channel */ -static void fsl_dma_memcpy_issue_pending(struct dma_chan *chan) +static void fsl_dma_memcpy_issue_pending(struct dma_chan *dchan) { - struct fsl_dma_chan *fsl_chan = to_fsl_chan(chan); - -#ifdef FSL_DMA_LD_DEBUG - struct fsl_desc_sw *ld; - unsigned long flags; - - spin_lock_irqsave(&fsl_chan->desc_lock, flags); - if (list_empty(&fsl_chan->ld_queue)) { - spin_unlock_irqrestore(&fsl_chan->desc_lock, flags); - return; - } - - dev_dbg(fsl_chan->dev, "--memcpy issue--\n"); - list_for_each_entry(ld, &fsl_chan->ld_queue, node) { - int i; - dev_dbg(fsl_chan->dev, "Ch %d, LD %08x\n", - fsl_chan->id, ld->async_tx.phys); - for (i = 0; i < 8; i++) - dev_dbg(fsl_chan->dev, "LD offset %d: %08x\n", - i, *(((u32 *)&ld->hw) + i)); - } - dev_dbg(fsl_chan->dev, "----------------\n"); - spin_unlock_irqrestore(&fsl_chan->desc_lock, flags); -#endif - - fsl_chan_xfer_ld_queue(fsl_chan); + struct fsldma_chan *chan = to_fsl_chan(dchan); + fsl_chan_xfer_ld_queue(chan); } /** - * fsl_dma_is_complete - Determine the DMA status - * @fsl_chan : Freescale DMA channel + * fsl_tx_status - Determine the DMA status + * @chan : Freescale DMA channel */ -static enum dma_status fsl_dma_is_complete(struct dma_chan *chan, +static enum dma_status fsl_tx_status(struct dma_chan *dchan, dma_cookie_t cookie, - dma_cookie_t *done, - dma_cookie_t *used) + struct dma_tx_state *txstate) { - struct fsl_dma_chan *fsl_chan = to_fsl_chan(chan); + struct fsldma_chan *chan = to_fsl_chan(dchan); dma_cookie_t last_used; dma_cookie_t last_complete; - fsl_chan_ld_cleanup(fsl_chan); + fsl_chan_ld_cleanup(chan); - last_used = chan->cookie; - last_complete = fsl_chan->completed_cookie; + last_used = dchan->cookie; + last_complete = chan->completed_cookie; - if (done) - *done = last_complete; - - if (used) - *used = last_used; + dma_set_tx_state(txstate, last_complete, last_used, 0); return dma_async_is_complete(cookie, last_complete, last_used); } -static irqreturn_t fsl_dma_chan_do_interrupt(int irq, void *data) +/*----------------------------------------------------------------------------*/ +/* Interrupt Handling */ +/*----------------------------------------------------------------------------*/ + +static irqreturn_t fsldma_chan_irq(int irq, void *data) { - struct fsl_dma_chan *fsl_chan = (struct fsl_dma_chan *)data; - u32 stat; + struct fsldma_chan *chan = data; int update_cookie = 0; int xfer_ld_q = 0; + u32 stat; - stat = get_sr(fsl_chan); - dev_dbg(fsl_chan->dev, "event: channel %d, stat = 0x%x\n", - fsl_chan->id, stat); - set_sr(fsl_chan, stat); /* Clear the event register */ + /* save and clear the status register */ + stat = get_sr(chan); + set_sr(chan, stat); + dev_dbg(chan->dev, "irq: channel %d, stat = 0x%x\n", chan->id, stat); stat &= ~(FSL_DMA_SR_CB | FSL_DMA_SR_CH); if (!stat) return IRQ_NONE; if (stat & FSL_DMA_SR_TE) - dev_err(fsl_chan->dev, "Transfer Error!\n"); + dev_err(chan->dev, "Transfer Error!\n"); - /* Programming Error + /* + * Programming Error * The DMA_INTERRUPT async_tx is a NULL transfer, which will * triger a PE interrupt. */ if (stat & FSL_DMA_SR_PE) { - dev_dbg(fsl_chan->dev, "event: Programming Error INT\n"); - if (get_bcr(fsl_chan) == 0) { + dev_dbg(chan->dev, "irq: Programming Error INT\n"); + if (get_bcr(chan) == 0) { /* BCR register is 0, this is a DMA_INTERRUPT async_tx. * Now, update the completed cookie, and continue the * next uncompleted transfer. @@ -996,208 +1034,296 @@ static irqreturn_t fsl_dma_chan_do_interrupt(int irq, void *data) stat &= ~FSL_DMA_SR_PE; } - /* If the link descriptor segment transfer finishes, + /* + * If the link descriptor segment transfer finishes, * we will recycle the used descriptor. */ if (stat & FSL_DMA_SR_EOSI) { - dev_dbg(fsl_chan->dev, "event: End-of-segments INT\n"); - dev_dbg(fsl_chan->dev, "event: clndar 0x%llx, nlndar 0x%llx\n", - (unsigned long long)get_cdar(fsl_chan), - (unsigned long long)get_ndar(fsl_chan)); + dev_dbg(chan->dev, "irq: End-of-segments INT\n"); + dev_dbg(chan->dev, "irq: clndar 0x%llx, nlndar 0x%llx\n", + (unsigned long long)get_cdar(chan), + (unsigned long long)get_ndar(chan)); stat &= ~FSL_DMA_SR_EOSI; update_cookie = 1; } - /* For MPC8349, EOCDI event need to update cookie + /* + * For MPC8349, EOCDI event need to update cookie * and start the next transfer if it exist. */ if (stat & FSL_DMA_SR_EOCDI) { - dev_dbg(fsl_chan->dev, "event: End-of-Chain link INT\n"); + dev_dbg(chan->dev, "irq: End-of-Chain link INT\n"); stat &= ~FSL_DMA_SR_EOCDI; update_cookie = 1; xfer_ld_q = 1; } - /* If it current transfer is the end-of-transfer, + /* + * If it current transfer is the end-of-transfer, * we should clear the Channel Start bit for * prepare next transfer. */ if (stat & FSL_DMA_SR_EOLNI) { - dev_dbg(fsl_chan->dev, "event: End-of-link INT\n"); + dev_dbg(chan->dev, "irq: End-of-link INT\n"); stat &= ~FSL_DMA_SR_EOLNI; xfer_ld_q = 1; } if (update_cookie) - fsl_dma_update_completed_cookie(fsl_chan); + fsl_dma_update_completed_cookie(chan); if (xfer_ld_q) - fsl_chan_xfer_ld_queue(fsl_chan); + fsl_chan_xfer_ld_queue(chan); if (stat) - dev_dbg(fsl_chan->dev, "event: unhandled sr 0x%02x\n", - stat); + dev_dbg(chan->dev, "irq: unhandled sr 0x%02x\n", stat); - dev_dbg(fsl_chan->dev, "event: Exit\n"); - tasklet_schedule(&fsl_chan->tasklet); + dev_dbg(chan->dev, "irq: Exit\n"); + tasklet_schedule(&chan->tasklet); return IRQ_HANDLED; } -static irqreturn_t fsl_dma_do_interrupt(int irq, void *data) +static void dma_do_tasklet(unsigned long data) { - struct fsl_dma_device *fdev = (struct fsl_dma_device *)data; - u32 gsr; - int ch_nr; + struct fsldma_chan *chan = (struct fsldma_chan *)data; + fsl_chan_ld_cleanup(chan); +} + +static irqreturn_t fsldma_ctrl_irq(int irq, void *data) +{ + struct fsldma_device *fdev = data; + struct fsldma_chan *chan; + unsigned int handled = 0; + u32 gsr, mask; + int i; - gsr = (fdev->feature & FSL_DMA_BIG_ENDIAN) ? in_be32(fdev->reg_base) - : in_le32(fdev->reg_base); - ch_nr = (32 - ffs(gsr)) / 8; + gsr = (fdev->feature & FSL_DMA_BIG_ENDIAN) ? in_be32(fdev->regs) + : in_le32(fdev->regs); + mask = 0xff000000; + dev_dbg(fdev->dev, "IRQ: gsr 0x%.8x\n", gsr); - return fdev->chan[ch_nr] ? fsl_dma_chan_do_interrupt(irq, - fdev->chan[ch_nr]) : IRQ_NONE; + for (i = 0; i < FSL_DMA_MAX_CHANS_PER_DEVICE; i++) { + chan = fdev->chan[i]; + if (!chan) + continue; + + if (gsr & mask) { + dev_dbg(fdev->dev, "IRQ: chan %d\n", chan->id); + fsldma_chan_irq(irq, chan); + handled++; + } + + gsr &= ~mask; + mask >>= 8; + } + + return IRQ_RETVAL(handled); } -static void dma_do_tasklet(unsigned long data) +static void fsldma_free_irqs(struct fsldma_device *fdev) { - struct fsl_dma_chan *fsl_chan = (struct fsl_dma_chan *)data; - fsl_chan_ld_cleanup(fsl_chan); + struct fsldma_chan *chan; + int i; + + if (fdev->irq != NO_IRQ) { + dev_dbg(fdev->dev, "free per-controller IRQ\n"); + free_irq(fdev->irq, fdev); + return; + } + + for (i = 0; i < FSL_DMA_MAX_CHANS_PER_DEVICE; i++) { + chan = fdev->chan[i]; + if (chan && chan->irq != NO_IRQ) { + dev_dbg(fdev->dev, "free channel %d IRQ\n", chan->id); + free_irq(chan->irq, chan); + } + } +} + +static int fsldma_request_irqs(struct fsldma_device *fdev) +{ + struct fsldma_chan *chan; + int ret; + int i; + + /* if we have a per-controller IRQ, use that */ + if (fdev->irq != NO_IRQ) { + dev_dbg(fdev->dev, "request per-controller IRQ\n"); + ret = request_irq(fdev->irq, fsldma_ctrl_irq, IRQF_SHARED, + "fsldma-controller", fdev); + return ret; + } + + /* no per-controller IRQ, use the per-channel IRQs */ + for (i = 0; i < FSL_DMA_MAX_CHANS_PER_DEVICE; i++) { + chan = fdev->chan[i]; + if (!chan) + continue; + + if (chan->irq == NO_IRQ) { + dev_err(fdev->dev, "no interrupts property defined for " + "DMA channel %d. Please fix your " + "device tree\n", chan->id); + ret = -ENODEV; + goto out_unwind; + } + + dev_dbg(fdev->dev, "request channel %d IRQ\n", chan->id); + ret = request_irq(chan->irq, fsldma_chan_irq, IRQF_SHARED, + "fsldma-chan", chan); + if (ret) { + dev_err(fdev->dev, "unable to request IRQ for DMA " + "channel %d\n", chan->id); + goto out_unwind; + } + } + + return 0; + +out_unwind: + for (/* none */; i >= 0; i--) { + chan = fdev->chan[i]; + if (!chan) + continue; + + if (chan->irq == NO_IRQ) + continue; + + free_irq(chan->irq, chan); + } + + return ret; } -static int __devinit fsl_dma_chan_probe(struct fsl_dma_device *fdev, +/*----------------------------------------------------------------------------*/ +/* OpenFirmware Subsystem */ +/*----------------------------------------------------------------------------*/ + +static int __devinit fsl_dma_chan_probe(struct fsldma_device *fdev, struct device_node *node, u32 feature, const char *compatible) { - struct fsl_dma_chan *new_fsl_chan; + struct fsldma_chan *chan; + struct resource res; int err; /* alloc channel */ - new_fsl_chan = kzalloc(sizeof(struct fsl_dma_chan), GFP_KERNEL); - if (!new_fsl_chan) { - dev_err(fdev->dev, "No free memory for allocating " - "dma channels!\n"); - return -ENOMEM; + chan = kzalloc(sizeof(*chan), GFP_KERNEL); + if (!chan) { + dev_err(fdev->dev, "no free memory for DMA channels!\n"); + err = -ENOMEM; + goto out_return; } - /* get dma channel register base */ - err = of_address_to_resource(node, 0, &new_fsl_chan->reg); - if (err) { - dev_err(fdev->dev, "Can't get %s property 'reg'\n", - node->full_name); - goto err_no_reg; + /* ioremap registers for use */ + chan->regs = of_iomap(node, 0); + if (!chan->regs) { + dev_err(fdev->dev, "unable to ioremap registers\n"); + err = -ENOMEM; + goto out_free_chan; } - new_fsl_chan->feature = feature; + err = of_address_to_resource(node, 0, &res); + if (err) { + dev_err(fdev->dev, "unable to find 'reg' property\n"); + goto out_iounmap_regs; + } + chan->feature = feature; if (!fdev->feature) - fdev->feature = new_fsl_chan->feature; + fdev->feature = chan->feature; - /* If the DMA device's feature is different than its channels', - * report the bug. + /* + * If the DMA device's feature is different than the feature + * of its channels, report the bug */ - WARN_ON(fdev->feature != new_fsl_chan->feature); - - new_fsl_chan->dev = fdev->dev; - new_fsl_chan->reg_base = ioremap(new_fsl_chan->reg.start, - new_fsl_chan->reg.end - new_fsl_chan->reg.start + 1); + WARN_ON(fdev->feature != chan->feature); - new_fsl_chan->id = ((new_fsl_chan->reg.start - 0x100) & 0xfff) >> 7; - if (new_fsl_chan->id >= FSL_DMA_MAX_CHANS_PER_DEVICE) { - dev_err(fdev->dev, "There is no %d channel!\n", - new_fsl_chan->id); + chan->dev = fdev->dev; + chan->id = ((res.start - 0x100) & 0xfff) >> 7; + if (chan->id >= FSL_DMA_MAX_CHANS_PER_DEVICE) { + dev_err(fdev->dev, "too many channels for device\n"); err = -EINVAL; - goto err_no_chan; + goto out_iounmap_regs; } - fdev->chan[new_fsl_chan->id] = new_fsl_chan; - tasklet_init(&new_fsl_chan->tasklet, dma_do_tasklet, - (unsigned long)new_fsl_chan); - /* Init the channel */ - dma_init(new_fsl_chan); + fdev->chan[chan->id] = chan; + tasklet_init(&chan->tasklet, dma_do_tasklet, (unsigned long)chan); + + /* Initialize the channel */ + dma_init(chan); /* Clear cdar registers */ - set_cdar(new_fsl_chan, 0); + set_cdar(chan, 0); - switch (new_fsl_chan->feature & FSL_DMA_IP_MASK) { + switch (chan->feature & FSL_DMA_IP_MASK) { case FSL_DMA_IP_85XX: - new_fsl_chan->toggle_ext_pause = fsl_chan_toggle_ext_pause; + chan->toggle_ext_pause = fsl_chan_toggle_ext_pause; case FSL_DMA_IP_83XX: - new_fsl_chan->toggle_ext_start = fsl_chan_toggle_ext_start; - new_fsl_chan->set_src_loop_size = fsl_chan_set_src_loop_size; - new_fsl_chan->set_dest_loop_size = fsl_chan_set_dest_loop_size; - new_fsl_chan->set_request_count = fsl_chan_set_request_count; + chan->toggle_ext_start = fsl_chan_toggle_ext_start; + chan->set_src_loop_size = fsl_chan_set_src_loop_size; + chan->set_dst_loop_size = fsl_chan_set_dst_loop_size; + chan->set_request_count = fsl_chan_set_request_count; } - spin_lock_init(&new_fsl_chan->desc_lock); - INIT_LIST_HEAD(&new_fsl_chan->ld_queue); + spin_lock_init(&chan->desc_lock); + INIT_LIST_HEAD(&chan->ld_pending); + INIT_LIST_HEAD(&chan->ld_running); - new_fsl_chan->common.device = &fdev->common; + chan->common.device = &fdev->common; + + /* find the IRQ line, if it exists in the device tree */ + chan->irq = irq_of_parse_and_map(node, 0); /* Add the channel to DMA device channel list */ - list_add_tail(&new_fsl_chan->common.device_node, - &fdev->common.channels); + list_add_tail(&chan->common.device_node, &fdev->common.channels); fdev->common.chancnt++; - new_fsl_chan->irq = irq_of_parse_and_map(node, 0); - if (new_fsl_chan->irq != NO_IRQ) { - err = request_irq(new_fsl_chan->irq, - &fsl_dma_chan_do_interrupt, IRQF_SHARED, - "fsldma-channel", new_fsl_chan); - if (err) { - dev_err(fdev->dev, "DMA channel %s request_irq error " - "with return %d\n", node->full_name, err); - goto err_no_irq; - } - } - - dev_info(fdev->dev, "#%d (%s), irq %d\n", new_fsl_chan->id, - compatible, - new_fsl_chan->irq != NO_IRQ ? new_fsl_chan->irq : fdev->irq); + dev_info(fdev->dev, "#%d (%s), irq %d\n", chan->id, compatible, + chan->irq != NO_IRQ ? chan->irq : fdev->irq); return 0; -err_no_irq: - list_del(&new_fsl_chan->common.device_node); -err_no_chan: - iounmap(new_fsl_chan->reg_base); -err_no_reg: - kfree(new_fsl_chan); +out_iounmap_regs: + iounmap(chan->regs); +out_free_chan: + kfree(chan); +out_return: return err; } -static void fsl_dma_chan_remove(struct fsl_dma_chan *fchan) +static void fsl_dma_chan_remove(struct fsldma_chan *chan) { - if (fchan->irq != NO_IRQ) - free_irq(fchan->irq, fchan); - list_del(&fchan->common.device_node); - iounmap(fchan->reg_base); - kfree(fchan); + irq_dispose_mapping(chan->irq); + list_del(&chan->common.device_node); + iounmap(chan->regs); + kfree(chan); } -static int __devinit of_fsl_dma_probe(struct of_device *dev, +static int __devinit fsldma_of_probe(struct of_device *op, const struct of_device_id *match) { - int err; - struct fsl_dma_device *fdev; + struct fsldma_device *fdev; struct device_node *child; + int err; - fdev = kzalloc(sizeof(struct fsl_dma_device), GFP_KERNEL); + fdev = kzalloc(sizeof(*fdev), GFP_KERNEL); if (!fdev) { - dev_err(&dev->dev, "No enough memory for 'priv'\n"); - return -ENOMEM; + dev_err(&op->dev, "No enough memory for 'priv'\n"); + err = -ENOMEM; + goto out_return; } - fdev->dev = &dev->dev; + + fdev->dev = &op->dev; INIT_LIST_HEAD(&fdev->common.channels); - /* get DMA controller register base */ - err = of_address_to_resource(dev->node, 0, &fdev->reg); - if (err) { - dev_err(&dev->dev, "Can't get %s property 'reg'\n", - dev->node->full_name); - goto err_no_reg; + /* ioremap the registers for use */ + fdev->regs = of_iomap(op->dev.of_node, 0); + if (!fdev->regs) { + dev_err(&op->dev, "unable to ioremap registers\n"); + err = -ENOMEM; + goto out_free_fdev; } - dev_info(&dev->dev, "Probe the Freescale DMA driver for %s " - "controller at 0x%llx...\n", - match->compatible, (unsigned long long)fdev->reg.start); - fdev->reg_base = ioremap(fdev->reg.start, fdev->reg.end - - fdev->reg.start + 1); + /* map the channel IRQ if it exists, but don't hookup the handler yet */ + fdev->irq = irq_of_parse_and_map(op->dev.of_node, 0); dma_cap_set(DMA_MEMCPY, fdev->common.cap_mask); dma_cap_set(DMA_INTERRUPT, fdev->common.cap_mask); @@ -1206,107 +1332,118 @@ static int __devinit of_fsl_dma_probe(struct of_device *dev, fdev->common.device_free_chan_resources = fsl_dma_free_chan_resources; fdev->common.device_prep_dma_interrupt = fsl_dma_prep_interrupt; fdev->common.device_prep_dma_memcpy = fsl_dma_prep_memcpy; - fdev->common.device_is_tx_complete = fsl_dma_is_complete; + fdev->common.device_tx_status = fsl_tx_status; fdev->common.device_issue_pending = fsl_dma_memcpy_issue_pending; fdev->common.device_prep_slave_sg = fsl_dma_prep_slave_sg; - fdev->common.device_terminate_all = fsl_dma_device_terminate_all; - fdev->common.dev = &dev->dev; - - fdev->irq = irq_of_parse_and_map(dev->node, 0); - if (fdev->irq != NO_IRQ) { - err = request_irq(fdev->irq, &fsl_dma_do_interrupt, IRQF_SHARED, - "fsldma-device", fdev); - if (err) { - dev_err(&dev->dev, "DMA device request_irq error " - "with return %d\n", err); - goto err; - } - } + fdev->common.device_control = fsl_dma_device_control; + fdev->common.dev = &op->dev; - dev_set_drvdata(&(dev->dev), fdev); + dev_set_drvdata(&op->dev, fdev); - /* We cannot use of_platform_bus_probe() because there is no - * of_platform_bus_remove. Instead, we manually instantiate every DMA + /* + * We cannot use of_platform_bus_probe() because there is no + * of_platform_bus_remove(). Instead, we manually instantiate every DMA * channel object. */ - for_each_child_of_node(dev->node, child) { - if (of_device_is_compatible(child, "fsl,eloplus-dma-channel")) + for_each_child_of_node(op->dev.of_node, child) { + if (of_device_is_compatible(child, "fsl,eloplus-dma-channel")) { fsl_dma_chan_probe(fdev, child, FSL_DMA_IP_85XX | FSL_DMA_BIG_ENDIAN, "fsl,eloplus-dma-channel"); - if (of_device_is_compatible(child, "fsl,elo-dma-channel")) + } + + if (of_device_is_compatible(child, "fsl,elo-dma-channel")) { fsl_dma_chan_probe(fdev, child, FSL_DMA_IP_83XX | FSL_DMA_LITTLE_ENDIAN, "fsl,elo-dma-channel"); + } + } + + /* + * Hookup the IRQ handler(s) + * + * If we have a per-controller interrupt, we prefer that to the + * per-channel interrupts to reduce the number of shared interrupt + * handlers on the same IRQ line + */ + err = fsldma_request_irqs(fdev); + if (err) { + dev_err(fdev->dev, "unable to request IRQs\n"); + goto out_free_fdev; } dma_async_device_register(&fdev->common); return 0; -err: - iounmap(fdev->reg_base); -err_no_reg: +out_free_fdev: + irq_dispose_mapping(fdev->irq); kfree(fdev); +out_return: return err; } -static int of_fsl_dma_remove(struct of_device *of_dev) +static int fsldma_of_remove(struct of_device *op) { - struct fsl_dma_device *fdev; + struct fsldma_device *fdev; unsigned int i; - fdev = dev_get_drvdata(&of_dev->dev); - + fdev = dev_get_drvdata(&op->dev); dma_async_device_unregister(&fdev->common); - for (i = 0; i < FSL_DMA_MAX_CHANS_PER_DEVICE; i++) + fsldma_free_irqs(fdev); + + for (i = 0; i < FSL_DMA_MAX_CHANS_PER_DEVICE; i++) { if (fdev->chan[i]) fsl_dma_chan_remove(fdev->chan[i]); + } - if (fdev->irq != NO_IRQ) - free_irq(fdev->irq, fdev); - - iounmap(fdev->reg_base); - + iounmap(fdev->regs); + dev_set_drvdata(&op->dev, NULL); kfree(fdev); - dev_set_drvdata(&of_dev->dev, NULL); return 0; } -static struct of_device_id of_fsl_dma_ids[] = { +static const struct of_device_id fsldma_of_ids[] = { { .compatible = "fsl,eloplus-dma", }, { .compatible = "fsl,elo-dma", }, {} }; -static struct of_platform_driver of_fsl_dma_driver = { - .name = "fsl-elo-dma", - .match_table = of_fsl_dma_ids, - .probe = of_fsl_dma_probe, - .remove = of_fsl_dma_remove, +static struct of_platform_driver fsldma_of_driver = { + .driver = { + .name = "fsl-elo-dma", + .owner = THIS_MODULE, + .of_match_table = fsldma_of_ids, + }, + .probe = fsldma_of_probe, + .remove = fsldma_of_remove, }; -static __init int of_fsl_dma_init(void) +/*----------------------------------------------------------------------------*/ +/* Module Init / Exit */ +/*----------------------------------------------------------------------------*/ + +static __init int fsldma_init(void) { int ret; pr_info("Freescale Elo / Elo Plus DMA driver\n"); - ret = of_register_platform_driver(&of_fsl_dma_driver); + ret = of_register_platform_driver(&fsldma_of_driver); if (ret) pr_err("fsldma: failed to register platform driver\n"); return ret; } -static void __exit of_fsl_dma_exit(void) +static void __exit fsldma_exit(void) { - of_unregister_platform_driver(&of_fsl_dma_driver); + of_unregister_platform_driver(&fsldma_of_driver); } -subsys_initcall(of_fsl_dma_init); -module_exit(of_fsl_dma_exit); +subsys_initcall(fsldma_init); +module_exit(fsldma_exit); MODULE_DESCRIPTION("Freescale Elo / Elo Plus DMA driver"); MODULE_LICENSE("GPL"); diff --git a/drivers/dma/fsldma.h b/drivers/dma/fsldma.h index 0df14cbb8ca3..cb4d6ff51597 100644 --- a/drivers/dma/fsldma.h +++ b/drivers/dma/fsldma.h @@ -92,11 +92,9 @@ struct fsl_desc_sw { struct list_head node; struct list_head tx_list; struct dma_async_tx_descriptor async_tx; - struct list_head *ld; - void *priv; } __attribute__((aligned(32))); -struct fsl_dma_chan_regs { +struct fsldma_chan_regs { u32 mr; /* 0x00 - Mode Register */ u32 sr; /* 0x04 - Status Register */ u64 cdar; /* 0x08 - Current descriptor address register */ @@ -106,20 +104,19 @@ struct fsl_dma_chan_regs { u64 ndar; /* 0x24 - Next Descriptor Address Register */ }; -struct fsl_dma_chan; +struct fsldma_chan; #define FSL_DMA_MAX_CHANS_PER_DEVICE 4 -struct fsl_dma_device { - void __iomem *reg_base; /* DGSR register base */ - struct resource reg; /* Resource for register */ +struct fsldma_device { + void __iomem *regs; /* DGSR register base */ struct device *dev; struct dma_device common; - struct fsl_dma_chan *chan[FSL_DMA_MAX_CHANS_PER_DEVICE]; + struct fsldma_chan *chan[FSL_DMA_MAX_CHANS_PER_DEVICE]; u32 feature; /* The same as DMA channels */ int irq; /* Channel IRQ */ }; -/* Define macros for fsl_dma_chan->feature property */ +/* Define macros for fsldma_chan->feature property */ #define FSL_DMA_LITTLE_ENDIAN 0x00000000 #define FSL_DMA_BIG_ENDIAN 0x00000001 @@ -130,28 +127,28 @@ struct fsl_dma_device { #define FSL_DMA_CHAN_PAUSE_EXT 0x00001000 #define FSL_DMA_CHAN_START_EXT 0x00002000 -struct fsl_dma_chan { - struct fsl_dma_chan_regs __iomem *reg_base; +struct fsldma_chan { + struct fsldma_chan_regs __iomem *regs; dma_cookie_t completed_cookie; /* The maximum cookie completed */ spinlock_t desc_lock; /* Descriptor operation lock */ - struct list_head ld_queue; /* Link descriptors queue */ + struct list_head ld_pending; /* Link descriptors queue */ + struct list_head ld_running; /* Link descriptors queue */ struct dma_chan common; /* DMA common channel */ struct dma_pool *desc_pool; /* Descriptors pool */ struct device *dev; /* Channel device */ - struct resource reg; /* Resource for register */ int irq; /* Channel IRQ */ int id; /* Raw id of this channel */ struct tasklet_struct tasklet; u32 feature; - void (*toggle_ext_pause)(struct fsl_dma_chan *fsl_chan, int enable); - void (*toggle_ext_start)(struct fsl_dma_chan *fsl_chan, int enable); - void (*set_src_loop_size)(struct fsl_dma_chan *fsl_chan, int size); - void (*set_dest_loop_size)(struct fsl_dma_chan *fsl_chan, int size); - void (*set_request_count)(struct fsl_dma_chan *fsl_chan, int size); + void (*toggle_ext_pause)(struct fsldma_chan *fsl_chan, int enable); + void (*toggle_ext_start)(struct fsldma_chan *fsl_chan, int enable); + void (*set_src_loop_size)(struct fsldma_chan *fsl_chan, int size); + void (*set_dst_loop_size)(struct fsldma_chan *fsl_chan, int size); + void (*set_request_count)(struct fsldma_chan *fsl_chan, int size); }; -#define to_fsl_chan(chan) container_of(chan, struct fsl_dma_chan, common) +#define to_fsl_chan(chan) container_of(chan, struct fsldma_chan, common) #define to_fsl_desc(lh) container_of(lh, struct fsl_desc_sw, node) #define tx_to_fsl_desc(tx) container_of(tx, struct fsl_desc_sw, async_tx) diff --git a/drivers/dma/ioat/dma.c b/drivers/dma/ioat/dma.c index dcc4ab78b32b..c9213ead4a26 100644 --- a/drivers/dma/ioat/dma.c +++ b/drivers/dma/ioat/dma.c @@ -27,6 +27,7 @@ #include <linux/init.h> #include <linux/module.h> +#include <linux/slab.h> #include <linux/pci.h> #include <linux/interrupt.h> #include <linux/dmaengine.h> @@ -71,7 +72,7 @@ static irqreturn_t ioat_dma_do_interrupt(int irq, void *data) } attnstatus = readl(instance->reg_base + IOAT_ATTNSTATUS_OFFSET); - for_each_bit(bit, &attnstatus, BITS_PER_LONG) { + for_each_set_bit(bit, &attnstatus, BITS_PER_LONG) { chan = ioat_chan_by_index(instance, bit); tasklet_schedule(&chan->cleanup_task); } @@ -94,16 +95,12 @@ static irqreturn_t ioat_dma_do_interrupt_msix(int irq, void *data) return IRQ_HANDLED; } -static void ioat1_cleanup_tasklet(unsigned long data); - /* common channel initialization */ -void ioat_init_channel(struct ioatdma_device *device, - struct ioat_chan_common *chan, int idx, - void (*timer_fn)(unsigned long), - void (*tasklet)(unsigned long), - unsigned long ioat) +void ioat_init_channel(struct ioatdma_device *device, struct ioat_chan_common *chan, int idx) { struct dma_device *dma = &device->common; + struct dma_chan *c = &chan->common; + unsigned long data = (unsigned long) c; chan->device = device; chan->reg_base = device->reg_base + (0x80 * (idx + 1)); @@ -112,14 +109,12 @@ void ioat_init_channel(struct ioatdma_device *device, list_add_tail(&chan->common.device_node, &dma->channels); device->idx[idx] = chan; init_timer(&chan->timer); - chan->timer.function = timer_fn; - chan->timer.data = ioat; - tasklet_init(&chan->cleanup_task, tasklet, ioat); + chan->timer.function = device->timer_fn; + chan->timer.data = data; + tasklet_init(&chan->cleanup_task, device->cleanup_fn, data); tasklet_disable(&chan->cleanup_task); } -static void ioat1_timer_event(unsigned long data); - /** * ioat1_dma_enumerate_channels - find and initialize the device's channels * @device: the device to be enumerated @@ -155,10 +150,7 @@ static int ioat1_enumerate_channels(struct ioatdma_device *device) if (!ioat) break; - ioat_init_channel(device, &ioat->base, i, - ioat1_timer_event, - ioat1_cleanup_tasklet, - (unsigned long) ioat); + ioat_init_channel(device, &ioat->base, i); ioat->xfercap = xfercap; spin_lock_init(&ioat->desc_lock); INIT_LIST_HEAD(&ioat->free_desc); @@ -532,12 +524,12 @@ ioat1_dma_prep_memcpy(struct dma_chan *c, dma_addr_t dma_dest, return &desc->txd; } -static void ioat1_cleanup_tasklet(unsigned long data) +static void ioat1_cleanup_event(unsigned long data) { - struct ioat_dma_chan *chan = (void *)data; + struct ioat_dma_chan *ioat = to_ioat_chan((void *) data); - ioat1_cleanup(chan); - writew(IOAT_CHANCTRL_RUN, chan->base.reg_base + IOAT_CHANCTRL_OFFSET); + ioat1_cleanup(ioat); + writew(IOAT_CHANCTRL_RUN, ioat->base.reg_base + IOAT_CHANCTRL_OFFSET); } void ioat_dma_unmap(struct ioat_chan_common *chan, enum dma_ctrl_flags flags, @@ -687,7 +679,7 @@ static void ioat1_cleanup(struct ioat_dma_chan *ioat) static void ioat1_timer_event(unsigned long data) { - struct ioat_dma_chan *ioat = (void *) data; + struct ioat_dma_chan *ioat = to_ioat_chan((void *) data); struct ioat_chan_common *chan = &ioat->base; dev_dbg(to_dev(chan), "%s: state: %lx\n", __func__, chan->state); @@ -734,18 +726,19 @@ static void ioat1_timer_event(unsigned long data) spin_unlock_bh(&chan->cleanup_lock); } -static enum dma_status -ioat1_dma_is_complete(struct dma_chan *c, dma_cookie_t cookie, - dma_cookie_t *done, dma_cookie_t *used) +enum dma_status +ioat_dma_tx_status(struct dma_chan *c, dma_cookie_t cookie, + struct dma_tx_state *txstate) { - struct ioat_dma_chan *ioat = to_ioat_chan(c); + struct ioat_chan_common *chan = to_chan_common(c); + struct ioatdma_device *device = chan->device; - if (ioat_is_complete(c, cookie, done, used) == DMA_SUCCESS) + if (ioat_tx_status(c, cookie, txstate) == DMA_SUCCESS) return DMA_SUCCESS; - ioat1_cleanup(ioat); + device->cleanup_fn((unsigned long) c); - return ioat_is_complete(c, cookie, done, used); + return ioat_tx_status(c, cookie, txstate); } static void ioat1_dma_start_null_desc(struct ioat_dma_chan *ioat) @@ -865,7 +858,7 @@ int __devinit ioat_dma_self_test(struct ioatdma_device *device) tmo = wait_for_completion_timeout(&cmp, msecs_to_jiffies(3000)); if (tmo == 0 || - dma->device_is_tx_complete(dma_chan, cookie, NULL, NULL) + dma->device_tx_status(dma_chan, cookie, NULL) != DMA_SUCCESS) { dev_err(dev, "Self-test copy timed out, disabling\n"); err = -ENODEV; @@ -1146,7 +1139,7 @@ ioat_attr_show(struct kobject *kobj, struct attribute *attr, char *page) return entry->show(&chan->common, page); } -struct sysfs_ops ioat_sysfs_ops = { +const struct sysfs_ops ioat_sysfs_ops = { .show = ioat_attr_show, }; @@ -1199,12 +1192,14 @@ int __devinit ioat1_dma_probe(struct ioatdma_device *device, int dca) device->intr_quirk = ioat1_intr_quirk; device->enumerate_channels = ioat1_enumerate_channels; device->self_test = ioat_dma_self_test; + device->timer_fn = ioat1_timer_event; + device->cleanup_fn = ioat1_cleanup_event; dma = &device->common; dma->device_prep_dma_memcpy = ioat1_dma_prep_memcpy; dma->device_issue_pending = ioat1_dma_memcpy_issue_pending; dma->device_alloc_chan_resources = ioat1_dma_alloc_chan_resources; dma->device_free_chan_resources = ioat1_dma_free_chan_resources; - dma->device_is_tx_complete = ioat1_dma_is_complete; + dma->device_tx_status = ioat_dma_tx_status; err = ioat_probe(device); if (err) diff --git a/drivers/dma/ioat/dma.h b/drivers/dma/ioat/dma.h index bbc3e78ef333..6d3a73b57e54 100644 --- a/drivers/dma/ioat/dma.h +++ b/drivers/dma/ioat/dma.h @@ -61,7 +61,7 @@ * @intr_quirk: interrupt setup quirk (for ioat_v1 devices) * @enumerate_channels: hw version specific channel enumeration * @reset_hw: hw version specific channel (re)initialization - * @cleanup_tasklet: select between the v2 and v3 cleanup routines + * @cleanup_fn: select between the v2 and v3 cleanup routines * @timer_fn: select between the v2 and v3 timer watchdog routines * @self_test: hardware version specific self test for each supported op type * @@ -80,7 +80,7 @@ struct ioatdma_device { void (*intr_quirk)(struct ioatdma_device *device); int (*enumerate_channels)(struct ioatdma_device *device); int (*reset_hw)(struct ioat_chan_common *chan); - void (*cleanup_tasklet)(unsigned long data); + void (*cleanup_fn)(unsigned long data); void (*timer_fn)(unsigned long data); int (*self_test)(struct ioatdma_device *device); }; @@ -96,6 +96,7 @@ struct ioat_chan_common { #define IOAT_COMPLETION_ACK 1 #define IOAT_RESET_PENDING 2 #define IOAT_KOBJ_INIT_FAIL 3 + #define IOAT_RESHAPE_PENDING 4 struct timer_list timer; #define COMPLETION_TIMEOUT msecs_to_jiffies(100) #define IDLE_TIMEOUT msecs_to_jiffies(2000) @@ -142,15 +143,14 @@ static inline struct ioat_dma_chan *to_ioat_chan(struct dma_chan *c) } /** - * ioat_is_complete - poll the status of an ioat transaction + * ioat_tx_status - poll the status of an ioat transaction * @c: channel handle * @cookie: transaction identifier - * @done: if set, updated with last completed transaction - * @used: if set, updated with last used transaction + * @txstate: if set, updated with the transaction state */ static inline enum dma_status -ioat_is_complete(struct dma_chan *c, dma_cookie_t cookie, - dma_cookie_t *done, dma_cookie_t *used) +ioat_tx_status(struct dma_chan *c, dma_cookie_t cookie, + struct dma_tx_state *txstate) { struct ioat_chan_common *chan = to_chan_common(c); dma_cookie_t last_used; @@ -159,10 +159,7 @@ ioat_is_complete(struct dma_chan *c, dma_cookie_t cookie, last_used = c->cookie; last_complete = chan->completed_cookie; - if (done) - *done = last_complete; - if (used) - *used = last_used; + dma_set_tx_state(txstate, last_complete, last_used, 0); return dma_async_is_complete(cookie, last_complete, last_used); } @@ -337,17 +334,16 @@ struct dca_provider * __devinit ioat_dca_init(struct pci_dev *pdev, void __iomem *iobase); unsigned long ioat_get_current_completion(struct ioat_chan_common *chan); void ioat_init_channel(struct ioatdma_device *device, - struct ioat_chan_common *chan, int idx, - void (*timer_fn)(unsigned long), - void (*tasklet)(unsigned long), - unsigned long ioat); + struct ioat_chan_common *chan, int idx); +enum dma_status ioat_dma_tx_status(struct dma_chan *c, dma_cookie_t cookie, + struct dma_tx_state *txstate); void ioat_dma_unmap(struct ioat_chan_common *chan, enum dma_ctrl_flags flags, size_t len, struct ioat_dma_descriptor *hw); bool ioat_cleanup_preamble(struct ioat_chan_common *chan, unsigned long *phys_complete); void ioat_kobject_add(struct ioatdma_device *device, struct kobj_type *type); void ioat_kobject_del(struct ioatdma_device *device); -extern struct sysfs_ops ioat_sysfs_ops; +extern const struct sysfs_ops ioat_sysfs_ops; extern struct ioat_sysfs_entry ioat_version_attr; extern struct ioat_sysfs_entry ioat_cap_attr; #endif /* IOATDMA_H */ diff --git a/drivers/dma/ioat/dma_v2.c b/drivers/dma/ioat/dma_v2.c index 5cc37afe2bc1..3c8b32a83794 100644 --- a/drivers/dma/ioat/dma_v2.c +++ b/drivers/dma/ioat/dma_v2.c @@ -27,6 +27,7 @@ #include <linux/init.h> #include <linux/module.h> +#include <linux/slab.h> #include <linux/pci.h> #include <linux/interrupt.h> #include <linux/dmaengine.h> @@ -51,55 +52,44 @@ MODULE_PARM_DESC(ioat_ring_max_alloc_order, void __ioat2_issue_pending(struct ioat2_dma_chan *ioat) { - void * __iomem reg_base = ioat->base.reg_base; + struct ioat_chan_common *chan = &ioat->base; - ioat->pending = 0; ioat->dmacount += ioat2_ring_pending(ioat); ioat->issued = ioat->head; - /* make descriptor updates globally visible before notifying channel */ - wmb(); - writew(ioat->dmacount, reg_base + IOAT_CHAN_DMACOUNT_OFFSET); - dev_dbg(to_dev(&ioat->base), + writew(ioat->dmacount, chan->reg_base + IOAT_CHAN_DMACOUNT_OFFSET); + dev_dbg(to_dev(chan), "%s: head: %#x tail: %#x issued: %#x count: %#x\n", __func__, ioat->head, ioat->tail, ioat->issued, ioat->dmacount); } -void ioat2_issue_pending(struct dma_chan *chan) +void ioat2_issue_pending(struct dma_chan *c) { - struct ioat2_dma_chan *ioat = to_ioat2_chan(chan); + struct ioat2_dma_chan *ioat = to_ioat2_chan(c); - spin_lock_bh(&ioat->ring_lock); - if (ioat->pending == 1) + if (ioat2_ring_pending(ioat)) { + spin_lock_bh(&ioat->prep_lock); __ioat2_issue_pending(ioat); - spin_unlock_bh(&ioat->ring_lock); + spin_unlock_bh(&ioat->prep_lock); + } } /** * ioat2_update_pending - log pending descriptors * @ioat: ioat2+ channel * - * set pending to '1' unless pending is already set to '2', pending == 2 - * indicates that submission is temporarily blocked due to an in-flight - * reset. If we are already above the ioat_pending_level threshold then - * just issue pending. - * - * called with ring_lock held + * Check if the number of unsubmitted descriptors has exceeded the + * watermark. Called with prep_lock held */ static void ioat2_update_pending(struct ioat2_dma_chan *ioat) { - if (unlikely(ioat->pending == 2)) - return; - else if (ioat2_ring_pending(ioat) > ioat_pending_level) + if (ioat2_ring_pending(ioat) > ioat_pending_level) __ioat2_issue_pending(ioat); - else - ioat->pending = 1; } static void __ioat2_start_null_desc(struct ioat2_dma_chan *ioat) { struct ioat_ring_ent *desc; struct ioat_dma_descriptor *hw; - int idx; if (ioat2_ring_space(ioat) < 1) { dev_err(to_dev(&ioat->base), @@ -109,8 +99,7 @@ static void __ioat2_start_null_desc(struct ioat2_dma_chan *ioat) dev_dbg(to_dev(&ioat->base), "%s: head: %#x tail: %#x issued: %#x\n", __func__, ioat->head, ioat->tail, ioat->issued); - idx = ioat2_desc_alloc(ioat, 1); - desc = ioat2_get_ring_ent(ioat, idx); + desc = ioat2_get_ring_ent(ioat, ioat->head); hw = desc->hw; hw->ctl = 0; @@ -124,14 +113,16 @@ static void __ioat2_start_null_desc(struct ioat2_dma_chan *ioat) async_tx_ack(&desc->txd); ioat2_set_chainaddr(ioat, desc->txd.phys); dump_desc_dbg(ioat, desc); + wmb(); + ioat->head += 1; __ioat2_issue_pending(ioat); } static void ioat2_start_null_desc(struct ioat2_dma_chan *ioat) { - spin_lock_bh(&ioat->ring_lock); + spin_lock_bh(&ioat->prep_lock); __ioat2_start_null_desc(ioat); - spin_unlock_bh(&ioat->ring_lock); + spin_unlock_bh(&ioat->prep_lock); } static void __cleanup(struct ioat2_dma_chan *ioat, unsigned long phys_complete) @@ -141,15 +132,16 @@ static void __cleanup(struct ioat2_dma_chan *ioat, unsigned long phys_complete) struct ioat_ring_ent *desc; bool seen_current = false; u16 active; - int i; + int idx = ioat->tail, i; dev_dbg(to_dev(chan), "%s: head: %#x tail: %#x issued: %#x\n", __func__, ioat->head, ioat->tail, ioat->issued); active = ioat2_ring_active(ioat); for (i = 0; i < active && !seen_current; i++) { - prefetch(ioat2_get_ring_ent(ioat, ioat->tail + i + 1)); - desc = ioat2_get_ring_ent(ioat, ioat->tail + i); + smp_read_barrier_depends(); + prefetch(ioat2_get_ring_ent(ioat, idx + i + 1)); + desc = ioat2_get_ring_ent(ioat, idx + i); tx = &desc->txd; dump_desc_dbg(ioat, desc); if (tx->cookie) { @@ -165,11 +157,12 @@ static void __cleanup(struct ioat2_dma_chan *ioat, unsigned long phys_complete) if (tx->phys == phys_complete) seen_current = true; } - ioat->tail += i; - BUG_ON(!seen_current); /* no active descs have written a completion? */ + smp_mb(); /* finish all descriptor reads before incrementing tail */ + ioat->tail = idx + i; + BUG_ON(active && !seen_current); /* no active descs have written a completion? */ chan->last_completion = phys_complete; - if (ioat->head == ioat->tail) { + if (active - i == 0) { dev_dbg(to_dev(chan), "%s: cancel completion timeout\n", __func__); clear_bit(IOAT_COMPLETION_PENDING, &chan->state); @@ -186,30 +179,15 @@ static void ioat2_cleanup(struct ioat2_dma_chan *ioat) struct ioat_chan_common *chan = &ioat->base; unsigned long phys_complete; - prefetch(chan->completion); - - if (!spin_trylock_bh(&chan->cleanup_lock)) - return; - - if (!ioat_cleanup_preamble(chan, &phys_complete)) { - spin_unlock_bh(&chan->cleanup_lock); - return; - } - - if (!spin_trylock_bh(&ioat->ring_lock)) { - spin_unlock_bh(&chan->cleanup_lock); - return; - } - - __cleanup(ioat, phys_complete); - - spin_unlock_bh(&ioat->ring_lock); + spin_lock_bh(&chan->cleanup_lock); + if (ioat_cleanup_preamble(chan, &phys_complete)) + __cleanup(ioat, phys_complete); spin_unlock_bh(&chan->cleanup_lock); } -void ioat2_cleanup_tasklet(unsigned long data) +void ioat2_cleanup_event(unsigned long data) { - struct ioat2_dma_chan *ioat = (void *) data; + struct ioat2_dma_chan *ioat = to_ioat2_chan((void *) data); ioat2_cleanup(ioat); writew(IOAT_CHANCTRL_RUN, ioat->base.reg_base + IOAT_CHANCTRL_OFFSET); @@ -291,15 +269,13 @@ static void ioat2_restart_channel(struct ioat2_dma_chan *ioat) void ioat2_timer_event(unsigned long data) { - struct ioat2_dma_chan *ioat = (void *) data; + struct ioat2_dma_chan *ioat = to_ioat2_chan((void *) data); struct ioat_chan_common *chan = &ioat->base; - spin_lock_bh(&chan->cleanup_lock); if (test_bit(IOAT_COMPLETION_PENDING, &chan->state)) { unsigned long phys_complete; u64 status; - spin_lock_bh(&ioat->ring_lock); status = ioat_chansts(chan); /* when halted due to errors check for channel @@ -318,26 +294,31 @@ void ioat2_timer_event(unsigned long data) * acknowledged a pending completion once, then be more * forceful with a restart */ - if (ioat_cleanup_preamble(chan, &phys_complete)) + spin_lock_bh(&chan->cleanup_lock); + if (ioat_cleanup_preamble(chan, &phys_complete)) { __cleanup(ioat, phys_complete); - else if (test_bit(IOAT_COMPLETION_ACK, &chan->state)) + } else if (test_bit(IOAT_COMPLETION_ACK, &chan->state)) { + spin_lock_bh(&ioat->prep_lock); ioat2_restart_channel(ioat); - else { + spin_unlock_bh(&ioat->prep_lock); + } else { set_bit(IOAT_COMPLETION_ACK, &chan->state); mod_timer(&chan->timer, jiffies + COMPLETION_TIMEOUT); } - spin_unlock_bh(&ioat->ring_lock); + spin_unlock_bh(&chan->cleanup_lock); } else { u16 active; /* if the ring is idle, empty, and oversized try to step * down the size */ - spin_lock_bh(&ioat->ring_lock); + spin_lock_bh(&chan->cleanup_lock); + spin_lock_bh(&ioat->prep_lock); active = ioat2_ring_active(ioat); if (active == 0 && ioat->alloc_order > ioat_get_alloc_order()) reshape_ring(ioat, ioat->alloc_order-1); - spin_unlock_bh(&ioat->ring_lock); + spin_unlock_bh(&ioat->prep_lock); + spin_unlock_bh(&chan->cleanup_lock); /* keep shrinking until we get back to our minimum * default size @@ -345,7 +326,6 @@ void ioat2_timer_event(unsigned long data) if (ioat->alloc_order > ioat_get_alloc_order()) mod_timer(&chan->timer, jiffies + IDLE_TIMEOUT); } - spin_unlock_bh(&chan->cleanup_lock); } static int ioat2_reset_hw(struct ioat_chan_common *chan) @@ -397,12 +377,9 @@ int ioat2_enumerate_channels(struct ioatdma_device *device) if (!ioat) break; - ioat_init_channel(device, &ioat->base, i, - device->timer_fn, - device->cleanup_tasklet, - (unsigned long) ioat); + ioat_init_channel(device, &ioat->base, i); ioat->xfercap_log = xfercap_log; - spin_lock_init(&ioat->ring_lock); + spin_lock_init(&ioat->prep_lock); if (device->reset_hw(&ioat->base)) { i = 0; break; @@ -428,8 +405,17 @@ static dma_cookie_t ioat2_tx_submit_unlock(struct dma_async_tx_descriptor *tx) if (!test_and_set_bit(IOAT_COMPLETION_PENDING, &chan->state)) mod_timer(&chan->timer, jiffies + COMPLETION_TIMEOUT); + + /* make descriptor updates visible before advancing ioat->head, + * this is purposefully not smp_wmb() since we are also + * publishing the descriptor updates to a dma device + */ + wmb(); + + ioat->head += ioat->produce; + ioat2_update_pending(ioat); - spin_unlock_bh(&ioat->ring_lock); + spin_unlock_bh(&ioat->prep_lock); return cookie; } @@ -541,14 +527,15 @@ int ioat2_alloc_chan_resources(struct dma_chan *c) if (!ring) return -ENOMEM; - spin_lock_bh(&ioat->ring_lock); + spin_lock_bh(&chan->cleanup_lock); + spin_lock_bh(&ioat->prep_lock); ioat->ring = ring; ioat->head = 0; ioat->issued = 0; ioat->tail = 0; - ioat->pending = 0; ioat->alloc_order = order; - spin_unlock_bh(&ioat->ring_lock); + spin_unlock_bh(&ioat->prep_lock); + spin_unlock_bh(&chan->cleanup_lock); tasklet_enable(&chan->cleanup_task); ioat2_start_null_desc(ioat); @@ -564,7 +551,7 @@ bool reshape_ring(struct ioat2_dma_chan *ioat, int order) */ struct ioat_chan_common *chan = &ioat->base; struct dma_chan *c = &chan->common; - const u16 curr_size = ioat2_ring_mask(ioat) + 1; + const u16 curr_size = ioat2_ring_size(ioat); const u16 active = ioat2_ring_active(ioat); const u16 new_size = 1 << order; struct ioat_ring_ent **ring; @@ -664,54 +651,61 @@ bool reshape_ring(struct ioat2_dma_chan *ioat, int order) } /** - * ioat2_alloc_and_lock - common descriptor alloc boilerplate for ioat2,3 ops - * @idx: gets starting descriptor index on successful allocation + * ioat2_check_space_lock - verify space and grab ring producer lock * @ioat: ioat2,3 channel (ring) to operate on * @num_descs: allocation length */ -int ioat2_alloc_and_lock(u16 *idx, struct ioat2_dma_chan *ioat, int num_descs) +int ioat2_check_space_lock(struct ioat2_dma_chan *ioat, int num_descs) { struct ioat_chan_common *chan = &ioat->base; + bool retry; - spin_lock_bh(&ioat->ring_lock); + retry: + spin_lock_bh(&ioat->prep_lock); /* never allow the last descriptor to be consumed, we need at * least one free at all times to allow for on-the-fly ring * resizing. */ - while (unlikely(ioat2_ring_space(ioat) <= num_descs)) { - if (reshape_ring(ioat, ioat->alloc_order + 1) && - ioat2_ring_space(ioat) > num_descs) - break; - - if (printk_ratelimit()) - dev_dbg(to_dev(chan), - "%s: ring full! num_descs: %d (%x:%x:%x)\n", - __func__, num_descs, ioat->head, ioat->tail, - ioat->issued); - spin_unlock_bh(&ioat->ring_lock); - - /* progress reclaim in the allocation failure case we - * may be called under bh_disabled so we need to trigger - * the timer event directly - */ - spin_lock_bh(&chan->cleanup_lock); - if (jiffies > chan->timer.expires && - timer_pending(&chan->timer)) { - struct ioatdma_device *device = chan->device; - - mod_timer(&chan->timer, jiffies + COMPLETION_TIMEOUT); - spin_unlock_bh(&chan->cleanup_lock); - device->timer_fn((unsigned long) ioat); - } else - spin_unlock_bh(&chan->cleanup_lock); - return -ENOMEM; + if (likely(ioat2_ring_space(ioat) > num_descs)) { + dev_dbg(to_dev(chan), "%s: num_descs: %d (%x:%x:%x)\n", + __func__, num_descs, ioat->head, ioat->tail, ioat->issued); + ioat->produce = num_descs; + return 0; /* with ioat->prep_lock held */ } + retry = test_and_set_bit(IOAT_RESHAPE_PENDING, &chan->state); + spin_unlock_bh(&ioat->prep_lock); + + /* is another cpu already trying to expand the ring? */ + if (retry) + goto retry; + + spin_lock_bh(&chan->cleanup_lock); + spin_lock_bh(&ioat->prep_lock); + retry = reshape_ring(ioat, ioat->alloc_order + 1); + clear_bit(IOAT_RESHAPE_PENDING, &chan->state); + spin_unlock_bh(&ioat->prep_lock); + spin_unlock_bh(&chan->cleanup_lock); - dev_dbg(to_dev(chan), "%s: num_descs: %d (%x:%x:%x)\n", - __func__, num_descs, ioat->head, ioat->tail, ioat->issued); + /* if we were able to expand the ring retry the allocation */ + if (retry) + goto retry; - *idx = ioat2_desc_alloc(ioat, num_descs); - return 0; /* with ioat->ring_lock held */ + if (printk_ratelimit()) + dev_dbg(to_dev(chan), "%s: ring full! num_descs: %d (%x:%x:%x)\n", + __func__, num_descs, ioat->head, ioat->tail, ioat->issued); + + /* progress reclaim in the allocation failure case we may be + * called under bh_disabled so we need to trigger the timer + * event directly + */ + if (jiffies > chan->timer.expires && timer_pending(&chan->timer)) { + struct ioatdma_device *device = chan->device; + + mod_timer(&chan->timer, jiffies + COMPLETION_TIMEOUT); + device->timer_fn((unsigned long) &chan->common); + } + + return -ENOMEM; } struct dma_async_tx_descriptor * @@ -724,14 +718,11 @@ ioat2_dma_prep_memcpy_lock(struct dma_chan *c, dma_addr_t dma_dest, dma_addr_t dst = dma_dest; dma_addr_t src = dma_src; size_t total_len = len; - int num_descs; - u16 idx; - int i; + int num_descs, idx, i; num_descs = ioat2_xferlen_to_descs(ioat, len); - if (likely(num_descs) && - ioat2_alloc_and_lock(&idx, ioat, num_descs) == 0) - /* pass */; + if (likely(num_descs) && ioat2_check_space_lock(ioat, num_descs) == 0) + idx = ioat->head; else return NULL; i = 0; @@ -785,10 +776,11 @@ void ioat2_free_chan_resources(struct dma_chan *c) tasklet_disable(&chan->cleanup_task); del_timer_sync(&chan->timer); - device->cleanup_tasklet((unsigned long) ioat); + device->cleanup_fn((unsigned long) c); device->reset_hw(chan); - spin_lock_bh(&ioat->ring_lock); + spin_lock_bh(&chan->cleanup_lock); + spin_lock_bh(&ioat->prep_lock); descs = ioat2_ring_space(ioat); dev_dbg(to_dev(chan), "freeing %d idle descriptors\n", descs); for (i = 0; i < descs; i++) { @@ -811,29 +803,14 @@ void ioat2_free_chan_resources(struct dma_chan *c) ioat->alloc_order = 0; pci_pool_free(device->completion_pool, chan->completion, chan->completion_dma); - spin_unlock_bh(&ioat->ring_lock); + spin_unlock_bh(&ioat->prep_lock); + spin_unlock_bh(&chan->cleanup_lock); chan->last_completion = 0; chan->completion_dma = 0; - ioat->pending = 0; ioat->dmacount = 0; } -enum dma_status -ioat2_is_complete(struct dma_chan *c, dma_cookie_t cookie, - dma_cookie_t *done, dma_cookie_t *used) -{ - struct ioat2_dma_chan *ioat = to_ioat2_chan(c); - struct ioatdma_device *device = ioat->base.device; - - if (ioat_is_complete(c, cookie, done, used) == DMA_SUCCESS) - return DMA_SUCCESS; - - device->cleanup_tasklet((unsigned long) ioat); - - return ioat_is_complete(c, cookie, done, used); -} - static ssize_t ring_size_show(struct dma_chan *c, char *page) { struct ioat2_dma_chan *ioat = to_ioat2_chan(c); @@ -874,7 +851,7 @@ int __devinit ioat2_dma_probe(struct ioatdma_device *device, int dca) device->enumerate_channels = ioat2_enumerate_channels; device->reset_hw = ioat2_reset_hw; - device->cleanup_tasklet = ioat2_cleanup_tasklet; + device->cleanup_fn = ioat2_cleanup_event; device->timer_fn = ioat2_timer_event; device->self_test = ioat_dma_self_test; dma = &device->common; @@ -882,7 +859,7 @@ int __devinit ioat2_dma_probe(struct ioatdma_device *device, int dca) dma->device_issue_pending = ioat2_issue_pending; dma->device_alloc_chan_resources = ioat2_alloc_chan_resources; dma->device_free_chan_resources = ioat2_free_chan_resources; - dma->device_is_tx_complete = ioat2_is_complete; + dma->device_tx_status = ioat_tx_status; err = ioat_probe(device); if (err) diff --git a/drivers/dma/ioat/dma_v2.h b/drivers/dma/ioat/dma_v2.h index 3afad8da43cc..a2c413b2b8d8 100644 --- a/drivers/dma/ioat/dma_v2.h +++ b/drivers/dma/ioat/dma_v2.h @@ -22,6 +22,7 @@ #define IOATDMA_V2_H #include <linux/dmaengine.h> +#include <linux/circ_buf.h> #include "dma.h" #include "hw.h" @@ -47,11 +48,11 @@ extern int ioat_ring_alloc_order; * @head: allocated index * @issued: hardware notification point * @tail: cleanup index - * @pending: lock free indicator for issued != head * @dmacount: identical to 'head' except for occasionally resetting to zero * @alloc_order: log2 of the number of allocated descriptors + * @produce: number of descriptors to produce at submit time * @ring: software ring buffer implementation of hardware ring - * @ring_lock: protects ring attributes + * @prep_lock: serializes descriptor preparation (producers) */ struct ioat2_dma_chan { struct ioat_chan_common base; @@ -61,9 +62,9 @@ struct ioat2_dma_chan { u16 tail; u16 dmacount; u16 alloc_order; - int pending; + u16 produce; struct ioat_ring_ent **ring; - spinlock_t ring_lock; + spinlock_t prep_lock; }; static inline struct ioat2_dma_chan *to_ioat2_chan(struct dma_chan *c) @@ -73,38 +74,26 @@ static inline struct ioat2_dma_chan *to_ioat2_chan(struct dma_chan *c) return container_of(chan, struct ioat2_dma_chan, base); } -static inline u16 ioat2_ring_mask(struct ioat2_dma_chan *ioat) +static inline u16 ioat2_ring_size(struct ioat2_dma_chan *ioat) { - return (1 << ioat->alloc_order) - 1; + return 1 << ioat->alloc_order; } /* count of descriptors in flight with the engine */ static inline u16 ioat2_ring_active(struct ioat2_dma_chan *ioat) { - return (ioat->head - ioat->tail) & ioat2_ring_mask(ioat); + return CIRC_CNT(ioat->head, ioat->tail, ioat2_ring_size(ioat)); } /* count of descriptors pending submission to hardware */ static inline u16 ioat2_ring_pending(struct ioat2_dma_chan *ioat) { - return (ioat->head - ioat->issued) & ioat2_ring_mask(ioat); + return CIRC_CNT(ioat->head, ioat->issued, ioat2_ring_size(ioat)); } static inline u16 ioat2_ring_space(struct ioat2_dma_chan *ioat) { - u16 num_descs = ioat2_ring_mask(ioat) + 1; - u16 active = ioat2_ring_active(ioat); - - BUG_ON(active > num_descs); - - return num_descs - active; -} - -/* assumes caller already checked space */ -static inline u16 ioat2_desc_alloc(struct ioat2_dma_chan *ioat, u16 len) -{ - ioat->head += len; - return ioat->head - len; + return ioat2_ring_size(ioat) - ioat2_ring_active(ioat); } static inline u16 ioat2_xferlen_to_descs(struct ioat2_dma_chan *ioat, size_t len) @@ -153,7 +142,7 @@ struct ioat_ring_ent { static inline struct ioat_ring_ent * ioat2_get_ring_ent(struct ioat2_dma_chan *ioat, u16 idx) { - return ioat->ring[idx & ioat2_ring_mask(ioat)]; + return ioat->ring[idx & (ioat2_ring_size(ioat) - 1)]; } static inline void ioat2_set_chainaddr(struct ioat2_dma_chan *ioat, u64 addr) @@ -170,7 +159,7 @@ int __devinit ioat2_dma_probe(struct ioatdma_device *dev, int dca); int __devinit ioat3_dma_probe(struct ioatdma_device *dev, int dca); struct dca_provider * __devinit ioat2_dca_init(struct pci_dev *pdev, void __iomem *iobase); struct dca_provider * __devinit ioat3_dca_init(struct pci_dev *pdev, void __iomem *iobase); -int ioat2_alloc_and_lock(u16 *idx, struct ioat2_dma_chan *ioat, int num_descs); +int ioat2_check_space_lock(struct ioat2_dma_chan *ioat, int num_descs); int ioat2_enumerate_channels(struct ioatdma_device *device); struct dma_async_tx_descriptor * ioat2_dma_prep_memcpy_lock(struct dma_chan *c, dma_addr_t dma_dest, @@ -178,12 +167,10 @@ ioat2_dma_prep_memcpy_lock(struct dma_chan *c, dma_addr_t dma_dest, void ioat2_issue_pending(struct dma_chan *chan); int ioat2_alloc_chan_resources(struct dma_chan *c); void ioat2_free_chan_resources(struct dma_chan *c); -enum dma_status ioat2_is_complete(struct dma_chan *c, dma_cookie_t cookie, - dma_cookie_t *done, dma_cookie_t *used); void __ioat2_restart_chan(struct ioat2_dma_chan *ioat); bool reshape_ring(struct ioat2_dma_chan *ioat, int order); void __ioat2_issue_pending(struct ioat2_dma_chan *ioat); -void ioat2_cleanup_tasklet(unsigned long data); +void ioat2_cleanup_event(unsigned long data); void ioat2_timer_event(unsigned long data); int ioat2_quiesce(struct ioat_chan_common *chan, unsigned long tmo); int ioat2_reset_sync(struct ioat_chan_common *chan, unsigned long tmo); diff --git a/drivers/dma/ioat/dma_v3.c b/drivers/dma/ioat/dma_v3.c index 9908c9e94b2d..1cdd22e1051b 100644 --- a/drivers/dma/ioat/dma_v3.c +++ b/drivers/dma/ioat/dma_v3.c @@ -57,6 +57,7 @@ */ #include <linux/pci.h> +#include <linux/gfp.h> #include <linux/dmaengine.h> #include <linux/dma-mapping.h> #include "registers.h" @@ -259,8 +260,8 @@ static void __cleanup(struct ioat2_dma_chan *ioat, unsigned long phys_complete) struct ioat_chan_common *chan = &ioat->base; struct ioat_ring_ent *desc; bool seen_current = false; + int idx = ioat->tail, i; u16 active; - int i; dev_dbg(to_dev(chan), "%s: head: %#x tail: %#x issued: %#x\n", __func__, ioat->head, ioat->tail, ioat->issued); @@ -269,13 +270,14 @@ static void __cleanup(struct ioat2_dma_chan *ioat, unsigned long phys_complete) for (i = 0; i < active && !seen_current; i++) { struct dma_async_tx_descriptor *tx; - prefetch(ioat2_get_ring_ent(ioat, ioat->tail + i + 1)); - desc = ioat2_get_ring_ent(ioat, ioat->tail + i); + smp_read_barrier_depends(); + prefetch(ioat2_get_ring_ent(ioat, idx + i + 1)); + desc = ioat2_get_ring_ent(ioat, idx + i); dump_desc_dbg(ioat, desc); tx = &desc->txd; if (tx->cookie) { chan->completed_cookie = tx->cookie; - ioat3_dma_unmap(ioat, desc, ioat->tail + i); + ioat3_dma_unmap(ioat, desc, idx + i); tx->cookie = 0; if (tx->callback) { tx->callback(tx->callback_param); @@ -292,15 +294,20 @@ static void __cleanup(struct ioat2_dma_chan *ioat, unsigned long phys_complete) i++; } } - ioat->tail += i; - BUG_ON(!seen_current); /* no active descs have written a completion? */ + smp_mb(); /* finish all descriptor reads before incrementing tail */ + ioat->tail = idx + i; + BUG_ON(active && !seen_current); /* no active descs have written a completion? */ chan->last_completion = phys_complete; - if (ioat->head == ioat->tail) { + + if (active - i == 0) { dev_dbg(to_dev(chan), "%s: cancel completion timeout\n", __func__); clear_bit(IOAT_COMPLETION_PENDING, &chan->state); mod_timer(&chan->timer, jiffies + IDLE_TIMEOUT); } + /* 5 microsecond delay per pending descriptor */ + writew(min((5 * (active - i)), IOAT_INTRDELAY_MASK), + chan->device->reg_base + IOAT_INTRDELAY_OFFSET); } static void ioat3_cleanup(struct ioat2_dma_chan *ioat) @@ -308,50 +315,26 @@ static void ioat3_cleanup(struct ioat2_dma_chan *ioat) struct ioat_chan_common *chan = &ioat->base; unsigned long phys_complete; - prefetch(chan->completion); - - if (!spin_trylock_bh(&chan->cleanup_lock)) - return; - - if (!ioat_cleanup_preamble(chan, &phys_complete)) { - spin_unlock_bh(&chan->cleanup_lock); - return; - } - - if (!spin_trylock_bh(&ioat->ring_lock)) { - spin_unlock_bh(&chan->cleanup_lock); - return; - } - - __cleanup(ioat, phys_complete); - - spin_unlock_bh(&ioat->ring_lock); + spin_lock_bh(&chan->cleanup_lock); + if (ioat_cleanup_preamble(chan, &phys_complete)) + __cleanup(ioat, phys_complete); spin_unlock_bh(&chan->cleanup_lock); } -static void ioat3_cleanup_tasklet(unsigned long data) +static void ioat3_cleanup_event(unsigned long data) { - struct ioat2_dma_chan *ioat = (void *) data; + struct ioat2_dma_chan *ioat = to_ioat2_chan((void *) data); ioat3_cleanup(ioat); - writew(IOAT_CHANCTRL_RUN | IOAT3_CHANCTRL_COMPL_DCA_EN, - ioat->base.reg_base + IOAT_CHANCTRL_OFFSET); + writew(IOAT_CHANCTRL_RUN, ioat->base.reg_base + IOAT_CHANCTRL_OFFSET); } static void ioat3_restart_channel(struct ioat2_dma_chan *ioat) { struct ioat_chan_common *chan = &ioat->base; unsigned long phys_complete; - u32 status; - - status = ioat_chansts(chan); - if (is_ioat_active(status) || is_ioat_idle(status)) - ioat_suspend(chan); - while (is_ioat_active(status) || is_ioat_idle(status)) { - status = ioat_chansts(chan); - cpu_relax(); - } + ioat2_quiesce(chan, 0); if (ioat_cleanup_preamble(chan, &phys_complete)) __cleanup(ioat, phys_complete); @@ -360,15 +343,13 @@ static void ioat3_restart_channel(struct ioat2_dma_chan *ioat) static void ioat3_timer_event(unsigned long data) { - struct ioat2_dma_chan *ioat = (void *) data; + struct ioat2_dma_chan *ioat = to_ioat2_chan((void *) data); struct ioat_chan_common *chan = &ioat->base; - spin_lock_bh(&chan->cleanup_lock); if (test_bit(IOAT_COMPLETION_PENDING, &chan->state)) { unsigned long phys_complete; u64 status; - spin_lock_bh(&ioat->ring_lock); status = ioat_chansts(chan); /* when halted due to errors check for channel @@ -387,26 +368,31 @@ static void ioat3_timer_event(unsigned long data) * acknowledged a pending completion once, then be more * forceful with a restart */ + spin_lock_bh(&chan->cleanup_lock); if (ioat_cleanup_preamble(chan, &phys_complete)) __cleanup(ioat, phys_complete); - else if (test_bit(IOAT_COMPLETION_ACK, &chan->state)) + else if (test_bit(IOAT_COMPLETION_ACK, &chan->state)) { + spin_lock_bh(&ioat->prep_lock); ioat3_restart_channel(ioat); - else { + spin_unlock_bh(&ioat->prep_lock); + } else { set_bit(IOAT_COMPLETION_ACK, &chan->state); mod_timer(&chan->timer, jiffies + COMPLETION_TIMEOUT); } - spin_unlock_bh(&ioat->ring_lock); + spin_unlock_bh(&chan->cleanup_lock); } else { u16 active; /* if the ring is idle, empty, and oversized try to step * down the size */ - spin_lock_bh(&ioat->ring_lock); + spin_lock_bh(&chan->cleanup_lock); + spin_lock_bh(&ioat->prep_lock); active = ioat2_ring_active(ioat); if (active == 0 && ioat->alloc_order > ioat_get_alloc_order()) reshape_ring(ioat, ioat->alloc_order-1); - spin_unlock_bh(&ioat->ring_lock); + spin_unlock_bh(&ioat->prep_lock); + spin_unlock_bh(&chan->cleanup_lock); /* keep shrinking until we get back to our minimum * default size @@ -414,21 +400,20 @@ static void ioat3_timer_event(unsigned long data) if (ioat->alloc_order > ioat_get_alloc_order()) mod_timer(&chan->timer, jiffies + IDLE_TIMEOUT); } - spin_unlock_bh(&chan->cleanup_lock); } static enum dma_status -ioat3_is_complete(struct dma_chan *c, dma_cookie_t cookie, - dma_cookie_t *done, dma_cookie_t *used) +ioat3_tx_status(struct dma_chan *c, dma_cookie_t cookie, + struct dma_tx_state *txstate) { struct ioat2_dma_chan *ioat = to_ioat2_chan(c); - if (ioat_is_complete(c, cookie, done, used) == DMA_SUCCESS) + if (ioat_tx_status(c, cookie, txstate) == DMA_SUCCESS) return DMA_SUCCESS; ioat3_cleanup(ioat); - return ioat_is_complete(c, cookie, done, used); + return ioat_tx_status(c, cookie, txstate); } static struct dma_async_tx_descriptor * @@ -439,15 +424,12 @@ ioat3_prep_memset_lock(struct dma_chan *c, dma_addr_t dest, int value, struct ioat_ring_ent *desc; size_t total_len = len; struct ioat_fill_descriptor *fill; - int num_descs; u64 src_data = (0x0101010101010101ULL) * (value & 0xff); - u16 idx; - int i; + int num_descs, idx, i; num_descs = ioat2_xferlen_to_descs(ioat, len); - if (likely(num_descs) && - ioat2_alloc_and_lock(&idx, ioat, num_descs) == 0) - /* pass */; + if (likely(num_descs) && ioat2_check_space_lock(ioat, num_descs) == 0) + idx = ioat->head; else return NULL; i = 0; @@ -492,11 +474,8 @@ __ioat3_prep_xor_lock(struct dma_chan *c, enum sum_check_flags *result, struct ioat_xor_descriptor *xor; struct ioat_xor_ext_descriptor *xor_ex = NULL; struct ioat_dma_descriptor *hw; + int num_descs, with_ext, idx, i; u32 offset = 0; - int num_descs; - int with_ext; - int i; - u16 idx; u8 op = result ? IOAT_OP_XOR_VAL : IOAT_OP_XOR; BUG_ON(src_cnt < 2); @@ -516,9 +495,8 @@ __ioat3_prep_xor_lock(struct dma_chan *c, enum sum_check_flags *result, * (legacy) descriptor to ensure all completion writes arrive in * order. */ - if (likely(num_descs) && - ioat2_alloc_and_lock(&idx, ioat, num_descs+1) == 0) - /* pass */; + if (likely(num_descs) && ioat2_check_space_lock(ioat, num_descs+1) == 0) + idx = ioat->head; else return NULL; i = 0; @@ -636,11 +614,8 @@ __ioat3_prep_pq_lock(struct dma_chan *c, enum sum_check_flags *result, struct ioat_pq_ext_descriptor *pq_ex = NULL; struct ioat_dma_descriptor *hw; u32 offset = 0; - int num_descs; - int with_ext; - int i, s; - u16 idx; u8 op = result ? IOAT_OP_PQ_VAL : IOAT_OP_PQ; + int i, s, idx, with_ext, num_descs; dev_dbg(to_dev(chan), "%s\n", __func__); /* the engine requires at least two sources (we provide @@ -666,8 +641,8 @@ __ioat3_prep_pq_lock(struct dma_chan *c, enum sum_check_flags *result, * order. */ if (likely(num_descs) && - ioat2_alloc_and_lock(&idx, ioat, num_descs+1) == 0) - /* pass */; + ioat2_check_space_lock(ioat, num_descs+1) == 0) + idx = ioat->head; else return NULL; i = 0; @@ -830,10 +805,9 @@ ioat3_prep_interrupt_lock(struct dma_chan *c, unsigned long flags) struct ioat2_dma_chan *ioat = to_ioat2_chan(c); struct ioat_ring_ent *desc; struct ioat_dma_descriptor *hw; - u16 idx; - if (ioat2_alloc_and_lock(&idx, ioat, 1) == 0) - desc = ioat2_get_ring_ent(ioat, idx); + if (ioat2_check_space_lock(ioat, 1) == 0) + desc = ioat2_get_ring_ent(ioat, ioat->head); else return NULL; @@ -956,7 +930,7 @@ static int __devinit ioat_xor_val_self_test(struct ioatdma_device *device) tmo = wait_for_completion_timeout(&cmp, msecs_to_jiffies(3000)); - if (dma->device_is_tx_complete(dma_chan, cookie, NULL, NULL) != DMA_SUCCESS) { + if (dma->device_tx_status(dma_chan, cookie, NULL) != DMA_SUCCESS) { dev_err(dev, "Self-test xor timed out\n"); err = -ENODEV; goto free_resources; @@ -1010,7 +984,7 @@ static int __devinit ioat_xor_val_self_test(struct ioatdma_device *device) tmo = wait_for_completion_timeout(&cmp, msecs_to_jiffies(3000)); - if (dma->device_is_tx_complete(dma_chan, cookie, NULL, NULL) != DMA_SUCCESS) { + if (dma->device_tx_status(dma_chan, cookie, NULL) != DMA_SUCCESS) { dev_err(dev, "Self-test validate timed out\n"); err = -ENODEV; goto free_resources; @@ -1051,7 +1025,7 @@ static int __devinit ioat_xor_val_self_test(struct ioatdma_device *device) tmo = wait_for_completion_timeout(&cmp, msecs_to_jiffies(3000)); - if (dma->device_is_tx_complete(dma_chan, cookie, NULL, NULL) != DMA_SUCCESS) { + if (dma->device_tx_status(dma_chan, cookie, NULL) != DMA_SUCCESS) { dev_err(dev, "Self-test memset timed out\n"); err = -ENODEV; goto free_resources; @@ -1094,7 +1068,7 @@ static int __devinit ioat_xor_val_self_test(struct ioatdma_device *device) tmo = wait_for_completion_timeout(&cmp, msecs_to_jiffies(3000)); - if (dma->device_is_tx_complete(dma_chan, cookie, NULL, NULL) != DMA_SUCCESS) { + if (dma->device_tx_status(dma_chan, cookie, NULL) != DMA_SUCCESS) { dev_err(dev, "Self-test 2nd validate timed out\n"); err = -ENODEV; goto free_resources; @@ -1201,7 +1175,7 @@ int __devinit ioat3_dma_probe(struct ioatdma_device *device, int dca) if (cap & IOAT_CAP_XOR) { is_raid_device = true; dma->max_xor = 8; - dma->xor_align = 2; + dma->xor_align = 6; dma_cap_set(DMA_XOR, dma->cap_mask); dma->device_prep_dma_xor = ioat3_prep_xor; @@ -1212,7 +1186,7 @@ int __devinit ioat3_dma_probe(struct ioatdma_device *device, int dca) if (cap & IOAT_CAP_PQ) { is_raid_device = true; dma_set_maxpq(dma, 8, 0); - dma->pq_align = 2; + dma->pq_align = 6; dma_cap_set(DMA_PQ, dma->cap_mask); dma->device_prep_dma_pq = ioat3_prep_pq; @@ -1222,7 +1196,7 @@ int __devinit ioat3_dma_probe(struct ioatdma_device *device, int dca) if (!(cap & IOAT_CAP_XOR)) { dma->max_xor = 8; - dma->xor_align = 2; + dma->xor_align = 6; dma_cap_set(DMA_XOR, dma->cap_mask); dma->device_prep_dma_xor = ioat3_prep_pqxor; @@ -1238,12 +1212,12 @@ int __devinit ioat3_dma_probe(struct ioatdma_device *device, int dca) if (is_raid_device) { - dma->device_is_tx_complete = ioat3_is_complete; - device->cleanup_tasklet = ioat3_cleanup_tasklet; + dma->device_tx_status = ioat3_tx_status; + device->cleanup_fn = ioat3_cleanup_event; device->timer_fn = ioat3_timer_event; } else { - dma->device_is_tx_complete = ioat2_is_complete; - device->cleanup_tasklet = ioat2_cleanup_tasklet; + dma->device_tx_status = ioat_dma_tx_status; + device->cleanup_fn = ioat2_cleanup_event; device->timer_fn = ioat2_timer_event; } diff --git a/drivers/dma/ioat/pci.c b/drivers/dma/ioat/pci.c index d545fae30f37..fab37d1cf48d 100644 --- a/drivers/dma/ioat/pci.c +++ b/drivers/dma/ioat/pci.c @@ -30,6 +30,7 @@ #include <linux/pci.h> #include <linux/interrupt.h> #include <linux/dca.h> +#include <linux/slab.h> #include "dma.h" #include "dma_v2.h" #include "registers.h" @@ -137,15 +138,10 @@ static int __devinit ioat_pci_probe(struct pci_dev *pdev, const struct pci_devic if (err) return err; - device = devm_kzalloc(dev, sizeof(*device), GFP_KERNEL); - if (!device) - return -ENOMEM; - - pci_set_master(pdev); - device = alloc_ioatdma(pdev, iomap[IOAT_MMIO_BAR]); if (!device) return -ENOMEM; + pci_set_master(pdev); pci_set_drvdata(pdev, device); device->version = readb(device->reg_base + IOAT_VER_OFFSET); diff --git a/drivers/dma/ioat/registers.h b/drivers/dma/ioat/registers.h index e8ae63baf588..1391798542b6 100644 --- a/drivers/dma/ioat/registers.h +++ b/drivers/dma/ioat/registers.h @@ -60,7 +60,7 @@ #define IOAT_PERPORTOFFSET_OFFSET 0x0A /* 16-bit */ #define IOAT_INTRDELAY_OFFSET 0x0C /* 16-bit */ -#define IOAT_INTRDELAY_INT_DELAY_MASK 0x3FFF /* Interrupt Delay Time */ +#define IOAT_INTRDELAY_MASK 0x3FFF /* Interrupt Delay Time */ #define IOAT_INTRDELAY_COALESE_SUPPORT 0x8000 /* Interrupt Coalescing Supported */ #define IOAT_DEVICE_STATUS_OFFSET 0x0E /* 16-bit */ diff --git a/drivers/dma/iop-adma.c b/drivers/dma/iop-adma.c index ca6e6a0cb793..161c452923b8 100644 --- a/drivers/dma/iop-adma.c +++ b/drivers/dma/iop-adma.c @@ -32,6 +32,7 @@ #include <linux/memory.h> #include <linux/ioport.h> #include <linux/raid/pq.h> +#include <linux/slab.h> #include <mach/adma.h> @@ -893,14 +894,14 @@ static void iop_adma_free_chan_resources(struct dma_chan *chan) } /** - * iop_adma_is_complete - poll the status of an ADMA transaction + * iop_adma_status - poll the status of an ADMA transaction * @chan: ADMA channel handle * @cookie: ADMA transaction identifier + * @txstate: a holder for the current state of the channel or NULL */ -static enum dma_status iop_adma_is_complete(struct dma_chan *chan, +static enum dma_status iop_adma_status(struct dma_chan *chan, dma_cookie_t cookie, - dma_cookie_t *done, - dma_cookie_t *used) + struct dma_tx_state *txstate) { struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan); dma_cookie_t last_used; @@ -909,12 +910,7 @@ static enum dma_status iop_adma_is_complete(struct dma_chan *chan, last_used = chan->cookie; last_complete = iop_chan->completed_cookie; - - if (done) - *done = last_complete; - if (used) - *used = last_used; - + dma_set_tx_state(txstate, last_complete, last_used, 0); ret = dma_async_is_complete(cookie, last_complete, last_used); if (ret == DMA_SUCCESS) return ret; @@ -923,11 +919,7 @@ static enum dma_status iop_adma_is_complete(struct dma_chan *chan, last_used = chan->cookie; last_complete = iop_chan->completed_cookie; - - if (done) - *done = last_complete; - if (used) - *used = last_used; + dma_set_tx_state(txstate, last_complete, last_used, 0); return dma_async_is_complete(cookie, last_complete, last_used); } @@ -1042,7 +1034,7 @@ static int __devinit iop_adma_memcpy_self_test(struct iop_adma_device *device) iop_adma_issue_pending(dma_chan); msleep(1); - if (iop_adma_is_complete(dma_chan, cookie, NULL, NULL) != + if (iop_adma_status(dma_chan, cookie, NULL) != DMA_SUCCESS) { dev_printk(KERN_ERR, dma_chan->device->dev, "Self-test copy timed out, disabling\n"); @@ -1142,7 +1134,7 @@ iop_adma_xor_val_self_test(struct iop_adma_device *device) iop_adma_issue_pending(dma_chan); msleep(8); - if (iop_adma_is_complete(dma_chan, cookie, NULL, NULL) != + if (iop_adma_status(dma_chan, cookie, NULL) != DMA_SUCCESS) { dev_printk(KERN_ERR, dma_chan->device->dev, "Self-test xor timed out, disabling\n"); @@ -1189,7 +1181,7 @@ iop_adma_xor_val_self_test(struct iop_adma_device *device) iop_adma_issue_pending(dma_chan); msleep(8); - if (iop_adma_is_complete(dma_chan, cookie, NULL, NULL) != DMA_SUCCESS) { + if (iop_adma_status(dma_chan, cookie, NULL) != DMA_SUCCESS) { dev_printk(KERN_ERR, dma_chan->device->dev, "Self-test zero sum timed out, disabling\n"); err = -ENODEV; @@ -1213,7 +1205,7 @@ iop_adma_xor_val_self_test(struct iop_adma_device *device) iop_adma_issue_pending(dma_chan); msleep(8); - if (iop_adma_is_complete(dma_chan, cookie, NULL, NULL) != DMA_SUCCESS) { + if (iop_adma_status(dma_chan, cookie, NULL) != DMA_SUCCESS) { dev_printk(KERN_ERR, dma_chan->device->dev, "Self-test memset timed out, disabling\n"); err = -ENODEV; @@ -1245,7 +1237,7 @@ iop_adma_xor_val_self_test(struct iop_adma_device *device) iop_adma_issue_pending(dma_chan); msleep(8); - if (iop_adma_is_complete(dma_chan, cookie, NULL, NULL) != DMA_SUCCESS) { + if (iop_adma_status(dma_chan, cookie, NULL) != DMA_SUCCESS) { dev_printk(KERN_ERR, dma_chan->device->dev, "Self-test non-zero sum timed out, disabling\n"); err = -ENODEV; @@ -1340,7 +1332,7 @@ iop_adma_pq_zero_sum_self_test(struct iop_adma_device *device) iop_adma_issue_pending(dma_chan); msleep(8); - if (iop_adma_is_complete(dma_chan, cookie, NULL, NULL) != + if (iop_adma_status(dma_chan, cookie, NULL) != DMA_SUCCESS) { dev_err(dev, "Self-test pq timed out, disabling\n"); err = -ENODEV; @@ -1377,7 +1369,7 @@ iop_adma_pq_zero_sum_self_test(struct iop_adma_device *device) iop_adma_issue_pending(dma_chan); msleep(8); - if (iop_adma_is_complete(dma_chan, cookie, NULL, NULL) != + if (iop_adma_status(dma_chan, cookie, NULL) != DMA_SUCCESS) { dev_err(dev, "Self-test pq-zero-sum timed out, disabling\n"); err = -ENODEV; @@ -1409,7 +1401,7 @@ iop_adma_pq_zero_sum_self_test(struct iop_adma_device *device) iop_adma_issue_pending(dma_chan); msleep(8); - if (iop_adma_is_complete(dma_chan, cookie, NULL, NULL) != + if (iop_adma_status(dma_chan, cookie, NULL) != DMA_SUCCESS) { dev_err(dev, "Self-test !pq-zero-sum timed out, disabling\n"); err = -ENODEV; @@ -1507,7 +1499,7 @@ static int __devinit iop_adma_probe(struct platform_device *pdev) /* set base routines */ dma_dev->device_alloc_chan_resources = iop_adma_alloc_chan_resources; dma_dev->device_free_chan_resources = iop_adma_free_chan_resources; - dma_dev->device_is_tx_complete = iop_adma_is_complete; + dma_dev->device_tx_status = iop_adma_status; dma_dev->device_issue_pending = iop_adma_issue_pending; dma_dev->dev = &pdev->dev; diff --git a/drivers/dma/iovlock.c b/drivers/dma/iovlock.c index c0a272c73682..bb48a57c2fc1 100644 --- a/drivers/dma/iovlock.c +++ b/drivers/dma/iovlock.c @@ -27,6 +27,7 @@ #include <linux/dmaengine.h> #include <linux/pagemap.h> +#include <linux/slab.h> #include <net/tcp.h> /* for memcpy_toiovec */ #include <asm/io.h> #include <asm/uaccess.h> diff --git a/drivers/dma/ipu/ipu_idmac.c b/drivers/dma/ipu/ipu_idmac.c index e80bae1673fa..cb26ee9773d6 100644 --- a/drivers/dma/ipu/ipu_idmac.c +++ b/drivers/dma/ipu/ipu_idmac.c @@ -348,6 +348,7 @@ static void ipu_ch_param_set_size(union chan_param_mem *params, break; case IPU_PIX_FMT_BGRA32: case IPU_PIX_FMT_BGR32: + case IPU_PIX_FMT_ABGR32: params->ip.bpp = 0; params->ip.pfs = 4; params->ip.npb = 7; @@ -376,20 +377,6 @@ static void ipu_ch_param_set_size(union chan_param_mem *params, params->ip.wid2 = 7; /* Blue bit width - 1 */ params->ip.wid3 = 7; /* Alpha bit width - 1 */ break; - case IPU_PIX_FMT_ABGR32: - params->ip.bpp = 0; - params->ip.pfs = 4; - params->ip.npb = 7; - params->ip.sat = 2; /* SAT = 32-bit access */ - params->ip.ofs0 = 8; /* Red bit offset */ - params->ip.ofs1 = 16; /* Green bit offset */ - params->ip.ofs2 = 24; /* Blue bit offset */ - params->ip.ofs3 = 0; /* Alpha bit offset */ - params->ip.wid0 = 7; /* Red bit width - 1 */ - params->ip.wid1 = 7; /* Green bit width - 1 */ - params->ip.wid2 = 7; /* Blue bit width - 1 */ - params->ip.wid3 = 7; /* Alpha bit width - 1 */ - break; case IPU_PIX_FMT_UYVY: params->ip.bpp = 2; params->ip.pfs = 6; @@ -1485,13 +1472,18 @@ static void idmac_issue_pending(struct dma_chan *chan) */ } -static void __idmac_terminate_all(struct dma_chan *chan) +static int __idmac_control(struct dma_chan *chan, enum dma_ctrl_cmd cmd, + unsigned long arg) { struct idmac_channel *ichan = to_idmac_chan(chan); struct idmac *idmac = to_idmac(chan->device); unsigned long flags; int i; + /* Only supports DMA_TERMINATE_ALL */ + if (cmd != DMA_TERMINATE_ALL) + return -ENXIO; + ipu_disable_channel(idmac, ichan, ichan->status >= IPU_CHANNEL_ENABLED); @@ -1518,17 +1510,23 @@ static void __idmac_terminate_all(struct dma_chan *chan) tasklet_enable(&to_ipu(idmac)->tasklet); ichan->status = IPU_CHANNEL_INITIALIZED; + + return 0; } -static void idmac_terminate_all(struct dma_chan *chan) +static int idmac_control(struct dma_chan *chan, enum dma_ctrl_cmd cmd, + unsigned long arg) { struct idmac_channel *ichan = to_idmac_chan(chan); + int ret; mutex_lock(&ichan->chan_mutex); - __idmac_terminate_all(chan); + ret = __idmac_control(chan, cmd, arg); mutex_unlock(&ichan->chan_mutex); + + return ret; } #ifdef DEBUG @@ -1620,7 +1618,7 @@ static void idmac_free_chan_resources(struct dma_chan *chan) mutex_lock(&ichan->chan_mutex); - __idmac_terminate_all(chan); + __idmac_control(chan, DMA_TERMINATE_ALL, 0); if (ichan->status > IPU_CHANNEL_FREE) { #ifdef DEBUG @@ -1650,15 +1648,12 @@ static void idmac_free_chan_resources(struct dma_chan *chan) tasklet_schedule(&to_ipu(idmac)->tasklet); } -static enum dma_status idmac_is_tx_complete(struct dma_chan *chan, - dma_cookie_t cookie, dma_cookie_t *done, dma_cookie_t *used) +static enum dma_status idmac_tx_status(struct dma_chan *chan, + dma_cookie_t cookie, struct dma_tx_state *txstate) { struct idmac_channel *ichan = to_idmac_chan(chan); - if (done) - *done = ichan->completed; - if (used) - *used = chan->cookie; + dma_set_tx_state(txstate, ichan->completed, chan->cookie, 0); if (cookie != chan->cookie) return DMA_ERROR; return DMA_SUCCESS; @@ -1677,12 +1672,12 @@ static int __init ipu_idmac_init(struct ipu *ipu) dma->dev = ipu->dev; dma->device_alloc_chan_resources = idmac_alloc_chan_resources; dma->device_free_chan_resources = idmac_free_chan_resources; - dma->device_is_tx_complete = idmac_is_tx_complete; + dma->device_tx_status = idmac_tx_status; dma->device_issue_pending = idmac_issue_pending; /* Compulsory for DMA_SLAVE fields */ dma->device_prep_slave_sg = idmac_prep_slave_sg; - dma->device_terminate_all = idmac_terminate_all; + dma->device_control = idmac_control; INIT_LIST_HEAD(&dma->channels); for (i = 0; i < IPU_CHANNELS_NUM; i++) { @@ -1716,7 +1711,7 @@ static void __exit ipu_idmac_exit(struct ipu *ipu) for (i = 0; i < IPU_CHANNELS_NUM; i++) { struct idmac_channel *ichan = ipu->channel + i; - idmac_terminate_all(&ichan->dma_chan); + idmac_control(&ichan->dma_chan, DMA_TERMINATE_ALL, 0); idmac_prep_slave_sg(&ichan->dma_chan, NULL, 0, DMA_NONE, 0); } diff --git a/drivers/dma/mpc512x_dma.c b/drivers/dma/mpc512x_dma.c new file mode 100644 index 000000000000..14a8c0f1698e --- /dev/null +++ b/drivers/dma/mpc512x_dma.c @@ -0,0 +1,796 @@ +/* + * Copyright (C) Freescale Semicondutor, Inc. 2007, 2008. + * Copyright (C) Semihalf 2009 + * + * Written by Piotr Ziecik <kosmo@semihalf.com>. Hardware description + * (defines, structures and comments) was taken from MPC5121 DMA driver + * written by Hongjun Chen <hong-jun.chen@freescale.com>. + * + * Approved as OSADL project by a majority of OSADL members and funded + * by OSADL membership fees in 2009; for details see www.osadl.org. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; either version 2 of the License, or (at your option) + * any later version. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * this program; if not, write to the Free Software Foundation, Inc., 59 + * Temple Place - Suite 330, Boston, MA 02111-1307, USA. + * + * The full GNU General Public License is included in this distribution in the + * file called COPYING. + */ + +/* + * This is initial version of MPC5121 DMA driver. Only memory to memory + * transfers are supported (tested using dmatest module). + */ + +#include <linux/module.h> +#include <linux/dmaengine.h> +#include <linux/dma-mapping.h> +#include <linux/interrupt.h> +#include <linux/io.h> +#include <linux/slab.h> +#include <linux/of_device.h> +#include <linux/of_platform.h> + +#include <linux/random.h> + +/* Number of DMA Transfer descriptors allocated per channel */ +#define MPC_DMA_DESCRIPTORS 64 + +/* Macro definitions */ +#define MPC_DMA_CHANNELS 64 +#define MPC_DMA_TCD_OFFSET 0x1000 + +/* Arbitration mode of group and channel */ +#define MPC_DMA_DMACR_EDCG (1 << 31) +#define MPC_DMA_DMACR_ERGA (1 << 3) +#define MPC_DMA_DMACR_ERCA (1 << 2) + +/* Error codes */ +#define MPC_DMA_DMAES_VLD (1 << 31) +#define MPC_DMA_DMAES_GPE (1 << 15) +#define MPC_DMA_DMAES_CPE (1 << 14) +#define MPC_DMA_DMAES_ERRCHN(err) \ + (((err) >> 8) & 0x3f) +#define MPC_DMA_DMAES_SAE (1 << 7) +#define MPC_DMA_DMAES_SOE (1 << 6) +#define MPC_DMA_DMAES_DAE (1 << 5) +#define MPC_DMA_DMAES_DOE (1 << 4) +#define MPC_DMA_DMAES_NCE (1 << 3) +#define MPC_DMA_DMAES_SGE (1 << 2) +#define MPC_DMA_DMAES_SBE (1 << 1) +#define MPC_DMA_DMAES_DBE (1 << 0) + +#define MPC_DMA_TSIZE_1 0x00 +#define MPC_DMA_TSIZE_2 0x01 +#define MPC_DMA_TSIZE_4 0x02 +#define MPC_DMA_TSIZE_16 0x04 +#define MPC_DMA_TSIZE_32 0x05 + +/* MPC5121 DMA engine registers */ +struct __attribute__ ((__packed__)) mpc_dma_regs { + /* 0x00 */ + u32 dmacr; /* DMA control register */ + u32 dmaes; /* DMA error status */ + /* 0x08 */ + u32 dmaerqh; /* DMA enable request high(channels 63~32) */ + u32 dmaerql; /* DMA enable request low(channels 31~0) */ + u32 dmaeeih; /* DMA enable error interrupt high(ch63~32) */ + u32 dmaeeil; /* DMA enable error interrupt low(ch31~0) */ + /* 0x18 */ + u8 dmaserq; /* DMA set enable request */ + u8 dmacerq; /* DMA clear enable request */ + u8 dmaseei; /* DMA set enable error interrupt */ + u8 dmaceei; /* DMA clear enable error interrupt */ + /* 0x1c */ + u8 dmacint; /* DMA clear interrupt request */ + u8 dmacerr; /* DMA clear error */ + u8 dmassrt; /* DMA set start bit */ + u8 dmacdne; /* DMA clear DONE status bit */ + /* 0x20 */ + u32 dmainth; /* DMA interrupt request high(ch63~32) */ + u32 dmaintl; /* DMA interrupt request low(ch31~0) */ + u32 dmaerrh; /* DMA error high(ch63~32) */ + u32 dmaerrl; /* DMA error low(ch31~0) */ + /* 0x30 */ + u32 dmahrsh; /* DMA hw request status high(ch63~32) */ + u32 dmahrsl; /* DMA hardware request status low(ch31~0) */ + u32 dmaihsa; /* DMA interrupt high select AXE(ch63~32) */ + u32 dmailsa; /* DMA interrupt low select AXE(ch31~0) */ + /* 0x40 ~ 0xff */ + u32 reserve0[48]; /* Reserved */ + /* 0x100 */ + u8 dchpri[MPC_DMA_CHANNELS]; + /* DMA channels(0~63) priority */ +}; + +struct __attribute__ ((__packed__)) mpc_dma_tcd { + /* 0x00 */ + u32 saddr; /* Source address */ + + u32 smod:5; /* Source address modulo */ + u32 ssize:3; /* Source data transfer size */ + u32 dmod:5; /* Destination address modulo */ + u32 dsize:3; /* Destination data transfer size */ + u32 soff:16; /* Signed source address offset */ + + /* 0x08 */ + u32 nbytes; /* Inner "minor" byte count */ + u32 slast; /* Last source address adjustment */ + u32 daddr; /* Destination address */ + + /* 0x14 */ + u32 citer_elink:1; /* Enable channel-to-channel linking on + * minor loop complete + */ + u32 citer_linkch:6; /* Link channel for minor loop complete */ + u32 citer:9; /* Current "major" iteration count */ + u32 doff:16; /* Signed destination address offset */ + + /* 0x18 */ + u32 dlast_sga; /* Last Destination address adjustment/scatter + * gather address + */ + + /* 0x1c */ + u32 biter_elink:1; /* Enable channel-to-channel linking on major + * loop complete + */ + u32 biter_linkch:6; + u32 biter:9; /* Beginning "major" iteration count */ + u32 bwc:2; /* Bandwidth control */ + u32 major_linkch:6; /* Link channel number */ + u32 done:1; /* Channel done */ + u32 active:1; /* Channel active */ + u32 major_elink:1; /* Enable channel-to-channel linking on major + * loop complete + */ + u32 e_sg:1; /* Enable scatter/gather processing */ + u32 d_req:1; /* Disable request */ + u32 int_half:1; /* Enable an interrupt when major counter is + * half complete + */ + u32 int_maj:1; /* Enable an interrupt when major iteration + * count completes + */ + u32 start:1; /* Channel start */ +}; + +struct mpc_dma_desc { + struct dma_async_tx_descriptor desc; + struct mpc_dma_tcd *tcd; + dma_addr_t tcd_paddr; + int error; + struct list_head node; +}; + +struct mpc_dma_chan { + struct dma_chan chan; + struct list_head free; + struct list_head prepared; + struct list_head queued; + struct list_head active; + struct list_head completed; + struct mpc_dma_tcd *tcd; + dma_addr_t tcd_paddr; + dma_cookie_t completed_cookie; + + /* Lock for this structure */ + spinlock_t lock; +}; + +struct mpc_dma { + struct dma_device dma; + struct tasklet_struct tasklet; + struct mpc_dma_chan channels[MPC_DMA_CHANNELS]; + struct mpc_dma_regs __iomem *regs; + struct mpc_dma_tcd __iomem *tcd; + int irq; + uint error_status; + + /* Lock for error_status field in this structure */ + spinlock_t error_status_lock; +}; + +#define DRV_NAME "mpc512x_dma" + +/* Convert struct dma_chan to struct mpc_dma_chan */ +static inline struct mpc_dma_chan *dma_chan_to_mpc_dma_chan(struct dma_chan *c) +{ + return container_of(c, struct mpc_dma_chan, chan); +} + +/* Convert struct dma_chan to struct mpc_dma */ +static inline struct mpc_dma *dma_chan_to_mpc_dma(struct dma_chan *c) +{ + struct mpc_dma_chan *mchan = dma_chan_to_mpc_dma_chan(c); + return container_of(mchan, struct mpc_dma, channels[c->chan_id]); +} + +/* + * Execute all queued DMA descriptors. + * + * Following requirements must be met while calling mpc_dma_execute(): + * a) mchan->lock is acquired, + * b) mchan->active list is empty, + * c) mchan->queued list contains at least one entry. + */ +static void mpc_dma_execute(struct mpc_dma_chan *mchan) +{ + struct mpc_dma *mdma = dma_chan_to_mpc_dma(&mchan->chan); + struct mpc_dma_desc *first = NULL; + struct mpc_dma_desc *prev = NULL; + struct mpc_dma_desc *mdesc; + int cid = mchan->chan.chan_id; + + /* Move all queued descriptors to active list */ + list_splice_tail_init(&mchan->queued, &mchan->active); + + /* Chain descriptors into one transaction */ + list_for_each_entry(mdesc, &mchan->active, node) { + if (!first) + first = mdesc; + + if (!prev) { + prev = mdesc; + continue; + } + + prev->tcd->dlast_sga = mdesc->tcd_paddr; + prev->tcd->e_sg = 1; + mdesc->tcd->start = 1; + + prev = mdesc; + } + + prev->tcd->start = 0; + prev->tcd->int_maj = 1; + + /* Send first descriptor in chain into hardware */ + memcpy_toio(&mdma->tcd[cid], first->tcd, sizeof(struct mpc_dma_tcd)); + out_8(&mdma->regs->dmassrt, cid); +} + +/* Handle interrupt on one half of DMA controller (32 channels) */ +static void mpc_dma_irq_process(struct mpc_dma *mdma, u32 is, u32 es, int off) +{ + struct mpc_dma_chan *mchan; + struct mpc_dma_desc *mdesc; + u32 status = is | es; + int ch; + + while ((ch = fls(status) - 1) >= 0) { + status &= ~(1 << ch); + mchan = &mdma->channels[ch + off]; + + spin_lock(&mchan->lock); + + /* Check error status */ + if (es & (1 << ch)) + list_for_each_entry(mdesc, &mchan->active, node) + mdesc->error = -EIO; + + /* Execute queued descriptors */ + list_splice_tail_init(&mchan->active, &mchan->completed); + if (!list_empty(&mchan->queued)) + mpc_dma_execute(mchan); + + spin_unlock(&mchan->lock); + } +} + +/* Interrupt handler */ +static irqreturn_t mpc_dma_irq(int irq, void *data) +{ + struct mpc_dma *mdma = data; + uint es; + + /* Save error status register */ + es = in_be32(&mdma->regs->dmaes); + spin_lock(&mdma->error_status_lock); + if ((es & MPC_DMA_DMAES_VLD) && mdma->error_status == 0) + mdma->error_status = es; + spin_unlock(&mdma->error_status_lock); + + /* Handle interrupt on each channel */ + mpc_dma_irq_process(mdma, in_be32(&mdma->regs->dmainth), + in_be32(&mdma->regs->dmaerrh), 32); + mpc_dma_irq_process(mdma, in_be32(&mdma->regs->dmaintl), + in_be32(&mdma->regs->dmaerrl), 0); + + /* Ack interrupt on all channels */ + out_be32(&mdma->regs->dmainth, 0xFFFFFFFF); + out_be32(&mdma->regs->dmaintl, 0xFFFFFFFF); + out_be32(&mdma->regs->dmaerrh, 0xFFFFFFFF); + out_be32(&mdma->regs->dmaerrl, 0xFFFFFFFF); + + /* Schedule tasklet */ + tasklet_schedule(&mdma->tasklet); + + return IRQ_HANDLED; +} + +/* DMA Tasklet */ +static void mpc_dma_tasklet(unsigned long data) +{ + struct mpc_dma *mdma = (void *)data; + dma_cookie_t last_cookie = 0; + struct mpc_dma_chan *mchan; + struct mpc_dma_desc *mdesc; + struct dma_async_tx_descriptor *desc; + unsigned long flags; + LIST_HEAD(list); + uint es; + int i; + + spin_lock_irqsave(&mdma->error_status_lock, flags); + es = mdma->error_status; + mdma->error_status = 0; + spin_unlock_irqrestore(&mdma->error_status_lock, flags); + + /* Print nice error report */ + if (es) { + dev_err(mdma->dma.dev, + "Hardware reported following error(s) on channel %u:\n", + MPC_DMA_DMAES_ERRCHN(es)); + + if (es & MPC_DMA_DMAES_GPE) + dev_err(mdma->dma.dev, "- Group Priority Error\n"); + if (es & MPC_DMA_DMAES_CPE) + dev_err(mdma->dma.dev, "- Channel Priority Error\n"); + if (es & MPC_DMA_DMAES_SAE) + dev_err(mdma->dma.dev, "- Source Address Error\n"); + if (es & MPC_DMA_DMAES_SOE) + dev_err(mdma->dma.dev, "- Source Offset" + " Configuration Error\n"); + if (es & MPC_DMA_DMAES_DAE) + dev_err(mdma->dma.dev, "- Destination Address" + " Error\n"); + if (es & MPC_DMA_DMAES_DOE) + dev_err(mdma->dma.dev, "- Destination Offset" + " Configuration Error\n"); + if (es & MPC_DMA_DMAES_NCE) + dev_err(mdma->dma.dev, "- NBytes/Citter" + " Configuration Error\n"); + if (es & MPC_DMA_DMAES_SGE) + dev_err(mdma->dma.dev, "- Scatter/Gather" + " Configuration Error\n"); + if (es & MPC_DMA_DMAES_SBE) + dev_err(mdma->dma.dev, "- Source Bus Error\n"); + if (es & MPC_DMA_DMAES_DBE) + dev_err(mdma->dma.dev, "- Destination Bus Error\n"); + } + + for (i = 0; i < mdma->dma.chancnt; i++) { + mchan = &mdma->channels[i]; + + /* Get all completed descriptors */ + spin_lock_irqsave(&mchan->lock, flags); + if (!list_empty(&mchan->completed)) + list_splice_tail_init(&mchan->completed, &list); + spin_unlock_irqrestore(&mchan->lock, flags); + + if (list_empty(&list)) + continue; + + /* Execute callbacks and run dependencies */ + list_for_each_entry(mdesc, &list, node) { + desc = &mdesc->desc; + + if (desc->callback) + desc->callback(desc->callback_param); + + last_cookie = desc->cookie; + dma_run_dependencies(desc); + } + + /* Free descriptors */ + spin_lock_irqsave(&mchan->lock, flags); + list_splice_tail_init(&list, &mchan->free); + mchan->completed_cookie = last_cookie; + spin_unlock_irqrestore(&mchan->lock, flags); + } +} + +/* Submit descriptor to hardware */ +static dma_cookie_t mpc_dma_tx_submit(struct dma_async_tx_descriptor *txd) +{ + struct mpc_dma_chan *mchan = dma_chan_to_mpc_dma_chan(txd->chan); + struct mpc_dma_desc *mdesc; + unsigned long flags; + dma_cookie_t cookie; + + mdesc = container_of(txd, struct mpc_dma_desc, desc); + + spin_lock_irqsave(&mchan->lock, flags); + + /* Move descriptor to queue */ + list_move_tail(&mdesc->node, &mchan->queued); + + /* If channel is idle, execute all queued descriptors */ + if (list_empty(&mchan->active)) + mpc_dma_execute(mchan); + + /* Update cookie */ + cookie = mchan->chan.cookie + 1; + if (cookie <= 0) + cookie = 1; + + mchan->chan.cookie = cookie; + mdesc->desc.cookie = cookie; + + spin_unlock_irqrestore(&mchan->lock, flags); + + return cookie; +} + +/* Alloc channel resources */ +static int mpc_dma_alloc_chan_resources(struct dma_chan *chan) +{ + struct mpc_dma *mdma = dma_chan_to_mpc_dma(chan); + struct mpc_dma_chan *mchan = dma_chan_to_mpc_dma_chan(chan); + struct mpc_dma_desc *mdesc; + struct mpc_dma_tcd *tcd; + dma_addr_t tcd_paddr; + unsigned long flags; + LIST_HEAD(descs); + int i; + + /* Alloc DMA memory for Transfer Control Descriptors */ + tcd = dma_alloc_coherent(mdma->dma.dev, + MPC_DMA_DESCRIPTORS * sizeof(struct mpc_dma_tcd), + &tcd_paddr, GFP_KERNEL); + if (!tcd) + return -ENOMEM; + + /* Alloc descriptors for this channel */ + for (i = 0; i < MPC_DMA_DESCRIPTORS; i++) { + mdesc = kzalloc(sizeof(struct mpc_dma_desc), GFP_KERNEL); + if (!mdesc) { + dev_notice(mdma->dma.dev, "Memory allocation error. " + "Allocated only %u descriptors\n", i); + break; + } + + dma_async_tx_descriptor_init(&mdesc->desc, chan); + mdesc->desc.flags = DMA_CTRL_ACK; + mdesc->desc.tx_submit = mpc_dma_tx_submit; + + mdesc->tcd = &tcd[i]; + mdesc->tcd_paddr = tcd_paddr + (i * sizeof(struct mpc_dma_tcd)); + + list_add_tail(&mdesc->node, &descs); + } + + /* Return error only if no descriptors were allocated */ + if (i == 0) { + dma_free_coherent(mdma->dma.dev, + MPC_DMA_DESCRIPTORS * sizeof(struct mpc_dma_tcd), + tcd, tcd_paddr); + return -ENOMEM; + } + + spin_lock_irqsave(&mchan->lock, flags); + mchan->tcd = tcd; + mchan->tcd_paddr = tcd_paddr; + list_splice_tail_init(&descs, &mchan->free); + spin_unlock_irqrestore(&mchan->lock, flags); + + /* Enable Error Interrupt */ + out_8(&mdma->regs->dmaseei, chan->chan_id); + + return 0; +} + +/* Free channel resources */ +static void mpc_dma_free_chan_resources(struct dma_chan *chan) +{ + struct mpc_dma *mdma = dma_chan_to_mpc_dma(chan); + struct mpc_dma_chan *mchan = dma_chan_to_mpc_dma_chan(chan); + struct mpc_dma_desc *mdesc, *tmp; + struct mpc_dma_tcd *tcd; + dma_addr_t tcd_paddr; + unsigned long flags; + LIST_HEAD(descs); + + spin_lock_irqsave(&mchan->lock, flags); + + /* Channel must be idle */ + BUG_ON(!list_empty(&mchan->prepared)); + BUG_ON(!list_empty(&mchan->queued)); + BUG_ON(!list_empty(&mchan->active)); + BUG_ON(!list_empty(&mchan->completed)); + + /* Move data */ + list_splice_tail_init(&mchan->free, &descs); + tcd = mchan->tcd; + tcd_paddr = mchan->tcd_paddr; + + spin_unlock_irqrestore(&mchan->lock, flags); + + /* Free DMA memory used by descriptors */ + dma_free_coherent(mdma->dma.dev, + MPC_DMA_DESCRIPTORS * sizeof(struct mpc_dma_tcd), + tcd, tcd_paddr); + + /* Free descriptors */ + list_for_each_entry_safe(mdesc, tmp, &descs, node) + kfree(mdesc); + + /* Disable Error Interrupt */ + out_8(&mdma->regs->dmaceei, chan->chan_id); +} + +/* Send all pending descriptor to hardware */ +static void mpc_dma_issue_pending(struct dma_chan *chan) +{ + /* + * We are posting descriptors to the hardware as soon as + * they are ready, so this function does nothing. + */ +} + +/* Check request completion status */ +static enum dma_status +mpc_dma_tx_status(struct dma_chan *chan, dma_cookie_t cookie, + struct dma_tx_state *txstate) +{ + struct mpc_dma_chan *mchan = dma_chan_to_mpc_dma_chan(chan); + unsigned long flags; + dma_cookie_t last_used; + dma_cookie_t last_complete; + + spin_lock_irqsave(&mchan->lock, flags); + last_used = mchan->chan.cookie; + last_complete = mchan->completed_cookie; + spin_unlock_irqrestore(&mchan->lock, flags); + + dma_set_tx_state(txstate, last_complete, last_used, 0); + return dma_async_is_complete(cookie, last_complete, last_used); +} + +/* Prepare descriptor for memory to memory copy */ +static struct dma_async_tx_descriptor * +mpc_dma_prep_memcpy(struct dma_chan *chan, dma_addr_t dst, dma_addr_t src, + size_t len, unsigned long flags) +{ + struct mpc_dma_chan *mchan = dma_chan_to_mpc_dma_chan(chan); + struct mpc_dma_desc *mdesc = NULL; + struct mpc_dma_tcd *tcd; + unsigned long iflags; + + /* Get free descriptor */ + spin_lock_irqsave(&mchan->lock, iflags); + if (!list_empty(&mchan->free)) { + mdesc = list_first_entry(&mchan->free, struct mpc_dma_desc, + node); + list_del(&mdesc->node); + } + spin_unlock_irqrestore(&mchan->lock, iflags); + + if (!mdesc) + return NULL; + + mdesc->error = 0; + tcd = mdesc->tcd; + + /* Prepare Transfer Control Descriptor for this transaction */ + memset(tcd, 0, sizeof(struct mpc_dma_tcd)); + + if (IS_ALIGNED(src | dst | len, 32)) { + tcd->ssize = MPC_DMA_TSIZE_32; + tcd->dsize = MPC_DMA_TSIZE_32; + tcd->soff = 32; + tcd->doff = 32; + } else if (IS_ALIGNED(src | dst | len, 16)) { + tcd->ssize = MPC_DMA_TSIZE_16; + tcd->dsize = MPC_DMA_TSIZE_16; + tcd->soff = 16; + tcd->doff = 16; + } else if (IS_ALIGNED(src | dst | len, 4)) { + tcd->ssize = MPC_DMA_TSIZE_4; + tcd->dsize = MPC_DMA_TSIZE_4; + tcd->soff = 4; + tcd->doff = 4; + } else if (IS_ALIGNED(src | dst | len, 2)) { + tcd->ssize = MPC_DMA_TSIZE_2; + tcd->dsize = MPC_DMA_TSIZE_2; + tcd->soff = 2; + tcd->doff = 2; + } else { + tcd->ssize = MPC_DMA_TSIZE_1; + tcd->dsize = MPC_DMA_TSIZE_1; + tcd->soff = 1; + tcd->doff = 1; + } + + tcd->saddr = src; + tcd->daddr = dst; + tcd->nbytes = len; + tcd->biter = 1; + tcd->citer = 1; + + /* Place descriptor in prepared list */ + spin_lock_irqsave(&mchan->lock, iflags); + list_add_tail(&mdesc->node, &mchan->prepared); + spin_unlock_irqrestore(&mchan->lock, iflags); + + return &mdesc->desc; +} + +static int __devinit mpc_dma_probe(struct of_device *op, + const struct of_device_id *match) +{ + struct device_node *dn = op->dev.of_node; + struct device *dev = &op->dev; + struct dma_device *dma; + struct mpc_dma *mdma; + struct mpc_dma_chan *mchan; + struct resource res; + ulong regs_start, regs_size; + int retval, i; + + mdma = devm_kzalloc(dev, sizeof(struct mpc_dma), GFP_KERNEL); + if (!mdma) { + dev_err(dev, "Memory exhausted!\n"); + return -ENOMEM; + } + + mdma->irq = irq_of_parse_and_map(dn, 0); + if (mdma->irq == NO_IRQ) { + dev_err(dev, "Error mapping IRQ!\n"); + return -EINVAL; + } + + retval = of_address_to_resource(dn, 0, &res); + if (retval) { + dev_err(dev, "Error parsing memory region!\n"); + return retval; + } + + regs_start = res.start; + regs_size = resource_size(&res); + + if (!devm_request_mem_region(dev, regs_start, regs_size, DRV_NAME)) { + dev_err(dev, "Error requesting memory region!\n"); + return -EBUSY; + } + + mdma->regs = devm_ioremap(dev, regs_start, regs_size); + if (!mdma->regs) { + dev_err(dev, "Error mapping memory region!\n"); + return -ENOMEM; + } + + mdma->tcd = (struct mpc_dma_tcd *)((u8 *)(mdma->regs) + + MPC_DMA_TCD_OFFSET); + + retval = devm_request_irq(dev, mdma->irq, &mpc_dma_irq, 0, DRV_NAME, + mdma); + if (retval) { + dev_err(dev, "Error requesting IRQ!\n"); + return -EINVAL; + } + + spin_lock_init(&mdma->error_status_lock); + + dma = &mdma->dma; + dma->dev = dev; + dma->chancnt = MPC_DMA_CHANNELS; + dma->device_alloc_chan_resources = mpc_dma_alloc_chan_resources; + dma->device_free_chan_resources = mpc_dma_free_chan_resources; + dma->device_issue_pending = mpc_dma_issue_pending; + dma->device_tx_status = mpc_dma_tx_status; + dma->device_prep_dma_memcpy = mpc_dma_prep_memcpy; + + INIT_LIST_HEAD(&dma->channels); + dma_cap_set(DMA_MEMCPY, dma->cap_mask); + + for (i = 0; i < dma->chancnt; i++) { + mchan = &mdma->channels[i]; + + mchan->chan.device = dma; + mchan->chan.chan_id = i; + mchan->chan.cookie = 1; + mchan->completed_cookie = mchan->chan.cookie; + + INIT_LIST_HEAD(&mchan->free); + INIT_LIST_HEAD(&mchan->prepared); + INIT_LIST_HEAD(&mchan->queued); + INIT_LIST_HEAD(&mchan->active); + INIT_LIST_HEAD(&mchan->completed); + + spin_lock_init(&mchan->lock); + list_add_tail(&mchan->chan.device_node, &dma->channels); + } + + tasklet_init(&mdma->tasklet, mpc_dma_tasklet, (unsigned long)mdma); + + /* + * Configure DMA Engine: + * - Dynamic clock, + * - Round-robin group arbitration, + * - Round-robin channel arbitration. + */ + out_be32(&mdma->regs->dmacr, MPC_DMA_DMACR_EDCG | + MPC_DMA_DMACR_ERGA | MPC_DMA_DMACR_ERCA); + + /* Disable hardware DMA requests */ + out_be32(&mdma->regs->dmaerqh, 0); + out_be32(&mdma->regs->dmaerql, 0); + + /* Disable error interrupts */ + out_be32(&mdma->regs->dmaeeih, 0); + out_be32(&mdma->regs->dmaeeil, 0); + + /* Clear interrupts status */ + out_be32(&mdma->regs->dmainth, 0xFFFFFFFF); + out_be32(&mdma->regs->dmaintl, 0xFFFFFFFF); + out_be32(&mdma->regs->dmaerrh, 0xFFFFFFFF); + out_be32(&mdma->regs->dmaerrl, 0xFFFFFFFF); + + /* Route interrupts to IPIC */ + out_be32(&mdma->regs->dmaihsa, 0); + out_be32(&mdma->regs->dmailsa, 0); + + /* Register DMA engine */ + dev_set_drvdata(dev, mdma); + retval = dma_async_device_register(dma); + if (retval) { + devm_free_irq(dev, mdma->irq, mdma); + irq_dispose_mapping(mdma->irq); + } + + return retval; +} + +static int __devexit mpc_dma_remove(struct of_device *op) +{ + struct device *dev = &op->dev; + struct mpc_dma *mdma = dev_get_drvdata(dev); + + dma_async_device_unregister(&mdma->dma); + devm_free_irq(dev, mdma->irq, mdma); + irq_dispose_mapping(mdma->irq); + + return 0; +} + +static struct of_device_id mpc_dma_match[] = { + { .compatible = "fsl,mpc5121-dma", }, + {}, +}; + +static struct of_platform_driver mpc_dma_driver = { + .probe = mpc_dma_probe, + .remove = __devexit_p(mpc_dma_remove), + .driver = { + .name = DRV_NAME, + .owner = THIS_MODULE, + .of_match_table = mpc_dma_match, + }, +}; + +static int __init mpc_dma_init(void) +{ + return of_register_platform_driver(&mpc_dma_driver); +} +module_init(mpc_dma_init); + +static void __exit mpc_dma_exit(void) +{ + of_unregister_platform_driver(&mpc_dma_driver); +} +module_exit(mpc_dma_exit); + +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("Piotr Ziecik <kosmo@semihalf.com>"); diff --git a/drivers/dma/mv_xor.c b/drivers/dma/mv_xor.c index 466ab10c1ff1..86c5ae9fde34 100644 --- a/drivers/dma/mv_xor.c +++ b/drivers/dma/mv_xor.c @@ -18,6 +18,7 @@ #include <linux/init.h> #include <linux/module.h> +#include <linux/slab.h> #include <linux/delay.h> #include <linux/dma-mapping.h> #include <linux/spinlock.h> @@ -809,14 +810,14 @@ static void mv_xor_free_chan_resources(struct dma_chan *chan) } /** - * mv_xor_is_complete - poll the status of an XOR transaction + * mv_xor_status - poll the status of an XOR transaction * @chan: XOR channel handle * @cookie: XOR transaction identifier + * @txstate: XOR transactions state holder (or NULL) */ -static enum dma_status mv_xor_is_complete(struct dma_chan *chan, +static enum dma_status mv_xor_status(struct dma_chan *chan, dma_cookie_t cookie, - dma_cookie_t *done, - dma_cookie_t *used) + struct dma_tx_state *txstate) { struct mv_xor_chan *mv_chan = to_mv_xor_chan(chan); dma_cookie_t last_used; @@ -826,10 +827,7 @@ static enum dma_status mv_xor_is_complete(struct dma_chan *chan, last_used = chan->cookie; last_complete = mv_chan->completed_cookie; mv_chan->is_complete_cookie = cookie; - if (done) - *done = last_complete; - if (used) - *used = last_used; + dma_set_tx_state(txstate, last_complete, last_used, 0); ret = dma_async_is_complete(cookie, last_complete, last_used); if (ret == DMA_SUCCESS) { @@ -841,11 +839,7 @@ static enum dma_status mv_xor_is_complete(struct dma_chan *chan, last_used = chan->cookie; last_complete = mv_chan->completed_cookie; - if (done) - *done = last_complete; - if (used) - *used = last_used; - + dma_set_tx_state(txstate, last_complete, last_used, 0); return dma_async_is_complete(cookie, last_complete, last_used); } @@ -974,7 +968,7 @@ static int __devinit mv_xor_memcpy_self_test(struct mv_xor_device *device) async_tx_ack(tx); msleep(1); - if (mv_xor_is_complete(dma_chan, cookie, NULL, NULL) != + if (mv_xor_status(dma_chan, cookie, NULL) != DMA_SUCCESS) { dev_printk(KERN_ERR, dma_chan->device->dev, "Self-test copy timed out, disabling\n"); @@ -1072,7 +1066,7 @@ mv_xor_xor_self_test(struct mv_xor_device *device) async_tx_ack(tx); msleep(8); - if (mv_xor_is_complete(dma_chan, cookie, NULL, NULL) != + if (mv_xor_status(dma_chan, cookie, NULL) != DMA_SUCCESS) { dev_printk(KERN_ERR, dma_chan->device->dev, "Self-test xor timed out, disabling\n"); @@ -1167,7 +1161,7 @@ static int __devinit mv_xor_probe(struct platform_device *pdev) /* set base routines */ dma_dev->device_alloc_chan_resources = mv_xor_alloc_chan_resources; dma_dev->device_free_chan_resources = mv_xor_free_chan_resources; - dma_dev->device_is_tx_complete = mv_xor_is_complete; + dma_dev->device_tx_status = mv_xor_status; dma_dev->device_issue_pending = mv_xor_issue_pending; dma_dev->dev = &pdev->dev; diff --git a/drivers/dma/pl330.c b/drivers/dma/pl330.c new file mode 100644 index 000000000000..7c50f6dfd3f4 --- /dev/null +++ b/drivers/dma/pl330.c @@ -0,0 +1,866 @@ +/* linux/drivers/dma/pl330.c + * + * Copyright (C) 2010 Samsung Electronics Co. Ltd. + * Jaswinder Singh <jassi.brar@samsung.com> + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + */ + +#include <linux/io.h> +#include <linux/init.h> +#include <linux/slab.h> +#include <linux/module.h> +#include <linux/dmaengine.h> +#include <linux/interrupt.h> +#include <linux/amba/bus.h> +#include <linux/amba/pl330.h> + +#define NR_DEFAULT_DESC 16 + +enum desc_status { + /* In the DMAC pool */ + FREE, + /* + * Allocted to some channel during prep_xxx + * Also may be sitting on the work_list. + */ + PREP, + /* + * Sitting on the work_list and already submitted + * to the PL330 core. Not more than two descriptors + * of a channel can be BUSY at any time. + */ + BUSY, + /* + * Sitting on the channel work_list but xfer done + * by PL330 core + */ + DONE, +}; + +struct dma_pl330_chan { + /* Schedule desc completion */ + struct tasklet_struct task; + + /* DMA-Engine Channel */ + struct dma_chan chan; + + /* Last completed cookie */ + dma_cookie_t completed; + + /* List of to be xfered descriptors */ + struct list_head work_list; + + /* Pointer to the DMAC that manages this channel, + * NULL if the channel is available to be acquired. + * As the parent, this DMAC also provides descriptors + * to the channel. + */ + struct dma_pl330_dmac *dmac; + + /* To protect channel manipulation */ + spinlock_t lock; + + /* Token of a hardware channel thread of PL330 DMAC + * NULL if the channel is available to be acquired. + */ + void *pl330_chid; +}; + +struct dma_pl330_dmac { + struct pl330_info pif; + + /* DMA-Engine Device */ + struct dma_device ddma; + + /* Pool of descriptors available for the DMAC's channels */ + struct list_head desc_pool; + /* To protect desc_pool manipulation */ + spinlock_t pool_lock; + + /* Peripheral channels connected to this DMAC */ + struct dma_pl330_chan peripherals[0]; /* keep at end */ +}; + +struct dma_pl330_desc { + /* To attach to a queue as child */ + struct list_head node; + + /* Descriptor for the DMA Engine API */ + struct dma_async_tx_descriptor txd; + + /* Xfer for PL330 core */ + struct pl330_xfer px; + + struct pl330_reqcfg rqcfg; + struct pl330_req req; + + enum desc_status status; + + /* The channel which currently holds this desc */ + struct dma_pl330_chan *pchan; +}; + +static inline struct dma_pl330_chan * +to_pchan(struct dma_chan *ch) +{ + if (!ch) + return NULL; + + return container_of(ch, struct dma_pl330_chan, chan); +} + +static inline struct dma_pl330_desc * +to_desc(struct dma_async_tx_descriptor *tx) +{ + return container_of(tx, struct dma_pl330_desc, txd); +} + +static inline void free_desc_list(struct list_head *list) +{ + struct dma_pl330_dmac *pdmac; + struct dma_pl330_desc *desc; + struct dma_pl330_chan *pch; + unsigned long flags; + + if (list_empty(list)) + return; + + /* Finish off the work list */ + list_for_each_entry(desc, list, node) { + dma_async_tx_callback callback; + void *param; + + /* All desc in a list belong to same channel */ + pch = desc->pchan; + callback = desc->txd.callback; + param = desc->txd.callback_param; + + if (callback) + callback(param); + + desc->pchan = NULL; + } + + pdmac = pch->dmac; + + spin_lock_irqsave(&pdmac->pool_lock, flags); + list_splice_tail_init(list, &pdmac->desc_pool); + spin_unlock_irqrestore(&pdmac->pool_lock, flags); +} + +static inline void fill_queue(struct dma_pl330_chan *pch) +{ + struct dma_pl330_desc *desc; + int ret; + + list_for_each_entry(desc, &pch->work_list, node) { + + /* If already submitted */ + if (desc->status == BUSY) + break; + + ret = pl330_submit_req(pch->pl330_chid, + &desc->req); + if (!ret) { + desc->status = BUSY; + break; + } else if (ret == -EAGAIN) { + /* QFull or DMAC Dying */ + break; + } else { + /* Unacceptable request */ + desc->status = DONE; + dev_err(pch->dmac->pif.dev, "%s:%d Bad Desc(%d)\n", + __func__, __LINE__, desc->txd.cookie); + tasklet_schedule(&pch->task); + } + } +} + +static void pl330_tasklet(unsigned long data) +{ + struct dma_pl330_chan *pch = (struct dma_pl330_chan *)data; + struct dma_pl330_desc *desc, *_dt; + unsigned long flags; + LIST_HEAD(list); + + spin_lock_irqsave(&pch->lock, flags); + + /* Pick up ripe tomatoes */ + list_for_each_entry_safe(desc, _dt, &pch->work_list, node) + if (desc->status == DONE) { + pch->completed = desc->txd.cookie; + list_move_tail(&desc->node, &list); + } + + /* Try to submit a req imm. next to the last completed cookie */ + fill_queue(pch); + + /* Make sure the PL330 Channel thread is active */ + pl330_chan_ctrl(pch->pl330_chid, PL330_OP_START); + + spin_unlock_irqrestore(&pch->lock, flags); + + free_desc_list(&list); +} + +static void dma_pl330_rqcb(void *token, enum pl330_op_err err) +{ + struct dma_pl330_desc *desc = token; + struct dma_pl330_chan *pch = desc->pchan; + unsigned long flags; + + /* If desc aborted */ + if (!pch) + return; + + spin_lock_irqsave(&pch->lock, flags); + + desc->status = DONE; + + spin_unlock_irqrestore(&pch->lock, flags); + + tasklet_schedule(&pch->task); +} + +static int pl330_alloc_chan_resources(struct dma_chan *chan) +{ + struct dma_pl330_chan *pch = to_pchan(chan); + struct dma_pl330_dmac *pdmac = pch->dmac; + unsigned long flags; + + spin_lock_irqsave(&pch->lock, flags); + + pch->completed = chan->cookie = 1; + + pch->pl330_chid = pl330_request_channel(&pdmac->pif); + if (!pch->pl330_chid) { + spin_unlock_irqrestore(&pch->lock, flags); + return 0; + } + + tasklet_init(&pch->task, pl330_tasklet, (unsigned long) pch); + + spin_unlock_irqrestore(&pch->lock, flags); + + return 1; +} + +static int pl330_control(struct dma_chan *chan, enum dma_ctrl_cmd cmd, unsigned long arg) +{ + struct dma_pl330_chan *pch = to_pchan(chan); + struct dma_pl330_desc *desc; + unsigned long flags; + + /* Only supports DMA_TERMINATE_ALL */ + if (cmd != DMA_TERMINATE_ALL) + return -ENXIO; + + spin_lock_irqsave(&pch->lock, flags); + + /* FLUSH the PL330 Channel thread */ + pl330_chan_ctrl(pch->pl330_chid, PL330_OP_FLUSH); + + /* Mark all desc done */ + list_for_each_entry(desc, &pch->work_list, node) + desc->status = DONE; + + spin_unlock_irqrestore(&pch->lock, flags); + + pl330_tasklet((unsigned long) pch); + + return 0; +} + +static void pl330_free_chan_resources(struct dma_chan *chan) +{ + struct dma_pl330_chan *pch = to_pchan(chan); + unsigned long flags; + + spin_lock_irqsave(&pch->lock, flags); + + tasklet_kill(&pch->task); + + pl330_release_channel(pch->pl330_chid); + pch->pl330_chid = NULL; + + spin_unlock_irqrestore(&pch->lock, flags); +} + +static enum dma_status +pl330_tx_status(struct dma_chan *chan, dma_cookie_t cookie, + struct dma_tx_state *txstate) +{ + struct dma_pl330_chan *pch = to_pchan(chan); + dma_cookie_t last_done, last_used; + int ret; + + last_done = pch->completed; + last_used = chan->cookie; + + ret = dma_async_is_complete(cookie, last_done, last_used); + + dma_set_tx_state(txstate, last_done, last_used, 0); + + return ret; +} + +static void pl330_issue_pending(struct dma_chan *chan) +{ + pl330_tasklet((unsigned long) to_pchan(chan)); +} + +/* + * We returned the last one of the circular list of descriptor(s) + * from prep_xxx, so the argument to submit corresponds to the last + * descriptor of the list. + */ +static dma_cookie_t pl330_tx_submit(struct dma_async_tx_descriptor *tx) +{ + struct dma_pl330_desc *desc, *last = to_desc(tx); + struct dma_pl330_chan *pch = to_pchan(tx->chan); + dma_cookie_t cookie; + unsigned long flags; + + spin_lock_irqsave(&pch->lock, flags); + + /* Assign cookies to all nodes */ + cookie = tx->chan->cookie; + + while (!list_empty(&last->node)) { + desc = list_entry(last->node.next, struct dma_pl330_desc, node); + + if (++cookie < 0) + cookie = 1; + desc->txd.cookie = cookie; + + list_move_tail(&desc->node, &pch->work_list); + } + + if (++cookie < 0) + cookie = 1; + last->txd.cookie = cookie; + + list_add_tail(&last->node, &pch->work_list); + + tx->chan->cookie = cookie; + + spin_unlock_irqrestore(&pch->lock, flags); + + return cookie; +} + +static inline void _init_desc(struct dma_pl330_desc *desc) +{ + desc->pchan = NULL; + desc->req.x = &desc->px; + desc->req.token = desc; + desc->rqcfg.swap = SWAP_NO; + desc->rqcfg.privileged = 0; + desc->rqcfg.insnaccess = 0; + desc->rqcfg.scctl = SCCTRL0; + desc->rqcfg.dcctl = DCCTRL0; + desc->req.cfg = &desc->rqcfg; + desc->req.xfer_cb = dma_pl330_rqcb; + desc->txd.tx_submit = pl330_tx_submit; + + INIT_LIST_HEAD(&desc->node); +} + +/* Returns the number of descriptors added to the DMAC pool */ +int add_desc(struct dma_pl330_dmac *pdmac, gfp_t flg, int count) +{ + struct dma_pl330_desc *desc; + unsigned long flags; + int i; + + if (!pdmac) + return 0; + + desc = kmalloc(count * sizeof(*desc), flg); + if (!desc) + return 0; + + spin_lock_irqsave(&pdmac->pool_lock, flags); + + for (i = 0; i < count; i++) { + _init_desc(&desc[i]); + list_add_tail(&desc[i].node, &pdmac->desc_pool); + } + + spin_unlock_irqrestore(&pdmac->pool_lock, flags); + + return count; +} + +static struct dma_pl330_desc * +pluck_desc(struct dma_pl330_dmac *pdmac) +{ + struct dma_pl330_desc *desc = NULL; + unsigned long flags; + + if (!pdmac) + return NULL; + + spin_lock_irqsave(&pdmac->pool_lock, flags); + + if (!list_empty(&pdmac->desc_pool)) { + desc = list_entry(pdmac->desc_pool.next, + struct dma_pl330_desc, node); + + list_del_init(&desc->node); + + desc->status = PREP; + desc->txd.callback = NULL; + } + + spin_unlock_irqrestore(&pdmac->pool_lock, flags); + + return desc; +} + +static struct dma_pl330_desc *pl330_get_desc(struct dma_pl330_chan *pch) +{ + struct dma_pl330_dmac *pdmac = pch->dmac; + struct dma_pl330_peri *peri = pch->chan.private; + struct dma_pl330_desc *desc; + + /* Pluck one desc from the pool of DMAC */ + desc = pluck_desc(pdmac); + + /* If the DMAC pool is empty, alloc new */ + if (!desc) { + if (!add_desc(pdmac, GFP_ATOMIC, 1)) + return NULL; + + /* Try again */ + desc = pluck_desc(pdmac); + if (!desc) { + dev_err(pch->dmac->pif.dev, + "%s:%d ALERT!\n", __func__, __LINE__); + return NULL; + } + } + + /* Initialize the descriptor */ + desc->pchan = pch; + desc->txd.cookie = 0; + async_tx_ack(&desc->txd); + + desc->req.rqtype = peri->rqtype; + desc->req.peri = peri->peri_id; + + dma_async_tx_descriptor_init(&desc->txd, &pch->chan); + + return desc; +} + +static inline void fill_px(struct pl330_xfer *px, + dma_addr_t dst, dma_addr_t src, size_t len) +{ + px->next = NULL; + px->bytes = len; + px->dst_addr = dst; + px->src_addr = src; +} + +static struct dma_pl330_desc * +__pl330_prep_dma_memcpy(struct dma_pl330_chan *pch, dma_addr_t dst, + dma_addr_t src, size_t len) +{ + struct dma_pl330_desc *desc = pl330_get_desc(pch); + + if (!desc) { + dev_err(pch->dmac->pif.dev, "%s:%d Unable to fetch desc\n", + __func__, __LINE__); + return NULL; + } + + /* + * Ideally we should lookout for reqs bigger than + * those that can be programmed with 256 bytes of + * MC buffer, but considering a req size is seldom + * going to be word-unaligned and more than 200MB, + * we take it easy. + * Also, should the limit is reached we'd rather + * have the platform increase MC buffer size than + * complicating this API driver. + */ + fill_px(&desc->px, dst, src, len); + + return desc; +} + +/* Call after fixing burst size */ +static inline int get_burst_len(struct dma_pl330_desc *desc, size_t len) +{ + struct dma_pl330_chan *pch = desc->pchan; + struct pl330_info *pi = &pch->dmac->pif; + int burst_len; + + burst_len = pi->pcfg.data_bus_width / 8; + burst_len *= pi->pcfg.data_buf_dep; + burst_len >>= desc->rqcfg.brst_size; + + /* src/dst_burst_len can't be more than 16 */ + if (burst_len > 16) + burst_len = 16; + + while (burst_len > 1) { + if (!(len % (burst_len << desc->rqcfg.brst_size))) + break; + burst_len--; + } + + return burst_len; +} + +static struct dma_async_tx_descriptor * +pl330_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dst, + dma_addr_t src, size_t len, unsigned long flags) +{ + struct dma_pl330_desc *desc; + struct dma_pl330_chan *pch = to_pchan(chan); + struct dma_pl330_peri *peri = chan->private; + struct pl330_info *pi; + int burst; + + if (unlikely(!pch || !len || !peri)) + return NULL; + + if (peri->rqtype != MEMTOMEM) + return NULL; + + pi = &pch->dmac->pif; + + desc = __pl330_prep_dma_memcpy(pch, dst, src, len); + if (!desc) + return NULL; + + desc->rqcfg.src_inc = 1; + desc->rqcfg.dst_inc = 1; + + /* Select max possible burst size */ + burst = pi->pcfg.data_bus_width / 8; + + while (burst > 1) { + if (!(len % burst)) + break; + burst /= 2; + } + + desc->rqcfg.brst_size = 0; + while (burst != (1 << desc->rqcfg.brst_size)) + desc->rqcfg.brst_size++; + + desc->rqcfg.brst_len = get_burst_len(desc, len); + + desc->txd.flags = flags; + + return &desc->txd; +} + +static struct dma_async_tx_descriptor * +pl330_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl, + unsigned int sg_len, enum dma_data_direction direction, + unsigned long flg) +{ + struct dma_pl330_desc *first, *desc = NULL; + struct dma_pl330_chan *pch = to_pchan(chan); + struct dma_pl330_peri *peri = chan->private; + struct scatterlist *sg; + unsigned long flags; + int i, burst_size; + dma_addr_t addr; + + if (unlikely(!pch || !sgl || !sg_len)) + return NULL; + + /* Make sure the direction is consistent */ + if ((direction == DMA_TO_DEVICE && + peri->rqtype != MEMTODEV) || + (direction == DMA_FROM_DEVICE && + peri->rqtype != DEVTOMEM)) { + dev_err(pch->dmac->pif.dev, "%s:%d Invalid Direction\n", + __func__, __LINE__); + return NULL; + } + + addr = peri->fifo_addr; + burst_size = peri->burst_sz; + + first = NULL; + + for_each_sg(sgl, sg, sg_len, i) { + + desc = pl330_get_desc(pch); + if (!desc) { + struct dma_pl330_dmac *pdmac = pch->dmac; + + dev_err(pch->dmac->pif.dev, + "%s:%d Unable to fetch desc\n", + __func__, __LINE__); + if (!first) + return NULL; + + spin_lock_irqsave(&pdmac->pool_lock, flags); + + while (!list_empty(&first->node)) { + desc = list_entry(first->node.next, + struct dma_pl330_desc, node); + list_move_tail(&desc->node, &pdmac->desc_pool); + } + + list_move_tail(&first->node, &pdmac->desc_pool); + + spin_unlock_irqrestore(&pdmac->pool_lock, flags); + + return NULL; + } + + if (!first) + first = desc; + else + list_add_tail(&desc->node, &first->node); + + if (direction == DMA_TO_DEVICE) { + desc->rqcfg.src_inc = 1; + desc->rqcfg.dst_inc = 0; + fill_px(&desc->px, + addr, sg_dma_address(sg), sg_dma_len(sg)); + } else { + desc->rqcfg.src_inc = 0; + desc->rqcfg.dst_inc = 1; + fill_px(&desc->px, + sg_dma_address(sg), addr, sg_dma_len(sg)); + } + + desc->rqcfg.brst_size = burst_size; + desc->rqcfg.brst_len = 1; + } + + /* Return the last desc in the chain */ + desc->txd.flags = flg; + return &desc->txd; +} + +static irqreturn_t pl330_irq_handler(int irq, void *data) +{ + if (pl330_update(data)) + return IRQ_HANDLED; + else + return IRQ_NONE; +} + +static int __devinit +pl330_probe(struct amba_device *adev, struct amba_id *id) +{ + struct dma_pl330_platdata *pdat; + struct dma_pl330_dmac *pdmac; + struct dma_pl330_chan *pch; + struct pl330_info *pi; + struct dma_device *pd; + struct resource *res; + int i, ret, irq; + + pdat = adev->dev.platform_data; + + if (!pdat || !pdat->nr_valid_peri) { + dev_err(&adev->dev, "platform data missing\n"); + return -ENODEV; + } + + /* Allocate a new DMAC and its Channels */ + pdmac = kzalloc(pdat->nr_valid_peri * sizeof(*pch) + + sizeof(*pdmac), GFP_KERNEL); + if (!pdmac) { + dev_err(&adev->dev, "unable to allocate mem\n"); + return -ENOMEM; + } + + pi = &pdmac->pif; + pi->dev = &adev->dev; + pi->pl330_data = NULL; + pi->mcbufsz = pdat->mcbuf_sz; + + res = &adev->res; + request_mem_region(res->start, resource_size(res), "dma-pl330"); + + pi->base = ioremap(res->start, resource_size(res)); + if (!pi->base) { + ret = -ENXIO; + goto probe_err1; + } + + irq = adev->irq[0]; + ret = request_irq(irq, pl330_irq_handler, 0, + dev_name(&adev->dev), pi); + if (ret) + goto probe_err2; + + ret = pl330_add(pi); + if (ret) + goto probe_err3; + + INIT_LIST_HEAD(&pdmac->desc_pool); + spin_lock_init(&pdmac->pool_lock); + + /* Create a descriptor pool of default size */ + if (!add_desc(pdmac, GFP_KERNEL, NR_DEFAULT_DESC)) + dev_warn(&adev->dev, "unable to allocate desc\n"); + + pd = &pdmac->ddma; + INIT_LIST_HEAD(&pd->channels); + + /* Initialize channel parameters */ + for (i = 0; i < pdat->nr_valid_peri; i++) { + struct dma_pl330_peri *peri = &pdat->peri[i]; + pch = &pdmac->peripherals[i]; + + switch (peri->rqtype) { + case MEMTOMEM: + dma_cap_set(DMA_MEMCPY, pd->cap_mask); + break; + case MEMTODEV: + case DEVTOMEM: + dma_cap_set(DMA_SLAVE, pd->cap_mask); + break; + default: + dev_err(&adev->dev, "DEVTODEV Not Supported\n"); + continue; + } + + INIT_LIST_HEAD(&pch->work_list); + spin_lock_init(&pch->lock); + pch->pl330_chid = NULL; + pch->chan.private = peri; + pch->chan.device = pd; + pch->chan.chan_id = i; + pch->dmac = pdmac; + + /* Add the channel to the DMAC list */ + pd->chancnt++; + list_add_tail(&pch->chan.device_node, &pd->channels); + } + + pd->dev = &adev->dev; + + pd->device_alloc_chan_resources = pl330_alloc_chan_resources; + pd->device_free_chan_resources = pl330_free_chan_resources; + pd->device_prep_dma_memcpy = pl330_prep_dma_memcpy; + pd->device_tx_status = pl330_tx_status; + pd->device_prep_slave_sg = pl330_prep_slave_sg; + pd->device_control = pl330_control; + pd->device_issue_pending = pl330_issue_pending; + + ret = dma_async_device_register(pd); + if (ret) { + dev_err(&adev->dev, "unable to register DMAC\n"); + goto probe_err4; + } + + amba_set_drvdata(adev, pdmac); + + dev_info(&adev->dev, + "Loaded driver for PL330 DMAC-%d\n", adev->periphid); + dev_info(&adev->dev, + "\tDBUFF-%ux%ubytes Num_Chans-%u Num_Peri-%u Num_Events-%u\n", + pi->pcfg.data_buf_dep, + pi->pcfg.data_bus_width / 8, pi->pcfg.num_chan, + pi->pcfg.num_peri, pi->pcfg.num_events); + + return 0; + +probe_err4: + pl330_del(pi); +probe_err3: + free_irq(irq, pi); +probe_err2: + iounmap(pi->base); +probe_err1: + release_mem_region(res->start, resource_size(res)); + kfree(pdmac); + + return ret; +} + +static int __devexit pl330_remove(struct amba_device *adev) +{ + struct dma_pl330_dmac *pdmac = amba_get_drvdata(adev); + struct dma_pl330_chan *pch, *_p; + struct pl330_info *pi; + struct resource *res; + int irq; + + if (!pdmac) + return 0; + + amba_set_drvdata(adev, NULL); + + /* Idle the DMAC */ + list_for_each_entry_safe(pch, _p, &pdmac->ddma.channels, + chan.device_node) { + + /* Remove the channel */ + list_del(&pch->chan.device_node); + + /* Flush the channel */ + pl330_control(&pch->chan, DMA_TERMINATE_ALL, 0); + pl330_free_chan_resources(&pch->chan); + } + + pi = &pdmac->pif; + + pl330_del(pi); + + irq = adev->irq[0]; + free_irq(irq, pi); + + iounmap(pi->base); + + res = &adev->res; + release_mem_region(res->start, resource_size(res)); + + kfree(pdmac); + + return 0; +} + +static struct amba_id pl330_ids[] = { + { + .id = 0x00041330, + .mask = 0x000fffff, + }, + { 0, 0 }, +}; + +static struct amba_driver pl330_driver = { + .drv = { + .owner = THIS_MODULE, + .name = "dma-pl330", + }, + .id_table = pl330_ids, + .probe = pl330_probe, + .remove = pl330_remove, +}; + +static int __init pl330_init(void) +{ + return amba_driver_register(&pl330_driver); +} +module_init(pl330_init); + +static void __exit pl330_exit(void) +{ + amba_driver_unregister(&pl330_driver); + return; +} +module_exit(pl330_exit); + +MODULE_AUTHOR("Jaswinder Singh <jassi.brar@samsung.com>"); +MODULE_DESCRIPTION("API Driver for PL330 DMAC"); +MODULE_LICENSE("GPL"); diff --git a/drivers/dma/ppc4xx/adma.c b/drivers/dma/ppc4xx/adma.c index 0a3478e910f0..7c3747902a37 100644 --- a/drivers/dma/ppc4xx/adma.c +++ b/drivers/dma/ppc4xx/adma.c @@ -38,6 +38,7 @@ #include <linux/dma-mapping.h> #include <linux/spinlock.h> #include <linux/interrupt.h> +#include <linux/slab.h> #include <linux/uaccess.h> #include <linux/proc_fs.h> #include <linux/of.h> @@ -3934,12 +3935,13 @@ static void ppc440spe_adma_free_chan_resources(struct dma_chan *chan) } /** - * ppc440spe_adma_is_complete - poll the status of an ADMA transaction + * ppc440spe_adma_tx_status - poll the status of an ADMA transaction * @chan: ADMA channel handle * @cookie: ADMA transaction identifier + * @txstate: a holder for the current state of the channel */ -static enum dma_status ppc440spe_adma_is_complete(struct dma_chan *chan, - dma_cookie_t cookie, dma_cookie_t *done, dma_cookie_t *used) +static enum dma_status ppc440spe_adma_tx_status(struct dma_chan *chan, + dma_cookie_t cookie, struct dma_tx_state *txstate) { struct ppc440spe_adma_chan *ppc440spe_chan; dma_cookie_t last_used; @@ -3950,10 +3952,7 @@ static enum dma_status ppc440spe_adma_is_complete(struct dma_chan *chan, last_used = chan->cookie; last_complete = ppc440spe_chan->completed_cookie; - if (done) - *done = last_complete; - if (used) - *used = last_used; + dma_set_tx_state(txstate, last_complete, last_used, 0); ret = dma_async_is_complete(cookie, last_complete, last_used); if (ret == DMA_SUCCESS) @@ -3964,10 +3963,7 @@ static enum dma_status ppc440spe_adma_is_complete(struct dma_chan *chan, last_used = chan->cookie; last_complete = ppc440spe_chan->completed_cookie; - if (done) - *done = last_complete; - if (used) - *used = last_used; + dma_set_tx_state(txstate, last_complete, last_used, 0); return dma_async_is_complete(cookie, last_complete, last_used); } @@ -4179,7 +4175,7 @@ static void ppc440spe_adma_init_capabilities(struct ppc440spe_adma_device *adev) ppc440spe_adma_alloc_chan_resources; adev->common.device_free_chan_resources = ppc440spe_adma_free_chan_resources; - adev->common.device_is_tx_complete = ppc440spe_adma_is_complete; + adev->common.device_tx_status = ppc440spe_adma_tx_status; adev->common.device_issue_pending = ppc440spe_adma_issue_pending; /* Set prep routines based on capability */ @@ -4261,10 +4257,12 @@ static int ppc440spe_adma_setup_irqs(struct ppc440spe_adma_device *adev, struct ppc440spe_adma_chan *chan, int *initcode) { + struct of_device *ofdev; struct device_node *np; int ret; - np = container_of(adev->dev, struct of_device, dev)->node; + ofdev = container_of(adev->dev, struct of_device, dev); + np = ofdev->dev.of_node; if (adev->id != PPC440SPE_XOR_ID) { adev->err_irq = irq_of_parse_and_map(np, 1); if (adev->err_irq == NO_IRQ) { @@ -4398,7 +4396,7 @@ static void ppc440spe_adma_release_irqs(struct ppc440spe_adma_device *adev, static int __devinit ppc440spe_adma_probe(struct of_device *ofdev, const struct of_device_id *match) { - struct device_node *np = ofdev->node; + struct device_node *np = ofdev->dev.of_node; struct resource res; struct ppc440spe_adma_device *adev; struct ppc440spe_adma_chan *chan; @@ -4630,7 +4628,7 @@ out: static int __devexit ppc440spe_adma_remove(struct of_device *ofdev) { struct ppc440spe_adma_device *adev = dev_get_drvdata(&ofdev->dev); - struct device_node *np = ofdev->node; + struct device_node *np = ofdev->dev.of_node; struct resource res; struct dma_chan *chan, *_chan; struct ppc_dma_chan_ref *ref, *_ref; @@ -4940,7 +4938,7 @@ out_free: return ret; } -static struct of_device_id __devinitdata ppc440spe_adma_of_match[] = { +static const struct of_device_id ppc440spe_adma_of_match[] __devinitconst = { { .compatible = "ibm,dma-440spe", }, { .compatible = "amcc,xor-accelerator", }, {}, @@ -4948,12 +4946,12 @@ static struct of_device_id __devinitdata ppc440spe_adma_of_match[] = { MODULE_DEVICE_TABLE(of, ppc440spe_adma_of_match); static struct of_platform_driver ppc440spe_adma_driver = { - .match_table = ppc440spe_adma_of_match, .probe = ppc440spe_adma_probe, .remove = __devexit_p(ppc440spe_adma_remove), .driver = { .name = "PPC440SP(E)-ADMA", .owner = THIS_MODULE, + .of_match_table = ppc440spe_adma_of_match, }, }; diff --git a/drivers/dma/shdma.c b/drivers/dma/shdma.c index d10cc899c460..a2a519fd2a24 100644 --- a/drivers/dma/shdma.c +++ b/drivers/dma/shdma.c @@ -19,13 +19,15 @@ #include <linux/init.h> #include <linux/module.h> +#include <linux/slab.h> #include <linux/interrupt.h> #include <linux/dmaengine.h> #include <linux/delay.h> #include <linux/dma-mapping.h> #include <linux/platform_device.h> -#include <cpu/dma.h> -#include <asm/dma-sh.h> +#include <linux/pm_runtime.h> +#include <linux/sh_dma.h> + #include "shdma.h" /* DMA descriptor control */ @@ -38,33 +40,32 @@ enum sh_dmae_desc_status { }; #define NR_DESCS_PER_CHANNEL 32 -/* - * Define the default configuration for dual address memory-memory transfer. - * The 0x400 value represents auto-request, external->external. - * - * And this driver set 4byte burst mode. - * If you want to change mode, you need to change RS_DEFAULT of value. - * (ex 1byte burst mode -> (RS_DUAL & ~TS_32) - */ -#define RS_DEFAULT (RS_DUAL) +/* Default MEMCPY transfer size = 2^2 = 4 bytes */ +#define LOG2_DEFAULT_XFER_SIZE 2 + +/* A bitmask with bits enough for enum sh_dmae_slave_chan_id */ +static unsigned long sh_dmae_slave_used[BITS_TO_LONGS(SH_DMA_SLAVE_NUMBER)]; static void sh_dmae_chan_ld_cleanup(struct sh_dmae_chan *sh_chan, bool all); -#define SH_DMAC_CHAN_BASE(id) (dma_base_addr[id]) static void sh_dmae_writel(struct sh_dmae_chan *sh_dc, u32 data, u32 reg) { - ctrl_outl(data, (SH_DMAC_CHAN_BASE(sh_dc->id) + reg)); + __raw_writel(data, sh_dc->base + reg / sizeof(u32)); } static u32 sh_dmae_readl(struct sh_dmae_chan *sh_dc, u32 reg) { - return ctrl_inl((SH_DMAC_CHAN_BASE(sh_dc->id) + reg)); + return __raw_readl(sh_dc->base + reg / sizeof(u32)); } -static void dmae_init(struct sh_dmae_chan *sh_chan) +static u16 dmaor_read(struct sh_dmae_device *shdev) { - u32 chcr = RS_DEFAULT; /* default is DUAL mode */ - sh_dmae_writel(sh_chan, chcr, CHCR); + return __raw_readw(shdev->chan_reg + DMAOR / sizeof(u32)); +} + +static void dmaor_write(struct sh_dmae_device *shdev, u16 data) +{ + __raw_writew(data, shdev->chan_reg + DMAOR / sizeof(u32)); } /* @@ -72,50 +73,75 @@ static void dmae_init(struct sh_dmae_chan *sh_chan) * * SH7780 has two DMAOR register */ -static void sh_dmae_ctl_stop(int id) +static void sh_dmae_ctl_stop(struct sh_dmae_device *shdev) { - unsigned short dmaor = dmaor_read_reg(id); + unsigned short dmaor = dmaor_read(shdev); - dmaor &= ~(DMAOR_NMIF | DMAOR_AE); - dmaor_write_reg(id, dmaor); + dmaor_write(shdev, dmaor & ~(DMAOR_NMIF | DMAOR_AE | DMAOR_DME)); } -static int sh_dmae_rst(int id) +static int sh_dmae_rst(struct sh_dmae_device *shdev) { unsigned short dmaor; - sh_dmae_ctl_stop(id); - dmaor = dmaor_read_reg(id) | DMAOR_INIT; + sh_dmae_ctl_stop(shdev); + dmaor = dmaor_read(shdev) | shdev->pdata->dmaor_init; - dmaor_write_reg(id, dmaor); - if (dmaor_read_reg(id) & (DMAOR_AE | DMAOR_NMIF)) { - pr_warning(KERN_ERR "dma-sh: Can't initialize DMAOR.\n"); + dmaor_write(shdev, dmaor); + if (dmaor_read(shdev) & (DMAOR_AE | DMAOR_NMIF)) { + pr_warning("dma-sh: Can't initialize DMAOR.\n"); return -EINVAL; } return 0; } -static int dmae_is_busy(struct sh_dmae_chan *sh_chan) +static bool dmae_is_busy(struct sh_dmae_chan *sh_chan) { u32 chcr = sh_dmae_readl(sh_chan, CHCR); - if (chcr & CHCR_DE) { - if (!(chcr & CHCR_TE)) - return -EBUSY; /* working */ - } - return 0; /* waiting */ + + if ((chcr & (CHCR_DE | CHCR_TE)) == CHCR_DE) + return true; /* working */ + + return false; /* waiting */ } -static inline unsigned int calc_xmit_shift(struct sh_dmae_chan *sh_chan) +static unsigned int calc_xmit_shift(struct sh_dmae_chan *sh_chan, u32 chcr) { - u32 chcr = sh_dmae_readl(sh_chan, CHCR); - return ts_shift[(chcr & CHCR_TS_MASK) >> CHCR_TS_SHIFT]; + struct sh_dmae_device *shdev = container_of(sh_chan->common.device, + struct sh_dmae_device, common); + struct sh_dmae_pdata *pdata = shdev->pdata; + int cnt = ((chcr & pdata->ts_low_mask) >> pdata->ts_low_shift) | + ((chcr & pdata->ts_high_mask) >> pdata->ts_high_shift); + + if (cnt >= pdata->ts_shift_num) + cnt = 0; + + return pdata->ts_shift[cnt]; +} + +static u32 log2size_to_chcr(struct sh_dmae_chan *sh_chan, int l2size) +{ + struct sh_dmae_device *shdev = container_of(sh_chan->common.device, + struct sh_dmae_device, common); + struct sh_dmae_pdata *pdata = shdev->pdata; + int i; + + for (i = 0; i < pdata->ts_shift_num; i++) + if (pdata->ts_shift[i] == l2size) + break; + + if (i == pdata->ts_shift_num) + i = 0; + + return ((i << pdata->ts_low_shift) & pdata->ts_low_mask) | + ((i << pdata->ts_high_shift) & pdata->ts_high_mask); } static void dmae_set_reg(struct sh_dmae_chan *sh_chan, struct sh_dmae_regs *hw) { sh_dmae_writel(sh_chan, hw->sar, SAR); sh_dmae_writel(sh_chan, hw->dar, DAR); - sh_dmae_writel(sh_chan, hw->tcr >> calc_xmit_shift(sh_chan), TCR); + sh_dmae_writel(sh_chan, hw->tcr >> sh_chan->xmit_shift, TCR); } static void dmae_start(struct sh_dmae_chan *sh_chan) @@ -123,7 +149,7 @@ static void dmae_start(struct sh_dmae_chan *sh_chan) u32 chcr = sh_dmae_readl(sh_chan, CHCR); chcr |= CHCR_DE | CHCR_IE; - sh_dmae_writel(sh_chan, chcr, CHCR); + sh_dmae_writel(sh_chan, chcr & ~CHCR_TE, CHCR); } static void dmae_halt(struct sh_dmae_chan *sh_chan) @@ -134,55 +160,44 @@ static void dmae_halt(struct sh_dmae_chan *sh_chan) sh_dmae_writel(sh_chan, chcr, CHCR); } +static void dmae_init(struct sh_dmae_chan *sh_chan) +{ + /* + * Default configuration for dual address memory-memory transfer. + * 0x400 represents auto-request. + */ + u32 chcr = DM_INC | SM_INC | 0x400 | log2size_to_chcr(sh_chan, + LOG2_DEFAULT_XFER_SIZE); + sh_chan->xmit_shift = calc_xmit_shift(sh_chan, chcr); + sh_dmae_writel(sh_chan, chcr, CHCR); +} + static int dmae_set_chcr(struct sh_dmae_chan *sh_chan, u32 val) { - int ret = dmae_is_busy(sh_chan); /* When DMA was working, can not set data to CHCR */ - if (ret) - return ret; + if (dmae_is_busy(sh_chan)) + return -EBUSY; + sh_chan->xmit_shift = calc_xmit_shift(sh_chan, val); sh_dmae_writel(sh_chan, val, CHCR); + return 0; } -#define DMARS1_ADDR 0x04 -#define DMARS2_ADDR 0x08 -#define DMARS_SHIFT 8 -#define DMARS_CHAN_MSK 0x01 static int dmae_set_dmars(struct sh_dmae_chan *sh_chan, u16 val) { - u32 addr; - int shift = 0; - int ret = dmae_is_busy(sh_chan); - if (ret) - return ret; - - if (sh_chan->id & DMARS_CHAN_MSK) - shift = DMARS_SHIFT; - - switch (sh_chan->id) { - /* DMARS0 */ - case 0: - case 1: - addr = SH_DMARS_BASE; - break; - /* DMARS1 */ - case 2: - case 3: - addr = (SH_DMARS_BASE + DMARS1_ADDR); - break; - /* DMARS2 */ - case 4: - case 5: - addr = (SH_DMARS_BASE + DMARS2_ADDR); - break; - default: - return -EINVAL; - } + struct sh_dmae_device *shdev = container_of(sh_chan->common.device, + struct sh_dmae_device, common); + struct sh_dmae_pdata *pdata = shdev->pdata; + const struct sh_dmae_channel *chan_pdata = &pdata->channel[sh_chan->id]; + u16 __iomem *addr = shdev->dmars + chan_pdata->dmars / sizeof(u16); + int shift = chan_pdata->dmars_bit; - ctrl_outw((val << shift) | - (ctrl_inw(addr) & (shift ? 0xFF00 : 0x00FF)), - addr); + if (dmae_is_busy(sh_chan)) + return -EBUSY; + + __raw_writew((__raw_readw(addr) & (0xff00 >> shift)) | (val << shift), + addr); return 0; } @@ -250,10 +265,59 @@ static struct sh_desc *sh_dmae_get_desc(struct sh_dmae_chan *sh_chan) return NULL; } +static const struct sh_dmae_slave_config *sh_dmae_find_slave( + struct sh_dmae_chan *sh_chan, struct sh_dmae_slave *param) +{ + struct dma_device *dma_dev = sh_chan->common.device; + struct sh_dmae_device *shdev = container_of(dma_dev, + struct sh_dmae_device, common); + struct sh_dmae_pdata *pdata = shdev->pdata; + int i; + + if (param->slave_id >= SH_DMA_SLAVE_NUMBER) + return NULL; + + for (i = 0; i < pdata->slave_num; i++) + if (pdata->slave[i].slave_id == param->slave_id) + return pdata->slave + i; + + return NULL; +} + static int sh_dmae_alloc_chan_resources(struct dma_chan *chan) { struct sh_dmae_chan *sh_chan = to_sh_chan(chan); struct sh_desc *desc; + struct sh_dmae_slave *param = chan->private; + int ret; + + pm_runtime_get_sync(sh_chan->dev); + + /* + * This relies on the guarantee from dmaengine that alloc_chan_resources + * never runs concurrently with itself or free_chan_resources. + */ + if (param) { + const struct sh_dmae_slave_config *cfg; + + cfg = sh_dmae_find_slave(sh_chan, param); + if (!cfg) { + ret = -EINVAL; + goto efindslave; + } + + if (test_and_set_bit(param->slave_id, sh_dmae_slave_used)) { + ret = -EBUSY; + goto etestused; + } + + param->config = cfg; + + dmae_set_dmars(sh_chan, cfg->mid_rid); + dmae_set_chcr(sh_chan, cfg->chcr); + } else if ((sh_dmae_readl(sh_chan, CHCR) & 0xf00) != 0x400) { + dmae_init(sh_chan); + } spin_lock_bh(&sh_chan->desc_lock); while (sh_chan->descs_allocated < NR_DESCS_PER_CHANNEL) { @@ -274,7 +338,20 @@ static int sh_dmae_alloc_chan_resources(struct dma_chan *chan) } spin_unlock_bh(&sh_chan->desc_lock); + if (!sh_chan->descs_allocated) { + ret = -ENOMEM; + goto edescalloc; + } + return sh_chan->descs_allocated; + +edescalloc: + if (param) + clear_bit(param->slave_id, sh_dmae_slave_used); +etestused: +efindslave: + pm_runtime_put(sh_chan->dev); + return ret; } /* @@ -285,11 +362,20 @@ static void sh_dmae_free_chan_resources(struct dma_chan *chan) struct sh_dmae_chan *sh_chan = to_sh_chan(chan); struct sh_desc *desc, *_desc; LIST_HEAD(list); + int descs = sh_chan->descs_allocated; + + dmae_halt(sh_chan); /* Prepared and not submitted descriptors can still be on the queue */ if (!list_empty(&sh_chan->ld_queue)) sh_dmae_chan_ld_cleanup(sh_chan, true); + if (chan->private) { + /* The caller is holding dma_list_mutex */ + struct sh_dmae_slave *param = chan->private; + clear_bit(param->slave_id, sh_dmae_slave_used); + } + spin_lock_bh(&sh_chan->desc_lock); list_splice_init(&sh_chan->ld_free, &list); @@ -297,27 +383,104 @@ static void sh_dmae_free_chan_resources(struct dma_chan *chan) spin_unlock_bh(&sh_chan->desc_lock); + if (descs > 0) + pm_runtime_put(sh_chan->dev); + list_for_each_entry_safe(desc, _desc, &list, node) kfree(desc); } -static struct dma_async_tx_descriptor *sh_dmae_prep_memcpy( - struct dma_chan *chan, dma_addr_t dma_dest, dma_addr_t dma_src, - size_t len, unsigned long flags) +/** + * sh_dmae_add_desc - get, set up and return one transfer descriptor + * @sh_chan: DMA channel + * @flags: DMA transfer flags + * @dest: destination DMA address, incremented when direction equals + * DMA_FROM_DEVICE or DMA_BIDIRECTIONAL + * @src: source DMA address, incremented when direction equals + * DMA_TO_DEVICE or DMA_BIDIRECTIONAL + * @len: DMA transfer length + * @first: if NULL, set to the current descriptor and cookie set to -EBUSY + * @direction: needed for slave DMA to decide which address to keep constant, + * equals DMA_BIDIRECTIONAL for MEMCPY + * Returns 0 or an error + * Locks: called with desc_lock held + */ +static struct sh_desc *sh_dmae_add_desc(struct sh_dmae_chan *sh_chan, + unsigned long flags, dma_addr_t *dest, dma_addr_t *src, size_t *len, + struct sh_desc **first, enum dma_data_direction direction) { - struct sh_dmae_chan *sh_chan; - struct sh_desc *first = NULL, *prev = NULL, *new; + struct sh_desc *new; size_t copy_size; - LIST_HEAD(tx_list); - int chunks = (len + SH_DMA_TCR_MAX) / (SH_DMA_TCR_MAX + 1); - if (!chan) + if (!*len) return NULL; - if (!len) + /* Allocate the link descriptor from the free list */ + new = sh_dmae_get_desc(sh_chan); + if (!new) { + dev_err(sh_chan->dev, "No free link descriptor available\n"); return NULL; + } - sh_chan = to_sh_chan(chan); + copy_size = min(*len, (size_t)SH_DMA_TCR_MAX + 1); + + new->hw.sar = *src; + new->hw.dar = *dest; + new->hw.tcr = copy_size; + + if (!*first) { + /* First desc */ + new->async_tx.cookie = -EBUSY; + *first = new; + } else { + /* Other desc - invisible to the user */ + new->async_tx.cookie = -EINVAL; + } + + dev_dbg(sh_chan->dev, + "chaining (%u/%u)@%x -> %x with %p, cookie %d, shift %d\n", + copy_size, *len, *src, *dest, &new->async_tx, + new->async_tx.cookie, sh_chan->xmit_shift); + + new->mark = DESC_PREPARED; + new->async_tx.flags = flags; + new->direction = direction; + + *len -= copy_size; + if (direction == DMA_BIDIRECTIONAL || direction == DMA_TO_DEVICE) + *src += copy_size; + if (direction == DMA_BIDIRECTIONAL || direction == DMA_FROM_DEVICE) + *dest += copy_size; + + return new; +} + +/* + * sh_dmae_prep_sg - prepare transfer descriptors from an SG list + * + * Common routine for public (MEMCPY) and slave DMA. The MEMCPY case is also + * converted to scatter-gather to guarantee consistent locking and a correct + * list manipulation. For slave DMA direction carries the usual meaning, and, + * logically, the SG list is RAM and the addr variable contains slave address, + * e.g., the FIFO I/O register. For MEMCPY direction equals DMA_BIDIRECTIONAL + * and the SG list contains only one element and points at the source buffer. + */ +static struct dma_async_tx_descriptor *sh_dmae_prep_sg(struct sh_dmae_chan *sh_chan, + struct scatterlist *sgl, unsigned int sg_len, dma_addr_t *addr, + enum dma_data_direction direction, unsigned long flags) +{ + struct scatterlist *sg; + struct sh_desc *first = NULL, *new = NULL /* compiler... */; + LIST_HEAD(tx_list); + int chunks = 0; + int i; + + if (!sg_len) + return NULL; + + for_each_sg(sgl, sg, sg_len, i) + chunks += (sg_dma_len(sg) + SH_DMA_TCR_MAX) / + (SH_DMA_TCR_MAX + 1); /* Have to lock the whole loop to protect against concurrent release */ spin_lock_bh(&sh_chan->desc_lock); @@ -333,49 +496,32 @@ static struct dma_async_tx_descriptor *sh_dmae_prep_memcpy( * only during this function, then they are immediately spliced * back onto the free list in form of a chain */ - do { - /* Allocate the link descriptor from the free list */ - new = sh_dmae_get_desc(sh_chan); - if (!new) { - dev_err(sh_chan->dev, - "No free memory for link descriptor\n"); - list_for_each_entry(new, &tx_list, node) - new->mark = DESC_IDLE; - list_splice(&tx_list, &sh_chan->ld_free); - spin_unlock_bh(&sh_chan->desc_lock); - return NULL; - } - - copy_size = min(len, (size_t)SH_DMA_TCR_MAX + 1); - - new->hw.sar = dma_src; - new->hw.dar = dma_dest; - new->hw.tcr = copy_size; - if (!first) { - /* First desc */ - new->async_tx.cookie = -EBUSY; - first = new; - } else { - /* Other desc - invisible to the user */ - new->async_tx.cookie = -EINVAL; - } - - dev_dbg(sh_chan->dev, - "chaining %u of %u with %p, dst %x, cookie %d\n", - copy_size, len, &new->async_tx, dma_dest, - new->async_tx.cookie); - - new->mark = DESC_PREPARED; - new->async_tx.flags = flags; - new->chunks = chunks--; - - prev = new; - len -= copy_size; - dma_src += copy_size; - dma_dest += copy_size; - /* Insert the link descriptor to the LD ring */ - list_add_tail(&new->node, &tx_list); - } while (len); + for_each_sg(sgl, sg, sg_len, i) { + dma_addr_t sg_addr = sg_dma_address(sg); + size_t len = sg_dma_len(sg); + + if (!len) + goto err_get_desc; + + do { + dev_dbg(sh_chan->dev, "Add SG #%d@%p[%d], dma %llx\n", + i, sg, len, (unsigned long long)sg_addr); + + if (direction == DMA_FROM_DEVICE) + new = sh_dmae_add_desc(sh_chan, flags, + &sg_addr, addr, &len, &first, + direction); + else + new = sh_dmae_add_desc(sh_chan, flags, + addr, &sg_addr, &len, &first, + direction); + if (!new) + goto err_get_desc; + + new->chunks = chunks--; + list_add_tail(&new->node, &tx_list); + } while (len); + } if (new != first) new->async_tx.cookie = -ENOSPC; @@ -386,6 +532,99 @@ static struct dma_async_tx_descriptor *sh_dmae_prep_memcpy( spin_unlock_bh(&sh_chan->desc_lock); return &first->async_tx; + +err_get_desc: + list_for_each_entry(new, &tx_list, node) + new->mark = DESC_IDLE; + list_splice(&tx_list, &sh_chan->ld_free); + + spin_unlock_bh(&sh_chan->desc_lock); + + return NULL; +} + +static struct dma_async_tx_descriptor *sh_dmae_prep_memcpy( + struct dma_chan *chan, dma_addr_t dma_dest, dma_addr_t dma_src, + size_t len, unsigned long flags) +{ + struct sh_dmae_chan *sh_chan; + struct scatterlist sg; + + if (!chan || !len) + return NULL; + + chan->private = NULL; + + sh_chan = to_sh_chan(chan); + + sg_init_table(&sg, 1); + sg_set_page(&sg, pfn_to_page(PFN_DOWN(dma_src)), len, + offset_in_page(dma_src)); + sg_dma_address(&sg) = dma_src; + sg_dma_len(&sg) = len; + + return sh_dmae_prep_sg(sh_chan, &sg, 1, &dma_dest, DMA_BIDIRECTIONAL, + flags); +} + +static struct dma_async_tx_descriptor *sh_dmae_prep_slave_sg( + struct dma_chan *chan, struct scatterlist *sgl, unsigned int sg_len, + enum dma_data_direction direction, unsigned long flags) +{ + struct sh_dmae_slave *param; + struct sh_dmae_chan *sh_chan; + dma_addr_t slave_addr; + + if (!chan) + return NULL; + + sh_chan = to_sh_chan(chan); + param = chan->private; + slave_addr = param->config->addr; + + /* Someone calling slave DMA on a public channel? */ + if (!param || !sg_len) { + dev_warn(sh_chan->dev, "%s: bad parameter: %p, %d, %d\n", + __func__, param, sg_len, param ? param->slave_id : -1); + return NULL; + } + + /* + * if (param != NULL), this is a successfully requested slave channel, + * therefore param->config != NULL too. + */ + return sh_dmae_prep_sg(sh_chan, sgl, sg_len, &slave_addr, + direction, flags); +} + +static int sh_dmae_control(struct dma_chan *chan, enum dma_ctrl_cmd cmd, + unsigned long arg) +{ + struct sh_dmae_chan *sh_chan = to_sh_chan(chan); + + /* Only supports DMA_TERMINATE_ALL */ + if (cmd != DMA_TERMINATE_ALL) + return -ENXIO; + + if (!chan) + return -EINVAL; + + dmae_halt(sh_chan); + + spin_lock_bh(&sh_chan->desc_lock); + if (!list_empty(&sh_chan->ld_queue)) { + /* Record partial transfer */ + struct sh_desc *desc = list_entry(sh_chan->ld_queue.next, + struct sh_desc, node); + desc->partial = (desc->hw.tcr - sh_dmae_readl(sh_chan, TCR)) << + sh_chan->xmit_shift; + + } + spin_unlock_bh(&sh_chan->desc_lock); + + sh_dmae_chan_ld_cleanup(sh_chan, true); + + return 0; } static dma_async_tx_callback __ld_cleanup(struct sh_dmae_chan *sh_chan, bool all) @@ -419,7 +658,11 @@ static dma_async_tx_callback __ld_cleanup(struct sh_dmae_chan *sh_chan, bool all cookie = tx->cookie; if (desc->mark == DESC_COMPLETED && desc->chunks == 1) { - BUG_ON(sh_chan->completed_cookie != desc->cookie - 1); + if (sh_chan->completed_cookie != desc->cookie - 1) + dev_dbg(sh_chan->dev, + "Completing cookie %d, expected %d\n", + desc->cookie, + sh_chan->completed_cookie + 1); sh_chan->completed_cookie = desc->cookie; } @@ -479,11 +722,15 @@ static void sh_dmae_chan_ld_cleanup(struct sh_dmae_chan *sh_chan, bool all) { while (__ld_cleanup(sh_chan, all)) ; + + if (all) + /* Terminating - forgive uncompleted cookies */ + sh_chan->completed_cookie = sh_chan->common.cookie; } static void sh_chan_xfer_ld_queue(struct sh_dmae_chan *sh_chan) { - struct sh_desc *sd; + struct sh_desc *desc; spin_lock_bh(&sh_chan->desc_lock); /* DMA work check */ @@ -492,11 +739,14 @@ static void sh_chan_xfer_ld_queue(struct sh_dmae_chan *sh_chan) return; } - /* Find the first un-transfer desciptor */ - list_for_each_entry(sd, &sh_chan->ld_queue, node) - if (sd->mark == DESC_SUBMITTED) { + /* Find the first not transferred desciptor */ + list_for_each_entry(desc, &sh_chan->ld_queue, node) + if (desc->mark == DESC_SUBMITTED) { + dev_dbg(sh_chan->dev, "Queue #%d to %d: %u@%x -> %x\n", + desc->async_tx.cookie, sh_chan->id, + desc->hw.tcr, desc->hw.sar, desc->hw.dar); /* Get the ld start address from ld_queue */ - dmae_set_reg(sh_chan, &sd->hw); + dmae_set_reg(sh_chan, &desc->hw); dmae_start(sh_chan); break; } @@ -510,28 +760,43 @@ static void sh_dmae_memcpy_issue_pending(struct dma_chan *chan) sh_chan_xfer_ld_queue(sh_chan); } -static enum dma_status sh_dmae_is_complete(struct dma_chan *chan, +static enum dma_status sh_dmae_tx_status(struct dma_chan *chan, dma_cookie_t cookie, - dma_cookie_t *done, - dma_cookie_t *used) + struct dma_tx_state *txstate) { struct sh_dmae_chan *sh_chan = to_sh_chan(chan); dma_cookie_t last_used; dma_cookie_t last_complete; + enum dma_status status; sh_dmae_chan_ld_cleanup(sh_chan, false); last_used = chan->cookie; last_complete = sh_chan->completed_cookie; BUG_ON(last_complete < 0); + dma_set_tx_state(txstate, last_complete, last_used, 0); - if (done) - *done = last_complete; + spin_lock_bh(&sh_chan->desc_lock); + + status = dma_async_is_complete(cookie, last_complete, last_used); + + /* + * If we don't find cookie on the queue, it has been aborted and we have + * to report error + */ + if (status != DMA_SUCCESS) { + struct sh_desc *desc; + status = DMA_ERROR; + list_for_each_entry(desc, &sh_chan->ld_queue, node) + if (desc->cookie == cookie) { + status = DMA_IN_PROGRESS; + break; + } + } - if (used) - *used = last_used; + spin_unlock_bh(&sh_chan->desc_lock); - return dma_async_is_complete(cookie, last_complete, last_used); + return status; } static irqreturn_t sh_dmae_interrupt(int irq, void *data) @@ -554,40 +819,32 @@ static irqreturn_t sh_dmae_interrupt(int irq, void *data) #if defined(CONFIG_CPU_SH4) static irqreturn_t sh_dmae_err(int irq, void *data) { - int err = 0; struct sh_dmae_device *shdev = (struct sh_dmae_device *)data; + int i; - /* IRQ Multi */ - if (shdev->pdata.mode & SHDMA_MIX_IRQ) { - int cnt = 0; - switch (irq) { -#if defined(DMTE6_IRQ) && defined(DMAE1_IRQ) - case DMTE6_IRQ: - cnt++; -#endif - case DMTE0_IRQ: - if (dmaor_read_reg(cnt) & (DMAOR_NMIF | DMAOR_AE)) { - disable_irq(irq); - return IRQ_HANDLED; + /* halt the dma controller */ + sh_dmae_ctl_stop(shdev); + + /* We cannot detect, which channel caused the error, have to reset all */ + for (i = 0; i < SH_DMAC_MAX_CHANNELS; i++) { + struct sh_dmae_chan *sh_chan = shdev->chan[i]; + if (sh_chan) { + struct sh_desc *desc; + /* Stop the channel */ + dmae_halt(sh_chan); + /* Complete all */ + list_for_each_entry(desc, &sh_chan->ld_queue, node) { + struct dma_async_tx_descriptor *tx = &desc->async_tx; + desc->mark = DESC_IDLE; + if (tx->callback) + tx->callback(tx->callback_param); } - default: - return IRQ_NONE; - } - } else { - /* reset dma controller */ - err = sh_dmae_rst(0); - if (err) - return err; -#ifdef SH_DMAC_BASE1 - if (shdev->pdata.mode & SHDMA_DMAOR1) { - err = sh_dmae_rst(1); - if (err) - return err; + list_splice_init(&sh_chan->ld_queue, &sh_chan->ld_free); } -#endif - disable_irq(irq); - return IRQ_HANDLED; } + sh_dmae_rst(shdev); + + return IRQ_HANDLED; } #endif @@ -596,11 +853,14 @@ static void dmae_do_tasklet(unsigned long data) struct sh_dmae_chan *sh_chan = (struct sh_dmae_chan *)data; struct sh_desc *desc; u32 sar_buf = sh_dmae_readl(sh_chan, SAR); + u32 dar_buf = sh_dmae_readl(sh_chan, DAR); spin_lock(&sh_chan->desc_lock); list_for_each_entry(desc, &sh_chan->ld_queue, node) { - if ((desc->hw.sar + desc->hw.tcr) == sar_buf && - desc->mark == DESC_SUBMITTED) { + if (desc->mark == DESC_SUBMITTED && + ((desc->direction == DMA_FROM_DEVICE && + (desc->hw.dar + desc->hw.tcr) == dar_buf) || + (desc->hw.sar + desc->hw.tcr) == sar_buf)) { dev_dbg(sh_chan->dev, "done #%d@%p dst %u\n", desc->async_tx.cookie, &desc->async_tx, desc->hw.dar); @@ -615,19 +875,12 @@ static void dmae_do_tasklet(unsigned long data) sh_dmae_chan_ld_cleanup(sh_chan, false); } -static unsigned int get_dmae_irq(unsigned int id) -{ - unsigned int irq = 0; - if (id < ARRAY_SIZE(dmte_irq_map)) - irq = dmte_irq_map[id]; - return irq; -} - -static int __devinit sh_dmae_chan_probe(struct sh_dmae_device *shdev, int id) +static int __devinit sh_dmae_chan_probe(struct sh_dmae_device *shdev, int id, + int irq, unsigned long flags) { int err; - unsigned int irq = get_dmae_irq(id); - unsigned long irqflags = IRQF_DISABLED; + const struct sh_dmae_channel *chan_pdata = &shdev->pdata->channel[id]; + struct platform_device *pdev = to_platform_device(shdev->common.dev); struct sh_dmae_chan *new_sh_chan; /* alloc channel */ @@ -638,8 +891,13 @@ static int __devinit sh_dmae_chan_probe(struct sh_dmae_device *shdev, int id) return -ENOMEM; } + /* copy struct dma_device */ + new_sh_chan->common.device = &shdev->common; + new_sh_chan->dev = shdev->common.dev; new_sh_chan->id = id; + new_sh_chan->irq = irq; + new_sh_chan->base = shdev->chan_reg + chan_pdata->offset / sizeof(u32); /* Init DMA tasklet */ tasklet_init(&new_sh_chan->tasklet, dmae_do_tasklet, @@ -654,29 +912,20 @@ static int __devinit sh_dmae_chan_probe(struct sh_dmae_device *shdev, int id) INIT_LIST_HEAD(&new_sh_chan->ld_queue); INIT_LIST_HEAD(&new_sh_chan->ld_free); - /* copy struct dma_device */ - new_sh_chan->common.device = &shdev->common; - /* Add the channel to DMA device channel list */ list_add_tail(&new_sh_chan->common.device_node, &shdev->common.channels); shdev->common.chancnt++; - if (shdev->pdata.mode & SHDMA_MIX_IRQ) { - irqflags = IRQF_SHARED; -#if defined(DMTE6_IRQ) - if (irq >= DMTE6_IRQ) - irq = DMTE6_IRQ; - else -#endif - irq = DMTE0_IRQ; - } - - snprintf(new_sh_chan->dev_id, sizeof(new_sh_chan->dev_id), - "sh-dmae%d", new_sh_chan->id); + if (pdev->id >= 0) + snprintf(new_sh_chan->dev_id, sizeof(new_sh_chan->dev_id), + "sh-dmae%d.%d", pdev->id, new_sh_chan->id); + else + snprintf(new_sh_chan->dev_id, sizeof(new_sh_chan->dev_id), + "sh-dma%d", new_sh_chan->id); /* set up channel irq */ - err = request_irq(irq, &sh_dmae_interrupt, irqflags, + err = request_irq(irq, &sh_dmae_interrupt, flags, new_sh_chan->dev_id, new_sh_chan); if (err) { dev_err(shdev->common.dev, "DMA channel %d request_irq error " @@ -684,11 +933,6 @@ static int __devinit sh_dmae_chan_probe(struct sh_dmae_device *shdev, int id) goto err_no_irq; } - /* CHCR register control function */ - new_sh_chan->set_chcr = dmae_set_chcr; - /* DMARS register control function */ - new_sh_chan->set_dmars = dmae_set_dmars; - shdev->chan[id] = new_sh_chan; return 0; @@ -705,12 +949,12 @@ static void sh_dmae_chan_remove(struct sh_dmae_device *shdev) for (i = shdev->common.chancnt - 1 ; i >= 0 ; i--) { if (shdev->chan[i]) { - struct sh_dmae_chan *shchan = shdev->chan[i]; - if (!(shdev->pdata.mode & SHDMA_MIX_IRQ)) - free_irq(dmte_irq_map[i], shchan); + struct sh_dmae_chan *sh_chan = shdev->chan[i]; + + free_irq(sh_chan->irq, sh_chan); - list_del(&shchan->common.device_node); - kfree(shchan); + list_del(&sh_chan->common.device_node); + kfree(sh_chan); shdev->chan[i] = NULL; } } @@ -719,85 +963,164 @@ static void sh_dmae_chan_remove(struct sh_dmae_device *shdev) static int __init sh_dmae_probe(struct platform_device *pdev) { - int err = 0, cnt, ecnt; - unsigned long irqflags = IRQF_DISABLED; -#if defined(CONFIG_CPU_SH4) - int eirq[] = { DMAE0_IRQ, -#if defined(DMAE1_IRQ) - DMAE1_IRQ -#endif - }; -#endif + struct sh_dmae_pdata *pdata = pdev->dev.platform_data; + unsigned long irqflags = IRQF_DISABLED, + chan_flag[SH_DMAC_MAX_CHANNELS] = {}; + int errirq, chan_irq[SH_DMAC_MAX_CHANNELS]; + int err, i, irq_cnt = 0, irqres = 0; struct sh_dmae_device *shdev; + struct resource *chan, *dmars, *errirq_res, *chanirq_res; /* get platform data */ - if (!pdev->dev.platform_data) + if (!pdata || !pdata->channel_num) return -ENODEV; + chan = platform_get_resource(pdev, IORESOURCE_MEM, 0); + /* DMARS area is optional, if absent, this controller cannot do slave DMA */ + dmars = platform_get_resource(pdev, IORESOURCE_MEM, 1); + /* + * IRQ resources: + * 1. there always must be at least one IRQ IO-resource. On SH4 it is + * the error IRQ, in which case it is the only IRQ in this resource: + * start == end. If it is the only IRQ resource, all channels also + * use the same IRQ. + * 2. DMA channel IRQ resources can be specified one per resource or in + * ranges (start != end) + * 3. iff all events (channels and, optionally, error) on this + * controller use the same IRQ, only one IRQ resource can be + * specified, otherwise there must be one IRQ per channel, even if + * some of them are equal + * 4. if all IRQs on this controller are equal or if some specific IRQs + * specify IORESOURCE_IRQ_SHAREABLE in their resources, they will be + * requested with the IRQF_SHARED flag + */ + errirq_res = platform_get_resource(pdev, IORESOURCE_IRQ, 0); + if (!chan || !errirq_res) + return -ENODEV; + + if (!request_mem_region(chan->start, resource_size(chan), pdev->name)) { + dev_err(&pdev->dev, "DMAC register region already claimed\n"); + return -EBUSY; + } + + if (dmars && !request_mem_region(dmars->start, resource_size(dmars), pdev->name)) { + dev_err(&pdev->dev, "DMAC DMARS region already claimed\n"); + err = -EBUSY; + goto ermrdmars; + } + + err = -ENOMEM; shdev = kzalloc(sizeof(struct sh_dmae_device), GFP_KERNEL); if (!shdev) { - dev_err(&pdev->dev, "No enough memory\n"); - return -ENOMEM; + dev_err(&pdev->dev, "Not enough memory\n"); + goto ealloc; + } + + shdev->chan_reg = ioremap(chan->start, resource_size(chan)); + if (!shdev->chan_reg) + goto emapchan; + if (dmars) { + shdev->dmars = ioremap(dmars->start, resource_size(dmars)); + if (!shdev->dmars) + goto emapdmars; } /* platform data */ - memcpy(&shdev->pdata, pdev->dev.platform_data, - sizeof(struct sh_dmae_pdata)); + shdev->pdata = pdata; + + pm_runtime_enable(&pdev->dev); + pm_runtime_get_sync(&pdev->dev); /* reset dma controller */ - err = sh_dmae_rst(0); + err = sh_dmae_rst(shdev); if (err) goto rst_err; - /* SH7780/85/23 has DMAOR1 */ - if (shdev->pdata.mode & SHDMA_DMAOR1) { - err = sh_dmae_rst(1); - if (err) - goto rst_err; - } - INIT_LIST_HEAD(&shdev->common.channels); dma_cap_set(DMA_MEMCPY, shdev->common.cap_mask); + if (dmars) + dma_cap_set(DMA_SLAVE, shdev->common.cap_mask); + shdev->common.device_alloc_chan_resources = sh_dmae_alloc_chan_resources; shdev->common.device_free_chan_resources = sh_dmae_free_chan_resources; shdev->common.device_prep_dma_memcpy = sh_dmae_prep_memcpy; - shdev->common.device_is_tx_complete = sh_dmae_is_complete; + shdev->common.device_tx_status = sh_dmae_tx_status; shdev->common.device_issue_pending = sh_dmae_memcpy_issue_pending; + + /* Compulsory for DMA_SLAVE fields */ + shdev->common.device_prep_slave_sg = sh_dmae_prep_slave_sg; + shdev->common.device_control = sh_dmae_control; + shdev->common.dev = &pdev->dev; /* Default transfer size of 32 bytes requires 32-byte alignment */ - shdev->common.copy_align = 5; + shdev->common.copy_align = LOG2_DEFAULT_XFER_SIZE; #if defined(CONFIG_CPU_SH4) - /* Non Mix IRQ mode SH7722/SH7730 etc... */ - if (shdev->pdata.mode & SHDMA_MIX_IRQ) { + chanirq_res = platform_get_resource(pdev, IORESOURCE_IRQ, 1); + + if (!chanirq_res) + chanirq_res = errirq_res; + else + irqres++; + + if (chanirq_res == errirq_res || + (errirq_res->flags & IORESOURCE_BITS) == IORESOURCE_IRQ_SHAREABLE) irqflags = IRQF_SHARED; - eirq[0] = DMTE0_IRQ; -#if defined(DMTE6_IRQ) && defined(DMAE1_IRQ) - eirq[1] = DMTE6_IRQ; -#endif + + errirq = errirq_res->start; + + err = request_irq(errirq, sh_dmae_err, irqflags, + "DMAC Address Error", shdev); + if (err) { + dev_err(&pdev->dev, + "DMA failed requesting irq #%d, error %d\n", + errirq, err); + goto eirq_err; } - for (ecnt = 0 ; ecnt < ARRAY_SIZE(eirq); ecnt++) { - err = request_irq(eirq[ecnt], sh_dmae_err, irqflags, - "DMAC Address Error", shdev); - if (err) { - dev_err(&pdev->dev, "DMA device request_irq" - "error (irq %d) with return %d\n", - eirq[ecnt], err); - goto eirq_err; +#else + chanirq_res = errirq_res; +#endif /* CONFIG_CPU_SH4 */ + + if (chanirq_res->start == chanirq_res->end && + !platform_get_resource(pdev, IORESOURCE_IRQ, 1)) { + /* Special case - all multiplexed */ + for (; irq_cnt < pdata->channel_num; irq_cnt++) { + chan_irq[irq_cnt] = chanirq_res->start; + chan_flag[irq_cnt] = IRQF_SHARED; } + } else { + do { + for (i = chanirq_res->start; i <= chanirq_res->end; i++) { + if ((errirq_res->flags & IORESOURCE_BITS) == + IORESOURCE_IRQ_SHAREABLE) + chan_flag[irq_cnt] = IRQF_SHARED; + else + chan_flag[irq_cnt] = IRQF_DISABLED; + dev_dbg(&pdev->dev, + "Found IRQ %d for channel %d\n", + i, irq_cnt); + chan_irq[irq_cnt++] = i; + } + chanirq_res = platform_get_resource(pdev, + IORESOURCE_IRQ, ++irqres); + } while (irq_cnt < pdata->channel_num && chanirq_res); } -#endif /* CONFIG_CPU_SH4 */ + + if (irq_cnt < pdata->channel_num) + goto eirqres; /* Create DMA Channel */ - for (cnt = 0 ; cnt < MAX_DMA_CHANNELS ; cnt++) { - err = sh_dmae_chan_probe(shdev, cnt); + for (i = 0; i < pdata->channel_num; i++) { + err = sh_dmae_chan_probe(shdev, i, chan_irq[i], chan_flag[i]); if (err) goto chan_probe_err; } + pm_runtime_put(&pdev->dev); + platform_set_drvdata(pdev, shdev); dma_async_device_register(&shdev->common); @@ -805,13 +1128,24 @@ static int __init sh_dmae_probe(struct platform_device *pdev) chan_probe_err: sh_dmae_chan_remove(shdev); - +eirqres: +#if defined(CONFIG_CPU_SH4) + free_irq(errirq, shdev); eirq_err: - for (ecnt-- ; ecnt >= 0; ecnt--) - free_irq(eirq[ecnt], shdev); - +#endif rst_err: + pm_runtime_put(&pdev->dev); + if (dmars) + iounmap(shdev->dmars); +emapdmars: + iounmap(shdev->chan_reg); +emapchan: kfree(shdev); +ealloc: + if (dmars) + release_mem_region(dmars->start, resource_size(dmars)); +ermrdmars: + release_mem_region(chan->start, resource_size(chan)); return err; } @@ -819,42 +1153,46 @@ rst_err: static int __exit sh_dmae_remove(struct platform_device *pdev) { struct sh_dmae_device *shdev = platform_get_drvdata(pdev); + struct resource *res; + int errirq = platform_get_irq(pdev, 0); dma_async_device_unregister(&shdev->common); - if (shdev->pdata.mode & SHDMA_MIX_IRQ) { - free_irq(DMTE0_IRQ, shdev); -#if defined(DMTE6_IRQ) - free_irq(DMTE6_IRQ, shdev); -#endif - } + if (errirq > 0) + free_irq(errirq, shdev); /* channel data remove */ sh_dmae_chan_remove(shdev); - if (!(shdev->pdata.mode & SHDMA_MIX_IRQ)) { - free_irq(DMAE0_IRQ, shdev); -#if defined(DMAE1_IRQ) - free_irq(DMAE1_IRQ, shdev); -#endif - } + pm_runtime_disable(&pdev->dev); + + if (shdev->dmars) + iounmap(shdev->dmars); + iounmap(shdev->chan_reg); + kfree(shdev); + res = platform_get_resource(pdev, IORESOURCE_MEM, 0); + if (res) + release_mem_region(res->start, resource_size(res)); + res = platform_get_resource(pdev, IORESOURCE_MEM, 1); + if (res) + release_mem_region(res->start, resource_size(res)); + return 0; } static void sh_dmae_shutdown(struct platform_device *pdev) { struct sh_dmae_device *shdev = platform_get_drvdata(pdev); - sh_dmae_ctl_stop(0); - if (shdev->pdata.mode & SHDMA_DMAOR1) - sh_dmae_ctl_stop(1); + sh_dmae_ctl_stop(shdev); } static struct platform_driver sh_dmae_driver = { .remove = __exit_p(sh_dmae_remove), .shutdown = sh_dmae_shutdown, .driver = { + .owner = THIS_MODULE, .name = "sh-dma-engine", }, }; diff --git a/drivers/dma/shdma.h b/drivers/dma/shdma.h index 108f1cffb6f5..4021275a0a43 100644 --- a/drivers/dma/shdma.h +++ b/drivers/dma/shdma.h @@ -17,23 +17,10 @@ #include <linux/interrupt.h> #include <linux/list.h> +#define SH_DMAC_MAX_CHANNELS 6 +#define SH_DMA_SLAVE_NUMBER 256 #define SH_DMA_TCR_MAX 0x00FFFFFF /* 16MB */ -struct sh_dmae_regs { - u32 sar; /* SAR / source address */ - u32 dar; /* DAR / destination address */ - u32 tcr; /* TCR / transfer count */ -}; - -struct sh_desc { - struct sh_dmae_regs hw; - struct list_head node; - struct dma_async_tx_descriptor async_tx; - dma_cookie_t cookie; - int chunks; - int mark; -}; - struct device; struct sh_dmae_chan { @@ -45,19 +32,19 @@ struct sh_dmae_chan { struct device *dev; /* Channel device */ struct tasklet_struct tasklet; /* Tasklet */ int descs_allocated; /* desc count */ + int xmit_shift; /* log_2(bytes_per_xfer) */ + int irq; int id; /* Raw id of this channel */ + u32 __iomem *base; char dev_id[16]; /* unique name per DMAC of channel */ - - /* Set chcr */ - int (*set_chcr)(struct sh_dmae_chan *sh_chan, u32 regs); - /* Set DMA resource */ - int (*set_dmars)(struct sh_dmae_chan *sh_chan, u16 res); }; struct sh_dmae_device { struct dma_device common; - struct sh_dmae_chan *chan[MAX_DMA_CHANNELS]; - struct sh_dmae_pdata pdata; + struct sh_dmae_chan *chan[SH_DMAC_MAX_CHANNELS]; + struct sh_dmae_pdata *pdata; + u32 __iomem *chan_reg; + u16 __iomem *dmars; }; #define to_sh_chan(chan) container_of(chan, struct sh_dmae_chan, common) diff --git a/drivers/dma/ste_dma40.c b/drivers/dma/ste_dma40.c new file mode 100644 index 000000000000..c426829f6ab8 --- /dev/null +++ b/drivers/dma/ste_dma40.c @@ -0,0 +1,2657 @@ +/* + * driver/dma/ste_dma40.c + * + * Copyright (C) ST-Ericsson 2007-2010 + * License terms: GNU General Public License (GPL) version 2 + * Author: Per Friden <per.friden@stericsson.com> + * Author: Jonas Aaberg <jonas.aberg@stericsson.com> + * + */ + +#include <linux/kernel.h> +#include <linux/slab.h> +#include <linux/dmaengine.h> +#include <linux/platform_device.h> +#include <linux/clk.h> +#include <linux/delay.h> + +#include <plat/ste_dma40.h> + +#include "ste_dma40_ll.h" + +#define D40_NAME "dma40" + +#define D40_PHY_CHAN -1 + +/* For masking out/in 2 bit channel positions */ +#define D40_CHAN_POS(chan) (2 * (chan / 2)) +#define D40_CHAN_POS_MASK(chan) (0x3 << D40_CHAN_POS(chan)) + +/* Maximum iterations taken before giving up suspending a channel */ +#define D40_SUSPEND_MAX_IT 500 + +#define D40_ALLOC_FREE (1 << 31) +#define D40_ALLOC_PHY (1 << 30) +#define D40_ALLOC_LOG_FREE 0 + +/* The number of free d40_desc to keep in memory before starting + * to kfree() them */ +#define D40_DESC_CACHE_SIZE 50 + +/* Hardware designer of the block */ +#define D40_PERIPHID2_DESIGNER 0x8 + +/** + * enum 40_command - The different commands and/or statuses. + * + * @D40_DMA_STOP: DMA channel command STOP or status STOPPED, + * @D40_DMA_RUN: The DMA channel is RUNNING of the command RUN. + * @D40_DMA_SUSPEND_REQ: Request the DMA to SUSPEND as soon as possible. + * @D40_DMA_SUSPENDED: The DMA channel is SUSPENDED. + */ +enum d40_command { + D40_DMA_STOP = 0, + D40_DMA_RUN = 1, + D40_DMA_SUSPEND_REQ = 2, + D40_DMA_SUSPENDED = 3 +}; + +/** + * struct d40_lli_pool - Structure for keeping LLIs in memory + * + * @base: Pointer to memory area when the pre_alloc_lli's are not large + * enough, IE bigger than the most common case, 1 dst and 1 src. NULL if + * pre_alloc_lli is used. + * @size: The size in bytes of the memory at base or the size of pre_alloc_lli. + * @pre_alloc_lli: Pre allocated area for the most common case of transfers, + * one buffer to one buffer. + */ +struct d40_lli_pool { + void *base; + int size; + /* Space for dst and src, plus an extra for padding */ + u8 pre_alloc_lli[3 * sizeof(struct d40_phy_lli)]; +}; + +/** + * struct d40_desc - A descriptor is one DMA job. + * + * @lli_phy: LLI settings for physical channel. Both src and dst= + * points into the lli_pool, to base if lli_len > 1 or to pre_alloc_lli if + * lli_len equals one. + * @lli_log: Same as above but for logical channels. + * @lli_pool: The pool with two entries pre-allocated. + * @lli_len: Number of LLI's in lli_pool + * @lli_tcount: Number of LLIs processed in the transfer. When equals lli_len + * then this transfer job is done. + * @txd: DMA engine struct. Used for among other things for communication + * during a transfer. + * @node: List entry. + * @dir: The transfer direction of this job. + * @is_in_client_list: true if the client owns this descriptor. + * + * This descriptor is used for both logical and physical transfers. + */ + +struct d40_desc { + /* LLI physical */ + struct d40_phy_lli_bidir lli_phy; + /* LLI logical */ + struct d40_log_lli_bidir lli_log; + + struct d40_lli_pool lli_pool; + u32 lli_len; + u32 lli_tcount; + + struct dma_async_tx_descriptor txd; + struct list_head node; + + enum dma_data_direction dir; + bool is_in_client_list; +}; + +/** + * struct d40_lcla_pool - LCLA pool settings and data. + * + * @base: The virtual address of LCLA. + * @phy: Physical base address of LCLA. + * @base_size: size of lcla. + * @lock: Lock to protect the content in this struct. + * @alloc_map: Mapping between physical channel and LCLA entries. + * @num_blocks: The number of entries of alloc_map. Equals to the + * number of physical channels. + */ +struct d40_lcla_pool { + void *base; + dma_addr_t phy; + resource_size_t base_size; + spinlock_t lock; + u32 *alloc_map; + int num_blocks; +}; + +/** + * struct d40_phy_res - struct for handling eventlines mapped to physical + * channels. + * + * @lock: A lock protection this entity. + * @num: The physical channel number of this entity. + * @allocated_src: Bit mapped to show which src event line's are mapped to + * this physical channel. Can also be free or physically allocated. + * @allocated_dst: Same as for src but is dst. + * allocated_dst and allocated_src uses the D40_ALLOC* defines as well as + * event line number. Both allocated_src and allocated_dst can not be + * allocated to a physical channel, since the interrupt handler has then + * no way of figure out which one the interrupt belongs to. + */ +struct d40_phy_res { + spinlock_t lock; + int num; + u32 allocated_src; + u32 allocated_dst; +}; + +struct d40_base; + +/** + * struct d40_chan - Struct that describes a channel. + * + * @lock: A spinlock to protect this struct. + * @log_num: The logical number, if any of this channel. + * @completed: Starts with 1, after first interrupt it is set to dma engine's + * current cookie. + * @pending_tx: The number of pending transfers. Used between interrupt handler + * and tasklet. + * @busy: Set to true when transfer is ongoing on this channel. + * @phy_chan: Pointer to physical channel which this instance runs on. + * @chan: DMA engine handle. + * @tasklet: Tasklet that gets scheduled from interrupt context to complete a + * transfer and call client callback. + * @client: Cliented owned descriptor list. + * @active: Active descriptor. + * @queue: Queued jobs. + * @free: List of free descripts, ready to be reused. + * @free_len: Number of descriptors in the free list. + * @dma_cfg: The client configuration of this dma channel. + * @base: Pointer to the device instance struct. + * @src_def_cfg: Default cfg register setting for src. + * @dst_def_cfg: Default cfg register setting for dst. + * @log_def: Default logical channel settings. + * @lcla: Space for one dst src pair for logical channel transfers. + * @lcpa: Pointer to dst and src lcpa settings. + * + * This struct can either "be" a logical or a physical channel. + */ +struct d40_chan { + spinlock_t lock; + int log_num; + /* ID of the most recent completed transfer */ + int completed; + int pending_tx; + bool busy; + struct d40_phy_res *phy_chan; + struct dma_chan chan; + struct tasklet_struct tasklet; + struct list_head client; + struct list_head active; + struct list_head queue; + struct list_head free; + int free_len; + struct stedma40_chan_cfg dma_cfg; + struct d40_base *base; + /* Default register configurations */ + u32 src_def_cfg; + u32 dst_def_cfg; + struct d40_def_lcsp log_def; + struct d40_lcla_elem lcla; + struct d40_log_lli_full *lcpa; +}; + +/** + * struct d40_base - The big global struct, one for each probe'd instance. + * + * @interrupt_lock: Lock used to make sure one interrupt is handle a time. + * @execmd_lock: Lock for execute command usage since several channels share + * the same physical register. + * @dev: The device structure. + * @virtbase: The virtual base address of the DMA's register. + * @clk: Pointer to the DMA clock structure. + * @phy_start: Physical memory start of the DMA registers. + * @phy_size: Size of the DMA register map. + * @irq: The IRQ number. + * @num_phy_chans: The number of physical channels. Read from HW. This + * is the number of available channels for this driver, not counting "Secure + * mode" allocated physical channels. + * @num_log_chans: The number of logical channels. Calculated from + * num_phy_chans. + * @dma_both: dma_device channels that can do both memcpy and slave transfers. + * @dma_slave: dma_device channels that can do only do slave transfers. + * @dma_memcpy: dma_device channels that can do only do memcpy transfers. + * @phy_chans: Room for all possible physical channels in system. + * @log_chans: Room for all possible logical channels in system. + * @lookup_log_chans: Used to map interrupt number to logical channel. Points + * to log_chans entries. + * @lookup_phy_chans: Used to map interrupt number to physical channel. Points + * to phy_chans entries. + * @plat_data: Pointer to provided platform_data which is the driver + * configuration. + * @phy_res: Vector containing all physical channels. + * @lcla_pool: lcla pool settings and data. + * @lcpa_base: The virtual mapped address of LCPA. + * @phy_lcpa: The physical address of the LCPA. + * @lcpa_size: The size of the LCPA area. + */ +struct d40_base { + spinlock_t interrupt_lock; + spinlock_t execmd_lock; + struct device *dev; + void __iomem *virtbase; + struct clk *clk; + phys_addr_t phy_start; + resource_size_t phy_size; + int irq; + int num_phy_chans; + int num_log_chans; + struct dma_device dma_both; + struct dma_device dma_slave; + struct dma_device dma_memcpy; + struct d40_chan *phy_chans; + struct d40_chan *log_chans; + struct d40_chan **lookup_log_chans; + struct d40_chan **lookup_phy_chans; + struct stedma40_platform_data *plat_data; + /* Physical half channels */ + struct d40_phy_res *phy_res; + struct d40_lcla_pool lcla_pool; + void *lcpa_base; + dma_addr_t phy_lcpa; + resource_size_t lcpa_size; +}; + +/** + * struct d40_interrupt_lookup - lookup table for interrupt handler + * + * @src: Interrupt mask register. + * @clr: Interrupt clear register. + * @is_error: true if this is an error interrupt. + * @offset: start delta in the lookup_log_chans in d40_base. If equals to + * D40_PHY_CHAN, the lookup_phy_chans shall be used instead. + */ +struct d40_interrupt_lookup { + u32 src; + u32 clr; + bool is_error; + int offset; +}; + +/** + * struct d40_reg_val - simple lookup struct + * + * @reg: The register. + * @val: The value that belongs to the register in reg. + */ +struct d40_reg_val { + unsigned int reg; + unsigned int val; +}; + +static int d40_pool_lli_alloc(struct d40_desc *d40d, + int lli_len, bool is_log) +{ + u32 align; + void *base; + + if (is_log) + align = sizeof(struct d40_log_lli); + else + align = sizeof(struct d40_phy_lli); + + if (lli_len == 1) { + base = d40d->lli_pool.pre_alloc_lli; + d40d->lli_pool.size = sizeof(d40d->lli_pool.pre_alloc_lli); + d40d->lli_pool.base = NULL; + } else { + d40d->lli_pool.size = ALIGN(lli_len * 2 * align, align); + + base = kmalloc(d40d->lli_pool.size + align, GFP_NOWAIT); + d40d->lli_pool.base = base; + + if (d40d->lli_pool.base == NULL) + return -ENOMEM; + } + + if (is_log) { + d40d->lli_log.src = PTR_ALIGN((struct d40_log_lli *) base, + align); + d40d->lli_log.dst = PTR_ALIGN(d40d->lli_log.src + lli_len, + align); + } else { + d40d->lli_phy.src = PTR_ALIGN((struct d40_phy_lli *)base, + align); + d40d->lli_phy.dst = PTR_ALIGN(d40d->lli_phy.src + lli_len, + align); + + d40d->lli_phy.src_addr = virt_to_phys(d40d->lli_phy.src); + d40d->lli_phy.dst_addr = virt_to_phys(d40d->lli_phy.dst); + } + + return 0; +} + +static void d40_pool_lli_free(struct d40_desc *d40d) +{ + kfree(d40d->lli_pool.base); + d40d->lli_pool.base = NULL; + d40d->lli_pool.size = 0; + d40d->lli_log.src = NULL; + d40d->lli_log.dst = NULL; + d40d->lli_phy.src = NULL; + d40d->lli_phy.dst = NULL; + d40d->lli_phy.src_addr = 0; + d40d->lli_phy.dst_addr = 0; +} + +static dma_cookie_t d40_assign_cookie(struct d40_chan *d40c, + struct d40_desc *desc) +{ + dma_cookie_t cookie = d40c->chan.cookie; + + if (++cookie < 0) + cookie = 1; + + d40c->chan.cookie = cookie; + desc->txd.cookie = cookie; + + return cookie; +} + +static void d40_desc_reset(struct d40_desc *d40d) +{ + d40d->lli_tcount = 0; +} + +static void d40_desc_remove(struct d40_desc *d40d) +{ + list_del(&d40d->node); +} + +static struct d40_desc *d40_desc_get(struct d40_chan *d40c) +{ + struct d40_desc *desc; + struct d40_desc *d; + struct d40_desc *_d; + + if (!list_empty(&d40c->client)) { + list_for_each_entry_safe(d, _d, &d40c->client, node) + if (async_tx_test_ack(&d->txd)) { + d40_pool_lli_free(d); + d40_desc_remove(d); + desc = d; + goto out; + } + } + + if (list_empty(&d40c->free)) { + /* Alloc new desc because we're out of used ones */ + desc = kzalloc(sizeof(struct d40_desc), GFP_NOWAIT); + if (desc == NULL) + goto out; + INIT_LIST_HEAD(&desc->node); + } else { + /* Reuse an old desc. */ + desc = list_first_entry(&d40c->free, + struct d40_desc, + node); + list_del(&desc->node); + d40c->free_len--; + } +out: + return desc; +} + +static void d40_desc_free(struct d40_chan *d40c, struct d40_desc *d40d) +{ + if (d40c->free_len < D40_DESC_CACHE_SIZE) { + list_add_tail(&d40d->node, &d40c->free); + d40c->free_len++; + } else + kfree(d40d); +} + +static void d40_desc_submit(struct d40_chan *d40c, struct d40_desc *desc) +{ + list_add_tail(&desc->node, &d40c->active); +} + +static struct d40_desc *d40_first_active_get(struct d40_chan *d40c) +{ + struct d40_desc *d; + + if (list_empty(&d40c->active)) + return NULL; + + d = list_first_entry(&d40c->active, + struct d40_desc, + node); + return d; +} + +static void d40_desc_queue(struct d40_chan *d40c, struct d40_desc *desc) +{ + list_add_tail(&desc->node, &d40c->queue); +} + +static struct d40_desc *d40_first_queued(struct d40_chan *d40c) +{ + struct d40_desc *d; + + if (list_empty(&d40c->queue)) + return NULL; + + d = list_first_entry(&d40c->queue, + struct d40_desc, + node); + return d; +} + +/* Support functions for logical channels */ + +static int d40_lcla_id_get(struct d40_chan *d40c, + struct d40_lcla_pool *pool) +{ + int src_id = 0; + int dst_id = 0; + struct d40_log_lli *lcla_lidx_base = + pool->base + d40c->phy_chan->num * 1024; + int i; + int lli_per_log = d40c->base->plat_data->llis_per_log; + + if (d40c->lcla.src_id >= 0 && d40c->lcla.dst_id >= 0) + return 0; + + if (pool->num_blocks > 32) + return -EINVAL; + + spin_lock(&pool->lock); + + for (i = 0; i < pool->num_blocks; i++) { + if (!(pool->alloc_map[d40c->phy_chan->num] & (0x1 << i))) { + pool->alloc_map[d40c->phy_chan->num] |= (0x1 << i); + break; + } + } + src_id = i; + if (src_id >= pool->num_blocks) + goto err; + + for (; i < pool->num_blocks; i++) { + if (!(pool->alloc_map[d40c->phy_chan->num] & (0x1 << i))) { + pool->alloc_map[d40c->phy_chan->num] |= (0x1 << i); + break; + } + } + + dst_id = i; + if (dst_id == src_id) + goto err; + + d40c->lcla.src_id = src_id; + d40c->lcla.dst_id = dst_id; + d40c->lcla.dst = lcla_lidx_base + dst_id * lli_per_log + 1; + d40c->lcla.src = lcla_lidx_base + src_id * lli_per_log + 1; + + + spin_unlock(&pool->lock); + return 0; +err: + spin_unlock(&pool->lock); + return -EINVAL; +} + +static void d40_lcla_id_put(struct d40_chan *d40c, + struct d40_lcla_pool *pool, + int id) +{ + if (id < 0) + return; + + d40c->lcla.src_id = -1; + d40c->lcla.dst_id = -1; + + spin_lock(&pool->lock); + pool->alloc_map[d40c->phy_chan->num] &= (~(0x1 << id)); + spin_unlock(&pool->lock); +} + +static int d40_channel_execute_command(struct d40_chan *d40c, + enum d40_command command) +{ + int status, i; + void __iomem *active_reg; + int ret = 0; + unsigned long flags; + + spin_lock_irqsave(&d40c->base->execmd_lock, flags); + + if (d40c->phy_chan->num % 2 == 0) + active_reg = d40c->base->virtbase + D40_DREG_ACTIVE; + else + active_reg = d40c->base->virtbase + D40_DREG_ACTIVO; + + if (command == D40_DMA_SUSPEND_REQ) { + status = (readl(active_reg) & + D40_CHAN_POS_MASK(d40c->phy_chan->num)) >> + D40_CHAN_POS(d40c->phy_chan->num); + + if (status == D40_DMA_SUSPENDED || status == D40_DMA_STOP) + goto done; + } + + writel(command << D40_CHAN_POS(d40c->phy_chan->num), active_reg); + + if (command == D40_DMA_SUSPEND_REQ) { + + for (i = 0 ; i < D40_SUSPEND_MAX_IT; i++) { + status = (readl(active_reg) & + D40_CHAN_POS_MASK(d40c->phy_chan->num)) >> + D40_CHAN_POS(d40c->phy_chan->num); + + cpu_relax(); + /* + * Reduce the number of bus accesses while + * waiting for the DMA to suspend. + */ + udelay(3); + + if (status == D40_DMA_STOP || + status == D40_DMA_SUSPENDED) + break; + } + + if (i == D40_SUSPEND_MAX_IT) { + dev_err(&d40c->chan.dev->device, + "[%s]: unable to suspend the chl %d (log: %d) status %x\n", + __func__, d40c->phy_chan->num, d40c->log_num, + status); + dump_stack(); + ret = -EBUSY; + } + + } +done: + spin_unlock_irqrestore(&d40c->base->execmd_lock, flags); + return ret; +} + +static void d40_term_all(struct d40_chan *d40c) +{ + struct d40_desc *d40d; + struct d40_desc *d; + struct d40_desc *_d; + + /* Release active descriptors */ + while ((d40d = d40_first_active_get(d40c))) { + d40_desc_remove(d40d); + + /* Return desc to free-list */ + d40_desc_free(d40c, d40d); + } + + /* Release queued descriptors waiting for transfer */ + while ((d40d = d40_first_queued(d40c))) { + d40_desc_remove(d40d); + + /* Return desc to free-list */ + d40_desc_free(d40c, d40d); + } + + /* Release client owned descriptors */ + if (!list_empty(&d40c->client)) + list_for_each_entry_safe(d, _d, &d40c->client, node) { + d40_pool_lli_free(d); + d40_desc_remove(d); + /* Return desc to free-list */ + d40_desc_free(d40c, d40d); + } + + d40_lcla_id_put(d40c, &d40c->base->lcla_pool, + d40c->lcla.src_id); + d40_lcla_id_put(d40c, &d40c->base->lcla_pool, + d40c->lcla.dst_id); + + d40c->pending_tx = 0; + d40c->busy = false; +} + +static void d40_config_set_event(struct d40_chan *d40c, bool do_enable) +{ + u32 val; + unsigned long flags; + + if (do_enable) + val = D40_ACTIVATE_EVENTLINE; + else + val = D40_DEACTIVATE_EVENTLINE; + + spin_lock_irqsave(&d40c->phy_chan->lock, flags); + + /* Enable event line connected to device (or memcpy) */ + if ((d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_MEM) || + (d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_PERIPH)) { + u32 event = D40_TYPE_TO_EVENT(d40c->dma_cfg.src_dev_type); + + writel((val << D40_EVENTLINE_POS(event)) | + ~D40_EVENTLINE_MASK(event), + d40c->base->virtbase + D40_DREG_PCBASE + + d40c->phy_chan->num * D40_DREG_PCDELTA + + D40_CHAN_REG_SSLNK); + } + if (d40c->dma_cfg.dir != STEDMA40_PERIPH_TO_MEM) { + u32 event = D40_TYPE_TO_EVENT(d40c->dma_cfg.dst_dev_type); + + writel((val << D40_EVENTLINE_POS(event)) | + ~D40_EVENTLINE_MASK(event), + d40c->base->virtbase + D40_DREG_PCBASE + + d40c->phy_chan->num * D40_DREG_PCDELTA + + D40_CHAN_REG_SDLNK); + } + + spin_unlock_irqrestore(&d40c->phy_chan->lock, flags); +} + +static u32 d40_chan_has_events(struct d40_chan *d40c) +{ + u32 val = 0; + + /* If SSLNK or SDLNK is zero all events are disabled */ + if ((d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_MEM) || + (d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_PERIPH)) + val = readl(d40c->base->virtbase + D40_DREG_PCBASE + + d40c->phy_chan->num * D40_DREG_PCDELTA + + D40_CHAN_REG_SSLNK); + + if (d40c->dma_cfg.dir != STEDMA40_PERIPH_TO_MEM) + val = readl(d40c->base->virtbase + D40_DREG_PCBASE + + d40c->phy_chan->num * D40_DREG_PCDELTA + + D40_CHAN_REG_SDLNK); + return val; +} + +static void d40_config_enable_lidx(struct d40_chan *d40c) +{ + /* Set LIDX for lcla */ + writel((d40c->phy_chan->num << D40_SREG_ELEM_LOG_LIDX_POS) & + D40_SREG_ELEM_LOG_LIDX_MASK, + d40c->base->virtbase + D40_DREG_PCBASE + + d40c->phy_chan->num * D40_DREG_PCDELTA + D40_CHAN_REG_SDELT); + + writel((d40c->phy_chan->num << D40_SREG_ELEM_LOG_LIDX_POS) & + D40_SREG_ELEM_LOG_LIDX_MASK, + d40c->base->virtbase + D40_DREG_PCBASE + + d40c->phy_chan->num * D40_DREG_PCDELTA + D40_CHAN_REG_SSELT); +} + +static int d40_config_write(struct d40_chan *d40c) +{ + u32 addr_base; + u32 var; + int res; + + res = d40_channel_execute_command(d40c, D40_DMA_SUSPEND_REQ); + if (res) + return res; + + /* Odd addresses are even addresses + 4 */ + addr_base = (d40c->phy_chan->num % 2) * 4; + /* Setup channel mode to logical or physical */ + var = ((u32)(d40c->log_num != D40_PHY_CHAN) + 1) << + D40_CHAN_POS(d40c->phy_chan->num); + writel(var, d40c->base->virtbase + D40_DREG_PRMSE + addr_base); + + /* Setup operational mode option register */ + var = ((d40c->dma_cfg.channel_type >> STEDMA40_INFO_CH_MODE_OPT_POS) & + 0x3) << D40_CHAN_POS(d40c->phy_chan->num); + + writel(var, d40c->base->virtbase + D40_DREG_PRMOE + addr_base); + + if (d40c->log_num != D40_PHY_CHAN) { + /* Set default config for CFG reg */ + writel(d40c->src_def_cfg, + d40c->base->virtbase + D40_DREG_PCBASE + + d40c->phy_chan->num * D40_DREG_PCDELTA + + D40_CHAN_REG_SSCFG); + writel(d40c->dst_def_cfg, + d40c->base->virtbase + D40_DREG_PCBASE + + d40c->phy_chan->num * D40_DREG_PCDELTA + + D40_CHAN_REG_SDCFG); + + d40_config_enable_lidx(d40c); + } + return res; +} + +static void d40_desc_load(struct d40_chan *d40c, struct d40_desc *d40d) +{ + + if (d40d->lli_phy.dst && d40d->lli_phy.src) { + d40_phy_lli_write(d40c->base->virtbase, + d40c->phy_chan->num, + d40d->lli_phy.dst, + d40d->lli_phy.src); + d40d->lli_tcount = d40d->lli_len; + } else if (d40d->lli_log.dst && d40d->lli_log.src) { + u32 lli_len; + struct d40_log_lli *src = d40d->lli_log.src; + struct d40_log_lli *dst = d40d->lli_log.dst; + + src += d40d->lli_tcount; + dst += d40d->lli_tcount; + + if (d40d->lli_len <= d40c->base->plat_data->llis_per_log) + lli_len = d40d->lli_len; + else + lli_len = d40c->base->plat_data->llis_per_log; + d40d->lli_tcount += lli_len; + d40_log_lli_write(d40c->lcpa, d40c->lcla.src, + d40c->lcla.dst, + dst, src, + d40c->base->plat_data->llis_per_log); + } +} + +static dma_cookie_t d40_tx_submit(struct dma_async_tx_descriptor *tx) +{ + struct d40_chan *d40c = container_of(tx->chan, + struct d40_chan, + chan); + struct d40_desc *d40d = container_of(tx, struct d40_desc, txd); + unsigned long flags; + + spin_lock_irqsave(&d40c->lock, flags); + + tx->cookie = d40_assign_cookie(d40c, d40d); + + d40_desc_queue(d40c, d40d); + + spin_unlock_irqrestore(&d40c->lock, flags); + + return tx->cookie; +} + +static int d40_start(struct d40_chan *d40c) +{ + int err; + + if (d40c->log_num != D40_PHY_CHAN) { + err = d40_channel_execute_command(d40c, D40_DMA_SUSPEND_REQ); + if (err) + return err; + d40_config_set_event(d40c, true); + } + + err = d40_channel_execute_command(d40c, D40_DMA_RUN); + + return err; +} + +static struct d40_desc *d40_queue_start(struct d40_chan *d40c) +{ + struct d40_desc *d40d; + int err; + + /* Start queued jobs, if any */ + d40d = d40_first_queued(d40c); + + if (d40d != NULL) { + d40c->busy = true; + + /* Remove from queue */ + d40_desc_remove(d40d); + + /* Add to active queue */ + d40_desc_submit(d40c, d40d); + + /* Initiate DMA job */ + d40_desc_load(d40c, d40d); + + /* Start dma job */ + err = d40_start(d40c); + + if (err) + return NULL; + } + + return d40d; +} + +/* called from interrupt context */ +static void dma_tc_handle(struct d40_chan *d40c) +{ + struct d40_desc *d40d; + + if (!d40c->phy_chan) + return; + + /* Get first active entry from list */ + d40d = d40_first_active_get(d40c); + + if (d40d == NULL) + return; + + if (d40d->lli_tcount < d40d->lli_len) { + + d40_desc_load(d40c, d40d); + /* Start dma job */ + (void) d40_start(d40c); + return; + } + + if (d40_queue_start(d40c) == NULL) + d40c->busy = false; + + d40c->pending_tx++; + tasklet_schedule(&d40c->tasklet); + +} + +static void dma_tasklet(unsigned long data) +{ + struct d40_chan *d40c = (struct d40_chan *) data; + struct d40_desc *d40d_fin; + unsigned long flags; + dma_async_tx_callback callback; + void *callback_param; + + spin_lock_irqsave(&d40c->lock, flags); + + /* Get first active entry from list */ + d40d_fin = d40_first_active_get(d40c); + + if (d40d_fin == NULL) + goto err; + + d40c->completed = d40d_fin->txd.cookie; + + /* + * If terminating a channel pending_tx is set to zero. + * This prevents any finished active jobs to return to the client. + */ + if (d40c->pending_tx == 0) { + spin_unlock_irqrestore(&d40c->lock, flags); + return; + } + + /* Callback to client */ + callback = d40d_fin->txd.callback; + callback_param = d40d_fin->txd.callback_param; + + if (async_tx_test_ack(&d40d_fin->txd)) { + d40_pool_lli_free(d40d_fin); + d40_desc_remove(d40d_fin); + /* Return desc to free-list */ + d40_desc_free(d40c, d40d_fin); + } else { + d40_desc_reset(d40d_fin); + if (!d40d_fin->is_in_client_list) { + d40_desc_remove(d40d_fin); + list_add_tail(&d40d_fin->node, &d40c->client); + d40d_fin->is_in_client_list = true; + } + } + + d40c->pending_tx--; + + if (d40c->pending_tx) + tasklet_schedule(&d40c->tasklet); + + spin_unlock_irqrestore(&d40c->lock, flags); + + if (callback) + callback(callback_param); + + return; + + err: + /* Rescue manouver if receiving double interrupts */ + if (d40c->pending_tx > 0) + d40c->pending_tx--; + spin_unlock_irqrestore(&d40c->lock, flags); +} + +static irqreturn_t d40_handle_interrupt(int irq, void *data) +{ + static const struct d40_interrupt_lookup il[] = { + {D40_DREG_LCTIS0, D40_DREG_LCICR0, false, 0}, + {D40_DREG_LCTIS1, D40_DREG_LCICR1, false, 32}, + {D40_DREG_LCTIS2, D40_DREG_LCICR2, false, 64}, + {D40_DREG_LCTIS3, D40_DREG_LCICR3, false, 96}, + {D40_DREG_LCEIS0, D40_DREG_LCICR0, true, 0}, + {D40_DREG_LCEIS1, D40_DREG_LCICR1, true, 32}, + {D40_DREG_LCEIS2, D40_DREG_LCICR2, true, 64}, + {D40_DREG_LCEIS3, D40_DREG_LCICR3, true, 96}, + {D40_DREG_PCTIS, D40_DREG_PCICR, false, D40_PHY_CHAN}, + {D40_DREG_PCEIS, D40_DREG_PCICR, true, D40_PHY_CHAN}, + }; + + int i; + u32 regs[ARRAY_SIZE(il)]; + u32 tmp; + u32 idx; + u32 row; + long chan = -1; + struct d40_chan *d40c; + unsigned long flags; + struct d40_base *base = data; + + spin_lock_irqsave(&base->interrupt_lock, flags); + + /* Read interrupt status of both logical and physical channels */ + for (i = 0; i < ARRAY_SIZE(il); i++) + regs[i] = readl(base->virtbase + il[i].src); + + for (;;) { + + chan = find_next_bit((unsigned long *)regs, + BITS_PER_LONG * ARRAY_SIZE(il), chan + 1); + + /* No more set bits found? */ + if (chan == BITS_PER_LONG * ARRAY_SIZE(il)) + break; + + row = chan / BITS_PER_LONG; + idx = chan & (BITS_PER_LONG - 1); + + /* ACK interrupt */ + tmp = readl(base->virtbase + il[row].clr); + tmp |= 1 << idx; + writel(tmp, base->virtbase + il[row].clr); + + if (il[row].offset == D40_PHY_CHAN) + d40c = base->lookup_phy_chans[idx]; + else + d40c = base->lookup_log_chans[il[row].offset + idx]; + spin_lock(&d40c->lock); + + if (!il[row].is_error) + dma_tc_handle(d40c); + else + dev_err(base->dev, "[%s] IRQ chan: %ld offset %d idx %d\n", + __func__, chan, il[row].offset, idx); + + spin_unlock(&d40c->lock); + } + + spin_unlock_irqrestore(&base->interrupt_lock, flags); + + return IRQ_HANDLED; +} + + +static int d40_validate_conf(struct d40_chan *d40c, + struct stedma40_chan_cfg *conf) +{ + int res = 0; + u32 dst_event_group = D40_TYPE_TO_GROUP(conf->dst_dev_type); + u32 src_event_group = D40_TYPE_TO_GROUP(conf->src_dev_type); + bool is_log = (conf->channel_type & STEDMA40_CHANNEL_IN_OPER_MODE) + == STEDMA40_CHANNEL_IN_LOG_MODE; + + if (d40c->dma_cfg.dir == STEDMA40_MEM_TO_PERIPH && + dst_event_group == STEDMA40_DEV_DST_MEMORY) { + dev_err(&d40c->chan.dev->device, "[%s] Invalid dst\n", + __func__); + res = -EINVAL; + } + + if (d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_MEM && + src_event_group == STEDMA40_DEV_SRC_MEMORY) { + dev_err(&d40c->chan.dev->device, "[%s] Invalid src\n", + __func__); + res = -EINVAL; + } + + if (src_event_group == STEDMA40_DEV_SRC_MEMORY && + dst_event_group == STEDMA40_DEV_DST_MEMORY && is_log) { + dev_err(&d40c->chan.dev->device, + "[%s] No event line\n", __func__); + res = -EINVAL; + } + + if (conf->dir == STEDMA40_PERIPH_TO_PERIPH && + (src_event_group != dst_event_group)) { + dev_err(&d40c->chan.dev->device, + "[%s] Invalid event group\n", __func__); + res = -EINVAL; + } + + if (conf->dir == STEDMA40_PERIPH_TO_PERIPH) { + /* + * DMAC HW supports it. Will be added to this driver, + * in case any dma client requires it. + */ + dev_err(&d40c->chan.dev->device, + "[%s] periph to periph not supported\n", + __func__); + res = -EINVAL; + } + + return res; +} + +static bool d40_alloc_mask_set(struct d40_phy_res *phy, bool is_src, + int log_event_line, bool is_log) +{ + unsigned long flags; + spin_lock_irqsave(&phy->lock, flags); + if (!is_log) { + /* Physical interrupts are masked per physical full channel */ + if (phy->allocated_src == D40_ALLOC_FREE && + phy->allocated_dst == D40_ALLOC_FREE) { + phy->allocated_dst = D40_ALLOC_PHY; + phy->allocated_src = D40_ALLOC_PHY; + goto found; + } else + goto not_found; + } + + /* Logical channel */ + if (is_src) { + if (phy->allocated_src == D40_ALLOC_PHY) + goto not_found; + + if (phy->allocated_src == D40_ALLOC_FREE) + phy->allocated_src = D40_ALLOC_LOG_FREE; + + if (!(phy->allocated_src & (1 << log_event_line))) { + phy->allocated_src |= 1 << log_event_line; + goto found; + } else + goto not_found; + } else { + if (phy->allocated_dst == D40_ALLOC_PHY) + goto not_found; + + if (phy->allocated_dst == D40_ALLOC_FREE) + phy->allocated_dst = D40_ALLOC_LOG_FREE; + + if (!(phy->allocated_dst & (1 << log_event_line))) { + phy->allocated_dst |= 1 << log_event_line; + goto found; + } else + goto not_found; + } + +not_found: + spin_unlock_irqrestore(&phy->lock, flags); + return false; +found: + spin_unlock_irqrestore(&phy->lock, flags); + return true; +} + +static bool d40_alloc_mask_free(struct d40_phy_res *phy, bool is_src, + int log_event_line) +{ + unsigned long flags; + bool is_free = false; + + spin_lock_irqsave(&phy->lock, flags); + if (!log_event_line) { + /* Physical interrupts are masked per physical full channel */ + phy->allocated_dst = D40_ALLOC_FREE; + phy->allocated_src = D40_ALLOC_FREE; + is_free = true; + goto out; + } + + /* Logical channel */ + if (is_src) { + phy->allocated_src &= ~(1 << log_event_line); + if (phy->allocated_src == D40_ALLOC_LOG_FREE) + phy->allocated_src = D40_ALLOC_FREE; + } else { + phy->allocated_dst &= ~(1 << log_event_line); + if (phy->allocated_dst == D40_ALLOC_LOG_FREE) + phy->allocated_dst = D40_ALLOC_FREE; + } + + is_free = ((phy->allocated_src | phy->allocated_dst) == + D40_ALLOC_FREE); + +out: + spin_unlock_irqrestore(&phy->lock, flags); + + return is_free; +} + +static int d40_allocate_channel(struct d40_chan *d40c) +{ + int dev_type; + int event_group; + int event_line; + struct d40_phy_res *phys; + int i; + int j; + int log_num; + bool is_src; + bool is_log = (d40c->dma_cfg.channel_type & STEDMA40_CHANNEL_IN_OPER_MODE) + == STEDMA40_CHANNEL_IN_LOG_MODE; + + + phys = d40c->base->phy_res; + + if (d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_MEM) { + dev_type = d40c->dma_cfg.src_dev_type; + log_num = 2 * dev_type; + is_src = true; + } else if (d40c->dma_cfg.dir == STEDMA40_MEM_TO_PERIPH || + d40c->dma_cfg.dir == STEDMA40_MEM_TO_MEM) { + /* dst event lines are used for logical memcpy */ + dev_type = d40c->dma_cfg.dst_dev_type; + log_num = 2 * dev_type + 1; + is_src = false; + } else + return -EINVAL; + + event_group = D40_TYPE_TO_GROUP(dev_type); + event_line = D40_TYPE_TO_EVENT(dev_type); + + if (!is_log) { + if (d40c->dma_cfg.dir == STEDMA40_MEM_TO_MEM) { + /* Find physical half channel */ + for (i = 0; i < d40c->base->num_phy_chans; i++) { + + if (d40_alloc_mask_set(&phys[i], is_src, + 0, is_log)) + goto found_phy; + } + } else + for (j = 0; j < d40c->base->num_phy_chans; j += 8) { + int phy_num = j + event_group * 2; + for (i = phy_num; i < phy_num + 2; i++) { + if (d40_alloc_mask_set(&phys[i], is_src, + 0, is_log)) + goto found_phy; + } + } + return -EINVAL; +found_phy: + d40c->phy_chan = &phys[i]; + d40c->log_num = D40_PHY_CHAN; + goto out; + } + if (dev_type == -1) + return -EINVAL; + + /* Find logical channel */ + for (j = 0; j < d40c->base->num_phy_chans; j += 8) { + int phy_num = j + event_group * 2; + /* + * Spread logical channels across all available physical rather + * than pack every logical channel at the first available phy + * channels. + */ + if (is_src) { + for (i = phy_num; i < phy_num + 2; i++) { + if (d40_alloc_mask_set(&phys[i], is_src, + event_line, is_log)) + goto found_log; + } + } else { + for (i = phy_num + 1; i >= phy_num; i--) { + if (d40_alloc_mask_set(&phys[i], is_src, + event_line, is_log)) + goto found_log; + } + } + } + return -EINVAL; + +found_log: + d40c->phy_chan = &phys[i]; + d40c->log_num = log_num; +out: + + if (is_log) + d40c->base->lookup_log_chans[d40c->log_num] = d40c; + else + d40c->base->lookup_phy_chans[d40c->phy_chan->num] = d40c; + + return 0; + +} + +static int d40_config_chan(struct d40_chan *d40c, + struct stedma40_chan_cfg *info) +{ + + /* Fill in basic CFG register values */ + d40_phy_cfg(&d40c->dma_cfg, &d40c->src_def_cfg, + &d40c->dst_def_cfg, d40c->log_num != D40_PHY_CHAN); + + if (d40c->log_num != D40_PHY_CHAN) { + d40_log_cfg(&d40c->dma_cfg, + &d40c->log_def.lcsp1, &d40c->log_def.lcsp3); + + if (d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_MEM) + d40c->lcpa = d40c->base->lcpa_base + + d40c->dma_cfg.src_dev_type * 32; + else + d40c->lcpa = d40c->base->lcpa_base + + d40c->dma_cfg.dst_dev_type * 32 + 16; + } + + /* Write channel configuration to the DMA */ + return d40_config_write(d40c); +} + +static int d40_config_memcpy(struct d40_chan *d40c) +{ + dma_cap_mask_t cap = d40c->chan.device->cap_mask; + + if (dma_has_cap(DMA_MEMCPY, cap) && !dma_has_cap(DMA_SLAVE, cap)) { + d40c->dma_cfg = *d40c->base->plat_data->memcpy_conf_log; + d40c->dma_cfg.src_dev_type = STEDMA40_DEV_SRC_MEMORY; + d40c->dma_cfg.dst_dev_type = d40c->base->plat_data-> + memcpy[d40c->chan.chan_id]; + + } else if (dma_has_cap(DMA_MEMCPY, cap) && + dma_has_cap(DMA_SLAVE, cap)) { + d40c->dma_cfg = *d40c->base->plat_data->memcpy_conf_phy; + } else { + dev_err(&d40c->chan.dev->device, "[%s] No memcpy\n", + __func__); + return -EINVAL; + } + + return 0; +} + + +static int d40_free_dma(struct d40_chan *d40c) +{ + + int res = 0; + u32 event, dir; + struct d40_phy_res *phy = d40c->phy_chan; + bool is_src; + + /* Terminate all queued and active transfers */ + d40_term_all(d40c); + + if (phy == NULL) { + dev_err(&d40c->chan.dev->device, "[%s] phy == null\n", + __func__); + return -EINVAL; + } + + if (phy->allocated_src == D40_ALLOC_FREE && + phy->allocated_dst == D40_ALLOC_FREE) { + dev_err(&d40c->chan.dev->device, "[%s] channel already free\n", + __func__); + return -EINVAL; + } + + + res = d40_channel_execute_command(d40c, D40_DMA_SUSPEND_REQ); + if (res) { + dev_err(&d40c->chan.dev->device, "[%s] suspend\n", + __func__); + return res; + } + + if (d40c->dma_cfg.dir == STEDMA40_MEM_TO_PERIPH || + d40c->dma_cfg.dir == STEDMA40_MEM_TO_MEM) { + event = D40_TYPE_TO_EVENT(d40c->dma_cfg.dst_dev_type); + dir = D40_CHAN_REG_SDLNK; + is_src = false; + } else if (d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_MEM) { + event = D40_TYPE_TO_EVENT(d40c->dma_cfg.src_dev_type); + dir = D40_CHAN_REG_SSLNK; + is_src = true; + } else { + dev_err(&d40c->chan.dev->device, + "[%s] Unknown direction\n", __func__); + return -EINVAL; + } + + if (d40c->log_num != D40_PHY_CHAN) { + /* + * Release logical channel, deactivate the event line during + * the time physical res is suspended. + */ + writel((D40_DEACTIVATE_EVENTLINE << D40_EVENTLINE_POS(event)) & + D40_EVENTLINE_MASK(event), + d40c->base->virtbase + D40_DREG_PCBASE + + phy->num * D40_DREG_PCDELTA + dir); + + d40c->base->lookup_log_chans[d40c->log_num] = NULL; + + /* + * Check if there are more logical allocation + * on this phy channel. + */ + if (!d40_alloc_mask_free(phy, is_src, event)) { + /* Resume the other logical channels if any */ + if (d40_chan_has_events(d40c)) { + res = d40_channel_execute_command(d40c, + D40_DMA_RUN); + if (res) { + dev_err(&d40c->chan.dev->device, + "[%s] Executing RUN command\n", + __func__); + return res; + } + } + return 0; + } + } else + d40_alloc_mask_free(phy, is_src, 0); + + /* Release physical channel */ + res = d40_channel_execute_command(d40c, D40_DMA_STOP); + if (res) { + dev_err(&d40c->chan.dev->device, + "[%s] Failed to stop channel\n", __func__); + return res; + } + d40c->phy_chan = NULL; + /* Invalidate channel type */ + d40c->dma_cfg.channel_type = 0; + d40c->base->lookup_phy_chans[phy->num] = NULL; + + return 0; + + +} + +static int d40_pause(struct dma_chan *chan) +{ + struct d40_chan *d40c = + container_of(chan, struct d40_chan, chan); + int res; + + unsigned long flags; + + spin_lock_irqsave(&d40c->lock, flags); + + res = d40_channel_execute_command(d40c, D40_DMA_SUSPEND_REQ); + if (res == 0) { + if (d40c->log_num != D40_PHY_CHAN) { + d40_config_set_event(d40c, false); + /* Resume the other logical channels if any */ + if (d40_chan_has_events(d40c)) + res = d40_channel_execute_command(d40c, + D40_DMA_RUN); + } + } + + spin_unlock_irqrestore(&d40c->lock, flags); + return res; +} + +static bool d40_is_paused(struct d40_chan *d40c) +{ + bool is_paused = false; + unsigned long flags; + void __iomem *active_reg; + u32 status; + u32 event; + int res; + + spin_lock_irqsave(&d40c->lock, flags); + + if (d40c->log_num == D40_PHY_CHAN) { + if (d40c->phy_chan->num % 2 == 0) + active_reg = d40c->base->virtbase + D40_DREG_ACTIVE; + else + active_reg = d40c->base->virtbase + D40_DREG_ACTIVO; + + status = (readl(active_reg) & + D40_CHAN_POS_MASK(d40c->phy_chan->num)) >> + D40_CHAN_POS(d40c->phy_chan->num); + if (status == D40_DMA_SUSPENDED || status == D40_DMA_STOP) + is_paused = true; + + goto _exit; + } + + res = d40_channel_execute_command(d40c, D40_DMA_SUSPEND_REQ); + if (res != 0) + goto _exit; + + if (d40c->dma_cfg.dir == STEDMA40_MEM_TO_PERIPH || + d40c->dma_cfg.dir == STEDMA40_MEM_TO_MEM) + event = D40_TYPE_TO_EVENT(d40c->dma_cfg.dst_dev_type); + else if (d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_MEM) + event = D40_TYPE_TO_EVENT(d40c->dma_cfg.src_dev_type); + else { + dev_err(&d40c->chan.dev->device, + "[%s] Unknown direction\n", __func__); + goto _exit; + } + status = d40_chan_has_events(d40c); + status = (status & D40_EVENTLINE_MASK(event)) >> + D40_EVENTLINE_POS(event); + + if (status != D40_DMA_RUN) + is_paused = true; + + /* Resume the other logical channels if any */ + if (d40_chan_has_events(d40c)) + res = d40_channel_execute_command(d40c, + D40_DMA_RUN); + +_exit: + spin_unlock_irqrestore(&d40c->lock, flags); + return is_paused; + +} + + +static bool d40_tx_is_linked(struct d40_chan *d40c) +{ + bool is_link; + + if (d40c->log_num != D40_PHY_CHAN) + is_link = readl(&d40c->lcpa->lcsp3) & D40_MEM_LCSP3_DLOS_MASK; + else + is_link = readl(d40c->base->virtbase + D40_DREG_PCBASE + + d40c->phy_chan->num * D40_DREG_PCDELTA + + D40_CHAN_REG_SDLNK) & + D40_SREG_LNK_PHYS_LNK_MASK; + return is_link; +} + +static u32 d40_residue(struct d40_chan *d40c) +{ + u32 num_elt; + + if (d40c->log_num != D40_PHY_CHAN) + num_elt = (readl(&d40c->lcpa->lcsp2) & D40_MEM_LCSP2_ECNT_MASK) + >> D40_MEM_LCSP2_ECNT_POS; + else + num_elt = (readl(d40c->base->virtbase + D40_DREG_PCBASE + + d40c->phy_chan->num * D40_DREG_PCDELTA + + D40_CHAN_REG_SDELT) & + D40_SREG_ELEM_PHY_ECNT_MASK) >> D40_SREG_ELEM_PHY_ECNT_POS; + return num_elt * (1 << d40c->dma_cfg.dst_info.data_width); +} + +static int d40_resume(struct dma_chan *chan) +{ + struct d40_chan *d40c = + container_of(chan, struct d40_chan, chan); + int res = 0; + unsigned long flags; + + spin_lock_irqsave(&d40c->lock, flags); + + if (d40c->log_num != D40_PHY_CHAN) { + res = d40_channel_execute_command(d40c, D40_DMA_SUSPEND_REQ); + if (res) + goto out; + + /* If bytes left to transfer or linked tx resume job */ + if (d40_residue(d40c) || d40_tx_is_linked(d40c)) { + d40_config_set_event(d40c, true); + res = d40_channel_execute_command(d40c, D40_DMA_RUN); + } + } else if (d40_residue(d40c) || d40_tx_is_linked(d40c)) + res = d40_channel_execute_command(d40c, D40_DMA_RUN); + +out: + spin_unlock_irqrestore(&d40c->lock, flags); + return res; +} + +static u32 stedma40_residue(struct dma_chan *chan) +{ + struct d40_chan *d40c = + container_of(chan, struct d40_chan, chan); + u32 bytes_left; + unsigned long flags; + + spin_lock_irqsave(&d40c->lock, flags); + bytes_left = d40_residue(d40c); + spin_unlock_irqrestore(&d40c->lock, flags); + + return bytes_left; +} + +/* Public DMA functions in addition to the DMA engine framework */ + +int stedma40_set_psize(struct dma_chan *chan, + int src_psize, + int dst_psize) +{ + struct d40_chan *d40c = + container_of(chan, struct d40_chan, chan); + unsigned long flags; + + spin_lock_irqsave(&d40c->lock, flags); + + if (d40c->log_num != D40_PHY_CHAN) { + d40c->log_def.lcsp1 &= ~D40_MEM_LCSP1_SCFG_PSIZE_MASK; + d40c->log_def.lcsp3 &= ~D40_MEM_LCSP1_SCFG_PSIZE_MASK; + d40c->log_def.lcsp1 |= src_psize << D40_MEM_LCSP1_SCFG_PSIZE_POS; + d40c->log_def.lcsp3 |= dst_psize << D40_MEM_LCSP1_SCFG_PSIZE_POS; + goto out; + } + + if (src_psize == STEDMA40_PSIZE_PHY_1) + d40c->src_def_cfg &= ~(1 << D40_SREG_CFG_PHY_PEN_POS); + else { + d40c->src_def_cfg |= 1 << D40_SREG_CFG_PHY_PEN_POS; + d40c->src_def_cfg &= ~(STEDMA40_PSIZE_PHY_16 << + D40_SREG_CFG_PSIZE_POS); + d40c->src_def_cfg |= src_psize << D40_SREG_CFG_PSIZE_POS; + } + + if (dst_psize == STEDMA40_PSIZE_PHY_1) + d40c->dst_def_cfg &= ~(1 << D40_SREG_CFG_PHY_PEN_POS); + else { + d40c->dst_def_cfg |= 1 << D40_SREG_CFG_PHY_PEN_POS; + d40c->dst_def_cfg &= ~(STEDMA40_PSIZE_PHY_16 << + D40_SREG_CFG_PSIZE_POS); + d40c->dst_def_cfg |= dst_psize << D40_SREG_CFG_PSIZE_POS; + } +out: + spin_unlock_irqrestore(&d40c->lock, flags); + return 0; +} +EXPORT_SYMBOL(stedma40_set_psize); + +struct dma_async_tx_descriptor *stedma40_memcpy_sg(struct dma_chan *chan, + struct scatterlist *sgl_dst, + struct scatterlist *sgl_src, + unsigned int sgl_len, + unsigned long flags) +{ + int res; + struct d40_desc *d40d; + struct d40_chan *d40c = container_of(chan, struct d40_chan, + chan); + unsigned long flg; + int lli_max = d40c->base->plat_data->llis_per_log; + + + spin_lock_irqsave(&d40c->lock, flg); + d40d = d40_desc_get(d40c); + + if (d40d == NULL) + goto err; + + memset(d40d, 0, sizeof(struct d40_desc)); + d40d->lli_len = sgl_len; + + d40d->txd.flags = flags; + + if (d40c->log_num != D40_PHY_CHAN) { + if (sgl_len > 1) + /* + * Check if there is space available in lcla. If not, + * split list into 1-length and run only in lcpa + * space. + */ + if (d40_lcla_id_get(d40c, + &d40c->base->lcla_pool) != 0) + lli_max = 1; + + if (d40_pool_lli_alloc(d40d, sgl_len, true) < 0) { + dev_err(&d40c->chan.dev->device, + "[%s] Out of memory\n", __func__); + goto err; + } + + (void) d40_log_sg_to_lli(d40c->lcla.src_id, + sgl_src, + sgl_len, + d40d->lli_log.src, + d40c->log_def.lcsp1, + d40c->dma_cfg.src_info.data_width, + flags & DMA_PREP_INTERRUPT, lli_max, + d40c->base->plat_data->llis_per_log); + + (void) d40_log_sg_to_lli(d40c->lcla.dst_id, + sgl_dst, + sgl_len, + d40d->lli_log.dst, + d40c->log_def.lcsp3, + d40c->dma_cfg.dst_info.data_width, + flags & DMA_PREP_INTERRUPT, lli_max, + d40c->base->plat_data->llis_per_log); + + + } else { + if (d40_pool_lli_alloc(d40d, sgl_len, false) < 0) { + dev_err(&d40c->chan.dev->device, + "[%s] Out of memory\n", __func__); + goto err; + } + + res = d40_phy_sg_to_lli(sgl_src, + sgl_len, + 0, + d40d->lli_phy.src, + d40d->lli_phy.src_addr, + d40c->src_def_cfg, + d40c->dma_cfg.src_info.data_width, + d40c->dma_cfg.src_info.psize, + true); + + if (res < 0) + goto err; + + res = d40_phy_sg_to_lli(sgl_dst, + sgl_len, + 0, + d40d->lli_phy.dst, + d40d->lli_phy.dst_addr, + d40c->dst_def_cfg, + d40c->dma_cfg.dst_info.data_width, + d40c->dma_cfg.dst_info.psize, + true); + + if (res < 0) + goto err; + + (void) dma_map_single(d40c->base->dev, d40d->lli_phy.src, + d40d->lli_pool.size, DMA_TO_DEVICE); + } + + dma_async_tx_descriptor_init(&d40d->txd, chan); + + d40d->txd.tx_submit = d40_tx_submit; + + spin_unlock_irqrestore(&d40c->lock, flg); + + return &d40d->txd; +err: + spin_unlock_irqrestore(&d40c->lock, flg); + return NULL; +} +EXPORT_SYMBOL(stedma40_memcpy_sg); + +bool stedma40_filter(struct dma_chan *chan, void *data) +{ + struct stedma40_chan_cfg *info = data; + struct d40_chan *d40c = + container_of(chan, struct d40_chan, chan); + int err; + + if (data) { + err = d40_validate_conf(d40c, info); + if (!err) + d40c->dma_cfg = *info; + } else + err = d40_config_memcpy(d40c); + + return err == 0; +} +EXPORT_SYMBOL(stedma40_filter); + +/* DMA ENGINE functions */ +static int d40_alloc_chan_resources(struct dma_chan *chan) +{ + int err; + unsigned long flags; + struct d40_chan *d40c = + container_of(chan, struct d40_chan, chan); + + spin_lock_irqsave(&d40c->lock, flags); + + d40c->completed = chan->cookie = 1; + + /* + * If no dma configuration is set (channel_type == 0) + * use default configuration + */ + if (d40c->dma_cfg.channel_type == 0) { + err = d40_config_memcpy(d40c); + if (err) + goto err_alloc; + } + + err = d40_allocate_channel(d40c); + if (err) { + dev_err(&d40c->chan.dev->device, + "[%s] Failed to allocate channel\n", __func__); + goto err_alloc; + } + + err = d40_config_chan(d40c, &d40c->dma_cfg); + if (err) { + dev_err(&d40c->chan.dev->device, + "[%s] Failed to configure channel\n", + __func__); + goto err_config; + } + + spin_unlock_irqrestore(&d40c->lock, flags); + return 0; + + err_config: + (void) d40_free_dma(d40c); + err_alloc: + spin_unlock_irqrestore(&d40c->lock, flags); + dev_err(&d40c->chan.dev->device, + "[%s] Channel allocation failed\n", __func__); + return -EINVAL; +} + +static void d40_free_chan_resources(struct dma_chan *chan) +{ + struct d40_chan *d40c = + container_of(chan, struct d40_chan, chan); + int err; + unsigned long flags; + + spin_lock_irqsave(&d40c->lock, flags); + + err = d40_free_dma(d40c); + + if (err) + dev_err(&d40c->chan.dev->device, + "[%s] Failed to free channel\n", __func__); + spin_unlock_irqrestore(&d40c->lock, flags); +} + +static struct dma_async_tx_descriptor *d40_prep_memcpy(struct dma_chan *chan, + dma_addr_t dst, + dma_addr_t src, + size_t size, + unsigned long flags) +{ + struct d40_desc *d40d; + struct d40_chan *d40c = container_of(chan, struct d40_chan, + chan); + unsigned long flg; + int err = 0; + + spin_lock_irqsave(&d40c->lock, flg); + d40d = d40_desc_get(d40c); + + if (d40d == NULL) { + dev_err(&d40c->chan.dev->device, + "[%s] Descriptor is NULL\n", __func__); + goto err; + } + + memset(d40d, 0, sizeof(struct d40_desc)); + + d40d->txd.flags = flags; + + dma_async_tx_descriptor_init(&d40d->txd, chan); + + d40d->txd.tx_submit = d40_tx_submit; + + if (d40c->log_num != D40_PHY_CHAN) { + + if (d40_pool_lli_alloc(d40d, 1, true) < 0) { + dev_err(&d40c->chan.dev->device, + "[%s] Out of memory\n", __func__); + goto err; + } + d40d->lli_len = 1; + + d40_log_fill_lli(d40d->lli_log.src, + src, + size, + 0, + d40c->log_def.lcsp1, + d40c->dma_cfg.src_info.data_width, + true, true); + + d40_log_fill_lli(d40d->lli_log.dst, + dst, + size, + 0, + d40c->log_def.lcsp3, + d40c->dma_cfg.dst_info.data_width, + true, true); + + } else { + + if (d40_pool_lli_alloc(d40d, 1, false) < 0) { + dev_err(&d40c->chan.dev->device, + "[%s] Out of memory\n", __func__); + goto err; + } + + err = d40_phy_fill_lli(d40d->lli_phy.src, + src, + size, + d40c->dma_cfg.src_info.psize, + 0, + d40c->src_def_cfg, + true, + d40c->dma_cfg.src_info.data_width, + false); + if (err) + goto err_fill_lli; + + err = d40_phy_fill_lli(d40d->lli_phy.dst, + dst, + size, + d40c->dma_cfg.dst_info.psize, + 0, + d40c->dst_def_cfg, + true, + d40c->dma_cfg.dst_info.data_width, + false); + + if (err) + goto err_fill_lli; + + (void) dma_map_single(d40c->base->dev, d40d->lli_phy.src, + d40d->lli_pool.size, DMA_TO_DEVICE); + } + + spin_unlock_irqrestore(&d40c->lock, flg); + return &d40d->txd; + +err_fill_lli: + dev_err(&d40c->chan.dev->device, + "[%s] Failed filling in PHY LLI\n", __func__); + d40_pool_lli_free(d40d); +err: + spin_unlock_irqrestore(&d40c->lock, flg); + return NULL; +} + +static int d40_prep_slave_sg_log(struct d40_desc *d40d, + struct d40_chan *d40c, + struct scatterlist *sgl, + unsigned int sg_len, + enum dma_data_direction direction, + unsigned long flags) +{ + dma_addr_t dev_addr = 0; + int total_size; + int lli_max = d40c->base->plat_data->llis_per_log; + + if (d40_pool_lli_alloc(d40d, sg_len, true) < 0) { + dev_err(&d40c->chan.dev->device, + "[%s] Out of memory\n", __func__); + return -ENOMEM; + } + + d40d->lli_len = sg_len; + d40d->lli_tcount = 0; + + if (sg_len > 1) + /* + * Check if there is space available in lcla. + * If not, split list into 1-length and run only + * in lcpa space. + */ + if (d40_lcla_id_get(d40c, &d40c->base->lcla_pool) != 0) + lli_max = 1; + + if (direction == DMA_FROM_DEVICE) { + dev_addr = d40c->base->plat_data->dev_rx[d40c->dma_cfg.src_dev_type]; + total_size = d40_log_sg_to_dev(&d40c->lcla, + sgl, sg_len, + &d40d->lli_log, + &d40c->log_def, + d40c->dma_cfg.src_info.data_width, + d40c->dma_cfg.dst_info.data_width, + direction, + flags & DMA_PREP_INTERRUPT, + dev_addr, lli_max, + d40c->base->plat_data->llis_per_log); + } else if (direction == DMA_TO_DEVICE) { + dev_addr = d40c->base->plat_data->dev_tx[d40c->dma_cfg.dst_dev_type]; + total_size = d40_log_sg_to_dev(&d40c->lcla, + sgl, sg_len, + &d40d->lli_log, + &d40c->log_def, + d40c->dma_cfg.src_info.data_width, + d40c->dma_cfg.dst_info.data_width, + direction, + flags & DMA_PREP_INTERRUPT, + dev_addr, lli_max, + d40c->base->plat_data->llis_per_log); + } else + return -EINVAL; + if (total_size < 0) + return -EINVAL; + + return 0; +} + +static int d40_prep_slave_sg_phy(struct d40_desc *d40d, + struct d40_chan *d40c, + struct scatterlist *sgl, + unsigned int sgl_len, + enum dma_data_direction direction, + unsigned long flags) +{ + dma_addr_t src_dev_addr; + dma_addr_t dst_dev_addr; + int res; + + if (d40_pool_lli_alloc(d40d, sgl_len, false) < 0) { + dev_err(&d40c->chan.dev->device, + "[%s] Out of memory\n", __func__); + return -ENOMEM; + } + + d40d->lli_len = sgl_len; + d40d->lli_tcount = 0; + + if (direction == DMA_FROM_DEVICE) { + dst_dev_addr = 0; + src_dev_addr = d40c->base->plat_data->dev_rx[d40c->dma_cfg.src_dev_type]; + } else if (direction == DMA_TO_DEVICE) { + dst_dev_addr = d40c->base->plat_data->dev_tx[d40c->dma_cfg.dst_dev_type]; + src_dev_addr = 0; + } else + return -EINVAL; + + res = d40_phy_sg_to_lli(sgl, + sgl_len, + src_dev_addr, + d40d->lli_phy.src, + d40d->lli_phy.src_addr, + d40c->src_def_cfg, + d40c->dma_cfg.src_info.data_width, + d40c->dma_cfg.src_info.psize, + true); + if (res < 0) + return res; + + res = d40_phy_sg_to_lli(sgl, + sgl_len, + dst_dev_addr, + d40d->lli_phy.dst, + d40d->lli_phy.dst_addr, + d40c->dst_def_cfg, + d40c->dma_cfg.dst_info.data_width, + d40c->dma_cfg.dst_info.psize, + true); + if (res < 0) + return res; + + (void) dma_map_single(d40c->base->dev, d40d->lli_phy.src, + d40d->lli_pool.size, DMA_TO_DEVICE); + return 0; +} + +static struct dma_async_tx_descriptor *d40_prep_slave_sg(struct dma_chan *chan, + struct scatterlist *sgl, + unsigned int sg_len, + enum dma_data_direction direction, + unsigned long flags) +{ + struct d40_desc *d40d; + struct d40_chan *d40c = container_of(chan, struct d40_chan, + chan); + unsigned long flg; + int err; + + if (d40c->dma_cfg.pre_transfer) + d40c->dma_cfg.pre_transfer(chan, + d40c->dma_cfg.pre_transfer_data, + sg_dma_len(sgl)); + + spin_lock_irqsave(&d40c->lock, flg); + d40d = d40_desc_get(d40c); + spin_unlock_irqrestore(&d40c->lock, flg); + + if (d40d == NULL) + return NULL; + + memset(d40d, 0, sizeof(struct d40_desc)); + + if (d40c->log_num != D40_PHY_CHAN) + err = d40_prep_slave_sg_log(d40d, d40c, sgl, sg_len, + direction, flags); + else + err = d40_prep_slave_sg_phy(d40d, d40c, sgl, sg_len, + direction, flags); + if (err) { + dev_err(&d40c->chan.dev->device, + "[%s] Failed to prepare %s slave sg job: %d\n", + __func__, + d40c->log_num != D40_PHY_CHAN ? "log" : "phy", err); + return NULL; + } + + d40d->txd.flags = flags; + + dma_async_tx_descriptor_init(&d40d->txd, chan); + + d40d->txd.tx_submit = d40_tx_submit; + + return &d40d->txd; +} + +static enum dma_status d40_tx_status(struct dma_chan *chan, + dma_cookie_t cookie, + struct dma_tx_state *txstate) +{ + struct d40_chan *d40c = container_of(chan, struct d40_chan, chan); + dma_cookie_t last_used; + dma_cookie_t last_complete; + int ret; + + last_complete = d40c->completed; + last_used = chan->cookie; + + if (d40_is_paused(d40c)) + ret = DMA_PAUSED; + else + ret = dma_async_is_complete(cookie, last_complete, last_used); + + dma_set_tx_state(txstate, last_complete, last_used, + stedma40_residue(chan)); + + return ret; +} + +static void d40_issue_pending(struct dma_chan *chan) +{ + struct d40_chan *d40c = container_of(chan, struct d40_chan, chan); + unsigned long flags; + + spin_lock_irqsave(&d40c->lock, flags); + + /* Busy means that pending jobs are already being processed */ + if (!d40c->busy) + (void) d40_queue_start(d40c); + + spin_unlock_irqrestore(&d40c->lock, flags); +} + +static int d40_control(struct dma_chan *chan, enum dma_ctrl_cmd cmd, + unsigned long arg) +{ + unsigned long flags; + struct d40_chan *d40c = container_of(chan, struct d40_chan, chan); + + switch (cmd) { + case DMA_TERMINATE_ALL: + spin_lock_irqsave(&d40c->lock, flags); + d40_term_all(d40c); + spin_unlock_irqrestore(&d40c->lock, flags); + return 0; + case DMA_PAUSE: + return d40_pause(chan); + case DMA_RESUME: + return d40_resume(chan); + } + + /* Other commands are unimplemented */ + return -ENXIO; +} + +/* Initialization functions */ + +static void __init d40_chan_init(struct d40_base *base, struct dma_device *dma, + struct d40_chan *chans, int offset, + int num_chans) +{ + int i = 0; + struct d40_chan *d40c; + + INIT_LIST_HEAD(&dma->channels); + + for (i = offset; i < offset + num_chans; i++) { + d40c = &chans[i]; + d40c->base = base; + d40c->chan.device = dma; + + /* Invalidate lcla element */ + d40c->lcla.src_id = -1; + d40c->lcla.dst_id = -1; + + spin_lock_init(&d40c->lock); + + d40c->log_num = D40_PHY_CHAN; + + INIT_LIST_HEAD(&d40c->free); + INIT_LIST_HEAD(&d40c->active); + INIT_LIST_HEAD(&d40c->queue); + INIT_LIST_HEAD(&d40c->client); + + d40c->free_len = 0; + + tasklet_init(&d40c->tasklet, dma_tasklet, + (unsigned long) d40c); + + list_add_tail(&d40c->chan.device_node, + &dma->channels); + } +} + +static int __init d40_dmaengine_init(struct d40_base *base, + int num_reserved_chans) +{ + int err ; + + d40_chan_init(base, &base->dma_slave, base->log_chans, + 0, base->num_log_chans); + + dma_cap_zero(base->dma_slave.cap_mask); + dma_cap_set(DMA_SLAVE, base->dma_slave.cap_mask); + + base->dma_slave.device_alloc_chan_resources = d40_alloc_chan_resources; + base->dma_slave.device_free_chan_resources = d40_free_chan_resources; + base->dma_slave.device_prep_dma_memcpy = d40_prep_memcpy; + base->dma_slave.device_prep_slave_sg = d40_prep_slave_sg; + base->dma_slave.device_tx_status = d40_tx_status; + base->dma_slave.device_issue_pending = d40_issue_pending; + base->dma_slave.device_control = d40_control; + base->dma_slave.dev = base->dev; + + err = dma_async_device_register(&base->dma_slave); + + if (err) { + dev_err(base->dev, + "[%s] Failed to register slave channels\n", + __func__); + goto failure1; + } + + d40_chan_init(base, &base->dma_memcpy, base->log_chans, + base->num_log_chans, base->plat_data->memcpy_len); + + dma_cap_zero(base->dma_memcpy.cap_mask); + dma_cap_set(DMA_MEMCPY, base->dma_memcpy.cap_mask); + + base->dma_memcpy.device_alloc_chan_resources = d40_alloc_chan_resources; + base->dma_memcpy.device_free_chan_resources = d40_free_chan_resources; + base->dma_memcpy.device_prep_dma_memcpy = d40_prep_memcpy; + base->dma_memcpy.device_prep_slave_sg = d40_prep_slave_sg; + base->dma_memcpy.device_tx_status = d40_tx_status; + base->dma_memcpy.device_issue_pending = d40_issue_pending; + base->dma_memcpy.device_control = d40_control; + base->dma_memcpy.dev = base->dev; + /* + * This controller can only access address at even + * 32bit boundaries, i.e. 2^2 + */ + base->dma_memcpy.copy_align = 2; + + err = dma_async_device_register(&base->dma_memcpy); + + if (err) { + dev_err(base->dev, + "[%s] Failed to regsiter memcpy only channels\n", + __func__); + goto failure2; + } + + d40_chan_init(base, &base->dma_both, base->phy_chans, + 0, num_reserved_chans); + + dma_cap_zero(base->dma_both.cap_mask); + dma_cap_set(DMA_SLAVE, base->dma_both.cap_mask); + dma_cap_set(DMA_MEMCPY, base->dma_both.cap_mask); + + base->dma_both.device_alloc_chan_resources = d40_alloc_chan_resources; + base->dma_both.device_free_chan_resources = d40_free_chan_resources; + base->dma_both.device_prep_dma_memcpy = d40_prep_memcpy; + base->dma_both.device_prep_slave_sg = d40_prep_slave_sg; + base->dma_both.device_tx_status = d40_tx_status; + base->dma_both.device_issue_pending = d40_issue_pending; + base->dma_both.device_control = d40_control; + base->dma_both.dev = base->dev; + base->dma_both.copy_align = 2; + err = dma_async_device_register(&base->dma_both); + + if (err) { + dev_err(base->dev, + "[%s] Failed to register logical and physical capable channels\n", + __func__); + goto failure3; + } + return 0; +failure3: + dma_async_device_unregister(&base->dma_memcpy); +failure2: + dma_async_device_unregister(&base->dma_slave); +failure1: + return err; +} + +/* Initialization functions. */ + +static int __init d40_phy_res_init(struct d40_base *base) +{ + int i; + int num_phy_chans_avail = 0; + u32 val[2]; + int odd_even_bit = -2; + + val[0] = readl(base->virtbase + D40_DREG_PRSME); + val[1] = readl(base->virtbase + D40_DREG_PRSMO); + + for (i = 0; i < base->num_phy_chans; i++) { + base->phy_res[i].num = i; + odd_even_bit += 2 * ((i % 2) == 0); + if (((val[i % 2] >> odd_even_bit) & 3) == 1) { + /* Mark security only channels as occupied */ + base->phy_res[i].allocated_src = D40_ALLOC_PHY; + base->phy_res[i].allocated_dst = D40_ALLOC_PHY; + } else { + base->phy_res[i].allocated_src = D40_ALLOC_FREE; + base->phy_res[i].allocated_dst = D40_ALLOC_FREE; + num_phy_chans_avail++; + } + spin_lock_init(&base->phy_res[i].lock); + } + dev_info(base->dev, "%d of %d physical DMA channels available\n", + num_phy_chans_avail, base->num_phy_chans); + + /* Verify settings extended vs standard */ + val[0] = readl(base->virtbase + D40_DREG_PRTYP); + + for (i = 0; i < base->num_phy_chans; i++) { + + if (base->phy_res[i].allocated_src == D40_ALLOC_FREE && + (val[0] & 0x3) != 1) + dev_info(base->dev, + "[%s] INFO: channel %d is misconfigured (%d)\n", + __func__, i, val[0] & 0x3); + + val[0] = val[0] >> 2; + } + + return num_phy_chans_avail; +} + +static struct d40_base * __init d40_hw_detect_init(struct platform_device *pdev) +{ + static const struct d40_reg_val dma_id_regs[] = { + /* Peripheral Id */ + { .reg = D40_DREG_PERIPHID0, .val = 0x0040}, + { .reg = D40_DREG_PERIPHID1, .val = 0x0000}, + /* + * D40_DREG_PERIPHID2 Depends on HW revision: + * MOP500/HREF ED has 0x0008, + * ? has 0x0018, + * HREF V1 has 0x0028 + */ + { .reg = D40_DREG_PERIPHID3, .val = 0x0000}, + + /* PCell Id */ + { .reg = D40_DREG_CELLID0, .val = 0x000d}, + { .reg = D40_DREG_CELLID1, .val = 0x00f0}, + { .reg = D40_DREG_CELLID2, .val = 0x0005}, + { .reg = D40_DREG_CELLID3, .val = 0x00b1} + }; + struct stedma40_platform_data *plat_data; + struct clk *clk = NULL; + void __iomem *virtbase = NULL; + struct resource *res = NULL; + struct d40_base *base = NULL; + int num_log_chans = 0; + int num_phy_chans; + int i; + + clk = clk_get(&pdev->dev, NULL); + + if (IS_ERR(clk)) { + dev_err(&pdev->dev, "[%s] No matching clock found\n", + __func__); + goto failure; + } + + clk_enable(clk); + + /* Get IO for DMAC base address */ + res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "base"); + if (!res) + goto failure; + + if (request_mem_region(res->start, resource_size(res), + D40_NAME " I/O base") == NULL) + goto failure; + + virtbase = ioremap(res->start, resource_size(res)); + if (!virtbase) + goto failure; + + /* HW version check */ + for (i = 0; i < ARRAY_SIZE(dma_id_regs); i++) { + if (dma_id_regs[i].val != + readl(virtbase + dma_id_regs[i].reg)) { + dev_err(&pdev->dev, + "[%s] Unknown hardware! Expected 0x%x at 0x%x but got 0x%x\n", + __func__, + dma_id_regs[i].val, + dma_id_regs[i].reg, + readl(virtbase + dma_id_regs[i].reg)); + goto failure; + } + } + + i = readl(virtbase + D40_DREG_PERIPHID2); + + if ((i & 0xf) != D40_PERIPHID2_DESIGNER) { + dev_err(&pdev->dev, + "[%s] Unknown designer! Got %x wanted %x\n", + __func__, i & 0xf, D40_PERIPHID2_DESIGNER); + goto failure; + } + + /* The number of physical channels on this HW */ + num_phy_chans = 4 * (readl(virtbase + D40_DREG_ICFG) & 0x7) + 4; + + dev_info(&pdev->dev, "hardware revision: %d @ 0x%x\n", + (i >> 4) & 0xf, res->start); + + plat_data = pdev->dev.platform_data; + + /* Count the number of logical channels in use */ + for (i = 0; i < plat_data->dev_len; i++) + if (plat_data->dev_rx[i] != 0) + num_log_chans++; + + for (i = 0; i < plat_data->dev_len; i++) + if (plat_data->dev_tx[i] != 0) + num_log_chans++; + + base = kzalloc(ALIGN(sizeof(struct d40_base), 4) + + (num_phy_chans + num_log_chans + plat_data->memcpy_len) * + sizeof(struct d40_chan), GFP_KERNEL); + + if (base == NULL) { + dev_err(&pdev->dev, "[%s] Out of memory\n", __func__); + goto failure; + } + + base->clk = clk; + base->num_phy_chans = num_phy_chans; + base->num_log_chans = num_log_chans; + base->phy_start = res->start; + base->phy_size = resource_size(res); + base->virtbase = virtbase; + base->plat_data = plat_data; + base->dev = &pdev->dev; + base->phy_chans = ((void *)base) + ALIGN(sizeof(struct d40_base), 4); + base->log_chans = &base->phy_chans[num_phy_chans]; + + base->phy_res = kzalloc(num_phy_chans * sizeof(struct d40_phy_res), + GFP_KERNEL); + if (!base->phy_res) + goto failure; + + base->lookup_phy_chans = kzalloc(num_phy_chans * + sizeof(struct d40_chan *), + GFP_KERNEL); + if (!base->lookup_phy_chans) + goto failure; + + if (num_log_chans + plat_data->memcpy_len) { + /* + * The max number of logical channels are event lines for all + * src devices and dst devices + */ + base->lookup_log_chans = kzalloc(plat_data->dev_len * 2 * + sizeof(struct d40_chan *), + GFP_KERNEL); + if (!base->lookup_log_chans) + goto failure; + } + base->lcla_pool.alloc_map = kzalloc(num_phy_chans * sizeof(u32), + GFP_KERNEL); + if (!base->lcla_pool.alloc_map) + goto failure; + + return base; + +failure: + if (clk) { + clk_disable(clk); + clk_put(clk); + } + if (virtbase) + iounmap(virtbase); + if (res) + release_mem_region(res->start, + resource_size(res)); + if (virtbase) + iounmap(virtbase); + + if (base) { + kfree(base->lcla_pool.alloc_map); + kfree(base->lookup_log_chans); + kfree(base->lookup_phy_chans); + kfree(base->phy_res); + kfree(base); + } + + return NULL; +} + +static void __init d40_hw_init(struct d40_base *base) +{ + + static const struct d40_reg_val dma_init_reg[] = { + /* Clock every part of the DMA block from start */ + { .reg = D40_DREG_GCC, .val = 0x0000ff01}, + + /* Interrupts on all logical channels */ + { .reg = D40_DREG_LCMIS0, .val = 0xFFFFFFFF}, + { .reg = D40_DREG_LCMIS1, .val = 0xFFFFFFFF}, + { .reg = D40_DREG_LCMIS2, .val = 0xFFFFFFFF}, + { .reg = D40_DREG_LCMIS3, .val = 0xFFFFFFFF}, + { .reg = D40_DREG_LCICR0, .val = 0xFFFFFFFF}, + { .reg = D40_DREG_LCICR1, .val = 0xFFFFFFFF}, + { .reg = D40_DREG_LCICR2, .val = 0xFFFFFFFF}, + { .reg = D40_DREG_LCICR3, .val = 0xFFFFFFFF}, + { .reg = D40_DREG_LCTIS0, .val = 0xFFFFFFFF}, + { .reg = D40_DREG_LCTIS1, .val = 0xFFFFFFFF}, + { .reg = D40_DREG_LCTIS2, .val = 0xFFFFFFFF}, + { .reg = D40_DREG_LCTIS3, .val = 0xFFFFFFFF} + }; + int i; + u32 prmseo[2] = {0, 0}; + u32 activeo[2] = {0xFFFFFFFF, 0xFFFFFFFF}; + u32 pcmis = 0; + u32 pcicr = 0; + + for (i = 0; i < ARRAY_SIZE(dma_init_reg); i++) + writel(dma_init_reg[i].val, + base->virtbase + dma_init_reg[i].reg); + + /* Configure all our dma channels to default settings */ + for (i = 0; i < base->num_phy_chans; i++) { + + activeo[i % 2] = activeo[i % 2] << 2; + + if (base->phy_res[base->num_phy_chans - i - 1].allocated_src + == D40_ALLOC_PHY) { + activeo[i % 2] |= 3; + continue; + } + + /* Enable interrupt # */ + pcmis = (pcmis << 1) | 1; + + /* Clear interrupt # */ + pcicr = (pcicr << 1) | 1; + + /* Set channel to physical mode */ + prmseo[i % 2] = prmseo[i % 2] << 2; + prmseo[i % 2] |= 1; + + } + + writel(prmseo[1], base->virtbase + D40_DREG_PRMSE); + writel(prmseo[0], base->virtbase + D40_DREG_PRMSO); + writel(activeo[1], base->virtbase + D40_DREG_ACTIVE); + writel(activeo[0], base->virtbase + D40_DREG_ACTIVO); + + /* Write which interrupt to enable */ + writel(pcmis, base->virtbase + D40_DREG_PCMIS); + + /* Write which interrupt to clear */ + writel(pcicr, base->virtbase + D40_DREG_PCICR); + +} + +static int __init d40_probe(struct platform_device *pdev) +{ + int err; + int ret = -ENOENT; + struct d40_base *base; + struct resource *res = NULL; + int num_reserved_chans; + u32 val; + + base = d40_hw_detect_init(pdev); + + if (!base) + goto failure; + + num_reserved_chans = d40_phy_res_init(base); + + platform_set_drvdata(pdev, base); + + spin_lock_init(&base->interrupt_lock); + spin_lock_init(&base->execmd_lock); + + /* Get IO for logical channel parameter address */ + res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "lcpa"); + if (!res) { + ret = -ENOENT; + dev_err(&pdev->dev, + "[%s] No \"lcpa\" memory resource\n", + __func__); + goto failure; + } + base->lcpa_size = resource_size(res); + base->phy_lcpa = res->start; + + if (request_mem_region(res->start, resource_size(res), + D40_NAME " I/O lcpa") == NULL) { + ret = -EBUSY; + dev_err(&pdev->dev, + "[%s] Failed to request LCPA region 0x%x-0x%x\n", + __func__, res->start, res->end); + goto failure; + } + + /* We make use of ESRAM memory for this. */ + val = readl(base->virtbase + D40_DREG_LCPA); + if (res->start != val && val != 0) { + dev_warn(&pdev->dev, + "[%s] Mismatch LCPA dma 0x%x, def 0x%x\n", + __func__, val, res->start); + } else + writel(res->start, base->virtbase + D40_DREG_LCPA); + + base->lcpa_base = ioremap(res->start, resource_size(res)); + if (!base->lcpa_base) { + ret = -ENOMEM; + dev_err(&pdev->dev, + "[%s] Failed to ioremap LCPA region\n", + __func__); + goto failure; + } + /* Get IO for logical channel link address */ + res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "lcla"); + if (!res) { + ret = -ENOENT; + dev_err(&pdev->dev, + "[%s] No \"lcla\" resource defined\n", + __func__); + goto failure; + } + + base->lcla_pool.base_size = resource_size(res); + base->lcla_pool.phy = res->start; + + if (request_mem_region(res->start, resource_size(res), + D40_NAME " I/O lcla") == NULL) { + ret = -EBUSY; + dev_err(&pdev->dev, + "[%s] Failed to request LCLA region 0x%x-0x%x\n", + __func__, res->start, res->end); + goto failure; + } + val = readl(base->virtbase + D40_DREG_LCLA); + if (res->start != val && val != 0) { + dev_warn(&pdev->dev, + "[%s] Mismatch LCLA dma 0x%x, def 0x%x\n", + __func__, val, res->start); + } else + writel(res->start, base->virtbase + D40_DREG_LCLA); + + base->lcla_pool.base = ioremap(res->start, resource_size(res)); + if (!base->lcla_pool.base) { + ret = -ENOMEM; + dev_err(&pdev->dev, + "[%s] Failed to ioremap LCLA 0x%x-0x%x\n", + __func__, res->start, res->end); + goto failure; + } + + spin_lock_init(&base->lcla_pool.lock); + + base->lcla_pool.num_blocks = base->num_phy_chans; + + base->irq = platform_get_irq(pdev, 0); + + ret = request_irq(base->irq, d40_handle_interrupt, 0, D40_NAME, base); + + if (ret) { + dev_err(&pdev->dev, "[%s] No IRQ defined\n", __func__); + goto failure; + } + + err = d40_dmaengine_init(base, num_reserved_chans); + if (err) + goto failure; + + d40_hw_init(base); + + dev_info(base->dev, "initialized\n"); + return 0; + +failure: + if (base) { + if (base->virtbase) + iounmap(base->virtbase); + if (base->lcla_pool.phy) + release_mem_region(base->lcla_pool.phy, + base->lcla_pool.base_size); + if (base->phy_lcpa) + release_mem_region(base->phy_lcpa, + base->lcpa_size); + if (base->phy_start) + release_mem_region(base->phy_start, + base->phy_size); + if (base->clk) { + clk_disable(base->clk); + clk_put(base->clk); + } + + kfree(base->lcla_pool.alloc_map); + kfree(base->lookup_log_chans); + kfree(base->lookup_phy_chans); + kfree(base->phy_res); + kfree(base); + } + + dev_err(&pdev->dev, "[%s] probe failed\n", __func__); + return ret; +} + +static struct platform_driver d40_driver = { + .driver = { + .owner = THIS_MODULE, + .name = D40_NAME, + }, +}; + +int __init stedma40_init(void) +{ + return platform_driver_probe(&d40_driver, d40_probe); +} +arch_initcall(stedma40_init); diff --git a/drivers/dma/ste_dma40_ll.c b/drivers/dma/ste_dma40_ll.c new file mode 100644 index 000000000000..561fdd8a80c1 --- /dev/null +++ b/drivers/dma/ste_dma40_ll.c @@ -0,0 +1,454 @@ +/* + * driver/dma/ste_dma40_ll.c + * + * Copyright (C) ST-Ericsson 2007-2010 + * License terms: GNU General Public License (GPL) version 2 + * Author: Per Friden <per.friden@stericsson.com> + * Author: Jonas Aaberg <jonas.aberg@stericsson.com> + */ + +#include <linux/kernel.h> +#include <plat/ste_dma40.h> + +#include "ste_dma40_ll.h" + +/* Sets up proper LCSP1 and LCSP3 register for a logical channel */ +void d40_log_cfg(struct stedma40_chan_cfg *cfg, + u32 *lcsp1, u32 *lcsp3) +{ + u32 l3 = 0; /* dst */ + u32 l1 = 0; /* src */ + + /* src is mem? -> increase address pos */ + if (cfg->dir == STEDMA40_MEM_TO_PERIPH || + cfg->dir == STEDMA40_MEM_TO_MEM) + l1 |= 1 << D40_MEM_LCSP1_SCFG_INCR_POS; + + /* dst is mem? -> increase address pos */ + if (cfg->dir == STEDMA40_PERIPH_TO_MEM || + cfg->dir == STEDMA40_MEM_TO_MEM) + l3 |= 1 << D40_MEM_LCSP3_DCFG_INCR_POS; + + /* src is hw? -> master port 1 */ + if (cfg->dir == STEDMA40_PERIPH_TO_MEM || + cfg->dir == STEDMA40_PERIPH_TO_PERIPH) + l1 |= 1 << D40_MEM_LCSP1_SCFG_MST_POS; + + /* dst is hw? -> master port 1 */ + if (cfg->dir == STEDMA40_MEM_TO_PERIPH || + cfg->dir == STEDMA40_PERIPH_TO_PERIPH) + l3 |= 1 << D40_MEM_LCSP3_DCFG_MST_POS; + + l3 |= 1 << D40_MEM_LCSP3_DCFG_TIM_POS; + l3 |= 1 << D40_MEM_LCSP3_DCFG_EIM_POS; + l3 |= cfg->dst_info.psize << D40_MEM_LCSP3_DCFG_PSIZE_POS; + l3 |= cfg->dst_info.data_width << D40_MEM_LCSP3_DCFG_ESIZE_POS; + l3 |= 1 << D40_MEM_LCSP3_DTCP_POS; + + l1 |= 1 << D40_MEM_LCSP1_SCFG_EIM_POS; + l1 |= cfg->src_info.psize << D40_MEM_LCSP1_SCFG_PSIZE_POS; + l1 |= cfg->src_info.data_width << D40_MEM_LCSP1_SCFG_ESIZE_POS; + l1 |= 1 << D40_MEM_LCSP1_STCP_POS; + + *lcsp1 = l1; + *lcsp3 = l3; + +} + +/* Sets up SRC and DST CFG register for both logical and physical channels */ +void d40_phy_cfg(struct stedma40_chan_cfg *cfg, + u32 *src_cfg, u32 *dst_cfg, bool is_log) +{ + u32 src = 0; + u32 dst = 0; + + if (!is_log) { + /* Physical channel */ + if ((cfg->dir == STEDMA40_PERIPH_TO_MEM) || + (cfg->dir == STEDMA40_PERIPH_TO_PERIPH)) { + /* Set master port to 1 */ + src |= 1 << D40_SREG_CFG_MST_POS; + src |= D40_TYPE_TO_EVENT(cfg->src_dev_type); + + if (cfg->src_info.flow_ctrl == STEDMA40_NO_FLOW_CTRL) + src |= 1 << D40_SREG_CFG_PHY_TM_POS; + else + src |= 3 << D40_SREG_CFG_PHY_TM_POS; + } + if ((cfg->dir == STEDMA40_MEM_TO_PERIPH) || + (cfg->dir == STEDMA40_PERIPH_TO_PERIPH)) { + /* Set master port to 1 */ + dst |= 1 << D40_SREG_CFG_MST_POS; + dst |= D40_TYPE_TO_EVENT(cfg->dst_dev_type); + + if (cfg->dst_info.flow_ctrl == STEDMA40_NO_FLOW_CTRL) + dst |= 1 << D40_SREG_CFG_PHY_TM_POS; + else + dst |= 3 << D40_SREG_CFG_PHY_TM_POS; + } + /* Interrupt on end of transfer for destination */ + dst |= 1 << D40_SREG_CFG_TIM_POS; + + /* Generate interrupt on error */ + src |= 1 << D40_SREG_CFG_EIM_POS; + dst |= 1 << D40_SREG_CFG_EIM_POS; + + /* PSIZE */ + if (cfg->src_info.psize != STEDMA40_PSIZE_PHY_1) { + src |= 1 << D40_SREG_CFG_PHY_PEN_POS; + src |= cfg->src_info.psize << D40_SREG_CFG_PSIZE_POS; + } + if (cfg->dst_info.psize != STEDMA40_PSIZE_PHY_1) { + dst |= 1 << D40_SREG_CFG_PHY_PEN_POS; + dst |= cfg->dst_info.psize << D40_SREG_CFG_PSIZE_POS; + } + + /* Element size */ + src |= cfg->src_info.data_width << D40_SREG_CFG_ESIZE_POS; + dst |= cfg->dst_info.data_width << D40_SREG_CFG_ESIZE_POS; + + } else { + /* Logical channel */ + dst |= 1 << D40_SREG_CFG_LOG_GIM_POS; + src |= 1 << D40_SREG_CFG_LOG_GIM_POS; + } + + if (cfg->channel_type & STEDMA40_HIGH_PRIORITY_CHANNEL) { + src |= 1 << D40_SREG_CFG_PRI_POS; + dst |= 1 << D40_SREG_CFG_PRI_POS; + } + + src |= cfg->src_info.endianess << D40_SREG_CFG_LBE_POS; + dst |= cfg->dst_info.endianess << D40_SREG_CFG_LBE_POS; + + *src_cfg = src; + *dst_cfg = dst; +} + +int d40_phy_fill_lli(struct d40_phy_lli *lli, + dma_addr_t data, + u32 data_size, + int psize, + dma_addr_t next_lli, + u32 reg_cfg, + bool term_int, + u32 data_width, + bool is_device) +{ + int num_elems; + + if (psize == STEDMA40_PSIZE_PHY_1) + num_elems = 1; + else + num_elems = 2 << psize; + + /* + * Size is 16bit. data_width is 8, 16, 32 or 64 bit + * Block large than 64 KiB must be split. + */ + if (data_size > (0xffff << data_width)) + return -EINVAL; + + /* Must be aligned */ + if (!IS_ALIGNED(data, 0x1 << data_width)) + return -EINVAL; + + /* Transfer size can't be smaller than (num_elms * elem_size) */ + if (data_size < num_elems * (0x1 << data_width)) + return -EINVAL; + + /* The number of elements. IE now many chunks */ + lli->reg_elt = (data_size >> data_width) << D40_SREG_ELEM_PHY_ECNT_POS; + + /* + * Distance to next element sized entry. + * Usually the size of the element unless you want gaps. + */ + if (!is_device) + lli->reg_elt |= (0x1 << data_width) << + D40_SREG_ELEM_PHY_EIDX_POS; + + /* Where the data is */ + lli->reg_ptr = data; + lli->reg_cfg = reg_cfg; + + /* If this scatter list entry is the last one, no next link */ + if (next_lli == 0) + lli->reg_lnk = 0x1 << D40_SREG_LNK_PHY_TCP_POS; + else + lli->reg_lnk = next_lli; + + /* Set/clear interrupt generation on this link item.*/ + if (term_int) + lli->reg_cfg |= 0x1 << D40_SREG_CFG_TIM_POS; + else + lli->reg_cfg &= ~(0x1 << D40_SREG_CFG_TIM_POS); + + /* Post link */ + lli->reg_lnk |= 0 << D40_SREG_LNK_PHY_PRE_POS; + + return 0; +} + +int d40_phy_sg_to_lli(struct scatterlist *sg, + int sg_len, + dma_addr_t target, + struct d40_phy_lli *lli, + dma_addr_t lli_phys, + u32 reg_cfg, + u32 data_width, + int psize, + bool term_int) +{ + int total_size = 0; + int i; + struct scatterlist *current_sg = sg; + dma_addr_t next_lli_phys; + dma_addr_t dst; + int err = 0; + + for_each_sg(sg, current_sg, sg_len, i) { + + total_size += sg_dma_len(current_sg); + + /* If this scatter list entry is the last one, no next link */ + if (sg_len - 1 == i) + next_lli_phys = 0; + else + next_lli_phys = ALIGN(lli_phys + (i + 1) * + sizeof(struct d40_phy_lli), + D40_LLI_ALIGN); + + if (target) + dst = target; + else + dst = sg_phys(current_sg); + + err = d40_phy_fill_lli(&lli[i], + dst, + sg_dma_len(current_sg), + psize, + next_lli_phys, + reg_cfg, + !next_lli_phys, + data_width, + target == dst); + if (err) + goto err; + } + + return total_size; + err: + return err; +} + + +void d40_phy_lli_write(void __iomem *virtbase, + u32 phy_chan_num, + struct d40_phy_lli *lli_dst, + struct d40_phy_lli *lli_src) +{ + + writel(lli_src->reg_cfg, virtbase + D40_DREG_PCBASE + + phy_chan_num * D40_DREG_PCDELTA + D40_CHAN_REG_SSCFG); + writel(lli_src->reg_elt, virtbase + D40_DREG_PCBASE + + phy_chan_num * D40_DREG_PCDELTA + D40_CHAN_REG_SSELT); + writel(lli_src->reg_ptr, virtbase + D40_DREG_PCBASE + + phy_chan_num * D40_DREG_PCDELTA + D40_CHAN_REG_SSPTR); + writel(lli_src->reg_lnk, virtbase + D40_DREG_PCBASE + + phy_chan_num * D40_DREG_PCDELTA + D40_CHAN_REG_SSLNK); + + writel(lli_dst->reg_cfg, virtbase + D40_DREG_PCBASE + + phy_chan_num * D40_DREG_PCDELTA + D40_CHAN_REG_SDCFG); + writel(lli_dst->reg_elt, virtbase + D40_DREG_PCBASE + + phy_chan_num * D40_DREG_PCDELTA + D40_CHAN_REG_SDELT); + writel(lli_dst->reg_ptr, virtbase + D40_DREG_PCBASE + + phy_chan_num * D40_DREG_PCDELTA + D40_CHAN_REG_SDPTR); + writel(lli_dst->reg_lnk, virtbase + D40_DREG_PCBASE + + phy_chan_num * D40_DREG_PCDELTA + D40_CHAN_REG_SDLNK); + +} + +/* DMA logical lli operations */ + +void d40_log_fill_lli(struct d40_log_lli *lli, + dma_addr_t data, u32 data_size, + u32 lli_next_off, u32 reg_cfg, + u32 data_width, + bool term_int, bool addr_inc) +{ + lli->lcsp13 = reg_cfg; + + /* The number of elements to transfer */ + lli->lcsp02 = ((data_size >> data_width) << + D40_MEM_LCSP0_ECNT_POS) & D40_MEM_LCSP0_ECNT_MASK; + /* 16 LSBs address of the current element */ + lli->lcsp02 |= data & D40_MEM_LCSP0_SPTR_MASK; + /* 16 MSBs address of the current element */ + lli->lcsp13 |= data & D40_MEM_LCSP1_SPTR_MASK; + + if (addr_inc) + lli->lcsp13 |= D40_MEM_LCSP1_SCFG_INCR_MASK; + + lli->lcsp13 |= D40_MEM_LCSP3_DTCP_MASK; + /* If this scatter list entry is the last one, no next link */ + lli->lcsp13 |= (lli_next_off << D40_MEM_LCSP1_SLOS_POS) & + D40_MEM_LCSP1_SLOS_MASK; + + if (term_int) + lli->lcsp13 |= D40_MEM_LCSP1_SCFG_TIM_MASK; + else + lli->lcsp13 &= ~D40_MEM_LCSP1_SCFG_TIM_MASK; +} + +int d40_log_sg_to_dev(struct d40_lcla_elem *lcla, + struct scatterlist *sg, + int sg_len, + struct d40_log_lli_bidir *lli, + struct d40_def_lcsp *lcsp, + u32 src_data_width, + u32 dst_data_width, + enum dma_data_direction direction, + bool term_int, dma_addr_t dev_addr, int max_len, + int llis_per_log) +{ + int total_size = 0; + struct scatterlist *current_sg = sg; + int i; + u32 next_lli_off_dst; + u32 next_lli_off_src; + + next_lli_off_src = 0; + next_lli_off_dst = 0; + + for_each_sg(sg, current_sg, sg_len, i) { + total_size += sg_dma_len(current_sg); + + /* + * If this scatter list entry is the last one or + * max length, terminate link. + */ + if (sg_len - 1 == i || ((i+1) % max_len == 0)) { + next_lli_off_src = 0; + next_lli_off_dst = 0; + } else { + if (next_lli_off_dst == 0 && + next_lli_off_src == 0) { + /* The first lli will be at next_lli_off */ + next_lli_off_dst = (lcla->dst_id * + llis_per_log + 1); + next_lli_off_src = (lcla->src_id * + llis_per_log + 1); + } else { + next_lli_off_dst++; + next_lli_off_src++; + } + } + + if (direction == DMA_TO_DEVICE) { + d40_log_fill_lli(&lli->src[i], + sg_phys(current_sg), + sg_dma_len(current_sg), + next_lli_off_src, + lcsp->lcsp1, src_data_width, + term_int && !next_lli_off_src, + true); + d40_log_fill_lli(&lli->dst[i], + dev_addr, + sg_dma_len(current_sg), + next_lli_off_dst, + lcsp->lcsp3, dst_data_width, + /* No next == terminal interrupt */ + term_int && !next_lli_off_dst, + false); + } else { + d40_log_fill_lli(&lli->dst[i], + sg_phys(current_sg), + sg_dma_len(current_sg), + next_lli_off_dst, + lcsp->lcsp3, dst_data_width, + /* No next == terminal interrupt */ + term_int && !next_lli_off_dst, + true); + d40_log_fill_lli(&lli->src[i], + dev_addr, + sg_dma_len(current_sg), + next_lli_off_src, + lcsp->lcsp1, src_data_width, + term_int && !next_lli_off_src, + false); + } + } + return total_size; +} + +int d40_log_sg_to_lli(int lcla_id, + struct scatterlist *sg, + int sg_len, + struct d40_log_lli *lli_sg, + u32 lcsp13, /* src or dst*/ + u32 data_width, + bool term_int, int max_len, int llis_per_log) +{ + int total_size = 0; + struct scatterlist *current_sg = sg; + int i; + u32 next_lli_off = 0; + + for_each_sg(sg, current_sg, sg_len, i) { + total_size += sg_dma_len(current_sg); + + /* + * If this scatter list entry is the last one or + * max length, terminate link. + */ + if (sg_len - 1 == i || ((i+1) % max_len == 0)) + next_lli_off = 0; + else { + if (next_lli_off == 0) + /* The first lli will be at next_lli_off */ + next_lli_off = lcla_id * llis_per_log + 1; + else + next_lli_off++; + } + + d40_log_fill_lli(&lli_sg[i], + sg_phys(current_sg), + sg_dma_len(current_sg), + next_lli_off, + lcsp13, data_width, + term_int && !next_lli_off, + true); + } + return total_size; +} + +void d40_log_lli_write(struct d40_log_lli_full *lcpa, + struct d40_log_lli *lcla_src, + struct d40_log_lli *lcla_dst, + struct d40_log_lli *lli_dst, + struct d40_log_lli *lli_src, + int llis_per_log) +{ + u32 slos = 0; + u32 dlos = 0; + int i; + + lcpa->lcsp0 = lli_src->lcsp02; + lcpa->lcsp1 = lli_src->lcsp13; + lcpa->lcsp2 = lli_dst->lcsp02; + lcpa->lcsp3 = lli_dst->lcsp13; + + slos = lli_src->lcsp13 & D40_MEM_LCSP1_SLOS_MASK; + dlos = lli_dst->lcsp13 & D40_MEM_LCSP3_DLOS_MASK; + + for (i = 0; (i < llis_per_log) && slos && dlos; i++) { + writel(lli_src[i+1].lcsp02, &lcla_src[i].lcsp02); + writel(lli_src[i+1].lcsp13, &lcla_src[i].lcsp13); + writel(lli_dst[i+1].lcsp02, &lcla_dst[i].lcsp02); + writel(lli_dst[i+1].lcsp13, &lcla_dst[i].lcsp13); + + slos = lli_src[i+1].lcsp13 & D40_MEM_LCSP1_SLOS_MASK; + dlos = lli_dst[i+1].lcsp13 & D40_MEM_LCSP3_DLOS_MASK; + } +} diff --git a/drivers/dma/ste_dma40_ll.h b/drivers/dma/ste_dma40_ll.h new file mode 100644 index 000000000000..2029280cb332 --- /dev/null +++ b/drivers/dma/ste_dma40_ll.h @@ -0,0 +1,354 @@ +/* + * driver/dma/ste_dma40_ll.h + * + * Copyright (C) ST-Ericsson 2007-2010 + * License terms: GNU General Public License (GPL) version 2 + * Author: Per Friden <per.friden@stericsson.com> + * Author: Jonas Aaberg <jonas.aberg@stericsson.com> + */ +#ifndef STE_DMA40_LL_H +#define STE_DMA40_LL_H + +#define D40_DREG_PCBASE 0x400 +#define D40_DREG_PCDELTA (8 * 4) +#define D40_LLI_ALIGN 16 /* LLI alignment must be 16 bytes. */ + +#define D40_TYPE_TO_GROUP(type) (type / 16) +#define D40_TYPE_TO_EVENT(type) (type % 16) + +/* Most bits of the CFG register are the same in log as in phy mode */ +#define D40_SREG_CFG_MST_POS 15 +#define D40_SREG_CFG_TIM_POS 14 +#define D40_SREG_CFG_EIM_POS 13 +#define D40_SREG_CFG_LOG_INCR_POS 12 +#define D40_SREG_CFG_PHY_PEN_POS 12 +#define D40_SREG_CFG_PSIZE_POS 10 +#define D40_SREG_CFG_ESIZE_POS 8 +#define D40_SREG_CFG_PRI_POS 7 +#define D40_SREG_CFG_LBE_POS 6 +#define D40_SREG_CFG_LOG_GIM_POS 5 +#define D40_SREG_CFG_LOG_MFU_POS 4 +#define D40_SREG_CFG_PHY_TM_POS 4 +#define D40_SREG_CFG_PHY_EVTL_POS 0 + + +/* Standard channel parameters - basic mode (element register) */ +#define D40_SREG_ELEM_PHY_ECNT_POS 16 +#define D40_SREG_ELEM_PHY_EIDX_POS 0 + +#define D40_SREG_ELEM_PHY_ECNT_MASK (0xFFFF << D40_SREG_ELEM_PHY_ECNT_POS) + +/* Standard channel parameters - basic mode (Link register) */ +#define D40_SREG_LNK_PHY_TCP_POS 0 +#define D40_SREG_LNK_PHY_LMP_POS 1 +#define D40_SREG_LNK_PHY_PRE_POS 2 +/* + * Source destination link address. Contains the + * 29-bit byte word aligned address of the reload area. + */ +#define D40_SREG_LNK_PHYS_LNK_MASK 0xFFFFFFF8UL + +/* Standard basic channel logical mode */ + +/* Element register */ +#define D40_SREG_ELEM_LOG_ECNT_POS 16 +#define D40_SREG_ELEM_LOG_LIDX_POS 8 +#define D40_SREG_ELEM_LOG_LOS_POS 1 +#define D40_SREG_ELEM_LOG_TCP_POS 0 + +#define D40_SREG_ELEM_LOG_LIDX_MASK (0xFF << D40_SREG_ELEM_LOG_LIDX_POS) + +/* Link register */ +#define D40_DEACTIVATE_EVENTLINE 0x0 +#define D40_ACTIVATE_EVENTLINE 0x1 +#define D40_EVENTLINE_POS(i) (2 * i) +#define D40_EVENTLINE_MASK(i) (0x3 << D40_EVENTLINE_POS(i)) + +/* Standard basic channel logical params in memory */ + +/* LCSP0 */ +#define D40_MEM_LCSP0_ECNT_POS 16 +#define D40_MEM_LCSP0_SPTR_POS 0 + +#define D40_MEM_LCSP0_ECNT_MASK (0xFFFF << D40_MEM_LCSP0_ECNT_POS) +#define D40_MEM_LCSP0_SPTR_MASK (0xFFFF << D40_MEM_LCSP0_SPTR_POS) + +/* LCSP1 */ +#define D40_MEM_LCSP1_SPTR_POS 16 +#define D40_MEM_LCSP1_SCFG_MST_POS 15 +#define D40_MEM_LCSP1_SCFG_TIM_POS 14 +#define D40_MEM_LCSP1_SCFG_EIM_POS 13 +#define D40_MEM_LCSP1_SCFG_INCR_POS 12 +#define D40_MEM_LCSP1_SCFG_PSIZE_POS 10 +#define D40_MEM_LCSP1_SCFG_ESIZE_POS 8 +#define D40_MEM_LCSP1_SLOS_POS 1 +#define D40_MEM_LCSP1_STCP_POS 0 + +#define D40_MEM_LCSP1_SPTR_MASK (0xFFFF << D40_MEM_LCSP1_SPTR_POS) +#define D40_MEM_LCSP1_SCFG_TIM_MASK (0x1 << D40_MEM_LCSP1_SCFG_TIM_POS) +#define D40_MEM_LCSP1_SCFG_INCR_MASK (0x1 << D40_MEM_LCSP1_SCFG_INCR_POS) +#define D40_MEM_LCSP1_SCFG_PSIZE_MASK (0x3 << D40_MEM_LCSP1_SCFG_PSIZE_POS) +#define D40_MEM_LCSP1_SLOS_MASK (0x7F << D40_MEM_LCSP1_SLOS_POS) +#define D40_MEM_LCSP1_STCP_MASK (0x1 << D40_MEM_LCSP1_STCP_POS) + +/* LCSP2 */ +#define D40_MEM_LCSP2_ECNT_POS 16 + +#define D40_MEM_LCSP2_ECNT_MASK (0xFFFF << D40_MEM_LCSP2_ECNT_POS) + +/* LCSP3 */ +#define D40_MEM_LCSP3_DCFG_MST_POS 15 +#define D40_MEM_LCSP3_DCFG_TIM_POS 14 +#define D40_MEM_LCSP3_DCFG_EIM_POS 13 +#define D40_MEM_LCSP3_DCFG_INCR_POS 12 +#define D40_MEM_LCSP3_DCFG_PSIZE_POS 10 +#define D40_MEM_LCSP3_DCFG_ESIZE_POS 8 +#define D40_MEM_LCSP3_DLOS_POS 1 +#define D40_MEM_LCSP3_DTCP_POS 0 + +#define D40_MEM_LCSP3_DLOS_MASK (0x7F << D40_MEM_LCSP3_DLOS_POS) +#define D40_MEM_LCSP3_DTCP_MASK (0x1 << D40_MEM_LCSP3_DTCP_POS) + + +/* Standard channel parameter register offsets */ +#define D40_CHAN_REG_SSCFG 0x00 +#define D40_CHAN_REG_SSELT 0x04 +#define D40_CHAN_REG_SSPTR 0x08 +#define D40_CHAN_REG_SSLNK 0x0C +#define D40_CHAN_REG_SDCFG 0x10 +#define D40_CHAN_REG_SDELT 0x14 +#define D40_CHAN_REG_SDPTR 0x18 +#define D40_CHAN_REG_SDLNK 0x1C + +/* DMA Register Offsets */ +#define D40_DREG_GCC 0x000 +#define D40_DREG_PRTYP 0x004 +#define D40_DREG_PRSME 0x008 +#define D40_DREG_PRSMO 0x00C +#define D40_DREG_PRMSE 0x010 +#define D40_DREG_PRMSO 0x014 +#define D40_DREG_PRMOE 0x018 +#define D40_DREG_PRMOO 0x01C +#define D40_DREG_LCPA 0x020 +#define D40_DREG_LCLA 0x024 +#define D40_DREG_ACTIVE 0x050 +#define D40_DREG_ACTIVO 0x054 +#define D40_DREG_FSEB1 0x058 +#define D40_DREG_FSEB2 0x05C +#define D40_DREG_PCMIS 0x060 +#define D40_DREG_PCICR 0x064 +#define D40_DREG_PCTIS 0x068 +#define D40_DREG_PCEIS 0x06C +#define D40_DREG_LCMIS0 0x080 +#define D40_DREG_LCMIS1 0x084 +#define D40_DREG_LCMIS2 0x088 +#define D40_DREG_LCMIS3 0x08C +#define D40_DREG_LCICR0 0x090 +#define D40_DREG_LCICR1 0x094 +#define D40_DREG_LCICR2 0x098 +#define D40_DREG_LCICR3 0x09C +#define D40_DREG_LCTIS0 0x0A0 +#define D40_DREG_LCTIS1 0x0A4 +#define D40_DREG_LCTIS2 0x0A8 +#define D40_DREG_LCTIS3 0x0AC +#define D40_DREG_LCEIS0 0x0B0 +#define D40_DREG_LCEIS1 0x0B4 +#define D40_DREG_LCEIS2 0x0B8 +#define D40_DREG_LCEIS3 0x0BC +#define D40_DREG_STFU 0xFC8 +#define D40_DREG_ICFG 0xFCC +#define D40_DREG_PERIPHID0 0xFE0 +#define D40_DREG_PERIPHID1 0xFE4 +#define D40_DREG_PERIPHID2 0xFE8 +#define D40_DREG_PERIPHID3 0xFEC +#define D40_DREG_CELLID0 0xFF0 +#define D40_DREG_CELLID1 0xFF4 +#define D40_DREG_CELLID2 0xFF8 +#define D40_DREG_CELLID3 0xFFC + +/* LLI related structures */ + +/** + * struct d40_phy_lli - The basic configration register for each physical + * channel. + * + * @reg_cfg: The configuration register. + * @reg_elt: The element register. + * @reg_ptr: The pointer register. + * @reg_lnk: The link register. + * + * These registers are set up for both physical and logical transfers + * Note that the bit in each register means differently in logical and + * physical(standard) mode. + * + * This struct must be 16 bytes aligned, and only contain physical registers + * since it will be directly accessed by the DMA. + */ +struct d40_phy_lli { + u32 reg_cfg; + u32 reg_elt; + u32 reg_ptr; + u32 reg_lnk; +}; + +/** + * struct d40_phy_lli_bidir - struct for a transfer. + * + * @src: Register settings for src channel. + * @dst: Register settings for dst channel. + * @dst_addr: Physical destination address. + * @src_addr: Physical source address. + * + * All DMA transfers have a source and a destination. + */ + +struct d40_phy_lli_bidir { + struct d40_phy_lli *src; + struct d40_phy_lli *dst; + dma_addr_t dst_addr; + dma_addr_t src_addr; +}; + + +/** + * struct d40_log_lli - logical lli configuration + * + * @lcsp02: Either maps to register lcsp0 if src or lcsp2 if dst. + * @lcsp13: Either maps to register lcsp1 if src or lcsp3 if dst. + * + * This struct must be 8 bytes aligned since it will be accessed directy by + * the DMA. Never add any none hw mapped registers to this struct. + */ + +struct d40_log_lli { + u32 lcsp02; + u32 lcsp13; +}; + +/** + * struct d40_log_lli_bidir - For both src and dst + * + * @src: pointer to src lli configuration. + * @dst: pointer to dst lli configuration. + * + * You always have a src and a dst when doing DMA transfers. + */ + +struct d40_log_lli_bidir { + struct d40_log_lli *src; + struct d40_log_lli *dst; +}; + +/** + * struct d40_log_lli_full - LCPA layout + * + * @lcsp0: Logical Channel Standard Param 0 - Src. + * @lcsp1: Logical Channel Standard Param 1 - Src. + * @lcsp2: Logical Channel Standard Param 2 - Dst. + * @lcsp3: Logical Channel Standard Param 3 - Dst. + * + * This struct maps to LCPA physical memory layout. Must map to + * the hw. + */ +struct d40_log_lli_full { + u32 lcsp0; + u32 lcsp1; + u32 lcsp2; + u32 lcsp3; +}; + +/** + * struct d40_def_lcsp - Default LCSP1 and LCSP3 settings + * + * @lcsp3: The default configuration for dst. + * @lcsp1: The default configuration for src. + */ +struct d40_def_lcsp { + u32 lcsp3; + u32 lcsp1; +}; + +/** + * struct d40_lcla_elem - Info for one LCA element. + * + * @src_id: logical channel src id + * @dst_id: logical channel dst id + * @src: LCPA formated src parameters + * @dst: LCPA formated dst parameters + * + */ +struct d40_lcla_elem { + int src_id; + int dst_id; + struct d40_log_lli *src; + struct d40_log_lli *dst; +}; + +/* Physical channels */ + +void d40_phy_cfg(struct stedma40_chan_cfg *cfg, + u32 *src_cfg, u32 *dst_cfg, bool is_log); + +void d40_log_cfg(struct stedma40_chan_cfg *cfg, + u32 *lcsp1, u32 *lcsp2); + +int d40_phy_sg_to_lli(struct scatterlist *sg, + int sg_len, + dma_addr_t target, + struct d40_phy_lli *lli, + dma_addr_t lli_phys, + u32 reg_cfg, + u32 data_width, + int psize, + bool term_int); + +int d40_phy_fill_lli(struct d40_phy_lli *lli, + dma_addr_t data, + u32 data_size, + int psize, + dma_addr_t next_lli, + u32 reg_cfg, + bool term_int, + u32 data_width, + bool is_device); + +void d40_phy_lli_write(void __iomem *virtbase, + u32 phy_chan_num, + struct d40_phy_lli *lli_dst, + struct d40_phy_lli *lli_src); + +/* Logical channels */ + +void d40_log_fill_lli(struct d40_log_lli *lli, + dma_addr_t data, u32 data_size, + u32 lli_next_off, u32 reg_cfg, + u32 data_width, + bool term_int, bool addr_inc); + +int d40_log_sg_to_dev(struct d40_lcla_elem *lcla, + struct scatterlist *sg, + int sg_len, + struct d40_log_lli_bidir *lli, + struct d40_def_lcsp *lcsp, + u32 src_data_width, + u32 dst_data_width, + enum dma_data_direction direction, + bool term_int, dma_addr_t dev_addr, int max_len, + int llis_per_log); + +void d40_log_lli_write(struct d40_log_lli_full *lcpa, + struct d40_log_lli *lcla_src, + struct d40_log_lli *lcla_dst, + struct d40_log_lli *lli_dst, + struct d40_log_lli *lli_src, + int llis_per_log); + +int d40_log_sg_to_lli(int lcla_id, + struct scatterlist *sg, + int sg_len, + struct d40_log_lli *lli_sg, + u32 lcsp13, /* src or dst*/ + u32 data_width, + bool term_int, int max_len, int llis_per_log); + +#endif /* STE_DMA40_LLI_H */ diff --git a/drivers/dma/timb_dma.c b/drivers/dma/timb_dma.c new file mode 100644 index 000000000000..a1bf77c1993f --- /dev/null +++ b/drivers/dma/timb_dma.c @@ -0,0 +1,860 @@ +/* + * timb_dma.c timberdale FPGA DMA driver + * Copyright (c) 2010 Intel Corporation + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. + */ + +/* Supports: + * Timberdale FPGA DMA engine + */ + +#include <linux/dmaengine.h> +#include <linux/dma-mapping.h> +#include <linux/init.h> +#include <linux/interrupt.h> +#include <linux/io.h> +#include <linux/module.h> +#include <linux/platform_device.h> +#include <linux/slab.h> + +#include <linux/timb_dma.h> + +#define DRIVER_NAME "timb-dma" + +/* Global DMA registers */ +#define TIMBDMA_ACR 0x34 +#define TIMBDMA_32BIT_ADDR 0x01 + +#define TIMBDMA_ISR 0x080000 +#define TIMBDMA_IPR 0x080004 +#define TIMBDMA_IER 0x080008 + +/* Channel specific registers */ +/* RX instances base addresses are 0x00, 0x40, 0x80 ... + * TX instances base addresses are 0x18, 0x58, 0x98 ... + */ +#define TIMBDMA_INSTANCE_OFFSET 0x40 +#define TIMBDMA_INSTANCE_TX_OFFSET 0x18 + +/* RX registers, relative the instance base */ +#define TIMBDMA_OFFS_RX_DHAR 0x00 +#define TIMBDMA_OFFS_RX_DLAR 0x04 +#define TIMBDMA_OFFS_RX_LR 0x0C +#define TIMBDMA_OFFS_RX_BLR 0x10 +#define TIMBDMA_OFFS_RX_ER 0x14 +#define TIMBDMA_RX_EN 0x01 +/* bytes per Row, video specific register + * which is placed after the TX registers... + */ +#define TIMBDMA_OFFS_RX_BPRR 0x30 + +/* TX registers, relative the instance base */ +#define TIMBDMA_OFFS_TX_DHAR 0x00 +#define TIMBDMA_OFFS_TX_DLAR 0x04 +#define TIMBDMA_OFFS_TX_BLR 0x0C +#define TIMBDMA_OFFS_TX_LR 0x14 + + +#define TIMB_DMA_DESC_SIZE 8 + +struct timb_dma_desc { + struct list_head desc_node; + struct dma_async_tx_descriptor txd; + u8 *desc_list; + unsigned int desc_list_len; + bool interrupt; +}; + +struct timb_dma_chan { + struct dma_chan chan; + void __iomem *membase; + spinlock_t lock; /* Used to protect data structures, + especially the lists and descriptors, + from races between the tasklet and calls + from above */ + dma_cookie_t last_completed_cookie; + bool ongoing; + struct list_head active_list; + struct list_head queue; + struct list_head free_list; + unsigned int bytes_per_line; + enum dma_data_direction direction; + unsigned int descs; /* Descriptors to allocate */ + unsigned int desc_elems; /* number of elems per descriptor */ +}; + +struct timb_dma { + struct dma_device dma; + void __iomem *membase; + struct tasklet_struct tasklet; + struct timb_dma_chan channels[0]; +}; + +static struct device *chan2dev(struct dma_chan *chan) +{ + return &chan->dev->device; +} +static struct device *chan2dmadev(struct dma_chan *chan) +{ + return chan2dev(chan)->parent->parent; +} + +static struct timb_dma *tdchantotd(struct timb_dma_chan *td_chan) +{ + int id = td_chan->chan.chan_id; + return (struct timb_dma *)((u8 *)td_chan - + id * sizeof(struct timb_dma_chan) - sizeof(struct timb_dma)); +} + +/* Must be called with the spinlock held */ +static void __td_enable_chan_irq(struct timb_dma_chan *td_chan) +{ + int id = td_chan->chan.chan_id; + struct timb_dma *td = tdchantotd(td_chan); + u32 ier; + + /* enable interrupt for this channel */ + ier = ioread32(td->membase + TIMBDMA_IER); + ier |= 1 << id; + dev_dbg(chan2dev(&td_chan->chan), "Enabling irq: %d, IER: 0x%x\n", id, + ier); + iowrite32(ier, td->membase + TIMBDMA_IER); +} + +/* Should be called with the spinlock held */ +static bool __td_dma_done_ack(struct timb_dma_chan *td_chan) +{ + int id = td_chan->chan.chan_id; + struct timb_dma *td = (struct timb_dma *)((u8 *)td_chan - + id * sizeof(struct timb_dma_chan) - sizeof(struct timb_dma)); + u32 isr; + bool done = false; + + dev_dbg(chan2dev(&td_chan->chan), "Checking irq: %d, td: %p\n", id, td); + + isr = ioread32(td->membase + TIMBDMA_ISR) & (1 << id); + if (isr) { + iowrite32(isr, td->membase + TIMBDMA_ISR); + done = true; + } + + return done; +} + +static void __td_unmap_desc(struct timb_dma_chan *td_chan, const u8 *dma_desc, + bool single) +{ + dma_addr_t addr; + int len; + + addr = (dma_desc[7] << 24) | (dma_desc[6] << 16) | (dma_desc[5] << 8) | + dma_desc[4]; + + len = (dma_desc[3] << 8) | dma_desc[2]; + + if (single) + dma_unmap_single(chan2dev(&td_chan->chan), addr, len, + td_chan->direction); + else + dma_unmap_page(chan2dev(&td_chan->chan), addr, len, + td_chan->direction); +} + +static void __td_unmap_descs(struct timb_dma_desc *td_desc, bool single) +{ + struct timb_dma_chan *td_chan = container_of(td_desc->txd.chan, + struct timb_dma_chan, chan); + u8 *descs; + + for (descs = td_desc->desc_list; ; descs += TIMB_DMA_DESC_SIZE) { + __td_unmap_desc(td_chan, descs, single); + if (descs[0] & 0x02) + break; + } +} + +static int td_fill_desc(struct timb_dma_chan *td_chan, u8 *dma_desc, + struct scatterlist *sg, bool last) +{ + if (sg_dma_len(sg) > USHRT_MAX) { + dev_err(chan2dev(&td_chan->chan), "Too big sg element\n"); + return -EINVAL; + } + + /* length must be word aligned */ + if (sg_dma_len(sg) % sizeof(u32)) { + dev_err(chan2dev(&td_chan->chan), "Incorrect length: %d\n", + sg_dma_len(sg)); + return -EINVAL; + } + + dev_dbg(chan2dev(&td_chan->chan), "desc: %p, addr: %p\n", + dma_desc, (void *)sg_dma_address(sg)); + + dma_desc[7] = (sg_dma_address(sg) >> 24) & 0xff; + dma_desc[6] = (sg_dma_address(sg) >> 16) & 0xff; + dma_desc[5] = (sg_dma_address(sg) >> 8) & 0xff; + dma_desc[4] = (sg_dma_address(sg) >> 0) & 0xff; + + dma_desc[3] = (sg_dma_len(sg) >> 8) & 0xff; + dma_desc[2] = (sg_dma_len(sg) >> 0) & 0xff; + + dma_desc[1] = 0x00; + dma_desc[0] = 0x21 | (last ? 0x02 : 0); /* tran, valid */ + + return 0; +} + +/* Must be called with the spinlock held */ +static void __td_start_dma(struct timb_dma_chan *td_chan) +{ + struct timb_dma_desc *td_desc; + + if (td_chan->ongoing) { + dev_err(chan2dev(&td_chan->chan), + "Transfer already ongoing\n"); + return; + } + + td_desc = list_entry(td_chan->active_list.next, struct timb_dma_desc, + desc_node); + + dev_dbg(chan2dev(&td_chan->chan), + "td_chan: %p, chan: %d, membase: %p\n", + td_chan, td_chan->chan.chan_id, td_chan->membase); + + if (td_chan->direction == DMA_FROM_DEVICE) { + + /* descriptor address */ + iowrite32(0, td_chan->membase + TIMBDMA_OFFS_RX_DHAR); + iowrite32(td_desc->txd.phys, td_chan->membase + + TIMBDMA_OFFS_RX_DLAR); + /* Bytes per line */ + iowrite32(td_chan->bytes_per_line, td_chan->membase + + TIMBDMA_OFFS_RX_BPRR); + /* enable RX */ + iowrite32(TIMBDMA_RX_EN, td_chan->membase + TIMBDMA_OFFS_RX_ER); + } else { + /* address high */ + iowrite32(0, td_chan->membase + TIMBDMA_OFFS_TX_DHAR); + iowrite32(td_desc->txd.phys, td_chan->membase + + TIMBDMA_OFFS_TX_DLAR); + } + + td_chan->ongoing = true; + + if (td_desc->interrupt) + __td_enable_chan_irq(td_chan); +} + +static void __td_finish(struct timb_dma_chan *td_chan) +{ + dma_async_tx_callback callback; + void *param; + struct dma_async_tx_descriptor *txd; + struct timb_dma_desc *td_desc; + + /* can happen if the descriptor is canceled */ + if (list_empty(&td_chan->active_list)) + return; + + td_desc = list_entry(td_chan->active_list.next, struct timb_dma_desc, + desc_node); + txd = &td_desc->txd; + + dev_dbg(chan2dev(&td_chan->chan), "descriptor %u complete\n", + txd->cookie); + + /* make sure to stop the transfer */ + if (td_chan->direction == DMA_FROM_DEVICE) + iowrite32(0, td_chan->membase + TIMBDMA_OFFS_RX_ER); +/* Currently no support for stopping DMA transfers + else + iowrite32(0, td_chan->membase + TIMBDMA_OFFS_TX_DLAR); +*/ + td_chan->last_completed_cookie = txd->cookie; + td_chan->ongoing = false; + + callback = txd->callback; + param = txd->callback_param; + + list_move(&td_desc->desc_node, &td_chan->free_list); + + if (!(txd->flags & DMA_COMPL_SKIP_SRC_UNMAP)) + __td_unmap_descs(td_desc, + txd->flags & DMA_COMPL_SRC_UNMAP_SINGLE); + + /* + * The API requires that no submissions are done from a + * callback, so we don't need to drop the lock here + */ + if (callback) + callback(param); +} + +static u32 __td_ier_mask(struct timb_dma *td) +{ + int i; + u32 ret = 0; + + for (i = 0; i < td->dma.chancnt; i++) { + struct timb_dma_chan *td_chan = td->channels + i; + if (td_chan->ongoing) { + struct timb_dma_desc *td_desc = + list_entry(td_chan->active_list.next, + struct timb_dma_desc, desc_node); + if (td_desc->interrupt) + ret |= 1 << i; + } + } + + return ret; +} + +static void __td_start_next(struct timb_dma_chan *td_chan) +{ + struct timb_dma_desc *td_desc; + + BUG_ON(list_empty(&td_chan->queue)); + BUG_ON(td_chan->ongoing); + + td_desc = list_entry(td_chan->queue.next, struct timb_dma_desc, + desc_node); + + dev_dbg(chan2dev(&td_chan->chan), "%s: started %u\n", + __func__, td_desc->txd.cookie); + + list_move(&td_desc->desc_node, &td_chan->active_list); + __td_start_dma(td_chan); +} + +static dma_cookie_t td_tx_submit(struct dma_async_tx_descriptor *txd) +{ + struct timb_dma_desc *td_desc = container_of(txd, struct timb_dma_desc, + txd); + struct timb_dma_chan *td_chan = container_of(txd->chan, + struct timb_dma_chan, chan); + dma_cookie_t cookie; + + spin_lock_bh(&td_chan->lock); + + cookie = txd->chan->cookie; + if (++cookie < 0) + cookie = 1; + txd->chan->cookie = cookie; + txd->cookie = cookie; + + if (list_empty(&td_chan->active_list)) { + dev_dbg(chan2dev(txd->chan), "%s: started %u\n", __func__, + txd->cookie); + list_add_tail(&td_desc->desc_node, &td_chan->active_list); + __td_start_dma(td_chan); + } else { + dev_dbg(chan2dev(txd->chan), "tx_submit: queued %u\n", + txd->cookie); + + list_add_tail(&td_desc->desc_node, &td_chan->queue); + } + + spin_unlock_bh(&td_chan->lock); + + return cookie; +} + +static struct timb_dma_desc *td_alloc_init_desc(struct timb_dma_chan *td_chan) +{ + struct dma_chan *chan = &td_chan->chan; + struct timb_dma_desc *td_desc; + int err; + + td_desc = kzalloc(sizeof(struct timb_dma_desc), GFP_KERNEL); + if (!td_desc) { + dev_err(chan2dev(chan), "Failed to alloc descriptor\n"); + goto err; + } + + td_desc->desc_list_len = td_chan->desc_elems * TIMB_DMA_DESC_SIZE; + + td_desc->desc_list = kzalloc(td_desc->desc_list_len, GFP_KERNEL); + if (!td_desc->desc_list) { + dev_err(chan2dev(chan), "Failed to alloc descriptor\n"); + goto err; + } + + dma_async_tx_descriptor_init(&td_desc->txd, chan); + td_desc->txd.tx_submit = td_tx_submit; + td_desc->txd.flags = DMA_CTRL_ACK; + + td_desc->txd.phys = dma_map_single(chan2dmadev(chan), + td_desc->desc_list, td_desc->desc_list_len, DMA_TO_DEVICE); + + err = dma_mapping_error(chan2dmadev(chan), td_desc->txd.phys); + if (err) { + dev_err(chan2dev(chan), "DMA mapping error: %d\n", err); + goto err; + } + + return td_desc; +err: + kfree(td_desc->desc_list); + kfree(td_desc); + + return NULL; + +} + +static void td_free_desc(struct timb_dma_desc *td_desc) +{ + dev_dbg(chan2dev(td_desc->txd.chan), "Freeing desc: %p\n", td_desc); + dma_unmap_single(chan2dmadev(td_desc->txd.chan), td_desc->txd.phys, + td_desc->desc_list_len, DMA_TO_DEVICE); + + kfree(td_desc->desc_list); + kfree(td_desc); +} + +static void td_desc_put(struct timb_dma_chan *td_chan, + struct timb_dma_desc *td_desc) +{ + dev_dbg(chan2dev(&td_chan->chan), "Putting desc: %p\n", td_desc); + + spin_lock_bh(&td_chan->lock); + list_add(&td_desc->desc_node, &td_chan->free_list); + spin_unlock_bh(&td_chan->lock); +} + +static struct timb_dma_desc *td_desc_get(struct timb_dma_chan *td_chan) +{ + struct timb_dma_desc *td_desc, *_td_desc; + struct timb_dma_desc *ret = NULL; + + spin_lock_bh(&td_chan->lock); + list_for_each_entry_safe(td_desc, _td_desc, &td_chan->free_list, + desc_node) { + if (async_tx_test_ack(&td_desc->txd)) { + list_del(&td_desc->desc_node); + ret = td_desc; + break; + } + dev_dbg(chan2dev(&td_chan->chan), "desc %p not ACKed\n", + td_desc); + } + spin_unlock_bh(&td_chan->lock); + + return ret; +} + +static int td_alloc_chan_resources(struct dma_chan *chan) +{ + struct timb_dma_chan *td_chan = + container_of(chan, struct timb_dma_chan, chan); + int i; + + dev_dbg(chan2dev(chan), "%s: entry\n", __func__); + + BUG_ON(!list_empty(&td_chan->free_list)); + for (i = 0; i < td_chan->descs; i++) { + struct timb_dma_desc *td_desc = td_alloc_init_desc(td_chan); + if (!td_desc) { + if (i) + break; + else { + dev_err(chan2dev(chan), + "Couldnt allocate any descriptors\n"); + return -ENOMEM; + } + } + + td_desc_put(td_chan, td_desc); + } + + spin_lock_bh(&td_chan->lock); + td_chan->last_completed_cookie = 1; + chan->cookie = 1; + spin_unlock_bh(&td_chan->lock); + + return 0; +} + +static void td_free_chan_resources(struct dma_chan *chan) +{ + struct timb_dma_chan *td_chan = + container_of(chan, struct timb_dma_chan, chan); + struct timb_dma_desc *td_desc, *_td_desc; + LIST_HEAD(list); + + dev_dbg(chan2dev(chan), "%s: Entry\n", __func__); + + /* check that all descriptors are free */ + BUG_ON(!list_empty(&td_chan->active_list)); + BUG_ON(!list_empty(&td_chan->queue)); + + spin_lock_bh(&td_chan->lock); + list_splice_init(&td_chan->free_list, &list); + spin_unlock_bh(&td_chan->lock); + + list_for_each_entry_safe(td_desc, _td_desc, &list, desc_node) { + dev_dbg(chan2dev(chan), "%s: Freeing desc: %p\n", __func__, + td_desc); + td_free_desc(td_desc); + } +} + +static enum dma_status td_tx_status(struct dma_chan *chan, dma_cookie_t cookie, + struct dma_tx_state *txstate) +{ + struct timb_dma_chan *td_chan = + container_of(chan, struct timb_dma_chan, chan); + dma_cookie_t last_used; + dma_cookie_t last_complete; + int ret; + + dev_dbg(chan2dev(chan), "%s: Entry\n", __func__); + + last_complete = td_chan->last_completed_cookie; + last_used = chan->cookie; + + ret = dma_async_is_complete(cookie, last_complete, last_used); + + dma_set_tx_state(txstate, last_complete, last_used, 0); + + dev_dbg(chan2dev(chan), + "%s: exit, ret: %d, last_complete: %d, last_used: %d\n", + __func__, ret, last_complete, last_used); + + return ret; +} + +static void td_issue_pending(struct dma_chan *chan) +{ + struct timb_dma_chan *td_chan = + container_of(chan, struct timb_dma_chan, chan); + + dev_dbg(chan2dev(chan), "%s: Entry\n", __func__); + spin_lock_bh(&td_chan->lock); + + if (!list_empty(&td_chan->active_list)) + /* transfer ongoing */ + if (__td_dma_done_ack(td_chan)) + __td_finish(td_chan); + + if (list_empty(&td_chan->active_list) && !list_empty(&td_chan->queue)) + __td_start_next(td_chan); + + spin_unlock_bh(&td_chan->lock); +} + +static struct dma_async_tx_descriptor *td_prep_slave_sg(struct dma_chan *chan, + struct scatterlist *sgl, unsigned int sg_len, + enum dma_data_direction direction, unsigned long flags) +{ + struct timb_dma_chan *td_chan = + container_of(chan, struct timb_dma_chan, chan); + struct timb_dma_desc *td_desc; + struct scatterlist *sg; + unsigned int i; + unsigned int desc_usage = 0; + + if (!sgl || !sg_len) { + dev_err(chan2dev(chan), "%s: No SG list\n", __func__); + return NULL; + } + + /* even channels are for RX, odd for TX */ + if (td_chan->direction != direction) { + dev_err(chan2dev(chan), + "Requesting channel in wrong direction\n"); + return NULL; + } + + td_desc = td_desc_get(td_chan); + if (!td_desc) { + dev_err(chan2dev(chan), "Not enough descriptors available\n"); + return NULL; + } + + td_desc->interrupt = (flags & DMA_PREP_INTERRUPT) != 0; + + for_each_sg(sgl, sg, sg_len, i) { + int err; + if (desc_usage > td_desc->desc_list_len) { + dev_err(chan2dev(chan), "No descriptor space\n"); + return NULL; + } + + err = td_fill_desc(td_chan, td_desc->desc_list + desc_usage, sg, + i == (sg_len - 1)); + if (err) { + dev_err(chan2dev(chan), "Failed to update desc: %d\n", + err); + td_desc_put(td_chan, td_desc); + return NULL; + } + desc_usage += TIMB_DMA_DESC_SIZE; + } + + dma_sync_single_for_device(chan2dmadev(chan), td_desc->txd.phys, + td_desc->desc_list_len, DMA_TO_DEVICE); + + return &td_desc->txd; +} + +static int td_control(struct dma_chan *chan, enum dma_ctrl_cmd cmd, + unsigned long arg) +{ + struct timb_dma_chan *td_chan = + container_of(chan, struct timb_dma_chan, chan); + struct timb_dma_desc *td_desc, *_td_desc; + + dev_dbg(chan2dev(chan), "%s: Entry\n", __func__); + + if (cmd != DMA_TERMINATE_ALL) + return -ENXIO; + + /* first the easy part, put the queue into the free list */ + spin_lock_bh(&td_chan->lock); + list_for_each_entry_safe(td_desc, _td_desc, &td_chan->queue, + desc_node) + list_move(&td_desc->desc_node, &td_chan->free_list); + + /* now tear down the runnning */ + __td_finish(td_chan); + spin_unlock_bh(&td_chan->lock); + + return 0; +} + +static void td_tasklet(unsigned long data) +{ + struct timb_dma *td = (struct timb_dma *)data; + u32 isr; + u32 ipr; + u32 ier; + int i; + + isr = ioread32(td->membase + TIMBDMA_ISR); + ipr = isr & __td_ier_mask(td); + + /* ack the interrupts */ + iowrite32(ipr, td->membase + TIMBDMA_ISR); + + for (i = 0; i < td->dma.chancnt; i++) + if (ipr & (1 << i)) { + struct timb_dma_chan *td_chan = td->channels + i; + spin_lock(&td_chan->lock); + __td_finish(td_chan); + if (!list_empty(&td_chan->queue)) + __td_start_next(td_chan); + spin_unlock(&td_chan->lock); + } + + ier = __td_ier_mask(td); + iowrite32(ier, td->membase + TIMBDMA_IER); +} + + +static irqreturn_t td_irq(int irq, void *devid) +{ + struct timb_dma *td = devid; + u32 ipr = ioread32(td->membase + TIMBDMA_IPR); + + if (ipr) { + /* disable interrupts, will be re-enabled in tasklet */ + iowrite32(0, td->membase + TIMBDMA_IER); + + tasklet_schedule(&td->tasklet); + + return IRQ_HANDLED; + } else + return IRQ_NONE; +} + + +static int __devinit td_probe(struct platform_device *pdev) +{ + struct timb_dma_platform_data *pdata = pdev->dev.platform_data; + struct timb_dma *td; + struct resource *iomem; + int irq; + int err; + int i; + + if (!pdata) { + dev_err(&pdev->dev, "No platform data\n"); + return -EINVAL; + } + + iomem = platform_get_resource(pdev, IORESOURCE_MEM, 0); + if (!iomem) + return -EINVAL; + + irq = platform_get_irq(pdev, 0); + if (irq < 0) + return irq; + + if (!request_mem_region(iomem->start, resource_size(iomem), + DRIVER_NAME)) + return -EBUSY; + + td = kzalloc(sizeof(struct timb_dma) + + sizeof(struct timb_dma_chan) * pdata->nr_channels, GFP_KERNEL); + if (!td) { + err = -ENOMEM; + goto err_release_region; + } + + dev_dbg(&pdev->dev, "Allocated TD: %p\n", td); + + td->membase = ioremap(iomem->start, resource_size(iomem)); + if (!td->membase) { + dev_err(&pdev->dev, "Failed to remap I/O memory\n"); + err = -ENOMEM; + goto err_free_mem; + } + + /* 32bit addressing */ + iowrite32(TIMBDMA_32BIT_ADDR, td->membase + TIMBDMA_ACR); + + /* disable and clear any interrupts */ + iowrite32(0x0, td->membase + TIMBDMA_IER); + iowrite32(0xFFFFFFFF, td->membase + TIMBDMA_ISR); + + tasklet_init(&td->tasklet, td_tasklet, (unsigned long)td); + + err = request_irq(irq, td_irq, IRQF_SHARED, DRIVER_NAME, td); + if (err) { + dev_err(&pdev->dev, "Failed to request IRQ\n"); + goto err_tasklet_kill; + } + + td->dma.device_alloc_chan_resources = td_alloc_chan_resources; + td->dma.device_free_chan_resources = td_free_chan_resources; + td->dma.device_tx_status = td_tx_status; + td->dma.device_issue_pending = td_issue_pending; + + dma_cap_set(DMA_SLAVE, td->dma.cap_mask); + dma_cap_set(DMA_PRIVATE, td->dma.cap_mask); + td->dma.device_prep_slave_sg = td_prep_slave_sg; + td->dma.device_control = td_control; + + td->dma.dev = &pdev->dev; + + INIT_LIST_HEAD(&td->dma.channels); + + for (i = 0; i < pdata->nr_channels; i++, td->dma.chancnt++) { + struct timb_dma_chan *td_chan = &td->channels[i]; + struct timb_dma_platform_data_channel *pchan = + pdata->channels + i; + + /* even channels are RX, odd are TX */ + if (((i % 2) && pchan->rx) || (!(i % 2) && !pchan->rx)) { + dev_err(&pdev->dev, "Wrong channel configuration\n"); + err = -EINVAL; + goto err_tasklet_kill; + } + + td_chan->chan.device = &td->dma; + td_chan->chan.cookie = 1; + td_chan->chan.chan_id = i; + spin_lock_init(&td_chan->lock); + INIT_LIST_HEAD(&td_chan->active_list); + INIT_LIST_HEAD(&td_chan->queue); + INIT_LIST_HEAD(&td_chan->free_list); + + td_chan->descs = pchan->descriptors; + td_chan->desc_elems = pchan->descriptor_elements; + td_chan->bytes_per_line = pchan->bytes_per_line; + td_chan->direction = pchan->rx ? DMA_FROM_DEVICE : + DMA_TO_DEVICE; + + td_chan->membase = td->membase + + (i / 2) * TIMBDMA_INSTANCE_OFFSET + + (pchan->rx ? 0 : TIMBDMA_INSTANCE_TX_OFFSET); + + dev_dbg(&pdev->dev, "Chan: %d, membase: %p\n", + i, td_chan->membase); + + list_add_tail(&td_chan->chan.device_node, &td->dma.channels); + } + + err = dma_async_device_register(&td->dma); + if (err) { + dev_err(&pdev->dev, "Failed to register async device\n"); + goto err_free_irq; + } + + platform_set_drvdata(pdev, td); + + dev_dbg(&pdev->dev, "Probe result: %d\n", err); + return err; + +err_free_irq: + free_irq(irq, td); +err_tasklet_kill: + tasklet_kill(&td->tasklet); + iounmap(td->membase); +err_free_mem: + kfree(td); +err_release_region: + release_mem_region(iomem->start, resource_size(iomem)); + + return err; + +} + +static int __devexit td_remove(struct platform_device *pdev) +{ + struct timb_dma *td = platform_get_drvdata(pdev); + struct resource *iomem = platform_get_resource(pdev, IORESOURCE_MEM, 0); + int irq = platform_get_irq(pdev, 0); + + dma_async_device_unregister(&td->dma); + free_irq(irq, td); + tasklet_kill(&td->tasklet); + iounmap(td->membase); + kfree(td); + release_mem_region(iomem->start, resource_size(iomem)); + + platform_set_drvdata(pdev, NULL); + + dev_dbg(&pdev->dev, "Removed...\n"); + return 0; +} + +static struct platform_driver td_driver = { + .driver = { + .name = DRIVER_NAME, + .owner = THIS_MODULE, + }, + .probe = td_probe, + .remove = __exit_p(td_remove), +}; + +static int __init td_init(void) +{ + return platform_driver_register(&td_driver); +} +module_init(td_init); + +static void __exit td_exit(void) +{ + platform_driver_unregister(&td_driver); +} +module_exit(td_exit); + +MODULE_LICENSE("GPL v2"); +MODULE_DESCRIPTION("Timberdale DMA controller driver"); +MODULE_AUTHOR("Pelagicore AB <info@pelagicore.com>"); +MODULE_ALIAS("platform:"DRIVER_NAME); diff --git a/drivers/dma/txx9dmac.c b/drivers/dma/txx9dmac.c index 3ebc61067e54..cbd83e362b5e 100644 --- a/drivers/dma/txx9dmac.c +++ b/drivers/dma/txx9dmac.c @@ -938,12 +938,17 @@ txx9dmac_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl, return &first->txd; } -static void txx9dmac_terminate_all(struct dma_chan *chan) +static int txx9dmac_control(struct dma_chan *chan, enum dma_ctrl_cmd cmd, + unsigned long arg) { struct txx9dmac_chan *dc = to_txx9dmac_chan(chan); struct txx9dmac_desc *desc, *_desc; LIST_HEAD(list); + /* Only supports DMA_TERMINATE_ALL */ + if (cmd != DMA_TERMINATE_ALL) + return -EINVAL; + dev_vdbg(chan2dev(chan), "terminate_all\n"); spin_lock_bh(&dc->lock); @@ -958,12 +963,13 @@ static void txx9dmac_terminate_all(struct dma_chan *chan) /* Flush all pending and queued descriptors */ list_for_each_entry_safe(desc, _desc, &list, desc_node) txx9dmac_descriptor_complete(dc, desc); + + return 0; } static enum dma_status -txx9dmac_is_tx_complete(struct dma_chan *chan, - dma_cookie_t cookie, - dma_cookie_t *done, dma_cookie_t *used) +txx9dmac_tx_status(struct dma_chan *chan, dma_cookie_t cookie, + struct dma_tx_state *txstate) { struct txx9dmac_chan *dc = to_txx9dmac_chan(chan); dma_cookie_t last_used; @@ -985,10 +991,7 @@ txx9dmac_is_tx_complete(struct dma_chan *chan, ret = dma_async_is_complete(cookie, last_complete, last_used); } - if (done) - *done = last_complete; - if (used) - *used = last_used; + dma_set_tx_state(txstate, last_complete, last_used, 0); return ret; } @@ -1153,8 +1156,8 @@ static int __init txx9dmac_chan_probe(struct platform_device *pdev) dc->dma.dev = &pdev->dev; dc->dma.device_alloc_chan_resources = txx9dmac_alloc_chan_resources; dc->dma.device_free_chan_resources = txx9dmac_free_chan_resources; - dc->dma.device_terminate_all = txx9dmac_terminate_all; - dc->dma.device_is_tx_complete = txx9dmac_is_tx_complete; + dc->dma.device_control = txx9dmac_control; + dc->dma.device_tx_status = txx9dmac_tx_status; dc->dma.device_issue_pending = txx9dmac_issue_pending; if (pdata && pdata->memcpy_chan == ch) { dc->dma.device_prep_dma_memcpy = txx9dmac_prep_dma_memcpy; @@ -1359,3 +1362,5 @@ module_exit(txx9dmac_exit); MODULE_LICENSE("GPL"); MODULE_DESCRIPTION("TXx9 DMA Controller driver"); MODULE_AUTHOR("Atsushi Nemoto <anemo@mba.ocn.ne.jp>"); +MODULE_ALIAS("platform:txx9dmac"); +MODULE_ALIAS("platform:txx9dmac-chan"); |