diff options
Diffstat (limited to 'drivers/dma')
47 files changed, 2381 insertions, 489 deletions
diff --git a/drivers/dma/Kconfig b/drivers/dma/Kconfig index 141aefbe37ec..d01d59812cf3 100644 --- a/drivers/dma/Kconfig +++ b/drivers/dma/Kconfig @@ -157,7 +157,7 @@ config DMA_SUN4I config DMA_SUN6I tristate "Allwinner A31 SoCs DMA support" - depends on MACH_SUN6I || MACH_SUN8I || COMPILE_TEST + depends on MACH_SUN6I || MACH_SUN8I || (ARM64 && ARCH_SUNXI) || COMPILE_TEST depends on RESET_CONTROLLER select DMA_ENGINE select DMA_VIRTUAL_CHANNELS @@ -436,6 +436,20 @@ config STE_DMA40 help Support for ST-Ericsson DMA40 controller +config ST_FDMA + tristate "ST FDMA dmaengine support" + depends on ARCH_STI + depends on REMOTEPROC + select ST_SLIM_REMOTEPROC + select DMA_ENGINE + select DMA_VIRTUAL_CHANNELS + help + Enable support for ST FDMA controller. + It supports 16 independent DMA channels, accepts up to 32 DMA requests + + Say Y here if you have such a chipset. + If unsure, say N. + config STM32_DMA bool "STMicroelectronics STM32 DMA support" depends on ARCH_STM32 || COMPILE_TEST @@ -444,7 +458,7 @@ config STM32_DMA help Enable support for the on-chip DMA controller on STMicroelectronics STM32 MCUs. - If you have a board based on such a MCU and wish to use DMA say Y or M + If you have a board based on such a MCU and wish to use DMA say Y here. config S3C24XX_DMAC @@ -480,7 +494,7 @@ config TEGRA20_APB_DMA or vice versa. It does not support memory to memory data transfer. config TEGRA210_ADMA - bool "NVIDIA Tegra210 ADMA support" + tristate "NVIDIA Tegra210 ADMA support" depends on (ARCH_TEGRA_210_SOC || COMPILE_TEST) && PM_CLK select DMA_ENGINE select DMA_VIRTUAL_CHANNELS @@ -557,12 +571,12 @@ config XILINX_ZYNQMP_DMA Enable support for Xilinx ZynqMP DMA controller. config ZX_DMA - tristate "ZTE ZX296702 DMA support" + tristate "ZTE ZX DMA support" depends on ARCH_ZX || COMPILE_TEST select DMA_ENGINE select DMA_VIRTUAL_CHANNELS help - Support the DMA engine for ZTE ZX296702 platform devices. + Support the DMA engine for ZTE ZX family platform devices. # driver files diff --git a/drivers/dma/Makefile b/drivers/dma/Makefile index e4dc9cac7ee8..0b723e94d9e6 100644 --- a/drivers/dma/Makefile +++ b/drivers/dma/Makefile @@ -66,7 +66,8 @@ obj-$(CONFIG_TI_CPPI41) += cppi41.o obj-$(CONFIG_TI_DMA_CROSSBAR) += ti-dma-crossbar.o obj-$(CONFIG_TI_EDMA) += edma.o obj-$(CONFIG_XGENE_DMA) += xgene-dma.o -obj-$(CONFIG_ZX_DMA) += zx296702_dma.o +obj-$(CONFIG_ZX_DMA) += zx_dma.o +obj-$(CONFIG_ST_FDMA) += st_fdma.o obj-y += qcom/ obj-y += xilinx/ diff --git a/drivers/dma/amba-pl08x.c b/drivers/dma/amba-pl08x.c index 939a7c31f760..0b7c6ce629a6 100644 --- a/drivers/dma/amba-pl08x.c +++ b/drivers/dma/amba-pl08x.c @@ -1793,6 +1793,13 @@ bool pl08x_filter_id(struct dma_chan *chan, void *chan_id) } EXPORT_SYMBOL_GPL(pl08x_filter_id); +static bool pl08x_filter_fn(struct dma_chan *chan, void *chan_id) +{ + struct pl08x_dma_chan *plchan = to_pl08x_chan(chan); + + return plchan->cd == chan_id; +} + /* * Just check that the device is there and active * TODO: turn this bit on/off depending on the number of physical channels @@ -2307,6 +2314,10 @@ static int pl08x_probe(struct amba_device *adev, const struct amba_id *id) ret = -EINVAL; goto out_no_platdata; } + } else { + pl08x->slave.filter.map = pl08x->pd->slave_map; + pl08x->slave.filter.mapcnt = pl08x->pd->slave_map_len; + pl08x->slave.filter.fn = pl08x_filter_fn; } /* By default, AHB1 only. If dualmaster, from platform */ diff --git a/drivers/dma/at_hdmac.c b/drivers/dma/at_hdmac.c index a4c8f80db29d..1baf3404a365 100644 --- a/drivers/dma/at_hdmac.c +++ b/drivers/dma/at_hdmac.c @@ -111,9 +111,8 @@ static struct at_desc *atc_alloc_descriptor(struct dma_chan *chan, struct at_dma *atdma = to_at_dma(chan->device); dma_addr_t phys; - desc = dma_pool_alloc(atdma->dma_desc_pool, gfp_flags, &phys); + desc = dma_pool_zalloc(atdma->dma_desc_pool, gfp_flags, &phys); if (desc) { - memset(desc, 0, sizeof(struct at_desc)); INIT_LIST_HEAD(&desc->tx_list); dma_async_tx_descriptor_init(&desc->txd, chan); /* txd.flags will be overwritten in prep functions */ diff --git a/drivers/dma/at_xdmac.c b/drivers/dma/at_xdmac.c index b7d7f2d443a1..7d4e0bcda9af 100644 --- a/drivers/dma/at_xdmac.c +++ b/drivers/dma/at_xdmac.c @@ -221,7 +221,6 @@ struct at_xdmac { int irq; struct clk *clk; u32 save_gim; - u32 save_gs; struct dma_pool *at_xdmac_desc_pool; struct at_xdmac_chan chan[0]; }; @@ -444,9 +443,8 @@ static struct at_xdmac_desc *at_xdmac_alloc_desc(struct dma_chan *chan, struct at_xdmac *atxdmac = to_at_xdmac(chan->device); dma_addr_t phys; - desc = dma_pool_alloc(atxdmac->at_xdmac_desc_pool, gfp_flags, &phys); + desc = dma_pool_zalloc(atxdmac->at_xdmac_desc_pool, gfp_flags, &phys); if (desc) { - memset(desc, 0, sizeof(*desc)); INIT_LIST_HEAD(&desc->descs_list); dma_async_tx_descriptor_init(&desc->tx_dma_desc, chan); desc->tx_dma_desc.tx_submit = at_xdmac_tx_submit; @@ -1896,7 +1894,6 @@ static int atmel_xdmac_resume(struct device *dev) } at_xdmac_write(atxdmac, AT_XDMAC_GIE, atxdmac->save_gim); - at_xdmac_write(atxdmac, AT_XDMAC_GE, atxdmac->save_gs); list_for_each_entry_safe(chan, _chan, &atxdmac->dma.channels, device_node) { atchan = to_at_xdmac_chan(chan); at_xdmac_chan_write(atchan, AT_XDMAC_CC, atchan->save_cc); diff --git a/drivers/dma/cppi41.c b/drivers/dma/cppi41.c index d5ba43a87a68..d74cee077842 100644 --- a/drivers/dma/cppi41.c +++ b/drivers/dma/cppi41.c @@ -79,14 +79,6 @@ #define QMGR_QUEUE_C(n) (0x2008 + (n) * 0x10) #define QMGR_QUEUE_D(n) (0x200c + (n) * 0x10) -/* Glue layer specific */ -/* USBSS / USB AM335x */ -#define USBSS_IRQ_STATUS 0x28 -#define USBSS_IRQ_ENABLER 0x2c -#define USBSS_IRQ_CLEARR 0x30 - -#define USBSS_IRQ_PD_COMP (1 << 2) - /* Packet Descriptor */ #define PD2_ZERO_LENGTH (1 << 19) @@ -153,6 +145,8 @@ struct cppi41_dd { /* context for suspend/resume */ unsigned int dma_tdfdq; + + bool is_suspended; }; #define FIST_COMPLETION_QUEUE 93 @@ -257,6 +251,10 @@ static struct cppi41_channel *desc_to_chan(struct cppi41_dd *cdd, u32 desc) BUG_ON(desc_num >= ALLOC_DECS_NUM); c = cdd->chan_busy[desc_num]; cdd->chan_busy[desc_num] = NULL; + + /* Usecount for chan_busy[], paired with push_desc_queue() */ + pm_runtime_put(cdd->ddev.dev); + return c; } @@ -288,14 +286,8 @@ static irqreturn_t cppi41_irq(int irq, void *data) { struct cppi41_dd *cdd = data; struct cppi41_channel *c; - u32 status; int i; - status = cppi_readl(cdd->usbss_mem + USBSS_IRQ_STATUS); - if (!(status & USBSS_IRQ_PD_COMP)) - return IRQ_NONE; - cppi_writel(status, cdd->usbss_mem + USBSS_IRQ_STATUS); - for (i = QMGR_PENDING_SLOT_Q(FIST_COMPLETION_QUEUE); i < QMGR_NUM_PEND; i++) { u32 val; @@ -317,12 +309,12 @@ static irqreturn_t cppi41_irq(int irq, void *data) while (val) { u32 desc, len; - int error; - error = pm_runtime_get(cdd->ddev.dev); - if (error < 0) - dev_err(cdd->ddev.dev, "%s pm runtime get: %i\n", - __func__, error); + /* + * This should never trigger, see the comments in + * push_desc_queue() + */ + WARN_ON(cdd->is_suspended); q_num = __fls(val); val &= ~(1 << q_num); @@ -343,9 +335,6 @@ static irqreturn_t cppi41_irq(int irq, void *data) c->residue = pd_trans_len(c->desc->pd6) - len; dma_cookie_complete(&c->txd); dmaengine_desc_get_callback_invoke(&c->txd, NULL); - - pm_runtime_mark_last_busy(cdd->ddev.dev); - pm_runtime_put_autosuspend(cdd->ddev.dev); } } return IRQ_HANDLED; @@ -447,6 +436,15 @@ static void push_desc_queue(struct cppi41_channel *c) */ __iowmb(); + /* + * DMA transfers can take at least 200ms to complete with USB mass + * storage connected. To prevent autosuspend timeouts, we must use + * pm_runtime_get/put() when chan_busy[] is modified. This will get + * cleared in desc_to_chan() or cppi41_stop_chan() depending on the + * outcome of the transfer. + */ + pm_runtime_get(cdd->ddev.dev); + desc_phys = lower_32_bits(c->desc_phys); desc_num = (desc_phys - cdd->descs_phys) / sizeof(struct cppi41_desc); WARN_ON(cdd->chan_busy[desc_num]); @@ -457,20 +455,26 @@ static void push_desc_queue(struct cppi41_channel *c) cppi_writel(reg, cdd->qmgr_mem + QMGR_QUEUE_D(c->q_num)); } -static void pending_desc(struct cppi41_channel *c) +/* + * Caller must hold cdd->lock to prevent push_desc_queue() + * getting called out of order. We have both cppi41_dma_issue_pending() + * and cppi41_runtime_resume() call this function. + */ +static void cppi41_run_queue(struct cppi41_dd *cdd) { - struct cppi41_dd *cdd = c->cdd; - unsigned long flags; + struct cppi41_channel *c, *_c; - spin_lock_irqsave(&cdd->lock, flags); - list_add_tail(&c->node, &cdd->pending); - spin_unlock_irqrestore(&cdd->lock, flags); + list_for_each_entry_safe(c, _c, &cdd->pending, node) { + push_desc_queue(c); + list_del(&c->node); + } } static void cppi41_dma_issue_pending(struct dma_chan *chan) { struct cppi41_channel *c = to_cpp41_chan(chan); struct cppi41_dd *cdd = c->cdd; + unsigned long flags; int error; error = pm_runtime_get(cdd->ddev.dev); @@ -482,10 +486,11 @@ static void cppi41_dma_issue_pending(struct dma_chan *chan) return; } - if (likely(pm_runtime_active(cdd->ddev.dev))) - push_desc_queue(c); - else - pending_desc(c); + spin_lock_irqsave(&cdd->lock, flags); + list_add_tail(&c->node, &cdd->pending); + if (!cdd->is_suspended) + cppi41_run_queue(cdd); + spin_unlock_irqrestore(&cdd->lock, flags); pm_runtime_mark_last_busy(cdd->ddev.dev); pm_runtime_put_autosuspend(cdd->ddev.dev); @@ -599,6 +604,7 @@ static void cppi41_compute_td_desc(struct cppi41_desc *d) static int cppi41_tear_down_chan(struct cppi41_channel *c) { + struct dmaengine_result abort_result; struct cppi41_dd *cdd = c->cdd; struct cppi41_desc *td; u32 reg; @@ -682,6 +688,12 @@ static int cppi41_tear_down_chan(struct cppi41_channel *c) c->td_seen = 0; c->td_desc_seen = 0; cppi_writel(0, c->gcr_reg); + + /* Invoke the callback to do the necessary clean-up */ + abort_result.result = DMA_TRANS_ABORTED; + dma_cookie_complete(&c->txd); + dmaengine_desc_get_callback_invoke(&c->txd, &abort_result); + return 0; } @@ -705,6 +717,9 @@ static int cppi41_stop_chan(struct dma_chan *chan) WARN_ON(!cdd->chan_busy[desc_num]); cdd->chan_busy[desc_num] = NULL; + /* Usecount for chan_busy[], paired with push_desc_queue() */ + pm_runtime_put(cdd->ddev.dev); + return 0; } @@ -1044,8 +1059,6 @@ static int cppi41_dma_probe(struct platform_device *pdev) goto err_irq; } - cppi_writel(USBSS_IRQ_PD_COMP, cdd->usbss_mem + USBSS_IRQ_ENABLER); - ret = devm_request_irq(&pdev->dev, irq, glue_info->isr, IRQF_SHARED, dev_name(dev), cdd); if (ret) @@ -1069,7 +1082,6 @@ err_of: dma_async_device_unregister(&cdd->ddev); err_dma_reg: err_irq: - cppi_writel(0, cdd->usbss_mem + USBSS_IRQ_CLEARR); cleanup_chans(cdd); err_chans: deinit_cppi41(dev, cdd); @@ -1097,7 +1109,6 @@ static int cppi41_dma_remove(struct platform_device *pdev) of_dma_controller_free(pdev->dev.of_node); dma_async_device_unregister(&cdd->ddev); - cppi_writel(0, cdd->usbss_mem + USBSS_IRQ_CLEARR); devm_free_irq(&pdev->dev, cdd->irq, cdd); cleanup_chans(cdd); deinit_cppi41(&pdev->dev, cdd); @@ -1116,7 +1127,6 @@ static int __maybe_unused cppi41_suspend(struct device *dev) struct cppi41_dd *cdd = dev_get_drvdata(dev); cdd->dma_tdfdq = cppi_readl(cdd->ctrl_mem + DMA_TDFDQ); - cppi_writel(0, cdd->usbss_mem + USBSS_IRQ_CLEARR); disable_sched(cdd); return 0; @@ -1142,16 +1152,18 @@ static int __maybe_unused cppi41_resume(struct device *dev) cppi_writel(QMGR_SCRATCH_SIZE, cdd->qmgr_mem + QMGR_LRAM_SIZE); cppi_writel(0, cdd->qmgr_mem + QMGR_LRAM1_BASE); - cppi_writel(USBSS_IRQ_PD_COMP, cdd->usbss_mem + USBSS_IRQ_ENABLER); - return 0; } static int __maybe_unused cppi41_runtime_suspend(struct device *dev) { struct cppi41_dd *cdd = dev_get_drvdata(dev); + unsigned long flags; + spin_lock_irqsave(&cdd->lock, flags); + cdd->is_suspended = true; WARN_ON(!list_empty(&cdd->pending)); + spin_unlock_irqrestore(&cdd->lock, flags); return 0; } @@ -1159,14 +1171,11 @@ static int __maybe_unused cppi41_runtime_suspend(struct device *dev) static int __maybe_unused cppi41_runtime_resume(struct device *dev) { struct cppi41_dd *cdd = dev_get_drvdata(dev); - struct cppi41_channel *c, *_c; unsigned long flags; spin_lock_irqsave(&cdd->lock, flags); - list_for_each_entry_safe(c, _c, &cdd->pending, node) { - push_desc_queue(c); - list_del(&c->node); - } + cdd->is_suspended = false; + cppi41_run_queue(cdd); spin_unlock_irqrestore(&cdd->lock, flags); return 0; diff --git a/drivers/dma/dmaengine.c b/drivers/dma/dmaengine.c index 6b535262ac5d..24e0221fd66d 100644 --- a/drivers/dma/dmaengine.c +++ b/drivers/dma/dmaengine.c @@ -65,7 +65,7 @@ #include <linux/mempool.h> static DEFINE_MUTEX(dma_list_mutex); -static DEFINE_IDR(dma_idr); +static DEFINE_IDA(dma_ida); static LIST_HEAD(dma_device_list); static long dmaengine_ref_count; @@ -162,7 +162,7 @@ static void chan_dev_release(struct device *dev) chan_dev = container_of(dev, typeof(*chan_dev), device); if (atomic_dec_and_test(chan_dev->idr_ref)) { mutex_lock(&dma_list_mutex); - idr_remove(&dma_idr, chan_dev->dev_id); + ida_remove(&dma_ida, chan_dev->dev_id); mutex_unlock(&dma_list_mutex); kfree(chan_dev->idr_ref); } @@ -898,14 +898,15 @@ static int get_dma_id(struct dma_device *device) { int rc; - mutex_lock(&dma_list_mutex); - - rc = idr_alloc(&dma_idr, NULL, 0, 0, GFP_KERNEL); - if (rc >= 0) - device->dev_id = rc; + do { + if (!ida_pre_get(&dma_ida, GFP_KERNEL)) + return -ENOMEM; + mutex_lock(&dma_list_mutex); + rc = ida_get_new(&dma_ida, &device->dev_id); + mutex_unlock(&dma_list_mutex); + } while (rc == -EAGAIN); - mutex_unlock(&dma_list_mutex); - return rc < 0 ? rc : 0; + return rc; } /** @@ -1035,7 +1036,7 @@ err_out: /* if we never registered a channel just release the idr */ if (atomic_read(idr_ref) == 0) { mutex_lock(&dma_list_mutex); - idr_remove(&dma_idr, device->dev_id); + ida_remove(&dma_ida, device->dev_id); mutex_unlock(&dma_list_mutex); kfree(idr_ref); return rc; diff --git a/drivers/dma/dmatest.c b/drivers/dma/dmatest.c index cf76fc6149e5..54d581d407aa 100644 --- a/drivers/dma/dmatest.c +++ b/drivers/dma/dmatest.c @@ -16,6 +16,7 @@ #include <linux/freezer.h> #include <linux/init.h> #include <linux/kthread.h> +#include <linux/sched/task.h> #include <linux/module.h> #include <linux/moduleparam.h> #include <linux/random.h> @@ -164,7 +165,9 @@ struct dmatest_thread { struct task_struct *task; struct dma_chan *chan; u8 **srcs; + u8 **usrcs; u8 **dsts; + u8 **udsts; enum dma_transaction_type type; bool done; }; @@ -427,10 +430,11 @@ static int dmatest_func(void *data) int dst_cnt; int i; ktime_t ktime, start, diff; - ktime_t filltime = ktime_set(0, 0); - ktime_t comparetime = ktime_set(0, 0); + ktime_t filltime = 0; + ktime_t comparetime = 0; s64 runtime = 0; unsigned long long total_len = 0; + u8 align = 0; set_freezable(); @@ -441,20 +445,24 @@ static int dmatest_func(void *data) params = &info->params; chan = thread->chan; dev = chan->device; - if (thread->type == DMA_MEMCPY) + if (thread->type == DMA_MEMCPY) { + align = dev->copy_align; src_cnt = dst_cnt = 1; - else if (thread->type == DMA_SG) + } else if (thread->type == DMA_SG) { + align = dev->copy_align; src_cnt = dst_cnt = sg_buffers; - else if (thread->type == DMA_XOR) { + } else if (thread->type == DMA_XOR) { /* force odd to ensure dst = src */ src_cnt = min_odd(params->xor_sources | 1, dev->max_xor); dst_cnt = 1; + align = dev->xor_align; } else if (thread->type == DMA_PQ) { /* force odd to ensure dst = src */ src_cnt = min_odd(params->pq_sources | 1, dma_maxpq(dev, 0)); dst_cnt = 2; + align = dev->pq_align; - pq_coefs = kmalloc(params->pq_sources+1, GFP_KERNEL); + pq_coefs = kmalloc(params->pq_sources + 1, GFP_KERNEL); if (!pq_coefs) goto err_thread_type; @@ -463,23 +471,47 @@ static int dmatest_func(void *data) } else goto err_thread_type; - thread->srcs = kcalloc(src_cnt+1, sizeof(u8 *), GFP_KERNEL); + thread->srcs = kcalloc(src_cnt + 1, sizeof(u8 *), GFP_KERNEL); if (!thread->srcs) goto err_srcs; + + thread->usrcs = kcalloc(src_cnt + 1, sizeof(u8 *), GFP_KERNEL); + if (!thread->usrcs) + goto err_usrcs; + for (i = 0; i < src_cnt; i++) { - thread->srcs[i] = kmalloc(params->buf_size, GFP_KERNEL); - if (!thread->srcs[i]) + thread->usrcs[i] = kmalloc(params->buf_size + align, + GFP_KERNEL); + if (!thread->usrcs[i]) goto err_srcbuf; + + /* align srcs to alignment restriction */ + if (align) + thread->srcs[i] = PTR_ALIGN(thread->usrcs[i], align); + else + thread->srcs[i] = thread->usrcs[i]; } thread->srcs[i] = NULL; - thread->dsts = kcalloc(dst_cnt+1, sizeof(u8 *), GFP_KERNEL); + thread->dsts = kcalloc(dst_cnt + 1, sizeof(u8 *), GFP_KERNEL); if (!thread->dsts) goto err_dsts; + + thread->udsts = kcalloc(dst_cnt + 1, sizeof(u8 *), GFP_KERNEL); + if (!thread->udsts) + goto err_udsts; + for (i = 0; i < dst_cnt; i++) { - thread->dsts[i] = kmalloc(params->buf_size, GFP_KERNEL); - if (!thread->dsts[i]) + thread->udsts[i] = kmalloc(params->buf_size + align, + GFP_KERNEL); + if (!thread->udsts[i]) goto err_dstbuf; + + /* align dsts to alignment restriction */ + if (align) + thread->dsts[i] = PTR_ALIGN(thread->udsts[i], align); + else + thread->dsts[i] = thread->udsts[i]; } thread->dsts[i] = NULL; @@ -498,20 +530,11 @@ static int dmatest_func(void *data) dma_addr_t srcs[src_cnt]; dma_addr_t *dsts; unsigned int src_off, dst_off, len; - u8 align = 0; struct scatterlist tx_sg[src_cnt]; struct scatterlist rx_sg[src_cnt]; total_tests++; - /* honor alignment restrictions */ - if (thread->type == DMA_MEMCPY || thread->type == DMA_SG) - align = dev->copy_align; - else if (thread->type == DMA_XOR) - align = dev->xor_align; - else if (thread->type == DMA_PQ) - align = dev->pq_align; - if (1 << align > params->buf_size) { pr_err("%u-byte buffer too small for %d-byte alignment\n", params->buf_size, 1 << align); @@ -549,7 +572,7 @@ static int dmatest_func(void *data) filltime = ktime_add(filltime, diff); } - um = dmaengine_get_unmap_data(dev->dev, src_cnt+dst_cnt, + um = dmaengine_get_unmap_data(dev->dev, src_cnt + dst_cnt, GFP_KERNEL); if (!um) { failed_tests++; @@ -729,13 +752,17 @@ static int dmatest_func(void *data) ret = 0; err_dstbuf: - for (i = 0; thread->dsts[i]; i++) - kfree(thread->dsts[i]); + for (i = 0; thread->udsts[i]; i++) + kfree(thread->udsts[i]); + kfree(thread->udsts); +err_udsts: kfree(thread->dsts); err_dsts: err_srcbuf: - for (i = 0; thread->srcs[i]; i++) - kfree(thread->srcs[i]); + for (i = 0; thread->usrcs[i]; i++) + kfree(thread->usrcs[i]); + kfree(thread->usrcs); +err_usrcs: kfree(thread->srcs); err_srcs: kfree(pq_coefs); diff --git a/drivers/dma/dw/Kconfig b/drivers/dma/dw/Kconfig index e00c9b022964..5a37b9fcf40d 100644 --- a/drivers/dma/dw/Kconfig +++ b/drivers/dma/dw/Kconfig @@ -24,5 +24,5 @@ config DW_DMAC_PCI select DW_DMAC_CORE help Support the Synopsys DesignWare AHB DMA controller on the - platfroms that enumerate it as a PCI device. For example, + platforms that enumerate it as a PCI device. For example, Intel Medfield has integrated this GPDMA controller. diff --git a/drivers/dma/dw/core.c b/drivers/dma/dw/core.c index c2c0a613cb7a..e500950dad82 100644 --- a/drivers/dma/dw/core.c +++ b/drivers/dma/dw/core.c @@ -138,16 +138,32 @@ static void dwc_desc_put(struct dw_dma_chan *dwc, struct dw_desc *desc) dwc->descs_allocated--; } -static void dwc_initialize(struct dw_dma_chan *dwc) +static void dwc_initialize_chan_idma32(struct dw_dma_chan *dwc) +{ + u32 cfghi = 0; + u32 cfglo = 0; + + /* Set default burst alignment */ + cfglo |= IDMA32C_CFGL_DST_BURST_ALIGN | IDMA32C_CFGL_SRC_BURST_ALIGN; + + /* Low 4 bits of the request lines */ + cfghi |= IDMA32C_CFGH_DST_PER(dwc->dws.dst_id & 0xf); + cfghi |= IDMA32C_CFGH_SRC_PER(dwc->dws.src_id & 0xf); + + /* Request line extension (2 bits) */ + cfghi |= IDMA32C_CFGH_DST_PER_EXT(dwc->dws.dst_id >> 4 & 0x3); + cfghi |= IDMA32C_CFGH_SRC_PER_EXT(dwc->dws.src_id >> 4 & 0x3); + + channel_writel(dwc, CFG_LO, cfglo); + channel_writel(dwc, CFG_HI, cfghi); +} + +static void dwc_initialize_chan_dw(struct dw_dma_chan *dwc) { - struct dw_dma *dw = to_dw_dma(dwc->chan.device); u32 cfghi = DWC_CFGH_FIFO_MODE; u32 cfglo = DWC_CFGL_CH_PRIOR(dwc->priority); bool hs_polarity = dwc->dws.hs_polarity; - if (test_bit(DW_DMA_IS_INITIALIZED, &dwc->flags)) - return; - cfghi |= DWC_CFGH_DST_PER(dwc->dws.dst_id); cfghi |= DWC_CFGH_SRC_PER(dwc->dws.src_id); @@ -156,6 +172,19 @@ static void dwc_initialize(struct dw_dma_chan *dwc) channel_writel(dwc, CFG_LO, cfglo); channel_writel(dwc, CFG_HI, cfghi); +} + +static void dwc_initialize(struct dw_dma_chan *dwc) +{ + struct dw_dma *dw = to_dw_dma(dwc->chan.device); + + if (test_bit(DW_DMA_IS_INITIALIZED, &dwc->flags)) + return; + + if (dw->pdata->is_idma32) + dwc_initialize_chan_idma32(dwc); + else + dwc_initialize_chan_dw(dwc); /* Enable interrupts */ channel_set_bit(dw, MASK.XFER, dwc->mask); @@ -184,6 +213,37 @@ static inline void dwc_chan_disable(struct dw_dma *dw, struct dw_dma_chan *dwc) cpu_relax(); } +static u32 bytes2block(struct dw_dma_chan *dwc, size_t bytes, + unsigned int width, size_t *len) +{ + struct dw_dma *dw = to_dw_dma(dwc->chan.device); + u32 block; + + /* Always in bytes for iDMA 32-bit */ + if (dw->pdata->is_idma32) + width = 0; + + if ((bytes >> width) > dwc->block_size) { + block = dwc->block_size; + *len = block << width; + } else { + block = bytes >> width; + *len = bytes; + } + + return block; +} + +static size_t block2bytes(struct dw_dma_chan *dwc, u32 block, u32 width) +{ + struct dw_dma *dw = to_dw_dma(dwc->chan.device); + + if (dw->pdata->is_idma32) + return IDMA32C_CTLH_BLOCK_TS(block); + + return DWC_CTLH_BLOCK_TS(block) << width; +} + /*----------------------------------------------------------------------*/ /* Perform single block transfer */ @@ -332,7 +392,7 @@ static inline u32 dwc_get_sent(struct dw_dma_chan *dwc) u32 ctlhi = channel_readl(dwc, CTL_HI); u32 ctllo = channel_readl(dwc, CTL_LO); - return (ctlhi & DWC_CTLH_BLOCK_TS_MASK) * (1 << (ctllo >> 4 & 7)); + return block2bytes(dwc, ctlhi, ctllo >> 4 & 7); } static void dwc_scan_descriptors(struct dw_dma *dw, struct dw_dma_chan *dwc) @@ -692,10 +752,7 @@ dwc_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src, | DWC_CTLL_FC_M2M; prev = first = NULL; - for (offset = 0; offset < len; offset += xfer_count << src_width) { - xfer_count = min_t(size_t, (len - offset) >> src_width, - dwc->block_size); - + for (offset = 0; offset < len; offset += xfer_count) { desc = dwc_desc_get(dwc); if (!desc) goto err_desc_get; @@ -703,8 +760,8 @@ dwc_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src, lli_write(desc, sar, src + offset); lli_write(desc, dar, dest + offset); lli_write(desc, ctllo, ctllo); - lli_write(desc, ctlhi, xfer_count); - desc->len = xfer_count << src_width; + lli_write(desc, ctlhi, bytes2block(dwc, len - offset, src_width, &xfer_count)); + desc->len = xfer_count; if (!first) { first = desc; @@ -775,7 +832,8 @@ dwc_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl, for_each_sg(sgl, sg, sg_len, i) { struct dw_desc *desc; - u32 len, dlen, mem; + u32 len, mem; + size_t dlen; mem = sg_dma_address(sg); len = sg_dma_len(sg); @@ -789,17 +847,8 @@ slave_sg_todev_fill_desc: lli_write(desc, sar, mem); lli_write(desc, dar, reg); + lli_write(desc, ctlhi, bytes2block(dwc, len, mem_width, &dlen)); lli_write(desc, ctllo, ctllo | DWC_CTLL_SRC_WIDTH(mem_width)); - if ((len >> mem_width) > dwc->block_size) { - dlen = dwc->block_size << mem_width; - mem += dlen; - len -= dlen; - } else { - dlen = len; - len = 0; - } - - lli_write(desc, ctlhi, dlen >> mem_width); desc->len = dlen; if (!first) { @@ -809,6 +858,9 @@ slave_sg_todev_fill_desc: list_add_tail(&desc->desc_node, &first->tx_list); } prev = desc; + + mem += dlen; + len -= dlen; total_len += dlen; if (len) @@ -828,13 +880,12 @@ slave_sg_todev_fill_desc: for_each_sg(sgl, sg, sg_len, i) { struct dw_desc *desc; - u32 len, dlen, mem; + u32 len, mem; + size_t dlen; mem = sg_dma_address(sg); len = sg_dma_len(sg); - mem_width = __ffs(data_width | mem | len); - slave_sg_fromdev_fill_desc: desc = dwc_desc_get(dwc); if (!desc) @@ -842,16 +893,9 @@ slave_sg_fromdev_fill_desc: lli_write(desc, sar, reg); lli_write(desc, dar, mem); + lli_write(desc, ctlhi, bytes2block(dwc, len, reg_width, &dlen)); + mem_width = __ffs(data_width | mem | dlen); lli_write(desc, ctllo, ctllo | DWC_CTLL_DST_WIDTH(mem_width)); - if ((len >> reg_width) > dwc->block_size) { - dlen = dwc->block_size << reg_width; - mem += dlen; - len -= dlen; - } else { - dlen = len; - len = 0; - } - lli_write(desc, ctlhi, dlen >> reg_width); desc->len = dlen; if (!first) { @@ -861,6 +905,9 @@ slave_sg_fromdev_fill_desc: list_add_tail(&desc->desc_node, &first->tx_list); } prev = desc; + + mem += dlen; + len -= dlen; total_len += dlen; if (len) @@ -903,25 +950,20 @@ bool dw_dma_filter(struct dma_chan *chan, void *param) } EXPORT_SYMBOL_GPL(dw_dma_filter); -/* - * Fix sconfig's burst size according to dw_dmac. We need to convert them as: - * 1 -> 0, 4 -> 1, 8 -> 2, 16 -> 3. - * - * NOTE: burst size 2 is not supported by controller. - * - * This can be done by finding least significant bit set: n & (n - 1) - */ -static inline void convert_burst(u32 *maxburst) -{ - if (*maxburst > 1) - *maxburst = fls(*maxburst) - 2; - else - *maxburst = 0; -} - static int dwc_config(struct dma_chan *chan, struct dma_slave_config *sconfig) { struct dw_dma_chan *dwc = to_dw_dma_chan(chan); + struct dma_slave_config *sc = &dwc->dma_sconfig; + struct dw_dma *dw = to_dw_dma(chan->device); + /* + * Fix sconfig's burst size according to dw_dmac. We need to convert + * them as: + * 1 -> 0, 4 -> 1, 8 -> 2, 16 -> 3. + * + * NOTE: burst size 2 is not supported by DesignWare controller. + * iDMA 32-bit supports it. + */ + u32 s = dw->pdata->is_idma32 ? 1 : 2; /* Check if chan will be configured for slave transfers */ if (!is_slave_direction(sconfig->direction)) @@ -930,28 +972,39 @@ static int dwc_config(struct dma_chan *chan, struct dma_slave_config *sconfig) memcpy(&dwc->dma_sconfig, sconfig, sizeof(*sconfig)); dwc->direction = sconfig->direction; - convert_burst(&dwc->dma_sconfig.src_maxburst); - convert_burst(&dwc->dma_sconfig.dst_maxburst); + sc->src_maxburst = sc->src_maxburst > 1 ? fls(sc->src_maxburst) - s : 0; + sc->dst_maxburst = sc->dst_maxburst > 1 ? fls(sc->dst_maxburst) - s : 0; return 0; } -static int dwc_pause(struct dma_chan *chan) +static void dwc_chan_pause(struct dw_dma_chan *dwc, bool drain) { - struct dw_dma_chan *dwc = to_dw_dma_chan(chan); - unsigned long flags; + struct dw_dma *dw = to_dw_dma(dwc->chan.device); unsigned int count = 20; /* timeout iterations */ u32 cfglo; - spin_lock_irqsave(&dwc->lock, flags); - cfglo = channel_readl(dwc, CFG_LO); + if (dw->pdata->is_idma32) { + if (drain) + cfglo |= IDMA32C_CFGL_CH_DRAIN; + else + cfglo &= ~IDMA32C_CFGL_CH_DRAIN; + } channel_writel(dwc, CFG_LO, cfglo | DWC_CFGL_CH_SUSP); while (!(channel_readl(dwc, CFG_LO) & DWC_CFGL_FIFO_EMPTY) && count--) udelay(2); set_bit(DW_DMA_IS_PAUSED, &dwc->flags); +} +static int dwc_pause(struct dma_chan *chan) +{ + struct dw_dma_chan *dwc = to_dw_dma_chan(chan); + unsigned long flags; + + spin_lock_irqsave(&dwc->lock, flags); + dwc_chan_pause(dwc, false); spin_unlock_irqrestore(&dwc->lock, flags); return 0; @@ -993,6 +1046,8 @@ static int dwc_terminate_all(struct dma_chan *chan) clear_bit(DW_DMA_IS_SOFT_LLP, &dwc->flags); + dwc_chan_pause(dwc, true); + dwc_chan_disable(dw, dwc); dwc_chan_resume(dwc); @@ -1085,6 +1140,32 @@ static void dwc_issue_pending(struct dma_chan *chan) /*----------------------------------------------------------------------*/ +/* + * Program FIFO size of channels. + * + * By default full FIFO (1024 bytes) is assigned to channel 0. Here we + * slice FIFO on equal parts between channels. + */ +static void idma32_fifo_partition(struct dw_dma *dw) +{ + u64 value = IDMA32C_FP_PSIZE_CH0(128) | IDMA32C_FP_PSIZE_CH1(128) | + IDMA32C_FP_UPDATE; + u64 fifo_partition = 0; + + if (!dw->pdata->is_idma32) + return; + + /* Fill FIFO_PARTITION low bits (Channels 0..1, 4..5) */ + fifo_partition |= value << 0; + + /* Fill FIFO_PARTITION high bits (Channels 2..3, 6..7) */ + fifo_partition |= value << 32; + + /* Program FIFO Partition registers - 128 bytes for each channel */ + idma32_writeq(dw, FIFO_PARTITION1, fifo_partition); + idma32_writeq(dw, FIFO_PARTITION0, fifo_partition); +} + static void dw_dma_off(struct dw_dma *dw) { unsigned int i; @@ -1504,8 +1585,16 @@ int dw_dma_probe(struct dw_dma_chip *chip) /* Force dma off, just in case */ dw_dma_off(dw); + idma32_fifo_partition(dw); + + /* Device and instance ID for IRQ and DMA pool */ + if (pdata->is_idma32) + snprintf(dw->name, sizeof(dw->name), "idma32:dmac%d", chip->id); + else + snprintf(dw->name, sizeof(dw->name), "dw:dmac%d", chip->id); + /* Create a pool of consistent memory blocks for hardware descriptors */ - dw->desc_pool = dmam_pool_create("dw_dmac_desc_pool", chip->dev, + dw->desc_pool = dmam_pool_create(dw->name, chip->dev, sizeof(struct dw_desc), 4, 0); if (!dw->desc_pool) { dev_err(chip->dev, "No memory for descriptors dma pool\n"); @@ -1516,7 +1605,7 @@ int dw_dma_probe(struct dw_dma_chip *chip) tasklet_init(&dw->tasklet, dw_dma_tasklet, (unsigned long)dw); err = request_irq(chip->irq, dw_dma_interrupt, IRQF_SHARED, - "dw_dmac", dw); + dw->name, dw); if (err) goto err_pdata; @@ -1569,7 +1658,7 @@ int dw_dma_probe(struct dw_dma_chip *chip) (dwc_params >> DWC_PARAMS_MBLK_EN & 0x1) == 0; } else { dwc->block_size = pdata->block_size; - dwc->nollp = pdata->is_nollp; + dwc->nollp = !pdata->multi_block[i]; } } @@ -1665,6 +1754,8 @@ int dw_dma_enable(struct dw_dma_chip *chip) { struct dw_dma *dw = chip->dw; + idma32_fifo_partition(dw); + dw_dma_on(dw); return 0; } diff --git a/drivers/dma/dw/pci.c b/drivers/dma/dw/pci.c index 0ae6c3b1d34e..7778ed705a1a 100644 --- a/drivers/dma/dw/pci.c +++ b/drivers/dma/dw/pci.c @@ -15,6 +15,18 @@ #include "internal.h" +static struct dw_dma_platform_data mrfld_pdata = { + .nr_channels = 8, + .is_private = true, + .is_memcpy = true, + .is_idma32 = true, + .chan_allocation_order = CHAN_ALLOCATION_ASCENDING, + .chan_priority = CHAN_PRIORITY_ASCENDING, + .block_size = 131071, + .nr_masters = 1, + .data_width = {4}, +}; + static int dw_pci_probe(struct pci_dev *pdev, const struct pci_device_id *pid) { const struct dw_dma_platform_data *pdata = (void *)pid->driver_data; @@ -47,6 +59,7 @@ static int dw_pci_probe(struct pci_dev *pdev, const struct pci_device_id *pid) return -ENOMEM; chip->dev = &pdev->dev; + chip->id = pdev->devfn; chip->regs = pcim_iomap_table(pdev)[0]; chip->irq = pdev->irq; chip->pdata = pdata; @@ -95,14 +108,16 @@ static const struct dev_pm_ops dw_pci_dev_pm_ops = { }; static const struct pci_device_id dw_pci_id_table[] = { - /* Medfield */ + /* Medfield (GPDMA) */ { PCI_VDEVICE(INTEL, 0x0827) }, - { PCI_VDEVICE(INTEL, 0x0830) }, /* BayTrail */ { PCI_VDEVICE(INTEL, 0x0f06) }, { PCI_VDEVICE(INTEL, 0x0f40) }, + /* Merrifield iDMA 32-bit (GPDMA) */ + { PCI_VDEVICE(INTEL, 0x11a2), (kernel_ulong_t)&mrfld_pdata }, + /* Braswell */ { PCI_VDEVICE(INTEL, 0x2286) }, { PCI_VDEVICE(INTEL, 0x22c0) }, diff --git a/drivers/dma/dw/platform.c b/drivers/dma/dw/platform.c index 5bda0eb9f393..c639c60b825a 100644 --- a/drivers/dma/dw/platform.c +++ b/drivers/dma/dw/platform.c @@ -102,7 +102,7 @@ dw_dma_parse_dt(struct platform_device *pdev) { struct device_node *np = pdev->dev.of_node; struct dw_dma_platform_data *pdata; - u32 tmp, arr[DW_DMA_MAX_NR_MASTERS]; + u32 tmp, arr[DW_DMA_MAX_NR_MASTERS], mb[DW_DMA_MAX_NR_CHANNELS]; u32 nr_masters; u32 nr_channels; @@ -118,6 +118,8 @@ dw_dma_parse_dt(struct platform_device *pdev) if (of_property_read_u32(np, "dma-channels", &nr_channels)) return NULL; + if (nr_channels > DW_DMA_MAX_NR_CHANNELS) + return NULL; pdata = devm_kzalloc(&pdev->dev, sizeof(*pdata), GFP_KERNEL); if (!pdata) @@ -129,6 +131,12 @@ dw_dma_parse_dt(struct platform_device *pdev) if (of_property_read_bool(np, "is_private")) pdata->is_private = true; + /* + * All known devices, which use DT for configuration, support + * memory-to-memory transfers. So enable it by default. + */ + pdata->is_memcpy = true; + if (!of_property_read_u32(np, "chan_allocation_order", &tmp)) pdata->chan_allocation_order = (unsigned char)tmp; @@ -146,6 +154,14 @@ dw_dma_parse_dt(struct platform_device *pdev) pdata->data_width[tmp] = BIT(arr[tmp] & 0x07); } + if (!of_property_read_u32_array(np, "multi-block", mb, nr_channels)) { + for (tmp = 0; tmp < nr_channels; tmp++) + pdata->multi_block[tmp] = mb[tmp]; + } else { + for (tmp = 0; tmp < nr_channels; tmp++) + pdata->multi_block[tmp] = 1; + } + return pdata; } #else @@ -186,6 +202,7 @@ static int dw_probe(struct platform_device *pdev) pdata = dw_dma_parse_dt(pdev); chip->dev = dev; + chip->id = pdev->id; chip->pdata = pdata; chip->clk = devm_clk_get(chip->dev, "hclk"); diff --git a/drivers/dma/dw/regs.h b/drivers/dma/dw/regs.h index f65dd104479f..32a328721c88 100644 --- a/drivers/dma/dw/regs.h +++ b/drivers/dma/dw/regs.h @@ -3,16 +3,21 @@ * * Copyright (C) 2005-2007 Atmel Corporation * Copyright (C) 2010-2011 ST Microelectronics + * Copyright (C) 2016 Intel Corporation * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 as * published by the Free Software Foundation. */ +#include <linux/bitops.h> #include <linux/interrupt.h> #include <linux/dmaengine.h> -#define DW_DMA_MAX_NR_CHANNELS 8 +#include <linux/io-64-nonatomic-hi-lo.h> + +#include "internal.h" + #define DW_DMA_MAX_NR_REQUESTS 16 /* flow controller */ @@ -84,9 +89,9 @@ struct dw_dma_regs { DW_REG(ID); DW_REG(TEST); - /* reserved */ - DW_REG(__reserved0); - DW_REG(__reserved1); + /* iDMA 32-bit support */ + DW_REG(CLASS_PRIORITY0); + DW_REG(CLASS_PRIORITY1); /* optional encoded params, 0x3c8..0x3f7 */ u32 __reserved; @@ -98,6 +103,17 @@ struct dw_dma_regs { /* top-level parameters */ u32 DW_PARAMS; + + /* component ID */ + u32 COMP_TYPE; + u32 COMP_VERSION; + + /* iDMA 32-bit support */ + DW_REG(FIFO_PARTITION0); + DW_REG(FIFO_PARTITION1); + + DW_REG(SAI_ERR); + DW_REG(GLOBAL_CFG); }; /* @@ -169,8 +185,9 @@ enum dw_dma_msize { #define DWC_CTLL_LLP_S_EN (1 << 28) /* src block chain */ /* Bitfields in CTL_HI */ -#define DWC_CTLH_DONE 0x00001000 -#define DWC_CTLH_BLOCK_TS_MASK 0x00000fff +#define DWC_CTLH_BLOCK_TS_MASK GENMASK(11, 0) +#define DWC_CTLH_BLOCK_TS(x) ((x) & DWC_CTLH_BLOCK_TS_MASK) +#define DWC_CTLH_DONE (1 << 12) /* Bitfields in CFG_LO */ #define DWC_CFGL_CH_PRIOR_MASK (0x7 << 5) /* priority mask */ @@ -213,6 +230,33 @@ enum dw_dma_msize { /* Bitfields in CFG */ #define DW_CFG_DMA_EN (1 << 0) +/* iDMA 32-bit support */ + +/* Bitfields in CTL_HI */ +#define IDMA32C_CTLH_BLOCK_TS_MASK GENMASK(16, 0) +#define IDMA32C_CTLH_BLOCK_TS(x) ((x) & IDMA32C_CTLH_BLOCK_TS_MASK) +#define IDMA32C_CTLH_DONE (1 << 17) + +/* Bitfields in CFG_LO */ +#define IDMA32C_CFGL_DST_BURST_ALIGN (1 << 0) /* dst burst align */ +#define IDMA32C_CFGL_SRC_BURST_ALIGN (1 << 1) /* src burst align */ +#define IDMA32C_CFGL_CH_DRAIN (1 << 10) /* drain FIFO */ +#define IDMA32C_CFGL_DST_OPT_BL (1 << 20) /* optimize dst burst length */ +#define IDMA32C_CFGL_SRC_OPT_BL (1 << 21) /* optimize src burst length */ + +/* Bitfields in CFG_HI */ +#define IDMA32C_CFGH_SRC_PER(x) ((x) << 0) +#define IDMA32C_CFGH_DST_PER(x) ((x) << 4) +#define IDMA32C_CFGH_RD_ISSUE_THD(x) ((x) << 8) +#define IDMA32C_CFGH_RW_ISSUE_THD(x) ((x) << 18) +#define IDMA32C_CFGH_SRC_PER_EXT(x) ((x) << 28) /* src peripheral extension */ +#define IDMA32C_CFGH_DST_PER_EXT(x) ((x) << 30) /* dst peripheral extension */ + +/* Bitfields in FIFO_PARTITION */ +#define IDMA32C_FP_PSIZE_CH0(x) ((x) << 0) +#define IDMA32C_FP_PSIZE_CH1(x) ((x) << 13) +#define IDMA32C_FP_UPDATE (1 << 26) + enum dw_dmac_flags { DW_DMA_IS_CYCLIC = 0, DW_DMA_IS_SOFT_LLP = 1, @@ -269,6 +313,7 @@ static inline struct dw_dma_chan *to_dw_dma_chan(struct dma_chan *chan) struct dw_dma { struct dma_device dma; + char name[20]; void __iomem *regs; struct dma_pool *desc_pool; struct tasklet_struct tasklet; @@ -292,6 +337,11 @@ static inline struct dw_dma_regs __iomem *__dw_regs(struct dw_dma *dw) #define dma_writel(dw, name, val) \ dma_writel_native((val), &(__dw_regs(dw)->name)) +#define idma32_readq(dw, name) \ + hi_lo_readq(&(__dw_regs(dw)->name)) +#define idma32_writeq(dw, name, val) \ + hi_lo_writeq((val), &(__dw_regs(dw)->name)) + #define channel_set_bit(dw, reg, mask) \ dma_writel(dw, reg, ((mask) << 8) | (mask)) #define channel_clear_bit(dw, reg, mask) \ diff --git a/drivers/dma/edma.c b/drivers/dma/edma.c index 77242b37ef87..3879f80a4815 100644 --- a/drivers/dma/edma.c +++ b/drivers/dma/edma.c @@ -2451,6 +2451,9 @@ static int edma_pm_resume(struct device *dev) int i; s8 (*queue_priority_mapping)[2]; + /* re initialize dummy slot to dummy param set */ + edma_write_slot(ecc, ecc->dummy_slot, &dummy_paramset); + queue_priority_mapping = ecc->info->queue_priority_mapping; /* Event queue priority mapping */ diff --git a/drivers/dma/fsl_raid.c b/drivers/dma/fsl_raid.c index db2f9e1653a2..90d29f90acfb 100644 --- a/drivers/dma/fsl_raid.c +++ b/drivers/dma/fsl_raid.c @@ -881,6 +881,7 @@ static struct of_device_id fsl_re_ids[] = { { .compatible = "fsl,raideng-v1.0", }, {} }; +MODULE_DEVICE_TABLE(of, fsl_re_ids); static struct platform_driver fsl_re_driver = { .driver = { diff --git a/drivers/dma/hsu/pci.c b/drivers/dma/hsu/pci.c index b51639f045ed..ad45cd344bba 100644 --- a/drivers/dma/hsu/pci.c +++ b/drivers/dma/hsu/pci.c @@ -23,15 +23,28 @@ #define HSU_PCI_CHAN_OFFSET 0x100 +#define PCI_DEVICE_ID_INTEL_MFLD_HSU_DMA 0x081e +#define PCI_DEVICE_ID_INTEL_MRFLD_HSU_DMA 0x1192 + static irqreturn_t hsu_pci_irq(int irq, void *dev) { struct hsu_dma_chip *chip = dev; + struct pci_dev *pdev = to_pci_dev(chip->dev); u32 dmaisr; u32 status; unsigned short i; int ret = 0; int err; + /* + * On Intel Tangier B0 and Anniedale the interrupt line, disregarding + * to have different numbers, is shared between HSU DMA and UART IPs. + * Thus on such SoCs we are expecting that IRQ handler is called in + * UART driver only. + */ + if (pdev->device == PCI_DEVICE_ID_INTEL_MRFLD_HSU_DMA) + return IRQ_HANDLED; + dmaisr = readl(chip->regs + HSU_PCI_DMAISR); for (i = 0; i < chip->hsu->nr_channels; i++) { if (dmaisr & 0x1) { @@ -77,13 +90,15 @@ static int hsu_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) if (!chip) return -ENOMEM; + ret = pci_alloc_irq_vectors(pdev, 1, 1, PCI_IRQ_ALL_TYPES); + if (ret < 0) + return ret; + chip->dev = &pdev->dev; chip->regs = pcim_iomap_table(pdev)[0]; chip->length = pci_resource_len(pdev, 0); chip->offset = HSU_PCI_CHAN_OFFSET; - chip->irq = pdev->irq; - - pci_enable_msi(pdev); + chip->irq = pci_irq_vector(pdev, 0); ret = hsu_dma_probe(chip); if (ret) @@ -111,8 +126,8 @@ static void hsu_pci_remove(struct pci_dev *pdev) } static const struct pci_device_id hsu_pci_id_table[] = { - { PCI_VDEVICE(INTEL, 0x081e), 0 }, - { PCI_VDEVICE(INTEL, 0x1192), 0 }, + { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_MFLD_HSU_DMA), 0 }, + { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_MRFLD_HSU_DMA), 0 }, { } }; MODULE_DEVICE_TABLE(pci, hsu_pci_id_table); diff --git a/drivers/dma/img-mdc-dma.c b/drivers/dma/img-mdc-dma.c index 624f1e1e9c55..54db1411ce73 100644 --- a/drivers/dma/img-mdc-dma.c +++ b/drivers/dma/img-mdc-dma.c @@ -292,7 +292,7 @@ static struct dma_async_tx_descriptor *mdc_prep_dma_memcpy( struct mdc_dma *mdma = mchan->mdma; struct mdc_tx_desc *mdesc; struct mdc_hw_list_desc *curr, *prev = NULL; - dma_addr_t curr_phys, prev_phys; + dma_addr_t curr_phys; if (!len) return NULL; @@ -324,7 +324,6 @@ static struct dma_async_tx_descriptor *mdc_prep_dma_memcpy( xfer_size); prev = curr; - prev_phys = curr_phys; mdesc->list_len++; src += xfer_size; @@ -375,7 +374,7 @@ static struct dma_async_tx_descriptor *mdc_prep_dma_cyclic( struct mdc_dma *mdma = mchan->mdma; struct mdc_tx_desc *mdesc; struct mdc_hw_list_desc *curr, *prev = NULL; - dma_addr_t curr_phys, prev_phys; + dma_addr_t curr_phys; if (!buf_len && !period_len) return NULL; @@ -430,7 +429,6 @@ static struct dma_async_tx_descriptor *mdc_prep_dma_cyclic( } prev = curr; - prev_phys = curr_phys; mdesc->list_len++; buf_addr += xfer_size; @@ -458,7 +456,7 @@ static struct dma_async_tx_descriptor *mdc_prep_slave_sg( struct mdc_tx_desc *mdesc; struct scatterlist *sg; struct mdc_hw_list_desc *curr, *prev = NULL; - dma_addr_t curr_phys, prev_phys; + dma_addr_t curr_phys; unsigned int i; if (!sgl) @@ -509,7 +507,6 @@ static struct dma_async_tx_descriptor *mdc_prep_slave_sg( } prev = curr; - prev_phys = curr_phys; mdesc->list_len++; mdesc->list_xfer_size += xfer_size; diff --git a/drivers/dma/imx-sdma.c b/drivers/dma/imx-sdma.c index b9629b2bfc05..d1651a50c349 100644 --- a/drivers/dma/imx-sdma.c +++ b/drivers/dma/imx-sdma.c @@ -298,6 +298,7 @@ struct sdma_engine; * @event_id1 for channels that use 2 events * @word_size peripheral access size * @buf_tail ID of the buffer that was processed + * @buf_ptail ID of the previous buffer that was processed * @num_bd max NUM_BD. number of descriptors currently handling */ struct sdma_channel { @@ -309,6 +310,7 @@ struct sdma_channel { unsigned int event_id1; enum dma_slave_buswidth word_size; unsigned int buf_tail; + unsigned int buf_ptail; unsigned int num_bd; unsigned int period_len; struct sdma_buffer_descriptor *bd; @@ -700,6 +702,8 @@ static void sdma_update_channel_loop(struct sdma_channel *sdmac) sdmac->chn_real_count = bd->mode.count; bd->mode.status |= BD_DONE; bd->mode.count = sdmac->period_len; + sdmac->buf_ptail = sdmac->buf_tail; + sdmac->buf_tail = (sdmac->buf_tail + 1) % sdmac->num_bd; /* * The callback is called from the interrupt context in order @@ -710,9 +714,6 @@ static void sdma_update_channel_loop(struct sdma_channel *sdmac) dmaengine_desc_get_callback_invoke(&sdmac->desc, NULL); - sdmac->buf_tail++; - sdmac->buf_tail %= sdmac->num_bd; - if (error) sdmac->status = old_status; } @@ -1186,6 +1187,8 @@ static struct dma_async_tx_descriptor *sdma_prep_slave_sg( sdmac->flags = 0; sdmac->buf_tail = 0; + sdmac->buf_ptail = 0; + sdmac->chn_real_count = 0; dev_dbg(sdma->dev, "setting up %d entries for channel %d.\n", sg_len, channel); @@ -1288,6 +1291,8 @@ static struct dma_async_tx_descriptor *sdma_prep_dma_cyclic( sdmac->status = DMA_IN_PROGRESS; sdmac->buf_tail = 0; + sdmac->buf_ptail = 0; + sdmac->chn_real_count = 0; sdmac->period_len = period_len; sdmac->flags |= IMX_DMA_SG_LOOP; @@ -1385,7 +1390,7 @@ static enum dma_status sdma_tx_status(struct dma_chan *chan, u32 residue; if (sdmac->flags & IMX_DMA_SG_LOOP) - residue = (sdmac->num_bd - sdmac->buf_tail) * + residue = (sdmac->num_bd - sdmac->buf_ptail) * sdmac->period_len - sdmac->chn_real_count; else residue = sdmac->chn_count - sdmac->chn_real_count; diff --git a/drivers/dma/ioat/dma.c b/drivers/dma/ioat/dma.c index 49386ce04bf5..a371b07a0981 100644 --- a/drivers/dma/ioat/dma.c +++ b/drivers/dma/ioat/dma.c @@ -39,6 +39,7 @@ #include "../dmaengine.h" static char *chanerr_str[] = { + "DMA Transfer Source Address Error", "DMA Transfer Destination Address Error", "Next Descriptor Address Error", "Descriptor Error", @@ -66,7 +67,6 @@ static char *chanerr_str[] = { "Result Guard Tag verification Error", "Result Application Tag verification Error", "Result Reference Tag verification Error", - NULL }; static void ioat_eh(struct ioatdma_chan *ioat_chan); @@ -75,13 +75,10 @@ static void ioat_print_chanerrs(struct ioatdma_chan *ioat_chan, u32 chanerr) { int i; - for (i = 0; i < 32; i++) { + for (i = 0; i < ARRAY_SIZE(chanerr_str); i++) { if ((chanerr >> i) & 1) { - if (chanerr_str[i]) { - dev_err(to_dev(ioat_chan), "Err(%d): %s\n", - i, chanerr_str[i]); - } else - break; + dev_err(to_dev(ioat_chan), "Err(%d): %s\n", + i, chanerr_str[i]); } } } @@ -341,15 +338,12 @@ ioat_alloc_ring_ent(struct dma_chan *chan, int idx, gfp_t flags) { struct ioat_dma_descriptor *hw; struct ioat_ring_ent *desc; - struct ioatdma_device *ioat_dma; struct ioatdma_chan *ioat_chan = to_ioat_chan(chan); int chunk; dma_addr_t phys; u8 *pos; off_t offs; - ioat_dma = to_ioatdma_device(chan->device); - chunk = idx / IOAT_DESCS_PER_2M; idx &= (IOAT_DESCS_PER_2M - 1); offs = idx * IOAT_DESC_SZ; @@ -614,11 +608,8 @@ static void __cleanup(struct ioatdma_chan *ioat_chan, dma_addr_t phys_complete) tx = &desc->txd; if (tx->cookie) { - struct dmaengine_result res; - dma_cookie_complete(tx); dma_descriptor_unmap(tx); - res.result = DMA_TRANS_NOERROR; dmaengine_desc_get_callback_invoke(tx, NULL); tx->callback = NULL; tx->callback_result = NULL; diff --git a/drivers/dma/ioat/hw.h b/drivers/dma/ioat/hw.h index 8e67895bcca3..abcc51b343ce 100644 --- a/drivers/dma/ioat/hw.h +++ b/drivers/dma/ioat/hw.h @@ -64,6 +64,8 @@ #define PCI_DEVICE_ID_INTEL_IOAT_BDX8 0x6f2e #define PCI_DEVICE_ID_INTEL_IOAT_BDX9 0x6f2f +#define PCI_DEVICE_ID_INTEL_IOAT_SKX 0x2021 + #define IOAT_VER_1_2 0x12 /* Version 1.2 */ #define IOAT_VER_2_0 0x20 /* Version 2.0 */ #define IOAT_VER_3_0 0x30 /* Version 3.0 */ diff --git a/drivers/dma/ioat/init.c b/drivers/dma/ioat/init.c index 015f7110b96d..cc5259b881d4 100644 --- a/drivers/dma/ioat/init.c +++ b/drivers/dma/ioat/init.c @@ -106,6 +106,8 @@ static struct pci_device_id ioat_pci_tbl[] = { { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_BDX8) }, { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_BDX9) }, + { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_SKX) }, + /* I/OAT v3.3 platforms */ { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_BWD0) }, { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_BWD1) }, @@ -243,10 +245,15 @@ static bool is_bdx_ioat(struct pci_dev *pdev) } } +static inline bool is_skx_ioat(struct pci_dev *pdev) +{ + return (pdev->device == PCI_DEVICE_ID_INTEL_IOAT_SKX) ? true : false; +} + static bool is_xeon_cb32(struct pci_dev *pdev) { return is_jf_ioat(pdev) || is_snb_ioat(pdev) || is_ivb_ioat(pdev) || - is_hsw_ioat(pdev) || is_bdx_ioat(pdev); + is_hsw_ioat(pdev) || is_bdx_ioat(pdev) || is_skx_ioat(pdev); } bool is_bwd_ioat(struct pci_dev *pdev) @@ -340,11 +347,13 @@ static int ioat_dma_self_test(struct ioatdma_device *ioat_dma) dma_src = dma_map_single(dev, src, IOAT_TEST_SIZE, DMA_TO_DEVICE); if (dma_mapping_error(dev, dma_src)) { dev_err(dev, "mapping src buffer failed\n"); + err = -ENOMEM; goto free_resources; } dma_dest = dma_map_single(dev, dest, IOAT_TEST_SIZE, DMA_FROM_DEVICE); if (dma_mapping_error(dev, dma_dest)) { dev_err(dev, "mapping dest buffer failed\n"); + err = -ENOMEM; goto unmap_src; } flags = DMA_PREP_INTERRUPT; @@ -691,7 +700,7 @@ static int ioat_alloc_chan_resources(struct dma_chan *c) /* doing 2 32bit writes to mmio since 1 64b write doesn't work */ ioat_chan->completion = dma_pool_zalloc(ioat_chan->ioat_dma->completion_pool, - GFP_KERNEL, &ioat_chan->completion_dma); + GFP_NOWAIT, &ioat_chan->completion_dma); if (!ioat_chan->completion) return -ENOMEM; @@ -701,7 +710,7 @@ static int ioat_alloc_chan_resources(struct dma_chan *c) ioat_chan->reg_base + IOAT_CHANCMP_OFFSET_HIGH); order = IOAT_MAX_ORDER; - ring = ioat_alloc_ring(c, order, GFP_KERNEL); + ring = ioat_alloc_ring(c, order, GFP_NOWAIT); if (!ring) return -ENOMEM; @@ -827,16 +836,20 @@ static int ioat_xor_val_self_test(struct ioatdma_device *ioat_dma) op = IOAT_OP_XOR; dest_dma = dma_map_page(dev, dest, 0, PAGE_SIZE, DMA_FROM_DEVICE); - if (dma_mapping_error(dev, dest_dma)) + if (dma_mapping_error(dev, dest_dma)) { + err = -ENOMEM; goto free_resources; + } for (i = 0; i < IOAT_NUM_SRC_TEST; i++) dma_srcs[i] = DMA_ERROR_CODE; for (i = 0; i < IOAT_NUM_SRC_TEST; i++) { dma_srcs[i] = dma_map_page(dev, xor_srcs[i], 0, PAGE_SIZE, DMA_TO_DEVICE); - if (dma_mapping_error(dev, dma_srcs[i])) + if (dma_mapping_error(dev, dma_srcs[i])) { + err = -ENOMEM; goto dma_unmap; + } } tx = dma->device_prep_dma_xor(dma_chan, dest_dma, dma_srcs, IOAT_NUM_SRC_TEST, PAGE_SIZE, @@ -904,8 +917,10 @@ static int ioat_xor_val_self_test(struct ioatdma_device *ioat_dma) for (i = 0; i < IOAT_NUM_SRC_TEST + 1; i++) { dma_srcs[i] = dma_map_page(dev, xor_val_srcs[i], 0, PAGE_SIZE, DMA_TO_DEVICE); - if (dma_mapping_error(dev, dma_srcs[i])) + if (dma_mapping_error(dev, dma_srcs[i])) { + err = -ENOMEM; goto dma_unmap; + } } tx = dma->device_prep_dma_xor_val(dma_chan, dma_srcs, IOAT_NUM_SRC_TEST + 1, PAGE_SIZE, @@ -957,8 +972,10 @@ static int ioat_xor_val_self_test(struct ioatdma_device *ioat_dma) for (i = 0; i < IOAT_NUM_SRC_TEST + 1; i++) { dma_srcs[i] = dma_map_page(dev, xor_val_srcs[i], 0, PAGE_SIZE, DMA_TO_DEVICE); - if (dma_mapping_error(dev, dma_srcs[i])) + if (dma_mapping_error(dev, dma_srcs[i])) { + err = -ENOMEM; goto dma_unmap; + } } tx = dma->device_prep_dma_xor_val(dma_chan, dma_srcs, IOAT_NUM_SRC_TEST + 1, PAGE_SIZE, @@ -1071,7 +1088,6 @@ static int ioat3_dma_probe(struct ioatdma_device *ioat_dma, int dca) struct dma_device *dma; struct dma_chan *c; struct ioatdma_chan *ioat_chan; - bool is_raid_device = false; int err; u16 val16; @@ -1095,7 +1111,6 @@ static int ioat3_dma_probe(struct ioatdma_device *ioat_dma, int dca) ioat_dma->cap &= ~(IOAT_CAP_XOR|IOAT_CAP_PQ); if (ioat_dma->cap & IOAT_CAP_XOR) { - is_raid_device = true; dma->max_xor = 8; dma_cap_set(DMA_XOR, dma->cap_mask); @@ -1106,7 +1121,6 @@ static int ioat3_dma_probe(struct ioatdma_device *ioat_dma, int dca) } if (ioat_dma->cap & IOAT_CAP_PQ) { - is_raid_device = true; dma->device_prep_dma_pq = ioat_prep_pq; dma->device_prep_dma_pq_val = ioat_prep_pq_val; @@ -1350,6 +1364,8 @@ static int ioat_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) device->version = readb(device->reg_base + IOAT_VER_OFFSET); if (device->version >= IOAT_VER_3_0) { + if (is_skx_ioat(pdev)) + device->version = IOAT_VER_3_2; err = ioat3_dma_probe(device, ioat_dca_enabled); if (device->version >= IOAT_VER_3_3) diff --git a/drivers/dma/ioat/registers.h b/drivers/dma/ioat/registers.h index 48fa4cf9f64a..2f3bbc88ff2a 100644 --- a/drivers/dma/ioat/registers.h +++ b/drivers/dma/ioat/registers.h @@ -106,8 +106,6 @@ #define IOAT_DMA_COMP_V1 0x0001 /* Compatibility with DMA version 1 */ #define IOAT_DMA_COMP_V2 0x0002 /* Compatibility with DMA version 2 */ -/* IOAT1 define left for i7300_idle driver to not fail compiling */ -#define IOAT1_CHANSTS_OFFSET 0x04 #define IOAT_CHANSTS_OFFSET 0x08 /* 64-bit Channel Status Register */ #define IOAT_CHANSTS_COMPLETED_DESCRIPTOR_ADDR (~0x3fULL) #define IOAT_CHANSTS_SOFT_ERR 0x10ULL diff --git a/drivers/dma/ipu/ipu_irq.c b/drivers/dma/ipu/ipu_irq.c index dd184b50e5b4..284627806b88 100644 --- a/drivers/dma/ipu/ipu_irq.c +++ b/drivers/dma/ipu/ipu_irq.c @@ -272,7 +272,7 @@ static void ipu_irq_handler(struct irq_desc *desc) u32 status; int i, line; - for (i = IPU_IRQ_NR_FN_BANKS; i < IPU_IRQ_NR_BANKS; i++) { + for (i = 0; i < IPU_IRQ_NR_BANKS; i++) { struct ipu_irq_bank *bank = irq_bank + i; raw_spin_lock(&bank_lock); diff --git a/drivers/dma/k3dma.c b/drivers/dma/k3dma.c index aabcb7934b05..01e25c68dd5a 100644 --- a/drivers/dma/k3dma.c +++ b/drivers/dma/k3dma.c @@ -458,13 +458,12 @@ static struct k3_dma_desc_sw *k3_dma_alloc_desc_resource(int num, if (!ds) return NULL; - ds->desc_hw = dma_pool_alloc(d->pool, GFP_NOWAIT, &ds->desc_hw_lli); + ds->desc_hw = dma_pool_zalloc(d->pool, GFP_NOWAIT, &ds->desc_hw_lli); if (!ds->desc_hw) { dev_dbg(chan->device->dev, "vch %p: dma alloc fail\n", &c->vc); kfree(ds); return NULL; } - memset(ds->desc_hw, 0, sizeof(struct k3_desc_hw) * num); ds->desc_num = num; return ds; } diff --git a/drivers/dma/mic_x100_dma.c b/drivers/dma/mic_x100_dma.c index 818255844a3c..5ba5714d0b7c 100644 --- a/drivers/dma/mic_x100_dma.c +++ b/drivers/dma/mic_x100_dma.c @@ -554,9 +554,7 @@ static int mic_dma_init(struct mic_dma_device *mic_dma_dev, int ret; for (i = first_chan; i < first_chan + MIC_DMA_NUM_CHAN; i++) { - unsigned long data; ch = &mic_dma_dev->mic_ch[i]; - data = (unsigned long)ch; ch->ch_num = i; ch->owner = owner; spin_lock_init(&ch->cleanup_lock); diff --git a/drivers/dma/mv_xor.c b/drivers/dma/mv_xor.c index 23f75285a4d9..0cb951b743a6 100644 --- a/drivers/dma/mv_xor.c +++ b/drivers/dma/mv_xor.c @@ -68,6 +68,36 @@ static void mv_desc_init(struct mv_xor_desc_slot *desc, hw_desc->byte_count = byte_count; } +/* Populate the descriptor */ +static void mv_xor_config_sg_ll_desc(struct mv_xor_desc_slot *desc, + dma_addr_t dma_src, dma_addr_t dma_dst, + u32 len, struct mv_xor_desc_slot *prev) +{ + struct mv_xor_desc *hw_desc = desc->hw_desc; + + hw_desc->status = XOR_DESC_DMA_OWNED; + hw_desc->phy_next_desc = 0; + /* Configure for XOR with only one src address -> MEMCPY */ + hw_desc->desc_command = XOR_DESC_OPERATION_XOR | (0x1 << 0); + hw_desc->phy_dest_addr = dma_dst; + hw_desc->phy_src_addr[0] = dma_src; + hw_desc->byte_count = len; + + if (prev) { + struct mv_xor_desc *hw_prev = prev->hw_desc; + + hw_prev->phy_next_desc = desc->async_tx.phys; + } +} + +static void mv_xor_desc_config_eod(struct mv_xor_desc_slot *desc) +{ + struct mv_xor_desc *hw_desc = desc->hw_desc; + + /* Enable end-of-descriptor interrupt */ + hw_desc->desc_command |= XOR_DESC_EOD_INT_EN; +} + static void mv_desc_set_mode(struct mv_xor_desc_slot *desc) { struct mv_xor_desc *hw_desc = desc->hw_desc; @@ -228,8 +258,13 @@ mv_chan_clean_completed_slots(struct mv_xor_chan *mv_chan) list_for_each_entry_safe(iter, _iter, &mv_chan->completed_slots, node) { - if (async_tx_test_ack(&iter->async_tx)) + if (async_tx_test_ack(&iter->async_tx)) { list_move_tail(&iter->node, &mv_chan->free_slots); + if (!list_empty(&iter->sg_tx_list)) { + list_splice_tail_init(&iter->sg_tx_list, + &mv_chan->free_slots); + } + } } return 0; } @@ -244,11 +279,20 @@ mv_desc_clean_slot(struct mv_xor_desc_slot *desc, /* the client is allowed to attach dependent operations * until 'ack' is set */ - if (!async_tx_test_ack(&desc->async_tx)) + if (!async_tx_test_ack(&desc->async_tx)) { /* move this slot to the completed_slots */ list_move_tail(&desc->node, &mv_chan->completed_slots); - else + if (!list_empty(&desc->sg_tx_list)) { + list_splice_tail_init(&desc->sg_tx_list, + &mv_chan->completed_slots); + } + } else { list_move_tail(&desc->node, &mv_chan->free_slots); + if (!list_empty(&desc->sg_tx_list)) { + list_splice_tail_init(&desc->sg_tx_list, + &mv_chan->free_slots); + } + } return 0; } @@ -450,6 +494,7 @@ static int mv_xor_alloc_chan_resources(struct dma_chan *chan) dma_async_tx_descriptor_init(&slot->async_tx, chan); slot->async_tx.tx_submit = mv_xor_tx_submit; INIT_LIST_HEAD(&slot->node); + INIT_LIST_HEAD(&slot->sg_tx_list); dma_desc = mv_chan->dma_desc_pool; slot->async_tx.phys = dma_desc + idx * MV_XOR_SLOT_SIZE; slot->idx = idx++; @@ -617,6 +662,132 @@ mv_xor_prep_dma_interrupt(struct dma_chan *chan, unsigned long flags) return mv_xor_prep_dma_xor(chan, dest, &src, 1, len, flags); } +/** + * mv_xor_prep_dma_sg - prepare descriptors for a memory sg transaction + * @chan: DMA channel + * @dst_sg: Destination scatter list + * @dst_sg_len: Number of entries in destination scatter list + * @src_sg: Source scatter list + * @src_sg_len: Number of entries in source scatter list + * @flags: transfer ack flags + * + * Return: Async transaction descriptor on success and NULL on failure + */ +static struct dma_async_tx_descriptor * +mv_xor_prep_dma_sg(struct dma_chan *chan, struct scatterlist *dst_sg, + unsigned int dst_sg_len, struct scatterlist *src_sg, + unsigned int src_sg_len, unsigned long flags) +{ + struct mv_xor_chan *mv_chan = to_mv_xor_chan(chan); + struct mv_xor_desc_slot *new; + struct mv_xor_desc_slot *first = NULL; + struct mv_xor_desc_slot *prev = NULL; + size_t len, dst_avail, src_avail; + dma_addr_t dma_dst, dma_src; + int desc_cnt = 0; + int ret; + + dev_dbg(mv_chan_to_devp(mv_chan), + "%s dst_sg_len: %d src_sg_len: %d flags: %ld\n", + __func__, dst_sg_len, src_sg_len, flags); + + dst_avail = sg_dma_len(dst_sg); + src_avail = sg_dma_len(src_sg); + + /* Run until we are out of scatterlist entries */ + while (true) { + /* Allocate and populate the descriptor */ + desc_cnt++; + new = mv_chan_alloc_slot(mv_chan); + if (!new) { + dev_err(mv_chan_to_devp(mv_chan), + "Out of descriptors (desc_cnt=%d)!\n", + desc_cnt); + goto err; + } + + len = min_t(size_t, src_avail, dst_avail); + len = min_t(size_t, len, MV_XOR_MAX_BYTE_COUNT); + if (len == 0) + goto fetch; + + if (len < MV_XOR_MIN_BYTE_COUNT) { + dev_err(mv_chan_to_devp(mv_chan), + "Transfer size of %zu too small!\n", len); + goto err; + } + + dma_dst = sg_dma_address(dst_sg) + sg_dma_len(dst_sg) - + dst_avail; + dma_src = sg_dma_address(src_sg) + sg_dma_len(src_sg) - + src_avail; + + /* Check if a new window needs to get added for 'dst' */ + ret = mv_xor_add_io_win(mv_chan, dma_dst); + if (ret) + goto err; + + /* Check if a new window needs to get added for 'src' */ + ret = mv_xor_add_io_win(mv_chan, dma_src); + if (ret) + goto err; + + /* Populate the descriptor */ + mv_xor_config_sg_ll_desc(new, dma_src, dma_dst, len, prev); + prev = new; + dst_avail -= len; + src_avail -= len; + + if (!first) + first = new; + else + list_move_tail(&new->node, &first->sg_tx_list); + +fetch: + /* Fetch the next dst scatterlist entry */ + if (dst_avail == 0) { + if (dst_sg_len == 0) + break; + + /* Fetch the next entry: if there are no more: done */ + dst_sg = sg_next(dst_sg); + if (dst_sg == NULL) + break; + + dst_sg_len--; + dst_avail = sg_dma_len(dst_sg); + } + + /* Fetch the next src scatterlist entry */ + if (src_avail == 0) { + if (src_sg_len == 0) + break; + + /* Fetch the next entry: if there are no more: done */ + src_sg = sg_next(src_sg); + if (src_sg == NULL) + break; + + src_sg_len--; + src_avail = sg_dma_len(src_sg); + } + } + + /* Set the EOD flag in the last descriptor */ + mv_xor_desc_config_eod(new); + first->async_tx.flags = flags; + + return &first->async_tx; + +err: + /* Cleanup: Move all descriptors back into the free list */ + spin_lock_bh(&mv_chan->lock); + mv_desc_clean_slot(first, mv_chan); + spin_unlock_bh(&mv_chan->lock); + + return NULL; +} + static void mv_xor_free_chan_resources(struct dma_chan *chan) { struct mv_xor_chan *mv_chan = to_mv_xor_chan(chan); @@ -1083,6 +1254,8 @@ mv_xor_channel_add(struct mv_xor_device *xordev, dma_dev->device_prep_dma_interrupt = mv_xor_prep_dma_interrupt; if (dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask)) dma_dev->device_prep_dma_memcpy = mv_xor_prep_dma_memcpy; + if (dma_has_cap(DMA_SG, dma_dev->cap_mask)) + dma_dev->device_prep_dma_sg = mv_xor_prep_dma_sg; if (dma_has_cap(DMA_XOR, dma_dev->cap_mask)) { dma_dev->max_xor = 8; dma_dev->device_prep_dma_xor = mv_xor_prep_dma_xor; @@ -1132,10 +1305,11 @@ mv_xor_channel_add(struct mv_xor_device *xordev, goto err_free_irq; } - dev_info(&pdev->dev, "Marvell XOR (%s): ( %s%s%s)\n", + dev_info(&pdev->dev, "Marvell XOR (%s): ( %s%s%s%s)\n", mv_chan->op_in_desc ? "Descriptor Mode" : "Registers Mode", dma_has_cap(DMA_XOR, dma_dev->cap_mask) ? "xor " : "", dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask) ? "cpy " : "", + dma_has_cap(DMA_SG, dma_dev->cap_mask) ? "sg " : "", dma_has_cap(DMA_INTERRUPT, dma_dev->cap_mask) ? "intr " : ""); dma_async_device_register(dma_dev); @@ -1378,6 +1552,7 @@ static int mv_xor_probe(struct platform_device *pdev) dma_cap_zero(cap_mask); dma_cap_set(DMA_MEMCPY, cap_mask); + dma_cap_set(DMA_SG, cap_mask); dma_cap_set(DMA_XOR, cap_mask); dma_cap_set(DMA_INTERRUPT, cap_mask); @@ -1455,12 +1630,7 @@ static struct platform_driver mv_xor_driver = { }, }; - -static int __init mv_xor_init(void) -{ - return platform_driver_register(&mv_xor_driver); -} -device_initcall(mv_xor_init); +builtin_platform_driver(mv_xor_driver); /* MODULE_AUTHOR("Saeed Bishara <saeed@marvell.com>"); diff --git a/drivers/dma/mv_xor.h b/drivers/dma/mv_xor.h index 88eeab222a23..cf921dd6af73 100644 --- a/drivers/dma/mv_xor.h +++ b/drivers/dma/mv_xor.h @@ -148,6 +148,7 @@ struct mv_xor_chan { */ struct mv_xor_desc_slot { struct list_head node; + struct list_head sg_tx_list; enum dma_transaction_type type; void *hw_desc; u16 idx; diff --git a/drivers/dma/nbpfaxi.c b/drivers/dma/nbpfaxi.c index 09de71519d37..3f45b9bdf201 100644 --- a/drivers/dma/nbpfaxi.c +++ b/drivers/dma/nbpfaxi.c @@ -225,6 +225,8 @@ struct nbpf_channel { struct nbpf_device { struct dma_device dma_dev; void __iomem *base; + u32 max_burst_mem_read; + u32 max_burst_mem_write; struct clk *clk; const struct nbpf_config *config; unsigned int eirq; @@ -425,10 +427,33 @@ static void nbpf_chan_configure(struct nbpf_channel *chan) nbpf_chan_write(chan, NBPF_CHAN_CFG, NBPF_CHAN_CFG_DMS | chan->dmarq_cfg); } -static u32 nbpf_xfer_ds(struct nbpf_device *nbpf, size_t size) +static u32 nbpf_xfer_ds(struct nbpf_device *nbpf, size_t size, + enum dma_transfer_direction direction) { + int max_burst = nbpf->config->buffer_size * 8; + + if (nbpf->max_burst_mem_read || nbpf->max_burst_mem_write) { + switch (direction) { + case DMA_MEM_TO_MEM: + max_burst = min_not_zero(nbpf->max_burst_mem_read, + nbpf->max_burst_mem_write); + break; + case DMA_MEM_TO_DEV: + if (nbpf->max_burst_mem_read) + max_burst = nbpf->max_burst_mem_read; + break; + case DMA_DEV_TO_MEM: + if (nbpf->max_burst_mem_write) + max_burst = nbpf->max_burst_mem_write; + break; + case DMA_DEV_TO_DEV: + default: + break; + } + } + /* Maximum supported bursts depend on the buffer size */ - return min_t(int, __ffs(size), ilog2(nbpf->config->buffer_size * 8)); + return min_t(int, __ffs(size), ilog2(max_burst)); } static size_t nbpf_xfer_size(struct nbpf_device *nbpf, @@ -458,7 +483,7 @@ static size_t nbpf_xfer_size(struct nbpf_device *nbpf, size = burst; } - return nbpf_xfer_ds(nbpf, size); + return nbpf_xfer_ds(nbpf, size, DMA_TRANS_NONE); } /* @@ -507,7 +532,7 @@ static int nbpf_prep_one(struct nbpf_link_desc *ldesc, * transfers we enable the SBE bit and terminate the transfer in our * .device_pause handler. */ - mem_xfer = nbpf_xfer_ds(chan->nbpf, size); + mem_xfer = nbpf_xfer_ds(chan->nbpf, size, direction); switch (direction) { case DMA_DEV_TO_MEM: @@ -1313,6 +1338,11 @@ static int nbpf_probe(struct platform_device *pdev) if (IS_ERR(nbpf->clk)) return PTR_ERR(nbpf->clk); + of_property_read_u32(np, "max-burst-mem-read", + &nbpf->max_burst_mem_read); + of_property_read_u32(np, "max-burst-mem-write", + &nbpf->max_burst_mem_write); + nbpf->config = cfg; for (i = 0; irqs < ARRAY_SIZE(irqbuf); i++) { diff --git a/drivers/dma/omap-dma.c b/drivers/dma/omap-dma.c index 7ca27d4b1c54..daf479cce691 100644 --- a/drivers/dma/omap-dma.c +++ b/drivers/dma/omap-dma.c @@ -166,6 +166,9 @@ enum { CSDP_DST_BURST_16 = 1 << 14, CSDP_DST_BURST_32 = 2 << 14, CSDP_DST_BURST_64 = 3 << 14, + CSDP_WRITE_NON_POSTED = 0 << 16, + CSDP_WRITE_POSTED = 1 << 16, + CSDP_WRITE_LAST_NON_POSTED = 2 << 16, CICR_TOUT_IE = BIT(0), /* OMAP1 only */ CICR_DROP_IE = BIT(1), @@ -422,7 +425,30 @@ static void omap_dma_start(struct omap_chan *c, struct omap_desc *d) c->running = true; } -static void omap_dma_stop(struct omap_chan *c) +static void omap_dma_drain_chan(struct omap_chan *c) +{ + int i; + u32 val; + + /* Wait for sDMA FIFO to drain */ + for (i = 0; ; i++) { + val = omap_dma_chan_read(c, CCR); + if (!(val & (CCR_RD_ACTIVE | CCR_WR_ACTIVE))) + break; + + if (i > 100) + break; + + udelay(5); + } + + if (val & (CCR_RD_ACTIVE | CCR_WR_ACTIVE)) + dev_err(c->vc.chan.device->dev, + "DMA drain did not complete on lch %d\n", + c->dma_ch); +} + +static int omap_dma_stop(struct omap_chan *c) { struct omap_dmadev *od = to_omap_dma_dev(c->vc.chan.device); uint32_t val; @@ -435,7 +461,6 @@ static void omap_dma_stop(struct omap_chan *c) val = omap_dma_chan_read(c, CCR); if (od->plat->errata & DMA_ERRATA_i541 && val & CCR_TRIGGER_SRC) { uint32_t sysconfig; - unsigned i; sysconfig = omap_dma_glbl_read(od, OCP_SYSCONFIG); val = sysconfig & ~DMA_SYSCONFIG_MIDLEMODE_MASK; @@ -446,27 +471,19 @@ static void omap_dma_stop(struct omap_chan *c) val &= ~CCR_ENABLE; omap_dma_chan_write(c, CCR, val); - /* Wait for sDMA FIFO to drain */ - for (i = 0; ; i++) { - val = omap_dma_chan_read(c, CCR); - if (!(val & (CCR_RD_ACTIVE | CCR_WR_ACTIVE))) - break; - - if (i > 100) - break; - - udelay(5); - } - - if (val & (CCR_RD_ACTIVE | CCR_WR_ACTIVE)) - dev_err(c->vc.chan.device->dev, - "DMA drain did not complete on lch %d\n", - c->dma_ch); + if (!(c->ccr & CCR_BUFFERING_DISABLE)) + omap_dma_drain_chan(c); omap_dma_glbl_write(od, OCP_SYSCONFIG, sysconfig); } else { + if (!(val & CCR_ENABLE)) + return -EINVAL; + val &= ~CCR_ENABLE; omap_dma_chan_write(c, CCR, val); + + if (!(c->ccr & CCR_BUFFERING_DISABLE)) + omap_dma_drain_chan(c); } mb(); @@ -481,8 +498,8 @@ static void omap_dma_stop(struct omap_chan *c) omap_dma_chan_write(c, CLNK_CTRL, val); } - c->running = false; + return 0; } static void omap_dma_start_sg(struct omap_chan *c, struct omap_desc *d) @@ -836,6 +853,8 @@ static enum dma_status omap_dma_tx_status(struct dma_chan *chan, } else { txstate->residue = 0; } + if (ret == DMA_IN_PROGRESS && c->paused) + ret = DMA_PAUSED; spin_unlock_irqrestore(&c->vc.lock, flags); return ret; @@ -865,15 +884,18 @@ static struct dma_async_tx_descriptor *omap_dma_prep_slave_sg( unsigned i, es, en, frame_bytes; bool ll_failed = false; u32 burst; + u32 port_window, port_window_bytes; if (dir == DMA_DEV_TO_MEM) { dev_addr = c->cfg.src_addr; dev_width = c->cfg.src_addr_width; burst = c->cfg.src_maxburst; + port_window = c->cfg.src_port_window_size; } else if (dir == DMA_MEM_TO_DEV) { dev_addr = c->cfg.dst_addr; dev_width = c->cfg.dst_addr_width; burst = c->cfg.dst_maxburst; + port_window = c->cfg.dst_port_window_size; } else { dev_err(chan->device->dev, "%s: bad direction?\n", __func__); return NULL; @@ -894,6 +916,12 @@ static struct dma_async_tx_descriptor *omap_dma_prep_slave_sg( return NULL; } + /* When the port_window is used, one frame must cover the window */ + if (port_window) { + burst = port_window; + port_window_bytes = port_window * es_bytes[es]; + } + /* Now allocate and setup the descriptor. */ d = kzalloc(sizeof(*d) + sglen * sizeof(d->sg[0]), GFP_ATOMIC); if (!d) @@ -905,11 +933,46 @@ static struct dma_async_tx_descriptor *omap_dma_prep_slave_sg( d->ccr = c->ccr | CCR_SYNC_FRAME; if (dir == DMA_DEV_TO_MEM) { - d->ccr |= CCR_DST_AMODE_POSTINC | CCR_SRC_AMODE_CONSTANT; d->csdp = CSDP_DST_BURST_64 | CSDP_DST_PACKED; + + d->ccr |= CCR_DST_AMODE_POSTINC; + if (port_window) { + d->ccr |= CCR_SRC_AMODE_DBLIDX; + + if (port_window_bytes >= 64) + d->csdp |= CSDP_SRC_BURST_64; + else if (port_window_bytes >= 32) + d->csdp |= CSDP_SRC_BURST_32; + else if (port_window_bytes >= 16) + d->csdp |= CSDP_SRC_BURST_16; + + } else { + d->ccr |= CCR_SRC_AMODE_CONSTANT; + } } else { - d->ccr |= CCR_DST_AMODE_CONSTANT | CCR_SRC_AMODE_POSTINC; d->csdp = CSDP_SRC_BURST_64 | CSDP_SRC_PACKED; + + d->ccr |= CCR_SRC_AMODE_POSTINC; + if (port_window) { + d->ccr |= CCR_DST_AMODE_DBLIDX; + d->ei = 1; + /* + * One frame covers the port_window and by configure + * the source frame index to be -1 * (port_window - 1) + * we instruct the sDMA that after a frame is processed + * it should move back to the start of the window. + */ + d->fi = -(port_window_bytes - 1); + + if (port_window_bytes >= 64) + d->csdp |= CSDP_DST_BURST_64; + else if (port_window_bytes >= 32) + d->csdp |= CSDP_DST_BURST_32; + else if (port_window_bytes >= 16) + d->csdp |= CSDP_DST_BURST_16; + } else { + d->ccr |= CCR_DST_AMODE_CONSTANT; + } } d->cicr = CICR_DROP_IE | CICR_BLOCK_IE; @@ -927,6 +990,9 @@ static struct dma_async_tx_descriptor *omap_dma_prep_slave_sg( d->ccr |= CCR_TRIGGER_SRC; d->cicr |= CICR_MISALIGNED_ERR_IE | CICR_TRANS_ERR_IE; + + if (port_window) + d->csdp |= CSDP_WRITE_LAST_NON_POSTED; } if (od->plat->errata & DMA_ERRATA_PARALLEL_CHANNELS) d->clnk_ctrl = c->dma_ch; @@ -952,6 +1018,16 @@ static struct dma_async_tx_descriptor *omap_dma_prep_slave_sg( osg->addr = sg_dma_address(sgent); osg->en = en; osg->fn = sg_dma_len(sgent) / frame_bytes; + if (port_window && dir == DMA_DEV_TO_MEM) { + osg->ei = 1; + /* + * One frame covers the port_window and by configure + * the source frame index to be -1 * (port_window - 1) + * we instruct the sDMA that after a frame is processed + * it should move back to the start of the window. + */ + osg->fi = -(port_window_bytes - 1); + } if (d->using_ll) { osg->t2_desc = dma_pool_alloc(od->desc_pool, GFP_ATOMIC, @@ -1247,10 +1323,8 @@ static int omap_dma_terminate_all(struct dma_chan *chan) omap_dma_stop(c); } - if (c->cyclic) { - c->cyclic = false; - c->paused = false; - } + c->cyclic = false; + c->paused = false; vchan_get_all_descriptors(&c->vc, &head); spin_unlock_irqrestore(&c->vc.lock, flags); @@ -1269,28 +1343,66 @@ static void omap_dma_synchronize(struct dma_chan *chan) static int omap_dma_pause(struct dma_chan *chan) { struct omap_chan *c = to_omap_dma_chan(chan); + struct omap_dmadev *od = to_omap_dma_dev(chan->device); + unsigned long flags; + int ret = -EINVAL; + bool can_pause = false; - /* Pause/Resume only allowed with cyclic mode */ - if (!c->cyclic) - return -EINVAL; + spin_lock_irqsave(&od->irq_lock, flags); + + if (!c->desc) + goto out; - if (!c->paused) { - omap_dma_stop(c); - c->paused = true; + if (c->cyclic) + can_pause = true; + + /* + * We do not allow DMA_MEM_TO_DEV transfers to be paused. + * From the AM572x TRM, 16.1.4.18 Disabling a Channel During Transfer: + * "When a channel is disabled during a transfer, the channel undergoes + * an abort, unless it is hardware-source-synchronized …". + * A source-synchronised channel is one where the fetching of data is + * under control of the device. In other words, a device-to-memory + * transfer. So, a destination-synchronised channel (which would be a + * memory-to-device transfer) undergoes an abort if the the CCR_ENABLE + * bit is cleared. + * From 16.1.4.20.4.6.2 Abort: "If an abort trigger occurs, the channel + * aborts immediately after completion of current read/write + * transactions and then the FIFO is cleaned up." The term "cleaned up" + * is not defined. TI recommends to check that RD_ACTIVE and WR_ACTIVE + * are both clear _before_ disabling the channel, otherwise data loss + * will occur. + * The problem is that if the channel is active, then device activity + * can result in DMA activity starting between reading those as both + * clear and the write to DMA_CCR to clear the enable bit hitting the + * hardware. If the DMA hardware can't drain the data in its FIFO to the + * destination, then data loss "might" occur (say if we write to an UART + * and the UART is not accepting any further data). + */ + else if (c->desc->dir == DMA_DEV_TO_MEM) + can_pause = true; + + if (can_pause && !c->paused) { + ret = omap_dma_stop(c); + if (!ret) + c->paused = true; } +out: + spin_unlock_irqrestore(&od->irq_lock, flags); - return 0; + return ret; } static int omap_dma_resume(struct dma_chan *chan) { struct omap_chan *c = to_omap_dma_chan(chan); + struct omap_dmadev *od = to_omap_dma_dev(chan->device); + unsigned long flags; + int ret = -EINVAL; - /* Pause/Resume only allowed with cyclic mode */ - if (!c->cyclic) - return -EINVAL; + spin_lock_irqsave(&od->irq_lock, flags); - if (c->paused) { + if (c->paused && c->desc) { mb(); /* Restore channel link register */ @@ -1298,9 +1410,11 @@ static int omap_dma_resume(struct dma_chan *chan) omap_dma_start(c, c->desc); c->paused = false; + ret = 0; } + spin_unlock_irqrestore(&od->irq_lock, flags); - return 0; + return ret; } static int omap_dma_chan_init(struct omap_dmadev *od) @@ -1339,6 +1453,7 @@ static int omap_dma_probe(struct platform_device *pdev) struct omap_dmadev *od; struct resource *res; int rc, i, irq; + u32 lch_count; od = devm_kzalloc(&pdev->dev, sizeof(*od), GFP_KERNEL); if (!od) @@ -1381,20 +1496,31 @@ static int omap_dma_probe(struct platform_device *pdev) spin_lock_init(&od->lock); spin_lock_init(&od->irq_lock); - if (!pdev->dev.of_node) { - od->dma_requests = od->plat->dma_attr->lch_count; - if (unlikely(!od->dma_requests)) - od->dma_requests = OMAP_SDMA_REQUESTS; - } else if (of_property_read_u32(pdev->dev.of_node, "dma-requests", - &od->dma_requests)) { + /* Number of DMA requests */ + od->dma_requests = OMAP_SDMA_REQUESTS; + if (pdev->dev.of_node && of_property_read_u32(pdev->dev.of_node, + "dma-requests", + &od->dma_requests)) { dev_info(&pdev->dev, "Missing dma-requests property, using %u.\n", OMAP_SDMA_REQUESTS); - od->dma_requests = OMAP_SDMA_REQUESTS; } - od->lch_map = devm_kcalloc(&pdev->dev, od->dma_requests, - sizeof(*od->lch_map), GFP_KERNEL); + /* Number of available logical channels */ + if (!pdev->dev.of_node) { + lch_count = od->plat->dma_attr->lch_count; + if (unlikely(!lch_count)) + lch_count = OMAP_SDMA_CHANNELS; + } else if (of_property_read_u32(pdev->dev.of_node, "dma-channels", + &lch_count)) { + dev_info(&pdev->dev, + "Missing dma-channels property, using %u.\n", + OMAP_SDMA_CHANNELS); + lch_count = OMAP_SDMA_CHANNELS; + } + + od->lch_map = devm_kcalloc(&pdev->dev, lch_count, sizeof(*od->lch_map), + GFP_KERNEL); if (!od->lch_map) return -ENOMEM; diff --git a/drivers/dma/pch_dma.c b/drivers/dma/pch_dma.c index df95727dc2fb..f9028e9d0dfc 100644 --- a/drivers/dma/pch_dma.c +++ b/drivers/dma/pch_dma.c @@ -417,10 +417,8 @@ static dma_cookie_t pd_tx_submit(struct dma_async_tx_descriptor *txd) { struct pch_dma_desc *desc = to_pd_desc(txd); struct pch_dma_chan *pd_chan = to_pd_chan(txd->chan); - dma_cookie_t cookie; spin_lock(&pd_chan->lock); - cookie = dma_cookie_assign(txd); if (list_empty(&pd_chan->active_list)) { list_add_tail(&desc->desc_node, &pd_chan->active_list); @@ -439,9 +437,8 @@ static struct pch_dma_desc *pdc_alloc_desc(struct dma_chan *chan, gfp_t flags) struct pch_dma *pd = to_pd(chan->device); dma_addr_t addr; - desc = pci_pool_alloc(pd->pool, flags, &addr); + desc = pci_pool_zalloc(pd->pool, flags, &addr); if (desc) { - memset(desc, 0, sizeof(struct pch_dma_desc)); INIT_LIST_HEAD(&desc->tx_list); dma_async_tx_descriptor_init(&desc->txd, chan); desc->txd.tx_submit = pd_tx_submit; diff --git a/drivers/dma/pl330.c b/drivers/dma/pl330.c index 030fe05ed43b..f37f4978dabb 100644 --- a/drivers/dma/pl330.c +++ b/drivers/dma/pl330.c @@ -448,6 +448,9 @@ struct dma_pl330_chan { /* for cyclic capability */ bool cyclic; + + /* for runtime pm tracking */ + bool active; }; struct pl330_dmac { @@ -570,7 +573,8 @@ static inline u32 _emit_ADDH(unsigned dry_run, u8 buf[], buf[0] = CMD_DMAADDH; buf[0] |= (da << 1); - *((__le16 *)&buf[1]) = cpu_to_le16(val); + buf[1] = val; + buf[2] = val >> 8; PL330_DBGCMD_DUMP(SZ_DMAADDH, "\tDMAADDH %s %u\n", da == 1 ? "DA" : "SA", val); @@ -724,7 +728,10 @@ static inline u32 _emit_MOV(unsigned dry_run, u8 buf[], buf[0] = CMD_DMAMOV; buf[1] = dst; - *((__le32 *)&buf[2]) = cpu_to_le32(val); + buf[2] = val; + buf[3] = val >> 8; + buf[4] = val >> 16; + buf[5] = val >> 24; PL330_DBGCMD_DUMP(SZ_DMAMOV, "\tDMAMOV %s 0x%x\n", dst == SAR ? "SAR" : (dst == DAR ? "DAR" : "CCR"), val); @@ -899,10 +906,11 @@ static inline u32 _emit_GO(unsigned dry_run, u8 buf[], buf[0] = CMD_DMAGO; buf[0] |= (ns << 1); - buf[1] = chan & 0x7; - - *((__le32 *)&buf[2]) = cpu_to_le32(addr); + buf[2] = addr; + buf[3] = addr >> 8; + buf[4] = addr >> 16; + buf[5] = addr >> 24; return SZ_DMAGO; } @@ -1691,7 +1699,6 @@ static bool _chan_ns(const struct pl330_dmac *pl330, int i) static struct pl330_thread *pl330_request_channel(struct pl330_dmac *pl330) { struct pl330_thread *thrd = NULL; - unsigned long flags; int chans, i; if (pl330->state == DYING) @@ -1699,8 +1706,6 @@ static struct pl330_thread *pl330_request_channel(struct pl330_dmac *pl330) chans = pl330->pcfg.num_chan; - spin_lock_irqsave(&pl330->lock, flags); - for (i = 0; i < chans; i++) { thrd = &pl330->channels[i]; if ((thrd->free) && (!_manager_ns(thrd) || @@ -1718,8 +1723,6 @@ static struct pl330_thread *pl330_request_channel(struct pl330_dmac *pl330) thrd = NULL; } - spin_unlock_irqrestore(&pl330->lock, flags); - return thrd; } @@ -1737,7 +1740,6 @@ static inline void _free_event(struct pl330_thread *thrd, int ev) static void pl330_release_channel(struct pl330_thread *thrd) { struct pl330_dmac *pl330; - unsigned long flags; if (!thrd || thrd->free) return; @@ -1749,10 +1751,8 @@ static void pl330_release_channel(struct pl330_thread *thrd) pl330 = thrd->dmac; - spin_lock_irqsave(&pl330->lock, flags); _free_event(thrd, thrd->ev); thrd->free = true; - spin_unlock_irqrestore(&pl330->lock, flags); } /* Initialize the structure for PL330 configuration, that can be used @@ -1859,9 +1859,10 @@ static int dmac_alloc_resources(struct pl330_dmac *pl330) * Alloc MicroCode buffer for 'chans' Channel threads. * A channel's buffer offset is (Channel_Id * MCODE_BUFF_PERCHAN) */ - pl330->mcode_cpu = dma_alloc_coherent(pl330->ddma.dev, + pl330->mcode_cpu = dma_alloc_attrs(pl330->ddma.dev, chans * pl330->mcbufsz, - &pl330->mcode_bus, GFP_KERNEL); + &pl330->mcode_bus, GFP_KERNEL, + DMA_ATTR_PRIVILEGED); if (!pl330->mcode_cpu) { dev_err(pl330->ddma.dev, "%s:%d Can't allocate memory!\n", __func__, __LINE__); @@ -1883,11 +1884,8 @@ static int dmac_alloc_resources(struct pl330_dmac *pl330) static int pl330_add(struct pl330_dmac *pl330) { - void __iomem *regs; int i, ret; - regs = pl330->base; - /* Check if we can handle this DMAC */ if ((pl330->pcfg.periph_id & 0xfffff) != PERIPH_ID_VAL) { dev_err(pl330->ddma.dev, "PERIPH_ID 0x%x !\n", @@ -2031,6 +2029,7 @@ static void pl330_tasklet(unsigned long data) _stop(pch->thread); spin_unlock(&pch->thread->dmac->lock); power_down = true; + pch->active = false; } else { /* Make sure the PL330 Channel thread is active */ spin_lock(&pch->thread->dmac->lock); @@ -2050,6 +2049,7 @@ static void pl330_tasklet(unsigned long data) desc->status = PREP; list_move_tail(&desc->node, &pch->work_list); if (power_down) { + pch->active = true; spin_lock(&pch->thread->dmac->lock); _start(pch->thread); spin_unlock(&pch->thread->dmac->lock); @@ -2115,20 +2115,20 @@ static int pl330_alloc_chan_resources(struct dma_chan *chan) struct pl330_dmac *pl330 = pch->dmac; unsigned long flags; - spin_lock_irqsave(&pch->lock, flags); + spin_lock_irqsave(&pl330->lock, flags); dma_cookie_init(chan); pch->cyclic = false; pch->thread = pl330_request_channel(pl330); if (!pch->thread) { - spin_unlock_irqrestore(&pch->lock, flags); + spin_unlock_irqrestore(&pl330->lock, flags); return -ENOMEM; } tasklet_init(&pch->task, pl330_tasklet, (unsigned long) pch); - spin_unlock_irqrestore(&pch->lock, flags); + spin_unlock_irqrestore(&pl330->lock, flags); return 1; } @@ -2164,6 +2164,7 @@ static int pl330_terminate_all(struct dma_chan *chan) unsigned long flags; struct pl330_dmac *pl330 = pch->dmac; LIST_HEAD(list); + bool power_down = false; pm_runtime_get_sync(pl330->ddma.dev); spin_lock_irqsave(&pch->lock, flags); @@ -2174,6 +2175,8 @@ static int pl330_terminate_all(struct dma_chan *chan) pch->thread->req[0].desc = NULL; pch->thread->req[1].desc = NULL; pch->thread->req_running = -1; + power_down = pch->active; + pch->active = false; /* Mark all desc done */ list_for_each_entry(desc, &pch->submitted_list, node) { @@ -2191,6 +2194,8 @@ static int pl330_terminate_all(struct dma_chan *chan) list_splice_tail_init(&pch->completed_list, &pl330->desc_pool); spin_unlock_irqrestore(&pch->lock, flags); pm_runtime_mark_last_busy(pl330->ddma.dev); + if (power_down) + pm_runtime_put_autosuspend(pl330->ddma.dev); pm_runtime_put_autosuspend(pl330->ddma.dev); return 0; @@ -2226,12 +2231,13 @@ static int pl330_pause(struct dma_chan *chan) static void pl330_free_chan_resources(struct dma_chan *chan) { struct dma_pl330_chan *pch = to_pchan(chan); + struct pl330_dmac *pl330 = pch->dmac; unsigned long flags; tasklet_kill(&pch->task); pm_runtime_get_sync(pch->dmac->ddma.dev); - spin_lock_irqsave(&pch->lock, flags); + spin_lock_irqsave(&pl330->lock, flags); pl330_release_channel(pch->thread); pch->thread = NULL; @@ -2239,7 +2245,7 @@ static void pl330_free_chan_resources(struct dma_chan *chan) if (pch->cyclic) list_splice_tail_init(&pch->work_list, &pch->dmac->desc_pool); - spin_unlock_irqrestore(&pch->lock, flags); + spin_unlock_irqrestore(&pl330->lock, flags); pm_runtime_mark_last_busy(pch->dmac->ddma.dev); pm_runtime_put_autosuspend(pch->dmac->ddma.dev); } @@ -2263,6 +2269,11 @@ static int pl330_get_current_xferred_count(struct dma_pl330_chan *pch, } pm_runtime_mark_last_busy(pch->dmac->ddma.dev); pm_runtime_put_autosuspend(pl330->ddma.dev); + + /* If DMAMOV hasn't finished yet, SAR/DAR can be zero */ + if (!val) + return 0; + return val - addr; } @@ -2350,6 +2361,7 @@ static void pl330_issue_pending(struct dma_chan *chan) * updated on work_list emptiness status. */ WARN_ON(list_empty(&pch->submitted_list)); + pch->active = true; pm_runtime_get_sync(pch->dmac->ddma.dev); } list_splice_tail_init(&pch->submitted_list, &pch->work_list); diff --git a/drivers/dma/pxa_dma.c b/drivers/dma/pxa_dma.c index 3f56f9ca4482..b53fb618bbf6 100644 --- a/drivers/dma/pxa_dma.c +++ b/drivers/dma/pxa_dma.c @@ -413,15 +413,6 @@ static inline void pxad_init_debugfs(struct pxad_device *pdev) {} static inline void pxad_cleanup_debugfs(struct pxad_device *pdev) {} #endif -/* - * In the transition phase where legacy pxa handling is done at the same time as - * mmp_dma, the DMA physical channel split between the 2 DMA providers is done - * through legacy_reserved. Legacy code reserves DMA channels by settings - * corresponding bits in legacy_reserved. - */ -static u32 legacy_reserved; -static u32 legacy_unavailable; - static struct pxad_phy *lookup_phy(struct pxad_chan *pchan) { int prio, i; @@ -442,14 +433,10 @@ static struct pxad_phy *lookup_phy(struct pxad_chan *pchan) for (i = 0; i < pdev->nr_chans; i++) { if (prio != (i & 0xf) >> 2) continue; - if ((i < 32) && (legacy_reserved & BIT(i))) - continue; phy = &pdev->phys[i]; if (!phy->vchan) { phy->vchan = pchan; found = phy; - if (i < 32) - legacy_unavailable |= BIT(i); goto out_unlock; } } @@ -469,7 +456,6 @@ static void pxad_free_phy(struct pxad_chan *chan) struct pxad_device *pdev = to_pxad_dev(chan->vc.chan.device); unsigned long flags; u32 reg; - int i; dev_dbg(&chan->vc.chan.dev->device, "%s(): freeing\n", __func__); @@ -483,9 +469,6 @@ static void pxad_free_phy(struct pxad_chan *chan) } spin_lock_irqsave(&pdev->phy_lock, flags); - for (i = 0; i < 32; i++) - if (chan->phy == &pdev->phys[i]) - legacy_unavailable &= ~BIT(i); chan->phy->vchan = NULL; chan->phy = NULL; spin_unlock_irqrestore(&pdev->phy_lock, flags); @@ -739,8 +722,6 @@ static irqreturn_t pxad_int_handler(int irq, void *dev_id) i = __ffs(dint); dint &= (dint - 1); phy = &pdev->phys[i]; - if ((i < 32) && (legacy_reserved & BIT(i))) - continue; if (pxad_chan_handler(irq, phy) == IRQ_HANDLED) ret = IRQ_HANDLED; } @@ -1522,15 +1503,6 @@ bool pxad_filter_fn(struct dma_chan *chan, void *param) } EXPORT_SYMBOL_GPL(pxad_filter_fn); -int pxad_toggle_reserved_channel(int legacy_channel) -{ - if (legacy_unavailable & (BIT(legacy_channel))) - return -EBUSY; - legacy_reserved ^= BIT(legacy_channel); - return 0; -} -EXPORT_SYMBOL_GPL(pxad_toggle_reserved_channel); - module_platform_driver(pxad_driver); MODULE_DESCRIPTION("Marvell PXA Peripheral DMA Driver"); diff --git a/drivers/dma/qcom/hidma.c b/drivers/dma/qcom/hidma.c index e244e10a94b5..3c982c96b4b7 100644 --- a/drivers/dma/qcom/hidma.c +++ b/drivers/dma/qcom/hidma.c @@ -56,6 +56,7 @@ #include <linux/irq.h> #include <linux/atomic.h> #include <linux/pm_runtime.h> +#include <linux/msi.h> #include "../dmaengine.h" #include "hidma.h" @@ -70,6 +71,7 @@ #define HIDMA_ERR_INFO_SW 0xFF #define HIDMA_ERR_CODE_UNEXPECTED_TERMINATE 0x0 #define HIDMA_NR_DEFAULT_DESC 10 +#define HIDMA_MSI_INTS 11 static inline struct hidma_dev *to_hidma_dev(struct dma_device *dmadev) { @@ -553,6 +555,17 @@ static irqreturn_t hidma_chirq_handler(int chirq, void *arg) return hidma_ll_inthandler(chirq, lldev); } +#ifdef CONFIG_GENERIC_MSI_IRQ_DOMAIN +static irqreturn_t hidma_chirq_handler_msi(int chirq, void *arg) +{ + struct hidma_lldev **lldevp = arg; + struct hidma_dev *dmadev = to_hidma_dev_from_lldev(lldevp); + + return hidma_ll_inthandler_msi(chirq, *lldevp, + 1 << (chirq - dmadev->msi_virqbase)); +} +#endif + static ssize_t hidma_show_values(struct device *dev, struct device_attribute *attr, char *buf) { @@ -567,8 +580,13 @@ static ssize_t hidma_show_values(struct device *dev, return strlen(buf); } -static int hidma_create_sysfs_entry(struct hidma_dev *dev, char *name, - int mode) +static inline void hidma_sysfs_uninit(struct hidma_dev *dev) +{ + device_remove_file(dev->ddev.dev, dev->chid_attrs); +} + +static struct device_attribute* +hidma_create_sysfs_entry(struct hidma_dev *dev, char *name, int mode) { struct device_attribute *attrs; char *name_copy; @@ -576,18 +594,125 @@ static int hidma_create_sysfs_entry(struct hidma_dev *dev, char *name, attrs = devm_kmalloc(dev->ddev.dev, sizeof(struct device_attribute), GFP_KERNEL); if (!attrs) - return -ENOMEM; + return NULL; name_copy = devm_kstrdup(dev->ddev.dev, name, GFP_KERNEL); if (!name_copy) - return -ENOMEM; + return NULL; attrs->attr.name = name_copy; attrs->attr.mode = mode; attrs->show = hidma_show_values; sysfs_attr_init(&attrs->attr); - return device_create_file(dev->ddev.dev, attrs); + return attrs; +} + +static int hidma_sysfs_init(struct hidma_dev *dev) +{ + dev->chid_attrs = hidma_create_sysfs_entry(dev, "chid", S_IRUGO); + if (!dev->chid_attrs) + return -ENOMEM; + + return device_create_file(dev->ddev.dev, dev->chid_attrs); +} + +#ifdef CONFIG_GENERIC_MSI_IRQ_DOMAIN +static void hidma_write_msi_msg(struct msi_desc *desc, struct msi_msg *msg) +{ + struct device *dev = msi_desc_to_dev(desc); + struct hidma_dev *dmadev = dev_get_drvdata(dev); + + if (!desc->platform.msi_index) { + writel(msg->address_lo, dmadev->dev_evca + 0x118); + writel(msg->address_hi, dmadev->dev_evca + 0x11C); + writel(msg->data, dmadev->dev_evca + 0x120); + } +} +#endif + +static void hidma_free_msis(struct hidma_dev *dmadev) +{ +#ifdef CONFIG_GENERIC_MSI_IRQ_DOMAIN + struct device *dev = dmadev->ddev.dev; + struct msi_desc *desc; + + /* free allocated MSI interrupts above */ + for_each_msi_entry(desc, dev) + devm_free_irq(dev, desc->irq, &dmadev->lldev); + + platform_msi_domain_free_irqs(dev); +#endif +} + +static int hidma_request_msi(struct hidma_dev *dmadev, + struct platform_device *pdev) +{ +#ifdef CONFIG_GENERIC_MSI_IRQ_DOMAIN + int rc; + struct msi_desc *desc; + struct msi_desc *failed_desc = NULL; + + rc = platform_msi_domain_alloc_irqs(&pdev->dev, HIDMA_MSI_INTS, + hidma_write_msi_msg); + if (rc) + return rc; + + for_each_msi_entry(desc, &pdev->dev) { + if (!desc->platform.msi_index) + dmadev->msi_virqbase = desc->irq; + + rc = devm_request_irq(&pdev->dev, desc->irq, + hidma_chirq_handler_msi, + 0, "qcom-hidma-msi", + &dmadev->lldev); + if (rc) { + failed_desc = desc; + break; + } + } + + if (rc) { + /* free allocated MSI interrupts above */ + for_each_msi_entry(desc, &pdev->dev) { + if (desc == failed_desc) + break; + devm_free_irq(&pdev->dev, desc->irq, + &dmadev->lldev); + } + } else { + /* Add callback to free MSIs on teardown */ + hidma_ll_setup_irq(dmadev->lldev, true); + + } + if (rc) + dev_warn(&pdev->dev, + "failed to request MSI irq, falling back to wired IRQ\n"); + return rc; +#else + return -EINVAL; +#endif +} + +static bool hidma_msi_capable(struct device *dev) +{ + struct acpi_device *adev = ACPI_COMPANION(dev); + const char *of_compat; + int ret = -EINVAL; + + if (!adev || acpi_disabled) { + ret = device_property_read_string(dev, "compatible", + &of_compat); + if (ret) + return false; + + ret = strcmp(of_compat, "qcom,hidma-1.1"); + } else { +#ifdef CONFIG_ACPI + ret = strcmp(acpi_device_hid(adev), "QCOM8062"); +#endif + } + return ret == 0; } static int hidma_probe(struct platform_device *pdev) @@ -599,6 +724,7 @@ static int hidma_probe(struct platform_device *pdev) void __iomem *evca; void __iomem *trca; int rc; + bool msi; pm_runtime_set_autosuspend_delay(&pdev->dev, HIDMA_AUTOSUSPEND_TIMEOUT); pm_runtime_use_autosuspend(&pdev->dev); @@ -660,6 +786,12 @@ static int hidma_probe(struct platform_device *pdev) dmadev->ddev.device_terminate_all = hidma_terminate_all; dmadev->ddev.copy_align = 8; + /* + * Determine the MSI capability of the platform. Old HW doesn't + * support MSI. + */ + msi = hidma_msi_capable(&pdev->dev); + device_property_read_u32(&pdev->dev, "desc-count", &dmadev->nr_descriptors); @@ -688,10 +820,17 @@ static int hidma_probe(struct platform_device *pdev) goto dmafree; } - rc = devm_request_irq(&pdev->dev, chirq, hidma_chirq_handler, 0, - "qcom-hidma", dmadev->lldev); - if (rc) - goto uninit; + platform_set_drvdata(pdev, dmadev); + if (msi) + rc = hidma_request_msi(dmadev, pdev); + + if (!msi || rc) { + hidma_ll_setup_irq(dmadev->lldev, false); + rc = devm_request_irq(&pdev->dev, chirq, hidma_chirq_handler, + 0, "qcom-hidma", dmadev->lldev); + if (rc) + goto uninit; + } INIT_LIST_HEAD(&dmadev->ddev.channels); rc = hidma_chan_init(dmadev, 0); @@ -705,14 +844,16 @@ static int hidma_probe(struct platform_device *pdev) dmadev->irq = chirq; tasklet_init(&dmadev->task, hidma_issue_task, (unsigned long)dmadev); hidma_debug_init(dmadev); - hidma_create_sysfs_entry(dmadev, "chid", S_IRUGO); + hidma_sysfs_init(dmadev); dev_info(&pdev->dev, "HI-DMA engine driver registration complete\n"); - platform_set_drvdata(pdev, dmadev); pm_runtime_mark_last_busy(dmadev->ddev.dev); pm_runtime_put_autosuspend(dmadev->ddev.dev); return 0; uninit: + if (msi) + hidma_free_msis(dmadev); + hidma_debug_uninit(dmadev); hidma_ll_uninit(dmadev->lldev); dmafree: @@ -730,8 +871,13 @@ static int hidma_remove(struct platform_device *pdev) pm_runtime_get_sync(dmadev->ddev.dev); dma_async_device_unregister(&dmadev->ddev); - devm_free_irq(dmadev->ddev.dev, dmadev->irq, dmadev->lldev); + if (!dmadev->lldev->msi_support) + devm_free_irq(dmadev->ddev.dev, dmadev->irq, dmadev->lldev); + else + hidma_free_msis(dmadev); + tasklet_kill(&dmadev->task); + hidma_sysfs_uninit(dmadev); hidma_debug_uninit(dmadev); hidma_ll_uninit(dmadev->lldev); hidma_free(dmadev); @@ -746,12 +892,15 @@ static int hidma_remove(struct platform_device *pdev) #if IS_ENABLED(CONFIG_ACPI) static const struct acpi_device_id hidma_acpi_ids[] = { {"QCOM8061"}, + {"QCOM8062"}, {}, }; +MODULE_DEVICE_TABLE(acpi, hidma_acpi_ids); #endif static const struct of_device_id hidma_match[] = { {.compatible = "qcom,hidma-1.0",}, + {.compatible = "qcom,hidma-1.1",}, {}, }; MODULE_DEVICE_TABLE(of, hidma_match); diff --git a/drivers/dma/qcom/hidma.h b/drivers/dma/qcom/hidma.h index e52e20716303..c7d014235c32 100644 --- a/drivers/dma/qcom/hidma.h +++ b/drivers/dma/qcom/hidma.h @@ -46,6 +46,7 @@ struct hidma_tre { }; struct hidma_lldev { + bool msi_support; /* flag indicating MSI support */ bool initialized; /* initialized flag */ u8 trch_state; /* trch_state of the device */ u8 evch_state; /* evch_state of the device */ @@ -58,7 +59,7 @@ struct hidma_lldev { void __iomem *evca; /* Event Channel address */ struct hidma_tre **pending_tre_list; /* Pointers to pending TREs */ - s32 pending_tre_count; /* Number of TREs pending */ + atomic_t pending_tre_count; /* Number of TREs pending */ void *tre_ring; /* TRE ring */ dma_addr_t tre_dma; /* TRE ring to be shared with HW */ @@ -114,6 +115,7 @@ struct hidma_dev { int irq; int chidx; u32 nr_descriptors; + int msi_virqbase; struct hidma_lldev *lldev; void __iomem *dev_trca; @@ -128,6 +130,9 @@ struct hidma_dev { struct dentry *debugfs; struct dentry *stats; + /* sysfs entry for the channel id */ + struct device_attribute *chid_attrs; + /* Task delivering issue_pending */ struct tasklet_struct task; }; @@ -145,12 +150,14 @@ int hidma_ll_disable(struct hidma_lldev *lldev); int hidma_ll_enable(struct hidma_lldev *llhndl); void hidma_ll_set_transfer_params(struct hidma_lldev *llhndl, u32 tre_ch, dma_addr_t src, dma_addr_t dest, u32 len, u32 flags); +void hidma_ll_setup_irq(struct hidma_lldev *lldev, bool msi); int hidma_ll_setup(struct hidma_lldev *lldev); struct hidma_lldev *hidma_ll_init(struct device *dev, u32 max_channels, void __iomem *trca, void __iomem *evca, u8 chidx); int hidma_ll_uninit(struct hidma_lldev *llhndl); irqreturn_t hidma_ll_inthandler(int irq, void *arg); +irqreturn_t hidma_ll_inthandler_msi(int irq, void *arg, int cause); void hidma_cleanup_pending_tre(struct hidma_lldev *llhndl, u8 err_info, u8 err_code); int hidma_debug_init(struct hidma_dev *dmadev); diff --git a/drivers/dma/qcom/hidma_dbg.c b/drivers/dma/qcom/hidma_dbg.c index fa827e5ffd68..3bdcb8056a36 100644 --- a/drivers/dma/qcom/hidma_dbg.c +++ b/drivers/dma/qcom/hidma_dbg.c @@ -74,7 +74,8 @@ static void hidma_ll_devstats(struct seq_file *s, void *llhndl) seq_printf(s, "tre_ring_handle=%pap\n", &lldev->tre_dma); seq_printf(s, "tre_ring_size = 0x%x\n", lldev->tre_ring_size); seq_printf(s, "tre_processed_off = 0x%x\n", lldev->tre_processed_off); - seq_printf(s, "pending_tre_count=%d\n", lldev->pending_tre_count); + seq_printf(s, "pending_tre_count=%d\n", + atomic_read(&lldev->pending_tre_count)); seq_printf(s, "evca=%p\n", lldev->evca); seq_printf(s, "evre_ring=%p\n", lldev->evre_ring); seq_printf(s, "evre_ring_handle=%pap\n", &lldev->evre_dma); @@ -164,7 +165,6 @@ static const struct file_operations hidma_dma_fops = { void hidma_debug_uninit(struct hidma_dev *dmadev) { debugfs_remove_recursive(dmadev->debugfs); - debugfs_remove_recursive(dmadev->stats); } int hidma_debug_init(struct hidma_dev *dmadev) diff --git a/drivers/dma/qcom/hidma_ll.c b/drivers/dma/qcom/hidma_ll.c index 3224f24c577b..6645bdf0d151 100644 --- a/drivers/dma/qcom/hidma_ll.c +++ b/drivers/dma/qcom/hidma_ll.c @@ -198,13 +198,16 @@ static void hidma_ll_tre_complete(unsigned long arg) } } -static int hidma_post_completed(struct hidma_lldev *lldev, int tre_iterator, - u8 err_info, u8 err_code) +static int hidma_post_completed(struct hidma_lldev *lldev, u8 err_info, + u8 err_code) { struct hidma_tre *tre; unsigned long flags; + u32 tre_iterator; spin_lock_irqsave(&lldev->lock, flags); + + tre_iterator = lldev->tre_processed_off; tre = lldev->pending_tre_list[tre_iterator / HIDMA_TRE_SIZE]; if (!tre) { spin_unlock_irqrestore(&lldev->lock, flags); @@ -218,12 +221,14 @@ static int hidma_post_completed(struct hidma_lldev *lldev, int tre_iterator, * Keep track of pending TREs that SW is expecting to receive * from HW. We got one now. Decrement our counter. */ - lldev->pending_tre_count--; - if (lldev->pending_tre_count < 0) { + if (atomic_dec_return(&lldev->pending_tre_count) < 0) { dev_warn(lldev->dev, "tre count mismatch on completion"); - lldev->pending_tre_count = 0; + atomic_set(&lldev->pending_tre_count, 0); } + HIDMA_INCREMENT_ITERATOR(tre_iterator, HIDMA_TRE_SIZE, + lldev->tre_ring_size); + lldev->tre_processed_off = tre_iterator; spin_unlock_irqrestore(&lldev->lock, flags); tre->err_info = err_info; @@ -245,13 +250,11 @@ static int hidma_post_completed(struct hidma_lldev *lldev, int tre_iterator, static int hidma_handle_tre_completion(struct hidma_lldev *lldev) { u32 evre_ring_size = lldev->evre_ring_size; - u32 tre_ring_size = lldev->tre_ring_size; u32 err_info, err_code, evre_write_off; - u32 tre_iterator, evre_iterator; + u32 evre_iterator; u32 num_completed = 0; evre_write_off = readl_relaxed(lldev->evca + HIDMA_EVCA_WRITE_PTR_REG); - tre_iterator = lldev->tre_processed_off; evre_iterator = lldev->evre_processed_off; if ((evre_write_off > evre_ring_size) || @@ -274,12 +277,9 @@ static int hidma_handle_tre_completion(struct hidma_lldev *lldev) err_code = (cfg >> HIDMA_EVRE_CODE_BIT_POS) & HIDMA_EVRE_CODE_MASK; - if (hidma_post_completed(lldev, tre_iterator, err_info, - err_code)) + if (hidma_post_completed(lldev, err_info, err_code)) break; - HIDMA_INCREMENT_ITERATOR(tre_iterator, HIDMA_TRE_SIZE, - tre_ring_size); HIDMA_INCREMENT_ITERATOR(evre_iterator, HIDMA_EVRE_SIZE, evre_ring_size); @@ -291,21 +291,22 @@ static int hidma_handle_tre_completion(struct hidma_lldev *lldev) evre_write_off = readl_relaxed(lldev->evca + HIDMA_EVCA_WRITE_PTR_REG); num_completed++; + + /* + * An error interrupt might have arrived while we are processing + * the completed interrupt. + */ + if (!hidma_ll_isenabled(lldev)) + break; } if (num_completed) { u32 evre_read_off = (lldev->evre_processed_off + HIDMA_EVRE_SIZE * num_completed); - u32 tre_read_off = (lldev->tre_processed_off + - HIDMA_TRE_SIZE * num_completed); - evre_read_off = evre_read_off % evre_ring_size; - tre_read_off = tre_read_off % tre_ring_size; - writel(evre_read_off, lldev->evca + HIDMA_EVCA_DOORBELL_REG); /* record the last processed tre offset */ - lldev->tre_processed_off = tre_read_off; lldev->evre_processed_off = evre_read_off; } @@ -315,27 +316,10 @@ static int hidma_handle_tre_completion(struct hidma_lldev *lldev) void hidma_cleanup_pending_tre(struct hidma_lldev *lldev, u8 err_info, u8 err_code) { - u32 tre_iterator; - u32 tre_ring_size = lldev->tre_ring_size; - int num_completed = 0; - u32 tre_read_off; - - tre_iterator = lldev->tre_processed_off; - while (lldev->pending_tre_count) { - if (hidma_post_completed(lldev, tre_iterator, err_info, - err_code)) + while (atomic_read(&lldev->pending_tre_count)) { + if (hidma_post_completed(lldev, err_info, err_code)) break; - HIDMA_INCREMENT_ITERATOR(tre_iterator, HIDMA_TRE_SIZE, - tre_ring_size); - num_completed++; } - tre_read_off = (lldev->tre_processed_off + - HIDMA_TRE_SIZE * num_completed); - - tre_read_off = tre_read_off % tre_ring_size; - - /* record the last processed tre offset */ - lldev->tre_processed_off = tre_read_off; } static int hidma_ll_reset(struct hidma_lldev *lldev) @@ -412,12 +396,24 @@ static int hidma_ll_reset(struct hidma_lldev *lldev) * requests traditionally to the destination, this concept does not apply * here for this HW. */ -irqreturn_t hidma_ll_inthandler(int chirq, void *arg) +static void hidma_ll_int_handler_internal(struct hidma_lldev *lldev, int cause) { - struct hidma_lldev *lldev = arg; - u32 status; - u32 enable; - u32 cause; + if (cause & HIDMA_ERR_INT_MASK) { + dev_err(lldev->dev, "error 0x%x, disabling...\n", + cause); + + /* Clear out pending interrupts */ + writel(cause, lldev->evca + HIDMA_EVCA_IRQ_CLR_REG); + + /* No further submissions. */ + hidma_ll_disable(lldev); + + /* Driver completes the txn and intimates the client.*/ + hidma_cleanup_pending_tre(lldev, 0xFF, + HIDMA_EVRE_STATUS_ERROR); + + return; + } /* * Fine tuned for this HW... @@ -426,35 +422,28 @@ irqreturn_t hidma_ll_inthandler(int chirq, void *arg) * read and write accessors are used for performance reasons due to * interrupt delivery guarantees. Do not copy this code blindly and * expect that to work. + * + * Try to consume as many EVREs as possible. */ + hidma_handle_tre_completion(lldev); + + /* We consumed TREs or there are pending TREs or EVREs. */ + writel_relaxed(cause, lldev->evca + HIDMA_EVCA_IRQ_CLR_REG); +} + +irqreturn_t hidma_ll_inthandler(int chirq, void *arg) +{ + struct hidma_lldev *lldev = arg; + u32 status; + u32 enable; + u32 cause; + status = readl_relaxed(lldev->evca + HIDMA_EVCA_IRQ_STAT_REG); enable = readl_relaxed(lldev->evca + HIDMA_EVCA_IRQ_EN_REG); cause = status & enable; while (cause) { - if (cause & HIDMA_ERR_INT_MASK) { - dev_err(lldev->dev, "error 0x%x, disabling...\n", - cause); - - /* Clear out pending interrupts */ - writel(cause, lldev->evca + HIDMA_EVCA_IRQ_CLR_REG); - - /* No further submissions. */ - hidma_ll_disable(lldev); - - /* Driver completes the txn and intimates the client.*/ - hidma_cleanup_pending_tre(lldev, 0xFF, - HIDMA_EVRE_STATUS_ERROR); - goto out; - } - - /* - * Try to consume as many EVREs as possible. - */ - hidma_handle_tre_completion(lldev); - - /* We consumed TREs or there are pending TREs or EVREs. */ - writel_relaxed(cause, lldev->evca + HIDMA_EVCA_IRQ_CLR_REG); + hidma_ll_int_handler_internal(lldev, cause); /* * Another interrupt might have arrived while we are @@ -465,7 +454,14 @@ irqreturn_t hidma_ll_inthandler(int chirq, void *arg) cause = status & enable; } -out: + return IRQ_HANDLED; +} + +irqreturn_t hidma_ll_inthandler_msi(int chirq, void *arg, int cause) +{ + struct hidma_lldev *lldev = arg; + + hidma_ll_int_handler_internal(lldev, cause); return IRQ_HANDLED; } @@ -548,7 +544,7 @@ void hidma_ll_queue_request(struct hidma_lldev *lldev, u32 tre_ch) tre->err_code = 0; tre->err_info = 0; tre->queued = 1; - lldev->pending_tre_count++; + atomic_inc(&lldev->pending_tre_count); lldev->tre_write_offset = (lldev->tre_write_offset + HIDMA_TRE_SIZE) % lldev->tre_ring_size; spin_unlock_irqrestore(&lldev->lock, flags); @@ -564,19 +560,8 @@ int hidma_ll_disable(struct hidma_lldev *lldev) u32 val; int ret; - val = readl(lldev->evca + HIDMA_EVCA_CTRLSTS_REG); - lldev->evch_state = HIDMA_CH_STATE(val); - val = readl(lldev->trca + HIDMA_TRCA_CTRLSTS_REG); - lldev->trch_state = HIDMA_CH_STATE(val); - - /* already suspended by this OS */ - if ((lldev->trch_state == HIDMA_CH_SUSPENDED) || - (lldev->evch_state == HIDMA_CH_SUSPENDED)) - return 0; - - /* already stopped by the manager */ - if ((lldev->trch_state == HIDMA_CH_STOPPED) || - (lldev->evch_state == HIDMA_CH_STOPPED)) + /* The channel needs to be in working state */ + if (!hidma_ll_isenabled(lldev)) return 0; val = readl(lldev->trca + HIDMA_TRCA_CTRLSTS_REG); @@ -654,7 +639,7 @@ int hidma_ll_setup(struct hidma_lldev *lldev) u32 val; u32 nr_tres = lldev->nr_tres; - lldev->pending_tre_count = 0; + atomic_set(&lldev->pending_tre_count, 0); lldev->tre_processed_off = 0; lldev->evre_processed_off = 0; lldev->tre_write_offset = 0; @@ -691,17 +676,36 @@ int hidma_ll_setup(struct hidma_lldev *lldev) writel(HIDMA_EVRE_SIZE * nr_tres, lldev->evca + HIDMA_EVCA_RING_LEN_REG); - /* support IRQ only for now */ + /* configure interrupts */ + hidma_ll_setup_irq(lldev, lldev->msi_support); + + rc = hidma_ll_enable(lldev); + if (rc) + return rc; + + return rc; +} + +void hidma_ll_setup_irq(struct hidma_lldev *lldev, bool msi) +{ + u32 val; + + lldev->msi_support = msi; + + /* disable interrupts again after reset */ + writel(0, lldev->evca + HIDMA_EVCA_IRQ_CLR_REG); + writel(0, lldev->evca + HIDMA_EVCA_IRQ_EN_REG); + + /* support IRQ by default */ val = readl(lldev->evca + HIDMA_EVCA_INTCTRL_REG); val &= ~0xF; - val |= 0x1; + if (!lldev->msi_support) + val = val | 0x1; writel(val, lldev->evca + HIDMA_EVCA_INTCTRL_REG); /* clear all pending interrupts and enable them */ writel(ENABLE_IRQS, lldev->evca + HIDMA_EVCA_IRQ_CLR_REG); writel(ENABLE_IRQS, lldev->evca + HIDMA_EVCA_IRQ_EN_REG); - - return hidma_ll_enable(lldev); } struct hidma_lldev *hidma_ll_init(struct device *dev, u32 nr_tres, @@ -816,7 +820,7 @@ int hidma_ll_uninit(struct hidma_lldev *lldev) tasklet_kill(&lldev->task); memset(lldev->trepool, 0, required_bytes); lldev->trepool = NULL; - lldev->pending_tre_count = 0; + atomic_set(&lldev->pending_tre_count, 0); lldev->tre_write_offset = 0; rc = hidma_ll_reset(lldev); diff --git a/drivers/dma/qcom/hidma_mgmt.c b/drivers/dma/qcom/hidma_mgmt.c index 82f36e466083..f847d32cc4b5 100644 --- a/drivers/dma/qcom/hidma_mgmt.c +++ b/drivers/dma/qcom/hidma_mgmt.c @@ -282,6 +282,7 @@ static const struct acpi_device_id hidma_mgmt_acpi_ids[] = { {"QCOM8060"}, {}, }; +MODULE_DEVICE_TABLE(acpi, hidma_mgmt_acpi_ids); #endif static const struct of_device_id hidma_mgmt_match[] = { @@ -375,8 +376,15 @@ static int __init hidma_mgmt_of_populate_channels(struct device_node *np) ret = PTR_ERR(new_pdev); goto out; } + of_node_get(child); + new_pdev->dev.of_node = child; of_dma_configure(&new_pdev->dev, child); - + /* + * It is assumed that calling of_msi_configure is safe on + * platforms with or without MSI support. + */ + of_msi_configure(&new_pdev->dev, child); + of_node_put(child); kfree(res); res = NULL; } @@ -395,7 +403,6 @@ static int __init hidma_mgmt_init(void) for_each_matching_node(child, hidma_mgmt_match) { /* device tree based firmware here */ hidma_mgmt_of_populate_channels(child); - of_node_put(child); } #endif platform_driver_register(&hidma_mgmt_driver); diff --git a/drivers/dma/s3c24xx-dma.c b/drivers/dma/s3c24xx-dma.c index 3c579abbabb7..f04c4702d98b 100644 --- a/drivers/dma/s3c24xx-dma.c +++ b/drivers/dma/s3c24xx-dma.c @@ -289,16 +289,11 @@ static struct s3c24xx_dma_phy *s3c24xx_dma_get_phy(struct s3c24xx_dma_chan *s3cchan) { struct s3c24xx_dma_engine *s3cdma = s3cchan->host; - const struct s3c24xx_dma_platdata *pdata = s3cdma->pdata; - struct s3c24xx_dma_channel *cdata; struct s3c24xx_dma_phy *phy = NULL; unsigned long flags; int i; int ret; - if (s3cchan->slave) - cdata = &pdata->channels[s3cchan->id]; - for (i = 0; i < s3cdma->pdata->num_phy_channels; i++) { phy = &s3cdma->phy_chans[i]; diff --git a/drivers/dma/sh/rcar-dmac.c b/drivers/dma/sh/rcar-dmac.c index 2e441d0ccd79..48b22d5c8602 100644 --- a/drivers/dma/sh/rcar-dmac.c +++ b/drivers/dma/sh/rcar-dmac.c @@ -986,6 +986,7 @@ static void rcar_dmac_free_chan_resources(struct dma_chan *chan) { struct rcar_dmac_chan *rchan = to_rcar_dmac_chan(chan); struct rcar_dmac *dmac = to_rcar_dmac(chan->device); + struct rcar_dmac_chan_map *map = &rchan->map; struct rcar_dmac_desc_page *page, *_page; struct rcar_dmac_desc *desc; LIST_HEAD(list); @@ -1019,6 +1020,13 @@ static void rcar_dmac_free_chan_resources(struct dma_chan *chan) free_page((unsigned long)page); } + /* Remove slave mapping if present. */ + if (map->slave.xfer_size) { + dma_unmap_resource(chan->device->dev, map->addr, + map->slave.xfer_size, map->dir, 0); + map->slave.xfer_size = 0; + } + pm_runtime_put(chan->device->dev); } @@ -1716,6 +1724,7 @@ static int rcar_dmac_probe(struct platform_device *pdev) dmac->dev = &pdev->dev; platform_set_drvdata(pdev, dmac); + dma_set_mask_and_coherent(dmac->dev, DMA_BIT_MASK(40)); ret = rcar_dmac_parse_of(&pdev->dev, dmac); if (ret < 0) diff --git a/drivers/dma/sh/usb-dmac.c b/drivers/dma/sh/usb-dmac.c index 06ecdc38cee0..72c649713ace 100644 --- a/drivers/dma/sh/usb-dmac.c +++ b/drivers/dma/sh/usb-dmac.c @@ -652,7 +652,6 @@ static bool usb_dmac_chan_filter(struct dma_chan *chan, void *arg) static struct dma_chan *usb_dmac_of_xlate(struct of_phandle_args *dma_spec, struct of_dma *ofdma) { - struct usb_dmac_chan *uchan; struct dma_chan *chan; dma_cap_mask_t mask; @@ -667,8 +666,6 @@ static struct dma_chan *usb_dmac_of_xlate(struct of_phandle_args *dma_spec, if (!chan) return NULL; - uchan = to_usb_dmac_chan(chan); - return chan; } diff --git a/drivers/dma/sirf-dma.c b/drivers/dma/sirf-dma.c index 8f62edad51be..a0733ac3edb1 100644 --- a/drivers/dma/sirf-dma.c +++ b/drivers/dma/sirf-dma.c @@ -1011,7 +1011,6 @@ static int __maybe_unused sirfsoc_dma_pm_suspend(struct device *dev) { struct sirfsoc_dma *sdma = dev_get_drvdata(dev); struct sirfsoc_dma_regs *save = &sdma->regs_save; - struct sirfsoc_dma_desc *sdesc; struct sirfsoc_dma_chan *schan; int ch; int ret; @@ -1044,9 +1043,6 @@ static int __maybe_unused sirfsoc_dma_pm_suspend(struct device *dev) schan = &sdma->channels[ch]; if (list_empty(&schan->active)) continue; - sdesc = list_first_entry(&schan->active, - struct sirfsoc_dma_desc, - node); save->ctrl[ch] = readl_relaxed(sdma->base + ch * 0x10 + SIRFSOC_DMA_CH_CTRL); } diff --git a/drivers/dma/st_fdma.c b/drivers/dma/st_fdma.c new file mode 100644 index 000000000000..bfb79bd0c6de --- /dev/null +++ b/drivers/dma/st_fdma.c @@ -0,0 +1,889 @@ +/* + * DMA driver for STMicroelectronics STi FDMA controller + * + * Copyright (C) 2014 STMicroelectronics + * + * Author: Ludovic Barre <Ludovic.barre@st.com> + * Peter Griffin <peter.griffin@linaro.org> + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + */ + +#include <linux/init.h> +#include <linux/module.h> +#include <linux/of_device.h> +#include <linux/of_dma.h> +#include <linux/platform_device.h> +#include <linux/interrupt.h> +#include <linux/remoteproc.h> + +#include "st_fdma.h" + +static inline struct st_fdma_chan *to_st_fdma_chan(struct dma_chan *c) +{ + return container_of(c, struct st_fdma_chan, vchan.chan); +} + +static struct st_fdma_desc *to_st_fdma_desc(struct virt_dma_desc *vd) +{ + return container_of(vd, struct st_fdma_desc, vdesc); +} + +static int st_fdma_dreq_get(struct st_fdma_chan *fchan) +{ + struct st_fdma_dev *fdev = fchan->fdev; + u32 req_line_cfg = fchan->cfg.req_line; + u32 dreq_line; + int try = 0; + + /* + * dreq_mask is shared for n channels of fdma, so all accesses must be + * atomic. if the dreq_mask is changed between ffz and set_bit, + * we retry + */ + do { + if (fdev->dreq_mask == ~0L) { + dev_err(fdev->dev, "No req lines available\n"); + return -EINVAL; + } + + if (try || req_line_cfg >= ST_FDMA_NR_DREQS) { + dev_err(fdev->dev, "Invalid or used req line\n"); + return -EINVAL; + } else { + dreq_line = req_line_cfg; + } + + try++; + } while (test_and_set_bit(dreq_line, &fdev->dreq_mask)); + + dev_dbg(fdev->dev, "get dreq_line:%d mask:%#lx\n", + dreq_line, fdev->dreq_mask); + + return dreq_line; +} + +static void st_fdma_dreq_put(struct st_fdma_chan *fchan) +{ + struct st_fdma_dev *fdev = fchan->fdev; + + dev_dbg(fdev->dev, "put dreq_line:%#x\n", fchan->dreq_line); + clear_bit(fchan->dreq_line, &fdev->dreq_mask); +} + +static void st_fdma_xfer_desc(struct st_fdma_chan *fchan) +{ + struct virt_dma_desc *vdesc; + unsigned long nbytes, ch_cmd, cmd; + + vdesc = vchan_next_desc(&fchan->vchan); + if (!vdesc) + return; + + fchan->fdesc = to_st_fdma_desc(vdesc); + nbytes = fchan->fdesc->node[0].desc->nbytes; + cmd = FDMA_CMD_START(fchan->vchan.chan.chan_id); + ch_cmd = fchan->fdesc->node[0].pdesc | FDMA_CH_CMD_STA_START; + + /* start the channel for the descriptor */ + fnode_write(fchan, nbytes, FDMA_CNTN_OFST); + fchan_write(fchan, ch_cmd, FDMA_CH_CMD_OFST); + writel(cmd, + fchan->fdev->slim_rproc->peri + FDMA_CMD_SET_OFST); + + dev_dbg(fchan->fdev->dev, "start chan:%d\n", fchan->vchan.chan.chan_id); +} + +static void st_fdma_ch_sta_update(struct st_fdma_chan *fchan, + unsigned long int_sta) +{ + unsigned long ch_sta, ch_err; + int ch_id = fchan->vchan.chan.chan_id; + struct st_fdma_dev *fdev = fchan->fdev; + + ch_sta = fchan_read(fchan, FDMA_CH_CMD_OFST); + ch_err = ch_sta & FDMA_CH_CMD_ERR_MASK; + ch_sta &= FDMA_CH_CMD_STA_MASK; + + if (int_sta & FDMA_INT_STA_ERR) { + dev_warn(fdev->dev, "chan:%d, error:%ld\n", ch_id, ch_err); + fchan->status = DMA_ERROR; + return; + } + + switch (ch_sta) { + case FDMA_CH_CMD_STA_PAUSED: + fchan->status = DMA_PAUSED; + break; + + case FDMA_CH_CMD_STA_RUNNING: + fchan->status = DMA_IN_PROGRESS; + break; + } +} + +static irqreturn_t st_fdma_irq_handler(int irq, void *dev_id) +{ + struct st_fdma_dev *fdev = dev_id; + irqreturn_t ret = IRQ_NONE; + struct st_fdma_chan *fchan = &fdev->chans[0]; + unsigned long int_sta, clr; + + int_sta = fdma_read(fdev, FDMA_INT_STA_OFST); + clr = int_sta; + + for (; int_sta != 0 ; int_sta >>= 2, fchan++) { + if (!(int_sta & (FDMA_INT_STA_CH | FDMA_INT_STA_ERR))) + continue; + + spin_lock(&fchan->vchan.lock); + st_fdma_ch_sta_update(fchan, int_sta); + + if (fchan->fdesc) { + if (!fchan->fdesc->iscyclic) { + list_del(&fchan->fdesc->vdesc.node); + vchan_cookie_complete(&fchan->fdesc->vdesc); + fchan->fdesc = NULL; + fchan->status = DMA_COMPLETE; + } else { + vchan_cyclic_callback(&fchan->fdesc->vdesc); + } + + /* Start the next descriptor (if available) */ + if (!fchan->fdesc) + st_fdma_xfer_desc(fchan); + } + + spin_unlock(&fchan->vchan.lock); + ret = IRQ_HANDLED; + } + + fdma_write(fdev, clr, FDMA_INT_CLR_OFST); + + return ret; +} + +static struct dma_chan *st_fdma_of_xlate(struct of_phandle_args *dma_spec, + struct of_dma *ofdma) +{ + struct st_fdma_dev *fdev = ofdma->of_dma_data; + struct dma_chan *chan; + struct st_fdma_chan *fchan; + int ret; + + if (dma_spec->args_count < 1) + return ERR_PTR(-EINVAL); + + if (fdev->dma_device.dev->of_node != dma_spec->np) + return ERR_PTR(-EINVAL); + + ret = rproc_boot(fdev->slim_rproc->rproc); + if (ret == -ENOENT) + return ERR_PTR(-EPROBE_DEFER); + else if (ret) + return ERR_PTR(ret); + + chan = dma_get_any_slave_channel(&fdev->dma_device); + if (!chan) + goto err_chan; + + fchan = to_st_fdma_chan(chan); + + fchan->cfg.of_node = dma_spec->np; + fchan->cfg.req_line = dma_spec->args[0]; + fchan->cfg.req_ctrl = 0; + fchan->cfg.type = ST_FDMA_TYPE_FREE_RUN; + + if (dma_spec->args_count > 1) + fchan->cfg.req_ctrl = dma_spec->args[1] + & FDMA_REQ_CTRL_CFG_MASK; + + if (dma_spec->args_count > 2) + fchan->cfg.type = dma_spec->args[2]; + + if (fchan->cfg.type == ST_FDMA_TYPE_FREE_RUN) { + fchan->dreq_line = 0; + } else { + fchan->dreq_line = st_fdma_dreq_get(fchan); + if (IS_ERR_VALUE(fchan->dreq_line)) { + chan = ERR_PTR(fchan->dreq_line); + goto err_chan; + } + } + + dev_dbg(fdev->dev, "xlate req_line:%d type:%d req_ctrl:%#lx\n", + fchan->cfg.req_line, fchan->cfg.type, fchan->cfg.req_ctrl); + + return chan; + +err_chan: + rproc_shutdown(fdev->slim_rproc->rproc); + return chan; + +} + +static void st_fdma_free_desc(struct virt_dma_desc *vdesc) +{ + struct st_fdma_desc *fdesc; + int i; + + fdesc = to_st_fdma_desc(vdesc); + for (i = 0; i < fdesc->n_nodes; i++) + dma_pool_free(fdesc->fchan->node_pool, fdesc->node[i].desc, + fdesc->node[i].pdesc); + kfree(fdesc); +} + +static struct st_fdma_desc *st_fdma_alloc_desc(struct st_fdma_chan *fchan, + int sg_len) +{ + struct st_fdma_desc *fdesc; + int i; + + fdesc = kzalloc(sizeof(*fdesc) + + sizeof(struct st_fdma_sw_node) * sg_len, GFP_NOWAIT); + if (!fdesc) + return NULL; + + fdesc->fchan = fchan; + fdesc->n_nodes = sg_len; + for (i = 0; i < sg_len; i++) { + fdesc->node[i].desc = dma_pool_alloc(fchan->node_pool, + GFP_NOWAIT, &fdesc->node[i].pdesc); + if (!fdesc->node[i].desc) + goto err; + } + return fdesc; + +err: + while (--i >= 0) + dma_pool_free(fchan->node_pool, fdesc->node[i].desc, + fdesc->node[i].pdesc); + kfree(fdesc); + return NULL; +} + +static int st_fdma_alloc_chan_res(struct dma_chan *chan) +{ + struct st_fdma_chan *fchan = to_st_fdma_chan(chan); + + /* Create the dma pool for descriptor allocation */ + fchan->node_pool = dma_pool_create(dev_name(&chan->dev->device), + fchan->fdev->dev, + sizeof(struct st_fdma_hw_node), + __alignof__(struct st_fdma_hw_node), + 0); + + if (!fchan->node_pool) { + dev_err(fchan->fdev->dev, "unable to allocate desc pool\n"); + return -ENOMEM; + } + + dev_dbg(fchan->fdev->dev, "alloc ch_id:%d type:%d\n", + fchan->vchan.chan.chan_id, fchan->cfg.type); + + return 0; +} + +static void st_fdma_free_chan_res(struct dma_chan *chan) +{ + struct st_fdma_chan *fchan = to_st_fdma_chan(chan); + struct rproc *rproc = fchan->fdev->slim_rproc->rproc; + unsigned long flags; + + LIST_HEAD(head); + + dev_dbg(fchan->fdev->dev, "%s: freeing chan:%d\n", + __func__, fchan->vchan.chan.chan_id); + + if (fchan->cfg.type != ST_FDMA_TYPE_FREE_RUN) + st_fdma_dreq_put(fchan); + + spin_lock_irqsave(&fchan->vchan.lock, flags); + fchan->fdesc = NULL; + spin_unlock_irqrestore(&fchan->vchan.lock, flags); + + dma_pool_destroy(fchan->node_pool); + fchan->node_pool = NULL; + memset(&fchan->cfg, 0, sizeof(struct st_fdma_cfg)); + + rproc_shutdown(rproc); +} + +static struct dma_async_tx_descriptor *st_fdma_prep_dma_memcpy( + struct dma_chan *chan, dma_addr_t dst, dma_addr_t src, + size_t len, unsigned long flags) +{ + struct st_fdma_chan *fchan; + struct st_fdma_desc *fdesc; + struct st_fdma_hw_node *hw_node; + + if (!len) + return NULL; + + fchan = to_st_fdma_chan(chan); + + /* We only require a single descriptor */ + fdesc = st_fdma_alloc_desc(fchan, 1); + if (!fdesc) { + dev_err(fchan->fdev->dev, "no memory for desc\n"); + return NULL; + } + + hw_node = fdesc->node[0].desc; + hw_node->next = 0; + hw_node->control = FDMA_NODE_CTRL_REQ_MAP_FREE_RUN; + hw_node->control |= FDMA_NODE_CTRL_SRC_INCR; + hw_node->control |= FDMA_NODE_CTRL_DST_INCR; + hw_node->control |= FDMA_NODE_CTRL_INT_EON; + hw_node->nbytes = len; + hw_node->saddr = src; + hw_node->daddr = dst; + hw_node->generic.length = len; + hw_node->generic.sstride = 0; + hw_node->generic.dstride = 0; + + return vchan_tx_prep(&fchan->vchan, &fdesc->vdesc, flags); +} + +static int config_reqctrl(struct st_fdma_chan *fchan, + enum dma_transfer_direction direction) +{ + u32 maxburst = 0, addr = 0; + enum dma_slave_buswidth width; + int ch_id = fchan->vchan.chan.chan_id; + struct st_fdma_dev *fdev = fchan->fdev; + + switch (direction) { + + case DMA_DEV_TO_MEM: + fchan->cfg.req_ctrl &= ~FDMA_REQ_CTRL_WNR; + maxburst = fchan->scfg.src_maxburst; + width = fchan->scfg.src_addr_width; + addr = fchan->scfg.src_addr; + break; + + case DMA_MEM_TO_DEV: + fchan->cfg.req_ctrl |= FDMA_REQ_CTRL_WNR; + maxburst = fchan->scfg.dst_maxburst; + width = fchan->scfg.dst_addr_width; + addr = fchan->scfg.dst_addr; + break; + + default: + return -EINVAL; + } + + fchan->cfg.req_ctrl &= ~FDMA_REQ_CTRL_OPCODE_MASK; + + switch (width) { + + case DMA_SLAVE_BUSWIDTH_1_BYTE: + fchan->cfg.req_ctrl |= FDMA_REQ_CTRL_OPCODE_LD_ST1; + break; + + case DMA_SLAVE_BUSWIDTH_2_BYTES: + fchan->cfg.req_ctrl |= FDMA_REQ_CTRL_OPCODE_LD_ST2; + break; + + case DMA_SLAVE_BUSWIDTH_4_BYTES: + fchan->cfg.req_ctrl |= FDMA_REQ_CTRL_OPCODE_LD_ST4; + break; + + case DMA_SLAVE_BUSWIDTH_8_BYTES: + fchan->cfg.req_ctrl |= FDMA_REQ_CTRL_OPCODE_LD_ST8; + break; + + default: + return -EINVAL; + } + + fchan->cfg.req_ctrl &= ~FDMA_REQ_CTRL_NUM_OPS_MASK; + fchan->cfg.req_ctrl |= FDMA_REQ_CTRL_NUM_OPS(maxburst-1); + dreq_write(fchan, fchan->cfg.req_ctrl, FDMA_REQ_CTRL_OFST); + + fchan->cfg.dev_addr = addr; + fchan->cfg.dir = direction; + + dev_dbg(fdev->dev, "chan:%d config_reqctrl:%#x req_ctrl:%#lx\n", + ch_id, addr, fchan->cfg.req_ctrl); + + return 0; +} + +static void fill_hw_node(struct st_fdma_hw_node *hw_node, + struct st_fdma_chan *fchan, + enum dma_transfer_direction direction) +{ + if (direction == DMA_MEM_TO_DEV) { + hw_node->control |= FDMA_NODE_CTRL_SRC_INCR; + hw_node->control |= FDMA_NODE_CTRL_DST_STATIC; + hw_node->daddr = fchan->cfg.dev_addr; + } else { + hw_node->control |= FDMA_NODE_CTRL_SRC_STATIC; + hw_node->control |= FDMA_NODE_CTRL_DST_INCR; + hw_node->saddr = fchan->cfg.dev_addr; + } + + hw_node->generic.sstride = 0; + hw_node->generic.dstride = 0; +} + +static inline struct st_fdma_chan *st_fdma_prep_common(struct dma_chan *chan, + size_t len, enum dma_transfer_direction direction) +{ + struct st_fdma_chan *fchan; + + if (!chan || !len) + return NULL; + + fchan = to_st_fdma_chan(chan); + + if (!is_slave_direction(direction)) { + dev_err(fchan->fdev->dev, "bad direction?\n"); + return NULL; + } + + return fchan; +} + +static struct dma_async_tx_descriptor *st_fdma_prep_dma_cyclic( + struct dma_chan *chan, dma_addr_t buf_addr, size_t len, + size_t period_len, enum dma_transfer_direction direction, + unsigned long flags) +{ + struct st_fdma_chan *fchan; + struct st_fdma_desc *fdesc; + int sg_len, i; + + fchan = st_fdma_prep_common(chan, len, direction); + if (!fchan) + return NULL; + + if (!period_len) + return NULL; + + if (config_reqctrl(fchan, direction)) { + dev_err(fchan->fdev->dev, "bad width or direction\n"); + return NULL; + } + + /* the buffer length must be a multiple of period_len */ + if (len % period_len != 0) { + dev_err(fchan->fdev->dev, "len is not multiple of period\n"); + return NULL; + } + + sg_len = len / period_len; + fdesc = st_fdma_alloc_desc(fchan, sg_len); + if (!fdesc) { + dev_err(fchan->fdev->dev, "no memory for desc\n"); + return NULL; + } + + fdesc->iscyclic = true; + + for (i = 0; i < sg_len; i++) { + struct st_fdma_hw_node *hw_node = fdesc->node[i].desc; + + hw_node->next = fdesc->node[(i + 1) % sg_len].pdesc; + + hw_node->control = + FDMA_NODE_CTRL_REQ_MAP_DREQ(fchan->dreq_line); + hw_node->control |= FDMA_NODE_CTRL_INT_EON; + + fill_hw_node(hw_node, fchan, direction); + + if (direction == DMA_MEM_TO_DEV) + hw_node->saddr = buf_addr + (i * period_len); + else + hw_node->daddr = buf_addr + (i * period_len); + + hw_node->nbytes = period_len; + hw_node->generic.length = period_len; + } + + return vchan_tx_prep(&fchan->vchan, &fdesc->vdesc, flags); +} + +static struct dma_async_tx_descriptor *st_fdma_prep_slave_sg( + struct dma_chan *chan, struct scatterlist *sgl, + unsigned int sg_len, enum dma_transfer_direction direction, + unsigned long flags, void *context) +{ + struct st_fdma_chan *fchan; + struct st_fdma_desc *fdesc; + struct st_fdma_hw_node *hw_node; + struct scatterlist *sg; + int i; + + fchan = st_fdma_prep_common(chan, sg_len, direction); + if (!fchan) + return NULL; + + if (!sgl) + return NULL; + + fdesc = st_fdma_alloc_desc(fchan, sg_len); + if (!fdesc) { + dev_err(fchan->fdev->dev, "no memory for desc\n"); + return NULL; + } + + fdesc->iscyclic = false; + + for_each_sg(sgl, sg, sg_len, i) { + hw_node = fdesc->node[i].desc; + + hw_node->next = fdesc->node[(i + 1) % sg_len].pdesc; + hw_node->control = FDMA_NODE_CTRL_REQ_MAP_DREQ(fchan->dreq_line); + + fill_hw_node(hw_node, fchan, direction); + + if (direction == DMA_MEM_TO_DEV) + hw_node->saddr = sg_dma_address(sg); + else + hw_node->daddr = sg_dma_address(sg); + + hw_node->nbytes = sg_dma_len(sg); + hw_node->generic.length = sg_dma_len(sg); + } + + /* interrupt at end of last node */ + hw_node->control |= FDMA_NODE_CTRL_INT_EON; + + return vchan_tx_prep(&fchan->vchan, &fdesc->vdesc, flags); +} + +static size_t st_fdma_desc_residue(struct st_fdma_chan *fchan, + struct virt_dma_desc *vdesc, + bool in_progress) +{ + struct st_fdma_desc *fdesc = fchan->fdesc; + size_t residue = 0; + dma_addr_t cur_addr = 0; + int i; + + if (in_progress) { + cur_addr = fchan_read(fchan, FDMA_CH_CMD_OFST); + cur_addr &= FDMA_CH_CMD_DATA_MASK; + } + + for (i = fchan->fdesc->n_nodes - 1 ; i >= 0; i--) { + if (cur_addr == fdesc->node[i].pdesc) { + residue += fnode_read(fchan, FDMA_CNTN_OFST); + break; + } + residue += fdesc->node[i].desc->nbytes; + } + + return residue; +} + +static enum dma_status st_fdma_tx_status(struct dma_chan *chan, + dma_cookie_t cookie, + struct dma_tx_state *txstate) +{ + struct st_fdma_chan *fchan = to_st_fdma_chan(chan); + struct virt_dma_desc *vd; + enum dma_status ret; + unsigned long flags; + + ret = dma_cookie_status(chan, cookie, txstate); + if (ret == DMA_COMPLETE || !txstate) + return ret; + + spin_lock_irqsave(&fchan->vchan.lock, flags); + vd = vchan_find_desc(&fchan->vchan, cookie); + if (fchan->fdesc && cookie == fchan->fdesc->vdesc.tx.cookie) + txstate->residue = st_fdma_desc_residue(fchan, vd, true); + else if (vd) + txstate->residue = st_fdma_desc_residue(fchan, vd, false); + else + txstate->residue = 0; + + spin_unlock_irqrestore(&fchan->vchan.lock, flags); + + return ret; +} + +static void st_fdma_issue_pending(struct dma_chan *chan) +{ + struct st_fdma_chan *fchan = to_st_fdma_chan(chan); + unsigned long flags; + + spin_lock_irqsave(&fchan->vchan.lock, flags); + + if (vchan_issue_pending(&fchan->vchan) && !fchan->fdesc) + st_fdma_xfer_desc(fchan); + + spin_unlock_irqrestore(&fchan->vchan.lock, flags); +} + +static int st_fdma_pause(struct dma_chan *chan) +{ + unsigned long flags; + LIST_HEAD(head); + struct st_fdma_chan *fchan = to_st_fdma_chan(chan); + int ch_id = fchan->vchan.chan.chan_id; + unsigned long cmd = FDMA_CMD_PAUSE(ch_id); + + dev_dbg(fchan->fdev->dev, "pause chan:%d\n", ch_id); + + spin_lock_irqsave(&fchan->vchan.lock, flags); + if (fchan->fdesc) + fdma_write(fchan->fdev, cmd, FDMA_CMD_SET_OFST); + spin_unlock_irqrestore(&fchan->vchan.lock, flags); + + return 0; +} + +static int st_fdma_resume(struct dma_chan *chan) +{ + unsigned long flags; + unsigned long val; + struct st_fdma_chan *fchan = to_st_fdma_chan(chan); + int ch_id = fchan->vchan.chan.chan_id; + + dev_dbg(fchan->fdev->dev, "resume chan:%d\n", ch_id); + + spin_lock_irqsave(&fchan->vchan.lock, flags); + if (fchan->fdesc) { + val = fchan_read(fchan, FDMA_CH_CMD_OFST); + val &= FDMA_CH_CMD_DATA_MASK; + fchan_write(fchan, val, FDMA_CH_CMD_OFST); + } + spin_unlock_irqrestore(&fchan->vchan.lock, flags); + + return 0; +} + +static int st_fdma_terminate_all(struct dma_chan *chan) +{ + unsigned long flags; + LIST_HEAD(head); + struct st_fdma_chan *fchan = to_st_fdma_chan(chan); + int ch_id = fchan->vchan.chan.chan_id; + unsigned long cmd = FDMA_CMD_PAUSE(ch_id); + + dev_dbg(fchan->fdev->dev, "terminate chan:%d\n", ch_id); + + spin_lock_irqsave(&fchan->vchan.lock, flags); + fdma_write(fchan->fdev, cmd, FDMA_CMD_SET_OFST); + fchan->fdesc = NULL; + vchan_get_all_descriptors(&fchan->vchan, &head); + spin_unlock_irqrestore(&fchan->vchan.lock, flags); + vchan_dma_desc_free_list(&fchan->vchan, &head); + + return 0; +} + +static int st_fdma_slave_config(struct dma_chan *chan, + struct dma_slave_config *slave_cfg) +{ + struct st_fdma_chan *fchan = to_st_fdma_chan(chan); + + memcpy(&fchan->scfg, slave_cfg, sizeof(fchan->scfg)); + return 0; +} + +static const struct st_fdma_driverdata fdma_mpe31_stih407_11 = { + .name = "STiH407", + .id = 0, +}; + +static const struct st_fdma_driverdata fdma_mpe31_stih407_12 = { + .name = "STiH407", + .id = 1, +}; + +static const struct st_fdma_driverdata fdma_mpe31_stih407_13 = { + .name = "STiH407", + .id = 2, +}; + +static const struct of_device_id st_fdma_match[] = { + { .compatible = "st,stih407-fdma-mpe31-11" + , .data = &fdma_mpe31_stih407_11 }, + { .compatible = "st,stih407-fdma-mpe31-12" + , .data = &fdma_mpe31_stih407_12 }, + { .compatible = "st,stih407-fdma-mpe31-13" + , .data = &fdma_mpe31_stih407_13 }, + {}, +}; +MODULE_DEVICE_TABLE(of, st_fdma_match); + +static int st_fdma_parse_dt(struct platform_device *pdev, + const struct st_fdma_driverdata *drvdata, + struct st_fdma_dev *fdev) +{ + snprintf(fdev->fw_name, FW_NAME_SIZE, "fdma_%s_%d.elf", + drvdata->name, drvdata->id); + + return of_property_read_u32(pdev->dev.of_node, "dma-channels", + &fdev->nr_channels); +} +#define FDMA_DMA_BUSWIDTHS (BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | \ + BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | \ + BIT(DMA_SLAVE_BUSWIDTH_3_BYTES) | \ + BIT(DMA_SLAVE_BUSWIDTH_4_BYTES)) + +static void st_fdma_free(struct st_fdma_dev *fdev) +{ + struct st_fdma_chan *fchan; + int i; + + for (i = 0; i < fdev->nr_channels; i++) { + fchan = &fdev->chans[i]; + list_del(&fchan->vchan.chan.device_node); + tasklet_kill(&fchan->vchan.task); + } +} + +static int st_fdma_probe(struct platform_device *pdev) +{ + struct st_fdma_dev *fdev; + const struct of_device_id *match; + struct device_node *np = pdev->dev.of_node; + const struct st_fdma_driverdata *drvdata; + int ret, i; + + match = of_match_device((st_fdma_match), &pdev->dev); + if (!match || !match->data) { + dev_err(&pdev->dev, "No device match found\n"); + return -ENODEV; + } + + drvdata = match->data; + + fdev = devm_kzalloc(&pdev->dev, sizeof(*fdev), GFP_KERNEL); + if (!fdev) + return -ENOMEM; + + ret = st_fdma_parse_dt(pdev, drvdata, fdev); + if (ret) { + dev_err(&pdev->dev, "unable to find platform data\n"); + goto err; + } + + fdev->chans = devm_kcalloc(&pdev->dev, fdev->nr_channels, + sizeof(struct st_fdma_chan), GFP_KERNEL); + if (!fdev->chans) + return -ENOMEM; + + fdev->dev = &pdev->dev; + fdev->drvdata = drvdata; + platform_set_drvdata(pdev, fdev); + + fdev->irq = platform_get_irq(pdev, 0); + if (fdev->irq < 0) { + dev_err(&pdev->dev, "Failed to get irq resource\n"); + return -EINVAL; + } + + ret = devm_request_irq(&pdev->dev, fdev->irq, st_fdma_irq_handler, 0, + dev_name(&pdev->dev), fdev); + if (ret) { + dev_err(&pdev->dev, "Failed to request irq (%d)\n", ret); + goto err; + } + + fdev->slim_rproc = st_slim_rproc_alloc(pdev, fdev->fw_name); + if (IS_ERR(fdev->slim_rproc)) { + ret = PTR_ERR(fdev->slim_rproc); + dev_err(&pdev->dev, "slim_rproc_alloc failed (%d)\n", ret); + goto err; + } + + /* Initialise list of FDMA channels */ + INIT_LIST_HEAD(&fdev->dma_device.channels); + for (i = 0; i < fdev->nr_channels; i++) { + struct st_fdma_chan *fchan = &fdev->chans[i]; + + fchan->fdev = fdev; + fchan->vchan.desc_free = st_fdma_free_desc; + vchan_init(&fchan->vchan, &fdev->dma_device); + } + + /* Initialise the FDMA dreq (reserve 0 & 31 for FDMA use) */ + fdev->dreq_mask = BIT(0) | BIT(31); + + dma_cap_set(DMA_SLAVE, fdev->dma_device.cap_mask); + dma_cap_set(DMA_CYCLIC, fdev->dma_device.cap_mask); + dma_cap_set(DMA_MEMCPY, fdev->dma_device.cap_mask); + + fdev->dma_device.dev = &pdev->dev; + fdev->dma_device.device_alloc_chan_resources = st_fdma_alloc_chan_res; + fdev->dma_device.device_free_chan_resources = st_fdma_free_chan_res; + fdev->dma_device.device_prep_dma_cyclic = st_fdma_prep_dma_cyclic; + fdev->dma_device.device_prep_slave_sg = st_fdma_prep_slave_sg; + fdev->dma_device.device_prep_dma_memcpy = st_fdma_prep_dma_memcpy; + fdev->dma_device.device_tx_status = st_fdma_tx_status; + fdev->dma_device.device_issue_pending = st_fdma_issue_pending; + fdev->dma_device.device_terminate_all = st_fdma_terminate_all; + fdev->dma_device.device_config = st_fdma_slave_config; + fdev->dma_device.device_pause = st_fdma_pause; + fdev->dma_device.device_resume = st_fdma_resume; + + fdev->dma_device.src_addr_widths = FDMA_DMA_BUSWIDTHS; + fdev->dma_device.dst_addr_widths = FDMA_DMA_BUSWIDTHS; + fdev->dma_device.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV); + fdev->dma_device.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST; + + ret = dma_async_device_register(&fdev->dma_device); + if (ret) { + dev_err(&pdev->dev, + "Failed to register DMA device (%d)\n", ret); + goto err_rproc; + } + + ret = of_dma_controller_register(np, st_fdma_of_xlate, fdev); + if (ret) { + dev_err(&pdev->dev, + "Failed to register controller (%d)\n", ret); + goto err_dma_dev; + } + + dev_info(&pdev->dev, "ST FDMA engine driver, irq:%d\n", fdev->irq); + + return 0; + +err_dma_dev: + dma_async_device_unregister(&fdev->dma_device); +err_rproc: + st_fdma_free(fdev); + st_slim_rproc_put(fdev->slim_rproc); +err: + return ret; +} + +static int st_fdma_remove(struct platform_device *pdev) +{ + struct st_fdma_dev *fdev = platform_get_drvdata(pdev); + + devm_free_irq(&pdev->dev, fdev->irq, fdev); + st_slim_rproc_put(fdev->slim_rproc); + of_dma_controller_free(pdev->dev.of_node); + dma_async_device_unregister(&fdev->dma_device); + + return 0; +} + +static struct platform_driver st_fdma_platform_driver = { + .driver = { + .name = DRIVER_NAME, + .of_match_table = st_fdma_match, + }, + .probe = st_fdma_probe, + .remove = st_fdma_remove, +}; +module_platform_driver(st_fdma_platform_driver); + +MODULE_LICENSE("GPL v2"); +MODULE_DESCRIPTION("STMicroelectronics FDMA engine driver"); +MODULE_AUTHOR("Ludovic.barre <Ludovic.barre@st.com>"); +MODULE_AUTHOR("Peter Griffin <peter.griffin@linaro.org>"); +MODULE_ALIAS("platform: " DRIVER_NAME); diff --git a/drivers/dma/st_fdma.h b/drivers/dma/st_fdma.h new file mode 100644 index 000000000000..c58e00d4ab37 --- /dev/null +++ b/drivers/dma/st_fdma.h @@ -0,0 +1,249 @@ +/* + * DMA driver header for STMicroelectronics STi FDMA controller + * + * Copyright (C) 2014 STMicroelectronics + * + * Author: Ludovic Barre <Ludovic.barre@st.com> + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + */ +#ifndef __DMA_ST_FDMA_H +#define __DMA_ST_FDMA_H + +#include <linux/dmaengine.h> +#include <linux/dmapool.h> +#include <linux/io.h> +#include <linux/remoteproc/st_slim_rproc.h> +#include "virt-dma.h" + +#define ST_FDMA_NR_DREQS 32 +#define FW_NAME_SIZE 30 +#define DRIVER_NAME "st-fdma" + +/** + * struct st_fdma_generic_node - Free running/paced generic node + * + * @length: Length in bytes of a line in a 2D mem to mem + * @sstride: Stride, in bytes, between source lines in a 2D data move + * @dstride: Stride, in bytes, between destination lines in a 2D data move + */ +struct st_fdma_generic_node { + u32 length; + u32 sstride; + u32 dstride; +}; + +/** + * struct st_fdma_hw_node - Node structure used by fdma hw + * + * @next: Pointer to next node + * @control: Transfer Control Parameters + * @nbytes: Number of Bytes to read + * @saddr: Source address + * @daddr: Destination address + * + * @generic: generic node for free running/paced transfert type + * 2 others transfert type are possible, but not yet implemented + * + * The NODE structures must be aligned to a 32 byte boundary + */ +struct st_fdma_hw_node { + u32 next; + u32 control; + u32 nbytes; + u32 saddr; + u32 daddr; + union { + struct st_fdma_generic_node generic; + }; +} __aligned(32); + +/* + * node control parameters + */ +#define FDMA_NODE_CTRL_REQ_MAP_MASK GENMASK(4, 0) +#define FDMA_NODE_CTRL_REQ_MAP_FREE_RUN 0x0 +#define FDMA_NODE_CTRL_REQ_MAP_DREQ(n) ((n)&FDMA_NODE_CTRL_REQ_MAP_MASK) +#define FDMA_NODE_CTRL_REQ_MAP_EXT FDMA_NODE_CTRL_REQ_MAP_MASK +#define FDMA_NODE_CTRL_SRC_MASK GENMASK(6, 5) +#define FDMA_NODE_CTRL_SRC_STATIC BIT(5) +#define FDMA_NODE_CTRL_SRC_INCR BIT(6) +#define FDMA_NODE_CTRL_DST_MASK GENMASK(8, 7) +#define FDMA_NODE_CTRL_DST_STATIC BIT(7) +#define FDMA_NODE_CTRL_DST_INCR BIT(8) +#define FDMA_NODE_CTRL_SECURE BIT(15) +#define FDMA_NODE_CTRL_PAUSE_EON BIT(30) +#define FDMA_NODE_CTRL_INT_EON BIT(31) + +/** + * struct st_fdma_sw_node - descriptor structure for link list + * + * @pdesc: Physical address of desc + * @node: link used for putting this into a channel queue + */ +struct st_fdma_sw_node { + dma_addr_t pdesc; + struct st_fdma_hw_node *desc; +}; + +#define NAME_SZ 10 + +struct st_fdma_driverdata { + u32 id; + char name[NAME_SZ]; +}; + +struct st_fdma_desc { + struct virt_dma_desc vdesc; + struct st_fdma_chan *fchan; + bool iscyclic; + unsigned int n_nodes; + struct st_fdma_sw_node node[]; +}; + +enum st_fdma_type { + ST_FDMA_TYPE_FREE_RUN, + ST_FDMA_TYPE_PACED, +}; + +struct st_fdma_cfg { + struct device_node *of_node; + enum st_fdma_type type; + dma_addr_t dev_addr; + enum dma_transfer_direction dir; + int req_line; /* request line */ + long req_ctrl; /* Request control */ +}; + +struct st_fdma_chan { + struct st_fdma_dev *fdev; + struct dma_pool *node_pool; + struct dma_slave_config scfg; + struct st_fdma_cfg cfg; + + int dreq_line; + + struct virt_dma_chan vchan; + struct st_fdma_desc *fdesc; + enum dma_status status; +}; + +struct st_fdma_dev { + struct device *dev; + const struct st_fdma_driverdata *drvdata; + struct dma_device dma_device; + + struct st_slim_rproc *slim_rproc; + + int irq; + + struct st_fdma_chan *chans; + + spinlock_t dreq_lock; + unsigned long dreq_mask; + + u32 nr_channels; + char fw_name[FW_NAME_SIZE]; +}; + +/* Peripheral Registers*/ + +#define FDMA_CMD_STA_OFST 0xFC0 +#define FDMA_CMD_SET_OFST 0xFC4 +#define FDMA_CMD_CLR_OFST 0xFC8 +#define FDMA_CMD_MASK_OFST 0xFCC +#define FDMA_CMD_START(ch) (0x1 << (ch << 1)) +#define FDMA_CMD_PAUSE(ch) (0x2 << (ch << 1)) +#define FDMA_CMD_FLUSH(ch) (0x3 << (ch << 1)) + +#define FDMA_INT_STA_OFST 0xFD0 +#define FDMA_INT_STA_CH 0x1 +#define FDMA_INT_STA_ERR 0x2 + +#define FDMA_INT_SET_OFST 0xFD4 +#define FDMA_INT_CLR_OFST 0xFD8 +#define FDMA_INT_MASK_OFST 0xFDC + +#define fdma_read(fdev, name) \ + readl((fdev)->slim_rproc->peri + name) + +#define fdma_write(fdev, val, name) \ + writel((val), (fdev)->slim_rproc->peri + name) + +/* fchan interface (dmem) */ +#define FDMA_CH_CMD_OFST 0x200 +#define FDMA_CH_CMD_STA_MASK GENMASK(1, 0) +#define FDMA_CH_CMD_STA_IDLE (0x0) +#define FDMA_CH_CMD_STA_START (0x1) +#define FDMA_CH_CMD_STA_RUNNING (0x2) +#define FDMA_CH_CMD_STA_PAUSED (0x3) +#define FDMA_CH_CMD_ERR_MASK GENMASK(4, 2) +#define FDMA_CH_CMD_ERR_INT (0x0 << 2) +#define FDMA_CH_CMD_ERR_NAND (0x1 << 2) +#define FDMA_CH_CMD_ERR_MCHI (0x2 << 2) +#define FDMA_CH_CMD_DATA_MASK GENMASK(31, 5) +#define fchan_read(fchan, name) \ + readl((fchan)->fdev->slim_rproc->mem[ST_SLIM_DMEM].cpu_addr \ + + (fchan)->vchan.chan.chan_id * 0x4 \ + + name) + +#define fchan_write(fchan, val, name) \ + writel((val), (fchan)->fdev->slim_rproc->mem[ST_SLIM_DMEM].cpu_addr \ + + (fchan)->vchan.chan.chan_id * 0x4 \ + + name) + +/* req interface */ +#define FDMA_REQ_CTRL_OFST 0x240 +#define dreq_write(fchan, val, name) \ + writel((val), (fchan)->fdev->slim_rproc->mem[ST_SLIM_DMEM].cpu_addr \ + + fchan->dreq_line * 0x04 \ + + name) +/* node interface */ +#define FDMA_NODE_SZ 128 +#define FDMA_PTRN_OFST 0x800 +#define FDMA_CNTN_OFST 0x808 +#define FDMA_SADDRN_OFST 0x80c +#define FDMA_DADDRN_OFST 0x810 +#define fnode_read(fchan, name) \ + readl((fchan)->fdev->slim_rproc->mem[ST_SLIM_DMEM].cpu_addr \ + + (fchan)->vchan.chan.chan_id * FDMA_NODE_SZ \ + + name) + +#define fnode_write(fchan, val, name) \ + writel((val), (fchan)->fdev->slim_rproc->mem[ST_SLIM_DMEM].cpu_addr \ + + (fchan)->vchan.chan.chan_id * FDMA_NODE_SZ \ + + name) + +/* + * request control bits + */ +#define FDMA_REQ_CTRL_NUM_OPS_MASK GENMASK(31, 24) +#define FDMA_REQ_CTRL_NUM_OPS(n) (FDMA_REQ_CTRL_NUM_OPS_MASK & \ + ((n) << 24)) +#define FDMA_REQ_CTRL_INITIATOR_MASK BIT(22) +#define FDMA_REQ_CTRL_INIT0 (0x0 << 22) +#define FDMA_REQ_CTRL_INIT1 (0x1 << 22) +#define FDMA_REQ_CTRL_INC_ADDR_ON BIT(21) +#define FDMA_REQ_CTRL_DATA_SWAP_ON BIT(17) +#define FDMA_REQ_CTRL_WNR BIT(14) +#define FDMA_REQ_CTRL_OPCODE_MASK GENMASK(7, 4) +#define FDMA_REQ_CTRL_OPCODE_LD_ST1 (0x0 << 4) +#define FDMA_REQ_CTRL_OPCODE_LD_ST2 (0x1 << 4) +#define FDMA_REQ_CTRL_OPCODE_LD_ST4 (0x2 << 4) +#define FDMA_REQ_CTRL_OPCODE_LD_ST8 (0x3 << 4) +#define FDMA_REQ_CTRL_OPCODE_LD_ST16 (0x4 << 4) +#define FDMA_REQ_CTRL_OPCODE_LD_ST32 (0x5 << 4) +#define FDMA_REQ_CTRL_OPCODE_LD_ST64 (0x6 << 4) +#define FDMA_REQ_CTRL_HOLDOFF_MASK GENMASK(2, 0) +#define FDMA_REQ_CTRL_HOLDOFF(n) ((n) & FDMA_REQ_CTRL_HOLDOFF_MASK) + +/* bits used by client to configure request control */ +#define FDMA_REQ_CTRL_CFG_MASK (FDMA_REQ_CTRL_HOLDOFF_MASK | \ + FDMA_REQ_CTRL_DATA_SWAP_ON | \ + FDMA_REQ_CTRL_INC_ADDR_ON | \ + FDMA_REQ_CTRL_INITIATOR_MASK) + +#endif /* __DMA_ST_FDMA_H */ diff --git a/drivers/dma/ste_dma40.c b/drivers/dma/ste_dma40.c index 8684d11b29bb..a6620b671d1d 100644 --- a/drivers/dma/ste_dma40.c +++ b/drivers/dma/ste_dma40.c @@ -2809,12 +2809,14 @@ static void __init d40_chan_init(struct d40_base *base, struct dma_device *dma, static void d40_ops_init(struct d40_base *base, struct dma_device *dev) { - if (dma_has_cap(DMA_SLAVE, dev->cap_mask)) + if (dma_has_cap(DMA_SLAVE, dev->cap_mask)) { dev->device_prep_slave_sg = d40_prep_slave_sg; + dev->directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV); + } if (dma_has_cap(DMA_MEMCPY, dev->cap_mask)) { dev->device_prep_dma_memcpy = d40_prep_memcpy; - + dev->directions = BIT(DMA_MEM_TO_MEM); /* * This controller can only access address at even * 32bit boundaries, i.e. 2^2 @@ -2836,6 +2838,7 @@ static void d40_ops_init(struct d40_base *base, struct dma_device *dev) dev->device_pause = d40_pause; dev->device_resume = d40_resume; dev->device_terminate_all = d40_terminate_all; + dev->residue_granularity = DMA_RESIDUE_GRANULARITY_BURST; dev->dev = base->dev; } diff --git a/drivers/dma/stm32-dma.c b/drivers/dma/stm32-dma.c index 307547f4848d..49f86cabcfec 100644 --- a/drivers/dma/stm32-dma.c +++ b/drivers/dma/stm32-dma.c @@ -114,6 +114,7 @@ #define STM32_DMA_MAX_CHANNELS 0x08 #define STM32_DMA_MAX_REQUEST_ID 0x08 #define STM32_DMA_MAX_DATA_PARAM 0x03 +#define STM32_DMA_MAX_BURST 16 enum stm32_dma_width { STM32_DMA_BYTE, @@ -403,6 +404,13 @@ static int stm32_dma_terminate_all(struct dma_chan *c) return 0; } +static void stm32_dma_synchronize(struct dma_chan *c) +{ + struct stm32_dma_chan *chan = to_stm32_dma_chan(c); + + vchan_synchronize(&chan->vchan); +} + static void stm32_dma_dump_reg(struct stm32_dma_chan *chan) { struct stm32_dma_device *dmadev = stm32_dma_get_dev(chan); @@ -421,7 +429,7 @@ static void stm32_dma_dump_reg(struct stm32_dma_chan *chan) dev_dbg(chan2dev(chan), "SFCR: 0x%08x\n", sfcr); } -static int stm32_dma_start_transfer(struct stm32_dma_chan *chan) +static void stm32_dma_start_transfer(struct stm32_dma_chan *chan) { struct stm32_dma_device *dmadev = stm32_dma_get_dev(chan); struct virt_dma_desc *vdesc; @@ -432,12 +440,12 @@ static int stm32_dma_start_transfer(struct stm32_dma_chan *chan) ret = stm32_dma_disable_chan(chan); if (ret < 0) - return ret; + return; if (!chan->desc) { vdesc = vchan_next_desc(&chan->vchan); if (!vdesc) - return -EPERM; + return; chan->desc = to_stm32_dma_desc(vdesc); chan->next_sg = 0; @@ -471,7 +479,7 @@ static int stm32_dma_start_transfer(struct stm32_dma_chan *chan) chan->busy = true; - return 0; + dev_dbg(chan2dev(chan), "vchan %p: started\n", &chan->vchan); } static void stm32_dma_configure_next_sg(struct stm32_dma_chan *chan) @@ -500,8 +508,6 @@ static void stm32_dma_configure_next_sg(struct stm32_dma_chan *chan) dev_dbg(chan2dev(chan), "CT=0 <=> SM1AR: 0x%08x\n", stm32_dma_read(dmadev, STM32_DMA_SM1AR(id))); } - - chan->next_sg++; } } @@ -510,6 +516,7 @@ static void stm32_dma_handle_chan_done(struct stm32_dma_chan *chan) if (chan->desc) { if (chan->desc->cyclic) { vchan_cyclic_callback(&chan->desc->vdesc); + chan->next_sg++; stm32_dma_configure_next_sg(chan); } else { chan->busy = false; @@ -527,13 +534,12 @@ static irqreturn_t stm32_dma_chan_irq(int irq, void *devid) { struct stm32_dma_chan *chan = devid; struct stm32_dma_device *dmadev = stm32_dma_get_dev(chan); - u32 status, scr, sfcr; + u32 status, scr; spin_lock(&chan->vchan.lock); status = stm32_dma_irq_status(chan); scr = stm32_dma_read(dmadev, STM32_DMA_SCR(chan->id)); - sfcr = stm32_dma_read(dmadev, STM32_DMA_SFCR(chan->id)); if ((status & STM32_DMA_TCI) && (scr & STM32_DMA_SCR_TCIE)) { stm32_dma_irq_clear(chan, STM32_DMA_TCI); @@ -553,15 +559,13 @@ static void stm32_dma_issue_pending(struct dma_chan *c) { struct stm32_dma_chan *chan = to_stm32_dma_chan(c); unsigned long flags; - int ret; spin_lock_irqsave(&chan->vchan.lock, flags); - if (!chan->busy) { - if (vchan_issue_pending(&chan->vchan) && !chan->desc) { - ret = stm32_dma_start_transfer(chan); - if ((!ret) && (chan->desc->cyclic)) - stm32_dma_configure_next_sg(chan); - } + if (vchan_issue_pending(&chan->vchan) && !chan->desc && !chan->busy) { + dev_dbg(chan2dev(chan), "vchan %p: issued\n", &chan->vchan); + stm32_dma_start_transfer(chan); + if (chan->desc->cyclic) + stm32_dma_configure_next_sg(chan); } spin_unlock_irqrestore(&chan->vchan.lock, flags); } @@ -574,15 +578,12 @@ static int stm32_dma_set_xfer_param(struct stm32_dma_chan *chan, int src_bus_width, dst_bus_width; int src_burst_size, dst_burst_size; u32 src_maxburst, dst_maxburst; - dma_addr_t src_addr, dst_addr; u32 dma_scr = 0; src_addr_width = chan->dma_sconfig.src_addr_width; dst_addr_width = chan->dma_sconfig.dst_addr_width; src_maxburst = chan->dma_sconfig.src_maxburst; dst_maxburst = chan->dma_sconfig.dst_maxburst; - src_addr = chan->dma_sconfig.src_addr; - dst_addr = chan->dma_sconfig.dst_addr; switch (direction) { case DMA_MEM_TO_DEV: @@ -852,26 +853,40 @@ static struct dma_async_tx_descriptor *stm32_dma_prep_dma_memcpy( return vchan_tx_prep(&chan->vchan, &desc->vdesc, flags); } +static u32 stm32_dma_get_remaining_bytes(struct stm32_dma_chan *chan) +{ + u32 dma_scr, width, ndtr; + struct stm32_dma_device *dmadev = stm32_dma_get_dev(chan); + + dma_scr = stm32_dma_read(dmadev, STM32_DMA_SCR(chan->id)); + width = STM32_DMA_SCR_PSIZE_GET(dma_scr); + ndtr = stm32_dma_read(dmadev, STM32_DMA_SNDTR(chan->id)); + + return ndtr << width; +} + static size_t stm32_dma_desc_residue(struct stm32_dma_chan *chan, struct stm32_dma_desc *desc, u32 next_sg) { - struct stm32_dma_device *dmadev = stm32_dma_get_dev(chan); - u32 dma_scr, width, residue, count; + u32 residue = 0; int i; - residue = 0; + /* + * In cyclic mode, for the last period, residue = remaining bytes from + * NDTR + */ + if (chan->desc->cyclic && next_sg == 0) + return stm32_dma_get_remaining_bytes(chan); + /* + * For all other periods in cyclic mode, and in sg mode, + * residue = remaining bytes from NDTR + remaining periods/sg to be + * transferred + */ for (i = next_sg; i < desc->num_sgs; i++) residue += desc->sg_req[i].len; - - if (next_sg != 0) { - dma_scr = stm32_dma_read(dmadev, STM32_DMA_SCR(chan->id)); - width = STM32_DMA_SCR_PSIZE_GET(dma_scr); - count = stm32_dma_read(dmadev, STM32_DMA_SNDTR(chan->id)); - - residue += count << width; - } + residue += stm32_dma_get_remaining_bytes(chan); return residue; } @@ -884,7 +899,7 @@ static enum dma_status stm32_dma_tx_status(struct dma_chan *c, struct virt_dma_desc *vdesc; enum dma_status status; unsigned long flags; - u32 residue; + u32 residue = 0; status = dma_cookie_status(c, cookie, state); if ((status == DMA_COMPLETE) || (!state)) @@ -892,16 +907,12 @@ static enum dma_status stm32_dma_tx_status(struct dma_chan *c, spin_lock_irqsave(&chan->vchan.lock, flags); vdesc = vchan_find_desc(&chan->vchan, cookie); - if (cookie == chan->desc->vdesc.tx.cookie) { + if (chan->desc && cookie == chan->desc->vdesc.tx.cookie) residue = stm32_dma_desc_residue(chan, chan->desc, chan->next_sg); - } else if (vdesc) { + else if (vdesc) residue = stm32_dma_desc_residue(chan, to_stm32_dma_desc(vdesc), 0); - } else { - residue = 0; - } - dma_set_residue(state, residue); spin_unlock_irqrestore(&chan->vchan.lock, flags); @@ -972,30 +983,36 @@ static struct dma_chan *stm32_dma_of_xlate(struct of_phandle_args *dma_spec, struct of_dma *ofdma) { struct stm32_dma_device *dmadev = ofdma->of_dma_data; + struct device *dev = dmadev->ddev.dev; struct stm32_dma_cfg cfg; struct stm32_dma_chan *chan; struct dma_chan *c; - if (dma_spec->args_count < 3) + if (dma_spec->args_count < 4) { + dev_err(dev, "Bad number of cells\n"); return NULL; + } cfg.channel_id = dma_spec->args[0]; cfg.request_line = dma_spec->args[1]; cfg.stream_config = dma_spec->args[2]; - cfg.threshold = 0; + cfg.threshold = dma_spec->args[3]; - if ((cfg.channel_id >= STM32_DMA_MAX_CHANNELS) || (cfg.request_line >= - STM32_DMA_MAX_REQUEST_ID)) + if ((cfg.channel_id >= STM32_DMA_MAX_CHANNELS) || + (cfg.request_line >= STM32_DMA_MAX_REQUEST_ID)) { + dev_err(dev, "Bad channel and/or request id\n"); return NULL; - - if (dma_spec->args_count > 3) - cfg.threshold = dma_spec->args[3]; + } chan = &dmadev->chan[cfg.channel_id]; c = dma_get_slave_channel(&chan->vchan.chan); - if (c) - stm32_dma_set_config(chan, &cfg); + if (!c) { + dev_err(dev, "No more channel avalaible\n"); + return NULL; + } + + stm32_dma_set_config(chan, &cfg); return c; } @@ -1059,6 +1076,7 @@ static int stm32_dma_probe(struct platform_device *pdev) dd->device_prep_dma_cyclic = stm32_dma_prep_dma_cyclic; dd->device_config = stm32_dma_slave_config; dd->device_terminate_all = stm32_dma_terminate_all; + dd->device_synchronize = stm32_dma_synchronize; dd->src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | BIT(DMA_SLAVE_BUSWIDTH_4_BYTES); @@ -1067,6 +1085,7 @@ static int stm32_dma_probe(struct platform_device *pdev) BIT(DMA_SLAVE_BUSWIDTH_4_BYTES); dd->directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV); dd->residue_granularity = DMA_RESIDUE_GRANULARITY_BURST; + dd->max_burst = STM32_DMA_MAX_BURST; dd->dev = &pdev->dev; INIT_LIST_HEAD(&dd->channels); diff --git a/drivers/dma/ti-dma-crossbar.c b/drivers/dma/ti-dma-crossbar.c index 3f24aeb48c0e..2403475a37cf 100644 --- a/drivers/dma/ti-dma-crossbar.c +++ b/drivers/dma/ti-dma-crossbar.c @@ -149,6 +149,7 @@ static int ti_am335x_xbar_probe(struct platform_device *pdev) match = of_match_node(ti_am335x_master_match, dma_node); if (!match) { dev_err(&pdev->dev, "DMA master is not supported\n"); + of_node_put(dma_node); return -EINVAL; } @@ -339,6 +340,7 @@ static int ti_dra7_xbar_probe(struct platform_device *pdev) match = of_match_node(ti_dra7_master_match, dma_node); if (!match) { dev_err(&pdev->dev, "DMA master is not supported\n"); + of_node_put(dma_node); return -EINVAL; } diff --git a/drivers/dma/zx296702_dma.c b/drivers/dma/zx_dma.c index 245d759d5ffc..2bb695315300 100644 --- a/drivers/dma/zx296702_dma.c +++ b/drivers/dma/zx_dma.c @@ -26,7 +26,7 @@ #define DRIVER_NAME "zx-dma" #define DMA_ALIGN 4 -#define DMA_MAX_SIZE (0x10000 - PAGE_SIZE) +#define DMA_MAX_SIZE (0x10000 - 512) #define LLI_BLOCK_SIZE (4 * PAGE_SIZE) #define REG_ZX_SRC_ADDR 0x00 @@ -365,7 +365,8 @@ static enum dma_status zx_dma_tx_status(struct dma_chan *chan, bytes = 0; clli = zx_dma_get_curr_lli(p); - index = (clli - ds->desc_hw_lli) / sizeof(struct zx_desc_hw); + index = (clli - ds->desc_hw_lli) / + sizeof(struct zx_desc_hw) + 1; for (; index < ds->desc_num; index++) { bytes += ds->desc_hw[index].src_x; /* end of lli */ @@ -435,13 +436,12 @@ static struct zx_dma_desc_sw *zx_alloc_desc_resource(int num, if (!ds) return NULL; - ds->desc_hw = dma_pool_alloc(d->pool, GFP_NOWAIT, &ds->desc_hw_lli); + ds->desc_hw = dma_pool_zalloc(d->pool, GFP_NOWAIT, &ds->desc_hw_lli); if (!ds->desc_hw) { dev_dbg(chan->device->dev, "vch %p: dma alloc fail\n", &c->vc); kfree(ds); return NULL; } - memset(ds->desc_hw, 0, sizeof(struct zx_desc_hw) * num); ds->desc_num = num; return ds; } @@ -813,6 +813,7 @@ static int zx_dma_probe(struct platform_device *op) INIT_LIST_HEAD(&d->slave.channels); dma_cap_set(DMA_SLAVE, d->slave.cap_mask); dma_cap_set(DMA_MEMCPY, d->slave.cap_mask); + dma_cap_set(DMA_CYCLIC, d->slave.cap_mask); dma_cap_set(DMA_PRIVATE, d->slave.cap_mask); d->slave.dev = &op->dev; d->slave.device_free_chan_resources = zx_dma_free_chan_resources; |