diff options
Diffstat (limited to 'drivers/dma')
49 files changed, 5446 insertions, 584 deletions
diff --git a/drivers/dma/Kconfig b/drivers/dma/Kconfig index 91eced044321..fd7ac13f2574 100644 --- a/drivers/dma/Kconfig +++ b/drivers/dma/Kconfig @@ -112,6 +112,17 @@ config FSL_DMA EloPlus is on mpc85xx and mpc86xx and Pxxx parts, and the Elo3 is on some Txxx and Bxxx parts. +config FSL_RAID + tristate "Freescale RAID engine Support" + depends on FSL_SOC && !ASYNC_TX_ENABLE_CHANNEL_SWITCH + select DMA_ENGINE + select DMA_ENGINE_RAID + ---help--- + Enable support for Freescale RAID Engine. RAID Engine is + available on some QorIQ SoCs (like P5020/P5040). It has + the capability to offload memcpy, xor and pq computation + for raid5/6. + source "drivers/dma/hsu/Kconfig" config MPC512X_DMA @@ -347,6 +358,16 @@ config DMA_JZ4740 select DMA_ENGINE select DMA_VIRTUAL_CHANNELS +config DMA_JZ4780 + tristate "JZ4780 DMA support" + depends on MACH_JZ4780 + select DMA_ENGINE + select DMA_VIRTUAL_CHANNELS + help + This selects support for the DMA controller in Ingenic JZ4780 SoCs. + If you have a board based on such a SoC and wish to use DMA for + devices which can use the DMA controller, say Y or M here. + config K3_DMA tristate "Hisilicon K3 DMA support" depends on ARCH_HI3xxx @@ -414,6 +435,14 @@ config IMG_MDC_DMA help Enable support for the IMG multi-threaded DMA controller (MDC). +config XGENE_DMA + tristate "APM X-Gene DMA support" + select DMA_ENGINE + select DMA_ENGINE_RAID + select ASYNC_TX_ENABLE_CHANNEL_SWITCH + help + Enable support for the APM X-Gene SoC DMA engine. + config DMA_ENGINE bool diff --git a/drivers/dma/Makefile b/drivers/dma/Makefile index 7e8301cb489d..69f77d5ba53b 100644 --- a/drivers/dma/Makefile +++ b/drivers/dma/Makefile @@ -41,9 +41,11 @@ obj-$(CONFIG_DMA_OMAP) += omap-dma.o obj-$(CONFIG_DMA_BCM2835) += bcm2835-dma.o obj-$(CONFIG_MMP_PDMA) += mmp_pdma.o obj-$(CONFIG_DMA_JZ4740) += dma-jz4740.o +obj-$(CONFIG_DMA_JZ4780) += dma-jz4780.o obj-$(CONFIG_TI_CPPI41) += cppi41.o obj-$(CONFIG_K3_DMA) += k3dma.o obj-$(CONFIG_MOXART_DMA) += moxart-dma.o +obj-$(CONFIG_FSL_RAID) += fsl_raid.o obj-$(CONFIG_FSL_EDMA) += fsl-edma.o obj-$(CONFIG_QCOM_BAM_DMA) += qcom_bam_dma.o obj-y += xilinx/ @@ -51,3 +53,4 @@ obj-$(CONFIG_INTEL_MIC_X100_DMA) += mic_x100_dma.o obj-$(CONFIG_NBPFAXI_DMA) += nbpfaxi.o obj-$(CONFIG_DMA_SUN6I) += sun6i-dma.o obj-$(CONFIG_IMG_MDC_DMA) += img-mdc-dma.o +obj-$(CONFIG_XGENE_DMA) += xgene-dma.o diff --git a/drivers/dma/amba-pl08x.c b/drivers/dma/amba-pl08x.c index 83aa55d6fa5d..49d396ec06e5 100644 --- a/drivers/dma/amba-pl08x.c +++ b/drivers/dma/amba-pl08x.c @@ -15,10 +15,6 @@ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for * more details. * - * You should have received a copy of the GNU General Public License along with - * this program; if not, write to the Free Software Foundation, Inc., 59 - * Temple Place - Suite 330, Boston, MA 02111-1307, USA. - * * The full GNU General Public License is in this distribution in the file * called COPYING. * @@ -1195,11 +1191,6 @@ static void pl08x_free_txd_list(struct pl08x_driver_data *pl08x, /* * The DMA ENGINE API */ -static int pl08x_alloc_chan_resources(struct dma_chan *chan) -{ - return 0; -} - static void pl08x_free_chan_resources(struct dma_chan *chan) { /* Ensure all queued descriptors are freed */ @@ -2066,7 +2057,6 @@ static int pl08x_probe(struct amba_device *adev, const struct amba_id *id) /* Initialize memcpy engine */ dma_cap_set(DMA_MEMCPY, pl08x->memcpy.cap_mask); pl08x->memcpy.dev = &adev->dev; - pl08x->memcpy.device_alloc_chan_resources = pl08x_alloc_chan_resources; pl08x->memcpy.device_free_chan_resources = pl08x_free_chan_resources; pl08x->memcpy.device_prep_dma_memcpy = pl08x_prep_dma_memcpy; pl08x->memcpy.device_prep_dma_interrupt = pl08x_prep_dma_interrupt; @@ -2085,7 +2075,6 @@ static int pl08x_probe(struct amba_device *adev, const struct amba_id *id) dma_cap_set(DMA_SLAVE, pl08x->slave.cap_mask); dma_cap_set(DMA_CYCLIC, pl08x->slave.cap_mask); pl08x->slave.dev = &adev->dev; - pl08x->slave.device_alloc_chan_resources = pl08x_alloc_chan_resources; pl08x->slave.device_free_chan_resources = pl08x_free_chan_resources; pl08x->slave.device_prep_dma_interrupt = pl08x_prep_dma_interrupt; pl08x->slave.device_tx_status = pl08x_dma_tx_status; diff --git a/drivers/dma/at_hdmac.c b/drivers/dma/at_hdmac.c index 0b4fc6fb48ce..57b2141ddddc 100644 --- a/drivers/dma/at_hdmac.c +++ b/drivers/dma/at_hdmac.c @@ -65,6 +65,21 @@ static void atc_issue_pending(struct dma_chan *chan); /*----------------------------------------------------------------------*/ +static inline unsigned int atc_get_xfer_width(dma_addr_t src, dma_addr_t dst, + size_t len) +{ + unsigned int width; + + if (!((src | dst | len) & 3)) + width = 2; + else if (!((src | dst | len) & 1)) + width = 1; + else + width = 0; + + return width; +} + static struct at_desc *atc_first_active(struct at_dma_chan *atchan) { return list_first_entry(&atchan->active_list, @@ -659,16 +674,10 @@ atc_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src, * We can be a lot more clever here, but this should take care * of the most common optimization. */ - if (!((src | dest | len) & 3)) { - ctrla = ATC_SRC_WIDTH_WORD | ATC_DST_WIDTH_WORD; - src_width = dst_width = 2; - } else if (!((src | dest | len) & 1)) { - ctrla = ATC_SRC_WIDTH_HALFWORD | ATC_DST_WIDTH_HALFWORD; - src_width = dst_width = 1; - } else { - ctrla = ATC_SRC_WIDTH_BYTE | ATC_DST_WIDTH_BYTE; - src_width = dst_width = 0; - } + src_width = dst_width = atc_get_xfer_width(src, dest, len); + + ctrla = ATC_SRC_WIDTH(src_width) | + ATC_DST_WIDTH(dst_width); for (offset = 0; offset < len; offset += xfer_count << src_width) { xfer_count = min_t(size_t, (len - offset) >> src_width, @@ -862,6 +871,144 @@ err: } /** + * atc_prep_dma_sg - prepare memory to memory scather-gather operation + * @chan: the channel to prepare operation on + * @dst_sg: destination scatterlist + * @dst_nents: number of destination scatterlist entries + * @src_sg: source scatterlist + * @src_nents: number of source scatterlist entries + * @flags: tx descriptor status flags + */ +static struct dma_async_tx_descriptor * +atc_prep_dma_sg(struct dma_chan *chan, + struct scatterlist *dst_sg, unsigned int dst_nents, + struct scatterlist *src_sg, unsigned int src_nents, + unsigned long flags) +{ + struct at_dma_chan *atchan = to_at_dma_chan(chan); + struct at_desc *desc = NULL; + struct at_desc *first = NULL; + struct at_desc *prev = NULL; + unsigned int src_width; + unsigned int dst_width; + size_t xfer_count; + u32 ctrla; + u32 ctrlb; + size_t dst_len = 0, src_len = 0; + dma_addr_t dst = 0, src = 0; + size_t len = 0, total_len = 0; + + if (unlikely(dst_nents == 0 || src_nents == 0)) + return NULL; + + if (unlikely(dst_sg == NULL || src_sg == NULL)) + return NULL; + + ctrlb = ATC_DEFAULT_CTRLB | ATC_IEN + | ATC_SRC_ADDR_MODE_INCR + | ATC_DST_ADDR_MODE_INCR + | ATC_FC_MEM2MEM; + + /* + * loop until there is either no more source or no more destination + * scatterlist entry + */ + while (true) { + + /* prepare the next transfer */ + if (dst_len == 0) { + + /* no more destination scatterlist entries */ + if (!dst_sg || !dst_nents) + break; + + dst = sg_dma_address(dst_sg); + dst_len = sg_dma_len(dst_sg); + + dst_sg = sg_next(dst_sg); + dst_nents--; + } + + if (src_len == 0) { + + /* no more source scatterlist entries */ + if (!src_sg || !src_nents) + break; + + src = sg_dma_address(src_sg); + src_len = sg_dma_len(src_sg); + + src_sg = sg_next(src_sg); + src_nents--; + } + + len = min_t(size_t, src_len, dst_len); + if (len == 0) + continue; + + /* take care for the alignment */ + src_width = dst_width = atc_get_xfer_width(src, dst, len); + + ctrla = ATC_SRC_WIDTH(src_width) | + ATC_DST_WIDTH(dst_width); + + /* + * The number of transfers to set up refer to the source width + * that depends on the alignment. + */ + xfer_count = len >> src_width; + if (xfer_count > ATC_BTSIZE_MAX) { + xfer_count = ATC_BTSIZE_MAX; + len = ATC_BTSIZE_MAX << src_width; + } + + /* create the transfer */ + desc = atc_desc_get(atchan); + if (!desc) + goto err_desc_get; + + desc->lli.saddr = src; + desc->lli.daddr = dst; + desc->lli.ctrla = ctrla | xfer_count; + desc->lli.ctrlb = ctrlb; + + desc->txd.cookie = 0; + desc->len = len; + + /* + * Although we only need the transfer width for the first and + * the last descriptor, its easier to set it to all descriptors. + */ + desc->tx_width = src_width; + + atc_desc_chain(&first, &prev, desc); + + /* update the lengths and addresses for the next loop cycle */ + dst_len -= len; + src_len -= len; + dst += len; + src += len; + + total_len += len; + } + + /* First descriptor of the chain embedds additional information */ + first->txd.cookie = -EBUSY; + first->total_len = total_len; + + /* set end-of-link to the last link descriptor of list*/ + set_desc_eol(desc); + + first->txd.flags = flags; /* client is in control of this ack */ + + return &first->txd; + +err_desc_get: + atc_desc_put(atchan, first); + return NULL; +} + +/** * atc_dma_cyclic_check_values * Check for too big/unaligned periods and unaligned DMA buffer */ @@ -1461,8 +1608,10 @@ static int __init at_dma_probe(struct platform_device *pdev) /* setup platform data for each SoC */ dma_cap_set(DMA_MEMCPY, at91sam9rl_config.cap_mask); + dma_cap_set(DMA_SG, at91sam9rl_config.cap_mask); dma_cap_set(DMA_MEMCPY, at91sam9g45_config.cap_mask); dma_cap_set(DMA_SLAVE, at91sam9g45_config.cap_mask); + dma_cap_set(DMA_SG, at91sam9g45_config.cap_mask); /* get DMA parameters from controller type */ plat_dat = at_dma_get_driver_data(pdev); @@ -1582,11 +1731,15 @@ static int __init at_dma_probe(struct platform_device *pdev) atdma->dma_common.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST; } + if (dma_has_cap(DMA_SG, atdma->dma_common.cap_mask)) + atdma->dma_common.device_prep_dma_sg = atc_prep_dma_sg; + dma_writel(atdma, EN, AT_DMA_ENABLE); - dev_info(&pdev->dev, "Atmel AHB DMA Controller ( %s%s), %d channels\n", + dev_info(&pdev->dev, "Atmel AHB DMA Controller ( %s%s%s), %d channels\n", dma_has_cap(DMA_MEMCPY, atdma->dma_common.cap_mask) ? "cpy " : "", dma_has_cap(DMA_SLAVE, atdma->dma_common.cap_mask) ? "slave " : "", + dma_has_cap(DMA_SG, atdma->dma_common.cap_mask) ? "sg-cpy " : "", plat_dat->nr_channels); dma_async_device_register(&atdma->dma_common); diff --git a/drivers/dma/at_xdmac.c b/drivers/dma/at_xdmac.c index d9891d3461f6..933e4b338459 100644 --- a/drivers/dma/at_xdmac.c +++ b/drivers/dma/at_xdmac.c @@ -1154,8 +1154,10 @@ static int at_xdmac_device_resume(struct dma_chan *chan) dev_dbg(chan2dev(chan), "%s\n", __func__); spin_lock_bh(&atchan->lock); - if (!at_xdmac_chan_is_paused(atchan)) + if (!at_xdmac_chan_is_paused(atchan)) { + spin_unlock_bh(&atchan->lock); return 0; + } at_xdmac_write(atxdmac, AT_XDMAC_GRWR, atchan->mask); clear_bit(AT_XDMAC_CHAN_IS_PAUSED, &atchan->status); diff --git a/drivers/dma/bestcomm/bestcomm.c b/drivers/dma/bestcomm/bestcomm.c index fa378d88f6c8..180fedb418cc 100644 --- a/drivers/dma/bestcomm/bestcomm.c +++ b/drivers/dma/bestcomm/bestcomm.c @@ -30,7 +30,7 @@ #define DRIVER_NAME "bestcomm-core" /* MPC5200 device tree match tables */ -static struct of_device_id mpc52xx_sram_ids[] = { +static const struct of_device_id mpc52xx_sram_ids[] = { { .compatible = "fsl,mpc5200-sram", }, { .compatible = "mpc5200-sram", }, {} @@ -481,7 +481,7 @@ static int mpc52xx_bcom_remove(struct platform_device *op) return 0; } -static struct of_device_id mpc52xx_bcom_of_match[] = { +static const struct of_device_id mpc52xx_bcom_of_match[] = { { .compatible = "fsl,mpc5200-bestcomm", }, { .compatible = "mpc5200-bestcomm", }, {}, diff --git a/drivers/dma/dma-jz4740.c b/drivers/dma/dma-jz4740.c index 84884418fd30..7638b24ce8d0 100644 --- a/drivers/dma/dma-jz4740.c +++ b/drivers/dma/dma-jz4740.c @@ -7,10 +7,6 @@ * Free Software Foundation; either version 2 of the License, or (at your * option) any later version. * - * You should have received a copy of the GNU General Public License along - * with this program; if not, write to the Free Software Foundation, Inc., - * 675 Mass Ave, Cambridge, MA 02139, USA. - * */ #include <linux/dmaengine.h> @@ -343,7 +339,7 @@ static void jz4740_dma_chan_irq(struct jz4740_dmaengine_chan *chan) { spin_lock(&chan->vchan.lock); if (chan->desc) { - if (chan->desc && chan->desc->cyclic) { + if (chan->desc->cyclic) { vchan_cyclic_callback(&chan->desc->vdesc); } else { if (chan->next_sg == chan->desc->num_sgs) { @@ -496,11 +492,6 @@ static enum dma_status jz4740_dma_tx_status(struct dma_chan *c, return status; } -static int jz4740_dma_alloc_chan_resources(struct dma_chan *c) -{ - return 0; -} - static void jz4740_dma_free_chan_resources(struct dma_chan *c) { vchan_free_chan_resources(to_virt_chan(c)); @@ -543,7 +534,6 @@ static int jz4740_dma_probe(struct platform_device *pdev) dma_cap_set(DMA_SLAVE, dd->cap_mask); dma_cap_set(DMA_CYCLIC, dd->cap_mask); - dd->device_alloc_chan_resources = jz4740_dma_alloc_chan_resources; dd->device_free_chan_resources = jz4740_dma_free_chan_resources; dd->device_tx_status = jz4740_dma_tx_status; dd->device_issue_pending = jz4740_dma_issue_pending; diff --git a/drivers/dma/dma-jz4780.c b/drivers/dma/dma-jz4780.c new file mode 100644 index 000000000000..26d2f0e09ea3 --- /dev/null +++ b/drivers/dma/dma-jz4780.c @@ -0,0 +1,877 @@ +/* + * Ingenic JZ4780 DMA controller + * + * Copyright (c) 2015 Imagination Technologies + * Author: Alex Smith <alex@alex-smith.me.uk> + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the + * Free Software Foundation; either version 2 of the License, or (at your + * option) any later version. + */ + +#include <linux/clk.h> +#include <linux/dmapool.h> +#include <linux/init.h> +#include <linux/interrupt.h> +#include <linux/module.h> +#include <linux/of.h> +#include <linux/of_dma.h> +#include <linux/platform_device.h> +#include <linux/slab.h> + +#include "dmaengine.h" +#include "virt-dma.h" + +#define JZ_DMA_NR_CHANNELS 32 + +/* Global registers. */ +#define JZ_DMA_REG_DMAC 0x1000 +#define JZ_DMA_REG_DIRQP 0x1004 +#define JZ_DMA_REG_DDR 0x1008 +#define JZ_DMA_REG_DDRS 0x100c +#define JZ_DMA_REG_DMACP 0x101c +#define JZ_DMA_REG_DSIRQP 0x1020 +#define JZ_DMA_REG_DSIRQM 0x1024 +#define JZ_DMA_REG_DCIRQP 0x1028 +#define JZ_DMA_REG_DCIRQM 0x102c + +/* Per-channel registers. */ +#define JZ_DMA_REG_CHAN(n) (n * 0x20) +#define JZ_DMA_REG_DSA(n) (0x00 + JZ_DMA_REG_CHAN(n)) +#define JZ_DMA_REG_DTA(n) (0x04 + JZ_DMA_REG_CHAN(n)) +#define JZ_DMA_REG_DTC(n) (0x08 + JZ_DMA_REG_CHAN(n)) +#define JZ_DMA_REG_DRT(n) (0x0c + JZ_DMA_REG_CHAN(n)) +#define JZ_DMA_REG_DCS(n) (0x10 + JZ_DMA_REG_CHAN(n)) +#define JZ_DMA_REG_DCM(n) (0x14 + JZ_DMA_REG_CHAN(n)) +#define JZ_DMA_REG_DDA(n) (0x18 + JZ_DMA_REG_CHAN(n)) +#define JZ_DMA_REG_DSD(n) (0x1c + JZ_DMA_REG_CHAN(n)) + +#define JZ_DMA_DMAC_DMAE BIT(0) +#define JZ_DMA_DMAC_AR BIT(2) +#define JZ_DMA_DMAC_HLT BIT(3) +#define JZ_DMA_DMAC_FMSC BIT(31) + +#define JZ_DMA_DRT_AUTO 0x8 + +#define JZ_DMA_DCS_CTE BIT(0) +#define JZ_DMA_DCS_HLT BIT(2) +#define JZ_DMA_DCS_TT BIT(3) +#define JZ_DMA_DCS_AR BIT(4) +#define JZ_DMA_DCS_DES8 BIT(30) + +#define JZ_DMA_DCM_LINK BIT(0) +#define JZ_DMA_DCM_TIE BIT(1) +#define JZ_DMA_DCM_STDE BIT(2) +#define JZ_DMA_DCM_TSZ_SHIFT 8 +#define JZ_DMA_DCM_TSZ_MASK (0x7 << JZ_DMA_DCM_TSZ_SHIFT) +#define JZ_DMA_DCM_DP_SHIFT 12 +#define JZ_DMA_DCM_SP_SHIFT 14 +#define JZ_DMA_DCM_DAI BIT(22) +#define JZ_DMA_DCM_SAI BIT(23) + +#define JZ_DMA_SIZE_4_BYTE 0x0 +#define JZ_DMA_SIZE_1_BYTE 0x1 +#define JZ_DMA_SIZE_2_BYTE 0x2 +#define JZ_DMA_SIZE_16_BYTE 0x3 +#define JZ_DMA_SIZE_32_BYTE 0x4 +#define JZ_DMA_SIZE_64_BYTE 0x5 +#define JZ_DMA_SIZE_128_BYTE 0x6 + +#define JZ_DMA_WIDTH_32_BIT 0x0 +#define JZ_DMA_WIDTH_8_BIT 0x1 +#define JZ_DMA_WIDTH_16_BIT 0x2 + +#define JZ_DMA_BUSWIDTHS (BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | \ + BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | \ + BIT(DMA_SLAVE_BUSWIDTH_4_BYTES)) + +/** + * struct jz4780_dma_hwdesc - descriptor structure read by the DMA controller. + * @dcm: value for the DCM (channel command) register + * @dsa: source address + * @dta: target address + * @dtc: transfer count (number of blocks of the transfer size specified in DCM + * to transfer) in the low 24 bits, offset of the next descriptor from the + * descriptor base address in the upper 8 bits. + * @sd: target/source stride difference (in stride transfer mode). + * @drt: request type + */ +struct jz4780_dma_hwdesc { + uint32_t dcm; + uint32_t dsa; + uint32_t dta; + uint32_t dtc; + uint32_t sd; + uint32_t drt; + uint32_t reserved[2]; +}; + +/* Size of allocations for hardware descriptor blocks. */ +#define JZ_DMA_DESC_BLOCK_SIZE PAGE_SIZE +#define JZ_DMA_MAX_DESC \ + (JZ_DMA_DESC_BLOCK_SIZE / sizeof(struct jz4780_dma_hwdesc)) + +struct jz4780_dma_desc { + struct virt_dma_desc vdesc; + + struct jz4780_dma_hwdesc *desc; + dma_addr_t desc_phys; + unsigned int count; + enum dma_transaction_type type; + uint32_t status; +}; + +struct jz4780_dma_chan { + struct virt_dma_chan vchan; + unsigned int id; + struct dma_pool *desc_pool; + + uint32_t transfer_type; + uint32_t transfer_shift; + struct dma_slave_config config; + + struct jz4780_dma_desc *desc; + unsigned int curr_hwdesc; +}; + +struct jz4780_dma_dev { + struct dma_device dma_device; + void __iomem *base; + struct clk *clk; + unsigned int irq; + + uint32_t chan_reserved; + struct jz4780_dma_chan chan[JZ_DMA_NR_CHANNELS]; +}; + +struct jz4780_dma_data { + uint32_t transfer_type; + int channel; +}; + +static inline struct jz4780_dma_chan *to_jz4780_dma_chan(struct dma_chan *chan) +{ + return container_of(chan, struct jz4780_dma_chan, vchan.chan); +} + +static inline struct jz4780_dma_desc *to_jz4780_dma_desc( + struct virt_dma_desc *vdesc) +{ + return container_of(vdesc, struct jz4780_dma_desc, vdesc); +} + +static inline struct jz4780_dma_dev *jz4780_dma_chan_parent( + struct jz4780_dma_chan *jzchan) +{ + return container_of(jzchan->vchan.chan.device, struct jz4780_dma_dev, + dma_device); +} + +static inline uint32_t jz4780_dma_readl(struct jz4780_dma_dev *jzdma, + unsigned int reg) +{ + return readl(jzdma->base + reg); +} + +static inline void jz4780_dma_writel(struct jz4780_dma_dev *jzdma, + unsigned int reg, uint32_t val) +{ + writel(val, jzdma->base + reg); +} + +static struct jz4780_dma_desc *jz4780_dma_desc_alloc( + struct jz4780_dma_chan *jzchan, unsigned int count, + enum dma_transaction_type type) +{ + struct jz4780_dma_desc *desc; + + if (count > JZ_DMA_MAX_DESC) + return NULL; + + desc = kzalloc(sizeof(*desc), GFP_NOWAIT); + if (!desc) + return NULL; + + desc->desc = dma_pool_alloc(jzchan->desc_pool, GFP_NOWAIT, + &desc->desc_phys); + if (!desc->desc) { + kfree(desc); + return NULL; + } + + desc->count = count; + desc->type = type; + return desc; +} + +static void jz4780_dma_desc_free(struct virt_dma_desc *vdesc) +{ + struct jz4780_dma_desc *desc = to_jz4780_dma_desc(vdesc); + struct jz4780_dma_chan *jzchan = to_jz4780_dma_chan(vdesc->tx.chan); + + dma_pool_free(jzchan->desc_pool, desc->desc, desc->desc_phys); + kfree(desc); +} + +static uint32_t jz4780_dma_transfer_size(unsigned long val, int *ord) +{ + *ord = ffs(val) - 1; + + switch (*ord) { + case 0: + return JZ_DMA_SIZE_1_BYTE; + case 1: + return JZ_DMA_SIZE_2_BYTE; + case 2: + return JZ_DMA_SIZE_4_BYTE; + case 4: + return JZ_DMA_SIZE_16_BYTE; + case 5: + return JZ_DMA_SIZE_32_BYTE; + case 6: + return JZ_DMA_SIZE_64_BYTE; + case 7: + return JZ_DMA_SIZE_128_BYTE; + default: + return -EINVAL; + } +} + +static uint32_t jz4780_dma_setup_hwdesc(struct jz4780_dma_chan *jzchan, + struct jz4780_dma_hwdesc *desc, dma_addr_t addr, size_t len, + enum dma_transfer_direction direction) +{ + struct dma_slave_config *config = &jzchan->config; + uint32_t width, maxburst, tsz; + int ord; + + if (direction == DMA_MEM_TO_DEV) { + desc->dcm = JZ_DMA_DCM_SAI; + desc->dsa = addr; + desc->dta = config->dst_addr; + desc->drt = jzchan->transfer_type; + + width = config->dst_addr_width; + maxburst = config->dst_maxburst; + } else { + desc->dcm = JZ_DMA_DCM_DAI; + desc->dsa = config->src_addr; + desc->dta = addr; + desc->drt = jzchan->transfer_type; + + width = config->src_addr_width; + maxburst = config->src_maxburst; + } + + /* + * This calculates the maximum transfer size that can be used with the + * given address, length, width and maximum burst size. The address + * must be aligned to the transfer size, the total length must be + * divisible by the transfer size, and we must not use more than the + * maximum burst specified by the user. + */ + tsz = jz4780_dma_transfer_size(addr | len | (width * maxburst), &ord); + jzchan->transfer_shift = ord; + + switch (width) { + case DMA_SLAVE_BUSWIDTH_1_BYTE: + case DMA_SLAVE_BUSWIDTH_2_BYTES: + break; + case DMA_SLAVE_BUSWIDTH_4_BYTES: + width = JZ_DMA_WIDTH_32_BIT; + break; + default: + return -EINVAL; + } + + desc->dcm |= tsz << JZ_DMA_DCM_TSZ_SHIFT; + desc->dcm |= width << JZ_DMA_DCM_SP_SHIFT; + desc->dcm |= width << JZ_DMA_DCM_DP_SHIFT; + + desc->dtc = len >> ord; +} + +static struct dma_async_tx_descriptor *jz4780_dma_prep_slave_sg( + struct dma_chan *chan, struct scatterlist *sgl, unsigned int sg_len, + enum dma_transfer_direction direction, unsigned long flags) +{ + struct jz4780_dma_chan *jzchan = to_jz4780_dma_chan(chan); + struct jz4780_dma_desc *desc; + unsigned int i; + int err; + + desc = jz4780_dma_desc_alloc(jzchan, sg_len, DMA_SLAVE); + if (!desc) + return NULL; + + for (i = 0; i < sg_len; i++) { + err = jz4780_dma_setup_hwdesc(jzchan, &desc->desc[i], + sg_dma_address(&sgl[i]), + sg_dma_len(&sgl[i]), + direction); + if (err < 0) + return ERR_PTR(err); + + + desc->desc[i].dcm |= JZ_DMA_DCM_TIE; + + if (i != (sg_len - 1)) { + /* Automatically proceeed to the next descriptor. */ + desc->desc[i].dcm |= JZ_DMA_DCM_LINK; + + /* + * The upper 8 bits of the DTC field in the descriptor + * must be set to (offset from descriptor base of next + * descriptor >> 4). + */ + desc->desc[i].dtc |= + (((i + 1) * sizeof(*desc->desc)) >> 4) << 24; + } + } + + return vchan_tx_prep(&jzchan->vchan, &desc->vdesc, flags); +} + +static struct dma_async_tx_descriptor *jz4780_dma_prep_dma_cyclic( + struct dma_chan *chan, dma_addr_t buf_addr, size_t buf_len, + size_t period_len, enum dma_transfer_direction direction, + unsigned long flags) +{ + struct jz4780_dma_chan *jzchan = to_jz4780_dma_chan(chan); + struct jz4780_dma_desc *desc; + unsigned int periods, i; + int err; + + if (buf_len % period_len) + return NULL; + + periods = buf_len / period_len; + + desc = jz4780_dma_desc_alloc(jzchan, periods, DMA_CYCLIC); + if (!desc) + return NULL; + + for (i = 0; i < periods; i++) { + err = jz4780_dma_setup_hwdesc(jzchan, &desc->desc[i], buf_addr, + period_len, direction); + if (err < 0) + return ERR_PTR(err); + + buf_addr += period_len; + + /* + * Set the link bit to indicate that the controller should + * automatically proceed to the next descriptor. In + * jz4780_dma_begin(), this will be cleared if we need to issue + * an interrupt after each period. + */ + desc->desc[i].dcm |= JZ_DMA_DCM_TIE | JZ_DMA_DCM_LINK; + + /* + * The upper 8 bits of the DTC field in the descriptor must be + * set to (offset from descriptor base of next descriptor >> 4). + * If this is the last descriptor, link it back to the first, + * i.e. leave offset set to 0, otherwise point to the next one. + */ + if (i != (periods - 1)) { + desc->desc[i].dtc |= + (((i + 1) * sizeof(*desc->desc)) >> 4) << 24; + } + } + + return vchan_tx_prep(&jzchan->vchan, &desc->vdesc, flags); +} + +struct dma_async_tx_descriptor *jz4780_dma_prep_dma_memcpy( + struct dma_chan *chan, dma_addr_t dest, dma_addr_t src, + size_t len, unsigned long flags) +{ + struct jz4780_dma_chan *jzchan = to_jz4780_dma_chan(chan); + struct jz4780_dma_desc *desc; + uint32_t tsz; + int ord; + + desc = jz4780_dma_desc_alloc(jzchan, 1, DMA_MEMCPY); + if (!desc) + return NULL; + + tsz = jz4780_dma_transfer_size(dest | src | len, &ord); + if (tsz < 0) + return ERR_PTR(tsz); + + desc->desc[0].dsa = src; + desc->desc[0].dta = dest; + desc->desc[0].drt = JZ_DMA_DRT_AUTO; + desc->desc[0].dcm = JZ_DMA_DCM_TIE | JZ_DMA_DCM_SAI | JZ_DMA_DCM_DAI | + tsz << JZ_DMA_DCM_TSZ_SHIFT | + JZ_DMA_WIDTH_32_BIT << JZ_DMA_DCM_SP_SHIFT | + JZ_DMA_WIDTH_32_BIT << JZ_DMA_DCM_DP_SHIFT; + desc->desc[0].dtc = len >> ord; + + return vchan_tx_prep(&jzchan->vchan, &desc->vdesc, flags); +} + +static void jz4780_dma_begin(struct jz4780_dma_chan *jzchan) +{ + struct jz4780_dma_dev *jzdma = jz4780_dma_chan_parent(jzchan); + struct virt_dma_desc *vdesc; + unsigned int i; + dma_addr_t desc_phys; + + if (!jzchan->desc) { + vdesc = vchan_next_desc(&jzchan->vchan); + if (!vdesc) + return; + + list_del(&vdesc->node); + + jzchan->desc = to_jz4780_dma_desc(vdesc); + jzchan->curr_hwdesc = 0; + + if (jzchan->desc->type == DMA_CYCLIC && vdesc->tx.callback) { + /* + * The DMA controller doesn't support triggering an + * interrupt after processing each descriptor, only + * after processing an entire terminated list of + * descriptors. For a cyclic DMA setup the list of + * descriptors is not terminated so we can never get an + * interrupt. + * + * If the user requested a callback for a cyclic DMA + * setup then we workaround this hardware limitation + * here by degrading to a set of unlinked descriptors + * which we will submit in sequence in response to the + * completion of processing the previous descriptor. + */ + for (i = 0; i < jzchan->desc->count; i++) + jzchan->desc->desc[i].dcm &= ~JZ_DMA_DCM_LINK; + } + } else { + /* + * There is an existing transfer, therefore this must be one + * for which we unlinked the descriptors above. Advance to the + * next one in the list. + */ + jzchan->curr_hwdesc = + (jzchan->curr_hwdesc + 1) % jzchan->desc->count; + } + + /* Use 8-word descriptors. */ + jz4780_dma_writel(jzdma, JZ_DMA_REG_DCS(jzchan->id), JZ_DMA_DCS_DES8); + + /* Write descriptor address and initiate descriptor fetch. */ + desc_phys = jzchan->desc->desc_phys + + (jzchan->curr_hwdesc * sizeof(*jzchan->desc->desc)); + jz4780_dma_writel(jzdma, JZ_DMA_REG_DDA(jzchan->id), desc_phys); + jz4780_dma_writel(jzdma, JZ_DMA_REG_DDRS, BIT(jzchan->id)); + + /* Enable the channel. */ + jz4780_dma_writel(jzdma, JZ_DMA_REG_DCS(jzchan->id), + JZ_DMA_DCS_DES8 | JZ_DMA_DCS_CTE); +} + +static void jz4780_dma_issue_pending(struct dma_chan *chan) +{ + struct jz4780_dma_chan *jzchan = to_jz4780_dma_chan(chan); + unsigned long flags; + + spin_lock_irqsave(&jzchan->vchan.lock, flags); + + if (vchan_issue_pending(&jzchan->vchan) && !jzchan->desc) + jz4780_dma_begin(jzchan); + + spin_unlock_irqrestore(&jzchan->vchan.lock, flags); +} + +static int jz4780_dma_terminate_all(struct jz4780_dma_chan *jzchan) +{ + struct jz4780_dma_dev *jzdma = jz4780_dma_chan_parent(jzchan); + unsigned long flags; + LIST_HEAD(head); + + spin_lock_irqsave(&jzchan->vchan.lock, flags); + + /* Clear the DMA status and stop the transfer. */ + jz4780_dma_writel(jzdma, JZ_DMA_REG_DCS(jzchan->id), 0); + if (jzchan->desc) { + jz4780_dma_desc_free(&jzchan->desc->vdesc); + jzchan->desc = NULL; + } + + vchan_get_all_descriptors(&jzchan->vchan, &head); + + spin_unlock_irqrestore(&jzchan->vchan.lock, flags); + + vchan_dma_desc_free_list(&jzchan->vchan, &head); + return 0; +} + +static int jz4780_dma_slave_config(struct jz4780_dma_chan *jzchan, + const struct dma_slave_config *config) +{ + if ((config->src_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES) + || (config->dst_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES)) + return -EINVAL; + + /* Copy the reset of the slave configuration, it is used later. */ + memcpy(&jzchan->config, config, sizeof(jzchan->config)); + + return 0; +} + +static size_t jz4780_dma_desc_residue(struct jz4780_dma_chan *jzchan, + struct jz4780_dma_desc *desc, unsigned int next_sg) +{ + struct jz4780_dma_dev *jzdma = jz4780_dma_chan_parent(jzchan); + unsigned int residue, count; + unsigned int i; + + residue = 0; + + for (i = next_sg; i < desc->count; i++) + residue += desc->desc[i].dtc << jzchan->transfer_shift; + + if (next_sg != 0) { + count = jz4780_dma_readl(jzdma, + JZ_DMA_REG_DTC(jzchan->id)); + residue += count << jzchan->transfer_shift; + } + + return residue; +} + +static enum dma_status jz4780_dma_tx_status(struct dma_chan *chan, + dma_cookie_t cookie, struct dma_tx_state *txstate) +{ + struct jz4780_dma_chan *jzchan = to_jz4780_dma_chan(chan); + struct virt_dma_desc *vdesc; + enum dma_status status; + unsigned long flags; + + status = dma_cookie_status(chan, cookie, txstate); + if ((status == DMA_COMPLETE) || (txstate == NULL)) + return status; + + spin_lock_irqsave(&jzchan->vchan.lock, flags); + + vdesc = vchan_find_desc(&jzchan->vchan, cookie); + if (vdesc) { + /* On the issued list, so hasn't been processed yet */ + txstate->residue = jz4780_dma_desc_residue(jzchan, + to_jz4780_dma_desc(vdesc), 0); + } else if (cookie == jzchan->desc->vdesc.tx.cookie) { + txstate->residue = jz4780_dma_desc_residue(jzchan, jzchan->desc, + (jzchan->curr_hwdesc + 1) % jzchan->desc->count); + } else + txstate->residue = 0; + + if (vdesc && jzchan->desc && vdesc == &jzchan->desc->vdesc + && jzchan->desc->status & (JZ_DMA_DCS_AR | JZ_DMA_DCS_HLT)) + status = DMA_ERROR; + + spin_unlock_irqrestore(&jzchan->vchan.lock, flags); + return status; +} + +static void jz4780_dma_chan_irq(struct jz4780_dma_dev *jzdma, + struct jz4780_dma_chan *jzchan) +{ + uint32_t dcs; + + spin_lock(&jzchan->vchan.lock); + + dcs = jz4780_dma_readl(jzdma, JZ_DMA_REG_DCS(jzchan->id)); + jz4780_dma_writel(jzdma, JZ_DMA_REG_DCS(jzchan->id), 0); + + if (dcs & JZ_DMA_DCS_AR) { + dev_warn(&jzchan->vchan.chan.dev->device, + "address error (DCS=0x%x)\n", dcs); + } + + if (dcs & JZ_DMA_DCS_HLT) { + dev_warn(&jzchan->vchan.chan.dev->device, + "channel halt (DCS=0x%x)\n", dcs); + } + + if (jzchan->desc) { + jzchan->desc->status = dcs; + + if ((dcs & (JZ_DMA_DCS_AR | JZ_DMA_DCS_HLT)) == 0) { + if (jzchan->desc->type == DMA_CYCLIC) { + vchan_cyclic_callback(&jzchan->desc->vdesc); + } else { + vchan_cookie_complete(&jzchan->desc->vdesc); + jzchan->desc = NULL; + } + + jz4780_dma_begin(jzchan); + } + } else { + dev_err(&jzchan->vchan.chan.dev->device, + "channel IRQ with no active transfer\n"); + } + + spin_unlock(&jzchan->vchan.lock); +} + +static irqreturn_t jz4780_dma_irq_handler(int irq, void *data) +{ + struct jz4780_dma_dev *jzdma = data; + uint32_t pending, dmac; + int i; + + pending = jz4780_dma_readl(jzdma, JZ_DMA_REG_DIRQP); + + for (i = 0; i < JZ_DMA_NR_CHANNELS; i++) { + if (!(pending & (1<<i))) + continue; + + jz4780_dma_chan_irq(jzdma, &jzdma->chan[i]); + } + + /* Clear halt and address error status of all channels. */ + dmac = jz4780_dma_readl(jzdma, JZ_DMA_REG_DMAC); + dmac &= ~(JZ_DMA_DMAC_HLT | JZ_DMA_DMAC_AR); + jz4780_dma_writel(jzdma, JZ_DMA_REG_DMAC, dmac); + + /* Clear interrupt pending status. */ + jz4780_dma_writel(jzdma, JZ_DMA_REG_DIRQP, 0); + + return IRQ_HANDLED; +} + +static int jz4780_dma_alloc_chan_resources(struct dma_chan *chan) +{ + struct jz4780_dma_chan *jzchan = to_jz4780_dma_chan(chan); + + jzchan->desc_pool = dma_pool_create(dev_name(&chan->dev->device), + chan->device->dev, + JZ_DMA_DESC_BLOCK_SIZE, + PAGE_SIZE, 0); + if (!jzchan->desc_pool) { + dev_err(&chan->dev->device, + "failed to allocate descriptor pool\n"); + return -ENOMEM; + } + + return 0; +} + +static void jz4780_dma_free_chan_resources(struct dma_chan *chan) +{ + struct jz4780_dma_chan *jzchan = to_jz4780_dma_chan(chan); + + vchan_free_chan_resources(&jzchan->vchan); + dma_pool_destroy(jzchan->desc_pool); + jzchan->desc_pool = NULL; +} + +static bool jz4780_dma_filter_fn(struct dma_chan *chan, void *param) +{ + struct jz4780_dma_chan *jzchan = to_jz4780_dma_chan(chan); + struct jz4780_dma_dev *jzdma = jz4780_dma_chan_parent(jzchan); + struct jz4780_dma_data *data = param; + + if (data->channel > -1) { + if (data->channel != jzchan->id) + return false; + } else if (jzdma->chan_reserved & BIT(jzchan->id)) { + return false; + } + + jzchan->transfer_type = data->transfer_type; + + return true; +} + +static struct dma_chan *jz4780_of_dma_xlate(struct of_phandle_args *dma_spec, + struct of_dma *ofdma) +{ + struct jz4780_dma_dev *jzdma = ofdma->of_dma_data; + dma_cap_mask_t mask = jzdma->dma_device.cap_mask; + struct jz4780_dma_data data; + + if (dma_spec->args_count != 2) + return NULL; + + data.transfer_type = dma_spec->args[0]; + data.channel = dma_spec->args[1]; + + if (data.channel > -1) { + if (data.channel >= JZ_DMA_NR_CHANNELS) { + dev_err(jzdma->dma_device.dev, + "device requested non-existent channel %u\n", + data.channel); + return NULL; + } + + /* Can only select a channel marked as reserved. */ + if (!(jzdma->chan_reserved & BIT(data.channel))) { + dev_err(jzdma->dma_device.dev, + "device requested unreserved channel %u\n", + data.channel); + return NULL; + } + } + + return dma_request_channel(mask, jz4780_dma_filter_fn, &data); +} + +static int jz4780_dma_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct jz4780_dma_dev *jzdma; + struct jz4780_dma_chan *jzchan; + struct dma_device *dd; + struct resource *res; + int i, ret; + + jzdma = devm_kzalloc(dev, sizeof(*jzdma), GFP_KERNEL); + if (!jzdma) + return -ENOMEM; + + platform_set_drvdata(pdev, jzdma); + + res = platform_get_resource(pdev, IORESOURCE_MEM, 0); + if (!res) { + dev_err(dev, "failed to get I/O memory\n"); + return -EINVAL; + } + + jzdma->base = devm_ioremap_resource(dev, res); + if (IS_ERR(jzdma->base)) + return PTR_ERR(jzdma->base); + + jzdma->irq = platform_get_irq(pdev, 0); + if (jzdma->irq < 0) { + dev_err(dev, "failed to get IRQ: %d\n", ret); + return jzdma->irq; + } + + ret = devm_request_irq(dev, jzdma->irq, jz4780_dma_irq_handler, 0, + dev_name(dev), jzdma); + if (ret) { + dev_err(dev, "failed to request IRQ %u!\n", jzdma->irq); + return -EINVAL; + } + + jzdma->clk = devm_clk_get(dev, NULL); + if (IS_ERR(jzdma->clk)) { + dev_err(dev, "failed to get clock\n"); + return PTR_ERR(jzdma->clk); + } + + clk_prepare_enable(jzdma->clk); + + /* Property is optional, if it doesn't exist the value will remain 0. */ + of_property_read_u32_index(dev->of_node, "ingenic,reserved-channels", + 0, &jzdma->chan_reserved); + + dd = &jzdma->dma_device; + + dma_cap_set(DMA_MEMCPY, dd->cap_mask); + dma_cap_set(DMA_SLAVE, dd->cap_mask); + dma_cap_set(DMA_CYCLIC, dd->cap_mask); + + dd->dev = dev; + dd->copy_align = 2; /* 2^2 = 4 byte alignment */ + dd->device_alloc_chan_resources = jz4780_dma_alloc_chan_resources; + dd->device_free_chan_resources = jz4780_dma_free_chan_resources; + dd->device_prep_slave_sg = jz4780_dma_prep_slave_sg; + dd->device_prep_dma_cyclic = jz4780_dma_prep_dma_cyclic; + dd->device_prep_dma_memcpy = jz4780_dma_prep_dma_memcpy; + dd->device_config = jz4780_dma_slave_config; + dd->device_terminate_all = jz4780_dma_terminate_all; + dd->device_tx_status = jz4780_dma_tx_status; + dd->device_issue_pending = jz4780_dma_issue_pending; + dd->src_addr_widths = JZ_DMA_BUSWIDTHS; + dd->dst_addr_widths = JZ_DMA_BUSWIDTHS; + dd->directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV); + dd->residue_granularity = DMA_RESIDUE_GRANULARITY_BURST; + + + /* + * Enable DMA controller, mark all channels as not programmable. + * Also set the FMSC bit - it increases MSC performance, so it makes + * little sense not to enable it. + */ + jz4780_dma_writel(jzdma, JZ_DMA_REG_DMAC, + JZ_DMA_DMAC_DMAE | JZ_DMA_DMAC_FMSC); + jz4780_dma_writel(jzdma, JZ_DMA_REG_DMACP, 0); + + INIT_LIST_HEAD(&dd->channels); + + for (i = 0; i < JZ_DMA_NR_CHANNELS; i++) { + jzchan = &jzdma->chan[i]; + jzchan->id = i; + + vchan_init(&jzchan->vchan, dd); + jzchan->vchan.desc_free = jz4780_dma_desc_free; + } + + ret = dma_async_device_register(dd); + if (ret) { + dev_err(dev, "failed to register device\n"); + goto err_disable_clk; + } + + /* Register with OF DMA helpers. */ + ret = of_dma_controller_register(dev->of_node, jz4780_of_dma_xlate, + jzdma); + if (ret) { + dev_err(dev, "failed to register OF DMA controller\n"); + goto err_unregister_dev; + } + + dev_info(dev, "JZ4780 DMA controller initialised\n"); + return 0; + +err_unregister_dev: + dma_async_device_unregister(dd); + +err_disable_clk: + clk_disable_unprepare(jzdma->clk); + return ret; +} + +static int jz4780_dma_remove(struct platform_device *pdev) +{ + struct jz4780_dma_dev *jzdma = platform_get_drvdata(pdev); + + of_dma_controller_free(pdev->dev.of_node); + devm_free_irq(&pdev->dev, jzdma->irq, jzdma); + dma_async_device_unregister(&jzdma->dma_device); + return 0; +} + +static const struct of_device_id jz4780_dma_dt_match[] = { + { .compatible = "ingenic,jz4780-dma", .data = NULL }, + {}, +}; +MODULE_DEVICE_TABLE(of, jz4780_dma_dt_match); + +static struct platform_driver jz4780_dma_driver = { + .probe = jz4780_dma_probe, + .remove = jz4780_dma_remove, + .driver = { + .name = "jz4780-dma", + .of_match_table = of_match_ptr(jz4780_dma_dt_match), + }, +}; + +static int __init jz4780_dma_init(void) +{ + return platform_driver_register(&jz4780_dma_driver); +} +subsys_initcall(jz4780_dma_init); + +static void __exit jz4780_dma_exit(void) +{ + platform_driver_unregister(&jz4780_dma_driver); +} +module_exit(jz4780_dma_exit); + +MODULE_AUTHOR("Alex Smith <alex@alex-smith.me.uk>"); +MODULE_DESCRIPTION("Ingenic JZ4780 DMA controller driver"); +MODULE_LICENSE("GPL"); diff --git a/drivers/dma/dmaengine.c b/drivers/dma/dmaengine.c index ac336a961dea..0e035a8cf401 100644 --- a/drivers/dma/dmaengine.c +++ b/drivers/dma/dmaengine.c @@ -11,10 +11,6 @@ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for * more details. * - * You should have received a copy of the GNU General Public License along with - * this program; if not, write to the Free Software Foundation, Inc., 59 - * Temple Place - Suite 330, Boston, MA 02111-1307, USA. - * * The full GNU General Public License is included in this distribution in the * file called COPYING. */ @@ -355,20 +351,6 @@ struct dma_chan *dma_find_channel(enum dma_transaction_type tx_type) } EXPORT_SYMBOL(dma_find_channel); -/* - * net_dma_find_channel - find a channel for net_dma - * net_dma has alignment requirements - */ -struct dma_chan *net_dma_find_channel(void) -{ - struct dma_chan *chan = dma_find_channel(DMA_MEMCPY); - if (chan && !is_dma_copy_aligned(chan->device, 1, 1, 1)) - return NULL; - - return chan; -} -EXPORT_SYMBOL(net_dma_find_channel); - /** * dma_issue_pending_all - flush all pending operations across all channels */ diff --git a/drivers/dma/dw/Kconfig b/drivers/dma/dw/Kconfig index dcfe964cc8dc..36e02f0f645e 100644 --- a/drivers/dma/dw/Kconfig +++ b/drivers/dma/dw/Kconfig @@ -3,7 +3,7 @@ # config DW_DMAC_CORE - tristate "Synopsys DesignWare AHB DMA support" + tristate select DMA_ENGINE config DW_DMAC diff --git a/drivers/dma/dw/core.c b/drivers/dma/dw/core.c index a8ad05291b27..1022c2e1a2b0 100644 --- a/drivers/dma/dw/core.c +++ b/drivers/dma/dw/core.c @@ -230,7 +230,8 @@ static void dwc_dostart(struct dw_dma_chan *dwc, struct dw_desc *first) /* ASSERT: channel is idle */ if (dma_readl(dw, CH_EN) & dwc->mask) { dev_err(chan2dev(&dwc->chan), - "BUG: Attempted to start non-idle channel\n"); + "%s: BUG: Attempted to start non-idle channel\n", + __func__); dwc_dump_chan_regs(dwc); /* The tasklet will hopefully advance the queue... */ @@ -814,11 +815,8 @@ dwc_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl, slave_sg_todev_fill_desc: desc = dwc_desc_get(dwc); - if (!desc) { - dev_err(chan2dev(chan), - "not enough descriptors available\n"); + if (!desc) goto err_desc_get; - } desc->lli.sar = mem; desc->lli.dar = reg; @@ -874,11 +872,8 @@ slave_sg_todev_fill_desc: slave_sg_fromdev_fill_desc: desc = dwc_desc_get(dwc); - if (!desc) { - dev_err(chan2dev(chan), - "not enough descriptors available\n"); + if (!desc) goto err_desc_get; - } desc->lli.sar = reg; desc->lli.dar = mem; @@ -922,6 +917,8 @@ slave_sg_fromdev_fill_desc: return &first->txd; err_desc_get: + dev_err(chan2dev(chan), + "not enough descriptors available. Direction %d\n", direction); dwc_desc_put(dwc, first); return NULL; } @@ -1261,7 +1258,8 @@ int dw_dma_cyclic_start(struct dma_chan *chan) /* Assert channel is idle */ if (dma_readl(dw, CH_EN) & dwc->mask) { dev_err(chan2dev(&dwc->chan), - "BUG: Attempted to start non-idle channel\n"); + "%s: BUG: Attempted to start non-idle channel\n", + __func__); dwc_dump_chan_regs(dwc); spin_unlock_irqrestore(&dwc->lock, flags); return -EBUSY; diff --git a/drivers/dma/edma.c b/drivers/dma/edma.c index 53dbd3b3384c..bf09db7ca9ee 100644 --- a/drivers/dma/edma.c +++ b/drivers/dma/edma.c @@ -812,7 +812,7 @@ static int edma_alloc_chan_resources(struct dma_chan *chan) LIST_HEAD(descs); a_ch_num = edma_alloc_channel(echan->ch_num, edma_callback, - chan, EVENTQ_DEFAULT); + echan, EVENTQ_DEFAULT); if (a_ch_num < 0) { ret = -ENODEV; diff --git a/drivers/dma/fsl_raid.c b/drivers/dma/fsl_raid.c new file mode 100644 index 000000000000..4d9470f16552 --- /dev/null +++ b/drivers/dma/fsl_raid.c @@ -0,0 +1,904 @@ +/* + * drivers/dma/fsl_raid.c + * + * Freescale RAID Engine device driver + * + * Author: + * Harninder Rai <harninder.rai@freescale.com> + * Naveen Burmi <naveenburmi@freescale.com> + * + * Rewrite: + * Xuelin Shi <xuelin.shi@freescale.com> + * + * Copyright (c) 2010-2014 Freescale Semiconductor, Inc. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Freescale Semiconductor nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * ALTERNATIVELY, this software may be distributed under the terms of the + * GNU General Public License ("GPL") as published by the Free Software + * Foundation, either version 2 of that License or (at your option) any + * later version. + * + * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * Theory of operation: + * + * General capabilities: + * RAID Engine (RE) block is capable of offloading XOR, memcpy and P/Q + * calculations required in RAID5 and RAID6 operations. RE driver + * registers with Linux's ASYNC layer as dma driver. RE hardware + * maintains strict ordering of the requests through chained + * command queueing. + * + * Data flow: + * Software RAID layer of Linux (MD layer) maintains RAID partitions, + * strips, stripes etc. It sends requests to the underlying ASYNC layer + * which further passes it to RE driver. ASYNC layer decides which request + * goes to which job ring of RE hardware. For every request processed by + * RAID Engine, driver gets an interrupt unless coalescing is set. The + * per job ring interrupt handler checks the status register for errors, + * clears the interrupt and leave the post interrupt processing to the irq + * thread. + */ +#include <linux/interrupt.h> +#include <linux/module.h> +#include <linux/of_irq.h> +#include <linux/of_address.h> +#include <linux/of_platform.h> +#include <linux/dma-mapping.h> +#include <linux/dmapool.h> +#include <linux/dmaengine.h> +#include <linux/io.h> +#include <linux/spinlock.h> +#include <linux/slab.h> + +#include "dmaengine.h" +#include "fsl_raid.h" + +#define FSL_RE_MAX_XOR_SRCS 16 +#define FSL_RE_MAX_PQ_SRCS 16 +#define FSL_RE_MIN_DESCS 256 +#define FSL_RE_MAX_DESCS (4 * FSL_RE_MIN_DESCS) +#define FSL_RE_FRAME_FORMAT 0x1 +#define FSL_RE_MAX_DATA_LEN (1024*1024) + +#define to_fsl_re_dma_desc(tx) container_of(tx, struct fsl_re_desc, async_tx) + +/* Add descriptors into per chan software queue - submit_q */ +static dma_cookie_t fsl_re_tx_submit(struct dma_async_tx_descriptor *tx) +{ + struct fsl_re_desc *desc; + struct fsl_re_chan *re_chan; + dma_cookie_t cookie; + unsigned long flags; + + desc = to_fsl_re_dma_desc(tx); + re_chan = container_of(tx->chan, struct fsl_re_chan, chan); + + spin_lock_irqsave(&re_chan->desc_lock, flags); + cookie = dma_cookie_assign(tx); + list_add_tail(&desc->node, &re_chan->submit_q); + spin_unlock_irqrestore(&re_chan->desc_lock, flags); + + return cookie; +} + +/* Copy descriptor from per chan software queue into hardware job ring */ +static void fsl_re_issue_pending(struct dma_chan *chan) +{ + struct fsl_re_chan *re_chan; + int avail; + struct fsl_re_desc *desc, *_desc; + unsigned long flags; + + re_chan = container_of(chan, struct fsl_re_chan, chan); + + spin_lock_irqsave(&re_chan->desc_lock, flags); + avail = FSL_RE_SLOT_AVAIL( + in_be32(&re_chan->jrregs->inbring_slot_avail)); + + list_for_each_entry_safe(desc, _desc, &re_chan->submit_q, node) { + if (!avail) + break; + + list_move_tail(&desc->node, &re_chan->active_q); + + memcpy(&re_chan->inb_ring_virt_addr[re_chan->inb_count], + &desc->hwdesc, sizeof(struct fsl_re_hw_desc)); + + re_chan->inb_count = (re_chan->inb_count + 1) & + FSL_RE_RING_SIZE_MASK; + out_be32(&re_chan->jrregs->inbring_add_job, FSL_RE_ADD_JOB(1)); + avail--; + } + spin_unlock_irqrestore(&re_chan->desc_lock, flags); +} + +static void fsl_re_desc_done(struct fsl_re_desc *desc) +{ + dma_async_tx_callback callback; + void *callback_param; + + dma_cookie_complete(&desc->async_tx); + + callback = desc->async_tx.callback; + callback_param = desc->async_tx.callback_param; + if (callback) + callback(callback_param); + + dma_descriptor_unmap(&desc->async_tx); +} + +static void fsl_re_cleanup_descs(struct fsl_re_chan *re_chan) +{ + struct fsl_re_desc *desc, *_desc; + unsigned long flags; + + spin_lock_irqsave(&re_chan->desc_lock, flags); + list_for_each_entry_safe(desc, _desc, &re_chan->ack_q, node) { + if (async_tx_test_ack(&desc->async_tx)) + list_move_tail(&desc->node, &re_chan->free_q); + } + spin_unlock_irqrestore(&re_chan->desc_lock, flags); + + fsl_re_issue_pending(&re_chan->chan); +} + +static void fsl_re_dequeue(unsigned long data) +{ + struct fsl_re_chan *re_chan; + struct fsl_re_desc *desc, *_desc; + struct fsl_re_hw_desc *hwdesc; + unsigned long flags; + unsigned int count, oub_count; + int found; + + re_chan = dev_get_drvdata((struct device *)data); + + fsl_re_cleanup_descs(re_chan); + + spin_lock_irqsave(&re_chan->desc_lock, flags); + count = FSL_RE_SLOT_FULL(in_be32(&re_chan->jrregs->oubring_slot_full)); + while (count--) { + found = 0; + hwdesc = &re_chan->oub_ring_virt_addr[re_chan->oub_count]; + list_for_each_entry_safe(desc, _desc, &re_chan->active_q, + node) { + /* compare the hw dma addr to find the completed */ + if (desc->hwdesc.lbea32 == hwdesc->lbea32 && + desc->hwdesc.addr_low == hwdesc->addr_low) { + found = 1; + break; + } + } + + if (found) { + fsl_re_desc_done(desc); + list_move_tail(&desc->node, &re_chan->ack_q); + } else { + dev_err(re_chan->dev, + "found hwdesc not in sw queue, discard it\n"); + } + + oub_count = (re_chan->oub_count + 1) & FSL_RE_RING_SIZE_MASK; + re_chan->oub_count = oub_count; + + out_be32(&re_chan->jrregs->oubring_job_rmvd, + FSL_RE_RMVD_JOB(1)); + } + spin_unlock_irqrestore(&re_chan->desc_lock, flags); +} + +/* Per Job Ring interrupt handler */ +static irqreturn_t fsl_re_isr(int irq, void *data) +{ + struct fsl_re_chan *re_chan; + u32 irqstate, status; + + re_chan = dev_get_drvdata((struct device *)data); + + irqstate = in_be32(&re_chan->jrregs->jr_interrupt_status); + if (!irqstate) + return IRQ_NONE; + + /* + * There's no way in upper layer (read MD layer) to recover from + * error conditions except restart everything. In long term we + * need to do something more than just crashing + */ + if (irqstate & FSL_RE_ERROR) { + status = in_be32(&re_chan->jrregs->jr_status); + dev_err(re_chan->dev, "chan error irqstate: %x, status: %x\n", + irqstate, status); + } + + /* Clear interrupt */ + out_be32(&re_chan->jrregs->jr_interrupt_status, FSL_RE_CLR_INTR); + + tasklet_schedule(&re_chan->irqtask); + + return IRQ_HANDLED; +} + +static enum dma_status fsl_re_tx_status(struct dma_chan *chan, + dma_cookie_t cookie, + struct dma_tx_state *txstate) +{ + return dma_cookie_status(chan, cookie, txstate); +} + +static void fill_cfd_frame(struct fsl_re_cmpnd_frame *cf, u8 index, + size_t length, dma_addr_t addr, bool final) +{ + u32 efrl = length & FSL_RE_CF_LENGTH_MASK; + + efrl |= final << FSL_RE_CF_FINAL_SHIFT; + cf[index].efrl32 = efrl; + cf[index].addr_high = upper_32_bits(addr); + cf[index].addr_low = lower_32_bits(addr); +} + +static struct fsl_re_desc *fsl_re_init_desc(struct fsl_re_chan *re_chan, + struct fsl_re_desc *desc, + void *cf, dma_addr_t paddr) +{ + desc->re_chan = re_chan; + desc->async_tx.tx_submit = fsl_re_tx_submit; + dma_async_tx_descriptor_init(&desc->async_tx, &re_chan->chan); + INIT_LIST_HEAD(&desc->node); + + desc->hwdesc.fmt32 = FSL_RE_FRAME_FORMAT << FSL_RE_HWDESC_FMT_SHIFT; + desc->hwdesc.lbea32 = upper_32_bits(paddr); + desc->hwdesc.addr_low = lower_32_bits(paddr); + desc->cf_addr = cf; + desc->cf_paddr = paddr; + + desc->cdb_addr = (void *)(cf + FSL_RE_CF_DESC_SIZE); + desc->cdb_paddr = paddr + FSL_RE_CF_DESC_SIZE; + + return desc; +} + +static struct fsl_re_desc *fsl_re_chan_alloc_desc(struct fsl_re_chan *re_chan, + unsigned long flags) +{ + struct fsl_re_desc *desc = NULL; + void *cf; + dma_addr_t paddr; + unsigned long lock_flag; + + fsl_re_cleanup_descs(re_chan); + + spin_lock_irqsave(&re_chan->desc_lock, lock_flag); + if (!list_empty(&re_chan->free_q)) { + /* take one desc from free_q */ + desc = list_first_entry(&re_chan->free_q, + struct fsl_re_desc, node); + list_del(&desc->node); + + desc->async_tx.flags = flags; + } + spin_unlock_irqrestore(&re_chan->desc_lock, lock_flag); + + if (!desc) { + desc = kzalloc(sizeof(*desc), GFP_NOWAIT); + if (!desc) + return NULL; + + cf = dma_pool_alloc(re_chan->re_dev->cf_desc_pool, GFP_NOWAIT, + &paddr); + if (!cf) { + kfree(desc); + return NULL; + } + + desc = fsl_re_init_desc(re_chan, desc, cf, paddr); + desc->async_tx.flags = flags; + + spin_lock_irqsave(&re_chan->desc_lock, lock_flag); + re_chan->alloc_count++; + spin_unlock_irqrestore(&re_chan->desc_lock, lock_flag); + } + + return desc; +} + +static struct dma_async_tx_descriptor *fsl_re_prep_dma_genq( + struct dma_chan *chan, dma_addr_t dest, dma_addr_t *src, + unsigned int src_cnt, const unsigned char *scf, size_t len, + unsigned long flags) +{ + struct fsl_re_chan *re_chan; + struct fsl_re_desc *desc; + struct fsl_re_xor_cdb *xor; + struct fsl_re_cmpnd_frame *cf; + u32 cdb; + unsigned int i, j; + unsigned int save_src_cnt = src_cnt; + int cont_q = 0; + + re_chan = container_of(chan, struct fsl_re_chan, chan); + if (len > FSL_RE_MAX_DATA_LEN) { + dev_err(re_chan->dev, "genq tx length %lu, max length %d\n", + len, FSL_RE_MAX_DATA_LEN); + return NULL; + } + + desc = fsl_re_chan_alloc_desc(re_chan, flags); + if (desc <= 0) + return NULL; + + if (scf && (flags & DMA_PREP_CONTINUE)) { + cont_q = 1; + src_cnt += 1; + } + + /* Filling xor CDB */ + cdb = FSL_RE_XOR_OPCODE << FSL_RE_CDB_OPCODE_SHIFT; + cdb |= (src_cnt - 1) << FSL_RE_CDB_NRCS_SHIFT; + cdb |= FSL_RE_BLOCK_SIZE << FSL_RE_CDB_BLKSIZE_SHIFT; + cdb |= FSL_RE_INTR_ON_ERROR << FSL_RE_CDB_ERROR_SHIFT; + cdb |= FSL_RE_DATA_DEP << FSL_RE_CDB_DEPEND_SHIFT; + xor = desc->cdb_addr; + xor->cdb32 = cdb; + + if (scf) { + /* compute q = src0*coef0^src1*coef1^..., * is GF(8) mult */ + for (i = 0; i < save_src_cnt; i++) + xor->gfm[i] = scf[i]; + if (cont_q) + xor->gfm[i++] = 1; + } else { + /* compute P, that is XOR all srcs */ + for (i = 0; i < src_cnt; i++) + xor->gfm[i] = 1; + } + + /* Filling frame 0 of compound frame descriptor with CDB */ + cf = desc->cf_addr; + fill_cfd_frame(cf, 0, sizeof(*xor), desc->cdb_paddr, 0); + + /* Fill CFD's 1st frame with dest buffer */ + fill_cfd_frame(cf, 1, len, dest, 0); + + /* Fill CFD's rest of the frames with source buffers */ + for (i = 2, j = 0; j < save_src_cnt; i++, j++) + fill_cfd_frame(cf, i, len, src[j], 0); + + if (cont_q) + fill_cfd_frame(cf, i++, len, dest, 0); + + /* Setting the final bit in the last source buffer frame in CFD */ + cf[i - 1].efrl32 |= 1 << FSL_RE_CF_FINAL_SHIFT; + + return &desc->async_tx; +} + +/* + * Prep function for P parity calculation.In RAID Engine terminology, + * XOR calculation is called GenQ calculation done through GenQ command + */ +static struct dma_async_tx_descriptor *fsl_re_prep_dma_xor( + struct dma_chan *chan, dma_addr_t dest, dma_addr_t *src, + unsigned int src_cnt, size_t len, unsigned long flags) +{ + /* NULL let genq take all coef as 1 */ + return fsl_re_prep_dma_genq(chan, dest, src, src_cnt, NULL, len, flags); +} + +/* + * Prep function for P/Q parity calculation.In RAID Engine terminology, + * P/Q calculation is called GenQQ done through GenQQ command + */ +static struct dma_async_tx_descriptor *fsl_re_prep_dma_pq( + struct dma_chan *chan, dma_addr_t *dest, dma_addr_t *src, + unsigned int src_cnt, const unsigned char *scf, size_t len, + unsigned long flags) +{ + struct fsl_re_chan *re_chan; + struct fsl_re_desc *desc; + struct fsl_re_pq_cdb *pq; + struct fsl_re_cmpnd_frame *cf; + u32 cdb; + u8 *p; + int gfmq_len, i, j; + unsigned int save_src_cnt = src_cnt; + + re_chan = container_of(chan, struct fsl_re_chan, chan); + if (len > FSL_RE_MAX_DATA_LEN) { + dev_err(re_chan->dev, "pq tx length is %lu, max length is %d\n", + len, FSL_RE_MAX_DATA_LEN); + return NULL; + } + + /* + * RE requires at least 2 sources, if given only one source, we pass the + * second source same as the first one. + * With only one source, generating P is meaningless, only generate Q. + */ + if (src_cnt == 1) { + struct dma_async_tx_descriptor *tx; + dma_addr_t dma_src[2]; + unsigned char coef[2]; + + dma_src[0] = *src; + coef[0] = *scf; + dma_src[1] = *src; + coef[1] = 0; + tx = fsl_re_prep_dma_genq(chan, dest[1], dma_src, 2, coef, len, + flags); + if (tx) + desc = to_fsl_re_dma_desc(tx); + + return tx; + } + + /* + * During RAID6 array creation, Linux's MD layer gets P and Q + * calculated separately in two steps. But our RAID Engine has + * the capability to calculate both P and Q with a single command + * Hence to merge well with MD layer, we need to provide a hook + * here and call re_jq_prep_dma_genq() function + */ + + if (flags & DMA_PREP_PQ_DISABLE_P) + return fsl_re_prep_dma_genq(chan, dest[1], src, src_cnt, + scf, len, flags); + + if (flags & DMA_PREP_CONTINUE) + src_cnt += 3; + + desc = fsl_re_chan_alloc_desc(re_chan, flags); + if (desc <= 0) + return NULL; + + /* Filling GenQQ CDB */ + cdb = FSL_RE_PQ_OPCODE << FSL_RE_CDB_OPCODE_SHIFT; + cdb |= (src_cnt - 1) << FSL_RE_CDB_NRCS_SHIFT; + cdb |= FSL_RE_BLOCK_SIZE << FSL_RE_CDB_BLKSIZE_SHIFT; + cdb |= FSL_RE_BUFFER_OUTPUT << FSL_RE_CDB_BUFFER_SHIFT; + cdb |= FSL_RE_DATA_DEP << FSL_RE_CDB_DEPEND_SHIFT; + + pq = desc->cdb_addr; + pq->cdb32 = cdb; + + p = pq->gfm_q1; + /* Init gfm_q1[] */ + for (i = 0; i < src_cnt; i++) + p[i] = 1; + + /* Align gfm[] to 32bit */ + gfmq_len = ALIGN(src_cnt, 4); + + /* Init gfm_q2[] */ + p += gfmq_len; + for (i = 0; i < src_cnt; i++) + p[i] = scf[i]; + + /* Filling frame 0 of compound frame descriptor with CDB */ + cf = desc->cf_addr; + fill_cfd_frame(cf, 0, sizeof(struct fsl_re_pq_cdb), desc->cdb_paddr, 0); + + /* Fill CFD's 1st & 2nd frame with dest buffers */ + for (i = 1, j = 0; i < 3; i++, j++) + fill_cfd_frame(cf, i, len, dest[j], 0); + + /* Fill CFD's rest of the frames with source buffers */ + for (i = 3, j = 0; j < save_src_cnt; i++, j++) + fill_cfd_frame(cf, i, len, src[j], 0); + + /* PQ computation continuation */ + if (flags & DMA_PREP_CONTINUE) { + if (src_cnt - save_src_cnt == 3) { + p[save_src_cnt] = 0; + p[save_src_cnt + 1] = 0; + p[save_src_cnt + 2] = 1; + fill_cfd_frame(cf, i++, len, dest[0], 0); + fill_cfd_frame(cf, i++, len, dest[1], 0); + fill_cfd_frame(cf, i++, len, dest[1], 0); + } else { + dev_err(re_chan->dev, "PQ tx continuation error!\n"); + return NULL; + } + } + + /* Setting the final bit in the last source buffer frame in CFD */ + cf[i - 1].efrl32 |= 1 << FSL_RE_CF_FINAL_SHIFT; + + return &desc->async_tx; +} + +/* + * Prep function for memcpy. In RAID Engine, memcpy is done through MOVE + * command. Logic of this function will need to be modified once multipage + * support is added in Linux's MD/ASYNC Layer + */ +static struct dma_async_tx_descriptor *fsl_re_prep_dma_memcpy( + struct dma_chan *chan, dma_addr_t dest, dma_addr_t src, + size_t len, unsigned long flags) +{ + struct fsl_re_chan *re_chan; + struct fsl_re_desc *desc; + size_t length; + struct fsl_re_cmpnd_frame *cf; + struct fsl_re_move_cdb *move; + u32 cdb; + + re_chan = container_of(chan, struct fsl_re_chan, chan); + + if (len > FSL_RE_MAX_DATA_LEN) { + dev_err(re_chan->dev, "cp tx length is %lu, max length is %d\n", + len, FSL_RE_MAX_DATA_LEN); + return NULL; + } + + desc = fsl_re_chan_alloc_desc(re_chan, flags); + if (desc <= 0) + return NULL; + + /* Filling move CDB */ + cdb = FSL_RE_MOVE_OPCODE << FSL_RE_CDB_OPCODE_SHIFT; + cdb |= FSL_RE_BLOCK_SIZE << FSL_RE_CDB_BLKSIZE_SHIFT; + cdb |= FSL_RE_INTR_ON_ERROR << FSL_RE_CDB_ERROR_SHIFT; + cdb |= FSL_RE_DATA_DEP << FSL_RE_CDB_DEPEND_SHIFT; + + move = desc->cdb_addr; + move->cdb32 = cdb; + + /* Filling frame 0 of CFD with move CDB */ + cf = desc->cf_addr; + fill_cfd_frame(cf, 0, sizeof(*move), desc->cdb_paddr, 0); + + length = min_t(size_t, len, FSL_RE_MAX_DATA_LEN); + + /* Fill CFD's 1st frame with dest buffer */ + fill_cfd_frame(cf, 1, length, dest, 0); + + /* Fill CFD's 2nd frame with src buffer */ + fill_cfd_frame(cf, 2, length, src, 1); + + return &desc->async_tx; +} + +static int fsl_re_alloc_chan_resources(struct dma_chan *chan) +{ + struct fsl_re_chan *re_chan; + struct fsl_re_desc *desc; + void *cf; + dma_addr_t paddr; + int i; + + re_chan = container_of(chan, struct fsl_re_chan, chan); + for (i = 0; i < FSL_RE_MIN_DESCS; i++) { + desc = kzalloc(sizeof(*desc), GFP_KERNEL); + if (!desc) + break; + + cf = dma_pool_alloc(re_chan->re_dev->cf_desc_pool, GFP_KERNEL, + &paddr); + if (!cf) { + kfree(desc); + break; + } + + INIT_LIST_HEAD(&desc->node); + fsl_re_init_desc(re_chan, desc, cf, paddr); + + list_add_tail(&desc->node, &re_chan->free_q); + re_chan->alloc_count++; + } + return re_chan->alloc_count; +} + +static void fsl_re_free_chan_resources(struct dma_chan *chan) +{ + struct fsl_re_chan *re_chan; + struct fsl_re_desc *desc; + + re_chan = container_of(chan, struct fsl_re_chan, chan); + while (re_chan->alloc_count--) { + desc = list_first_entry(&re_chan->free_q, + struct fsl_re_desc, + node); + + list_del(&desc->node); + dma_pool_free(re_chan->re_dev->cf_desc_pool, desc->cf_addr, + desc->cf_paddr); + kfree(desc); + } + + if (!list_empty(&re_chan->free_q)) + dev_err(re_chan->dev, "chan resource cannot be cleaned!\n"); +} + +static int fsl_re_chan_probe(struct platform_device *ofdev, + struct device_node *np, u8 q, u32 off) +{ + struct device *dev, *chandev; + struct fsl_re_drv_private *re_priv; + struct fsl_re_chan *chan; + struct dma_device *dma_dev; + u32 ptr; + u32 status; + int ret = 0, rc; + struct platform_device *chan_ofdev; + + dev = &ofdev->dev; + re_priv = dev_get_drvdata(dev); + dma_dev = &re_priv->dma_dev; + + chan = devm_kzalloc(dev, sizeof(*chan), GFP_KERNEL); + if (!chan) + return -ENOMEM; + + /* create platform device for chan node */ + chan_ofdev = of_platform_device_create(np, NULL, dev); + if (!chan_ofdev) { + dev_err(dev, "Not able to create ofdev for jr %d\n", q); + ret = -EINVAL; + goto err_free; + } + + /* read reg property from dts */ + rc = of_property_read_u32(np, "reg", &ptr); + if (rc) { + dev_err(dev, "Reg property not found in jr %d\n", q); + ret = -ENODEV; + goto err_free; + } + + chan->jrregs = (struct fsl_re_chan_cfg *)((u8 *)re_priv->re_regs + + off + ptr); + + /* read irq property from dts */ + chan->irq = irq_of_parse_and_map(np, 0); + if (chan->irq == NO_IRQ) { + dev_err(dev, "No IRQ defined for JR %d\n", q); + ret = -ENODEV; + goto err_free; + } + + snprintf(chan->name, sizeof(chan->name), "re_jr%02d", q); + + chandev = &chan_ofdev->dev; + tasklet_init(&chan->irqtask, fsl_re_dequeue, (unsigned long)chandev); + + ret = request_irq(chan->irq, fsl_re_isr, 0, chan->name, chandev); + if (ret) { + dev_err(dev, "Unable to register interrupt for JR %d\n", q); + ret = -EINVAL; + goto err_free; + } + + re_priv->re_jrs[q] = chan; + chan->chan.device = dma_dev; + chan->chan.private = chan; + chan->dev = chandev; + chan->re_dev = re_priv; + + spin_lock_init(&chan->desc_lock); + INIT_LIST_HEAD(&chan->ack_q); + INIT_LIST_HEAD(&chan->active_q); + INIT_LIST_HEAD(&chan->submit_q); + INIT_LIST_HEAD(&chan->free_q); + + chan->inb_ring_virt_addr = dma_pool_alloc(chan->re_dev->hw_desc_pool, + GFP_KERNEL, &chan->inb_phys_addr); + if (!chan->inb_ring_virt_addr) { + dev_err(dev, "No dma memory for inb_ring_virt_addr\n"); + ret = -ENOMEM; + goto err_free; + } + + chan->oub_ring_virt_addr = dma_pool_alloc(chan->re_dev->hw_desc_pool, + GFP_KERNEL, &chan->oub_phys_addr); + if (!chan->oub_ring_virt_addr) { + dev_err(dev, "No dma memory for oub_ring_virt_addr\n"); + ret = -ENOMEM; + goto err_free_1; + } + + /* Program the Inbound/Outbound ring base addresses and size */ + out_be32(&chan->jrregs->inbring_base_h, + chan->inb_phys_addr & FSL_RE_ADDR_BIT_MASK); + out_be32(&chan->jrregs->oubring_base_h, + chan->oub_phys_addr & FSL_RE_ADDR_BIT_MASK); + out_be32(&chan->jrregs->inbring_base_l, + chan->inb_phys_addr >> FSL_RE_ADDR_BIT_SHIFT); + out_be32(&chan->jrregs->oubring_base_l, + chan->oub_phys_addr >> FSL_RE_ADDR_BIT_SHIFT); + out_be32(&chan->jrregs->inbring_size, + FSL_RE_RING_SIZE << FSL_RE_RING_SIZE_SHIFT); + out_be32(&chan->jrregs->oubring_size, + FSL_RE_RING_SIZE << FSL_RE_RING_SIZE_SHIFT); + + /* Read LIODN value from u-boot */ + status = in_be32(&chan->jrregs->jr_config_1) & FSL_RE_REG_LIODN_MASK; + + /* Program the CFG reg */ + out_be32(&chan->jrregs->jr_config_1, + FSL_RE_CFG1_CBSI | FSL_RE_CFG1_CBS0 | status); + + dev_set_drvdata(chandev, chan); + + /* Enable RE/CHAN */ + out_be32(&chan->jrregs->jr_command, FSL_RE_ENABLE); + + return 0; + +err_free_1: + dma_pool_free(chan->re_dev->hw_desc_pool, chan->inb_ring_virt_addr, + chan->inb_phys_addr); +err_free: + return ret; +} + +/* Probe function for RAID Engine */ +static int fsl_re_probe(struct platform_device *ofdev) +{ + struct fsl_re_drv_private *re_priv; + struct device_node *np; + struct device_node *child; + u32 off; + u8 ridx = 0; + struct dma_device *dma_dev; + struct resource *res; + int rc; + struct device *dev = &ofdev->dev; + + re_priv = devm_kzalloc(dev, sizeof(*re_priv), GFP_KERNEL); + if (!re_priv) + return -ENOMEM; + + res = platform_get_resource(ofdev, IORESOURCE_MEM, 0); + if (!res) + return -ENODEV; + + /* IOMAP the entire RAID Engine region */ + re_priv->re_regs = devm_ioremap(dev, res->start, resource_size(res)); + if (!re_priv->re_regs) + return -EBUSY; + + /* Program the RE mode */ + out_be32(&re_priv->re_regs->global_config, FSL_RE_NON_DPAA_MODE); + + /* Program Galois Field polynomial */ + out_be32(&re_priv->re_regs->galois_field_config, FSL_RE_GFM_POLY); + + dev_info(dev, "version %x, mode %x, gfp %x\n", + in_be32(&re_priv->re_regs->re_version_id), + in_be32(&re_priv->re_regs->global_config), + in_be32(&re_priv->re_regs->galois_field_config)); + + dma_dev = &re_priv->dma_dev; + dma_dev->dev = dev; + INIT_LIST_HEAD(&dma_dev->channels); + dma_set_mask(dev, DMA_BIT_MASK(40)); + + dma_dev->device_alloc_chan_resources = fsl_re_alloc_chan_resources; + dma_dev->device_tx_status = fsl_re_tx_status; + dma_dev->device_issue_pending = fsl_re_issue_pending; + + dma_dev->max_xor = FSL_RE_MAX_XOR_SRCS; + dma_dev->device_prep_dma_xor = fsl_re_prep_dma_xor; + dma_cap_set(DMA_XOR, dma_dev->cap_mask); + + dma_dev->max_pq = FSL_RE_MAX_PQ_SRCS; + dma_dev->device_prep_dma_pq = fsl_re_prep_dma_pq; + dma_cap_set(DMA_PQ, dma_dev->cap_mask); + + dma_dev->device_prep_dma_memcpy = fsl_re_prep_dma_memcpy; + dma_cap_set(DMA_MEMCPY, dma_dev->cap_mask); + + dma_dev->device_free_chan_resources = fsl_re_free_chan_resources; + + re_priv->total_chans = 0; + + re_priv->cf_desc_pool = dmam_pool_create("fsl_re_cf_desc_pool", dev, + FSL_RE_CF_CDB_SIZE, + FSL_RE_CF_CDB_ALIGN, 0); + + if (!re_priv->cf_desc_pool) { + dev_err(dev, "No memory for fsl re_cf desc pool\n"); + return -ENOMEM; + } + + re_priv->hw_desc_pool = dmam_pool_create("fsl_re_hw_desc_pool", dev, + sizeof(struct fsl_re_hw_desc) * FSL_RE_RING_SIZE, + FSL_RE_FRAME_ALIGN, 0); + if (!re_priv->hw_desc_pool) { + dev_err(dev, "No memory for fsl re_hw desc pool\n"); + return -ENOMEM; + } + + dev_set_drvdata(dev, re_priv); + + /* Parse Device tree to find out the total number of JQs present */ + for_each_compatible_node(np, NULL, "fsl,raideng-v1.0-job-queue") { + rc = of_property_read_u32(np, "reg", &off); + if (rc) { + dev_err(dev, "Reg property not found in JQ node\n"); + return -ENODEV; + } + /* Find out the Job Rings present under each JQ */ + for_each_child_of_node(np, child) { + rc = of_device_is_compatible(child, + "fsl,raideng-v1.0-job-ring"); + if (rc) { + fsl_re_chan_probe(ofdev, child, ridx++, off); + re_priv->total_chans++; + } + } + } + + dma_async_device_register(dma_dev); + + return 0; +} + +static void fsl_re_remove_chan(struct fsl_re_chan *chan) +{ + dma_pool_free(chan->re_dev->hw_desc_pool, chan->inb_ring_virt_addr, + chan->inb_phys_addr); + + dma_pool_free(chan->re_dev->hw_desc_pool, chan->oub_ring_virt_addr, + chan->oub_phys_addr); +} + +static int fsl_re_remove(struct platform_device *ofdev) +{ + struct fsl_re_drv_private *re_priv; + struct device *dev; + int i; + + dev = &ofdev->dev; + re_priv = dev_get_drvdata(dev); + + /* Cleanup chan related memory areas */ + for (i = 0; i < re_priv->total_chans; i++) + fsl_re_remove_chan(re_priv->re_jrs[i]); + + /* Unregister the driver */ + dma_async_device_unregister(&re_priv->dma_dev); + + return 0; +} + +static struct of_device_id fsl_re_ids[] = { + { .compatible = "fsl,raideng-v1.0", }, + {} +}; + +static struct platform_driver fsl_re_driver = { + .driver = { + .name = "fsl-raideng", + .owner = THIS_MODULE, + .of_match_table = fsl_re_ids, + }, + .probe = fsl_re_probe, + .remove = fsl_re_remove, +}; + +module_platform_driver(fsl_re_driver); + +MODULE_AUTHOR("Harninder Rai <harninder.rai@freescale.com>"); +MODULE_LICENSE("GPL v2"); +MODULE_DESCRIPTION("Freescale RAID Engine Device Driver"); diff --git a/drivers/dma/fsl_raid.h b/drivers/dma/fsl_raid.h new file mode 100644 index 000000000000..69d743c04973 --- /dev/null +++ b/drivers/dma/fsl_raid.h @@ -0,0 +1,306 @@ +/* + * drivers/dma/fsl_raid.h + * + * Freescale RAID Engine device driver + * + * Author: + * Harninder Rai <harninder.rai@freescale.com> + * Naveen Burmi <naveenburmi@freescale.com> + * + * Rewrite: + * Xuelin Shi <xuelin.shi@freescale.com> + + * Copyright (c) 2010-2012 Freescale Semiconductor, Inc. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Freescale Semiconductor nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * ALTERNATIVELY, this software may be distributed under the terms of the + * GNU General Public License ("GPL") as published by the Free Software + * Foundation, either version 2 of that License or (at your option) any + * later version. + * + * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#define FSL_RE_MAX_CHANS 4 +#define FSL_RE_DPAA_MODE BIT(30) +#define FSL_RE_NON_DPAA_MODE BIT(31) +#define FSL_RE_GFM_POLY 0x1d000000 +#define FSL_RE_ADD_JOB(x) ((x) << 16) +#define FSL_RE_RMVD_JOB(x) ((x) << 16) +#define FSL_RE_CFG1_CBSI 0x08000000 +#define FSL_RE_CFG1_CBS0 0x00080000 +#define FSL_RE_SLOT_FULL_SHIFT 8 +#define FSL_RE_SLOT_FULL(x) ((x) >> FSL_RE_SLOT_FULL_SHIFT) +#define FSL_RE_SLOT_AVAIL_SHIFT 8 +#define FSL_RE_SLOT_AVAIL(x) ((x) >> FSL_RE_SLOT_AVAIL_SHIFT) +#define FSL_RE_PQ_OPCODE 0x1B +#define FSL_RE_XOR_OPCODE 0x1A +#define FSL_RE_MOVE_OPCODE 0x8 +#define FSL_RE_FRAME_ALIGN 16 +#define FSL_RE_BLOCK_SIZE 0x3 /* 4096 bytes */ +#define FSL_RE_CACHEABLE_IO 0x0 +#define FSL_RE_BUFFER_OUTPUT 0x0 +#define FSL_RE_INTR_ON_ERROR 0x1 +#define FSL_RE_DATA_DEP 0x1 +#define FSL_RE_ENABLE_DPI 0x0 +#define FSL_RE_RING_SIZE 0x400 +#define FSL_RE_RING_SIZE_MASK (FSL_RE_RING_SIZE - 1) +#define FSL_RE_RING_SIZE_SHIFT 8 +#define FSL_RE_ADDR_BIT_SHIFT 4 +#define FSL_RE_ADDR_BIT_MASK (BIT(FSL_RE_ADDR_BIT_SHIFT) - 1) +#define FSL_RE_ERROR 0x40000000 +#define FSL_RE_INTR 0x80000000 +#define FSL_RE_CLR_INTR 0x80000000 +#define FSL_RE_PAUSE 0x80000000 +#define FSL_RE_ENABLE 0x80000000 +#define FSL_RE_REG_LIODN_MASK 0x00000FFF + +#define FSL_RE_CDB_OPCODE_MASK 0xF8000000 +#define FSL_RE_CDB_OPCODE_SHIFT 27 +#define FSL_RE_CDB_EXCLEN_MASK 0x03000000 +#define FSL_RE_CDB_EXCLEN_SHIFT 24 +#define FSL_RE_CDB_EXCLQ1_MASK 0x00F00000 +#define FSL_RE_CDB_EXCLQ1_SHIFT 20 +#define FSL_RE_CDB_EXCLQ2_MASK 0x000F0000 +#define FSL_RE_CDB_EXCLQ2_SHIFT 16 +#define FSL_RE_CDB_BLKSIZE_MASK 0x0000C000 +#define FSL_RE_CDB_BLKSIZE_SHIFT 14 +#define FSL_RE_CDB_CACHE_MASK 0x00003000 +#define FSL_RE_CDB_CACHE_SHIFT 12 +#define FSL_RE_CDB_BUFFER_MASK 0x00000800 +#define FSL_RE_CDB_BUFFER_SHIFT 11 +#define FSL_RE_CDB_ERROR_MASK 0x00000400 +#define FSL_RE_CDB_ERROR_SHIFT 10 +#define FSL_RE_CDB_NRCS_MASK 0x0000003C +#define FSL_RE_CDB_NRCS_SHIFT 6 +#define FSL_RE_CDB_DEPEND_MASK 0x00000008 +#define FSL_RE_CDB_DEPEND_SHIFT 3 +#define FSL_RE_CDB_DPI_MASK 0x00000004 +#define FSL_RE_CDB_DPI_SHIFT 2 + +/* + * the largest cf block is 19*sizeof(struct cmpnd_frame), which is 304 bytes. + * here 19 = 1(cdb)+2(dest)+16(src), align to 64bytes, that is 320 bytes. + * the largest cdb block: struct pq_cdb which is 180 bytes, adding to cf block + * 320+180=500, align to 64bytes, that is 512 bytes. + */ +#define FSL_RE_CF_DESC_SIZE 320 +#define FSL_RE_CF_CDB_SIZE 512 +#define FSL_RE_CF_CDB_ALIGN 64 + +struct fsl_re_ctrl { + /* General Configuration Registers */ + __be32 global_config; /* Global Configuration Register */ + u8 rsvd1[4]; + __be32 galois_field_config; /* Galois Field Configuration Register */ + u8 rsvd2[4]; + __be32 jq_wrr_config; /* WRR Configuration register */ + u8 rsvd3[4]; + __be32 crc_config; /* CRC Configuration register */ + u8 rsvd4[228]; + __be32 system_reset; /* System Reset Register */ + u8 rsvd5[252]; + __be32 global_status; /* Global Status Register */ + u8 rsvd6[832]; + __be32 re_liodn_base; /* LIODN Base Register */ + u8 rsvd7[1712]; + __be32 re_version_id; /* Version ID register of RE */ + __be32 re_version_id_2; /* Version ID 2 register of RE */ + u8 rsvd8[512]; + __be32 host_config; /* Host I/F Configuration Register */ +}; + +struct fsl_re_chan_cfg { + /* Registers for JR interface */ + __be32 jr_config_0; /* Job Queue Configuration 0 Register */ + __be32 jr_config_1; /* Job Queue Configuration 1 Register */ + __be32 jr_interrupt_status; /* Job Queue Interrupt Status Register */ + u8 rsvd1[4]; + __be32 jr_command; /* Job Queue Command Register */ + u8 rsvd2[4]; + __be32 jr_status; /* Job Queue Status Register */ + u8 rsvd3[228]; + + /* Input Ring */ + __be32 inbring_base_h; /* Inbound Ring Base Address Register - High */ + __be32 inbring_base_l; /* Inbound Ring Base Address Register - Low */ + __be32 inbring_size; /* Inbound Ring Size Register */ + u8 rsvd4[4]; + __be32 inbring_slot_avail; /* Inbound Ring Slot Available Register */ + u8 rsvd5[4]; + __be32 inbring_add_job; /* Inbound Ring Add Job Register */ + u8 rsvd6[4]; + __be32 inbring_cnsmr_indx; /* Inbound Ring Consumer Index Register */ + u8 rsvd7[220]; + + /* Output Ring */ + __be32 oubring_base_h; /* Outbound Ring Base Address Register - High */ + __be32 oubring_base_l; /* Outbound Ring Base Address Register - Low */ + __be32 oubring_size; /* Outbound Ring Size Register */ + u8 rsvd8[4]; + __be32 oubring_job_rmvd; /* Outbound Ring Job Removed Register */ + u8 rsvd9[4]; + __be32 oubring_slot_full; /* Outbound Ring Slot Full Register */ + u8 rsvd10[4]; + __be32 oubring_prdcr_indx; /* Outbound Ring Producer Index */ +}; + +/* + * Command Descriptor Block (CDB) for unicast move command. + * In RAID Engine terms, memcpy is done through move command + */ +struct fsl_re_move_cdb { + __be32 cdb32; +}; + +/* Data protection/integrity related fields */ +#define FSL_RE_DPI_APPS_MASK 0xC0000000 +#define FSL_RE_DPI_APPS_SHIFT 30 +#define FSL_RE_DPI_REF_MASK 0x30000000 +#define FSL_RE_DPI_REF_SHIFT 28 +#define FSL_RE_DPI_GUARD_MASK 0x0C000000 +#define FSL_RE_DPI_GUARD_SHIFT 26 +#define FSL_RE_DPI_ATTR_MASK 0x03000000 +#define FSL_RE_DPI_ATTR_SHIFT 24 +#define FSL_RE_DPI_META_MASK 0x0000FFFF + +struct fsl_re_dpi { + __be32 dpi32; + __be32 ref; +}; + +/* + * CDB for GenQ command. In RAID Engine terminology, XOR is + * done through this command + */ +struct fsl_re_xor_cdb { + __be32 cdb32; + u8 gfm[16]; + struct fsl_re_dpi dpi_dest_spec; + struct fsl_re_dpi dpi_src_spec[16]; +}; + +/* CDB for no-op command */ +struct fsl_re_noop_cdb { + __be32 cdb32; +}; + +/* + * CDB for GenQQ command. In RAID Engine terminology, P/Q is + * done through this command + */ +struct fsl_re_pq_cdb { + __be32 cdb32; + u8 gfm_q1[16]; + u8 gfm_q2[16]; + struct fsl_re_dpi dpi_dest_spec[2]; + struct fsl_re_dpi dpi_src_spec[16]; +}; + +/* Compound frame */ +#define FSL_RE_CF_ADDR_HIGH_MASK 0x000000FF +#define FSL_RE_CF_EXT_MASK 0x80000000 +#define FSL_RE_CF_EXT_SHIFT 31 +#define FSL_RE_CF_FINAL_MASK 0x40000000 +#define FSL_RE_CF_FINAL_SHIFT 30 +#define FSL_RE_CF_LENGTH_MASK 0x000FFFFF +#define FSL_RE_CF_BPID_MASK 0x00FF0000 +#define FSL_RE_CF_BPID_SHIFT 16 +#define FSL_RE_CF_OFFSET_MASK 0x00001FFF + +struct fsl_re_cmpnd_frame { + __be32 addr_high; + __be32 addr_low; + __be32 efrl32; + __be32 rbro32; +}; + +/* Frame descriptor */ +#define FSL_RE_HWDESC_LIODN_MASK 0x3F000000 +#define FSL_RE_HWDESC_LIODN_SHIFT 24 +#define FSL_RE_HWDESC_BPID_MASK 0x00FF0000 +#define FSL_RE_HWDESC_BPID_SHIFT 16 +#define FSL_RE_HWDESC_ELIODN_MASK 0x0000F000 +#define FSL_RE_HWDESC_ELIODN_SHIFT 12 +#define FSL_RE_HWDESC_FMT_SHIFT 29 +#define FSL_RE_HWDESC_FMT_MASK (0x3 << FSL_RE_HWDESC_FMT_SHIFT) + +struct fsl_re_hw_desc { + __be32 lbea32; + __be32 addr_low; + __be32 fmt32; + __be32 status; +}; + +/* Raid Engine device private data */ +struct fsl_re_drv_private { + u8 total_chans; + struct dma_device dma_dev; + struct fsl_re_ctrl *re_regs; + struct fsl_re_chan *re_jrs[FSL_RE_MAX_CHANS]; + struct dma_pool *cf_desc_pool; + struct dma_pool *hw_desc_pool; +}; + +/* Per job ring data structure */ +struct fsl_re_chan { + char name[16]; + spinlock_t desc_lock; /* queue lock */ + struct list_head ack_q; /* wait to acked queue */ + struct list_head active_q; /* already issued on hw, not completed */ + struct list_head submit_q; + struct list_head free_q; /* alloc available queue */ + struct device *dev; + struct fsl_re_drv_private *re_dev; + struct dma_chan chan; + struct fsl_re_chan_cfg *jrregs; + int irq; + struct tasklet_struct irqtask; + u32 alloc_count; + + /* hw descriptor ring for inbound queue*/ + dma_addr_t inb_phys_addr; + struct fsl_re_hw_desc *inb_ring_virt_addr; + u32 inb_count; + + /* hw descriptor ring for outbound queue */ + dma_addr_t oub_phys_addr; + struct fsl_re_hw_desc *oub_ring_virt_addr; + u32 oub_count; +}; + +/* Async transaction descriptor */ +struct fsl_re_desc { + struct dma_async_tx_descriptor async_tx; + struct list_head node; + struct fsl_re_hw_desc hwdesc; + struct fsl_re_chan *re_chan; + + /* hwdesc will point to cf_addr */ + void *cf_addr; + dma_addr_t cf_paddr; + + void *cdb_addr; + dma_addr_t cdb_paddr; + int status; +}; diff --git a/drivers/dma/img-mdc-dma.c b/drivers/dma/img-mdc-dma.c index ed045a9ad634..9ca56830cc63 100644 --- a/drivers/dma/img-mdc-dma.c +++ b/drivers/dma/img-mdc-dma.c @@ -689,11 +689,6 @@ static int mdc_slave_config(struct dma_chan *chan, return 0; } -static int mdc_alloc_chan_resources(struct dma_chan *chan) -{ - return 0; -} - static void mdc_free_chan_resources(struct dma_chan *chan) { struct mdc_chan *mchan = to_mdc_chan(chan); @@ -910,7 +905,6 @@ static int mdc_dma_probe(struct platform_device *pdev) mdma->dma_dev.device_prep_slave_sg = mdc_prep_slave_sg; mdma->dma_dev.device_prep_dma_cyclic = mdc_prep_dma_cyclic; mdma->dma_dev.device_prep_dma_memcpy = mdc_prep_dma_memcpy; - mdma->dma_dev.device_alloc_chan_resources = mdc_alloc_chan_resources; mdma->dma_dev.device_free_chan_resources = mdc_free_chan_resources; mdma->dma_dev.device_tx_status = mdc_tx_status; mdma->dma_dev.device_issue_pending = mdc_issue_pending; diff --git a/drivers/dma/imx-sdma.c b/drivers/dma/imx-sdma.c index 66a0efb9651d..62bbd79338e0 100644 --- a/drivers/dma/imx-sdma.c +++ b/drivers/dma/imx-sdma.c @@ -1260,6 +1260,7 @@ static void sdma_issue_pending(struct dma_chan *chan) #define SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V1 34 #define SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V2 38 +#define SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V3 41 static void sdma_add_scripts(struct sdma_engine *sdma, const struct sdma_script_start_addrs *addr) @@ -1306,6 +1307,9 @@ static void sdma_load_firmware(const struct firmware *fw, void *context) case 2: sdma->script_number = SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V2; break; + case 3: + sdma->script_number = SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V3; + break; default: dev_err(sdma->dev, "unknown firmware version\n"); goto err_firmware; diff --git a/drivers/dma/ioat/dca.c b/drivers/dma/ioat/dca.c index 3b55bb8d969a..ea1e107ae884 100644 --- a/drivers/dma/ioat/dca.c +++ b/drivers/dma/ioat/dca.c @@ -11,10 +11,6 @@ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for * more details. * - * You should have received a copy of the GNU General Public License along with - * this program; if not, write to the Free Software Foundation, Inc., - * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. - * * The full GNU General Public License is included in this distribution in * the file called "COPYING". * diff --git a/drivers/dma/ioat/dma.c b/drivers/dma/ioat/dma.c index 940c1502a8b5..ee0aa9f4ccfa 100644 --- a/drivers/dma/ioat/dma.c +++ b/drivers/dma/ioat/dma.c @@ -11,10 +11,6 @@ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for * more details. * - * You should have received a copy of the GNU General Public License along with - * this program; if not, write to the Free Software Foundation, Inc., - * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. - * * The full GNU General Public License is included in this distribution in * the file called "COPYING". * diff --git a/drivers/dma/ioat/dma.h b/drivers/dma/ioat/dma.h index d63f68b1aa35..30f5c7eede16 100644 --- a/drivers/dma/ioat/dma.h +++ b/drivers/dma/ioat/dma.h @@ -11,10 +11,6 @@ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for * more details. * - * You should have received a copy of the GNU General Public License along with - * this program; if not, write to the Free Software Foundation, Inc., 59 - * Temple Place - Suite 330, Boston, MA 02111-1307, USA. - * * The full GNU General Public License is included in this distribution in the * file called COPYING. */ diff --git a/drivers/dma/ioat/dma_v2.c b/drivers/dma/ioat/dma_v2.c index 695483e6be32..69c7dfcad023 100644 --- a/drivers/dma/ioat/dma_v2.c +++ b/drivers/dma/ioat/dma_v2.c @@ -11,10 +11,6 @@ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for * more details. * - * You should have received a copy of the GNU General Public License along with - * this program; if not, write to the Free Software Foundation, Inc., - * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. - * * The full GNU General Public License is included in this distribution in * the file called "COPYING". * diff --git a/drivers/dma/ioat/dma_v2.h b/drivers/dma/ioat/dma_v2.h index 470292767e68..bf24ebe874b0 100644 --- a/drivers/dma/ioat/dma_v2.h +++ b/drivers/dma/ioat/dma_v2.h @@ -11,10 +11,6 @@ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for * more details. * - * You should have received a copy of the GNU General Public License along with - * this program; if not, write to the Free Software Foundation, Inc., 59 - * Temple Place - Suite 330, Boston, MA 02111-1307, USA. - * * The full GNU General Public License is included in this distribution in the * file called COPYING. */ diff --git a/drivers/dma/ioat/dma_v3.c b/drivers/dma/ioat/dma_v3.c index 194ec20c9408..64790a45ef5d 100644 --- a/drivers/dma/ioat/dma_v3.c +++ b/drivers/dma/ioat/dma_v3.c @@ -15,10 +15,6 @@ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for * more details. * - * You should have received a copy of the GNU General Public License along with - * this program; if not, write to the Free Software Foundation, Inc., - * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. - * * The full GNU General Public License is included in this distribution in * the file called "COPYING". * diff --git a/drivers/dma/ioat/hw.h b/drivers/dma/ioat/hw.h index 02177ecf09f8..a3e731edce57 100644 --- a/drivers/dma/ioat/hw.h +++ b/drivers/dma/ioat/hw.h @@ -11,10 +11,6 @@ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for * more details. * - * You should have received a copy of the GNU General Public License along with - * this program; if not, write to the Free Software Foundation, Inc., 59 - * Temple Place - Suite 330, Boston, MA 02111-1307, USA. - * * The full GNU General Public License is included in this distribution in the * file called COPYING. */ diff --git a/drivers/dma/ioat/pci.c b/drivers/dma/ioat/pci.c index 5501eb072d69..76f0dc688a19 100644 --- a/drivers/dma/ioat/pci.c +++ b/drivers/dma/ioat/pci.c @@ -11,10 +11,6 @@ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for * more details. * - * You should have received a copy of the GNU General Public License along with - * this program; if not, write to the Free Software Foundation, Inc., - * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. - * * The full GNU General Public License is included in this distribution in * the file called "COPYING". * diff --git a/drivers/dma/ioat/registers.h b/drivers/dma/ioat/registers.h index 2f1cfa0f1f47..909352f74c89 100644 --- a/drivers/dma/ioat/registers.h +++ b/drivers/dma/ioat/registers.h @@ -11,10 +11,6 @@ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for * more details. * - * You should have received a copy of the GNU General Public License along with - * this program; if not, write to the Free Software Foundation, Inc., 59 - * Temple Place - Suite 330, Boston, MA 02111-1307, USA. - * * The full GNU General Public License is included in this distribution in the * file called COPYING. */ diff --git a/drivers/dma/iop-adma.c b/drivers/dma/iop-adma.c index 263d9f6a207e..998826854fdd 100644 --- a/drivers/dma/iop-adma.c +++ b/drivers/dma/iop-adma.c @@ -11,10 +11,6 @@ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for * more details. * - * You should have received a copy of the GNU General Public License along with - * this program; if not, write to the Free Software Foundation, Inc., - * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. - * */ /* diff --git a/drivers/dma/k3dma.c b/drivers/dma/k3dma.c index 6f7f43529ccb..647e362f01fd 100644 --- a/drivers/dma/k3dma.c +++ b/drivers/dma/k3dma.c @@ -313,11 +313,6 @@ static void k3_dma_tasklet(unsigned long arg) } } -static int k3_dma_alloc_chan_resources(struct dma_chan *chan) -{ - return 0; -} - static void k3_dma_free_chan_resources(struct dma_chan *chan) { struct k3_dma_chan *c = to_k3_chan(chan); @@ -654,7 +649,7 @@ static void k3_dma_free_desc(struct virt_dma_desc *vd) kfree(ds); } -static struct of_device_id k3_pdma_dt_ids[] = { +static const struct of_device_id k3_pdma_dt_ids[] = { { .compatible = "hisilicon,k3-dma-1.0", }, {} }; @@ -728,7 +723,6 @@ static int k3_dma_probe(struct platform_device *op) dma_cap_set(DMA_SLAVE, d->slave.cap_mask); dma_cap_set(DMA_MEMCPY, d->slave.cap_mask); d->slave.dev = &op->dev; - d->slave.device_alloc_chan_resources = k3_dma_alloc_chan_resources; d->slave.device_free_chan_resources = k3_dma_free_chan_resources; d->slave.device_tx_status = k3_dma_tx_status; d->slave.device_prep_dma_memcpy = k3_dma_prep_memcpy; diff --git a/drivers/dma/mmp_pdma.c b/drivers/dma/mmp_pdma.c index eb410044e1af..462a0229a743 100644 --- a/drivers/dma/mmp_pdma.c +++ b/drivers/dma/mmp_pdma.c @@ -973,7 +973,7 @@ static int mmp_pdma_chan_init(struct mmp_pdma_device *pdev, int idx, int irq) return 0; } -static struct of_device_id mmp_pdma_dt_ids[] = { +static const struct of_device_id mmp_pdma_dt_ids[] = { { .compatible = "marvell,pdma-1.0", }, {} }; diff --git a/drivers/dma/mmp_tdma.c b/drivers/dma/mmp_tdma.c index b6f4e1fc9c78..449e785def17 100644 --- a/drivers/dma/mmp_tdma.c +++ b/drivers/dma/mmp_tdma.c @@ -613,7 +613,7 @@ struct dma_chan *mmp_tdma_xlate(struct of_phandle_args *dma_spec, return dma_request_channel(mask, mmp_tdma_filter_fn, ¶m); } -static struct of_device_id mmp_tdma_dt_ids[] = { +static const struct of_device_id mmp_tdma_dt_ids[] = { { .compatible = "marvell,adma-1.0", .data = (void *)MMP_AUD_TDMA}, { .compatible = "marvell,pxa910-squ", .data = (void *)PXA910_SQU}, {} diff --git a/drivers/dma/mpc512x_dma.c b/drivers/dma/mpc512x_dma.c index 57d2457545f3..e6281e7aa46e 100644 --- a/drivers/dma/mpc512x_dma.c +++ b/drivers/dma/mpc512x_dma.c @@ -21,10 +21,6 @@ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for * more details. * - * You should have received a copy of the GNU General Public License along with - * this program; if not, write to the Free Software Foundation, Inc., 59 - * Temple Place - Suite 330, Boston, MA 02111-1307, USA. - * * The full GNU General Public License is included in this distribution in the * file called COPYING. */ @@ -1072,7 +1068,7 @@ static int mpc_dma_remove(struct platform_device *op) return 0; } -static struct of_device_id mpc_dma_match[] = { +static const struct of_device_id mpc_dma_match[] = { { .compatible = "fsl,mpc5121-dma", }, { .compatible = "fsl,mpc8308-dma", }, {}, diff --git a/drivers/dma/mv_xor.c b/drivers/dma/mv_xor.c index b03e8137b918..1c56001df676 100644 --- a/drivers/dma/mv_xor.c +++ b/drivers/dma/mv_xor.c @@ -10,10 +10,6 @@ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for * more details. - * - * You should have received a copy of the GNU General Public License along with - * this program; if not, write to the Free Software Foundation, Inc., - * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. */ #include <linux/init.h> @@ -1249,7 +1245,7 @@ static int mv_xor_remove(struct platform_device *pdev) } #ifdef CONFIG_OF -static struct of_device_id mv_xor_dt_ids[] = { +static const struct of_device_id mv_xor_dt_ids[] = { { .compatible = "marvell,orion-xor", }, {}, }; diff --git a/drivers/dma/mv_xor.h b/drivers/dma/mv_xor.h index 78edc7e44569..91958dba39a2 100644 --- a/drivers/dma/mv_xor.h +++ b/drivers/dma/mv_xor.h @@ -9,10 +9,6 @@ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. */ #ifndef MV_XOR_H diff --git a/drivers/dma/pch_dma.c b/drivers/dma/pch_dma.c index 35c143cb88da..b859792dde95 100644 --- a/drivers/dma/pch_dma.c +++ b/drivers/dma/pch_dma.c @@ -949,6 +949,7 @@ err_free_res: err_disable_pdev: pci_disable_device(pdev); err_free_mem: + kfree(pd); return err; } diff --git a/drivers/dma/pl330.c b/drivers/dma/pl330.c index 0e1f56772855..a7d9d3029b14 100644 --- a/drivers/dma/pl330.c +++ b/drivers/dma/pl330.c @@ -556,7 +556,7 @@ static inline u32 _emit_ADDH(unsigned dry_run, u8 buf[], buf[0] = CMD_DMAADDH; buf[0] |= (da << 1); - *((u16 *)&buf[1]) = val; + *((__le16 *)&buf[1]) = cpu_to_le16(val); PL330_DBGCMD_DUMP(SZ_DMAADDH, "\tDMAADDH %s %u\n", da == 1 ? "DA" : "SA", val); @@ -710,7 +710,7 @@ static inline u32 _emit_MOV(unsigned dry_run, u8 buf[], buf[0] = CMD_DMAMOV; buf[1] = dst; - *((u32 *)&buf[2]) = val; + *((__le32 *)&buf[2]) = cpu_to_le32(val); PL330_DBGCMD_DUMP(SZ_DMAMOV, "\tDMAMOV %s 0x%x\n", dst == SAR ? "SAR" : (dst == DAR ? "DAR" : "CCR"), val); @@ -888,7 +888,7 @@ static inline u32 _emit_GO(unsigned dry_run, u8 buf[], buf[1] = chan & 0x7; - *((u32 *)&buf[2]) = addr; + *((__le32 *)&buf[2]) = cpu_to_le32(addr); return SZ_DMAGO; } @@ -928,7 +928,7 @@ static inline void _execute_DBGINSN(struct pl330_thread *thrd, } writel(val, regs + DBGINST0); - val = *((u32 *)&insn[2]); + val = le32_to_cpu(*((__le32 *)&insn[2])); writel(val, regs + DBGINST1); /* If timed out due to halted state-machine */ @@ -2162,7 +2162,7 @@ static int pl330_terminate_all(struct dma_chan *chan) * DMA transfer again. This pause feature was implemented to * allow safely read residue before channel termination. */ -int pl330_pause(struct dma_chan *chan) +static int pl330_pause(struct dma_chan *chan) { struct dma_pl330_chan *pch = to_pchan(chan); struct pl330_dmac *pl330 = pch->dmac; @@ -2203,8 +2203,8 @@ static void pl330_free_chan_resources(struct dma_chan *chan) pm_runtime_put_autosuspend(pch->dmac->ddma.dev); } -int pl330_get_current_xferred_count(struct dma_pl330_chan *pch, - struct dma_pl330_desc *desc) +static int pl330_get_current_xferred_count(struct dma_pl330_chan *pch, + struct dma_pl330_desc *desc) { struct pl330_thread *thrd = pch->thread; struct pl330_dmac *pl330 = pch->dmac; @@ -2259,7 +2259,17 @@ pl330_tx_status(struct dma_chan *chan, dma_cookie_t cookie, transferred = 0; residual += desc->bytes_requested - transferred; if (desc->txd.cookie == cookie) { - ret = desc->status; + switch (desc->status) { + case DONE: + ret = DMA_COMPLETE; + break; + case PREP: + case BUSY: + ret = DMA_IN_PROGRESS; + break; + default: + WARN_ON(1); + } break; } if (desc->last) diff --git a/drivers/dma/ppc4xx/adma.c b/drivers/dma/ppc4xx/adma.c index fa764a39cd36..9217f893b0d1 100644 --- a/drivers/dma/ppc4xx/adma.c +++ b/drivers/dma/ppc4xx/adma.c @@ -16,10 +16,6 @@ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for * more details. * - * You should have received a copy of the GNU General Public License along with - * this program; if not, write to the Free Software Foundation, Inc., 59 - * Temple Place - Suite 330, Boston, MA 02111-1307, USA. - * * The full GNU General Public License is included in this distribution in the * file called COPYING. */ diff --git a/drivers/dma/qcom_bam_dma.c b/drivers/dma/qcom_bam_dma.c index 9c914d625906..5a250cdc8376 100644 --- a/drivers/dma/qcom_bam_dma.c +++ b/drivers/dma/qcom_bam_dma.c @@ -171,6 +171,35 @@ static const struct reg_offset_data bam_v1_4_reg_info[] = { [BAM_P_FIFO_SIZES] = { 0x1820, 0x00, 0x1000, 0x00 }, }; +static const struct reg_offset_data bam_v1_7_reg_info[] = { + [BAM_CTRL] = { 0x00000, 0x00, 0x00, 0x00 }, + [BAM_REVISION] = { 0x01000, 0x00, 0x00, 0x00 }, + [BAM_NUM_PIPES] = { 0x01008, 0x00, 0x00, 0x00 }, + [BAM_DESC_CNT_TRSHLD] = { 0x00008, 0x00, 0x00, 0x00 }, + [BAM_IRQ_SRCS] = { 0x03010, 0x00, 0x00, 0x00 }, + [BAM_IRQ_SRCS_MSK] = { 0x03014, 0x00, 0x00, 0x00 }, + [BAM_IRQ_SRCS_UNMASKED] = { 0x03018, 0x00, 0x00, 0x00 }, + [BAM_IRQ_STTS] = { 0x00014, 0x00, 0x00, 0x00 }, + [BAM_IRQ_CLR] = { 0x00018, 0x00, 0x00, 0x00 }, + [BAM_IRQ_EN] = { 0x0001C, 0x00, 0x00, 0x00 }, + [BAM_CNFG_BITS] = { 0x0007C, 0x00, 0x00, 0x00 }, + [BAM_IRQ_SRCS_EE] = { 0x03000, 0x00, 0x00, 0x1000 }, + [BAM_IRQ_SRCS_MSK_EE] = { 0x03004, 0x00, 0x00, 0x1000 }, + [BAM_P_CTRL] = { 0x13000, 0x1000, 0x00, 0x00 }, + [BAM_P_RST] = { 0x13004, 0x1000, 0x00, 0x00 }, + [BAM_P_HALT] = { 0x13008, 0x1000, 0x00, 0x00 }, + [BAM_P_IRQ_STTS] = { 0x13010, 0x1000, 0x00, 0x00 }, + [BAM_P_IRQ_CLR] = { 0x13014, 0x1000, 0x00, 0x00 }, + [BAM_P_IRQ_EN] = { 0x13018, 0x1000, 0x00, 0x00 }, + [BAM_P_EVNT_DEST_ADDR] = { 0x1382C, 0x00, 0x1000, 0x00 }, + [BAM_P_EVNT_REG] = { 0x13818, 0x00, 0x1000, 0x00 }, + [BAM_P_SW_OFSTS] = { 0x13800, 0x00, 0x1000, 0x00 }, + [BAM_P_DATA_FIFO_ADDR] = { 0x13824, 0x00, 0x1000, 0x00 }, + [BAM_P_DESC_FIFO_ADDR] = { 0x1381C, 0x00, 0x1000, 0x00 }, + [BAM_P_EVNT_GEN_TRSHLD] = { 0x13828, 0x00, 0x1000, 0x00 }, + [BAM_P_FIFO_SIZES] = { 0x13820, 0x00, 0x1000, 0x00 }, +}; + /* BAM CTRL */ #define BAM_SW_RST BIT(0) #define BAM_EN BIT(1) @@ -1051,6 +1080,7 @@ static void bam_channel_init(struct bam_device *bdev, struct bam_chan *bchan, static const struct of_device_id bam_of_match[] = { { .compatible = "qcom,bam-v1.3.0", .data = &bam_v1_3_reg_info }, { .compatible = "qcom,bam-v1.4.0", .data = &bam_v1_4_reg_info }, + { .compatible = "qcom,bam-v1.7.0", .data = &bam_v1_7_reg_info }, {} }; @@ -1113,7 +1143,7 @@ static int bam_dma_probe(struct platform_device *pdev) if (!bdev->channels) { ret = -ENOMEM; - goto err_disable_clk; + goto err_tasklet_kill; } /* allocate and initialize channels */ @@ -1125,7 +1155,7 @@ static int bam_dma_probe(struct platform_device *pdev) ret = devm_request_irq(bdev->dev, bdev->irq, bam_dma_irq, IRQF_TRIGGER_HIGH, "bam_dma", bdev); if (ret) - goto err_disable_clk; + goto err_bam_channel_exit; /* set max dma segment size */ bdev->common.dev = bdev->dev; @@ -1133,7 +1163,7 @@ static int bam_dma_probe(struct platform_device *pdev) ret = dma_set_max_seg_size(bdev->common.dev, BAM_MAX_DATA_SIZE); if (ret) { dev_err(bdev->dev, "cannot set maximum segment size\n"); - goto err_disable_clk; + goto err_bam_channel_exit; } platform_set_drvdata(pdev, bdev); @@ -1161,7 +1191,7 @@ static int bam_dma_probe(struct platform_device *pdev) ret = dma_async_device_register(&bdev->common); if (ret) { dev_err(bdev->dev, "failed to register dma async device\n"); - goto err_disable_clk; + goto err_bam_channel_exit; } ret = of_dma_controller_register(pdev->dev.of_node, bam_dma_xlate, @@ -1173,8 +1203,14 @@ static int bam_dma_probe(struct platform_device *pdev) err_unregister_dma: dma_async_device_unregister(&bdev->common); +err_bam_channel_exit: + for (i = 0; i < bdev->num_channels; i++) + tasklet_kill(&bdev->channels[i].vc.task); +err_tasklet_kill: + tasklet_kill(&bdev->task); err_disable_clk: clk_disable_unprepare(bdev->bamclk); + return ret; } diff --git a/drivers/dma/s3c24xx-dma.c b/drivers/dma/s3c24xx-dma.c index 2f91da3db836..01dcaf21b988 100644 --- a/drivers/dma/s3c24xx-dma.c +++ b/drivers/dma/s3c24xx-dma.c @@ -749,11 +749,6 @@ unlock: return ret; } -static int s3c24xx_dma_alloc_chan_resources(struct dma_chan *chan) -{ - return 0; -} - static void s3c24xx_dma_free_chan_resources(struct dma_chan *chan) { /* Ensure all queued descriptors are freed */ @@ -1238,7 +1233,7 @@ static int s3c24xx_dma_probe(struct platform_device *pdev) if (!s3cdma->phy_chans) return -ENOMEM; - /* aquire irqs and clocks for all physical channels */ + /* acquire irqs and clocks for all physical channels */ for (i = 0; i < pdata->num_phy_channels; i++) { struct s3c24xx_dma_phy *phy = &s3cdma->phy_chans[i]; char clk_name[6]; @@ -1266,7 +1261,7 @@ static int s3c24xx_dma_probe(struct platform_device *pdev) sprintf(clk_name, "dma.%d", i); phy->clk = devm_clk_get(&pdev->dev, clk_name); if (IS_ERR(phy->clk) && sdata->has_clocks) { - dev_err(&pdev->dev, "unable to aquire clock for channel %d, error %lu", + dev_err(&pdev->dev, "unable to acquire clock for channel %d, error %lu\n", i, PTR_ERR(phy->clk)); continue; } @@ -1290,8 +1285,6 @@ static int s3c24xx_dma_probe(struct platform_device *pdev) dma_cap_set(DMA_MEMCPY, s3cdma->memcpy.cap_mask); dma_cap_set(DMA_PRIVATE, s3cdma->memcpy.cap_mask); s3cdma->memcpy.dev = &pdev->dev; - s3cdma->memcpy.device_alloc_chan_resources = - s3c24xx_dma_alloc_chan_resources; s3cdma->memcpy.device_free_chan_resources = s3c24xx_dma_free_chan_resources; s3cdma->memcpy.device_prep_dma_memcpy = s3c24xx_dma_prep_memcpy; @@ -1305,8 +1298,6 @@ static int s3c24xx_dma_probe(struct platform_device *pdev) dma_cap_set(DMA_CYCLIC, s3cdma->slave.cap_mask); dma_cap_set(DMA_PRIVATE, s3cdma->slave.cap_mask); s3cdma->slave.dev = &pdev->dev; - s3cdma->slave.device_alloc_chan_resources = - s3c24xx_dma_alloc_chan_resources; s3cdma->slave.device_free_chan_resources = s3c24xx_dma_free_chan_resources; s3cdma->slave.device_tx_status = s3c24xx_dma_tx_status; diff --git a/drivers/dma/sa11x0-dma.c b/drivers/dma/sa11x0-dma.c index 5adf5407a8cb..43db255050d2 100644 --- a/drivers/dma/sa11x0-dma.c +++ b/drivers/dma/sa11x0-dma.c @@ -389,11 +389,6 @@ static void sa11x0_dma_tasklet(unsigned long arg) } -static int sa11x0_dma_alloc_chan_resources(struct dma_chan *chan) -{ - return 0; -} - static void sa11x0_dma_free_chan_resources(struct dma_chan *chan) { struct sa11x0_dma_chan *c = to_sa11x0_dma_chan(chan); @@ -835,7 +830,6 @@ static int sa11x0_dma_init_dmadev(struct dma_device *dmadev, INIT_LIST_HEAD(&dmadev->channels); dmadev->dev = dev; - dmadev->device_alloc_chan_resources = sa11x0_dma_alloc_chan_resources; dmadev->device_free_chan_resources = sa11x0_dma_free_chan_resources; dmadev->device_config = sa11x0_dma_device_config; dmadev->device_pause = sa11x0_dma_device_pause; @@ -948,6 +942,12 @@ static int sa11x0_dma_probe(struct platform_device *pdev) dma_cap_set(DMA_CYCLIC, d->slave.cap_mask); d->slave.device_prep_slave_sg = sa11x0_dma_prep_slave_sg; d->slave.device_prep_dma_cyclic = sa11x0_dma_prep_dma_cyclic; + d->slave.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV); + d->slave.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST; + d->slave.src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | + BIT(DMA_SLAVE_BUSWIDTH_2_BYTES); + d->slave.dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | + BIT(DMA_SLAVE_BUSWIDTH_2_BYTES); ret = sa11x0_dma_init_dmadev(&d->slave, &pdev->dev); if (ret) { dev_warn(d->slave.dev, "failed to register slave async device: %d\n", diff --git a/drivers/dma/sh/Kconfig b/drivers/dma/sh/Kconfig index 8190ad225a1b..0f371524a4d9 100644 --- a/drivers/dma/sh/Kconfig +++ b/drivers/dma/sh/Kconfig @@ -51,12 +51,6 @@ config RCAR_HPB_DMAE help Enable support for the Renesas R-Car series DMA controllers. -config RCAR_AUDMAC_PP - tristate "Renesas R-Car Audio DMAC Peripheral Peripheral support" - depends on SH_DMAE_BASE - help - Enable support for the Renesas R-Car Audio DMAC Peripheral Peripheral controllers. - config RCAR_DMAC tristate "Renesas R-Car Gen2 DMA Controller" depends on ARCH_SHMOBILE || COMPILE_TEST @@ -64,3 +58,12 @@ config RCAR_DMAC help This driver supports the general purpose DMA controller found in the Renesas R-Car second generation SoCs. + +config RENESAS_USB_DMAC + tristate "Renesas USB-DMA Controller" + depends on ARCH_SHMOBILE || COMPILE_TEST + select RENESAS_DMA + select DMA_VIRTUAL_CHANNELS + help + This driver supports the USB-DMA controller found in the Renesas + SoCs. diff --git a/drivers/dma/sh/Makefile b/drivers/dma/sh/Makefile index 2852f9db61a4..b8a598066ce2 100644 --- a/drivers/dma/sh/Makefile +++ b/drivers/dma/sh/Makefile @@ -15,5 +15,5 @@ obj-$(CONFIG_SH_DMAE) += shdma.o obj-$(CONFIG_SUDMAC) += sudmac.o obj-$(CONFIG_RCAR_HPB_DMAE) += rcar-hpbdma.o -obj-$(CONFIG_RCAR_AUDMAC_PP) += rcar-audmapp.o obj-$(CONFIG_RCAR_DMAC) += rcar-dmac.o +obj-$(CONFIG_RENESAS_USB_DMAC) += usb-dmac.o diff --git a/drivers/dma/sh/rcar-audmapp.c b/drivers/dma/sh/rcar-audmapp.c deleted file mode 100644 index d95bbdd721f4..000000000000 --- a/drivers/dma/sh/rcar-audmapp.c +++ /dev/null @@ -1,376 +0,0 @@ -/* - * This is for Renesas R-Car Audio-DMAC-peri-peri. - * - * Copyright (C) 2014 Renesas Electronics Corporation - * Copyright (C) 2014 Kuninori Morimoto <kuninori.morimoto.gx@renesas.com> - * - * based on the drivers/dma/sh/shdma.c - * - * Copyright (C) 2011-2012 Guennadi Liakhovetski <g.liakhovetski@gmx.de> - * Copyright (C) 2009 Nobuhiro Iwamatsu <iwamatsu.nobuhiro@renesas.com> - * Copyright (C) 2009 Renesas Solutions, Inc. All rights reserved. - * Copyright (C) 2007 Freescale Semiconductor, Inc. All rights reserved. - * - * This is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - */ -#include <linux/delay.h> -#include <linux/init.h> -#include <linux/module.h> -#include <linux/slab.h> -#include <linux/dmaengine.h> -#include <linux/of_dma.h> -#include <linux/platform_data/dma-rcar-audmapp.h> -#include <linux/platform_device.h> -#include <linux/shdma-base.h> - -/* - * DMA register - */ -#define PDMASAR 0x00 -#define PDMADAR 0x04 -#define PDMACHCR 0x0c - -/* PDMACHCR */ -#define PDMACHCR_DE (1 << 0) - -#define AUDMAPP_MAX_CHANNELS 29 - -/* Default MEMCPY transfer size = 2^2 = 4 bytes */ -#define LOG2_DEFAULT_XFER_SIZE 2 -#define AUDMAPP_SLAVE_NUMBER 256 -#define AUDMAPP_LEN_MAX (16 * 1024 * 1024) - -struct audmapp_chan { - struct shdma_chan shdma_chan; - void __iomem *base; - dma_addr_t slave_addr; - u32 chcr; -}; - -struct audmapp_device { - struct shdma_dev shdma_dev; - struct audmapp_pdata *pdata; - struct device *dev; - void __iomem *chan_reg; -}; - -struct audmapp_desc { - struct shdma_desc shdma_desc; - dma_addr_t src; - dma_addr_t dst; -}; - -#define to_shdma_chan(c) container_of(c, struct shdma_chan, dma_chan) - -#define to_chan(chan) container_of(chan, struct audmapp_chan, shdma_chan) -#define to_desc(sdesc) container_of(sdesc, struct audmapp_desc, shdma_desc) -#define to_dev(chan) container_of(chan->shdma_chan.dma_chan.device, \ - struct audmapp_device, shdma_dev.dma_dev) - -static void audmapp_write(struct audmapp_chan *auchan, u32 data, u32 reg) -{ - struct audmapp_device *audev = to_dev(auchan); - struct device *dev = audev->dev; - - dev_dbg(dev, "w %p : %08x\n", auchan->base + reg, data); - - iowrite32(data, auchan->base + reg); -} - -static u32 audmapp_read(struct audmapp_chan *auchan, u32 reg) -{ - return ioread32(auchan->base + reg); -} - -static void audmapp_halt(struct shdma_chan *schan) -{ - struct audmapp_chan *auchan = to_chan(schan); - int i; - - audmapp_write(auchan, 0, PDMACHCR); - - for (i = 0; i < 1024; i++) { - if (0 == audmapp_read(auchan, PDMACHCR)) - return; - udelay(1); - } -} - -static void audmapp_start_xfer(struct shdma_chan *schan, - struct shdma_desc *sdesc) -{ - struct audmapp_chan *auchan = to_chan(schan); - struct audmapp_device *audev = to_dev(auchan); - struct audmapp_desc *desc = to_desc(sdesc); - struct device *dev = audev->dev; - u32 chcr = auchan->chcr | PDMACHCR_DE; - - dev_dbg(dev, "src/dst/chcr = %pad/%pad/%08x\n", - &desc->src, &desc->dst, chcr); - - audmapp_write(auchan, desc->src, PDMASAR); - audmapp_write(auchan, desc->dst, PDMADAR); - audmapp_write(auchan, chcr, PDMACHCR); -} - -static int audmapp_get_config(struct audmapp_chan *auchan, int slave_id, - u32 *chcr, dma_addr_t *dst) -{ - struct audmapp_device *audev = to_dev(auchan); - struct audmapp_pdata *pdata = audev->pdata; - struct audmapp_slave_config *cfg; - int i; - - *chcr = 0; - *dst = 0; - - if (!pdata) { /* DT */ - *chcr = ((u32)slave_id) << 16; - auchan->shdma_chan.slave_id = (slave_id) >> 8; - return 0; - } - - /* non-DT */ - - if (slave_id >= AUDMAPP_SLAVE_NUMBER) - return -ENXIO; - - for (i = 0, cfg = pdata->slave; i < pdata->slave_num; i++, cfg++) - if (cfg->slave_id == slave_id) { - *chcr = cfg->chcr; - *dst = cfg->dst; - return 0; - } - - return -ENXIO; -} - -static int audmapp_set_slave(struct shdma_chan *schan, int slave_id, - dma_addr_t slave_addr, bool try) -{ - struct audmapp_chan *auchan = to_chan(schan); - u32 chcr; - dma_addr_t dst; - int ret; - - ret = audmapp_get_config(auchan, slave_id, &chcr, &dst); - if (ret < 0) - return ret; - - if (try) - return 0; - - auchan->chcr = chcr; - auchan->slave_addr = slave_addr ? : dst; - - return 0; -} - -static int audmapp_desc_setup(struct shdma_chan *schan, - struct shdma_desc *sdesc, - dma_addr_t src, dma_addr_t dst, size_t *len) -{ - struct audmapp_desc *desc = to_desc(sdesc); - - if (*len > (size_t)AUDMAPP_LEN_MAX) - *len = (size_t)AUDMAPP_LEN_MAX; - - desc->src = src; - desc->dst = dst; - - return 0; -} - -static void audmapp_setup_xfer(struct shdma_chan *schan, - int slave_id) -{ -} - -static dma_addr_t audmapp_slave_addr(struct shdma_chan *schan) -{ - struct audmapp_chan *auchan = to_chan(schan); - - return auchan->slave_addr; -} - -static bool audmapp_channel_busy(struct shdma_chan *schan) -{ - struct audmapp_chan *auchan = to_chan(schan); - u32 chcr = audmapp_read(auchan, PDMACHCR); - - return chcr & ~PDMACHCR_DE; -} - -static bool audmapp_desc_completed(struct shdma_chan *schan, - struct shdma_desc *sdesc) -{ - return true; -} - -static struct shdma_desc *audmapp_embedded_desc(void *buf, int i) -{ - return &((struct audmapp_desc *)buf)[i].shdma_desc; -} - -static const struct shdma_ops audmapp_shdma_ops = { - .halt_channel = audmapp_halt, - .desc_setup = audmapp_desc_setup, - .set_slave = audmapp_set_slave, - .start_xfer = audmapp_start_xfer, - .embedded_desc = audmapp_embedded_desc, - .setup_xfer = audmapp_setup_xfer, - .slave_addr = audmapp_slave_addr, - .channel_busy = audmapp_channel_busy, - .desc_completed = audmapp_desc_completed, -}; - -static int audmapp_chan_probe(struct platform_device *pdev, - struct audmapp_device *audev, int id) -{ - struct shdma_dev *sdev = &audev->shdma_dev; - struct audmapp_chan *auchan; - struct shdma_chan *schan; - struct device *dev = audev->dev; - - auchan = devm_kzalloc(dev, sizeof(*auchan), GFP_KERNEL); - if (!auchan) - return -ENOMEM; - - schan = &auchan->shdma_chan; - schan->max_xfer_len = AUDMAPP_LEN_MAX; - - shdma_chan_probe(sdev, schan, id); - - auchan->base = audev->chan_reg + 0x20 + (0x10 * id); - dev_dbg(dev, "%02d : %p / %p", id, auchan->base, audev->chan_reg); - - return 0; -} - -static void audmapp_chan_remove(struct audmapp_device *audev) -{ - struct shdma_chan *schan; - int i; - - shdma_for_each_chan(schan, &audev->shdma_dev, i) { - BUG_ON(!schan); - shdma_chan_remove(schan); - } -} - -static struct dma_chan *audmapp_of_xlate(struct of_phandle_args *dma_spec, - struct of_dma *ofdma) -{ - dma_cap_mask_t mask; - struct dma_chan *chan; - u32 chcr = dma_spec->args[0]; - - if (dma_spec->args_count != 1) - return NULL; - - dma_cap_zero(mask); - dma_cap_set(DMA_SLAVE, mask); - - chan = dma_request_channel(mask, shdma_chan_filter, NULL); - if (chan) - to_shdma_chan(chan)->hw_req = chcr; - - return chan; -} - -static int audmapp_probe(struct platform_device *pdev) -{ - struct audmapp_pdata *pdata = pdev->dev.platform_data; - struct device_node *np = pdev->dev.of_node; - struct audmapp_device *audev; - struct shdma_dev *sdev; - struct dma_device *dma_dev; - struct resource *res; - int err, i; - - if (np) - of_dma_controller_register(np, audmapp_of_xlate, pdev); - else if (!pdata) - return -ENODEV; - - res = platform_get_resource(pdev, IORESOURCE_MEM, 0); - - audev = devm_kzalloc(&pdev->dev, sizeof(*audev), GFP_KERNEL); - if (!audev) - return -ENOMEM; - - audev->dev = &pdev->dev; - audev->pdata = pdata; - audev->chan_reg = devm_ioremap_resource(&pdev->dev, res); - if (IS_ERR(audev->chan_reg)) - return PTR_ERR(audev->chan_reg); - - sdev = &audev->shdma_dev; - sdev->ops = &audmapp_shdma_ops; - sdev->desc_size = sizeof(struct audmapp_desc); - - dma_dev = &sdev->dma_dev; - dma_dev->copy_align = LOG2_DEFAULT_XFER_SIZE; - dma_cap_set(DMA_SLAVE, dma_dev->cap_mask); - - err = shdma_init(&pdev->dev, sdev, AUDMAPP_MAX_CHANNELS); - if (err < 0) - return err; - - platform_set_drvdata(pdev, audev); - - /* Create DMA Channel */ - for (i = 0; i < AUDMAPP_MAX_CHANNELS; i++) { - err = audmapp_chan_probe(pdev, audev, i); - if (err) - goto chan_probe_err; - } - - err = dma_async_device_register(dma_dev); - if (err < 0) - goto chan_probe_err; - - return err; - -chan_probe_err: - audmapp_chan_remove(audev); - shdma_cleanup(sdev); - - return err; -} - -static int audmapp_remove(struct platform_device *pdev) -{ - struct audmapp_device *audev = platform_get_drvdata(pdev); - struct dma_device *dma_dev = &audev->shdma_dev.dma_dev; - - dma_async_device_unregister(dma_dev); - - audmapp_chan_remove(audev); - shdma_cleanup(&audev->shdma_dev); - - return 0; -} - -static const struct of_device_id audmapp_of_match[] = { - { .compatible = "renesas,rcar-audmapp", }, - {}, -}; - -static struct platform_driver audmapp_driver = { - .probe = audmapp_probe, - .remove = audmapp_remove, - .driver = { - .name = "rcar-audmapp-engine", - .of_match_table = audmapp_of_match, - }, -}; -module_platform_driver(audmapp_driver); - -MODULE_AUTHOR("Kuninori Morimoto <kuninori.morimoto.gx@renesas.com>"); -MODULE_DESCRIPTION("Renesas R-Car Audio DMAC peri-peri driver"); -MODULE_LICENSE("GPL"); diff --git a/drivers/dma/sh/shdma-base.c b/drivers/dma/sh/shdma-base.c index 8ee383d339a5..10fcabad80f3 100644 --- a/drivers/dma/sh/shdma-base.c +++ b/drivers/dma/sh/shdma-base.c @@ -171,8 +171,7 @@ static struct shdma_desc *shdma_get_desc(struct shdma_chan *schan) return NULL; } -static int shdma_setup_slave(struct shdma_chan *schan, int slave_id, - dma_addr_t slave_addr) +static int shdma_setup_slave(struct shdma_chan *schan, dma_addr_t slave_addr) { struct shdma_dev *sdev = to_shdma_dev(schan->dma_chan.device); const struct shdma_ops *ops = sdev->ops; @@ -183,25 +182,23 @@ static int shdma_setup_slave(struct shdma_chan *schan, int slave_id, ret = ops->set_slave(schan, match, slave_addr, true); if (ret < 0) return ret; - - slave_id = schan->slave_id; } else { - match = slave_id; + match = schan->real_slave_id; } - if (slave_id < 0 || slave_id >= slave_num) + if (schan->real_slave_id < 0 || schan->real_slave_id >= slave_num) return -EINVAL; - if (test_and_set_bit(slave_id, shdma_slave_used)) + if (test_and_set_bit(schan->real_slave_id, shdma_slave_used)) return -EBUSY; ret = ops->set_slave(schan, match, slave_addr, false); if (ret < 0) { - clear_bit(slave_id, shdma_slave_used); + clear_bit(schan->real_slave_id, shdma_slave_used); return ret; } - schan->slave_id = slave_id; + schan->slave_id = schan->real_slave_id; return 0; } @@ -221,10 +218,12 @@ static int shdma_alloc_chan_resources(struct dma_chan *chan) */ if (slave) { /* Legacy mode: .private is set in filter */ - ret = shdma_setup_slave(schan, slave->slave_id, 0); + schan->real_slave_id = slave->slave_id; + ret = shdma_setup_slave(schan, 0); if (ret < 0) goto esetslave; } else { + /* Normal mode: real_slave_id was set by filter */ schan->slave_id = -EINVAL; } @@ -258,11 +257,14 @@ esetslave: /* * This is the standard shdma filter function to be used as a replacement to the - * "old" method, using the .private pointer. If for some reason you allocate a - * channel without slave data, use something like ERR_PTR(-EINVAL) as a filter + * "old" method, using the .private pointer. + * You always have to pass a valid slave id as the argument, old drivers that + * pass ERR_PTR(-EINVAL) as a filter parameter and set it up in dma_slave_config + * need to be updated so we can remove the slave_id field from dma_slave_config. * parameter. If this filter is used, the slave driver, after calling * dma_request_channel(), will also have to call dmaengine_slave_config() with - * .slave_id, .direction, and either .src_addr or .dst_addr set. + * .direction, and either .src_addr or .dst_addr set. + * * NOTE: this filter doesn't support multiple DMAC drivers with the DMA_SLAVE * capability! If this becomes a requirement, hardware glue drivers, using this * services would have to provide their own filters, which first would check @@ -276,7 +278,7 @@ bool shdma_chan_filter(struct dma_chan *chan, void *arg) { struct shdma_chan *schan; struct shdma_dev *sdev; - int match = (long)arg; + int slave_id = (long)arg; int ret; /* Only support channels handled by this driver. */ @@ -284,19 +286,39 @@ bool shdma_chan_filter(struct dma_chan *chan, void *arg) shdma_alloc_chan_resources) return false; - if (match < 0) + schan = to_shdma_chan(chan); + sdev = to_shdma_dev(chan->device); + + /* + * For DT, the schan->slave_id field is generated by the + * set_slave function from the slave ID that is passed in + * from xlate. For the non-DT case, the slave ID is + * directly passed into the filter function by the driver + */ + if (schan->dev->of_node) { + ret = sdev->ops->set_slave(schan, slave_id, 0, true); + if (ret < 0) + return false; + + schan->real_slave_id = schan->slave_id; + return true; + } + + if (slave_id < 0) { /* No slave requested - arbitrary channel */ + dev_warn(sdev->dma_dev.dev, "invalid slave ID passed to dma_request_slave\n"); return true; + } - schan = to_shdma_chan(chan); - if (!schan->dev->of_node && match >= slave_num) + if (slave_id >= slave_num) return false; - sdev = to_shdma_dev(schan->dma_chan.device); - ret = sdev->ops->set_slave(schan, match, 0, true); + ret = sdev->ops->set_slave(schan, slave_id, 0, true); if (ret < 0) return false; + schan->real_slave_id = slave_id; + return true; } EXPORT_SYMBOL(shdma_chan_filter); @@ -452,6 +474,8 @@ static void shdma_free_chan_resources(struct dma_chan *chan) chan->private = NULL; } + schan->real_slave_id = 0; + spin_lock_irq(&schan->chan_lock); list_splice_init(&schan->ld_free, &list); @@ -764,11 +788,20 @@ static int shdma_config(struct dma_chan *chan, */ if (!config) return -EINVAL; + + /* + * overriding the slave_id through dma_slave_config is deprecated, + * but possibly some out-of-tree drivers still do it. + */ + if (WARN_ON_ONCE(config->slave_id && + config->slave_id != schan->real_slave_id)) + schan->real_slave_id = config->slave_id; + /* * We could lock this, but you shouldn't be configuring the * channel, while using it... */ - return shdma_setup_slave(schan, config->slave_id, + return shdma_setup_slave(schan, config->direction == DMA_DEV_TO_MEM ? config->src_addr : config->dst_addr); } diff --git a/drivers/dma/sh/shdmac.c b/drivers/dma/sh/shdmac.c index 9f1d4c7dbab8..11707df1a689 100644 --- a/drivers/dma/sh/shdmac.c +++ b/drivers/dma/sh/shdmac.c @@ -443,7 +443,7 @@ static bool sh_dmae_reset(struct sh_dmae_device *shdev) return ret; } -#if defined(CONFIG_CPU_SH4) || defined(CONFIG_ARM) +#if defined(CONFIG_CPU_SH4) || defined(CONFIG_ARCH_SHMOBILE) static irqreturn_t sh_dmae_err(int irq, void *data) { struct sh_dmae_device *shdev = data; @@ -689,7 +689,7 @@ static int sh_dmae_probe(struct platform_device *pdev) const struct sh_dmae_pdata *pdata; unsigned long chan_flag[SH_DMAE_MAX_CHANNELS] = {}; int chan_irq[SH_DMAE_MAX_CHANNELS]; -#if defined(CONFIG_CPU_SH4) || defined(CONFIG_ARM) +#if defined(CONFIG_CPU_SH4) || defined(CONFIG_ARCH_SHMOBILE) unsigned long irqflags = 0; int errirq; #endif diff --git a/drivers/dma/sh/usb-dmac.c b/drivers/dma/sh/usb-dmac.c new file mode 100644 index 000000000000..f705798ce3eb --- /dev/null +++ b/drivers/dma/sh/usb-dmac.c @@ -0,0 +1,910 @@ +/* + * Renesas USB DMA Controller Driver + * + * Copyright (C) 2015 Renesas Electronics Corporation + * + * based on rcar-dmac.c + * Copyright (C) 2014 Renesas Electronics Inc. + * Author: Laurent Pinchart <laurent.pinchart@ideasonboard.com> + * + * This is free software; you can redistribute it and/or modify + * it under the terms of version 2 of the GNU General Public License as + * published by the Free Software Foundation. + */ + +#include <linux/delay.h> +#include <linux/dma-mapping.h> +#include <linux/dmaengine.h> +#include <linux/interrupt.h> +#include <linux/list.h> +#include <linux/module.h> +#include <linux/of.h> +#include <linux/of_dma.h> +#include <linux/of_platform.h> +#include <linux/platform_device.h> +#include <linux/pm_runtime.h> +#include <linux/slab.h> +#include <linux/spinlock.h> + +#include "../dmaengine.h" +#include "../virt-dma.h" + +/* + * struct usb_dmac_sg - Descriptor for a hardware transfer + * @mem_addr: memory address + * @size: transfer size in bytes + */ +struct usb_dmac_sg { + dma_addr_t mem_addr; + u32 size; +}; + +/* + * struct usb_dmac_desc - USB DMA Transfer Descriptor + * @vd: base virtual channel DMA transaction descriptor + * @direction: direction of the DMA transfer + * @sg_allocated_len: length of allocated sg + * @sg_len: length of sg + * @sg_index: index of sg + * @residue: residue after the DMAC completed a transfer + * @node: node for desc_got and desc_freed + * @done_cookie: cookie after the DMAC completed a transfer + * @sg: information for the transfer + */ +struct usb_dmac_desc { + struct virt_dma_desc vd; + enum dma_transfer_direction direction; + unsigned int sg_allocated_len; + unsigned int sg_len; + unsigned int sg_index; + u32 residue; + struct list_head node; + dma_cookie_t done_cookie; + struct usb_dmac_sg sg[0]; +}; + +#define to_usb_dmac_desc(vd) container_of(vd, struct usb_dmac_desc, vd) + +/* + * struct usb_dmac_chan - USB DMA Controller Channel + * @vc: base virtual DMA channel object + * @iomem: channel I/O memory base + * @index: index of this channel in the controller + * @irq: irq number of this channel + * @desc: the current descriptor + * @descs_allocated: number of descriptors allocated + * @desc_got: got descriptors + * @desc_freed: freed descriptors after the DMAC completed a transfer + */ +struct usb_dmac_chan { + struct virt_dma_chan vc; + void __iomem *iomem; + unsigned int index; + int irq; + struct usb_dmac_desc *desc; + int descs_allocated; + struct list_head desc_got; + struct list_head desc_freed; +}; + +#define to_usb_dmac_chan(c) container_of(c, struct usb_dmac_chan, vc.chan) + +/* + * struct usb_dmac - USB DMA Controller + * @engine: base DMA engine object + * @dev: the hardware device + * @iomem: remapped I/O memory base + * @n_channels: number of available channels + * @channels: array of DMAC channels + */ +struct usb_dmac { + struct dma_device engine; + struct device *dev; + void __iomem *iomem; + + unsigned int n_channels; + struct usb_dmac_chan *channels; +}; + +#define to_usb_dmac(d) container_of(d, struct usb_dmac, engine) + +/* ----------------------------------------------------------------------------- + * Registers + */ + +#define USB_DMAC_CHAN_OFFSET(i) (0x20 + 0x20 * (i)) + +#define USB_DMASWR 0x0008 +#define USB_DMASWR_SWR (1 << 0) +#define USB_DMAOR 0x0060 +#define USB_DMAOR_AE (1 << 2) +#define USB_DMAOR_DME (1 << 0) + +#define USB_DMASAR 0x0000 +#define USB_DMADAR 0x0004 +#define USB_DMATCR 0x0008 +#define USB_DMATCR_MASK 0x00ffffff +#define USB_DMACHCR 0x0014 +#define USB_DMACHCR_FTE (1 << 24) +#define USB_DMACHCR_NULLE (1 << 16) +#define USB_DMACHCR_NULL (1 << 12) +#define USB_DMACHCR_TS_8B ((0 << 7) | (0 << 6)) +#define USB_DMACHCR_TS_16B ((0 << 7) | (1 << 6)) +#define USB_DMACHCR_TS_32B ((1 << 7) | (0 << 6)) +#define USB_DMACHCR_IE (1 << 5) +#define USB_DMACHCR_SP (1 << 2) +#define USB_DMACHCR_TE (1 << 1) +#define USB_DMACHCR_DE (1 << 0) +#define USB_DMATEND 0x0018 + +/* Hardcode the xfer_shift to 5 (32bytes) */ +#define USB_DMAC_XFER_SHIFT 5 +#define USB_DMAC_XFER_SIZE (1 << USB_DMAC_XFER_SHIFT) +#define USB_DMAC_CHCR_TS USB_DMACHCR_TS_32B +#define USB_DMAC_SLAVE_BUSWIDTH DMA_SLAVE_BUSWIDTH_32_BYTES + +/* for descriptors */ +#define USB_DMAC_INITIAL_NR_DESC 16 +#define USB_DMAC_INITIAL_NR_SG 8 + +/* ----------------------------------------------------------------------------- + * Device access + */ + +static void usb_dmac_write(struct usb_dmac *dmac, u32 reg, u32 data) +{ + writel(data, dmac->iomem + reg); +} + +static u32 usb_dmac_read(struct usb_dmac *dmac, u32 reg) +{ + return readl(dmac->iomem + reg); +} + +static u32 usb_dmac_chan_read(struct usb_dmac_chan *chan, u32 reg) +{ + return readl(chan->iomem + reg); +} + +static void usb_dmac_chan_write(struct usb_dmac_chan *chan, u32 reg, u32 data) +{ + writel(data, chan->iomem + reg); +} + +/* ----------------------------------------------------------------------------- + * Initialization and configuration + */ + +static bool usb_dmac_chan_is_busy(struct usb_dmac_chan *chan) +{ + u32 chcr = usb_dmac_chan_read(chan, USB_DMACHCR); + + return (chcr & (USB_DMACHCR_DE | USB_DMACHCR_TE)) == USB_DMACHCR_DE; +} + +static u32 usb_dmac_calc_tend(u32 size) +{ + /* + * Please refer to the Figure "Example of Final Transaction Valid + * Data Transfer Enable (EDTEN) Setting" in the data sheet. + */ + return 0xffffffff << (32 - (size % USB_DMAC_XFER_SIZE ? : + USB_DMAC_XFER_SIZE)); +} + +/* This function is already held by vc.lock */ +static void usb_dmac_chan_start_sg(struct usb_dmac_chan *chan, + unsigned int index) +{ + struct usb_dmac_desc *desc = chan->desc; + struct usb_dmac_sg *sg = desc->sg + index; + dma_addr_t src_addr = 0, dst_addr = 0; + + WARN_ON_ONCE(usb_dmac_chan_is_busy(chan)); + + if (desc->direction == DMA_DEV_TO_MEM) + dst_addr = sg->mem_addr; + else + src_addr = sg->mem_addr; + + dev_dbg(chan->vc.chan.device->dev, + "chan%u: queue sg %p: %u@%pad -> %pad\n", + chan->index, sg, sg->size, &src_addr, &dst_addr); + + usb_dmac_chan_write(chan, USB_DMASAR, src_addr & 0xffffffff); + usb_dmac_chan_write(chan, USB_DMADAR, dst_addr & 0xffffffff); + usb_dmac_chan_write(chan, USB_DMATCR, + DIV_ROUND_UP(sg->size, USB_DMAC_XFER_SIZE)); + usb_dmac_chan_write(chan, USB_DMATEND, usb_dmac_calc_tend(sg->size)); + + usb_dmac_chan_write(chan, USB_DMACHCR, USB_DMAC_CHCR_TS | + USB_DMACHCR_NULLE | USB_DMACHCR_IE | USB_DMACHCR_DE); +} + +/* This function is already held by vc.lock */ +static void usb_dmac_chan_start_desc(struct usb_dmac_chan *chan) +{ + struct virt_dma_desc *vd; + + vd = vchan_next_desc(&chan->vc); + if (!vd) { + chan->desc = NULL; + return; + } + + /* + * Remove this request from vc->desc_issued. Otherwise, this driver + * will get the previous value from vchan_next_desc() after a transfer + * was completed. + */ + list_del(&vd->node); + + chan->desc = to_usb_dmac_desc(vd); + chan->desc->sg_index = 0; + usb_dmac_chan_start_sg(chan, 0); +} + +static int usb_dmac_init(struct usb_dmac *dmac) +{ + u16 dmaor; + + /* Clear all channels and enable the DMAC globally. */ + usb_dmac_write(dmac, USB_DMAOR, USB_DMAOR_DME); + + dmaor = usb_dmac_read(dmac, USB_DMAOR); + if ((dmaor & (USB_DMAOR_AE | USB_DMAOR_DME)) != USB_DMAOR_DME) { + dev_warn(dmac->dev, "DMAOR initialization failed.\n"); + return -EIO; + } + + return 0; +} + +/* ----------------------------------------------------------------------------- + * Descriptors allocation and free + */ +static int usb_dmac_desc_alloc(struct usb_dmac_chan *chan, unsigned int sg_len, + gfp_t gfp) +{ + struct usb_dmac_desc *desc; + unsigned long flags; + + desc = kzalloc(sizeof(*desc) + sg_len * sizeof(desc->sg[0]), gfp); + if (!desc) + return -ENOMEM; + + desc->sg_allocated_len = sg_len; + INIT_LIST_HEAD(&desc->node); + + spin_lock_irqsave(&chan->vc.lock, flags); + list_add_tail(&desc->node, &chan->desc_freed); + spin_unlock_irqrestore(&chan->vc.lock, flags); + + return 0; +} + +static void usb_dmac_desc_free(struct usb_dmac_chan *chan) +{ + struct usb_dmac_desc *desc, *_desc; + LIST_HEAD(list); + + list_splice_init(&chan->desc_freed, &list); + list_splice_init(&chan->desc_got, &list); + + list_for_each_entry_safe(desc, _desc, &list, node) { + list_del(&desc->node); + kfree(desc); + } + chan->descs_allocated = 0; +} + +static struct usb_dmac_desc *usb_dmac_desc_get(struct usb_dmac_chan *chan, + unsigned int sg_len, gfp_t gfp) +{ + struct usb_dmac_desc *desc = NULL; + unsigned long flags; + + /* Get a freed descritpor */ + spin_lock_irqsave(&chan->vc.lock, flags); + list_for_each_entry(desc, &chan->desc_freed, node) { + if (sg_len <= desc->sg_allocated_len) { + list_move_tail(&desc->node, &chan->desc_got); + spin_unlock_irqrestore(&chan->vc.lock, flags); + return desc; + } + } + spin_unlock_irqrestore(&chan->vc.lock, flags); + + /* Allocate a new descriptor */ + if (!usb_dmac_desc_alloc(chan, sg_len, gfp)) { + /* If allocated the desc, it was added to tail of the list */ + spin_lock_irqsave(&chan->vc.lock, flags); + desc = list_last_entry(&chan->desc_freed, struct usb_dmac_desc, + node); + list_move_tail(&desc->node, &chan->desc_got); + spin_unlock_irqrestore(&chan->vc.lock, flags); + return desc; + } + + return NULL; +} + +static void usb_dmac_desc_put(struct usb_dmac_chan *chan, + struct usb_dmac_desc *desc) +{ + unsigned long flags; + + spin_lock_irqsave(&chan->vc.lock, flags); + list_move_tail(&desc->node, &chan->desc_freed); + spin_unlock_irqrestore(&chan->vc.lock, flags); +} + +/* ----------------------------------------------------------------------------- + * Stop and reset + */ + +static void usb_dmac_soft_reset(struct usb_dmac_chan *uchan) +{ + struct dma_chan *chan = &uchan->vc.chan; + struct usb_dmac *dmac = to_usb_dmac(chan->device); + int i; + + /* Don't issue soft reset if any one of channels is busy */ + for (i = 0; i < dmac->n_channels; ++i) { + if (usb_dmac_chan_is_busy(uchan)) + return; + } + + usb_dmac_write(dmac, USB_DMAOR, 0); + usb_dmac_write(dmac, USB_DMASWR, USB_DMASWR_SWR); + udelay(100); + usb_dmac_write(dmac, USB_DMASWR, 0); + usb_dmac_write(dmac, USB_DMAOR, 1); +} + +static void usb_dmac_chan_halt(struct usb_dmac_chan *chan) +{ + u32 chcr = usb_dmac_chan_read(chan, USB_DMACHCR); + + chcr &= ~(USB_DMACHCR_IE | USB_DMACHCR_TE | USB_DMACHCR_DE); + usb_dmac_chan_write(chan, USB_DMACHCR, chcr); + + usb_dmac_soft_reset(chan); +} + +static void usb_dmac_stop(struct usb_dmac *dmac) +{ + usb_dmac_write(dmac, USB_DMAOR, 0); +} + +/* ----------------------------------------------------------------------------- + * DMA engine operations + */ + +static int usb_dmac_alloc_chan_resources(struct dma_chan *chan) +{ + struct usb_dmac_chan *uchan = to_usb_dmac_chan(chan); + int ret; + + while (uchan->descs_allocated < USB_DMAC_INITIAL_NR_DESC) { + ret = usb_dmac_desc_alloc(uchan, USB_DMAC_INITIAL_NR_SG, + GFP_KERNEL); + if (ret < 0) { + usb_dmac_desc_free(uchan); + return ret; + } + uchan->descs_allocated++; + } + + return pm_runtime_get_sync(chan->device->dev); +} + +static void usb_dmac_free_chan_resources(struct dma_chan *chan) +{ + struct usb_dmac_chan *uchan = to_usb_dmac_chan(chan); + unsigned long flags; + + /* Protect against ISR */ + spin_lock_irqsave(&uchan->vc.lock, flags); + usb_dmac_chan_halt(uchan); + spin_unlock_irqrestore(&uchan->vc.lock, flags); + + usb_dmac_desc_free(uchan); + vchan_free_chan_resources(&uchan->vc); + + pm_runtime_put(chan->device->dev); +} + +static struct dma_async_tx_descriptor * +usb_dmac_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl, + unsigned int sg_len, enum dma_transfer_direction dir, + unsigned long dma_flags, void *context) +{ + struct usb_dmac_chan *uchan = to_usb_dmac_chan(chan); + struct usb_dmac_desc *desc; + struct scatterlist *sg; + int i; + + if (!sg_len) { + dev_warn(chan->device->dev, + "%s: bad parameter: len=%d\n", __func__, sg_len); + return NULL; + } + + desc = usb_dmac_desc_get(uchan, sg_len, GFP_NOWAIT); + if (!desc) + return NULL; + + desc->direction = dir; + desc->sg_len = sg_len; + for_each_sg(sgl, sg, sg_len, i) { + desc->sg[i].mem_addr = sg_dma_address(sg); + desc->sg[i].size = sg_dma_len(sg); + } + + return vchan_tx_prep(&uchan->vc, &desc->vd, dma_flags); +} + +static int usb_dmac_chan_terminate_all(struct dma_chan *chan) +{ + struct usb_dmac_chan *uchan = to_usb_dmac_chan(chan); + struct usb_dmac_desc *desc; + unsigned long flags; + LIST_HEAD(head); + LIST_HEAD(list); + + spin_lock_irqsave(&uchan->vc.lock, flags); + usb_dmac_chan_halt(uchan); + vchan_get_all_descriptors(&uchan->vc, &head); + if (uchan->desc) + uchan->desc = NULL; + list_splice_init(&uchan->desc_got, &list); + list_for_each_entry(desc, &list, node) + list_move_tail(&desc->node, &uchan->desc_freed); + spin_unlock_irqrestore(&uchan->vc.lock, flags); + vchan_dma_desc_free_list(&uchan->vc, &head); + + return 0; +} + +static unsigned int usb_dmac_get_current_residue(struct usb_dmac_chan *chan, + struct usb_dmac_desc *desc, + int sg_index) +{ + struct usb_dmac_sg *sg = desc->sg + sg_index; + u32 mem_addr = sg->mem_addr & 0xffffffff; + unsigned int residue = sg->size; + + /* + * We cannot use USB_DMATCR to calculate residue because USB_DMATCR + * has unsuited value to calculate. + */ + if (desc->direction == DMA_DEV_TO_MEM) + residue -= usb_dmac_chan_read(chan, USB_DMADAR) - mem_addr; + else + residue -= usb_dmac_chan_read(chan, USB_DMASAR) - mem_addr; + + return residue; +} + +static u32 usb_dmac_chan_get_residue_if_complete(struct usb_dmac_chan *chan, + dma_cookie_t cookie) +{ + struct usb_dmac_desc *desc; + u32 residue = 0; + + list_for_each_entry_reverse(desc, &chan->desc_freed, node) { + if (desc->done_cookie == cookie) { + residue = desc->residue; + break; + } + } + + return residue; +} + +static u32 usb_dmac_chan_get_residue(struct usb_dmac_chan *chan, + dma_cookie_t cookie) +{ + u32 residue = 0; + struct virt_dma_desc *vd; + struct usb_dmac_desc *desc = chan->desc; + int i; + + if (!desc) { + vd = vchan_find_desc(&chan->vc, cookie); + if (!vd) + return 0; + desc = to_usb_dmac_desc(vd); + } + + /* Compute the size of all usb_dmac_sg still to be transferred */ + for (i = desc->sg_index + 1; i < desc->sg_len; i++) + residue += desc->sg[i].size; + + /* Add the residue for the current sg */ + residue += usb_dmac_get_current_residue(chan, desc, desc->sg_index); + + return residue; +} + +static enum dma_status usb_dmac_tx_status(struct dma_chan *chan, + dma_cookie_t cookie, + struct dma_tx_state *txstate) +{ + struct usb_dmac_chan *uchan = to_usb_dmac_chan(chan); + enum dma_status status; + unsigned int residue = 0; + unsigned long flags; + + status = dma_cookie_status(chan, cookie, txstate); + /* a client driver will get residue after DMA_COMPLETE */ + if (!txstate) + return status; + + spin_lock_irqsave(&uchan->vc.lock, flags); + if (status == DMA_COMPLETE) + residue = usb_dmac_chan_get_residue_if_complete(uchan, cookie); + else + residue = usb_dmac_chan_get_residue(uchan, cookie); + spin_unlock_irqrestore(&uchan->vc.lock, flags); + + dma_set_residue(txstate, residue); + + return status; +} + +static void usb_dmac_issue_pending(struct dma_chan *chan) +{ + struct usb_dmac_chan *uchan = to_usb_dmac_chan(chan); + unsigned long flags; + + spin_lock_irqsave(&uchan->vc.lock, flags); + if (vchan_issue_pending(&uchan->vc) && !uchan->desc) + usb_dmac_chan_start_desc(uchan); + spin_unlock_irqrestore(&uchan->vc.lock, flags); +} + +static void usb_dmac_virt_desc_free(struct virt_dma_desc *vd) +{ + struct usb_dmac_desc *desc = to_usb_dmac_desc(vd); + struct usb_dmac_chan *chan = to_usb_dmac_chan(vd->tx.chan); + + usb_dmac_desc_put(chan, desc); +} + +/* ----------------------------------------------------------------------------- + * IRQ handling + */ + +static void usb_dmac_isr_transfer_end(struct usb_dmac_chan *chan) +{ + struct usb_dmac_desc *desc = chan->desc; + + BUG_ON(!desc); + + if (++desc->sg_index < desc->sg_len) { + usb_dmac_chan_start_sg(chan, desc->sg_index); + } else { + desc->residue = usb_dmac_get_current_residue(chan, desc, + desc->sg_index - 1); + desc->done_cookie = desc->vd.tx.cookie; + vchan_cookie_complete(&desc->vd); + + /* Restart the next transfer if this driver has a next desc */ + usb_dmac_chan_start_desc(chan); + } +} + +static irqreturn_t usb_dmac_isr_channel(int irq, void *dev) +{ + struct usb_dmac_chan *chan = dev; + irqreturn_t ret = IRQ_NONE; + u32 mask = USB_DMACHCR_TE; + u32 check_bits = USB_DMACHCR_TE | USB_DMACHCR_SP; + u32 chcr; + + spin_lock(&chan->vc.lock); + + chcr = usb_dmac_chan_read(chan, USB_DMACHCR); + if (chcr & check_bits) + mask |= USB_DMACHCR_DE | check_bits; + if (chcr & USB_DMACHCR_NULL) { + /* An interruption of TE will happen after we set FTE */ + mask |= USB_DMACHCR_NULL; + chcr |= USB_DMACHCR_FTE; + ret |= IRQ_HANDLED; + } + usb_dmac_chan_write(chan, USB_DMACHCR, chcr & ~mask); + + if (chcr & check_bits) { + usb_dmac_isr_transfer_end(chan); + ret |= IRQ_HANDLED; + } + + spin_unlock(&chan->vc.lock); + + return ret; +} + +/* ----------------------------------------------------------------------------- + * OF xlate and channel filter + */ + +static bool usb_dmac_chan_filter(struct dma_chan *chan, void *arg) +{ + struct usb_dmac_chan *uchan = to_usb_dmac_chan(chan); + struct of_phandle_args *dma_spec = arg; + + if (dma_spec->np != chan->device->dev->of_node) + return false; + + /* USB-DMAC should be used with fixed usb controller's FIFO */ + if (uchan->index != dma_spec->args[0]) + return false; + + return true; +} + +static struct dma_chan *usb_dmac_of_xlate(struct of_phandle_args *dma_spec, + struct of_dma *ofdma) +{ + struct usb_dmac_chan *uchan; + struct dma_chan *chan; + dma_cap_mask_t mask; + + if (dma_spec->args_count != 1) + return NULL; + + /* Only slave DMA channels can be allocated via DT */ + dma_cap_zero(mask); + dma_cap_set(DMA_SLAVE, mask); + + chan = dma_request_channel(mask, usb_dmac_chan_filter, dma_spec); + if (!chan) + return NULL; + + uchan = to_usb_dmac_chan(chan); + + return chan; +} + +/* ----------------------------------------------------------------------------- + * Power management + */ + +static int usb_dmac_runtime_suspend(struct device *dev) +{ + struct usb_dmac *dmac = dev_get_drvdata(dev); + int i; + + for (i = 0; i < dmac->n_channels; ++i) + usb_dmac_chan_halt(&dmac->channels[i]); + + return 0; +} + +static int usb_dmac_runtime_resume(struct device *dev) +{ + struct usb_dmac *dmac = dev_get_drvdata(dev); + + return usb_dmac_init(dmac); +} + +static const struct dev_pm_ops usb_dmac_pm = { + SET_RUNTIME_PM_OPS(usb_dmac_runtime_suspend, usb_dmac_runtime_resume, + NULL) +}; + +/* ----------------------------------------------------------------------------- + * Probe and remove + */ + +static int usb_dmac_chan_probe(struct usb_dmac *dmac, + struct usb_dmac_chan *uchan, + unsigned int index) +{ + struct platform_device *pdev = to_platform_device(dmac->dev); + char pdev_irqname[5]; + char *irqname; + int ret; + + uchan->index = index; + uchan->iomem = dmac->iomem + USB_DMAC_CHAN_OFFSET(index); + + /* Request the channel interrupt. */ + sprintf(pdev_irqname, "ch%u", index); + uchan->irq = platform_get_irq_byname(pdev, pdev_irqname); + if (uchan->irq < 0) { + dev_err(dmac->dev, "no IRQ specified for channel %u\n", index); + return -ENODEV; + } + + irqname = devm_kasprintf(dmac->dev, GFP_KERNEL, "%s:%u", + dev_name(dmac->dev), index); + if (!irqname) + return -ENOMEM; + + ret = devm_request_irq(dmac->dev, uchan->irq, usb_dmac_isr_channel, + IRQF_SHARED, irqname, uchan); + if (ret) { + dev_err(dmac->dev, "failed to request IRQ %u (%d)\n", + uchan->irq, ret); + return ret; + } + + uchan->vc.desc_free = usb_dmac_virt_desc_free; + vchan_init(&uchan->vc, &dmac->engine); + INIT_LIST_HEAD(&uchan->desc_freed); + INIT_LIST_HEAD(&uchan->desc_got); + + return 0; +} + +static int usb_dmac_parse_of(struct device *dev, struct usb_dmac *dmac) +{ + struct device_node *np = dev->of_node; + int ret; + + ret = of_property_read_u32(np, "dma-channels", &dmac->n_channels); + if (ret < 0) { + dev_err(dev, "unable to read dma-channels property\n"); + return ret; + } + + if (dmac->n_channels <= 0 || dmac->n_channels >= 100) { + dev_err(dev, "invalid number of channels %u\n", + dmac->n_channels); + return -EINVAL; + } + + return 0; +} + +static int usb_dmac_probe(struct platform_device *pdev) +{ + const enum dma_slave_buswidth widths = USB_DMAC_SLAVE_BUSWIDTH; + struct dma_device *engine; + struct usb_dmac *dmac; + struct resource *mem; + unsigned int i; + int ret; + + dmac = devm_kzalloc(&pdev->dev, sizeof(*dmac), GFP_KERNEL); + if (!dmac) + return -ENOMEM; + + dmac->dev = &pdev->dev; + platform_set_drvdata(pdev, dmac); + + ret = usb_dmac_parse_of(&pdev->dev, dmac); + if (ret < 0) + return ret; + + dmac->channels = devm_kcalloc(&pdev->dev, dmac->n_channels, + sizeof(*dmac->channels), GFP_KERNEL); + if (!dmac->channels) + return -ENOMEM; + + /* Request resources. */ + mem = platform_get_resource(pdev, IORESOURCE_MEM, 0); + dmac->iomem = devm_ioremap_resource(&pdev->dev, mem); + if (IS_ERR(dmac->iomem)) + return PTR_ERR(dmac->iomem); + + /* Enable runtime PM and initialize the device. */ + pm_runtime_enable(&pdev->dev); + ret = pm_runtime_get_sync(&pdev->dev); + if (ret < 0) { + dev_err(&pdev->dev, "runtime PM get sync failed (%d)\n", ret); + return ret; + } + + ret = usb_dmac_init(dmac); + pm_runtime_put(&pdev->dev); + + if (ret) { + dev_err(&pdev->dev, "failed to reset device\n"); + goto error; + } + + /* Initialize the channels. */ + INIT_LIST_HEAD(&dmac->engine.channels); + + for (i = 0; i < dmac->n_channels; ++i) { + ret = usb_dmac_chan_probe(dmac, &dmac->channels[i], i); + if (ret < 0) + goto error; + } + + /* Register the DMAC as a DMA provider for DT. */ + ret = of_dma_controller_register(pdev->dev.of_node, usb_dmac_of_xlate, + NULL); + if (ret < 0) + goto error; + + /* + * Register the DMA engine device. + * + * Default transfer size of 32 bytes requires 32-byte alignment. + */ + engine = &dmac->engine; + dma_cap_set(DMA_SLAVE, engine->cap_mask); + + engine->dev = &pdev->dev; + + engine->src_addr_widths = widths; + engine->dst_addr_widths = widths; + engine->directions = BIT(DMA_MEM_TO_DEV) | BIT(DMA_DEV_TO_MEM); + engine->residue_granularity = DMA_RESIDUE_GRANULARITY_BURST; + + engine->device_alloc_chan_resources = usb_dmac_alloc_chan_resources; + engine->device_free_chan_resources = usb_dmac_free_chan_resources; + engine->device_prep_slave_sg = usb_dmac_prep_slave_sg; + engine->device_terminate_all = usb_dmac_chan_terminate_all; + engine->device_tx_status = usb_dmac_tx_status; + engine->device_issue_pending = usb_dmac_issue_pending; + + ret = dma_async_device_register(engine); + if (ret < 0) + goto error; + + return 0; + +error: + of_dma_controller_free(pdev->dev.of_node); + pm_runtime_disable(&pdev->dev); + return ret; +} + +static void usb_dmac_chan_remove(struct usb_dmac *dmac, + struct usb_dmac_chan *uchan) +{ + usb_dmac_chan_halt(uchan); + devm_free_irq(dmac->dev, uchan->irq, uchan); +} + +static int usb_dmac_remove(struct platform_device *pdev) +{ + struct usb_dmac *dmac = platform_get_drvdata(pdev); + int i; + + for (i = 0; i < dmac->n_channels; ++i) + usb_dmac_chan_remove(dmac, &dmac->channels[i]); + of_dma_controller_free(pdev->dev.of_node); + dma_async_device_unregister(&dmac->engine); + + pm_runtime_disable(&pdev->dev); + + return 0; +} + +static void usb_dmac_shutdown(struct platform_device *pdev) +{ + struct usb_dmac *dmac = platform_get_drvdata(pdev); + + usb_dmac_stop(dmac); +} + +static const struct of_device_id usb_dmac_of_ids[] = { + { .compatible = "renesas,usb-dmac", }, + { /* Sentinel */ } +}; +MODULE_DEVICE_TABLE(of, usb_dmac_of_ids); + +static struct platform_driver usb_dmac_driver = { + .driver = { + .pm = &usb_dmac_pm, + .name = "usb-dmac", + .of_match_table = usb_dmac_of_ids, + }, + .probe = usb_dmac_probe, + .remove = usb_dmac_remove, + .shutdown = usb_dmac_shutdown, +}; + +module_platform_driver(usb_dmac_driver); + +MODULE_DESCRIPTION("Renesas USB DMA Controller Driver"); +MODULE_AUTHOR("Yoshihiro Shimoda <yoshihiro.shimoda.uh@renesas.com>"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/dma/sirf-dma.c b/drivers/dma/sirf-dma.c index d0086e9f2082..a1afda43b8ef 100644 --- a/drivers/dma/sirf-dma.c +++ b/drivers/dma/sirf-dma.c @@ -896,7 +896,7 @@ static const struct dev_pm_ops sirfsoc_dma_pm_ops = { SET_SYSTEM_SLEEP_PM_OPS(sirfsoc_dma_pm_suspend, sirfsoc_dma_pm_resume) }; -static struct of_device_id sirfsoc_dma_match[] = { +static const struct of_device_id sirfsoc_dma_match[] = { { .compatible = "sirf,prima2-dmac", }, { .compatible = "sirf,marco-dmac", }, {}, diff --git a/drivers/dma/ste_dma40.c b/drivers/dma/ste_dma40.c index 1332b1d4d541..3c10f034d4b9 100644 --- a/drivers/dma/ste_dma40.c +++ b/drivers/dma/ste_dma40.c @@ -2514,7 +2514,8 @@ static struct dma_async_tx_descriptor *d40_prep_memcpy(struct dma_chan *chan, sg_dma_len(&dst_sg) = size; sg_dma_len(&src_sg) = size; - return d40_prep_sg(chan, &src_sg, &dst_sg, 1, DMA_NONE, dma_flags); + return d40_prep_sg(chan, &src_sg, &dst_sg, 1, + DMA_MEM_TO_MEM, dma_flags); } static struct dma_async_tx_descriptor * @@ -2526,7 +2527,8 @@ d40_prep_memcpy_sg(struct dma_chan *chan, if (dst_nents != src_nents) return NULL; - return d40_prep_sg(chan, src_sg, dst_sg, src_nents, DMA_NONE, dma_flags); + return d40_prep_sg(chan, src_sg, dst_sg, src_nents, + DMA_MEM_TO_MEM, dma_flags); } static struct dma_async_tx_descriptor * diff --git a/drivers/dma/sun6i-dma.c b/drivers/dma/sun6i-dma.c index 7ebcf9bec698..11e536586812 100644 --- a/drivers/dma/sun6i-dma.c +++ b/drivers/dma/sun6i-dma.c @@ -796,11 +796,6 @@ static void sun6i_dma_issue_pending(struct dma_chan *chan) spin_unlock_irqrestore(&vchan->vc.lock, flags); } -static int sun6i_dma_alloc_chan_resources(struct dma_chan *chan) -{ - return 0; -} - static void sun6i_dma_free_chan_resources(struct dma_chan *chan) { struct sun6i_dma_dev *sdev = to_sun6i_dma_dev(chan->device); @@ -896,7 +891,7 @@ static struct sun6i_dma_config sun8i_a23_dma_cfg = { .nr_max_vchans = 37, }; -static struct of_device_id sun6i_dma_match[] = { +static const struct of_device_id sun6i_dma_match[] = { { .compatible = "allwinner,sun6i-a31-dma", .data = &sun6i_a31_dma_cfg }, { .compatible = "allwinner,sun8i-a23-dma", .data = &sun8i_a23_dma_cfg }, { /* sentinel */ } @@ -957,7 +952,6 @@ static int sun6i_dma_probe(struct platform_device *pdev) dma_cap_set(DMA_SLAVE, sdc->slave.cap_mask); INIT_LIST_HEAD(&sdc->slave.channels); - sdc->slave.device_alloc_chan_resources = sun6i_dma_alloc_chan_resources; sdc->slave.device_free_chan_resources = sun6i_dma_free_chan_resources; sdc->slave.device_tx_status = sun6i_dma_tx_status; sdc->slave.device_issue_pending = sun6i_dma_issue_pending; diff --git a/drivers/dma/xgene-dma.c b/drivers/dma/xgene-dma.c new file mode 100755 index 000000000000..f52e37502254 --- /dev/null +++ b/drivers/dma/xgene-dma.c @@ -0,0 +1,2089 @@ +/* + * Applied Micro X-Gene SoC DMA engine Driver + * + * Copyright (c) 2015, Applied Micro Circuits Corporation + * Authors: Rameshwar Prasad Sahu <rsahu@apm.com> + * Loc Ho <lho@apm.com> + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the + * Free Software Foundation; either version 2 of the License, or (at your + * option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + * NOTE: PM support is currently not available. + */ + +#include <linux/clk.h> +#include <linux/delay.h> +#include <linux/dma-mapping.h> +#include <linux/dmaengine.h> +#include <linux/dmapool.h> +#include <linux/interrupt.h> +#include <linux/io.h> +#include <linux/module.h> +#include <linux/of_device.h> + +#include "dmaengine.h" + +/* X-Gene DMA ring csr registers and bit definations */ +#define XGENE_DMA_RING_CONFIG 0x04 +#define XGENE_DMA_RING_ENABLE BIT(31) +#define XGENE_DMA_RING_ID 0x08 +#define XGENE_DMA_RING_ID_SETUP(v) ((v) | BIT(31)) +#define XGENE_DMA_RING_ID_BUF 0x0C +#define XGENE_DMA_RING_ID_BUF_SETUP(v) (((v) << 9) | BIT(21)) +#define XGENE_DMA_RING_THRESLD0_SET1 0x30 +#define XGENE_DMA_RING_THRESLD0_SET1_VAL 0X64 +#define XGENE_DMA_RING_THRESLD1_SET1 0x34 +#define XGENE_DMA_RING_THRESLD1_SET1_VAL 0xC8 +#define XGENE_DMA_RING_HYSTERESIS 0x68 +#define XGENE_DMA_RING_HYSTERESIS_VAL 0xFFFFFFFF +#define XGENE_DMA_RING_STATE 0x6C +#define XGENE_DMA_RING_STATE_WR_BASE 0x70 +#define XGENE_DMA_RING_NE_INT_MODE 0x017C +#define XGENE_DMA_RING_NE_INT_MODE_SET(m, v) \ + ((m) = ((m) & ~BIT(31 - (v))) | BIT(31 - (v))) +#define XGENE_DMA_RING_NE_INT_MODE_RESET(m, v) \ + ((m) &= (~BIT(31 - (v)))) +#define XGENE_DMA_RING_CLKEN 0xC208 +#define XGENE_DMA_RING_SRST 0xC200 +#define XGENE_DMA_RING_MEM_RAM_SHUTDOWN 0xD070 +#define XGENE_DMA_RING_BLK_MEM_RDY 0xD074 +#define XGENE_DMA_RING_BLK_MEM_RDY_VAL 0xFFFFFFFF +#define XGENE_DMA_RING_DESC_CNT(v) (((v) & 0x0001FFFE) >> 1) +#define XGENE_DMA_RING_ID_GET(owner, num) (((owner) << 6) | (num)) +#define XGENE_DMA_RING_DST_ID(v) ((1 << 10) | (v)) +#define XGENE_DMA_RING_CMD_OFFSET 0x2C +#define XGENE_DMA_RING_CMD_BASE_OFFSET(v) ((v) << 6) +#define XGENE_DMA_RING_COHERENT_SET(m) \ + (((u32 *)(m))[2] |= BIT(4)) +#define XGENE_DMA_RING_ADDRL_SET(m, v) \ + (((u32 *)(m))[2] |= (((v) >> 8) << 5)) +#define XGENE_DMA_RING_ADDRH_SET(m, v) \ + (((u32 *)(m))[3] |= ((v) >> 35)) +#define XGENE_DMA_RING_ACCEPTLERR_SET(m) \ + (((u32 *)(m))[3] |= BIT(19)) +#define XGENE_DMA_RING_SIZE_SET(m, v) \ + (((u32 *)(m))[3] |= ((v) << 23)) +#define XGENE_DMA_RING_RECOMBBUF_SET(m) \ + (((u32 *)(m))[3] |= BIT(27)) +#define XGENE_DMA_RING_RECOMTIMEOUTL_SET(m) \ + (((u32 *)(m))[3] |= (0x7 << 28)) +#define XGENE_DMA_RING_RECOMTIMEOUTH_SET(m) \ + (((u32 *)(m))[4] |= 0x3) +#define XGENE_DMA_RING_SELTHRSH_SET(m) \ + (((u32 *)(m))[4] |= BIT(3)) +#define XGENE_DMA_RING_TYPE_SET(m, v) \ + (((u32 *)(m))[4] |= ((v) << 19)) + +/* X-Gene DMA device csr registers and bit definitions */ +#define XGENE_DMA_IPBRR 0x0 +#define XGENE_DMA_DEV_ID_RD(v) ((v) & 0x00000FFF) +#define XGENE_DMA_BUS_ID_RD(v) (((v) >> 12) & 3) +#define XGENE_DMA_REV_NO_RD(v) (((v) >> 14) & 3) +#define XGENE_DMA_GCR 0x10 +#define XGENE_DMA_CH_SETUP(v) \ + ((v) = ((v) & ~0x000FFFFF) | 0x000AAFFF) +#define XGENE_DMA_ENABLE(v) ((v) |= BIT(31)) +#define XGENE_DMA_DISABLE(v) ((v) &= ~BIT(31)) +#define XGENE_DMA_RAID6_CONT 0x14 +#define XGENE_DMA_RAID6_MULTI_CTRL(v) ((v) << 24) +#define XGENE_DMA_INT 0x70 +#define XGENE_DMA_INT_MASK 0x74 +#define XGENE_DMA_INT_ALL_MASK 0xFFFFFFFF +#define XGENE_DMA_INT_ALL_UNMASK 0x0 +#define XGENE_DMA_INT_MASK_SHIFT 0x14 +#define XGENE_DMA_RING_INT0_MASK 0x90A0 +#define XGENE_DMA_RING_INT1_MASK 0x90A8 +#define XGENE_DMA_RING_INT2_MASK 0x90B0 +#define XGENE_DMA_RING_INT3_MASK 0x90B8 +#define XGENE_DMA_RING_INT4_MASK 0x90C0 +#define XGENE_DMA_CFG_RING_WQ_ASSOC 0x90E0 +#define XGENE_DMA_ASSOC_RING_MNGR1 0xFFFFFFFF +#define XGENE_DMA_MEM_RAM_SHUTDOWN 0xD070 +#define XGENE_DMA_BLK_MEM_RDY 0xD074 +#define XGENE_DMA_BLK_MEM_RDY_VAL 0xFFFFFFFF + +/* X-Gene SoC EFUSE csr register and bit defination */ +#define XGENE_SOC_JTAG1_SHADOW 0x18 +#define XGENE_DMA_PQ_DISABLE_MASK BIT(13) + +/* X-Gene DMA Descriptor format */ +#define XGENE_DMA_DESC_NV_BIT BIT_ULL(50) +#define XGENE_DMA_DESC_IN_BIT BIT_ULL(55) +#define XGENE_DMA_DESC_C_BIT BIT_ULL(63) +#define XGENE_DMA_DESC_DR_BIT BIT_ULL(61) +#define XGENE_DMA_DESC_ELERR_POS 46 +#define XGENE_DMA_DESC_RTYPE_POS 56 +#define XGENE_DMA_DESC_LERR_POS 60 +#define XGENE_DMA_DESC_FLYBY_POS 4 +#define XGENE_DMA_DESC_BUFLEN_POS 48 +#define XGENE_DMA_DESC_HOENQ_NUM_POS 48 + +#define XGENE_DMA_DESC_NV_SET(m) \ + (((u64 *)(m))[0] |= XGENE_DMA_DESC_NV_BIT) +#define XGENE_DMA_DESC_IN_SET(m) \ + (((u64 *)(m))[0] |= XGENE_DMA_DESC_IN_BIT) +#define XGENE_DMA_DESC_RTYPE_SET(m, v) \ + (((u64 *)(m))[0] |= ((u64)(v) << XGENE_DMA_DESC_RTYPE_POS)) +#define XGENE_DMA_DESC_BUFADDR_SET(m, v) \ + (((u64 *)(m))[0] |= (v)) +#define XGENE_DMA_DESC_BUFLEN_SET(m, v) \ + (((u64 *)(m))[0] |= ((u64)(v) << XGENE_DMA_DESC_BUFLEN_POS)) +#define XGENE_DMA_DESC_C_SET(m) \ + (((u64 *)(m))[1] |= XGENE_DMA_DESC_C_BIT) +#define XGENE_DMA_DESC_FLYBY_SET(m, v) \ + (((u64 *)(m))[2] |= ((v) << XGENE_DMA_DESC_FLYBY_POS)) +#define XGENE_DMA_DESC_MULTI_SET(m, v, i) \ + (((u64 *)(m))[2] |= ((u64)(v) << (((i) + 1) * 8))) +#define XGENE_DMA_DESC_DR_SET(m) \ + (((u64 *)(m))[2] |= XGENE_DMA_DESC_DR_BIT) +#define XGENE_DMA_DESC_DST_ADDR_SET(m, v) \ + (((u64 *)(m))[3] |= (v)) +#define XGENE_DMA_DESC_H0ENQ_NUM_SET(m, v) \ + (((u64 *)(m))[3] |= ((u64)(v) << XGENE_DMA_DESC_HOENQ_NUM_POS)) +#define XGENE_DMA_DESC_ELERR_RD(m) \ + (((m) >> XGENE_DMA_DESC_ELERR_POS) & 0x3) +#define XGENE_DMA_DESC_LERR_RD(m) \ + (((m) >> XGENE_DMA_DESC_LERR_POS) & 0x7) +#define XGENE_DMA_DESC_STATUS(elerr, lerr) \ + (((elerr) << 4) | (lerr)) + +/* X-Gene DMA descriptor empty s/w signature */ +#define XGENE_DMA_DESC_EMPTY_INDEX 0 +#define XGENE_DMA_DESC_EMPTY_SIGNATURE ~0ULL +#define XGENE_DMA_DESC_SET_EMPTY(m) \ + (((u64 *)(m))[XGENE_DMA_DESC_EMPTY_INDEX] = \ + XGENE_DMA_DESC_EMPTY_SIGNATURE) +#define XGENE_DMA_DESC_IS_EMPTY(m) \ + (((u64 *)(m))[XGENE_DMA_DESC_EMPTY_INDEX] == \ + XGENE_DMA_DESC_EMPTY_SIGNATURE) + +/* X-Gene DMA configurable parameters defines */ +#define XGENE_DMA_RING_NUM 512 +#define XGENE_DMA_BUFNUM 0x0 +#define XGENE_DMA_CPU_BUFNUM 0x18 +#define XGENE_DMA_RING_OWNER_DMA 0x03 +#define XGENE_DMA_RING_OWNER_CPU 0x0F +#define XGENE_DMA_RING_TYPE_REGULAR 0x01 +#define XGENE_DMA_RING_WQ_DESC_SIZE 32 /* 32 Bytes */ +#define XGENE_DMA_RING_NUM_CONFIG 5 +#define XGENE_DMA_MAX_CHANNEL 4 +#define XGENE_DMA_XOR_CHANNEL 0 +#define XGENE_DMA_PQ_CHANNEL 1 +#define XGENE_DMA_MAX_BYTE_CNT 0x4000 /* 16 KB */ +#define XGENE_DMA_MAX_64B_DESC_BYTE_CNT 0x14000 /* 80 KB */ +#define XGENE_DMA_XOR_ALIGNMENT 6 /* 64 Bytes */ +#define XGENE_DMA_MAX_XOR_SRC 5 +#define XGENE_DMA_16K_BUFFER_LEN_CODE 0x0 +#define XGENE_DMA_INVALID_LEN_CODE 0x7800 + +/* X-Gene DMA descriptor error codes */ +#define ERR_DESC_AXI 0x01 +#define ERR_BAD_DESC 0x02 +#define ERR_READ_DATA_AXI 0x03 +#define ERR_WRITE_DATA_AXI 0x04 +#define ERR_FBP_TIMEOUT 0x05 +#define ERR_ECC 0x06 +#define ERR_DIFF_SIZE 0x08 +#define ERR_SCT_GAT_LEN 0x09 +#define ERR_CRC_ERR 0x11 +#define ERR_CHKSUM 0x12 +#define ERR_DIF 0x13 + +/* X-Gene DMA error interrupt codes */ +#define ERR_DIF_SIZE_INT 0x0 +#define ERR_GS_ERR_INT 0x1 +#define ERR_FPB_TIMEO_INT 0x2 +#define ERR_WFIFO_OVF_INT 0x3 +#define ERR_RFIFO_OVF_INT 0x4 +#define ERR_WR_TIMEO_INT 0x5 +#define ERR_RD_TIMEO_INT 0x6 +#define ERR_WR_ERR_INT 0x7 +#define ERR_RD_ERR_INT 0x8 +#define ERR_BAD_DESC_INT 0x9 +#define ERR_DESC_DST_INT 0xA +#define ERR_DESC_SRC_INT 0xB + +/* X-Gene DMA flyby operation code */ +#define FLYBY_2SRC_XOR 0x8 +#define FLYBY_3SRC_XOR 0x9 +#define FLYBY_4SRC_XOR 0xA +#define FLYBY_5SRC_XOR 0xB + +/* X-Gene DMA SW descriptor flags */ +#define XGENE_DMA_FLAG_64B_DESC BIT(0) + +/* Define to dump X-Gene DMA descriptor */ +#define XGENE_DMA_DESC_DUMP(desc, m) \ + print_hex_dump(KERN_ERR, (m), \ + DUMP_PREFIX_ADDRESS, 16, 8, (desc), 32, 0) + +#define to_dma_desc_sw(tx) \ + container_of(tx, struct xgene_dma_desc_sw, tx) +#define to_dma_chan(dchan) \ + container_of(dchan, struct xgene_dma_chan, dma_chan) + +#define chan_dbg(chan, fmt, arg...) \ + dev_dbg(chan->dev, "%s: " fmt, chan->name, ##arg) +#define chan_err(chan, fmt, arg...) \ + dev_err(chan->dev, "%s: " fmt, chan->name, ##arg) + +struct xgene_dma_desc_hw { + u64 m0; + u64 m1; + u64 m2; + u64 m3; +}; + +enum xgene_dma_ring_cfgsize { + XGENE_DMA_RING_CFG_SIZE_512B, + XGENE_DMA_RING_CFG_SIZE_2KB, + XGENE_DMA_RING_CFG_SIZE_16KB, + XGENE_DMA_RING_CFG_SIZE_64KB, + XGENE_DMA_RING_CFG_SIZE_512KB, + XGENE_DMA_RING_CFG_SIZE_INVALID +}; + +struct xgene_dma_ring { + struct xgene_dma *pdma; + u8 buf_num; + u16 id; + u16 num; + u16 head; + u16 owner; + u16 slots; + u16 dst_ring_num; + u32 size; + void __iomem *cmd; + void __iomem *cmd_base; + dma_addr_t desc_paddr; + u32 state[XGENE_DMA_RING_NUM_CONFIG]; + enum xgene_dma_ring_cfgsize cfgsize; + union { + void *desc_vaddr; + struct xgene_dma_desc_hw *desc_hw; + }; +}; + +struct xgene_dma_desc_sw { + struct xgene_dma_desc_hw desc1; + struct xgene_dma_desc_hw desc2; + u32 flags; + struct list_head node; + struct list_head tx_list; + struct dma_async_tx_descriptor tx; +}; + +/** + * struct xgene_dma_chan - internal representation of an X-Gene DMA channel + * @dma_chan: dmaengine channel object member + * @pdma: X-Gene DMA device structure reference + * @dev: struct device reference for dma mapping api + * @id: raw id of this channel + * @rx_irq: channel IRQ + * @name: name of X-Gene DMA channel + * @lock: serializes enqueue/dequeue operations to the descriptor pool + * @pending: number of transaction request pushed to DMA controller for + * execution, but still waiting for completion, + * @max_outstanding: max number of outstanding request we can push to channel + * @ld_pending: descriptors which are queued to run, but have not yet been + * submitted to the hardware for execution + * @ld_running: descriptors which are currently being executing by the hardware + * @ld_completed: descriptors which have finished execution by the hardware. + * These descriptors have already had their cleanup actions run. They + * are waiting for the ACK bit to be set by the async tx API. + * @desc_pool: descriptor pool for DMA operations + * @tasklet: bottom half where all completed descriptors cleans + * @tx_ring: transmit ring descriptor that we use to prepare actual + * descriptors for further executions + * @rx_ring: receive ring descriptor that we use to get completed DMA + * descriptors during cleanup time + */ +struct xgene_dma_chan { + struct dma_chan dma_chan; + struct xgene_dma *pdma; + struct device *dev; + int id; + int rx_irq; + char name[10]; + spinlock_t lock; + int pending; + int max_outstanding; + struct list_head ld_pending; + struct list_head ld_running; + struct list_head ld_completed; + struct dma_pool *desc_pool; + struct tasklet_struct tasklet; + struct xgene_dma_ring tx_ring; + struct xgene_dma_ring rx_ring; +}; + +/** + * struct xgene_dma - internal representation of an X-Gene DMA device + * @err_irq: DMA error irq number + * @ring_num: start id number for DMA ring + * @csr_dma: base for DMA register access + * @csr_ring: base for DMA ring register access + * @csr_ring_cmd: base for DMA ring command register access + * @csr_efuse: base for efuse register access + * @dma_dev: embedded struct dma_device + * @chan: reference to X-Gene DMA channels + */ +struct xgene_dma { + struct device *dev; + struct clk *clk; + int err_irq; + int ring_num; + void __iomem *csr_dma; + void __iomem *csr_ring; + void __iomem *csr_ring_cmd; + void __iomem *csr_efuse; + struct dma_device dma_dev[XGENE_DMA_MAX_CHANNEL]; + struct xgene_dma_chan chan[XGENE_DMA_MAX_CHANNEL]; +}; + +static const char * const xgene_dma_desc_err[] = { + [ERR_DESC_AXI] = "AXI error when reading src/dst link list", + [ERR_BAD_DESC] = "ERR or El_ERR fields not set to zero in desc", + [ERR_READ_DATA_AXI] = "AXI error when reading data", + [ERR_WRITE_DATA_AXI] = "AXI error when writing data", + [ERR_FBP_TIMEOUT] = "Timeout on bufpool fetch", + [ERR_ECC] = "ECC double bit error", + [ERR_DIFF_SIZE] = "Bufpool too small to hold all the DIF result", + [ERR_SCT_GAT_LEN] = "Gather and scatter data length not same", + [ERR_CRC_ERR] = "CRC error", + [ERR_CHKSUM] = "Checksum error", + [ERR_DIF] = "DIF error", +}; + +static const char * const xgene_dma_err[] = { + [ERR_DIF_SIZE_INT] = "DIF size error", + [ERR_GS_ERR_INT] = "Gather scatter not same size error", + [ERR_FPB_TIMEO_INT] = "Free pool time out error", + [ERR_WFIFO_OVF_INT] = "Write FIFO over flow error", + [ERR_RFIFO_OVF_INT] = "Read FIFO over flow error", + [ERR_WR_TIMEO_INT] = "Write time out error", + [ERR_RD_TIMEO_INT] = "Read time out error", + [ERR_WR_ERR_INT] = "HBF bus write error", + [ERR_RD_ERR_INT] = "HBF bus read error", + [ERR_BAD_DESC_INT] = "Ring descriptor HE0 not set error", + [ERR_DESC_DST_INT] = "HFB reading dst link address error", + [ERR_DESC_SRC_INT] = "HFB reading src link address error", +}; + +static bool is_pq_enabled(struct xgene_dma *pdma) +{ + u32 val; + + val = ioread32(pdma->csr_efuse + XGENE_SOC_JTAG1_SHADOW); + return !(val & XGENE_DMA_PQ_DISABLE_MASK); +} + +static void xgene_dma_cpu_to_le64(u64 *desc, int count) +{ + int i; + + for (i = 0; i < count; i++) + desc[i] = cpu_to_le64(desc[i]); +} + +static u16 xgene_dma_encode_len(u32 len) +{ + return (len < XGENE_DMA_MAX_BYTE_CNT) ? + len : XGENE_DMA_16K_BUFFER_LEN_CODE; +} + +static u8 xgene_dma_encode_xor_flyby(u32 src_cnt) +{ + static u8 flyby_type[] = { + FLYBY_2SRC_XOR, /* Dummy */ + FLYBY_2SRC_XOR, /* Dummy */ + FLYBY_2SRC_XOR, + FLYBY_3SRC_XOR, + FLYBY_4SRC_XOR, + FLYBY_5SRC_XOR + }; + + return flyby_type[src_cnt]; +} + +static u32 xgene_dma_ring_desc_cnt(struct xgene_dma_ring *ring) +{ + u32 __iomem *cmd_base = ring->cmd_base; + u32 ring_state = ioread32(&cmd_base[1]); + + return XGENE_DMA_RING_DESC_CNT(ring_state); +} + +static void xgene_dma_set_src_buffer(void *ext8, size_t *len, + dma_addr_t *paddr) +{ + size_t nbytes = (*len < XGENE_DMA_MAX_BYTE_CNT) ? + *len : XGENE_DMA_MAX_BYTE_CNT; + + XGENE_DMA_DESC_BUFADDR_SET(ext8, *paddr); + XGENE_DMA_DESC_BUFLEN_SET(ext8, xgene_dma_encode_len(nbytes)); + *len -= nbytes; + *paddr += nbytes; +} + +static void xgene_dma_invalidate_buffer(void *ext8) +{ + XGENE_DMA_DESC_BUFLEN_SET(ext8, XGENE_DMA_INVALID_LEN_CODE); +} + +static void *xgene_dma_lookup_ext8(u64 *desc, int idx) +{ + return (idx % 2) ? (desc + idx - 1) : (desc + idx + 1); +} + +static void xgene_dma_init_desc(void *desc, u16 dst_ring_num) +{ + XGENE_DMA_DESC_C_SET(desc); /* Coherent IO */ + XGENE_DMA_DESC_IN_SET(desc); + XGENE_DMA_DESC_H0ENQ_NUM_SET(desc, dst_ring_num); + XGENE_DMA_DESC_RTYPE_SET(desc, XGENE_DMA_RING_OWNER_DMA); +} + +static void xgene_dma_prep_cpy_desc(struct xgene_dma_chan *chan, + struct xgene_dma_desc_sw *desc_sw, + dma_addr_t dst, dma_addr_t src, + size_t len) +{ + void *desc1, *desc2; + int i; + + /* Get 1st descriptor */ + desc1 = &desc_sw->desc1; + xgene_dma_init_desc(desc1, chan->tx_ring.dst_ring_num); + + /* Set destination address */ + XGENE_DMA_DESC_DR_SET(desc1); + XGENE_DMA_DESC_DST_ADDR_SET(desc1, dst); + + /* Set 1st source address */ + xgene_dma_set_src_buffer(desc1 + 8, &len, &src); + + if (len <= 0) { + desc2 = NULL; + goto skip_additional_src; + } + + /* + * We need to split this source buffer, + * and need to use 2nd descriptor + */ + desc2 = &desc_sw->desc2; + XGENE_DMA_DESC_NV_SET(desc1); + + /* Set 2nd to 5th source address */ + for (i = 0; i < 4 && len; i++) + xgene_dma_set_src_buffer(xgene_dma_lookup_ext8(desc2, i), + &len, &src); + + /* Invalidate unused source address field */ + for (; i < 4; i++) + xgene_dma_invalidate_buffer(xgene_dma_lookup_ext8(desc2, i)); + + /* Updated flag that we have prepared 64B descriptor */ + desc_sw->flags |= XGENE_DMA_FLAG_64B_DESC; + +skip_additional_src: + /* Hardware stores descriptor in little endian format */ + xgene_dma_cpu_to_le64(desc1, 4); + if (desc2) + xgene_dma_cpu_to_le64(desc2, 4); +} + +static void xgene_dma_prep_xor_desc(struct xgene_dma_chan *chan, + struct xgene_dma_desc_sw *desc_sw, + dma_addr_t *dst, dma_addr_t *src, + u32 src_cnt, size_t *nbytes, + const u8 *scf) +{ + void *desc1, *desc2; + size_t len = *nbytes; + int i; + + desc1 = &desc_sw->desc1; + desc2 = &desc_sw->desc2; + + /* Initialize DMA descriptor */ + xgene_dma_init_desc(desc1, chan->tx_ring.dst_ring_num); + + /* Set destination address */ + XGENE_DMA_DESC_DR_SET(desc1); + XGENE_DMA_DESC_DST_ADDR_SET(desc1, *dst); + + /* We have multiple source addresses, so need to set NV bit*/ + XGENE_DMA_DESC_NV_SET(desc1); + + /* Set flyby opcode */ + XGENE_DMA_DESC_FLYBY_SET(desc1, xgene_dma_encode_xor_flyby(src_cnt)); + + /* Set 1st to 5th source addresses */ + for (i = 0; i < src_cnt; i++) { + len = *nbytes; + xgene_dma_set_src_buffer((i == 0) ? (desc1 + 8) : + xgene_dma_lookup_ext8(desc2, i - 1), + &len, &src[i]); + XGENE_DMA_DESC_MULTI_SET(desc1, scf[i], i); + } + + /* Hardware stores descriptor in little endian format */ + xgene_dma_cpu_to_le64(desc1, 4); + xgene_dma_cpu_to_le64(desc2, 4); + + /* Update meta data */ + *nbytes = len; + *dst += XGENE_DMA_MAX_BYTE_CNT; + + /* We need always 64B descriptor to perform xor or pq operations */ + desc_sw->flags |= XGENE_DMA_FLAG_64B_DESC; +} + +static dma_cookie_t xgene_dma_tx_submit(struct dma_async_tx_descriptor *tx) +{ + struct xgene_dma_desc_sw *desc; + struct xgene_dma_chan *chan; + dma_cookie_t cookie; + + if (unlikely(!tx)) + return -EINVAL; + + chan = to_dma_chan(tx->chan); + desc = to_dma_desc_sw(tx); + + spin_lock_bh(&chan->lock); + + cookie = dma_cookie_assign(tx); + + /* Add this transaction list onto the tail of the pending queue */ + list_splice_tail_init(&desc->tx_list, &chan->ld_pending); + + spin_unlock_bh(&chan->lock); + + return cookie; +} + +static void xgene_dma_clean_descriptor(struct xgene_dma_chan *chan, + struct xgene_dma_desc_sw *desc) +{ + list_del(&desc->node); + chan_dbg(chan, "LD %p free\n", desc); + dma_pool_free(chan->desc_pool, desc, desc->tx.phys); +} + +static struct xgene_dma_desc_sw *xgene_dma_alloc_descriptor( + struct xgene_dma_chan *chan) +{ + struct xgene_dma_desc_sw *desc; + dma_addr_t phys; + + desc = dma_pool_alloc(chan->desc_pool, GFP_NOWAIT, &phys); + if (!desc) { + chan_err(chan, "Failed to allocate LDs\n"); + return NULL; + } + + memset(desc, 0, sizeof(*desc)); + + INIT_LIST_HEAD(&desc->tx_list); + desc->tx.phys = phys; + desc->tx.tx_submit = xgene_dma_tx_submit; + dma_async_tx_descriptor_init(&desc->tx, &chan->dma_chan); + + chan_dbg(chan, "LD %p allocated\n", desc); + + return desc; +} + +/** + * xgene_dma_clean_completed_descriptor - free all descriptors which + * has been completed and acked + * @chan: X-Gene DMA channel + * + * This function is used on all completed and acked descriptors. + */ +static void xgene_dma_clean_completed_descriptor(struct xgene_dma_chan *chan) +{ + struct xgene_dma_desc_sw *desc, *_desc; + + /* Run the callback for each descriptor, in order */ + list_for_each_entry_safe(desc, _desc, &chan->ld_completed, node) { + if (async_tx_test_ack(&desc->tx)) + xgene_dma_clean_descriptor(chan, desc); + } +} + +/** + * xgene_dma_run_tx_complete_actions - cleanup a single link descriptor + * @chan: X-Gene DMA channel + * @desc: descriptor to cleanup and free + * + * This function is used on a descriptor which has been executed by the DMA + * controller. It will run any callbacks, submit any dependencies. + */ +static void xgene_dma_run_tx_complete_actions(struct xgene_dma_chan *chan, + struct xgene_dma_desc_sw *desc) +{ + struct dma_async_tx_descriptor *tx = &desc->tx; + + /* + * If this is not the last transaction in the group, + * then no need to complete cookie and run any callback as + * this is not the tx_descriptor which had been sent to caller + * of this DMA request + */ + + if (tx->cookie == 0) + return; + + dma_cookie_complete(tx); + + /* Run the link descriptor callback function */ + if (tx->callback) + tx->callback(tx->callback_param); + + dma_descriptor_unmap(tx); + + /* Run any dependencies */ + dma_run_dependencies(tx); +} + +/** + * xgene_dma_clean_running_descriptor - move the completed descriptor from + * ld_running to ld_completed + * @chan: X-Gene DMA channel + * @desc: the descriptor which is completed + * + * Free the descriptor directly if acked by async_tx api, + * else move it to queue ld_completed. + */ +static void xgene_dma_clean_running_descriptor(struct xgene_dma_chan *chan, + struct xgene_dma_desc_sw *desc) +{ + /* Remove from the list of running transactions */ + list_del(&desc->node); + + /* + * the client is allowed to attach dependent operations + * until 'ack' is set + */ + if (!async_tx_test_ack(&desc->tx)) { + /* + * Move this descriptor to the list of descriptors which is + * completed, but still awaiting the 'ack' bit to be set. + */ + list_add_tail(&desc->node, &chan->ld_completed); + return; + } + + chan_dbg(chan, "LD %p free\n", desc); + dma_pool_free(chan->desc_pool, desc, desc->tx.phys); +} + +static int xgene_chan_xfer_request(struct xgene_dma_ring *ring, + struct xgene_dma_desc_sw *desc_sw) +{ + struct xgene_dma_desc_hw *desc_hw; + + /* Check if can push more descriptor to hw for execution */ + if (xgene_dma_ring_desc_cnt(ring) > (ring->slots - 2)) + return -EBUSY; + + /* Get hw descriptor from DMA tx ring */ + desc_hw = &ring->desc_hw[ring->head]; + + /* + * Increment the head count to point next + * descriptor for next time + */ + if (++ring->head == ring->slots) + ring->head = 0; + + /* Copy prepared sw descriptor data to hw descriptor */ + memcpy(desc_hw, &desc_sw->desc1, sizeof(*desc_hw)); + + /* + * Check if we have prepared 64B descriptor, + * in this case we need one more hw descriptor + */ + if (desc_sw->flags & XGENE_DMA_FLAG_64B_DESC) { + desc_hw = &ring->desc_hw[ring->head]; + + if (++ring->head == ring->slots) + ring->head = 0; + + memcpy(desc_hw, &desc_sw->desc2, sizeof(*desc_hw)); + } + + /* Notify the hw that we have descriptor ready for execution */ + iowrite32((desc_sw->flags & XGENE_DMA_FLAG_64B_DESC) ? + 2 : 1, ring->cmd); + + return 0; +} + +/** + * xgene_chan_xfer_ld_pending - push any pending transactions to hw + * @chan : X-Gene DMA channel + * + * LOCKING: must hold chan->desc_lock + */ +static void xgene_chan_xfer_ld_pending(struct xgene_dma_chan *chan) +{ + struct xgene_dma_desc_sw *desc_sw, *_desc_sw; + int ret; + + /* + * If the list of pending descriptors is empty, then we + * don't need to do any work at all + */ + if (list_empty(&chan->ld_pending)) { + chan_dbg(chan, "No pending LDs\n"); + return; + } + + /* + * Move elements from the queue of pending transactions onto the list + * of running transactions and push it to hw for further executions + */ + list_for_each_entry_safe(desc_sw, _desc_sw, &chan->ld_pending, node) { + /* + * Check if have pushed max number of transactions to hw + * as capable, so let's stop here and will push remaining + * elements from pening ld queue after completing some + * descriptors that we have already pushed + */ + if (chan->pending >= chan->max_outstanding) + return; + + ret = xgene_chan_xfer_request(&chan->tx_ring, desc_sw); + if (ret) + return; + + /* + * Delete this element from ld pending queue and append it to + * ld running queue + */ + list_move_tail(&desc_sw->node, &chan->ld_running); + + /* Increment the pending transaction count */ + chan->pending++; + } +} + +/** + * xgene_dma_cleanup_descriptors - cleanup link descriptors which are completed + * and move them to ld_completed to free until flag 'ack' is set + * @chan: X-Gene DMA channel + * + * This function is used on descriptors which have been executed by the DMA + * controller. It will run any callbacks, submit any dependencies, then + * free these descriptors if flag 'ack' is set. + */ +static void xgene_dma_cleanup_descriptors(struct xgene_dma_chan *chan) +{ + struct xgene_dma_ring *ring = &chan->rx_ring; + struct xgene_dma_desc_sw *desc_sw, *_desc_sw; + struct xgene_dma_desc_hw *desc_hw; + u8 status; + + /* Clean already completed and acked descriptors */ + xgene_dma_clean_completed_descriptor(chan); + + /* Run the callback for each descriptor, in order */ + list_for_each_entry_safe(desc_sw, _desc_sw, &chan->ld_running, node) { + /* Get subsequent hw descriptor from DMA rx ring */ + desc_hw = &ring->desc_hw[ring->head]; + + /* Check if this descriptor has been completed */ + if (unlikely(XGENE_DMA_DESC_IS_EMPTY(desc_hw))) + break; + + if (++ring->head == ring->slots) + ring->head = 0; + + /* Check if we have any error with DMA transactions */ + status = XGENE_DMA_DESC_STATUS( + XGENE_DMA_DESC_ELERR_RD(le64_to_cpu( + desc_hw->m0)), + XGENE_DMA_DESC_LERR_RD(le64_to_cpu( + desc_hw->m0))); + if (status) { + /* Print the DMA error type */ + chan_err(chan, "%s\n", xgene_dma_desc_err[status]); + + /* + * We have DMA transactions error here. Dump DMA Tx + * and Rx descriptors for this request */ + XGENE_DMA_DESC_DUMP(&desc_sw->desc1, + "X-Gene DMA TX DESC1: "); + + if (desc_sw->flags & XGENE_DMA_FLAG_64B_DESC) + XGENE_DMA_DESC_DUMP(&desc_sw->desc2, + "X-Gene DMA TX DESC2: "); + + XGENE_DMA_DESC_DUMP(desc_hw, + "X-Gene DMA RX ERR DESC: "); + } + + /* Notify the hw about this completed descriptor */ + iowrite32(-1, ring->cmd); + + /* Mark this hw descriptor as processed */ + XGENE_DMA_DESC_SET_EMPTY(desc_hw); + + xgene_dma_run_tx_complete_actions(chan, desc_sw); + + xgene_dma_clean_running_descriptor(chan, desc_sw); + + /* + * Decrement the pending transaction count + * as we have processed one + */ + chan->pending--; + } + + /* + * Start any pending transactions automatically + * In the ideal case, we keep the DMA controller busy while we go + * ahead and free the descriptors below. + */ + xgene_chan_xfer_ld_pending(chan); +} + +static int xgene_dma_alloc_chan_resources(struct dma_chan *dchan) +{ + struct xgene_dma_chan *chan = to_dma_chan(dchan); + + /* Has this channel already been allocated? */ + if (chan->desc_pool) + return 1; + + chan->desc_pool = dma_pool_create(chan->name, chan->dev, + sizeof(struct xgene_dma_desc_sw), + 0, 0); + if (!chan->desc_pool) { + chan_err(chan, "Failed to allocate descriptor pool\n"); + return -ENOMEM; + } + + chan_dbg(chan, "Allocate descripto pool\n"); + + return 1; +} + +/** + * xgene_dma_free_desc_list - Free all descriptors in a queue + * @chan: X-Gene DMA channel + * @list: the list to free + * + * LOCKING: must hold chan->desc_lock + */ +static void xgene_dma_free_desc_list(struct xgene_dma_chan *chan, + struct list_head *list) +{ + struct xgene_dma_desc_sw *desc, *_desc; + + list_for_each_entry_safe(desc, _desc, list, node) + xgene_dma_clean_descriptor(chan, desc); +} + +static void xgene_dma_free_tx_desc_list(struct xgene_dma_chan *chan, + struct list_head *list) +{ + struct xgene_dma_desc_sw *desc, *_desc; + + list_for_each_entry_safe(desc, _desc, list, node) + xgene_dma_clean_descriptor(chan, desc); +} + +static void xgene_dma_free_chan_resources(struct dma_chan *dchan) +{ + struct xgene_dma_chan *chan = to_dma_chan(dchan); + + chan_dbg(chan, "Free all resources\n"); + + if (!chan->desc_pool) + return; + + spin_lock_bh(&chan->lock); + + /* Process all running descriptor */ + xgene_dma_cleanup_descriptors(chan); + + /* Clean all link descriptor queues */ + xgene_dma_free_desc_list(chan, &chan->ld_pending); + xgene_dma_free_desc_list(chan, &chan->ld_running); + xgene_dma_free_desc_list(chan, &chan->ld_completed); + + spin_unlock_bh(&chan->lock); + + /* Delete this channel DMA pool */ + dma_pool_destroy(chan->desc_pool); + chan->desc_pool = NULL; +} + +static struct dma_async_tx_descriptor *xgene_dma_prep_memcpy( + struct dma_chan *dchan, dma_addr_t dst, dma_addr_t src, + size_t len, unsigned long flags) +{ + struct xgene_dma_desc_sw *first = NULL, *new; + struct xgene_dma_chan *chan; + size_t copy; + + if (unlikely(!dchan || !len)) + return NULL; + + chan = to_dma_chan(dchan); + + do { + /* Allocate the link descriptor from DMA pool */ + new = xgene_dma_alloc_descriptor(chan); + if (!new) + goto fail; + + /* Create the largest transaction possible */ + copy = min_t(size_t, len, XGENE_DMA_MAX_64B_DESC_BYTE_CNT); + + /* Prepare DMA descriptor */ + xgene_dma_prep_cpy_desc(chan, new, dst, src, copy); + + if (!first) + first = new; + + new->tx.cookie = 0; + async_tx_ack(&new->tx); + + /* Update metadata */ + len -= copy; + dst += copy; + src += copy; + + /* Insert the link descriptor to the LD ring */ + list_add_tail(&new->node, &first->tx_list); + } while (len); + + new->tx.flags = flags; /* client is in control of this ack */ + new->tx.cookie = -EBUSY; + list_splice(&first->tx_list, &new->tx_list); + + return &new->tx; + +fail: + if (!first) + return NULL; + + xgene_dma_free_tx_desc_list(chan, &first->tx_list); + return NULL; +} + +static struct dma_async_tx_descriptor *xgene_dma_prep_sg( + struct dma_chan *dchan, struct scatterlist *dst_sg, + u32 dst_nents, struct scatterlist *src_sg, + u32 src_nents, unsigned long flags) +{ + struct xgene_dma_desc_sw *first = NULL, *new = NULL; + struct xgene_dma_chan *chan; + size_t dst_avail, src_avail; + dma_addr_t dst, src; + size_t len; + + if (unlikely(!dchan)) + return NULL; + + if (unlikely(!dst_nents || !src_nents)) + return NULL; + + if (unlikely(!dst_sg || !src_sg)) + return NULL; + + chan = to_dma_chan(dchan); + + /* Get prepared for the loop */ + dst_avail = sg_dma_len(dst_sg); + src_avail = sg_dma_len(src_sg); + dst_nents--; + src_nents--; + + /* Run until we are out of scatterlist entries */ + while (true) { + /* Create the largest transaction possible */ + len = min_t(size_t, src_avail, dst_avail); + len = min_t(size_t, len, XGENE_DMA_MAX_64B_DESC_BYTE_CNT); + if (len == 0) + goto fetch; + + dst = sg_dma_address(dst_sg) + sg_dma_len(dst_sg) - dst_avail; + src = sg_dma_address(src_sg) + sg_dma_len(src_sg) - src_avail; + + /* Allocate the link descriptor from DMA pool */ + new = xgene_dma_alloc_descriptor(chan); + if (!new) + goto fail; + + /* Prepare DMA descriptor */ + xgene_dma_prep_cpy_desc(chan, new, dst, src, len); + + if (!first) + first = new; + + new->tx.cookie = 0; + async_tx_ack(&new->tx); + + /* update metadata */ + dst_avail -= len; + src_avail -= len; + + /* Insert the link descriptor to the LD ring */ + list_add_tail(&new->node, &first->tx_list); + +fetch: + /* fetch the next dst scatterlist entry */ + if (dst_avail == 0) { + /* no more entries: we're done */ + if (dst_nents == 0) + break; + + /* fetch the next entry: if there are no more: done */ + dst_sg = sg_next(dst_sg); + if (!dst_sg) + break; + + dst_nents--; + dst_avail = sg_dma_len(dst_sg); + } + + /* fetch the next src scatterlist entry */ + if (src_avail == 0) { + /* no more entries: we're done */ + if (src_nents == 0) + break; + + /* fetch the next entry: if there are no more: done */ + src_sg = sg_next(src_sg); + if (!src_sg) + break; + + src_nents--; + src_avail = sg_dma_len(src_sg); + } + } + + if (!new) + return NULL; + + new->tx.flags = flags; /* client is in control of this ack */ + new->tx.cookie = -EBUSY; + list_splice(&first->tx_list, &new->tx_list); + + return &new->tx; +fail: + if (!first) + return NULL; + + xgene_dma_free_tx_desc_list(chan, &first->tx_list); + return NULL; +} + +static struct dma_async_tx_descriptor *xgene_dma_prep_xor( + struct dma_chan *dchan, dma_addr_t dst, dma_addr_t *src, + u32 src_cnt, size_t len, unsigned long flags) +{ + struct xgene_dma_desc_sw *first = NULL, *new; + struct xgene_dma_chan *chan; + static u8 multi[XGENE_DMA_MAX_XOR_SRC] = { + 0x01, 0x01, 0x01, 0x01, 0x01}; + + if (unlikely(!dchan || !len)) + return NULL; + + chan = to_dma_chan(dchan); + + do { + /* Allocate the link descriptor from DMA pool */ + new = xgene_dma_alloc_descriptor(chan); + if (!new) + goto fail; + + /* Prepare xor DMA descriptor */ + xgene_dma_prep_xor_desc(chan, new, &dst, src, + src_cnt, &len, multi); + + if (!first) + first = new; + + new->tx.cookie = 0; + async_tx_ack(&new->tx); + + /* Insert the link descriptor to the LD ring */ + list_add_tail(&new->node, &first->tx_list); + } while (len); + + new->tx.flags = flags; /* client is in control of this ack */ + new->tx.cookie = -EBUSY; + list_splice(&first->tx_list, &new->tx_list); + + return &new->tx; + +fail: + if (!first) + return NULL; + + xgene_dma_free_tx_desc_list(chan, &first->tx_list); + return NULL; +} + +static struct dma_async_tx_descriptor *xgene_dma_prep_pq( + struct dma_chan *dchan, dma_addr_t *dst, dma_addr_t *src, + u32 src_cnt, const u8 *scf, size_t len, unsigned long flags) +{ + struct xgene_dma_desc_sw *first = NULL, *new; + struct xgene_dma_chan *chan; + size_t _len = len; + dma_addr_t _src[XGENE_DMA_MAX_XOR_SRC]; + static u8 multi[XGENE_DMA_MAX_XOR_SRC] = {0x01, 0x01, 0x01, 0x01, 0x01}; + + if (unlikely(!dchan || !len)) + return NULL; + + chan = to_dma_chan(dchan); + + /* + * Save source addresses on local variable, may be we have to + * prepare two descriptor to generate P and Q if both enabled + * in the flags by client + */ + memcpy(_src, src, sizeof(*src) * src_cnt); + + if (flags & DMA_PREP_PQ_DISABLE_P) + len = 0; + + if (flags & DMA_PREP_PQ_DISABLE_Q) + _len = 0; + + do { + /* Allocate the link descriptor from DMA pool */ + new = xgene_dma_alloc_descriptor(chan); + if (!new) + goto fail; + + if (!first) + first = new; + + new->tx.cookie = 0; + async_tx_ack(&new->tx); + + /* Insert the link descriptor to the LD ring */ + list_add_tail(&new->node, &first->tx_list); + + /* + * Prepare DMA descriptor to generate P, + * if DMA_PREP_PQ_DISABLE_P flag is not set + */ + if (len) { + xgene_dma_prep_xor_desc(chan, new, &dst[0], src, + src_cnt, &len, multi); + continue; + } + + /* + * Prepare DMA descriptor to generate Q, + * if DMA_PREP_PQ_DISABLE_Q flag is not set + */ + if (_len) { + xgene_dma_prep_xor_desc(chan, new, &dst[1], _src, + src_cnt, &_len, scf); + } + } while (len || _len); + + new->tx.flags = flags; /* client is in control of this ack */ + new->tx.cookie = -EBUSY; + list_splice(&first->tx_list, &new->tx_list); + + return &new->tx; + +fail: + if (!first) + return NULL; + + xgene_dma_free_tx_desc_list(chan, &first->tx_list); + return NULL; +} + +static void xgene_dma_issue_pending(struct dma_chan *dchan) +{ + struct xgene_dma_chan *chan = to_dma_chan(dchan); + + spin_lock_bh(&chan->lock); + xgene_chan_xfer_ld_pending(chan); + spin_unlock_bh(&chan->lock); +} + +static enum dma_status xgene_dma_tx_status(struct dma_chan *dchan, + dma_cookie_t cookie, + struct dma_tx_state *txstate) +{ + return dma_cookie_status(dchan, cookie, txstate); +} + +static void xgene_dma_tasklet_cb(unsigned long data) +{ + struct xgene_dma_chan *chan = (struct xgene_dma_chan *)data; + + spin_lock_bh(&chan->lock); + + /* Run all cleanup for descriptors which have been completed */ + xgene_dma_cleanup_descriptors(chan); + + /* Re-enable DMA channel IRQ */ + enable_irq(chan->rx_irq); + + spin_unlock_bh(&chan->lock); +} + +static irqreturn_t xgene_dma_chan_ring_isr(int irq, void *id) +{ + struct xgene_dma_chan *chan = (struct xgene_dma_chan *)id; + + BUG_ON(!chan); + + /* + * Disable DMA channel IRQ until we process completed + * descriptors + */ + disable_irq_nosync(chan->rx_irq); + + /* + * Schedule the tasklet to handle all cleanup of the current + * transaction. It will start a new transaction if there is + * one pending. + */ + tasklet_schedule(&chan->tasklet); + + return IRQ_HANDLED; +} + +static irqreturn_t xgene_dma_err_isr(int irq, void *id) +{ + struct xgene_dma *pdma = (struct xgene_dma *)id; + unsigned long int_mask; + u32 val, i; + + val = ioread32(pdma->csr_dma + XGENE_DMA_INT); + + /* Clear DMA interrupts */ + iowrite32(val, pdma->csr_dma + XGENE_DMA_INT); + + /* Print DMA error info */ + int_mask = val >> XGENE_DMA_INT_MASK_SHIFT; + for_each_set_bit(i, &int_mask, ARRAY_SIZE(xgene_dma_err)) + dev_err(pdma->dev, + "Interrupt status 0x%08X %s\n", val, xgene_dma_err[i]); + + return IRQ_HANDLED; +} + +static void xgene_dma_wr_ring_state(struct xgene_dma_ring *ring) +{ + int i; + + iowrite32(ring->num, ring->pdma->csr_ring + XGENE_DMA_RING_STATE); + + for (i = 0; i < XGENE_DMA_RING_NUM_CONFIG; i++) + iowrite32(ring->state[i], ring->pdma->csr_ring + + XGENE_DMA_RING_STATE_WR_BASE + (i * 4)); +} + +static void xgene_dma_clr_ring_state(struct xgene_dma_ring *ring) +{ + memset(ring->state, 0, sizeof(u32) * XGENE_DMA_RING_NUM_CONFIG); + xgene_dma_wr_ring_state(ring); +} + +static void xgene_dma_setup_ring(struct xgene_dma_ring *ring) +{ + void *ring_cfg = ring->state; + u64 addr = ring->desc_paddr; + void *desc; + u32 i, val; + + ring->slots = ring->size / XGENE_DMA_RING_WQ_DESC_SIZE; + + /* Clear DMA ring state */ + xgene_dma_clr_ring_state(ring); + + /* Set DMA ring type */ + XGENE_DMA_RING_TYPE_SET(ring_cfg, XGENE_DMA_RING_TYPE_REGULAR); + + if (ring->owner == XGENE_DMA_RING_OWNER_DMA) { + /* Set recombination buffer and timeout */ + XGENE_DMA_RING_RECOMBBUF_SET(ring_cfg); + XGENE_DMA_RING_RECOMTIMEOUTL_SET(ring_cfg); + XGENE_DMA_RING_RECOMTIMEOUTH_SET(ring_cfg); + } + + /* Initialize DMA ring state */ + XGENE_DMA_RING_SELTHRSH_SET(ring_cfg); + XGENE_DMA_RING_ACCEPTLERR_SET(ring_cfg); + XGENE_DMA_RING_COHERENT_SET(ring_cfg); + XGENE_DMA_RING_ADDRL_SET(ring_cfg, addr); + XGENE_DMA_RING_ADDRH_SET(ring_cfg, addr); + XGENE_DMA_RING_SIZE_SET(ring_cfg, ring->cfgsize); + + /* Write DMA ring configurations */ + xgene_dma_wr_ring_state(ring); + + /* Set DMA ring id */ + iowrite32(XGENE_DMA_RING_ID_SETUP(ring->id), + ring->pdma->csr_ring + XGENE_DMA_RING_ID); + + /* Set DMA ring buffer */ + iowrite32(XGENE_DMA_RING_ID_BUF_SETUP(ring->num), + ring->pdma->csr_ring + XGENE_DMA_RING_ID_BUF); + + if (ring->owner != XGENE_DMA_RING_OWNER_CPU) + return; + + /* Set empty signature to DMA Rx ring descriptors */ + for (i = 0; i < ring->slots; i++) { + desc = &ring->desc_hw[i]; + XGENE_DMA_DESC_SET_EMPTY(desc); + } + + /* Enable DMA Rx ring interrupt */ + val = ioread32(ring->pdma->csr_ring + XGENE_DMA_RING_NE_INT_MODE); + XGENE_DMA_RING_NE_INT_MODE_SET(val, ring->buf_num); + iowrite32(val, ring->pdma->csr_ring + XGENE_DMA_RING_NE_INT_MODE); +} + +static void xgene_dma_clear_ring(struct xgene_dma_ring *ring) +{ + u32 ring_id, val; + + if (ring->owner == XGENE_DMA_RING_OWNER_CPU) { + /* Disable DMA Rx ring interrupt */ + val = ioread32(ring->pdma->csr_ring + + XGENE_DMA_RING_NE_INT_MODE); + XGENE_DMA_RING_NE_INT_MODE_RESET(val, ring->buf_num); + iowrite32(val, ring->pdma->csr_ring + + XGENE_DMA_RING_NE_INT_MODE); + } + + /* Clear DMA ring state */ + ring_id = XGENE_DMA_RING_ID_SETUP(ring->id); + iowrite32(ring_id, ring->pdma->csr_ring + XGENE_DMA_RING_ID); + + iowrite32(0, ring->pdma->csr_ring + XGENE_DMA_RING_ID_BUF); + xgene_dma_clr_ring_state(ring); +} + +static void xgene_dma_set_ring_cmd(struct xgene_dma_ring *ring) +{ + ring->cmd_base = ring->pdma->csr_ring_cmd + + XGENE_DMA_RING_CMD_BASE_OFFSET((ring->num - + XGENE_DMA_RING_NUM)); + + ring->cmd = ring->cmd_base + XGENE_DMA_RING_CMD_OFFSET; +} + +static int xgene_dma_get_ring_size(struct xgene_dma_chan *chan, + enum xgene_dma_ring_cfgsize cfgsize) +{ + int size; + + switch (cfgsize) { + case XGENE_DMA_RING_CFG_SIZE_512B: + size = 0x200; + break; + case XGENE_DMA_RING_CFG_SIZE_2KB: + size = 0x800; + break; + case XGENE_DMA_RING_CFG_SIZE_16KB: + size = 0x4000; + break; + case XGENE_DMA_RING_CFG_SIZE_64KB: + size = 0x10000; + break; + case XGENE_DMA_RING_CFG_SIZE_512KB: + size = 0x80000; + break; + default: + chan_err(chan, "Unsupported cfg ring size %d\n", cfgsize); + return -EINVAL; + } + + return size; +} + +static void xgene_dma_delete_ring_one(struct xgene_dma_ring *ring) +{ + /* Clear DMA ring configurations */ + xgene_dma_clear_ring(ring); + + /* De-allocate DMA ring descriptor */ + if (ring->desc_vaddr) { + dma_free_coherent(ring->pdma->dev, ring->size, + ring->desc_vaddr, ring->desc_paddr); + ring->desc_vaddr = NULL; + } +} + +static void xgene_dma_delete_chan_rings(struct xgene_dma_chan *chan) +{ + xgene_dma_delete_ring_one(&chan->rx_ring); + xgene_dma_delete_ring_one(&chan->tx_ring); +} + +static int xgene_dma_create_ring_one(struct xgene_dma_chan *chan, + struct xgene_dma_ring *ring, + enum xgene_dma_ring_cfgsize cfgsize) +{ + /* Setup DMA ring descriptor variables */ + ring->pdma = chan->pdma; + ring->cfgsize = cfgsize; + ring->num = chan->pdma->ring_num++; + ring->id = XGENE_DMA_RING_ID_GET(ring->owner, ring->buf_num); + + ring->size = xgene_dma_get_ring_size(chan, cfgsize); + if (ring->size <= 0) + return ring->size; + + /* Allocate memory for DMA ring descriptor */ + ring->desc_vaddr = dma_zalloc_coherent(chan->dev, ring->size, + &ring->desc_paddr, GFP_KERNEL); + if (!ring->desc_vaddr) { + chan_err(chan, "Failed to allocate ring desc\n"); + return -ENOMEM; + } + + /* Configure and enable DMA ring */ + xgene_dma_set_ring_cmd(ring); + xgene_dma_setup_ring(ring); + + return 0; +} + +static int xgene_dma_create_chan_rings(struct xgene_dma_chan *chan) +{ + struct xgene_dma_ring *rx_ring = &chan->rx_ring; + struct xgene_dma_ring *tx_ring = &chan->tx_ring; + int ret; + + /* Create DMA Rx ring descriptor */ + rx_ring->owner = XGENE_DMA_RING_OWNER_CPU; + rx_ring->buf_num = XGENE_DMA_CPU_BUFNUM + chan->id; + + ret = xgene_dma_create_ring_one(chan, rx_ring, + XGENE_DMA_RING_CFG_SIZE_64KB); + if (ret) + return ret; + + chan_dbg(chan, "Rx ring id 0x%X num %d desc 0x%p\n", + rx_ring->id, rx_ring->num, rx_ring->desc_vaddr); + + /* Create DMA Tx ring descriptor */ + tx_ring->owner = XGENE_DMA_RING_OWNER_DMA; + tx_ring->buf_num = XGENE_DMA_BUFNUM + chan->id; + + ret = xgene_dma_create_ring_one(chan, tx_ring, + XGENE_DMA_RING_CFG_SIZE_64KB); + if (ret) { + xgene_dma_delete_ring_one(rx_ring); + return ret; + } + + tx_ring->dst_ring_num = XGENE_DMA_RING_DST_ID(rx_ring->num); + + chan_dbg(chan, + "Tx ring id 0x%X num %d desc 0x%p\n", + tx_ring->id, tx_ring->num, tx_ring->desc_vaddr); + + /* Set the max outstanding request possible to this channel */ + chan->max_outstanding = rx_ring->slots; + + return ret; +} + +static int xgene_dma_init_rings(struct xgene_dma *pdma) +{ + int ret, i, j; + + for (i = 0; i < XGENE_DMA_MAX_CHANNEL; i++) { + ret = xgene_dma_create_chan_rings(&pdma->chan[i]); + if (ret) { + for (j = 0; j < i; j++) + xgene_dma_delete_chan_rings(&pdma->chan[j]); + return ret; + } + } + + return ret; +} + +static void xgene_dma_enable(struct xgene_dma *pdma) +{ + u32 val; + + /* Configure and enable DMA engine */ + val = ioread32(pdma->csr_dma + XGENE_DMA_GCR); + XGENE_DMA_CH_SETUP(val); + XGENE_DMA_ENABLE(val); + iowrite32(val, pdma->csr_dma + XGENE_DMA_GCR); +} + +static void xgene_dma_disable(struct xgene_dma *pdma) +{ + u32 val; + + val = ioread32(pdma->csr_dma + XGENE_DMA_GCR); + XGENE_DMA_DISABLE(val); + iowrite32(val, pdma->csr_dma + XGENE_DMA_GCR); +} + +static void xgene_dma_mask_interrupts(struct xgene_dma *pdma) +{ + /* + * Mask DMA ring overflow, underflow and + * AXI write/read error interrupts + */ + iowrite32(XGENE_DMA_INT_ALL_MASK, + pdma->csr_dma + XGENE_DMA_RING_INT0_MASK); + iowrite32(XGENE_DMA_INT_ALL_MASK, + pdma->csr_dma + XGENE_DMA_RING_INT1_MASK); + iowrite32(XGENE_DMA_INT_ALL_MASK, + pdma->csr_dma + XGENE_DMA_RING_INT2_MASK); + iowrite32(XGENE_DMA_INT_ALL_MASK, + pdma->csr_dma + XGENE_DMA_RING_INT3_MASK); + iowrite32(XGENE_DMA_INT_ALL_MASK, + pdma->csr_dma + XGENE_DMA_RING_INT4_MASK); + + /* Mask DMA error interrupts */ + iowrite32(XGENE_DMA_INT_ALL_MASK, pdma->csr_dma + XGENE_DMA_INT_MASK); +} + +static void xgene_dma_unmask_interrupts(struct xgene_dma *pdma) +{ + /* + * Unmask DMA ring overflow, underflow and + * AXI write/read error interrupts + */ + iowrite32(XGENE_DMA_INT_ALL_UNMASK, + pdma->csr_dma + XGENE_DMA_RING_INT0_MASK); + iowrite32(XGENE_DMA_INT_ALL_UNMASK, + pdma->csr_dma + XGENE_DMA_RING_INT1_MASK); + iowrite32(XGENE_DMA_INT_ALL_UNMASK, + pdma->csr_dma + XGENE_DMA_RING_INT2_MASK); + iowrite32(XGENE_DMA_INT_ALL_UNMASK, + pdma->csr_dma + XGENE_DMA_RING_INT3_MASK); + iowrite32(XGENE_DMA_INT_ALL_UNMASK, + pdma->csr_dma + XGENE_DMA_RING_INT4_MASK); + + /* Unmask DMA error interrupts */ + iowrite32(XGENE_DMA_INT_ALL_UNMASK, + pdma->csr_dma + XGENE_DMA_INT_MASK); +} + +static void xgene_dma_init_hw(struct xgene_dma *pdma) +{ + u32 val; + + /* Associate DMA ring to corresponding ring HW */ + iowrite32(XGENE_DMA_ASSOC_RING_MNGR1, + pdma->csr_dma + XGENE_DMA_CFG_RING_WQ_ASSOC); + + /* Configure RAID6 polynomial control setting */ + if (is_pq_enabled(pdma)) + iowrite32(XGENE_DMA_RAID6_MULTI_CTRL(0x1D), + pdma->csr_dma + XGENE_DMA_RAID6_CONT); + else + dev_info(pdma->dev, "PQ is disabled in HW\n"); + + xgene_dma_enable(pdma); + xgene_dma_unmask_interrupts(pdma); + + /* Get DMA id and version info */ + val = ioread32(pdma->csr_dma + XGENE_DMA_IPBRR); + + /* DMA device info */ + dev_info(pdma->dev, + "X-Gene DMA v%d.%02d.%02d driver registered %d channels", + XGENE_DMA_REV_NO_RD(val), XGENE_DMA_BUS_ID_RD(val), + XGENE_DMA_DEV_ID_RD(val), XGENE_DMA_MAX_CHANNEL); +} + +static int xgene_dma_init_ring_mngr(struct xgene_dma *pdma) +{ + if (ioread32(pdma->csr_ring + XGENE_DMA_RING_CLKEN) && + (!ioread32(pdma->csr_ring + XGENE_DMA_RING_SRST))) + return 0; + + iowrite32(0x3, pdma->csr_ring + XGENE_DMA_RING_CLKEN); + iowrite32(0x0, pdma->csr_ring + XGENE_DMA_RING_SRST); + + /* Bring up memory */ + iowrite32(0x0, pdma->csr_ring + XGENE_DMA_RING_MEM_RAM_SHUTDOWN); + + /* Force a barrier */ + ioread32(pdma->csr_ring + XGENE_DMA_RING_MEM_RAM_SHUTDOWN); + + /* reset may take up to 1ms */ + usleep_range(1000, 1100); + + if (ioread32(pdma->csr_ring + XGENE_DMA_RING_BLK_MEM_RDY) + != XGENE_DMA_RING_BLK_MEM_RDY_VAL) { + dev_err(pdma->dev, + "Failed to release ring mngr memory from shutdown\n"); + return -ENODEV; + } + + /* program threshold set 1 and all hysteresis */ + iowrite32(XGENE_DMA_RING_THRESLD0_SET1_VAL, + pdma->csr_ring + XGENE_DMA_RING_THRESLD0_SET1); + iowrite32(XGENE_DMA_RING_THRESLD1_SET1_VAL, + pdma->csr_ring + XGENE_DMA_RING_THRESLD1_SET1); + iowrite32(XGENE_DMA_RING_HYSTERESIS_VAL, + pdma->csr_ring + XGENE_DMA_RING_HYSTERESIS); + + /* Enable QPcore and assign error queue */ + iowrite32(XGENE_DMA_RING_ENABLE, + pdma->csr_ring + XGENE_DMA_RING_CONFIG); + + return 0; +} + +static int xgene_dma_init_mem(struct xgene_dma *pdma) +{ + int ret; + + ret = xgene_dma_init_ring_mngr(pdma); + if (ret) + return ret; + + /* Bring up memory */ + iowrite32(0x0, pdma->csr_dma + XGENE_DMA_MEM_RAM_SHUTDOWN); + + /* Force a barrier */ + ioread32(pdma->csr_dma + XGENE_DMA_MEM_RAM_SHUTDOWN); + + /* reset may take up to 1ms */ + usleep_range(1000, 1100); + + if (ioread32(pdma->csr_dma + XGENE_DMA_BLK_MEM_RDY) + != XGENE_DMA_BLK_MEM_RDY_VAL) { + dev_err(pdma->dev, + "Failed to release DMA memory from shutdown\n"); + return -ENODEV; + } + + return 0; +} + +static int xgene_dma_request_irqs(struct xgene_dma *pdma) +{ + struct xgene_dma_chan *chan; + int ret, i, j; + + /* Register DMA error irq */ + ret = devm_request_irq(pdma->dev, pdma->err_irq, xgene_dma_err_isr, + 0, "dma_error", pdma); + if (ret) { + dev_err(pdma->dev, + "Failed to register error IRQ %d\n", pdma->err_irq); + return ret; + } + + /* Register DMA channel rx irq */ + for (i = 0; i < XGENE_DMA_MAX_CHANNEL; i++) { + chan = &pdma->chan[i]; + ret = devm_request_irq(chan->dev, chan->rx_irq, + xgene_dma_chan_ring_isr, + 0, chan->name, chan); + if (ret) { + chan_err(chan, "Failed to register Rx IRQ %d\n", + chan->rx_irq); + devm_free_irq(pdma->dev, pdma->err_irq, pdma); + + for (j = 0; j < i; j++) { + chan = &pdma->chan[i]; + devm_free_irq(chan->dev, chan->rx_irq, chan); + } + + return ret; + } + } + + return 0; +} + +static void xgene_dma_free_irqs(struct xgene_dma *pdma) +{ + struct xgene_dma_chan *chan; + int i; + + /* Free DMA device error irq */ + devm_free_irq(pdma->dev, pdma->err_irq, pdma); + + for (i = 0; i < XGENE_DMA_MAX_CHANNEL; i++) { + chan = &pdma->chan[i]; + devm_free_irq(chan->dev, chan->rx_irq, chan); + } +} + +static void xgene_dma_set_caps(struct xgene_dma_chan *chan, + struct dma_device *dma_dev) +{ + /* Initialize DMA device capability mask */ + dma_cap_zero(dma_dev->cap_mask); + + /* Set DMA device capability */ + dma_cap_set(DMA_MEMCPY, dma_dev->cap_mask); + dma_cap_set(DMA_SG, dma_dev->cap_mask); + + /* Basically here, the X-Gene SoC DMA engine channel 0 supports XOR + * and channel 1 supports XOR, PQ both. First thing here is we have + * mechanism in hw to enable/disable PQ/XOR supports on channel 1, + * we can make sure this by reading SoC Efuse register. + * Second thing, we have hw errata that if we run channel 0 and + * channel 1 simultaneously with executing XOR and PQ request, + * suddenly DMA engine hangs, So here we enable XOR on channel 0 only + * if XOR and PQ supports on channel 1 is disabled. + */ + if ((chan->id == XGENE_DMA_PQ_CHANNEL) && + is_pq_enabled(chan->pdma)) { + dma_cap_set(DMA_PQ, dma_dev->cap_mask); + dma_cap_set(DMA_XOR, dma_dev->cap_mask); + } else if ((chan->id == XGENE_DMA_XOR_CHANNEL) && + !is_pq_enabled(chan->pdma)) { + dma_cap_set(DMA_XOR, dma_dev->cap_mask); + } + + /* Set base and prep routines */ + dma_dev->dev = chan->dev; + dma_dev->device_alloc_chan_resources = xgene_dma_alloc_chan_resources; + dma_dev->device_free_chan_resources = xgene_dma_free_chan_resources; + dma_dev->device_issue_pending = xgene_dma_issue_pending; + dma_dev->device_tx_status = xgene_dma_tx_status; + dma_dev->device_prep_dma_memcpy = xgene_dma_prep_memcpy; + dma_dev->device_prep_dma_sg = xgene_dma_prep_sg; + + if (dma_has_cap(DMA_XOR, dma_dev->cap_mask)) { + dma_dev->device_prep_dma_xor = xgene_dma_prep_xor; + dma_dev->max_xor = XGENE_DMA_MAX_XOR_SRC; + dma_dev->xor_align = XGENE_DMA_XOR_ALIGNMENT; + } + + if (dma_has_cap(DMA_PQ, dma_dev->cap_mask)) { + dma_dev->device_prep_dma_pq = xgene_dma_prep_pq; + dma_dev->max_pq = XGENE_DMA_MAX_XOR_SRC; + dma_dev->pq_align = XGENE_DMA_XOR_ALIGNMENT; + } +} + +static int xgene_dma_async_register(struct xgene_dma *pdma, int id) +{ + struct xgene_dma_chan *chan = &pdma->chan[id]; + struct dma_device *dma_dev = &pdma->dma_dev[id]; + int ret; + + chan->dma_chan.device = dma_dev; + + spin_lock_init(&chan->lock); + INIT_LIST_HEAD(&chan->ld_pending); + INIT_LIST_HEAD(&chan->ld_running); + INIT_LIST_HEAD(&chan->ld_completed); + tasklet_init(&chan->tasklet, xgene_dma_tasklet_cb, + (unsigned long)chan); + + chan->pending = 0; + chan->desc_pool = NULL; + dma_cookie_init(&chan->dma_chan); + + /* Setup dma device capabilities and prep routines */ + xgene_dma_set_caps(chan, dma_dev); + + /* Initialize DMA device list head */ + INIT_LIST_HEAD(&dma_dev->channels); + list_add_tail(&chan->dma_chan.device_node, &dma_dev->channels); + + /* Register with Linux async DMA framework*/ + ret = dma_async_device_register(dma_dev); + if (ret) { + chan_err(chan, "Failed to register async device %d", ret); + tasklet_kill(&chan->tasklet); + + return ret; + } + + /* DMA capability info */ + dev_info(pdma->dev, + "%s: CAPABILITY ( %s%s%s%s)\n", dma_chan_name(&chan->dma_chan), + dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask) ? "MEMCPY " : "", + dma_has_cap(DMA_SG, dma_dev->cap_mask) ? "SGCPY " : "", + dma_has_cap(DMA_XOR, dma_dev->cap_mask) ? "XOR " : "", + dma_has_cap(DMA_PQ, dma_dev->cap_mask) ? "PQ " : ""); + + return 0; +} + +static int xgene_dma_init_async(struct xgene_dma *pdma) +{ + int ret, i, j; + + for (i = 0; i < XGENE_DMA_MAX_CHANNEL ; i++) { + ret = xgene_dma_async_register(pdma, i); + if (ret) { + for (j = 0; j < i; j++) { + dma_async_device_unregister(&pdma->dma_dev[j]); + tasklet_kill(&pdma->chan[j].tasklet); + } + + return ret; + } + } + + return ret; +} + +static void xgene_dma_async_unregister(struct xgene_dma *pdma) +{ + int i; + + for (i = 0; i < XGENE_DMA_MAX_CHANNEL; i++) + dma_async_device_unregister(&pdma->dma_dev[i]); +} + +static void xgene_dma_init_channels(struct xgene_dma *pdma) +{ + struct xgene_dma_chan *chan; + int i; + + pdma->ring_num = XGENE_DMA_RING_NUM; + + for (i = 0; i < XGENE_DMA_MAX_CHANNEL; i++) { + chan = &pdma->chan[i]; + chan->dev = pdma->dev; + chan->pdma = pdma; + chan->id = i; + snprintf(chan->name, sizeof(chan->name), "dmachan%d", chan->id); + } +} + +static int xgene_dma_get_resources(struct platform_device *pdev, + struct xgene_dma *pdma) +{ + struct resource *res; + int irq, i; + + /* Get DMA csr region */ + res = platform_get_resource(pdev, IORESOURCE_MEM, 0); + if (!res) { + dev_err(&pdev->dev, "Failed to get csr region\n"); + return -ENXIO; + } + + pdma->csr_dma = devm_ioremap(&pdev->dev, res->start, + resource_size(res)); + if (!pdma->csr_dma) { + dev_err(&pdev->dev, "Failed to ioremap csr region"); + return -ENOMEM; + } + + /* Get DMA ring csr region */ + res = platform_get_resource(pdev, IORESOURCE_MEM, 1); + if (!res) { + dev_err(&pdev->dev, "Failed to get ring csr region\n"); + return -ENXIO; + } + + pdma->csr_ring = devm_ioremap(&pdev->dev, res->start, + resource_size(res)); + if (!pdma->csr_ring) { + dev_err(&pdev->dev, "Failed to ioremap ring csr region"); + return -ENOMEM; + } + + /* Get DMA ring cmd csr region */ + res = platform_get_resource(pdev, IORESOURCE_MEM, 2); + if (!res) { + dev_err(&pdev->dev, "Failed to get ring cmd csr region\n"); + return -ENXIO; + } + + pdma->csr_ring_cmd = devm_ioremap(&pdev->dev, res->start, + resource_size(res)); + if (!pdma->csr_ring_cmd) { + dev_err(&pdev->dev, "Failed to ioremap ring cmd csr region"); + return -ENOMEM; + } + + /* Get efuse csr region */ + res = platform_get_resource(pdev, IORESOURCE_MEM, 3); + if (!res) { + dev_err(&pdev->dev, "Failed to get efuse csr region\n"); + return -ENXIO; + } + + pdma->csr_efuse = devm_ioremap(&pdev->dev, res->start, + resource_size(res)); + if (!pdma->csr_efuse) { + dev_err(&pdev->dev, "Failed to ioremap efuse csr region"); + return -ENOMEM; + } + + /* Get DMA error interrupt */ + irq = platform_get_irq(pdev, 0); + if (irq <= 0) { + dev_err(&pdev->dev, "Failed to get Error IRQ\n"); + return -ENXIO; + } + + pdma->err_irq = irq; + + /* Get DMA Rx ring descriptor interrupts for all DMA channels */ + for (i = 1; i <= XGENE_DMA_MAX_CHANNEL; i++) { + irq = platform_get_irq(pdev, i); + if (irq <= 0) { + dev_err(&pdev->dev, "Failed to get Rx IRQ\n"); + return -ENXIO; + } + + pdma->chan[i - 1].rx_irq = irq; + } + + return 0; +} + +static int xgene_dma_probe(struct platform_device *pdev) +{ + struct xgene_dma *pdma; + int ret, i; + + pdma = devm_kzalloc(&pdev->dev, sizeof(*pdma), GFP_KERNEL); + if (!pdma) + return -ENOMEM; + + pdma->dev = &pdev->dev; + platform_set_drvdata(pdev, pdma); + + ret = xgene_dma_get_resources(pdev, pdma); + if (ret) + return ret; + + pdma->clk = devm_clk_get(&pdev->dev, NULL); + if (IS_ERR(pdma->clk)) { + dev_err(&pdev->dev, "Failed to get clk\n"); + return PTR_ERR(pdma->clk); + } + + /* Enable clk before accessing registers */ + ret = clk_prepare_enable(pdma->clk); + if (ret) { + dev_err(&pdev->dev, "Failed to enable clk %d\n", ret); + return ret; + } + + /* Remove DMA RAM out of shutdown */ + ret = xgene_dma_init_mem(pdma); + if (ret) + goto err_clk_enable; + + ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(42)); + if (ret) { + dev_err(&pdev->dev, "No usable DMA configuration\n"); + goto err_dma_mask; + } + + /* Initialize DMA channels software state */ + xgene_dma_init_channels(pdma); + + /* Configue DMA rings */ + ret = xgene_dma_init_rings(pdma); + if (ret) + goto err_clk_enable; + + ret = xgene_dma_request_irqs(pdma); + if (ret) + goto err_request_irq; + + /* Configure and enable DMA engine */ + xgene_dma_init_hw(pdma); + + /* Register DMA device with linux async framework */ + ret = xgene_dma_init_async(pdma); + if (ret) + goto err_async_init; + + return 0; + +err_async_init: + xgene_dma_free_irqs(pdma); + +err_request_irq: + for (i = 0; i < XGENE_DMA_MAX_CHANNEL; i++) + xgene_dma_delete_chan_rings(&pdma->chan[i]); + +err_dma_mask: +err_clk_enable: + clk_disable_unprepare(pdma->clk); + + return ret; +} + +static int xgene_dma_remove(struct platform_device *pdev) +{ + struct xgene_dma *pdma = platform_get_drvdata(pdev); + struct xgene_dma_chan *chan; + int i; + + xgene_dma_async_unregister(pdma); + + /* Mask interrupts and disable DMA engine */ + xgene_dma_mask_interrupts(pdma); + xgene_dma_disable(pdma); + xgene_dma_free_irqs(pdma); + + for (i = 0; i < XGENE_DMA_MAX_CHANNEL; i++) { + chan = &pdma->chan[i]; + tasklet_kill(&chan->tasklet); + xgene_dma_delete_chan_rings(chan); + } + + clk_disable_unprepare(pdma->clk); + + return 0; +} + +static const struct of_device_id xgene_dma_of_match_ptr[] = { + {.compatible = "apm,xgene-storm-dma",}, + {}, +}; +MODULE_DEVICE_TABLE(of, xgene_dma_of_match_ptr); + +static struct platform_driver xgene_dma_driver = { + .probe = xgene_dma_probe, + .remove = xgene_dma_remove, + .driver = { + .name = "X-Gene-DMA", + .of_match_table = xgene_dma_of_match_ptr, + }, +}; + +module_platform_driver(xgene_dma_driver); + +MODULE_DESCRIPTION("APM X-Gene SoC DMA driver"); +MODULE_AUTHOR("Rameshwar Prasad Sahu <rsahu@apm.com>"); +MODULE_AUTHOR("Loc Ho <lho@apm.com>"); +MODULE_LICENSE("GPL"); +MODULE_VERSION("1.0"); diff --git a/drivers/dma/xilinx/xilinx_vdma.c b/drivers/dma/xilinx/xilinx_vdma.c index bdd2a5dd7220..d8434d465885 100644 --- a/drivers/dma/xilinx/xilinx_vdma.c +++ b/drivers/dma/xilinx/xilinx_vdma.c @@ -22,9 +22,9 @@ * (at your option) any later version. */ -#include <linux/amba/xilinx_dma.h> #include <linux/bitops.h> #include <linux/dmapool.h> +#include <linux/dma/xilinx_dma.h> #include <linux/init.h> #include <linux/interrupt.h> #include <linux/io.h> |