diff options
Diffstat (limited to 'target/linux/layerscape/patches-4.14/802-dma-support-layerscape.patch')
| -rw-r--r-- | target/linux/layerscape/patches-4.14/802-dma-support-layerscape.patch | 4263 |
1 files changed, 0 insertions, 4263 deletions
diff --git a/target/linux/layerscape/patches-4.14/802-dma-support-layerscape.patch b/target/linux/layerscape/patches-4.14/802-dma-support-layerscape.patch deleted file mode 100644 index e39bae0d1d..0000000000 --- a/target/linux/layerscape/patches-4.14/802-dma-support-layerscape.patch +++ /dev/null @@ -1,4263 +0,0 @@ -From 5cb4bc977d933323429050033da9c701b24df43e Mon Sep 17 00:00:00 2001 -From: Biwen Li <biwen.li@nxp.com> -Date: Wed, 17 Apr 2019 18:58:23 +0800 -Subject: [PATCH] dma: support layerscape -MIME-Version: 1.0 -Content-Type: text/plain; charset=UTF-8 -Content-Transfer-Encoding: 8bit - -This is an integrated patch of dma for layerscape - -Signed-off-by: Biwen Li <biwen.li@nxp.com> -Signed-off-by: Catalin Horghidan <catalin.horghidan@nxp.com> -Signed-off-by: Changming Huang <jerry.huang@nxp.com> -Signed-off-by: Horia Geantă <horia.geanta@nxp.com> -Signed-off-by: Ioana Ciornei <ioana.ciornei@nxp.com> -Signed-off-by: jiaheng.fan <jiaheng.fan@nxp.com> -Signed-off-by: Laurentiu Tudor <laurentiu.tudor@nxp.com> -Signed-off-by: Peng Ma <peng.ma@nxp.com> -Signed-off-by: Radu Alexe <radu.alexe@nxp.com> -Signed-off-by: Rajiv Vishwakarma <rajiv.vishwakarma@nxp.com> -Signed-off-by: Tudor Ambarus <tudor-dan.ambarus@nxp.com> -Signed-off-by: Wen He <wen.he_1@nxp.com> -Signed-off-by: Yuan Yao <yao.yuan@nxp.com> ---- - drivers/dma/Kconfig | 33 +- - drivers/dma/Makefile | 3 + - drivers/dma/caam_dma.c | 462 ++++++++ - drivers/dma/dpaa2-qdma/Kconfig | 8 + - drivers/dma/dpaa2-qdma/Makefile | 8 + - drivers/dma/dpaa2-qdma/dpaa2-qdma.c | 781 ++++++++++++++ - drivers/dma/dpaa2-qdma/dpaa2-qdma.h | 181 ++++ - drivers/dma/dpaa2-qdma/dpdmai.c | 515 +++++++++ - drivers/dma/dpaa2-qdma/fsl_dpdmai.h | 521 +++++++++ - drivers/dma/dpaa2-qdma/fsl_dpdmai_cmd.h | 222 ++++ - drivers/dma/fsl-edma.c | 66 +- - drivers/dma/fsl-qdma.c | 1278 +++++++++++++++++++++++ - 12 files changed, 4073 insertions(+), 5 deletions(-) - create mode 100644 drivers/dma/caam_dma.c - create mode 100644 drivers/dma/dpaa2-qdma/Kconfig - create mode 100644 drivers/dma/dpaa2-qdma/Makefile - create mode 100644 drivers/dma/dpaa2-qdma/dpaa2-qdma.c - create mode 100644 drivers/dma/dpaa2-qdma/dpaa2-qdma.h - create mode 100644 drivers/dma/dpaa2-qdma/dpdmai.c - create mode 100644 drivers/dma/dpaa2-qdma/fsl_dpdmai.h - create mode 100644 drivers/dma/dpaa2-qdma/fsl_dpdmai_cmd.h - create mode 100644 drivers/dma/fsl-qdma.c - ---- a/drivers/dma/Kconfig -+++ b/drivers/dma/Kconfig -@@ -129,6 +129,24 @@ config COH901318 - help - Enable support for ST-Ericsson COH 901 318 DMA. - -+config CRYPTO_DEV_FSL_CAAM_DMA -+ tristate "CAAM DMA engine support" -+ depends on CRYPTO_DEV_FSL_CAAM_JR -+ default n -+ select DMA_ENGINE -+ select ASYNC_CORE -+ select ASYNC_TX_ENABLE_CHANNEL_SWITCH -+ help -+ Selecting this will offload the DMA operations for users of -+ the scatter gather memcopy API to the CAAM via job rings. The -+ CAAM is a hardware module that provides hardware acceleration to -+ cryptographic operations. It has a built-in DMA controller that can -+ be programmed to read/write cryptographic data. This module defines -+ a DMA driver that uses the DMA capabilities of the CAAM. -+ -+ To compile this as a module, choose M here: the module -+ will be called caam_dma. -+ - config DMA_BCM2835 - tristate "BCM2835 DMA engine support" - depends on ARCH_BCM2835 -@@ -215,6 +233,20 @@ config FSL_EDMA - multiplexing capability for DMA request sources(slot). - This module can be found on Freescale Vybrid and LS-1 SoCs. - -+config FSL_QDMA -+ tristate "NXP Layerscape qDMA engine support" -+ select DMA_ENGINE -+ select DMA_VIRTUAL_CHANNELS -+ select DMA_ENGINE_RAID -+ select ASYNC_TX_ENABLE_CHANNEL_SWITCH -+ help -+ Support the NXP Layerscape qDMA engine with command queue and legacy mode. -+ Channel virtualization is supported through enqueuing of DMA jobs to, -+ or dequeuing DMA jobs from, different work queues. -+ This module can be found on NXP Layerscape SoCs. -+ -+source drivers/dma/dpaa2-qdma/Kconfig -+ - config FSL_RAID - tristate "Freescale RAID engine Support" - depends on FSL_SOC && !ASYNC_TX_ENABLE_CHANNEL_SWITCH -@@ -600,7 +632,6 @@ config ZX_DMA - help - Support the DMA engine for ZTE ZX family platform devices. - -- - # driver files - source "drivers/dma/bestcomm/Kconfig" - ---- a/drivers/dma/Makefile -+++ b/drivers/dma/Makefile -@@ -31,7 +31,9 @@ obj-$(CONFIG_DMA_SUN6I) += sun6i-dma.o - obj-$(CONFIG_DW_DMAC_CORE) += dw/ - obj-$(CONFIG_EP93XX_DMA) += ep93xx_dma.o - obj-$(CONFIG_FSL_DMA) += fsldma.o -+obj-$(CONFIG_FSL_DPAA2_QDMA) += dpaa2-qdma/ - obj-$(CONFIG_FSL_EDMA) += fsl-edma.o -+obj-$(CONFIG_FSL_QDMA) += fsl-qdma.o - obj-$(CONFIG_FSL_RAID) += fsl_raid.o - obj-$(CONFIG_HSU_DMA) += hsu/ - obj-$(CONFIG_IMG_MDC_DMA) += img-mdc-dma.o -@@ -71,6 +73,7 @@ obj-$(CONFIG_TI_EDMA) += edma.o - obj-$(CONFIG_XGENE_DMA) += xgene-dma.o - obj-$(CONFIG_ZX_DMA) += zx_dma.o - obj-$(CONFIG_ST_FDMA) += st_fdma.o -+obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM_DMA) += caam_dma.o - - obj-y += qcom/ - obj-y += xilinx/ ---- /dev/null -+++ b/drivers/dma/caam_dma.c -@@ -0,0 +1,462 @@ -+/* -+ * caam support for SG DMA -+ * -+ * Copyright 2016 Freescale Semiconductor, Inc -+ * Copyright 2017 NXP -+ * -+ * Redistribution and use in source and binary forms, with or without -+ * modification, are permitted provided that the following conditions are met: -+ * * Redistributions of source code must retain the above copyright -+ * notice, this list of conditions and the following disclaimer. -+ * * Redistributions in binary form must reproduce the above copyright -+ * notice, this list of conditions and the following disclaimer in the -+ * documentation and/or other materials provided with the distribution. -+ * * Neither the names of the above-listed copyright holders nor the -+ * names of any contributors may be used to endorse or promote products -+ * derived from this software without specific prior written permission. -+ * -+ * -+ * ALTERNATIVELY, this software may be distributed under the terms of the -+ * GNU General Public License ("GPL") as published by the Free Software -+ * Foundation, either version 2 of that License or (at your option) any -+ * later version. -+ * -+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE -+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -+ * POSSIBILITY OF SUCH DAMAGE. -+ */ -+ -+#include <linux/dma-mapping.h> -+#include <linux/dmaengine.h> -+#include <linux/module.h> -+#include <linux/platform_device.h> -+#include <linux/slab.h> -+ -+#include "dmaengine.h" -+ -+#include "../crypto/caam/regs.h" -+#include "../crypto/caam/jr.h" -+#include "../crypto/caam/error.h" -+#include "../crypto/caam/desc_constr.h" -+ -+#define DESC_DMA_MEMCPY_LEN ((CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN) / \ -+ CAAM_CMD_SZ) -+ -+/* -+ * This is max chunk size of a DMA transfer. If a buffer is larger than this -+ * value it is internally broken into chunks of max CAAM_DMA_CHUNK_SIZE bytes -+ * and for each chunk a DMA transfer request is issued. -+ * This value is the largest number on 16 bits that is a multiple of 256 bytes -+ * (the largest configurable CAAM DMA burst size). -+ */ -+#define CAAM_DMA_CHUNK_SIZE 65280 -+ -+struct caam_dma_sh_desc { -+ u32 desc[DESC_DMA_MEMCPY_LEN] ____cacheline_aligned; -+ dma_addr_t desc_dma; -+}; -+ -+/* caam dma extended descriptor */ -+struct caam_dma_edesc { -+ struct dma_async_tx_descriptor async_tx; -+ struct list_head node; -+ struct caam_dma_ctx *ctx; -+ dma_addr_t src_dma; -+ dma_addr_t dst_dma; -+ unsigned int src_len; -+ unsigned int dst_len; -+ u32 jd[] ____cacheline_aligned; -+}; -+ -+/* -+ * caam_dma_ctx - per jr/channel context -+ * @chan: dma channel used by async_tx API -+ * @node: list_head used to attach to the global dma_ctx_list -+ * @jrdev: Job Ring device -+ * @pending_q: queue of pending (submitted, but not enqueued) jobs -+ * @done_not_acked: jobs that have been completed by jr, but maybe not acked -+ * @edesc_lock: protects extended descriptor -+ */ -+struct caam_dma_ctx { -+ struct dma_chan chan; -+ struct list_head node; -+ struct device *jrdev; -+ struct list_head pending_q; -+ struct list_head done_not_acked; -+ spinlock_t edesc_lock; -+}; -+ -+static struct dma_device *dma_dev; -+static struct caam_dma_sh_desc *dma_sh_desc; -+static LIST_HEAD(dma_ctx_list); -+ -+static dma_cookie_t caam_dma_tx_submit(struct dma_async_tx_descriptor *tx) -+{ -+ struct caam_dma_edesc *edesc = NULL; -+ struct caam_dma_ctx *ctx = NULL; -+ dma_cookie_t cookie; -+ -+ edesc = container_of(tx, struct caam_dma_edesc, async_tx); -+ ctx = container_of(tx->chan, struct caam_dma_ctx, chan); -+ -+ spin_lock_bh(&ctx->edesc_lock); -+ -+ cookie = dma_cookie_assign(tx); -+ list_add_tail(&edesc->node, &ctx->pending_q); -+ -+ spin_unlock_bh(&ctx->edesc_lock); -+ -+ return cookie; -+} -+ -+static void caam_jr_chan_free_edesc(struct caam_dma_edesc *edesc) -+{ -+ struct caam_dma_ctx *ctx = edesc->ctx; -+ struct caam_dma_edesc *_edesc = NULL; -+ -+ spin_lock_bh(&ctx->edesc_lock); -+ -+ list_add_tail(&edesc->node, &ctx->done_not_acked); -+ list_for_each_entry_safe(edesc, _edesc, &ctx->done_not_acked, node) { -+ if (async_tx_test_ack(&edesc->async_tx)) { -+ list_del(&edesc->node); -+ kfree(edesc); -+ } -+ } -+ -+ spin_unlock_bh(&ctx->edesc_lock); -+} -+ -+static void caam_dma_done(struct device *dev, u32 *hwdesc, u32 err, -+ void *context) -+{ -+ struct caam_dma_edesc *edesc = context; -+ struct caam_dma_ctx *ctx = edesc->ctx; -+ dma_async_tx_callback callback; -+ void *callback_param; -+ -+ if (err) -+ caam_jr_strstatus(ctx->jrdev, err); -+ -+ dma_run_dependencies(&edesc->async_tx); -+ -+ spin_lock_bh(&ctx->edesc_lock); -+ dma_cookie_complete(&edesc->async_tx); -+ spin_unlock_bh(&ctx->edesc_lock); -+ -+ callback = edesc->async_tx.callback; -+ callback_param = edesc->async_tx.callback_param; -+ -+ dma_descriptor_unmap(&edesc->async_tx); -+ -+ caam_jr_chan_free_edesc(edesc); -+ -+ if (callback) -+ callback(callback_param); -+} -+ -+static void caam_dma_memcpy_init_job_desc(struct caam_dma_edesc *edesc) -+{ -+ u32 *jd = edesc->jd; -+ u32 *sh_desc = dma_sh_desc->desc; -+ dma_addr_t desc_dma = dma_sh_desc->desc_dma; -+ -+ /* init the job descriptor */ -+ init_job_desc_shared(jd, desc_dma, desc_len(sh_desc), HDR_REVERSE); -+ -+ /* set SEQIN PTR */ -+ append_seq_in_ptr(jd, edesc->src_dma, edesc->src_len, 0); -+ -+ /* set SEQOUT PTR */ -+ append_seq_out_ptr(jd, edesc->dst_dma, edesc->dst_len, 0); -+ -+ print_hex_dump_debug("caam dma desc@" __stringify(__LINE__) ": ", -+ DUMP_PREFIX_ADDRESS, 16, 4, jd, desc_bytes(jd), 1); -+} -+ -+static struct dma_async_tx_descriptor * -+caam_dma_prep_memcpy(struct dma_chan *chan, dma_addr_t dst, dma_addr_t src, -+ size_t len, unsigned long flags) -+{ -+ struct caam_dma_edesc *edesc; -+ struct caam_dma_ctx *ctx = container_of(chan, struct caam_dma_ctx, -+ chan); -+ -+ edesc = kzalloc(sizeof(*edesc) + DESC_JOB_IO_LEN, GFP_DMA | GFP_NOWAIT); -+ if (!edesc) -+ return ERR_PTR(-ENOMEM); -+ -+ dma_async_tx_descriptor_init(&edesc->async_tx, chan); -+ edesc->async_tx.tx_submit = caam_dma_tx_submit; -+ edesc->async_tx.flags = flags; -+ edesc->async_tx.cookie = -EBUSY; -+ -+ edesc->src_dma = src; -+ edesc->src_len = len; -+ edesc->dst_dma = dst; -+ edesc->dst_len = len; -+ edesc->ctx = ctx; -+ -+ caam_dma_memcpy_init_job_desc(edesc); -+ -+ return &edesc->async_tx; -+} -+ -+/* This function can be called in an interrupt context */ -+static void caam_dma_issue_pending(struct dma_chan *chan) -+{ -+ struct caam_dma_ctx *ctx = container_of(chan, struct caam_dma_ctx, -+ chan); -+ struct caam_dma_edesc *edesc, *_edesc; -+ -+ spin_lock_bh(&ctx->edesc_lock); -+ list_for_each_entry_safe(edesc, _edesc, &ctx->pending_q, node) { -+ if (caam_jr_enqueue(ctx->jrdev, edesc->jd, -+ caam_dma_done, edesc) < 0) -+ break; -+ list_del(&edesc->node); -+ } -+ spin_unlock_bh(&ctx->edesc_lock); -+} -+ -+static void caam_dma_free_chan_resources(struct dma_chan *chan) -+{ -+ struct caam_dma_ctx *ctx = container_of(chan, struct caam_dma_ctx, -+ chan); -+ struct caam_dma_edesc *edesc, *_edesc; -+ -+ spin_lock_bh(&ctx->edesc_lock); -+ list_for_each_entry_safe(edesc, _edesc, &ctx->pending_q, node) { -+ list_del(&edesc->node); -+ kfree(edesc); -+ } -+ list_for_each_entry_safe(edesc, _edesc, &ctx->done_not_acked, node) { -+ list_del(&edesc->node); -+ kfree(edesc); -+ } -+ spin_unlock_bh(&ctx->edesc_lock); -+} -+ -+static int caam_dma_jr_chan_bind(void) -+{ -+ struct device *jrdev; -+ struct caam_dma_ctx *ctx; -+ int bonds = 0; -+ int i; -+ -+ for (i = 0; i < caam_jr_driver_probed(); i++) { -+ jrdev = caam_jridx_alloc(i); -+ if (IS_ERR(jrdev)) { -+ pr_err("job ring device %d allocation failed\n", i); -+ continue; -+ } -+ -+ ctx = kzalloc(sizeof(*ctx), GFP_KERNEL); -+ if (!ctx) { -+ caam_jr_free(jrdev); -+ continue; -+ } -+ -+ ctx->chan.device = dma_dev; -+ ctx->chan.private = ctx; -+ -+ ctx->jrdev = jrdev; -+ -+ INIT_LIST_HEAD(&ctx->pending_q); -+ INIT_LIST_HEAD(&ctx->done_not_acked); -+ INIT_LIST_HEAD(&ctx->node); -+ spin_lock_init(&ctx->edesc_lock); -+ -+ dma_cookie_init(&ctx->chan); -+ -+ /* add the context of this channel to the context list */ -+ list_add_tail(&ctx->node, &dma_ctx_list); -+ -+ /* add this channel to the device chan list */ -+ list_add_tail(&ctx->chan.device_node, &dma_dev->channels); -+ -+ bonds++; -+ } -+ -+ return bonds; -+} -+ -+static inline void caam_jr_dma_free(struct dma_chan *chan) -+{ -+ struct caam_dma_ctx *ctx = container_of(chan, struct caam_dma_ctx, -+ chan); -+ -+ list_del(&ctx->node); -+ list_del(&chan->device_node); -+ caam_jr_free(ctx->jrdev); -+ kfree(ctx); -+} -+ -+static void set_caam_dma_desc(u32 *desc) -+{ -+ u32 *jmp_cmd; -+ -+ /* dma shared descriptor */ -+ init_sh_desc(desc, HDR_SHARE_NEVER | (1 << HDR_START_IDX_SHIFT)); -+ -+ /* REG1 = CAAM_DMA_CHUNK_SIZE */ -+ append_math_add_imm_u32(desc, REG1, ZERO, IMM, CAAM_DMA_CHUNK_SIZE); -+ -+ /* REG0 = SEQINLEN - CAAM_DMA_CHUNK_SIZE */ -+ append_math_sub_imm_u32(desc, REG0, SEQINLEN, IMM, CAAM_DMA_CHUNK_SIZE); -+ -+ /* -+ * if (REG0 > 0) -+ * jmp to LABEL1 -+ */ -+ jmp_cmd = append_jump(desc, JUMP_TEST_INVALL | JUMP_COND_MATH_N | -+ JUMP_COND_MATH_Z); -+ -+ /* REG1 = SEQINLEN */ -+ append_math_sub(desc, REG1, SEQINLEN, ZERO, CAAM_CMD_SZ); -+ -+ /* LABEL1 */ -+ set_jump_tgt_here(desc, jmp_cmd); -+ -+ /* VARSEQINLEN = REG1 */ -+ append_math_add(desc, VARSEQINLEN, REG1, ZERO, CAAM_CMD_SZ); -+ -+ /* VARSEQOUTLEN = REG1 */ -+ append_math_add(desc, VARSEQOUTLEN, REG1, ZERO, CAAM_CMD_SZ); -+ -+ /* do FIFO STORE */ -+ append_seq_fifo_store(desc, 0, FIFOST_TYPE_METADATA | LDST_VLF); -+ -+ /* do FIFO LOAD */ -+ append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | -+ FIFOLD_TYPE_IFIFO | LDST_VLF); -+ -+ /* -+ * if (REG0 > 0) -+ * jmp 0xF8 (after shared desc header) -+ */ -+ append_jump(desc, JUMP_TEST_INVALL | JUMP_COND_MATH_N | -+ JUMP_COND_MATH_Z | 0xF8); -+ -+ print_hex_dump_debug("caam dma shdesc@" __stringify(__LINE__) ": ", -+ DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), -+ 1); -+} -+ -+static int __init caam_dma_probe(struct platform_device *pdev) -+{ -+ struct device *dev = &pdev->dev; -+ struct device *ctrldev = dev->parent; -+ struct dma_chan *chan, *_chan; -+ u32 *sh_desc; -+ int err = -ENOMEM; -+ int bonds; -+ -+ if (!caam_jr_driver_probed()) { -+ dev_info(dev, "Defer probing after JR driver probing\n"); -+ return -EPROBE_DEFER; -+ } -+ -+ dma_dev = kzalloc(sizeof(*dma_dev), GFP_KERNEL); -+ if (!dma_dev) -+ return -ENOMEM; -+ -+ dma_sh_desc = kzalloc(sizeof(*dma_sh_desc), GFP_KERNEL | GFP_DMA); -+ if (!dma_sh_desc) -+ goto desc_err; -+ -+ sh_desc = dma_sh_desc->desc; -+ set_caam_dma_desc(sh_desc); -+ dma_sh_desc->desc_dma = dma_map_single(ctrldev, sh_desc, -+ desc_bytes(sh_desc), -+ DMA_TO_DEVICE); -+ if (dma_mapping_error(ctrldev, dma_sh_desc->desc_dma)) { -+ dev_err(dev, "unable to map dma descriptor\n"); -+ goto map_err; -+ } -+ -+ INIT_LIST_HEAD(&dma_dev->channels); -+ -+ bonds = caam_dma_jr_chan_bind(); -+ if (!bonds) { -+ err = -ENODEV; -+ goto jr_bind_err; -+ } -+ -+ dma_dev->dev = dev; -+ dma_dev->residue_granularity = DMA_RESIDUE_GRANULARITY_DESCRIPTOR; -+ dma_cap_set(DMA_MEMCPY, dma_dev->cap_mask); -+ dma_cap_set(DMA_PRIVATE, dma_dev->cap_mask); -+ dma_dev->device_tx_status = dma_cookie_status; -+ dma_dev->device_issue_pending = caam_dma_issue_pending; -+ dma_dev->device_prep_dma_memcpy = caam_dma_prep_memcpy; -+ dma_dev->device_free_chan_resources = caam_dma_free_chan_resources; -+ -+ err = dma_async_device_register(dma_dev); -+ if (err) { -+ dev_err(dev, "Failed to register CAAM DMA engine\n"); -+ goto jr_bind_err; -+ } -+ -+ dev_info(dev, "caam dma support with %d job rings\n", bonds); -+ -+ return err; -+ -+jr_bind_err: -+ list_for_each_entry_safe(chan, _chan, &dma_dev->channels, device_node) -+ caam_jr_dma_free(chan); -+ -+ dma_unmap_single(ctrldev, dma_sh_desc->desc_dma, desc_bytes(sh_desc), -+ DMA_TO_DEVICE); -+map_err: -+ kfree(dma_sh_desc); -+desc_err: -+ kfree(dma_dev); -+ return err; -+} -+ -+static int caam_dma_remove(struct platform_device *pdev) -+{ -+ struct device *dev = &pdev->dev; -+ struct device *ctrldev = dev->parent; -+ struct caam_dma_ctx *ctx, *_ctx; -+ -+ dma_async_device_unregister(dma_dev); -+ -+ list_for_each_entry_safe(ctx, _ctx, &dma_ctx_list, node) { -+ list_del(&ctx->node); -+ caam_jr_free(ctx->jrdev); -+ kfree(ctx); -+ } -+ -+ dma_unmap_single(ctrldev, dma_sh_desc->desc_dma, -+ desc_bytes(dma_sh_desc->desc), DMA_TO_DEVICE); -+ -+ kfree(dma_sh_desc); -+ kfree(dma_dev); -+ -+ dev_info(dev, "caam dma support disabled\n"); -+ return 0; -+} -+ -+static struct platform_driver caam_dma_driver = { -+ .driver = { -+ .name = "caam-dma", -+ }, -+ .probe = caam_dma_probe, -+ .remove = caam_dma_remove, -+}; -+module_platform_driver(caam_dma_driver); -+ -+MODULE_LICENSE("Dual BSD/GPL"); -+MODULE_DESCRIPTION("NXP CAAM support for DMA engine"); -+MODULE_AUTHOR("NXP Semiconductors"); -+MODULE_ALIAS("platform:caam-dma"); ---- /dev/null -+++ b/drivers/dma/dpaa2-qdma/Kconfig -@@ -0,0 +1,8 @@ -+menuconfig FSL_DPAA2_QDMA -+ tristate "NXP DPAA2 QDMA" -+ depends on FSL_MC_BUS && FSL_MC_DPIO -+ select DMA_ENGINE -+ select DMA_VIRTUAL_CHANNELS -+ ---help--- -+ NXP Data Path Acceleration Architecture 2 QDMA driver, -+ using the NXP MC bus driver. ---- /dev/null -+++ b/drivers/dma/dpaa2-qdma/Makefile -@@ -0,0 +1,8 @@ -+# -+# Makefile for the NXP DPAA2 CAAM controllers -+# -+ccflags-y += -DVERSION=\"\" -+ -+obj-$(CONFIG_FSL_DPAA2_QDMA) += fsl-dpaa2-qdma.o -+ -+fsl-dpaa2-qdma-objs := dpaa2-qdma.o dpdmai.o ---- /dev/null -+++ b/drivers/dma/dpaa2-qdma/dpaa2-qdma.c -@@ -0,0 +1,781 @@ -+/* -+ * drivers/dma/dpaa2-qdma/dpaa2-qdma.c -+ * -+ * Copyright 2015-2017 NXP Semiconductor, Inc. -+ * Author: Changming Huang <jerry.huang@nxp.com> -+ * -+ * Driver for the NXP QDMA engine with QMan mode. -+ * Channel virtualization is supported through enqueuing of DMA jobs to, -+ * or dequeuing DMA jobs from different work queues with QMan portal. -+ * This module can be found on NXP LS2 SoCs. -+ * -+ * This program is free software; you can redistribute it and/or modify it -+ * under the terms of the GNU General Public License as published by the -+ * Free Software Foundation; either version 2 of the License, or (at your -+ * option) any later version. -+ */ -+ -+#include <linux/init.h> -+#include <linux/module.h> -+#include <linux/interrupt.h> -+#include <linux/clk.h> -+#include <linux/dma-mapping.h> -+#include <linux/dmapool.h> -+#include <linux/slab.h> -+#include <linux/spinlock.h> -+#include <linux/of.h> -+#include <linux/of_device.h> -+#include <linux/of_address.h> -+#include <linux/of_irq.h> -+#include <linux/of_dma.h> -+#include <linux/types.h> -+#include <linux/delay.h> -+#include <linux/iommu.h> -+#include <linux/sys_soc.h> -+ -+#include "../virt-dma.h" -+ -+#include <linux/fsl/mc.h> -+#include "../../../drivers/staging/fsl-mc/include/dpaa2-io.h" -+#include "../../../drivers/staging/fsl-mc/include/dpaa2-fd.h" -+#include "fsl_dpdmai_cmd.h" -+#include "fsl_dpdmai.h" -+#include "dpaa2-qdma.h" -+ -+static bool smmu_disable = true; -+ -+static struct dpaa2_qdma_chan *to_dpaa2_qdma_chan(struct dma_chan *chan) -+{ -+ return container_of(chan, struct dpaa2_qdma_chan, vchan.chan); -+} -+ -+static struct dpaa2_qdma_comp *to_fsl_qdma_comp(struct virt_dma_desc *vd) -+{ -+ return container_of(vd, struct dpaa2_qdma_comp, vdesc); -+} -+ -+static int dpaa2_qdma_alloc_chan_resources(struct dma_chan *chan) -+{ -+ return 0; -+} -+ -+static void dpaa2_qdma_free_chan_resources(struct dma_chan *chan) -+{ -+ struct dpaa2_qdma_chan *dpaa2_chan = to_dpaa2_qdma_chan(chan); -+ unsigned long flags; -+ LIST_HEAD(head); -+ -+ spin_lock_irqsave(&dpaa2_chan->vchan.lock, flags); -+ vchan_get_all_descriptors(&dpaa2_chan->vchan, &head); -+ spin_unlock_irqrestore(&dpaa2_chan->vchan.lock, flags); -+ -+ vchan_dma_desc_free_list(&dpaa2_chan->vchan, &head); -+} -+ -+/* -+ * Request a command descriptor for enqueue. -+ */ -+static struct dpaa2_qdma_comp * -+dpaa2_qdma_request_desc(struct dpaa2_qdma_chan *dpaa2_chan) -+{ -+ struct dpaa2_qdma_comp *comp_temp = NULL; -+ unsigned long flags; -+ -+ spin_lock_irqsave(&dpaa2_chan->queue_lock, flags); -+ if (list_empty(&dpaa2_chan->comp_free)) { -+ spin_unlock_irqrestore(&dpaa2_chan->queue_lock, flags); -+ comp_temp = kzalloc(sizeof(*comp_temp), GFP_KERNEL); -+ if (!comp_temp) -+ goto err; -+ comp_temp->fd_virt_addr = dma_pool_alloc(dpaa2_chan->fd_pool, -+ GFP_NOWAIT, &comp_temp->fd_bus_addr); -+ if (!comp_temp->fd_virt_addr) -+ goto err; -+ -+ comp_temp->fl_virt_addr = -+ (void *)((struct dpaa2_fd *) -+ comp_temp->fd_virt_addr + 1); -+ comp_temp->fl_bus_addr = comp_temp->fd_bus_addr + -+ sizeof(struct dpaa2_fd); -+ comp_temp->desc_virt_addr = -+ (void *)((struct dpaa2_fl_entry *) -+ comp_temp->fl_virt_addr + 3); -+ comp_temp->desc_bus_addr = comp_temp->fl_bus_addr + -+ sizeof(struct dpaa2_fl_entry) * 3; -+ -+ comp_temp->qchan = dpaa2_chan; -+ return comp_temp; -+ } -+ comp_temp = list_first_entry(&dpaa2_chan->comp_free, -+ struct dpaa2_qdma_comp, list); -+ list_del(&comp_temp->list); -+ spin_unlock_irqrestore(&dpaa2_chan->queue_lock, flags); -+ -+ comp_temp->qchan = dpaa2_chan; -+err: -+ return comp_temp; -+} -+ -+static void dpaa2_qdma_populate_fd(uint32_t format, -+ struct dpaa2_qdma_comp *dpaa2_comp) -+{ -+ struct dpaa2_fd *fd; -+ -+ fd = (struct dpaa2_fd *)dpaa2_comp->fd_virt_addr; -+ memset(fd, 0, sizeof(struct dpaa2_fd)); -+ -+ /* fd populated */ -+ dpaa2_fd_set_addr(fd, dpaa2_comp->fl_bus_addr); -+ /* Bypass memory translation, Frame list format, short length disable */ -+ /* we need to disable BMT if fsl-mc use iova addr */ -+ if (smmu_disable) -+ dpaa2_fd_set_bpid(fd, QMAN_FD_BMT_ENABLE); -+ dpaa2_fd_set_format(fd, QMAN_FD_FMT_ENABLE | QMAN_FD_SL_DISABLE); -+ -+ dpaa2_fd_set_frc(fd, format | QDMA_SER_CTX); -+} -+ -+/* first frame list for descriptor buffer */ -+static void dpaa2_qdma_populate_first_framel( -+ struct dpaa2_fl_entry *f_list, -+ struct dpaa2_qdma_comp *dpaa2_comp, -+ bool wrt_changed) -+{ -+ struct dpaa2_qdma_sd_d *sdd; -+ -+ sdd = (struct dpaa2_qdma_sd_d *)dpaa2_comp->desc_virt_addr; -+ memset(sdd, 0, 2 * (sizeof(*sdd))); -+ /* source and destination descriptor */ -+ sdd->cmd = cpu_to_le32(QDMA_SD_CMD_RDTTYPE_COHERENT); /* source descriptor CMD */ -+ sdd++; -+ -+ /* dest descriptor CMD */ -+ if (wrt_changed) -+ sdd->cmd = cpu_to_le32(LX2160_QDMA_DD_CMD_WRTTYPE_COHERENT); -+ else -+ sdd->cmd = cpu_to_le32(QDMA_DD_CMD_WRTTYPE_COHERENT); -+ -+ memset(f_list, 0, sizeof(struct dpaa2_fl_entry)); -+ /* first frame list to source descriptor */ -+ -+ dpaa2_fl_set_addr(f_list, dpaa2_comp->desc_bus_addr); -+ dpaa2_fl_set_len(f_list, 0x20); -+ dpaa2_fl_set_format(f_list, QDMA_FL_FMT_SBF | QDMA_FL_SL_LONG); -+ -+ if (smmu_disable) -+ f_list->bpid = cpu_to_le16(QDMA_FL_BMT_ENABLE); /* bypass memory translation */ -+} -+ -+/* source and destination frame list */ -+static void dpaa2_qdma_populate_frames(struct dpaa2_fl_entry *f_list, -+ dma_addr_t dst, dma_addr_t src, size_t len, uint8_t fmt) -+{ -+ /* source frame list to source buffer */ -+ memset(f_list, 0, sizeof(struct dpaa2_fl_entry)); -+ -+ -+ dpaa2_fl_set_addr(f_list, src); -+ dpaa2_fl_set_len(f_list, len); -+ dpaa2_fl_set_format(f_list, (fmt | QDMA_FL_SL_LONG)); /* single buffer frame or scatter gather frame */ -+ if (smmu_disable) -+ f_list->bpid = cpu_to_le16(QDMA_FL_BMT_ENABLE); /* bypass memory translation */ -+ -+ f_list++; -+ /* destination frame list to destination buffer */ -+ memset(f_list, 0, sizeof(struct dpaa2_fl_entry)); -+ -+ dpaa2_fl_set_addr(f_list, dst); -+ dpaa2_fl_set_len(f_list, len); -+ dpaa2_fl_set_format(f_list, (fmt | QDMA_FL_SL_LONG)); -+ dpaa2_fl_set_final(f_list, QDMA_FL_F); /* single buffer frame or scatter gather frame */ -+ if (smmu_disable) -+ f_list->bpid = cpu_to_le16(QDMA_FL_BMT_ENABLE); /* bypass memory translation */ -+} -+ -+static struct dma_async_tx_descriptor *dpaa2_qdma_prep_memcpy( -+ struct dma_chan *chan, dma_addr_t dst, -+ dma_addr_t src, size_t len, unsigned long flags) -+{ -+ struct dpaa2_qdma_chan *dpaa2_chan = to_dpaa2_qdma_chan(chan); -+ struct dpaa2_qdma_engine *dpaa2_qdma; -+ struct dpaa2_qdma_comp *dpaa2_comp; -+ struct dpaa2_fl_entry *f_list; -+ bool wrt_changed; -+ uint32_t format; -+ -+ dpaa2_qdma = dpaa2_chan->qdma; -+ dpaa2_comp = dpaa2_qdma_request_desc(dpaa2_chan); -+ wrt_changed = dpaa2_qdma->qdma_wrtype_fixup; -+ -+#ifdef LONG_FORMAT -+ format = QDMA_FD_LONG_FORMAT; -+#else -+ format = QDMA_FD_SHORT_FORMAT; -+#endif -+ /* populate Frame descriptor */ -+ dpaa2_qdma_populate_fd(format, dpaa2_comp); -+ -+ f_list = (struct dpaa2_fl_entry *)dpaa2_comp->fl_virt_addr; -+ -+#ifdef LONG_FORMAT -+ /* first frame list for descriptor buffer (logn format) */ -+ dpaa2_qdma_populate_first_framel(f_list, dpaa2_comp, wrt_changed); -+ -+ f_list++; -+#endif -+ -+ dpaa2_qdma_populate_frames(f_list, dst, src, len, QDMA_FL_FMT_SBF); -+ -+ return vchan_tx_prep(&dpaa2_chan->vchan, &dpaa2_comp->vdesc, flags); -+} -+ -+static enum dma_status dpaa2_qdma_tx_status(struct dma_chan *chan, -+ dma_cookie_t cookie, struct dma_tx_state *txstate) -+{ -+ return dma_cookie_status(chan, cookie, txstate); -+} -+ -+static void dpaa2_qdma_free_desc(struct virt_dma_desc *vdesc) -+{ -+} -+ -+static void dpaa2_qdma_issue_pending(struct dma_chan *chan) -+{ -+ struct dpaa2_qdma_comp *dpaa2_comp; -+ struct dpaa2_qdma_chan *dpaa2_chan = to_dpaa2_qdma_chan(chan); -+ struct dpaa2_qdma_engine *dpaa2_qdma = dpaa2_chan->qdma; -+ struct dpaa2_qdma_priv *priv = dpaa2_qdma->priv; -+ struct virt_dma_desc *vdesc; -+ struct dpaa2_fd *fd; -+ int err; -+ unsigned long flags; -+ -+ spin_lock_irqsave(&dpaa2_chan->queue_lock, flags); -+ spin_lock(&dpaa2_chan->vchan.lock); -+ if (vchan_issue_pending(&dpaa2_chan->vchan)) { -+ vdesc = vchan_next_desc(&dpaa2_chan->vchan); -+ if (!vdesc) -+ goto err_enqueue; -+ dpaa2_comp = to_fsl_qdma_comp(vdesc); -+ -+ fd = (struct dpaa2_fd *)dpaa2_comp->fd_virt_addr; -+ -+ list_del(&vdesc->node); -+ list_add_tail(&dpaa2_comp->list, &dpaa2_chan->comp_used); -+ -+ /* TOBO: priority hard-coded to zero */ -+ err = dpaa2_io_service_enqueue_fq(NULL, -+ priv->tx_queue_attr[0].fqid, fd); -+ if (err) { -+ list_del(&dpaa2_comp->list); -+ list_add_tail(&dpaa2_comp->list, -+ &dpaa2_chan->comp_free); -+ } -+ -+ } -+err_enqueue: -+ spin_unlock(&dpaa2_chan->vchan.lock); -+ spin_unlock_irqrestore(&dpaa2_chan->queue_lock, flags); -+} -+ -+static int __cold dpaa2_qdma_setup(struct fsl_mc_device *ls_dev) -+{ -+ struct device *dev = &ls_dev->dev; -+ struct dpaa2_qdma_priv *priv; -+ struct dpaa2_qdma_priv_per_prio *ppriv; -+ uint8_t prio_def = DPDMAI_PRIO_NUM; -+ int err; -+ int i; -+ -+ priv = dev_get_drvdata(dev); -+ -+ priv->dev = dev; -+ priv->dpqdma_id = ls_dev->obj_desc.id; -+ -+ /*Get the handle for the DPDMAI this interface is associate with */ -+ err = dpdmai_open(priv->mc_io, 0, priv->dpqdma_id, &ls_dev->mc_handle); -+ if (err) { -+ dev_err(dev, "dpdmai_open() failed\n"); -+ return err; -+ } -+ dev_info(dev, "Opened dpdmai object successfully\n"); -+ -+ err = dpdmai_get_attributes(priv->mc_io, 0, ls_dev->mc_handle, -+ &priv->dpdmai_attr); -+ if (err) { -+ dev_err(dev, "dpdmai_get_attributes() failed\n"); -+ return err; -+ } -+ -+ if (priv->dpdmai_attr.version.major > DPDMAI_VER_MAJOR) { -+ dev_err(dev, "DPDMAI major version mismatch\n" -+ "Found %u.%u, supported version is %u.%u\n", -+ priv->dpdmai_attr.version.major, -+ priv->dpdmai_attr.version.minor, -+ DPDMAI_VER_MAJOR, DPDMAI_VER_MINOR); -+ } -+ -+ if (priv->dpdmai_attr.version.minor > DPDMAI_VER_MINOR) { -+ dev_err(dev, "DPDMAI minor version mismatch\n" -+ "Found %u.%u, supported version is %u.%u\n", -+ priv->dpdmai_attr.version.major, -+ priv->dpdmai_attr.version.minor, -+ DPDMAI_VER_MAJOR, DPDMAI_VER_MINOR); -+ } -+ -+ priv->num_pairs = min(priv->dpdmai_attr.num_of_priorities, prio_def); -+ ppriv = kcalloc(priv->num_pairs, sizeof(*ppriv), GFP_KERNEL); -+ if (!ppriv) { -+ dev_err(dev, "kzalloc for ppriv failed\n"); -+ return -1; -+ } -+ priv->ppriv = ppriv; -+ -+ for (i = 0; i < priv->num_pairs; i++) { -+ err = dpdmai_get_rx_queue(priv->mc_io, 0, ls_dev->mc_handle, -+ i, &priv->rx_queue_attr[i]); -+ if (err) { -+ dev_err(dev, "dpdmai_get_rx_queue() failed\n"); -+ return err; -+ } -+ ppriv->rsp_fqid = priv->rx_queue_attr[i].fqid; -+ -+ err = dpdmai_get_tx_queue(priv->mc_io, 0, ls_dev->mc_handle, -+ i, &priv->tx_queue_attr[i]); -+ if (err) { -+ dev_err(dev, "dpdmai_get_tx_queue() failed\n"); -+ return err; -+ } -+ ppriv->req_fqid = priv->tx_queue_attr[i].fqid; -+ ppriv->prio = i; -+ ppriv->priv = priv; -+ ppriv++; -+ } -+ -+ return 0; -+} -+ -+static void dpaa2_qdma_fqdan_cb(struct dpaa2_io_notification_ctx *ctx) -+{ -+ struct dpaa2_qdma_priv_per_prio *ppriv = container_of(ctx, -+ struct dpaa2_qdma_priv_per_prio, nctx); -+ struct dpaa2_qdma_priv *priv = ppriv->priv; -+ struct dpaa2_qdma_comp *dpaa2_comp, *_comp_tmp; -+ struct dpaa2_qdma_chan *qchan; -+ const struct dpaa2_fd *fd; -+ const struct dpaa2_fd *fd_eq; -+ struct dpaa2_dq *dq; -+ int err; -+ int is_last = 0; -+ uint8_t status; -+ int i; -+ int found; -+ uint32_t n_chans = priv->dpaa2_qdma->n_chans; -+ -+ do { -+ err = dpaa2_io_service_pull_fq(NULL, ppriv->rsp_fqid, -+ ppriv->store); -+ } while (err); -+ -+ while (!is_last) { -+ do { -+ dq = dpaa2_io_store_next(ppriv->store, &is_last); -+ } while (!is_last && !dq); -+ if (!dq) { -+ dev_err(priv->dev, "FQID returned no valid frames!\n"); -+ continue; -+ } -+ -+ /* obtain FD and process the error */ -+ fd = dpaa2_dq_fd(dq); -+ -+ status = dpaa2_fd_get_ctrl(fd) & 0xff; -+ if (status) -+ dev_err(priv->dev, "FD error occurred\n"); -+ found = 0; -+ for (i = 0; i < n_chans; i++) { -+ qchan = &priv->dpaa2_qdma->chans[i]; -+ spin_lock(&qchan->queue_lock); -+ if (list_empty(&qchan->comp_used)) { -+ spin_unlock(&qchan->queue_lock); -+ continue; -+ } -+ list_for_each_entry_safe(dpaa2_comp, _comp_tmp, -+ &qchan->comp_used, list) { -+ fd_eq = (struct dpaa2_fd *) -+ dpaa2_comp->fd_virt_addr; -+ -+ if (le64_to_cpu(fd_eq->simple.addr) == -+ le64_to_cpu(fd->simple.addr)) { -+ -+ list_del(&dpaa2_comp->list); -+ list_add_tail(&dpaa2_comp->list, -+ &qchan->comp_free); -+ -+ spin_lock(&qchan->vchan.lock); -+ vchan_cookie_complete( -+ &dpaa2_comp->vdesc); -+ spin_unlock(&qchan->vchan.lock); -+ found = 1; -+ break; -+ } -+ } -+ spin_unlock(&qchan->queue_lock); -+ if (found) -+ break; -+ } -+ } -+ -+ dpaa2_io_service_rearm(NULL, ctx); -+} -+ -+static int __cold dpaa2_qdma_dpio_setup(struct dpaa2_qdma_priv *priv) -+{ -+ int err, i, num; -+ struct device *dev = priv->dev; -+ struct dpaa2_qdma_priv_per_prio *ppriv; -+ -+ num = priv->num_pairs; -+ ppriv = priv->ppriv; -+ for (i = 0; i < num; i++) { -+ ppriv->nctx.is_cdan = 0; -+ ppriv->nctx.desired_cpu = 1; -+ ppriv->nctx.id = ppriv->rsp_fqid; -+ ppriv->nctx.cb = dpaa2_qdma_fqdan_cb; -+ err = dpaa2_io_service_register(NULL, &ppriv->nctx, dev); -+ if (err) { -+ dev_err(dev, "Notification register failed\n"); -+ goto err_service; -+ } -+ -+ ppriv->store = dpaa2_io_store_create(DPAA2_QDMA_STORE_SIZE, -+ dev); -+ if (!ppriv->store) { -+ dev_err(dev, "dpaa2_io_store_create() failed\n"); -+ goto err_store; -+ } -+ -+ ppriv++; -+ } -+ return 0; -+ -+err_store: -+ dpaa2_io_service_deregister(NULL, &ppriv->nctx, dev); -+err_service: -+ ppriv--; -+ while (ppriv >= priv->ppriv) { -+ dpaa2_io_service_deregister(NULL, &ppriv->nctx, dev); -+ dpaa2_io_store_destroy(ppriv->store); -+ ppriv--; -+ } -+ return -1; -+} -+ -+static void __cold dpaa2_dpmai_store_free(struct dpaa2_qdma_priv *priv) -+{ -+ struct dpaa2_qdma_priv_per_prio *ppriv = priv->ppriv; -+ int i; -+ -+ for (i = 0; i < priv->num_pairs; i++) { -+ dpaa2_io_store_destroy(ppriv->store); -+ ppriv++; -+ } -+} -+ -+static void __cold dpaa2_dpdmai_dpio_free(struct dpaa2_qdma_priv *priv) -+{ -+ struct dpaa2_qdma_priv_per_prio *ppriv = priv->ppriv; -+ struct device *dev = priv->dev; -+ int i; -+ -+ for (i = 0; i < priv->num_pairs; i++) { -+ dpaa2_io_service_deregister(NULL, &ppriv->nctx, dev); -+ ppriv++; -+ } -+} -+ -+static int __cold dpaa2_dpdmai_bind(struct dpaa2_qdma_priv *priv) -+{ -+ int err; -+ struct dpdmai_rx_queue_cfg rx_queue_cfg; -+ struct device *dev = priv->dev; -+ struct dpaa2_qdma_priv_per_prio *ppriv; -+ struct fsl_mc_device *ls_dev = to_fsl_mc_device(dev); -+ int i, num; -+ -+ num = priv->num_pairs; -+ ppriv = priv->ppriv; -+ for (i = 0; i < num; i++) { -+ rx_queue_cfg.options = DPDMAI_QUEUE_OPT_USER_CTX | -+ DPDMAI_QUEUE_OPT_DEST; -+ rx_queue_cfg.user_ctx = ppriv->nctx.qman64; -+ rx_queue_cfg.dest_cfg.dest_type = DPDMAI_DEST_DPIO; -+ rx_queue_cfg.dest_cfg.dest_id = ppriv->nctx.dpio_id; -+ rx_queue_cfg.dest_cfg.priority = ppriv->prio; -+ err = dpdmai_set_rx_queue(priv->mc_io, 0, ls_dev->mc_handle, -+ rx_queue_cfg.dest_cfg.priority, &rx_queue_cfg); -+ if (err) { -+ dev_err(dev, "dpdmai_set_rx_queue() failed\n"); -+ return err; -+ } -+ -+ ppriv++; -+ } -+ -+ return 0; -+} -+ -+static int __cold dpaa2_dpdmai_dpio_unbind(struct dpaa2_qdma_priv *priv) -+{ -+ int err = 0; -+ struct device *dev = priv->dev; -+ struct fsl_mc_device *ls_dev = to_fsl_mc_device(dev); -+ struct dpaa2_qdma_priv_per_prio *ppriv = priv->ppriv; -+ int i; -+ -+ for (i = 0; i < priv->num_pairs; i++) { -+ ppriv->nctx.qman64 = 0; -+ ppriv->nctx.dpio_id = 0; -+ ppriv++; -+ } -+ -+ err = dpdmai_reset(priv->mc_io, 0, ls_dev->mc_handle); -+ if (err) -+ dev_err(dev, "dpdmai_reset() failed\n"); -+ -+ return err; -+} -+ -+static void __cold dpaa2_dpdmai_free_comp(struct dpaa2_qdma_chan *qchan, -+ struct list_head *head) -+{ -+ struct dpaa2_qdma_comp *comp_tmp, *_comp_tmp; -+ /* free the QDMA comp resource */ -+ list_for_each_entry_safe(comp_tmp, _comp_tmp, -+ head, list) { -+ dma_pool_free(qchan->fd_pool, -+ comp_tmp->fd_virt_addr, -+ comp_tmp->fd_bus_addr); -+ list_del(&comp_tmp->list); -+ kfree(comp_tmp); -+ } -+ -+} -+ -+static void __cold dpaa2_dpdmai_free_channels( -+ struct dpaa2_qdma_engine *dpaa2_qdma) -+{ -+ struct dpaa2_qdma_chan *qchan; -+ int num, i; -+ -+ num = dpaa2_qdma->n_chans; -+ for (i = 0; i < num; i++) { -+ qchan = &dpaa2_qdma->chans[i]; -+ dpaa2_dpdmai_free_comp(qchan, &qchan->comp_used); -+ dpaa2_dpdmai_free_comp(qchan, &qchan->comp_free); -+ dma_pool_destroy(qchan->fd_pool); -+ } -+} -+ -+static int dpaa2_dpdmai_alloc_channels(struct dpaa2_qdma_engine *dpaa2_qdma) -+{ -+ struct dpaa2_qdma_chan *dpaa2_chan; -+ struct device *dev = &dpaa2_qdma->priv->dpdmai_dev->dev; -+ int i; -+ -+ INIT_LIST_HEAD(&dpaa2_qdma->dma_dev.channels); -+ for (i = 0; i < dpaa2_qdma->n_chans; i++) { -+ dpaa2_chan = &dpaa2_qdma->chans[i]; -+ dpaa2_chan->qdma = dpaa2_qdma; -+ dpaa2_chan->vchan.desc_free = dpaa2_qdma_free_desc; -+ vchan_init(&dpaa2_chan->vchan, &dpaa2_qdma->dma_dev); -+ -+ dpaa2_chan->fd_pool = dma_pool_create("fd_pool", -+ dev, FD_POOL_SIZE, 32, 0); -+ if (!dpaa2_chan->fd_pool) -+ return -1; -+ -+ spin_lock_init(&dpaa2_chan->queue_lock); -+ INIT_LIST_HEAD(&dpaa2_chan->comp_used); -+ INIT_LIST_HEAD(&dpaa2_chan->comp_free); -+ } -+ return 0; -+} -+ -+static int dpaa2_qdma_probe(struct fsl_mc_device *dpdmai_dev) -+{ -+ struct dpaa2_qdma_priv *priv; -+ struct device *dev = &dpdmai_dev->dev; -+ struct dpaa2_qdma_engine *dpaa2_qdma; -+ int err; -+ -+ priv = kzalloc(sizeof(*priv), GFP_KERNEL); -+ if (!priv) -+ return -ENOMEM; -+ dev_set_drvdata(dev, priv); -+ priv->dpdmai_dev = dpdmai_dev; -+ -+ priv->iommu_domain = iommu_get_domain_for_dev(dev); -+ if (priv->iommu_domain) -+ smmu_disable = false; -+ -+ /* obtain a MC portal */ -+ err = fsl_mc_portal_allocate(dpdmai_dev, 0, &priv->mc_io); -+ if (err) { -+ if (err == -ENXIO) -+ err = -EPROBE_DEFER; -+ else -+ dev_err(dev, "MC portal allocation failed\n"); -+ goto err_mcportal; -+ } -+ -+ /* DPDMAI initialization */ -+ err = dpaa2_qdma_setup(dpdmai_dev); -+ if (err) { -+ dev_err(dev, "dpaa2_dpdmai_setup() failed\n"); -+ goto err_dpdmai_setup; -+ } -+ -+ /* DPIO */ -+ err = dpaa2_qdma_dpio_setup(priv); -+ if (err) { -+ dev_err(dev, "dpaa2_dpdmai_dpio_setup() failed\n"); -+ goto err_dpio_setup; -+ } -+ -+ /* DPDMAI binding to DPIO */ -+ err = dpaa2_dpdmai_bind(priv); -+ if (err) { -+ dev_err(dev, "dpaa2_dpdmai_bind() failed\n"); -+ goto err_bind; -+ } -+ -+ /* DPDMAI enable */ -+ err = dpdmai_enable(priv->mc_io, 0, dpdmai_dev->mc_handle); -+ if (err) { -+ dev_err(dev, "dpdmai_enable() faile\n"); -+ goto err_enable; -+ } -+ -+ dpaa2_qdma = kzalloc(sizeof(*dpaa2_qdma), GFP_KERNEL); -+ if (!dpaa2_qdma) { -+ err = -ENOMEM; -+ goto err_eng; -+ } -+ -+ priv->dpaa2_qdma = dpaa2_qdma; -+ dpaa2_qdma->priv = priv; -+ -+ dpaa2_qdma->n_chans = NUM_CH; -+ -+ err = dpaa2_dpdmai_alloc_channels(dpaa2_qdma); -+ if (err) { -+ dev_err(dev, "QDMA alloc channels faile\n"); -+ goto err_reg; -+ } -+ -+ if (soc_device_match(soc_fixup_tuning)) -+ dpaa2_qdma->qdma_wrtype_fixup = true; -+ else -+ dpaa2_qdma->qdma_wrtype_fixup = false; -+ -+ dma_cap_set(DMA_PRIVATE, dpaa2_qdma->dma_dev.cap_mask); -+ dma_cap_set(DMA_SLAVE, dpaa2_qdma->dma_dev.cap_mask); -+ dma_cap_set(DMA_MEMCPY, dpaa2_qdma->dma_dev.cap_mask); -+ -+ dpaa2_qdma->dma_dev.dev = dev; -+ dpaa2_qdma->dma_dev.device_alloc_chan_resources -+ = dpaa2_qdma_alloc_chan_resources; -+ dpaa2_qdma->dma_dev.device_free_chan_resources -+ = dpaa2_qdma_free_chan_resources; -+ dpaa2_qdma->dma_dev.device_tx_status = dpaa2_qdma_tx_status; -+ dpaa2_qdma->dma_dev.device_prep_dma_memcpy = dpaa2_qdma_prep_memcpy; -+ dpaa2_qdma->dma_dev.device_issue_pending = dpaa2_qdma_issue_pending; -+ -+ err = dma_async_device_register(&dpaa2_qdma->dma_dev); -+ if (err) { -+ dev_err(dev, "Can't register NXP QDMA engine.\n"); -+ goto err_reg; -+ } -+ -+ return 0; -+ -+err_reg: -+ dpaa2_dpdmai_free_channels(dpaa2_qdma); -+ kfree(dpaa2_qdma); -+err_eng: -+ dpdmai_disable(priv->mc_io, 0, dpdmai_dev->mc_handle); -+err_enable: -+ dpaa2_dpdmai_dpio_unbind(priv); -+err_bind: -+ dpaa2_dpmai_store_free(priv); -+ dpaa2_dpdmai_dpio_free(priv); -+err_dpio_setup: -+ dpdmai_close(priv->mc_io, 0, dpdmai_dev->mc_handle); -+err_dpdmai_setup: -+ fsl_mc_portal_free(priv->mc_io); -+err_mcportal: -+ kfree(priv->ppriv); -+ kfree(priv); -+ dev_set_drvdata(dev, NULL); -+ return err; -+} -+ -+static int dpaa2_qdma_remove(struct fsl_mc_device *ls_dev) -+{ -+ struct device *dev; -+ struct dpaa2_qdma_priv *priv; -+ struct dpaa2_qdma_engine *dpaa2_qdma; -+ -+ dev = &ls_dev->dev; -+ priv = dev_get_drvdata(dev); -+ dpaa2_qdma = priv->dpaa2_qdma; -+ -+ dpdmai_disable(priv->mc_io, 0, ls_dev->mc_handle); -+ dpaa2_dpdmai_dpio_unbind(priv); -+ dpaa2_dpmai_store_free(priv); -+ dpaa2_dpdmai_dpio_free(priv); -+ dpdmai_close(priv->mc_io, 0, ls_dev->mc_handle); -+ fsl_mc_portal_free(priv->mc_io); -+ dev_set_drvdata(dev, NULL); -+ dpaa2_dpdmai_free_channels(dpaa2_qdma); -+ -+ dma_async_device_unregister(&dpaa2_qdma->dma_dev); -+ kfree(priv); -+ kfree(dpaa2_qdma); -+ -+ return 0; -+} -+ -+static const struct fsl_mc_device_id dpaa2_qdma_id_table[] = { -+ { -+ .vendor = FSL_MC_VENDOR_FREESCALE, -+ .obj_type = "dpdmai", -+ }, -+ { .vendor = 0x0 } -+}; -+ -+static struct fsl_mc_driver dpaa2_qdma_driver = { -+ .driver = { -+ .name = "dpaa2-qdma", -+ .owner = THIS_MODULE, -+ }, -+ .probe = dpaa2_qdma_probe, -+ .remove = dpaa2_qdma_remove, -+ .match_id_table = dpaa2_qdma_id_table -+}; -+ -+static int __init dpaa2_qdma_driver_init(void) -+{ -+ return fsl_mc_driver_register(&(dpaa2_qdma_driver)); -+} -+late_initcall(dpaa2_qdma_driver_init); -+ -+static void __exit fsl_qdma_exit(void) -+{ -+ fsl_mc_driver_unregister(&(dpaa2_qdma_driver)); -+} -+module_exit(fsl_qdma_exit); -+ -+MODULE_DESCRIPTION("NXP DPAA2 qDMA driver"); -+MODULE_LICENSE("Dual BSD/GPL"); ---- /dev/null -+++ b/drivers/dma/dpaa2-qdma/dpaa2-qdma.h -@@ -0,0 +1,181 @@ -+/* Copyright 2015 NXP Semiconductor Inc. -+ * -+ * Redistribution and use in source and binary forms, with or without -+ * modification, are permitted provided that the following conditions are met: -+ * * Redistributions of source code must retain the above copyright -+ * notice, this list of conditions and the following disclaimer. -+ * * Redistributions in binary form must reproduce the above copyright -+ * notice, this list of conditions and the following disclaimer in the -+ * documentation and/or other materials provided with the distribution. -+ * * Neither the name of NXP Semiconductor nor the -+ * names of its contributors may be used to endorse or promote products -+ * derived from this software without specific prior written permission. -+ * -+ * -+ * ALTERNATIVELY, this software may be distributed under the terms of the -+ * GNU General Public License ("GPL") as published by the Free Software -+ * Foundation, either version 2 of that License or (at your option) any -+ * later version. -+ * -+ * THIS SOFTWARE IS PROVIDED BY NXP Semiconductor ``AS IS'' AND ANY -+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -+ * DISCLAIMED. IN NO EVENT SHALL NXP Semiconductor BE LIABLE FOR ANY -+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -+ */ -+ -+#ifndef __DPAA2_QDMA_H -+#define __DPAA2_QDMA_H -+ -+#define LONG_FORMAT 1 -+ -+#define DPAA2_QDMA_STORE_SIZE 16 -+#define NUM_CH 8 -+ -+#define QDMA_DMR_OFFSET 0x0 -+#define QDMA_DQ_EN (0 << 30) -+#define QDMA_DQ_DIS (1 << 30) -+ -+#define QDMA_DSR_M_OFFSET 0x10004 -+ -+struct dpaa2_qdma_sd_d { -+ uint32_t rsv:32; -+ union { -+ struct { -+ uint32_t ssd:12; /* souce stride distance */ -+ uint32_t sss:12; /* souce stride size */ -+ uint32_t rsv1:8; -+ } sdf; -+ struct { -+ uint32_t dsd:12; /* Destination stride distance */ -+ uint32_t dss:12; /* Destination stride size */ -+ uint32_t rsv2:8; -+ } ddf; -+ } df; -+ uint32_t rbpcmd; /* Route-by-port command */ -+ uint32_t cmd; -+} __attribute__((__packed__)); -+/* Source descriptor command read transaction type for RBP=0: -+ coherent copy of cacheable memory */ -+#define QDMA_SD_CMD_RDTTYPE_COHERENT (0xb << 28) -+/* Destination descriptor command write transaction type for RBP=0: -+ coherent copy of cacheable memory */ -+#define QDMA_DD_CMD_WRTTYPE_COHERENT (0x6 << 28) -+#define LX2160_QDMA_DD_CMD_WRTTYPE_COHERENT (0xb << 28) -+ -+#define QMAN_FD_FMT_ENABLE (1) /* frame list table enable */ -+#define QMAN_FD_BMT_ENABLE (1 << 15) /* bypass memory translation */ -+#define QMAN_FD_BMT_DISABLE (0 << 15) /* bypass memory translation */ -+#define QMAN_FD_SL_DISABLE (0 << 14) /* short lengthe disabled */ -+#define QMAN_FD_SL_ENABLE (1 << 14) /* short lengthe enabled */ -+ -+#define QDMA_FINAL_BIT_DISABLE (0 << 31) /* final bit disable */ -+#define QDMA_FINAL_BIT_ENABLE (1 << 31) /* final bit enable */ -+ -+#define QDMA_FD_SHORT_FORMAT (1 << 11) /* short format */ -+#define QDMA_FD_LONG_FORMAT (0 << 11) /* long format */ -+#define QDMA_SER_DISABLE (0 << 8) /* no notification */ -+#define QDMA_SER_CTX (1 << 8) /* notification by FQD_CTX[fqid] */ -+#define QDMA_SER_DEST (2 << 8) /* notification by destination desc */ -+#define QDMA_SER_BOTH (3 << 8) /* soruce and dest notification */ -+#define QDMA_FD_SPF_ENALBE (1 << 30) /* source prefetch enable */ -+ -+#define QMAN_FD_VA_ENABLE (1 << 14) /* Address used is virtual address */ -+#define QMAN_FD_VA_DISABLE (0 << 14)/* Address used is a real address */ -+#define QMAN_FD_CBMT_ENABLE (1 << 15) /* Flow Context: 49bit physical address */ -+#define QMAN_FD_CBMT_DISABLE (0 << 15) /* Flow Context: 64bit virtual address */ -+#define QMAN_FD_SC_DISABLE (0 << 27) /* stashing control */ -+ -+#define QDMA_FL_FMT_SBF (0x0) /* Single buffer frame */ -+#define QDMA_FL_FMT_SGE 0x2 /* Scatter gather frame */ -+#define QDMA_FL_BMT_ENABLE (0x1 << 15)/* enable bypass memory translation */ -+#define QDMA_FL_BMT_DISABLE 0x0 /* enable bypass memory translation */ -+#define QDMA_FL_SL_LONG (0x0 << 2)/* long length */ -+#define QDMA_FL_SL_SHORT 0x1 /* short length */ -+#define QDMA_FL_F (0x1)/* last frame list bit */ -+/*Description of Frame list table structure*/ -+ -+struct dpaa2_qdma_chan { -+ struct virt_dma_chan vchan; -+ struct virt_dma_desc vdesc; -+ enum dma_status status; -+ struct dpaa2_qdma_engine *qdma; -+ -+ struct mutex dpaa2_queue_mutex; -+ spinlock_t queue_lock; -+ struct dma_pool *fd_pool; -+ -+ struct list_head comp_used; -+ struct list_head comp_free; -+ -+}; -+ -+struct dpaa2_qdma_comp { -+ dma_addr_t fd_bus_addr; -+ dma_addr_t fl_bus_addr; -+ dma_addr_t desc_bus_addr; -+ void *fd_virt_addr; -+ void *fl_virt_addr; -+ void *desc_virt_addr; -+ struct dpaa2_qdma_chan *qchan; -+ struct virt_dma_desc vdesc; -+ struct list_head list; -+}; -+ -+struct dpaa2_qdma_engine { -+ struct dma_device dma_dev; -+ u32 n_chans; -+ struct dpaa2_qdma_chan chans[NUM_CH]; -+ bool qdma_wrtype_fixup; -+ -+ struct dpaa2_qdma_priv *priv; -+}; -+ -+/* -+ * dpaa2_qdma_priv - driver private data -+ */ -+struct dpaa2_qdma_priv { -+ int dpqdma_id; -+ -+ struct iommu_domain *iommu_domain; -+ struct dpdmai_attr dpdmai_attr; -+ struct device *dev; -+ struct fsl_mc_io *mc_io; -+ struct fsl_mc_device *dpdmai_dev; -+ -+ struct dpdmai_rx_queue_attr rx_queue_attr[DPDMAI_PRIO_NUM]; -+ struct dpdmai_tx_queue_attr tx_queue_attr[DPDMAI_PRIO_NUM]; -+ -+ uint8_t num_pairs; -+ -+ struct dpaa2_qdma_engine *dpaa2_qdma; -+ struct dpaa2_qdma_priv_per_prio *ppriv; -+}; -+ -+struct dpaa2_qdma_priv_per_prio { -+ int req_fqid; -+ int rsp_fqid; -+ int prio; -+ -+ struct dpaa2_io_store *store; -+ struct dpaa2_io_notification_ctx nctx; -+ -+ struct dpaa2_qdma_priv *priv; -+}; -+ -+static struct soc_device_attribute soc_fixup_tuning[] = { -+ { .family = "QorIQ LX2160A"}, -+ { }, -+}; -+ -+/* FD pool size: one FD + 3 Frame list + 2 source/destination descriptor */ -+#define FD_POOL_SIZE (sizeof(struct dpaa2_fd) + \ -+ sizeof(struct dpaa2_fl_entry) * 3 + \ -+ sizeof(struct dpaa2_qdma_sd_d) * 2) -+ -+#endif /* __DPAA2_QDMA_H */ ---- /dev/null -+++ b/drivers/dma/dpaa2-qdma/dpdmai.c -@@ -0,0 +1,515 @@ -+/* Copyright 2013-2015 Freescale Semiconductor Inc. -+ * -+ * Redistribution and use in source and binary forms, with or without -+ * modification, are permitted provided that the following conditions are met: -+ * * Redistributions of source code must retain the above copyright -+ * notice, this list of conditions and the following disclaimer. -+ * * Redistributions in binary form must reproduce the above copyright -+ * notice, this list of conditions and the following disclaimer in the -+ * documentation and/or other materials provided with the distribution. -+ * * Neither the name of the above-listed copyright holders nor the -+ * names of any contributors may be used to endorse or promote products -+ * derived from this software without specific prior written permission. -+ * -+ * -+ * ALTERNATIVELY, this software may be distributed under the terms of the -+ * GNU General Public License ("GPL") as published by the Free Software -+ * Foundation, either version 2 of that License or (at your option) any -+ * later version. -+ * -+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE -+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -+ * POSSIBILITY OF SUCH DAMAGE. -+ */ -+#include <linux/types.h> -+#include <linux/io.h> -+#include "fsl_dpdmai.h" -+#include "fsl_dpdmai_cmd.h" -+#include <linux/fsl/mc.h> -+ -+struct dpdmai_cmd_open { -+ __le32 dpdmai_id; -+}; -+ -+struct dpdmai_rsp_get_attributes { -+ __le32 id; -+ u8 num_of_priorities; -+ u8 pad0[3]; -+ __le16 major; -+ __le16 minor; -+}; -+ -+ -+struct dpdmai_cmd_queue { -+ __le32 dest_id; -+ u8 priority; -+ u8 queue; -+ u8 dest_type; -+ u8 pad; -+ __le64 user_ctx; -+ union { -+ __le32 options; -+ __le32 fqid; -+ }; -+}; -+ -+struct dpdmai_rsp_get_tx_queue { -+ __le64 pad; -+ __le32 fqid; -+}; -+ -+ -+int dpdmai_open(struct fsl_mc_io *mc_io, -+ uint32_t cmd_flags, -+ int dpdmai_id, -+ uint16_t *token) -+{ -+ struct fsl_mc_command cmd = { 0 }; -+ struct dpdmai_cmd_open *cmd_params; -+ int err; -+ -+ /* prepare command */ -+ cmd.header = mc_encode_cmd_header(DPDMAI_CMDID_OPEN, -+ cmd_flags, -+ 0); -+ -+ cmd_params = (struct dpdmai_cmd_open *)cmd.params; -+ cmd_params->dpdmai_id = cpu_to_le32(dpdmai_id); -+ -+ /* send command to mc*/ -+ err = mc_send_command(mc_io, &cmd); -+ if (err) -+ return err; -+ -+ /* retrieve response parameters */ -+ *token = mc_cmd_hdr_read_token(&cmd); -+ return 0; -+} -+ -+int dpdmai_close(struct fsl_mc_io *mc_io, -+ uint32_t cmd_flags, -+ uint16_t token) -+{ -+ struct fsl_mc_command cmd = { 0 }; -+ -+ /* prepare command */ -+ cmd.header = mc_encode_cmd_header(DPDMAI_CMDID_CLOSE, -+ cmd_flags, token); -+ -+ /* send command to mc*/ -+ return mc_send_command(mc_io, &cmd); -+} -+ -+int dpdmai_create(struct fsl_mc_io *mc_io, -+ uint32_t cmd_flags, -+ const struct dpdmai_cfg *cfg, -+ uint16_t *token) -+{ -+ struct fsl_mc_command cmd = { 0 }; -+ int err; -+ -+ /* prepare command */ -+ cmd.header = mc_encode_cmd_header(DPDMAI_CMDID_CREATE, -+ cmd_flags, -+ 0); -+ DPDMAI_CMD_CREATE(cmd, cfg); -+ -+ /* send command to mc*/ -+ err = mc_send_command(mc_io, &cmd); -+ if (err) -+ return err; -+ -+ /* retrieve response parameters */ -+ *token = MC_CMD_HDR_READ_TOKEN(cmd.header); -+ -+ return 0; -+} -+ -+int dpdmai_destroy(struct fsl_mc_io *mc_io, -+ uint32_t cmd_flags, -+ uint16_t token) -+{ -+ struct fsl_mc_command cmd = { 0 }; -+ -+ /* prepare command */ -+ cmd.header = mc_encode_cmd_header(DPDMAI_CMDID_DESTROY, -+ cmd_flags, -+ token); -+ -+ /* send command to mc*/ -+ return mc_send_command(mc_io, &cmd); -+} -+ -+int dpdmai_enable(struct fsl_mc_io *mc_io, -+ uint32_t cmd_flags, -+ uint16_t token) -+{ -+ struct fsl_mc_command cmd = { 0 }; -+ -+ /* prepare command */ -+ cmd.header = mc_encode_cmd_header(DPDMAI_CMDID_ENABLE, -+ cmd_flags, -+ token); -+ -+ /* send command to mc*/ -+ return mc_send_command(mc_io, &cmd); -+} -+ -+int dpdmai_disable(struct fsl_mc_io *mc_io, -+ uint32_t cmd_flags, -+ uint16_t token) -+{ -+ struct fsl_mc_command cmd = { 0 }; -+ -+ /* prepare command */ -+ cmd.header = mc_encode_cmd_header(DPDMAI_CMDID_DISABLE, -+ cmd_flags, -+ token); -+ -+ /* send command to mc*/ -+ return mc_send_command(mc_io, &cmd); -+} -+ -+int dpdmai_is_enabled(struct fsl_mc_io *mc_io, -+ uint32_t cmd_flags, -+ uint16_t token, -+ int *en) -+{ -+ struct fsl_mc_command cmd = { 0 }; -+ int err; -+ /* prepare command */ -+ cmd.header = mc_encode_cmd_header(DPDMAI_CMDID_IS_ENABLED, -+ cmd_flags, -+ token); -+ -+ /* send command to mc*/ -+ err = mc_send_command(mc_io, &cmd); -+ if (err) -+ return err; -+ -+ /* retrieve response parameters */ -+ DPDMAI_RSP_IS_ENABLED(cmd, *en); -+ -+ return 0; -+} -+ -+int dpdmai_reset(struct fsl_mc_io *mc_io, -+ uint32_t cmd_flags, -+ uint16_t token) -+{ -+ struct fsl_mc_command cmd = { 0 }; -+ -+ /* prepare command */ -+ cmd.header = mc_encode_cmd_header(DPDMAI_CMDID_RESET, -+ cmd_flags, -+ token); -+ -+ /* send command to mc*/ -+ return mc_send_command(mc_io, &cmd); -+} -+ -+int dpdmai_get_irq(struct fsl_mc_io *mc_io, -+ uint32_t cmd_flags, -+ uint16_t token, -+ uint8_t irq_index, -+ int *type, -+ struct dpdmai_irq_cfg *irq_cfg) -+{ -+ struct fsl_mc_command cmd = { 0 }; -+ int err; -+ -+ /* prepare command */ -+ cmd.header = mc_encode_cmd_header(DPDMAI_CMDID_GET_IRQ, -+ cmd_flags, -+ token); -+ DPDMAI_CMD_GET_IRQ(cmd, irq_index); -+ -+ /* send command to mc*/ -+ err = mc_send_command(mc_io, &cmd); -+ if (err) -+ return err; -+ -+ /* retrieve response parameters */ -+ DPDMAI_RSP_GET_IRQ(cmd, *type, irq_cfg); -+ -+ return 0; -+} -+ -+int dpdmai_set_irq(struct fsl_mc_io *mc_io, -+ uint32_t cmd_flags, -+ uint16_t token, -+ uint8_t irq_index, -+ struct dpdmai_irq_cfg *irq_cfg) -+{ -+ struct fsl_mc_command cmd = { 0 }; -+ -+ /* prepare command */ -+ cmd.header = mc_encode_cmd_header(DPDMAI_CMDID_SET_IRQ, -+ cmd_flags, -+ token); -+ DPDMAI_CMD_SET_IRQ(cmd, irq_index, irq_cfg); -+ -+ /* send command to mc*/ -+ return mc_send_command(mc_io, &cmd); -+} -+ -+int dpdmai_get_irq_enable(struct fsl_mc_io *mc_io, -+ uint32_t cmd_flags, -+ uint16_t token, -+ uint8_t irq_index, -+ uint8_t *en) -+{ -+ struct fsl_mc_command cmd = { 0 }; -+ int err; -+ -+ /* prepare command */ -+ cmd.header = mc_encode_cmd_header(DPDMAI_CMDID_GET_IRQ_ENABLE, -+ cmd_flags, -+ token); -+ DPDMAI_CMD_GET_IRQ_ENABLE(cmd, irq_index); -+ -+ /* send command to mc*/ -+ err = mc_send_command(mc_io, &cmd); -+ if (err) -+ return err; -+ -+ /* retrieve response parameters */ -+ DPDMAI_RSP_GET_IRQ_ENABLE(cmd, *en); -+ -+ return 0; -+} -+ -+int dpdmai_set_irq_enable(struct fsl_mc_io *mc_io, -+ uint32_t cmd_flags, -+ uint16_t token, -+ uint8_t irq_index, -+ uint8_t en) -+{ -+ struct fsl_mc_command cmd = { 0 }; -+ -+ /* prepare command */ -+ cmd.header = mc_encode_cmd_header(DPDMAI_CMDID_SET_IRQ_ENABLE, -+ cmd_flags, -+ token); -+ DPDMAI_CMD_SET_IRQ_ENABLE(cmd, irq_index, en); -+ -+ /* send command to mc*/ -+ return mc_send_command(mc_io, &cmd); -+} -+ -+int dpdmai_get_irq_mask(struct fsl_mc_io *mc_io, -+ uint32_t cmd_flags, -+ uint16_t token, -+ uint8_t irq_index, -+ uint32_t *mask) -+{ -+ struct fsl_mc_command cmd = { 0 }; -+ int err; -+ -+ /* prepare command */ -+ cmd.header = mc_encode_cmd_header(DPDMAI_CMDID_GET_IRQ_MASK, -+ cmd_flags, -+ token); -+ DPDMAI_CMD_GET_IRQ_MASK(cmd, irq_index); -+ -+ /* send command to mc*/ -+ err = mc_send_command(mc_io, &cmd); -+ if (err) -+ return err; -+ -+ /* retrieve response parameters */ -+ DPDMAI_RSP_GET_IRQ_MASK(cmd, *mask); -+ -+ return 0; -+} -+ -+int dpdmai_set_irq_mask(struct fsl_mc_io *mc_io, -+ uint32_t cmd_flags, -+ uint16_t token, -+ uint8_t irq_index, -+ uint32_t mask) -+{ -+ struct fsl_mc_command cmd = { 0 }; -+ -+ /* prepare command */ -+ cmd.header = mc_encode_cmd_header(DPDMAI_CMDID_SET_IRQ_MASK, -+ cmd_flags, -+ token); -+ DPDMAI_CMD_SET_IRQ_MASK(cmd, irq_index, mask); -+ -+ /* send command to mc*/ -+ return mc_send_command(mc_io, &cmd); -+} -+ -+int dpdmai_get_irq_status(struct fsl_mc_io *mc_io, -+ uint32_t cmd_flags, -+ uint16_t token, -+ uint8_t irq_index, -+ uint32_t *status) -+{ -+ struct fsl_mc_command cmd = { 0 }; -+ int err; -+ -+ /* prepare command */ -+ cmd.header = mc_encode_cmd_header(DPDMAI_CMDID_GET_IRQ_STATUS, -+ cmd_flags, -+ token); -+ DPDMAI_CMD_GET_IRQ_STATUS(cmd, irq_index, *status); -+ -+ /* send command to mc*/ -+ err = mc_send_command(mc_io, &cmd); -+ if (err) -+ return err; -+ -+ /* retrieve response parameters */ -+ DPDMAI_RSP_GET_IRQ_STATUS(cmd, *status); -+ -+ return 0; -+} -+ -+int dpdmai_clear_irq_status(struct fsl_mc_io *mc_io, -+ uint32_t cmd_flags, -+ uint16_t token, -+ uint8_t irq_index, -+ uint32_t status) -+{ -+ struct fsl_mc_command cmd = { 0 }; -+ -+ /* prepare command */ -+ cmd.header = mc_encode_cmd_header(DPDMAI_CMDID_CLEAR_IRQ_STATUS, -+ cmd_flags, -+ token); -+ DPDMAI_CMD_CLEAR_IRQ_STATUS(cmd, irq_index, status); -+ -+ /* send command to mc*/ -+ return mc_send_command(mc_io, &cmd); -+} -+ -+int dpdmai_get_attributes(struct fsl_mc_io *mc_io, -+ uint32_t cmd_flags, -+ uint16_t token, -+ struct dpdmai_attr *attr) -+{ -+ struct fsl_mc_command cmd = { 0 }; -+ int err; -+ struct dpdmai_rsp_get_attributes *rsp_params; -+ -+ /* prepare command */ -+ cmd.header = mc_encode_cmd_header(DPDMAI_CMDID_GET_ATTR, -+ cmd_flags, -+ token); -+ -+ /* send command to mc*/ -+ err = mc_send_command(mc_io, &cmd); -+ if (err) -+ return err; -+ -+ /* retrieve response parameters */ -+ rsp_params = (struct dpdmai_rsp_get_attributes *)cmd.params; -+ attr->id = le32_to_cpu(rsp_params->id); -+ attr->version.major = le16_to_cpu(rsp_params->major); -+ attr->version.minor = le16_to_cpu(rsp_params->minor); -+ attr->num_of_priorities = rsp_params->num_of_priorities; -+ -+ -+ return 0; -+} -+ -+int dpdmai_set_rx_queue(struct fsl_mc_io *mc_io, -+ uint32_t cmd_flags, -+ uint16_t token, -+ uint8_t priority, -+ const struct dpdmai_rx_queue_cfg *cfg) -+{ -+ struct fsl_mc_command cmd = { 0 }; -+ struct dpdmai_cmd_queue *cmd_params; -+ -+ /* prepare command */ -+ cmd.header = mc_encode_cmd_header(DPDMAI_CMDID_SET_RX_QUEUE, -+ cmd_flags, -+ token); -+ -+ cmd_params = (struct dpdmai_cmd_queue *)cmd.params; -+ cmd_params->dest_id = cpu_to_le32(cfg->dest_cfg.dest_id); -+ cmd_params->priority = cfg->dest_cfg.priority; -+ cmd_params->queue = priority; -+ cmd_params->dest_type = cfg->dest_cfg.dest_type; -+ cmd_params->user_ctx = cpu_to_le64(cfg->user_ctx); -+ cmd_params->options = cpu_to_le32(cfg->options); -+ -+ -+ /* send command to mc*/ -+ return mc_send_command(mc_io, &cmd); -+} -+ -+int dpdmai_get_rx_queue(struct fsl_mc_io *mc_io, -+ uint32_t cmd_flags, -+ uint16_t token, -+ uint8_t priority, struct dpdmai_rx_queue_attr *attr) -+{ -+ struct fsl_mc_command cmd = { 0 }; -+ struct dpdmai_cmd_queue *cmd_params; -+ int err; -+ -+ /* prepare command */ -+ cmd.header = mc_encode_cmd_header(DPDMAI_CMDID_GET_RX_QUEUE, -+ cmd_flags, -+ token); -+ -+ cmd_params = (struct dpdmai_cmd_queue *)cmd.params; -+ cmd_params->queue = priority; -+ -+ /* send command to mc*/ -+ err = mc_send_command(mc_io, &cmd); -+ if (err) -+ return err; -+ -+ /* retrieve response parameters */ -+ attr->dest_cfg.dest_id = le32_to_cpu(cmd_params->dest_id); -+ attr->dest_cfg.priority = cmd_params->priority; -+ attr->dest_cfg.dest_type = cmd_params->dest_type; -+ attr->user_ctx = le64_to_cpu(cmd_params->user_ctx); -+ attr->fqid = le32_to_cpu(cmd_params->fqid); -+ -+ return 0; -+} -+ -+int dpdmai_get_tx_queue(struct fsl_mc_io *mc_io, -+ uint32_t cmd_flags, -+ uint16_t token, -+ uint8_t priority, -+ struct dpdmai_tx_queue_attr *attr) -+{ -+ struct fsl_mc_command cmd = { 0 }; -+ struct dpdmai_cmd_queue *cmd_params; -+ struct dpdmai_rsp_get_tx_queue *rsp_params; -+ int err; -+ -+ /* prepare command */ -+ cmd.header = mc_encode_cmd_header(DPDMAI_CMDID_GET_TX_QUEUE, -+ cmd_flags, -+ token); -+ -+ cmd_params = (struct dpdmai_cmd_queue *)cmd.params; -+ cmd_params->queue = priority; -+ -+ /* send command to mc*/ -+ err = mc_send_command(mc_io, &cmd); -+ if (err) -+ return err; -+ -+ /* retrieve response parameters */ -+ -+ rsp_params = (struct dpdmai_rsp_get_tx_queue *)cmd.params; -+ attr->fqid = le32_to_cpu(rsp_params->fqid); -+ -+ return 0; -+} ---- /dev/null -+++ b/drivers/dma/dpaa2-qdma/fsl_dpdmai.h -@@ -0,0 +1,521 @@ -+/* Copyright 2013-2015 Freescale Semiconductor Inc. -+ * -+ * Redistribution and use in source and binary forms, with or without -+ * modification, are permitted provided that the following conditions are met: -+ * * Redistributions of source code must retain the above copyright -+ * notice, this list of conditions and the following disclaimer. -+ * * Redistributions in binary form must reproduce the above copyright -+ * notice, this list of conditions and the following disclaimer in the -+ * documentation and/or other materials provided with the distribution. -+ * * Neither the name of the above-listed copyright holders nor the -+ * names of any contributors may be used to endorse or promote products -+ * derived from this software without specific prior written permission. -+ * -+ * -+ * ALTERNATIVELY, this software may be distributed under the terms of the -+ * GNU General Public License ("GPL") as published by the Free Software -+ * Foundation, either version 2 of that License or (at your option) any -+ * later version. -+ * -+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE -+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -+ * POSSIBILITY OF SUCH DAMAGE. -+ */ -+#ifndef __FSL_DPDMAI_H -+#define __FSL_DPDMAI_H -+ -+struct fsl_mc_io; -+ -+/* Data Path DMA Interface API -+ * Contains initialization APIs and runtime control APIs for DPDMAI -+ */ -+ -+/* General DPDMAI macros */ -+ -+/** -+ * Maximum number of Tx/Rx priorities per DPDMAI object -+ */ -+#define DPDMAI_PRIO_NUM 2 -+ -+/** -+ * All queues considered; see dpdmai_set_rx_queue() -+ */ -+#define DPDMAI_ALL_QUEUES (uint8_t)(-1) -+ -+/** -+ * dpdmai_open() - Open a control session for the specified object -+ * @mc_io: Pointer to MC portal's I/O object -+ * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' -+ * @dpdmai_id: DPDMAI unique ID -+ * @token: Returned token; use in subsequent API calls -+ * -+ * This function can be used to open a control session for an -+ * already created object; an object may have been declared in -+ * the DPL or by calling the dpdmai_create() function. -+ * This function returns a unique authentication token, -+ * associated with the specific object ID and the specific MC -+ * portal; this token must be used in all subsequent commands for -+ * this specific object. -+ * -+ * Return: '0' on Success; Error code otherwise. -+ */ -+int dpdmai_open(struct fsl_mc_io *mc_io, -+ uint32_t cmd_flags, -+ int dpdmai_id, -+ uint16_t *token); -+ -+/** -+ * dpdmai_close() - Close the control session of the object -+ * @mc_io: Pointer to MC portal's I/O object -+ * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' -+ * @token: Token of DPDMAI object -+ * -+ * After this function is called, no further operations are -+ * allowed on the object without opening a new control session. -+ * -+ * Return: '0' on Success; Error code otherwise. -+ */ -+int dpdmai_close(struct fsl_mc_io *mc_io, -+ uint32_t cmd_flags, -+ uint16_t token); -+ -+/** -+ * struct dpdmai_cfg - Structure representing DPDMAI configuration -+ * @priorities: Priorities for the DMA hardware processing; valid priorities are -+ * configured with values 1-8; the entry following last valid entry -+ * should be configured with 0 -+ */ -+struct dpdmai_cfg { -+ uint8_t priorities[DPDMAI_PRIO_NUM]; -+}; -+ -+/** -+ * dpdmai_create() - Create the DPDMAI object -+ * @mc_io: Pointer to MC portal's I/O object -+ * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' -+ * @cfg: Configuration structure -+ * @token: Returned token; use in subsequent API calls -+ * -+ * Create the DPDMAI object, allocate required resources and -+ * perform required initialization. -+ * -+ * The object can be created either by declaring it in the -+ * DPL file, or by calling this function. -+ * -+ * This function returns a unique authentication token, -+ * associated with the specific object ID and the specific MC -+ * portal; this token must be used in all subsequent calls to -+ * this specific object. For objects that are created using the -+ * DPL file, call dpdmai_open() function to get an authentication -+ * token first. -+ * -+ * Return: '0' on Success; Error code otherwise. -+ */ -+int dpdmai_create(struct fsl_mc_io *mc_io, -+ uint32_t cmd_flags, -+ const struct dpdmai_cfg *cfg, -+ uint16_t *token); -+ -+/** -+ * dpdmai_destroy() - Destroy the DPDMAI object and release all its resources. -+ * @mc_io: Pointer to MC portal's I/O object -+ * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' -+ * @token: Token of DPDMAI object -+ * -+ * Return: '0' on Success; error code otherwise. -+ */ -+int dpdmai_destroy(struct fsl_mc_io *mc_io, -+ uint32_t cmd_flags, -+ uint16_t token); -+ -+/** -+ * dpdmai_enable() - Enable the DPDMAI, allow sending and receiving frames. -+ * @mc_io: Pointer to MC portal's I/O object -+ * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' -+ * @token: Token of DPDMAI object -+ * -+ * Return: '0' on Success; Error code otherwise. -+ */ -+int dpdmai_enable(struct fsl_mc_io *mc_io, -+ uint32_t cmd_flags, -+ uint16_t token); -+ -+/** -+ * dpdmai_disable() - Disable the DPDMAI, stop sending and receiving frames. -+ * @mc_io: Pointer to MC portal's I/O object -+ * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' -+ * @token: Token of DPDMAI object -+ * -+ * Return: '0' on Success; Error code otherwise. -+ */ -+int dpdmai_disable(struct fsl_mc_io *mc_io, -+ uint32_t cmd_flags, -+ uint16_t token); -+ -+/** -+ * dpdmai_is_enabled() - Check if the DPDMAI is enabled. -+ * @mc_io: Pointer to MC portal's I/O object -+ * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' -+ * @token: Token of DPDMAI object -+ * @en: Returns '1' if object is enabled; '0' otherwise -+ * -+ * Return: '0' on Success; Error code otherwise. -+ */ -+int dpdmai_is_enabled(struct fsl_mc_io *mc_io, -+ uint32_t cmd_flags, -+ uint16_t token, -+ int *en); -+ -+/** -+ * dpdmai_reset() - Reset the DPDMAI, returns the object to initial state. -+ * @mc_io: Pointer to MC portal's I/O object -+ * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' -+ * @token: Token of DPDMAI object -+ * -+ * Return: '0' on Success; Error code otherwise. -+ */ -+int dpdmai_reset(struct fsl_mc_io *mc_io, -+ uint32_t cmd_flags, -+ uint16_t token); -+ -+/** -+ * struct dpdmai_irq_cfg - IRQ configuration -+ * @addr: Address that must be written to signal a message-based interrupt -+ * @val: Value to write into irq_addr address -+ * @irq_num: A user defined number associated with this IRQ -+ */ -+struct dpdmai_irq_cfg { -+ uint64_t addr; -+ uint32_t val; -+ int irq_num; -+}; -+ -+/** -+ * dpdmai_set_irq() - Set IRQ information for the DPDMAI to trigger an interrupt. -+ * @mc_io: Pointer to MC portal's I/O object -+ * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' -+ * @token: Token of DPDMAI object -+ * @irq_index: Identifies the interrupt index to configure -+ * @irq_cfg: IRQ configuration -+ * -+ * Return: '0' on Success; Error code otherwise. -+ */ -+int dpdmai_set_irq(struct fsl_mc_io *mc_io, -+ uint32_t cmd_flags, -+ uint16_t token, -+ uint8_t irq_index, -+ struct dpdmai_irq_cfg *irq_cfg); -+ -+/** -+ * dpdmai_get_irq() - Get IRQ information from the DPDMAI -+ * -+ * @mc_io: Pointer to MC portal's I/O object -+ * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' -+ * @token: Token of DPDMAI object -+ * @irq_index: The interrupt index to configure -+ * @type: Interrupt type: 0 represents message interrupt -+ * type (both irq_addr and irq_val are valid) -+ * @irq_cfg: IRQ attributes -+ * -+ * Return: '0' on Success; Error code otherwise. -+ */ -+int dpdmai_get_irq(struct fsl_mc_io *mc_io, -+ uint32_t cmd_flags, -+ uint16_t token, -+ uint8_t irq_index, -+ int *type, -+ struct dpdmai_irq_cfg *irq_cfg); -+ -+/** -+ * dpdmai_set_irq_enable() - Set overall interrupt state. -+ * @mc_io: Pointer to MC portal's I/O object -+ * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' -+ * @token: Token of DPDMAI object -+ * @irq_index: The interrupt index to configure -+ * @en: Interrupt state - enable = 1, disable = 0 -+ * -+ * Allows GPP software to control when interrupts are generated. -+ * Each interrupt can have up to 32 causes. The enable/disable control's the -+ * overall interrupt state. if the interrupt is disabled no causes will cause -+ * an interrupt -+ * -+ * Return: '0' on Success; Error code otherwise. -+ */ -+int dpdmai_set_irq_enable(struct fsl_mc_io *mc_io, -+ uint32_t cmd_flags, -+ uint16_t token, -+ uint8_t irq_index, -+ uint8_t en); -+ -+/** -+ * dpdmai_get_irq_enable() - Get overall interrupt state -+ * @mc_io: Pointer to MC portal's I/O object -+ * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' -+ * @token: Token of DPDMAI object -+ * @irq_index: The interrupt index to configure -+ * @en: Returned Interrupt state - enable = 1, disable = 0 -+ * -+ * Return: '0' on Success; Error code otherwise. -+ */ -+int dpdmai_get_irq_enable(struct fsl_mc_io *mc_io, -+ uint32_t cmd_flags, -+ uint16_t token, -+ uint8_t irq_index, -+ uint8_t *en); -+ -+/** -+ * dpdmai_set_irq_mask() - Set interrupt mask. -+ * @mc_io: Pointer to MC portal's I/O object -+ * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' -+ * @token: Token of DPDMAI object -+ * @irq_index: The interrupt index to configure -+ * @mask: event mask to trigger interrupt; -+ * each bit: -+ * 0 = ignore event -+ * 1 = consider event for asserting IRQ -+ * -+ * Every interrupt can have up to 32 causes and the interrupt model supports -+ * masking/unmasking each cause independently -+ * -+ * Return: '0' on Success; Error code otherwise. -+ */ -+int dpdmai_set_irq_mask(struct fsl_mc_io *mc_io, -+ uint32_t cmd_flags, -+ uint16_t token, -+ uint8_t irq_index, -+ uint32_t mask); -+ -+/** -+ * dpdmai_get_irq_mask() - Get interrupt mask. -+ * @mc_io: Pointer to MC portal's I/O object -+ * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' -+ * @token: Token of DPDMAI object -+ * @irq_index: The interrupt index to configure -+ * @mask: Returned event mask to trigger interrupt -+ * -+ * Every interrupt can have up to 32 causes and the interrupt model supports -+ * masking/unmasking each cause independently -+ * -+ * Return: '0' on Success; Error code otherwise. -+ */ -+int dpdmai_get_irq_mask(struct fsl_mc_io *mc_io, -+ uint32_t cmd_flags, -+ uint16_t token, -+ uint8_t irq_index, -+ uint32_t *mask); -+ -+/** -+ * dpdmai_get_irq_status() - Get the current status of any pending interrupts -+ * @mc_io: Pointer to MC portal's I/O object -+ * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' -+ * @token: Token of DPDMAI object -+ * @irq_index: The interrupt index to configure -+ * @status: Returned interrupts status - one bit per cause: -+ * 0 = no interrupt pending -+ * 1 = interrupt pending -+ * -+ * Return: '0' on Success; Error code otherwise. -+ */ -+int dpdmai_get_irq_status(struct fsl_mc_io *mc_io, -+ uint32_t cmd_flags, -+ uint16_t token, -+ uint8_t irq_index, -+ uint32_t *status); -+ -+/** -+ * dpdmai_clear_irq_status() - Clear a pending interrupt's status -+ * @mc_io: Pointer to MC portal's I/O object -+ * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' -+ * @token: Token of DPDMAI object -+ * @irq_index: The interrupt index to configure -+ * @status: bits to clear (W1C) - one bit per cause: -+ * 0 = don't change -+ * 1 = clear status bit -+ * -+ * Return: '0' on Success; Error code otherwise. -+ */ -+int dpdmai_clear_irq_status(struct fsl_mc_io *mc_io, -+ uint32_t cmd_flags, -+ uint16_t token, -+ uint8_t irq_index, -+ uint32_t status); -+ -+/** -+ * struct dpdmai_attr - Structure representing DPDMAI attributes -+ * @id: DPDMAI object ID -+ * @version: DPDMAI version -+ * @num_of_priorities: number of priorities -+ */ -+struct dpdmai_attr { -+ int id; -+ /** -+ * struct version - DPDMAI version -+ * @major: DPDMAI major version -+ * @minor: DPDMAI minor version -+ */ -+ struct { -+ uint16_t major; -+ uint16_t minor; -+ } version; -+ uint8_t num_of_priorities; -+}; -+ -+/** -+ * dpdmai_get_attributes() - Retrieve DPDMAI attributes. -+ * @mc_io: Pointer to MC portal's I/O object -+ * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' -+ * @token: Token of DPDMAI object -+ * @attr: Returned object's attributes -+ * -+ * Return: '0' on Success; Error code otherwise. -+ */ -+int dpdmai_get_attributes(struct fsl_mc_io *mc_io, -+ uint32_t cmd_flags, -+ uint16_t token, -+ struct dpdmai_attr *attr); -+ -+/** -+ * enum dpdmai_dest - DPDMAI destination types -+ * @DPDMAI_DEST_NONE: Unassigned destination; The queue is set in parked mode -+ * and does not generate FQDAN notifications; user is expected to dequeue -+ * from the queue based on polling or other user-defined method -+ * @DPDMAI_DEST_DPIO: The queue is set in schedule mode and generates FQDAN -+ * notifications to the specified DPIO; user is expected to dequeue -+ * from the queue only after notification is received -+ * @DPDMAI_DEST_DPCON: The queue is set in schedule mode and does not generate -+ * FQDAN notifications, but is connected to the specified DPCON object; -+ * user is expected to dequeue from the DPCON channel -+ */ -+enum dpdmai_dest { -+ DPDMAI_DEST_NONE = 0, -+ DPDMAI_DEST_DPIO = 1, -+ DPDMAI_DEST_DPCON = 2 -+}; -+ -+/** -+ * struct dpdmai_dest_cfg - Structure representing DPDMAI destination parameters -+ * @dest_type: Destination type -+ * @dest_id: Either DPIO ID or DPCON ID, depending on the destination type -+ * @priority: Priority selection within the DPIO or DPCON channel; valid values -+ * are 0-1 or 0-7, depending on the number of priorities in that -+ * channel; not relevant for 'DPDMAI_DEST_NONE' option -+ */ -+struct dpdmai_dest_cfg { -+ enum dpdmai_dest dest_type; -+ int dest_id; -+ uint8_t priority; -+}; -+ -+/* DPDMAI queue modification options */ -+ -+/** -+ * Select to modify the user's context associated with the queue -+ */ -+#define DPDMAI_QUEUE_OPT_USER_CTX 0x00000001 -+ -+/** -+ * Select to modify the queue's destination -+ */ -+#define DPDMAI_QUEUE_OPT_DEST 0x00000002 -+ -+/** -+ * struct dpdmai_rx_queue_cfg - DPDMAI RX queue configuration -+ * @options: Flags representing the suggested modifications to the queue; -+ * Use any combination of 'DPDMAI_QUEUE_OPT_<X>' flags -+ * @user_ctx: User context value provided in the frame descriptor of each -+ * dequeued frame; -+ * valid only if 'DPDMAI_QUEUE_OPT_USER_CTX' is contained in 'options' -+ * @dest_cfg: Queue destination parameters; -+ * valid only if 'DPDMAI_QUEUE_OPT_DEST' is contained in 'options' -+ */ -+struct dpdmai_rx_queue_cfg { -+ uint32_t options; -+ uint64_t user_ctx; -+ struct dpdmai_dest_cfg dest_cfg; -+ -+}; -+ -+/** -+ * dpdmai_set_rx_queue() - Set Rx queue configuration -+ * @mc_io: Pointer to MC portal's I/O object -+ * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' -+ * @token: Token of DPDMAI object -+ * @priority: Select the queue relative to number of -+ * priorities configured at DPDMAI creation; use -+ * DPDMAI_ALL_QUEUES to configure all Rx queues -+ * identically. -+ * @cfg: Rx queue configuration -+ * -+ * Return: '0' on Success; Error code otherwise. -+ */ -+int dpdmai_set_rx_queue(struct fsl_mc_io *mc_io, -+ uint32_t cmd_flags, -+ uint16_t token, -+ uint8_t priority, -+ const struct dpdmai_rx_queue_cfg *cfg); -+ -+/** -+ * struct dpdmai_rx_queue_attr - Structure representing attributes of Rx queues -+ * @user_ctx: User context value provided in the frame descriptor of each -+ * dequeued frame -+ * @dest_cfg: Queue destination configuration -+ * @fqid: Virtual FQID value to be used for dequeue operations -+ */ -+struct dpdmai_rx_queue_attr { -+ uint64_t user_ctx; -+ struct dpdmai_dest_cfg dest_cfg; -+ uint32_t fqid; -+}; -+ -+/** -+ * dpdmai_get_rx_queue() - Retrieve Rx queue attributes. -+ * @mc_io: Pointer to MC portal's I/O object -+ * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' -+ * @token: Token of DPDMAI object -+ * @priority: Select the queue relative to number of -+ * priorities configured at DPDMAI creation -+ * @attr: Returned Rx queue attributes -+ * -+ * Return: '0' on Success; Error code otherwise. -+ */ -+int dpdmai_get_rx_queue(struct fsl_mc_io *mc_io, -+ uint32_t cmd_flags, -+ uint16_t token, -+ uint8_t priority, -+ struct dpdmai_rx_queue_attr *attr); -+ -+/** -+ * struct dpdmai_tx_queue_attr - Structure representing attributes of Tx queues -+ * @fqid: Virtual FQID to be used for sending frames to DMA hardware -+ */ -+ -+struct dpdmai_tx_queue_attr { -+ uint32_t fqid; -+}; -+ -+/** -+ * dpdmai_get_tx_queue() - Retrieve Tx queue attributes. -+ * @mc_io: Pointer to MC portal's I/O object -+ * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' -+ * @token: Token of DPDMAI object -+ * @priority: Select the queue relative to number of -+ * priorities configured at DPDMAI creation -+ * @attr: Returned Tx queue attributes -+ * -+ * Return: '0' on Success; Error code otherwise. -+ */ -+int dpdmai_get_tx_queue(struct fsl_mc_io *mc_io, -+ uint32_t cmd_flags, -+ uint16_t token, -+ uint8_t priority, -+ struct dpdmai_tx_queue_attr *attr); -+ -+#endif /* __FSL_DPDMAI_H */ ---- /dev/null -+++ b/drivers/dma/dpaa2-qdma/fsl_dpdmai_cmd.h -@@ -0,0 +1,222 @@ -+/* Copyright 2013-2016 Freescale Semiconductor Inc. -+ * -+ * Redistribution and use in source and binary forms, with or without -+ * modification, are permitted provided that the following conditions are met: -+ * * Redistributions of source code must retain the above copyright -+ * notice, this list of conditions and the following disclaimer. -+ * * Redistributions in binary form must reproduce the above copyright -+ * notice, this list of conditions and the following disclaimer in the -+ * documentation and/or other materials provided with the distribution. -+ * * Neither the name of the above-listed copyright holders nor the -+ * names of any contributors may be used to endorse or promote products -+ * derived from this software without specific prior written permission. -+ * -+ * -+ * ALTERNATIVELY, this software may be distributed under the terms of the -+ * GNU General Public License ("GPL") as published by the Free Software -+ * Foundation, either version 2 of that License or (at your option) any -+ * later version. -+ * -+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE -+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -+ * POSSIBILITY OF SUCH DAMAGE. -+ */ -+#ifndef _FSL_DPDMAI_CMD_H -+#define _FSL_DPDMAI_CMD_H -+ -+/* DPDMAI Version */ -+#define DPDMAI_VER_MAJOR 2 -+#define DPDMAI_VER_MINOR 2 -+ -+#define DPDMAI_CMD_BASE_VERSION 0 -+#define DPDMAI_CMD_ID_OFFSET 4 -+ -+/* Command IDs */ -+#define DPDMAI_CMDID_CLOSE ((0x800 << DPDMAI_CMD_ID_OFFSET) | DPDMAI_CMD_BASE_VERSION) -+#define DPDMAI_CMDID_OPEN ((0x80E << DPDMAI_CMD_ID_OFFSET) | DPDMAI_CMD_BASE_VERSION) -+#define DPDMAI_CMDID_CREATE ((0x90E << DPDMAI_CMD_ID_OFFSET) | DPDMAI_CMD_BASE_VERSION) -+#define DPDMAI_CMDID_DESTROY ((0x900 << DPDMAI_CMD_ID_OFFSET) | DPDMAI_CMD_BASE_VERSION) -+ -+#define DPDMAI_CMDID_ENABLE ((0x002 << DPDMAI_CMD_ID_OFFSET) | DPDMAI_CMD_BASE_VERSION) -+#define DPDMAI_CMDID_DISABLE ((0x003 << DPDMAI_CMD_ID_OFFSET) | DPDMAI_CMD_BASE_VERSION) -+#define DPDMAI_CMDID_GET_ATTR ((0x004 << DPDMAI_CMD_ID_OFFSET) | DPDMAI_CMD_BASE_VERSION) -+#define DPDMAI_CMDID_RESET ((0x005 << DPDMAI_CMD_ID_OFFSET) | DPDMAI_CMD_BASE_VERSION) -+#define DPDMAI_CMDID_IS_ENABLED ((0x006 << DPDMAI_CMD_ID_OFFSET) | DPDMAI_CMD_BASE_VERSION) -+ -+#define DPDMAI_CMDID_SET_IRQ ((0x010 << DPDMAI_CMD_ID_OFFSET) | DPDMAI_CMD_BASE_VERSION) -+#define DPDMAI_CMDID_GET_IRQ ((0x011 << DPDMAI_CMD_ID_OFFSET) | DPDMAI_CMD_BASE_VERSION) -+#define DPDMAI_CMDID_SET_IRQ_ENABLE ((0x012 << DPDMAI_CMD_ID_OFFSET) | DPDMAI_CMD_BASE_VERSION) -+#define DPDMAI_CMDID_GET_IRQ_ENABLE ((0x013 << DPDMAI_CMD_ID_OFFSET) | DPDMAI_CMD_BASE_VERSION) -+#define DPDMAI_CMDID_SET_IRQ_MASK ((0x014 << DPDMAI_CMD_ID_OFFSET) | DPDMAI_CMD_BASE_VERSION) -+#define DPDMAI_CMDID_GET_IRQ_MASK ((0x015 << DPDMAI_CMD_ID_OFFSET) | DPDMAI_CMD_BASE_VERSION) -+#define DPDMAI_CMDID_GET_IRQ_STATUS ((0x016 << DPDMAI_CMD_ID_OFFSET) | DPDMAI_CMD_BASE_VERSION) -+#define DPDMAI_CMDID_CLEAR_IRQ_STATUS ((0x017 << DPDMAI_CMD_ID_OFFSET) | DPDMAI_CMD_BASE_VERSION) -+ -+#define DPDMAI_CMDID_SET_RX_QUEUE ((0x1A0 << DPDMAI_CMD_ID_OFFSET) | DPDMAI_CMD_BASE_VERSION) -+#define DPDMAI_CMDID_GET_RX_QUEUE ((0x1A1 << DPDMAI_CMD_ID_OFFSET) | DPDMAI_CMD_BASE_VERSION) -+#define DPDMAI_CMDID_GET_TX_QUEUE ((0x1A2 << DPDMAI_CMD_ID_OFFSET) | DPDMAI_CMD_BASE_VERSION) -+ -+ -+#define MC_CMD_HDR_TOKEN_O 32 /* Token field offset */ -+#define MC_CMD_HDR_TOKEN_S 16 /* Token field size */ -+ -+ -+#define MAKE_UMASK64(_width) \ -+ ((uint64_t)((_width) < 64 ? ((uint64_t)1 << (_width)) - 1 : \ -+ (uint64_t)-1)) -+ -+static inline uint64_t mc_enc(int lsoffset, int width, uint64_t val) -+{ -+ return (uint64_t)(((uint64_t)val & MAKE_UMASK64(width)) << lsoffset); -+} -+ -+static inline uint64_t mc_dec(uint64_t val, int lsoffset, int width) -+{ -+ return (uint64_t)((val >> lsoffset) & MAKE_UMASK64(width)); -+} -+ -+#define MC_CMD_OP(_cmd, _param, _offset, _width, _type, _arg) \ -+ ((_cmd).params[_param] |= mc_enc((_offset), (_width), _arg)) -+ -+#define MC_RSP_OP(_cmd, _param, _offset, _width, _type, _arg) \ -+ (_arg = (_type)mc_dec(_cmd.params[_param], (_offset), (_width))) -+ -+#define MC_CMD_HDR_READ_TOKEN(_hdr) \ -+ ((uint16_t)mc_dec((_hdr), MC_CMD_HDR_TOKEN_O, MC_CMD_HDR_TOKEN_S)) -+ -+/* cmd, param, offset, width, type, arg_name */ -+#define DPDMAI_CMD_OPEN(cmd, dpdmai_id) \ -+ MC_CMD_OP(cmd, 0, 0, 32, int, dpdmai_id) -+ -+/* cmd, param, offset, width, type, arg_name */ -+#define DPDMAI_CMD_CREATE(cmd, cfg) \ -+do { \ -+ MC_CMD_OP(cmd, 0, 8, 8, uint8_t, cfg->priorities[0]);\ -+ MC_CMD_OP(cmd, 0, 16, 8, uint8_t, cfg->priorities[1]);\ -+} while (0) -+ -+/* cmd, param, offset, width, type, arg_name */ -+#define DPDMAI_RSP_IS_ENABLED(cmd, en) \ -+ MC_RSP_OP(cmd, 0, 0, 1, int, en) -+ -+/* cmd, param, offset, width, type, arg_name */ -+#define DPDMAI_CMD_SET_IRQ(cmd, irq_index, irq_cfg) \ -+do { \ -+ MC_CMD_OP(cmd, 0, 0, 8, uint8_t, irq_index);\ -+ MC_CMD_OP(cmd, 0, 32, 32, uint32_t, irq_cfg->val);\ -+ MC_CMD_OP(cmd, 1, 0, 64, uint64_t, irq_cfg->addr);\ -+ MC_CMD_OP(cmd, 2, 0, 32, int, irq_cfg->irq_num); \ -+} while (0) -+ -+/* cmd, param, offset, width, type, arg_name */ -+#define DPDMAI_CMD_GET_IRQ(cmd, irq_index) \ -+ MC_CMD_OP(cmd, 0, 32, 8, uint8_t, irq_index) -+ -+/* cmd, param, offset, width, type, arg_name */ -+#define DPDMAI_RSP_GET_IRQ(cmd, type, irq_cfg) \ -+do { \ -+ MC_RSP_OP(cmd, 0, 0, 32, uint32_t, irq_cfg->val); \ -+ MC_RSP_OP(cmd, 1, 0, 64, uint64_t, irq_cfg->addr);\ -+ MC_RSP_OP(cmd, 2, 0, 32, int, irq_cfg->irq_num); \ -+ MC_RSP_OP(cmd, 2, 32, 32, int, type); \ -+} while (0) -+ -+/* cmd, param, offset, width, type, arg_name */ -+#define DPDMAI_CMD_SET_IRQ_ENABLE(cmd, irq_index, enable_state) \ -+do { \ -+ MC_CMD_OP(cmd, 0, 0, 8, uint8_t, enable_state); \ -+ MC_CMD_OP(cmd, 0, 32, 8, uint8_t, irq_index); \ -+} while (0) -+ -+/* cmd, param, offset, width, type, arg_name */ -+#define DPDMAI_CMD_GET_IRQ_ENABLE(cmd, irq_index) \ -+ MC_CMD_OP(cmd, 0, 32, 8, uint8_t, irq_index) -+ -+/* cmd, param, offset, width, type, arg_name */ -+#define DPDMAI_RSP_GET_IRQ_ENABLE(cmd, enable_state) \ -+ MC_RSP_OP(cmd, 0, 0, 8, uint8_t, enable_state) -+ -+/* cmd, param, offset, width, type, arg_name */ -+#define DPDMAI_CMD_SET_IRQ_MASK(cmd, irq_index, mask) \ -+do { \ -+ MC_CMD_OP(cmd, 0, 0, 32, uint32_t, mask); \ -+ MC_CMD_OP(cmd, 0, 32, 8, uint8_t, irq_index); \ -+} while (0) -+ -+/* cmd, param, offset, width, type, arg_name */ -+#define DPDMAI_CMD_GET_IRQ_MASK(cmd, irq_index) \ -+ MC_CMD_OP(cmd, 0, 32, 8, uint8_t, irq_index) -+ -+/* cmd, param, offset, width, type, arg_name */ -+#define DPDMAI_RSP_GET_IRQ_MASK(cmd, mask) \ -+ MC_RSP_OP(cmd, 0, 0, 32, uint32_t, mask) -+ -+/* cmd, param, offset, width, type, arg_name */ -+#define DPDMAI_CMD_GET_IRQ_STATUS(cmd, irq_index, status) \ -+do { \ -+ MC_CMD_OP(cmd, 0, 0, 32, uint32_t, status);\ -+ MC_CMD_OP(cmd, 0, 32, 8, uint8_t, irq_index);\ -+} while (0) -+ -+/* cmd, param, offset, width, type, arg_name */ -+#define DPDMAI_RSP_GET_IRQ_STATUS(cmd, status) \ -+ MC_RSP_OP(cmd, 0, 0, 32, uint32_t, status) -+ -+/* cmd, param, offset, width, type, arg_name */ -+#define DPDMAI_CMD_CLEAR_IRQ_STATUS(cmd, irq_index, status) \ -+do { \ -+ MC_CMD_OP(cmd, 0, 0, 32, uint32_t, status); \ -+ MC_CMD_OP(cmd, 0, 32, 8, uint8_t, irq_index); \ -+} while (0) -+ -+/* cmd, param, offset, width, type, arg_name */ -+#define DPDMAI_RSP_GET_ATTR(cmd, attr) \ -+do { \ -+ MC_RSP_OP(cmd, 0, 0, 32, int, attr->id); \ -+ MC_RSP_OP(cmd, 0, 32, 8, uint8_t, attr->num_of_priorities); \ -+ MC_RSP_OP(cmd, 1, 0, 16, uint16_t, attr->version.major);\ -+ MC_RSP_OP(cmd, 1, 16, 16, uint16_t, attr->version.minor);\ -+} while (0) -+ -+/* cmd, param, offset, width, type, arg_name */ -+#define DPDMAI_CMD_SET_RX_QUEUE(cmd, priority, cfg) \ -+do { \ -+ MC_CMD_OP(cmd, 0, 0, 32, int, cfg->dest_cfg.dest_id); \ -+ MC_CMD_OP(cmd, 0, 32, 8, uint8_t, cfg->dest_cfg.priority); \ -+ MC_CMD_OP(cmd, 0, 40, 8, uint8_t, priority); \ -+ MC_CMD_OP(cmd, 0, 48, 4, enum dpdmai_dest, cfg->dest_cfg.dest_type); \ -+ MC_CMD_OP(cmd, 1, 0, 64, uint64_t, cfg->user_ctx); \ -+ MC_CMD_OP(cmd, 2, 0, 32, uint32_t, cfg->options);\ -+} while (0) -+ -+/* cmd, param, offset, width, type, arg_name */ -+#define DPDMAI_CMD_GET_RX_QUEUE(cmd, priority) \ -+ MC_CMD_OP(cmd, 0, 40, 8, uint8_t, priority) -+ -+/* cmd, param, offset, width, type, arg_name */ -+#define DPDMAI_RSP_GET_RX_QUEUE(cmd, attr) \ -+do { \ -+ MC_RSP_OP(cmd, 0, 0, 32, int, attr->dest_cfg.dest_id);\ -+ MC_RSP_OP(cmd, 0, 32, 8, uint8_t, attr->dest_cfg.priority);\ -+ MC_RSP_OP(cmd, 0, 48, 4, enum dpdmai_dest, attr->dest_cfg.dest_type);\ -+ MC_RSP_OP(cmd, 1, 0, 64, uint64_t, attr->user_ctx);\ -+ MC_RSP_OP(cmd, 2, 0, 32, uint32_t, attr->fqid);\ -+} while (0) -+ -+/* cmd, param, offset, width, type, arg_name */ -+#define DPDMAI_CMD_GET_TX_QUEUE(cmd, priority) \ -+ MC_CMD_OP(cmd, 0, 40, 8, uint8_t, priority) -+ -+/* cmd, param, offset, width, type, arg_name */ -+#define DPDMAI_RSP_GET_TX_QUEUE(cmd, attr) \ -+ MC_RSP_OP(cmd, 1, 0, 32, uint32_t, attr->fqid) -+ -+#endif /* _FSL_DPDMAI_CMD_H */ ---- a/drivers/dma/fsl-edma.c -+++ b/drivers/dma/fsl-edma.c -@@ -146,6 +146,8 @@ struct fsl_edma_slave_config { - u32 dev_addr; - u32 burst; - u32 attr; -+ dma_addr_t dma_dev_addr; -+ enum dma_data_direction dma_dir; - }; - - struct fsl_edma_chan { -@@ -342,6 +344,53 @@ static int fsl_edma_resume(struct dma_ch - return 0; - } - -+static void fsl_edma_unprep_slave_dma(struct fsl_edma_chan *fsl_chan) -+{ -+ if (fsl_chan->fsc.dma_dir != DMA_NONE) -+ dma_unmap_resource(fsl_chan->vchan.chan.device->dev, -+ fsl_chan->fsc.dma_dev_addr, -+ fsl_chan->fsc.burst, fsl_chan->fsc.dma_dir, 0); -+ fsl_chan->fsc.dma_dir = DMA_NONE; -+} -+ -+static bool fsl_edma_prep_slave_dma(struct fsl_edma_chan *fsl_chan, -+ enum dma_transfer_direction dir) -+{ -+ struct device *dev = fsl_chan->vchan.chan.device->dev; -+ enum dma_data_direction dma_dir; -+ -+ switch (dir) { -+ case DMA_MEM_TO_DEV: -+ dma_dir = DMA_FROM_DEVICE; -+ break; -+ case DMA_DEV_TO_MEM: -+ dma_dir = DMA_TO_DEVICE; -+ break; -+ case DMA_DEV_TO_DEV: -+ dma_dir = DMA_BIDIRECTIONAL; -+ break; -+ default: -+ dma_dir = DMA_NONE; -+ break; -+ } -+ -+ /* Already mapped for this config? */ -+ if (fsl_chan->fsc.dma_dir == dma_dir) -+ return true; -+ -+ fsl_edma_unprep_slave_dma(fsl_chan); -+ fsl_chan->fsc.dma_dev_addr = dma_map_resource(dev, -+ fsl_chan->fsc.dev_addr, -+ fsl_chan->fsc.burst, -+ dma_dir, 0); -+ if (dma_mapping_error(dev, fsl_chan->fsc.dma_dev_addr)) -+ return false; -+ -+ fsl_chan->fsc.dma_dir = dma_dir; -+ -+ return true; -+} -+ - static int fsl_edma_slave_config(struct dma_chan *chan, - struct dma_slave_config *cfg) - { -@@ -361,6 +410,7 @@ static int fsl_edma_slave_config(struct - } else { - return -EINVAL; - } -+ fsl_edma_unprep_slave_dma(fsl_chan); - return 0; - } - -@@ -553,6 +603,9 @@ static struct dma_async_tx_descriptor *f - if (!is_slave_direction(fsl_chan->fsc.dir)) - return NULL; - -+ if (!fsl_edma_prep_slave_dma(fsl_chan, fsl_chan->fsc.dir)) -+ return NULL; -+ - sg_len = buf_len / period_len; - fsl_desc = fsl_edma_alloc_desc(fsl_chan, sg_len); - if (!fsl_desc) -@@ -572,11 +625,11 @@ static struct dma_async_tx_descriptor *f - - if (fsl_chan->fsc.dir == DMA_MEM_TO_DEV) { - src_addr = dma_buf_next; -- dst_addr = fsl_chan->fsc.dev_addr; -+ dst_addr = fsl_chan->fsc.dma_dev_addr; - soff = fsl_chan->fsc.addr_width; - doff = 0; - } else { -- src_addr = fsl_chan->fsc.dev_addr; -+ src_addr = fsl_chan->fsc.dma_dev_addr; - dst_addr = dma_buf_next; - soff = 0; - doff = fsl_chan->fsc.addr_width; -@@ -606,6 +659,9 @@ static struct dma_async_tx_descriptor *f - if (!is_slave_direction(fsl_chan->fsc.dir)) - return NULL; - -+ if (!fsl_edma_prep_slave_dma(fsl_chan, fsl_chan->fsc.dir)) -+ return NULL; -+ - fsl_desc = fsl_edma_alloc_desc(fsl_chan, sg_len); - if (!fsl_desc) - return NULL; -@@ -618,11 +674,11 @@ static struct dma_async_tx_descriptor *f - - if (fsl_chan->fsc.dir == DMA_MEM_TO_DEV) { - src_addr = sg_dma_address(sg); -- dst_addr = fsl_chan->fsc.dev_addr; -+ dst_addr = fsl_chan->fsc.dma_dev_addr; - soff = fsl_chan->fsc.addr_width; - doff = 0; - } else { -- src_addr = fsl_chan->fsc.dev_addr; -+ src_addr = fsl_chan->fsc.dma_dev_addr; - dst_addr = sg_dma_address(sg); - soff = 0; - doff = fsl_chan->fsc.addr_width; -@@ -802,6 +858,7 @@ static void fsl_edma_free_chan_resources - fsl_edma_chan_mux(fsl_chan, 0, false); - fsl_chan->edesc = NULL; - vchan_get_all_descriptors(&fsl_chan->vchan, &head); -+ fsl_edma_unprep_slave_dma(fsl_chan); - spin_unlock_irqrestore(&fsl_chan->vchan.lock, flags); - - vchan_dma_desc_free_list(&fsl_chan->vchan, &head); -@@ -937,6 +994,7 @@ static int fsl_edma_probe(struct platfor - fsl_chan->slave_id = 0; - fsl_chan->idle = true; - fsl_chan->vchan.desc_free = fsl_edma_free_desc; -+ fsl_chan->fsc.dma_dir = DMA_NONE; - vchan_init(&fsl_chan->vchan, &fsl_edma->dma_dev); - - edma_writew(fsl_edma, 0x0, fsl_edma->membase + EDMA_TCD_CSR(i)); ---- /dev/null -+++ b/drivers/dma/fsl-qdma.c -@@ -0,0 +1,1278 @@ -+/* -+ * Driver for NXP Layerscape Queue direct memory access controller (qDMA) -+ * -+ * Copyright 2017 NXP -+ * -+ * Author: -+ * Jiaheng Fan <jiaheng.fan@nxp.com> -+ * Wen He <wen.he_1@nxp.com> -+ * -+ * SPDX-License-Identifier: GPL-2.0+ -+ */ -+ -+#include <linux/interrupt.h> -+#include <linux/module.h> -+#include <linux/delay.h> -+#include <linux/of_irq.h> -+#include <linux/of_address.h> -+#include <linux/of_platform.h> -+#include <linux/of_dma.h> -+#include <linux/dma-mapping.h> -+#include <linux/dmapool.h> -+#include <linux/dmaengine.h> -+#include <linux/slab.h> -+#include <linux/spinlock.h> -+ -+#include "virt-dma.h" -+ -+#define FSL_QDMA_DMR 0x0 -+#define FSL_QDMA_DSR 0x4 -+#define FSL_QDMA_DEIER 0xe00 -+#define FSL_QDMA_DEDR 0xe04 -+#define FSL_QDMA_DECFDW0R 0xe10 -+#define FSL_QDMA_DECFDW1R 0xe14 -+#define FSL_QDMA_DECFDW2R 0xe18 -+#define FSL_QDMA_DECFDW3R 0xe1c -+#define FSL_QDMA_DECFQIDR 0xe30 -+#define FSL_QDMA_DECBR 0xe34 -+ -+#define FSL_QDMA_BCQMR(x) (0xc0 + 0x100 * (x)) -+#define FSL_QDMA_BCQSR(x) (0xc4 + 0x100 * (x)) -+#define FSL_QDMA_BCQEDPA_SADDR(x) (0xc8 + 0x100 * (x)) -+#define FSL_QDMA_BCQDPA_SADDR(x) (0xcc + 0x100 * (x)) -+#define FSL_QDMA_BCQEEPA_SADDR(x) (0xd0 + 0x100 * (x)) -+#define FSL_QDMA_BCQEPA_SADDR(x) (0xd4 + 0x100 * (x)) -+#define FSL_QDMA_BCQIER(x) (0xe0 + 0x100 * (x)) -+#define FSL_QDMA_BCQIDR(x) (0xe4 + 0x100 * (x)) -+ -+#define FSL_QDMA_SQDPAR 0x80c -+#define FSL_QDMA_SQEPAR 0x814 -+#define FSL_QDMA_BSQMR 0x800 -+#define FSL_QDMA_BSQSR 0x804 -+#define FSL_QDMA_BSQICR 0x828 -+#define FSL_QDMA_CQMR 0xa00 -+#define FSL_QDMA_CQDSCR1 0xa08 -+#define FSL_QDMA_CQDSCR2 0xa0c -+#define FSL_QDMA_CQIER 0xa10 -+#define FSL_QDMA_CQEDR 0xa14 -+#define FSL_QDMA_SQCCMR 0xa20 -+ -+#define FSL_QDMA_SQICR_ICEN -+ -+#define FSL_QDMA_CQIDR_CQT 0xff000000 -+#define FSL_QDMA_CQIDR_SQPE 0x800000 -+#define FSL_QDMA_CQIDR_SQT 0x8000 -+ -+#define FSL_QDMA_BCQIER_CQTIE 0x8000 -+#define FSL_QDMA_BCQIER_CQPEIE 0x800000 -+#define FSL_QDMA_BSQICR_ICEN 0x80000000 -+#define FSL_QDMA_BSQICR_ICST(x) ((x) << 16) -+#define FSL_QDMA_CQIER_MEIE 0x80000000 -+#define FSL_QDMA_CQIER_TEIE 0x1 -+#define FSL_QDMA_SQCCMR_ENTER_WM 0x200000 -+ -+#define FSL_QDMA_QUEUE_MAX 8 -+ -+#define FSL_QDMA_BCQMR_EN 0x80000000 -+#define FSL_QDMA_BCQMR_EI 0x40000000 -+#define FSL_QDMA_BCQMR_CD_THLD(x) ((x) << 20) -+#define FSL_QDMA_BCQMR_CQ_SIZE(x) ((x) << 16) -+ -+#define FSL_QDMA_BCQSR_QF 0x10000 -+#define FSL_QDMA_BCQSR_XOFF 0x1 -+ -+#define FSL_QDMA_BSQMR_EN 0x80000000 -+#define FSL_QDMA_BSQMR_DI 0x40000000 -+#define FSL_QDMA_BSQMR_CQ_SIZE(x) ((x) << 16) -+ -+#define FSL_QDMA_BSQSR_QE 0x20000 -+ -+#define FSL_QDMA_DMR_DQD 0x40000000 -+#define FSL_QDMA_DSR_DB 0x80000000 -+ -+#define FSL_QDMA_COMMAND_BUFFER_SIZE 64 -+#define FSL_QDMA_DESCRIPTOR_BUFFER_SIZE 32 -+#define FSL_QDMA_CIRCULAR_DESC_SIZE_MIN 64 -+#define FSL_QDMA_CIRCULAR_DESC_SIZE_MAX 16384 -+#define FSL_QDMA_QUEUE_NUM_MAX 8 -+ -+#define FSL_QDMA_CMD_RWTTYPE 0x4 -+#define FSL_QDMA_CMD_LWC 0x2 -+ -+#define FSL_QDMA_CMD_RWTTYPE_OFFSET 28 -+#define FSL_QDMA_CMD_NS_OFFSET 27 -+#define FSL_QDMA_CMD_DQOS_OFFSET 24 -+#define FSL_QDMA_CMD_WTHROTL_OFFSET 20 -+#define FSL_QDMA_CMD_DSEN_OFFSET 19 -+#define FSL_QDMA_CMD_LWC_OFFSET 16 -+ -+#define QDMA_CCDF_STATUS 20 -+#define QDMA_CCDF_OFFSET 20 -+#define QDMA_CCDF_MASK GENMASK(28, 20) -+#define QDMA_CCDF_FOTMAT BIT(29) -+#define QDMA_CCDF_SER BIT(30) -+ -+#define QDMA_SG_FIN BIT(30) -+#define QDMA_SG_EXT BIT(31) -+#define QDMA_SG_LEN_MASK GENMASK(29, 0) -+ -+#define QDMA_BIG_ENDIAN 0x00000001 -+#define COMP_TIMEOUT 1000 -+#define COMMAND_QUEUE_OVERFLLOW 10 -+ -+#define QDMA_IN(fsl_qdma_engine, addr) \ -+ (((fsl_qdma_engine)->big_endian & QDMA_BIG_ENDIAN) ? \ -+ ioread32be(addr) : ioread32(addr)) -+#define QDMA_OUT(fsl_qdma_engine, addr, val) \ -+ (((fsl_qdma_engine)->big_endian & QDMA_BIG_ENDIAN) ? \ -+ iowrite32be(val, addr) : iowrite32(val, addr)) -+ -+#define FSL_QDMA_BLOCK_BASE_OFFSET(fsl_qdma_engine, x) \ -+ (((fsl_qdma_engine)->block_offset) * (x)) -+ -+static DEFINE_PER_CPU(u64, pre_addr); -+static DEFINE_PER_CPU(u64, pre_queue); -+ -+/* qDMA Command Descriptor Fotmats */ -+ -+struct fsl_qdma_format { -+ __le32 status; /* ser, status */ -+ __le32 cfg; /* format, offset */ -+ union { -+ struct { -+ __le32 addr_lo; /* low 32-bits of 40-bit address */ -+ u8 addr_hi; /* high 8-bits of 40-bit address */ -+ u8 __reserved1[2]; -+ u8 cfg8b_w1; /* dd, queue */ -+ } __packed; -+ __le64 data; -+ }; -+} __packed; -+ -+static inline u64 -+qdma_ccdf_addr_get64(const struct fsl_qdma_format *ccdf) -+{ -+ return le64_to_cpu(ccdf->data) & 0xffffffffffLLU; -+} -+ -+static inline void -+qdma_desc_addr_set64(struct fsl_qdma_format *ccdf, u64 addr) -+{ -+ ccdf->addr_hi = upper_32_bits(addr); -+ ccdf->addr_lo = cpu_to_le32(lower_32_bits(addr)); -+} -+ -+static inline u64 -+qdma_ccdf_get_queue(const struct fsl_qdma_format *ccdf) -+{ -+ return ccdf->cfg8b_w1 & 0xff; -+} -+ -+static inline int -+qdma_ccdf_get_offset(const struct fsl_qdma_format *ccdf) -+{ -+ return (le32_to_cpu(ccdf->cfg) & QDMA_CCDF_MASK) >> QDMA_CCDF_OFFSET; -+} -+ -+static inline void -+qdma_ccdf_set_format(struct fsl_qdma_format *ccdf, int offset) -+{ -+ ccdf->cfg = cpu_to_le32(QDMA_CCDF_FOTMAT | offset); -+} -+ -+static inline int -+qdma_ccdf_get_status(const struct fsl_qdma_format *ccdf) -+{ -+ return (le32_to_cpu(ccdf->status) & QDMA_CCDF_MASK) >> QDMA_CCDF_STATUS; -+} -+ -+static inline void -+qdma_ccdf_set_ser(struct fsl_qdma_format *ccdf, int status) -+{ -+ ccdf->status = cpu_to_le32(QDMA_CCDF_SER | status); -+} -+ -+static inline void qdma_csgf_set_len(struct fsl_qdma_format *csgf, int len) -+{ -+ csgf->cfg = cpu_to_le32(len & QDMA_SG_LEN_MASK); -+} -+ -+static inline void qdma_csgf_set_f(struct fsl_qdma_format *csgf, int len) -+{ -+ csgf->cfg = cpu_to_le32(QDMA_SG_FIN | (len & QDMA_SG_LEN_MASK)); -+} -+ -+static inline void qdma_csgf_set_e(struct fsl_qdma_format *csgf, int len) -+{ -+ csgf->cfg = cpu_to_le32(QDMA_SG_EXT | (len & QDMA_SG_LEN_MASK)); -+} -+ -+/* qDMA Source Descriptor Format */ -+struct fsl_qdma_sdf { -+ __le32 rev3; -+ __le32 cfg; /* rev4, bit[0-11] - ssd, bit[12-23] sss */ -+ __le32 rev5; -+ __le32 cmd; -+} __packed; -+ -+/* qDMA Destination Descriptor Format */ -+struct fsl_qdma_ddf { -+ __le32 rev1; -+ __le32 cfg; /* rev2, bit[0-11] - dsd, bit[12-23] - dss */ -+ __le32 rev3; -+ __le32 cmd; -+} __packed; -+ -+struct fsl_qdma_chan { -+ struct virt_dma_chan vchan; -+ struct virt_dma_desc vdesc; -+ enum dma_status status; -+ struct fsl_qdma_engine *qdma; -+ struct fsl_qdma_queue *queue; -+}; -+ -+struct fsl_qdma_queue { -+ struct fsl_qdma_format *virt_head; -+ struct fsl_qdma_format *virt_tail; -+ struct list_head comp_used; -+ struct list_head comp_free; -+ struct dma_pool *comp_pool; -+ struct dma_pool *desc_pool; -+ spinlock_t queue_lock; -+ dma_addr_t bus_addr; -+ u32 n_cq; -+ u32 id; -+ struct fsl_qdma_format *cq; -+ void __iomem *block_base; -+}; -+ -+struct fsl_qdma_comp { -+ dma_addr_t bus_addr; -+ dma_addr_t desc_bus_addr; -+ void *virt_addr; -+ void *desc_virt_addr; -+ struct fsl_qdma_chan *qchan; -+ struct virt_dma_desc vdesc; -+ struct list_head list; -+}; -+ -+struct fsl_qdma_engine { -+ struct dma_device dma_dev; -+ void __iomem *ctrl_base; -+ void __iomem *status_base; -+ void __iomem *block_base; -+ u32 n_chans; -+ u32 n_queues; -+ struct mutex fsl_qdma_mutex; -+ int error_irq; -+ int *queue_irq; -+ bool big_endian; -+ struct fsl_qdma_queue *queue; -+ struct fsl_qdma_queue **status; -+ struct fsl_qdma_chan *chans; -+ int block_number; -+ int block_offset; -+ int irq_base; -+ int desc_allocated; -+ -+}; -+ -+static u32 qdma_readl(struct fsl_qdma_engine *qdma, void __iomem *addr) -+{ -+ return QDMA_IN(qdma, addr); -+} -+ -+static void qdma_writel(struct fsl_qdma_engine *qdma, u32 val, -+ void __iomem *addr) -+{ -+ QDMA_OUT(qdma, addr, val); -+} -+ -+static struct fsl_qdma_chan *to_fsl_qdma_chan(struct dma_chan *chan) -+{ -+ return container_of(chan, struct fsl_qdma_chan, vchan.chan); -+} -+ -+static struct fsl_qdma_comp *to_fsl_qdma_comp(struct virt_dma_desc *vd) -+{ -+ return container_of(vd, struct fsl_qdma_comp, vdesc); -+} -+ -+static void fsl_qdma_free_chan_resources(struct dma_chan *chan) -+{ -+ struct fsl_qdma_chan *fsl_chan = to_fsl_qdma_chan(chan); -+ struct fsl_qdma_queue *fsl_queue = fsl_chan->queue; -+ struct fsl_qdma_engine *fsl_qdma = fsl_chan->qdma; -+ struct fsl_qdma_comp *comp_temp, *_comp_temp; -+ unsigned long flags; -+ LIST_HEAD(head); -+ -+ spin_lock_irqsave(&fsl_chan->vchan.lock, flags); -+ vchan_get_all_descriptors(&fsl_chan->vchan, &head); -+ spin_unlock_irqrestore(&fsl_chan->vchan.lock, flags); -+ -+ vchan_dma_desc_free_list(&fsl_chan->vchan, &head); -+ -+ if (!fsl_queue->comp_pool && !fsl_queue->comp_pool) -+ return; -+ -+ list_for_each_entry_safe(comp_temp, _comp_temp, -+ &fsl_queue->comp_used, list) { -+ dma_pool_free(fsl_queue->comp_pool, -+ comp_temp->virt_addr, -+ comp_temp->bus_addr); -+ dma_pool_free(fsl_queue->desc_pool, -+ comp_temp->desc_virt_addr, -+ comp_temp->desc_bus_addr); -+ list_del(&comp_temp->list); -+ kfree(comp_temp); -+ } -+ -+ list_for_each_entry_safe(comp_temp, _comp_temp, -+ &fsl_queue->comp_free, list) { -+ dma_pool_free(fsl_queue->comp_pool, -+ comp_temp->virt_addr, -+ comp_temp->bus_addr); -+ dma_pool_free(fsl_queue->desc_pool, -+ comp_temp->desc_virt_addr, -+ comp_temp->desc_bus_addr); -+ list_del(&comp_temp->list); -+ kfree(comp_temp); -+ } -+ -+ dma_pool_destroy(fsl_queue->comp_pool); -+ dma_pool_destroy(fsl_queue->desc_pool); -+ -+ fsl_qdma->desc_allocated--; -+ fsl_queue->comp_pool = NULL; -+ fsl_queue->desc_pool = NULL; -+} -+ -+static void fsl_qdma_comp_fill_memcpy(struct fsl_qdma_comp *fsl_comp, -+ dma_addr_t dst, dma_addr_t src, u32 len) -+{ -+ struct fsl_qdma_format *ccdf, *csgf_desc, *csgf_src, *csgf_dest; -+ struct fsl_qdma_sdf *sdf; -+ struct fsl_qdma_ddf *ddf; -+ -+ ccdf = (struct fsl_qdma_format *)fsl_comp->virt_addr; -+ csgf_desc = (struct fsl_qdma_format *)fsl_comp->virt_addr + 1; -+ csgf_src = (struct fsl_qdma_format *)fsl_comp->virt_addr + 2; -+ csgf_dest = (struct fsl_qdma_format *)fsl_comp->virt_addr + 3; -+ sdf = (struct fsl_qdma_sdf *)fsl_comp->desc_virt_addr; -+ ddf = (struct fsl_qdma_ddf *)fsl_comp->desc_virt_addr + 1; -+ -+ memset(fsl_comp->virt_addr, 0, FSL_QDMA_COMMAND_BUFFER_SIZE); -+ memset(fsl_comp->desc_virt_addr, 0, FSL_QDMA_DESCRIPTOR_BUFFER_SIZE); -+ /* Head Command Descriptor(Frame Descriptor) */ -+ qdma_desc_addr_set64(ccdf, fsl_comp->bus_addr + 16); -+ qdma_ccdf_set_format(ccdf, qdma_ccdf_get_offset(ccdf)); -+ qdma_ccdf_set_ser(ccdf, qdma_ccdf_get_status(ccdf)); -+ /* Status notification is enqueued to status queue. */ -+ /* Compound Command Descriptor(Frame List Table) */ -+ qdma_desc_addr_set64(csgf_desc, fsl_comp->desc_bus_addr); -+ /* It must be 32 as Compound S/G Descriptor */ -+ qdma_csgf_set_len(csgf_desc, 32); -+ qdma_desc_addr_set64(csgf_src, src); -+ qdma_csgf_set_len(csgf_src, len); -+ qdma_desc_addr_set64(csgf_dest, dst); -+ qdma_csgf_set_len(csgf_dest, len); -+ /* This entry is the last entry. */ -+ qdma_csgf_set_f(csgf_dest, len); -+ /* Descriptor Buffer */ -+ sdf->cmd = cpu_to_le32( -+ FSL_QDMA_CMD_RWTTYPE << FSL_QDMA_CMD_RWTTYPE_OFFSET); -+ ddf->cmd = cpu_to_le32( -+ FSL_QDMA_CMD_RWTTYPE << FSL_QDMA_CMD_RWTTYPE_OFFSET); -+ ddf->cmd |= cpu_to_le32( -+ FSL_QDMA_CMD_LWC << FSL_QDMA_CMD_LWC_OFFSET); -+} -+ -+/* -+ * Pre-request command descriptor and compound S/G for enqueue. -+ */ -+static int fsl_qdma_pre_request_enqueue_comp_desc(struct fsl_qdma_queue *queue) -+{ -+ struct fsl_qdma_comp *comp_temp; -+ int i; -+ -+ for (i = 0; i < queue->n_cq + COMMAND_QUEUE_OVERFLLOW; i++) { -+ comp_temp = kzalloc(sizeof(*comp_temp), GFP_KERNEL); -+ if (!comp_temp) -+ return -ENOMEM; -+ comp_temp->virt_addr = dma_pool_alloc(queue->comp_pool, -+ GFP_KERNEL, -+ &comp_temp->bus_addr); -+ -+ if (!comp_temp->virt_addr) { -+ kfree(comp_temp); -+ return -ENOMEM; -+ } -+ -+ list_add_tail(&comp_temp->list, &queue->comp_free); -+ } -+ -+ return 0; -+} -+ -+/* -+ * Pre-request source and destination descriptor for enqueue. -+ */ -+static int fsl_qdma_pre_request_enqueue_sd_desc(struct fsl_qdma_queue *queue) -+{ -+ struct fsl_qdma_comp *comp_temp, *_comp_temp; -+ -+ list_for_each_entry_safe(comp_temp, _comp_temp, -+ &queue->comp_free, list) { -+ comp_temp->desc_virt_addr = dma_pool_alloc(queue->desc_pool, -+ GFP_KERNEL, -+ &comp_temp->desc_bus_addr); -+ if (!comp_temp->desc_virt_addr) -+ return -ENOMEM; -+ } -+ -+ return 0; -+} -+ -+/* -+ * Request a command descriptor for enqueue. -+ */ -+static struct fsl_qdma_comp *fsl_qdma_request_enqueue_desc( -+ struct fsl_qdma_chan *fsl_chan) -+{ -+ struct fsl_qdma_comp *comp_temp; -+ struct fsl_qdma_queue *queue = fsl_chan->queue; -+ unsigned long flags; -+ int timeout = COMP_TIMEOUT; -+ -+ while (timeout) { -+ spin_lock_irqsave(&queue->queue_lock, flags); -+ if (!list_empty(&queue->comp_free)) { -+ comp_temp = list_first_entry(&queue->comp_free, -+ struct fsl_qdma_comp, -+ list); -+ list_del(&comp_temp->list); -+ -+ spin_unlock_irqrestore(&queue->queue_lock, flags); -+ comp_temp->qchan = fsl_chan; -+ return comp_temp; -+ } -+ spin_unlock_irqrestore(&queue->queue_lock, flags); -+ udelay(1); -+ timeout--; -+ } -+ -+ return NULL; -+} -+ -+static struct fsl_qdma_queue *fsl_qdma_alloc_queue_resources( -+ struct platform_device *pdev, -+ struct fsl_qdma_engine *fsl_qdma) -+{ -+ struct fsl_qdma_queue *queue_head, *queue_temp; -+ int ret, len, i, j; -+ unsigned int queue_size[FSL_QDMA_QUEUE_MAX]; -+ int queue_num; -+ int block_number; -+ -+ queue_num = fsl_qdma->n_queues; -+ block_number = fsl_qdma->block_number; -+ -+ if (queue_num > FSL_QDMA_QUEUE_MAX) -+ queue_num = FSL_QDMA_QUEUE_MAX; -+ len = sizeof(*queue_head) * queue_num * block_number; -+ queue_head = devm_kzalloc(&pdev->dev, len, GFP_KERNEL); -+ if (!queue_head) -+ return NULL; -+ -+ ret = device_property_read_u32_array(&pdev->dev, "queue-sizes", -+ queue_size, queue_num); -+ if (ret) { -+ dev_err(&pdev->dev, "Can't get queue-sizes.\n"); -+ return NULL; -+ } -+ for (j = 0; j < block_number; j++) { -+ for (i = 0; i < queue_num; i++) { -+ if (queue_size[i] > FSL_QDMA_CIRCULAR_DESC_SIZE_MAX || -+ queue_size[i] < FSL_QDMA_CIRCULAR_DESC_SIZE_MIN) { -+ dev_err(&pdev->dev, -+ "Get wrong queue-sizes.\n"); -+ return NULL; -+ } -+ queue_temp = queue_head + i + (j * queue_num); -+ -+ queue_temp->cq = -+ dma_alloc_coherent(&pdev->dev, -+ sizeof(struct fsl_qdma_format) * -+ queue_size[i], -+ &queue_temp->bus_addr, -+ GFP_KERNEL); -+ if (!queue_temp->cq) -+ return NULL; -+ queue_temp->block_base = fsl_qdma->block_base + -+ FSL_QDMA_BLOCK_BASE_OFFSET(fsl_qdma, j); -+ queue_temp->n_cq = queue_size[i]; -+ queue_temp->id = i; -+ queue_temp->virt_head = queue_temp->cq; -+ queue_temp->virt_tail = queue_temp->cq; -+ /* -+ * List for queue command buffer -+ */ -+ INIT_LIST_HEAD(&queue_temp->comp_used); -+ spin_lock_init(&queue_temp->queue_lock); -+ } -+ } -+ return queue_head; -+} -+ -+static struct fsl_qdma_queue *fsl_qdma_prep_status_queue( -+ struct platform_device *pdev) -+{ -+ struct device_node *np = pdev->dev.of_node; -+ struct fsl_qdma_queue *status_head; -+ unsigned int status_size; -+ int ret; -+ -+ ret = of_property_read_u32(np, "status-sizes", &status_size); -+ if (ret) { -+ dev_err(&pdev->dev, "Can't get status-sizes.\n"); -+ return NULL; -+ } -+ if (status_size > FSL_QDMA_CIRCULAR_DESC_SIZE_MAX -+ || status_size < FSL_QDMA_CIRCULAR_DESC_SIZE_MIN) { -+ dev_err(&pdev->dev, "Get wrong status_size.\n"); -+ return NULL; -+ } -+ status_head = devm_kzalloc(&pdev->dev, sizeof(*status_head), -+ GFP_KERNEL); -+ if (!status_head) -+ return NULL; -+ -+ /* -+ * Buffer for queue command -+ */ -+ status_head->cq = dma_alloc_coherent(&pdev->dev, -+ sizeof(struct fsl_qdma_format) * -+ status_size, -+ &status_head->bus_addr, -+ GFP_KERNEL); -+ if (!status_head->cq) -+ return NULL; -+ status_head->n_cq = status_size; -+ status_head->virt_head = status_head->cq; -+ status_head->virt_tail = status_head->cq; -+ status_head->comp_pool = NULL; -+ -+ return status_head; -+} -+ -+static int fsl_qdma_halt(struct fsl_qdma_engine *fsl_qdma) -+{ -+ void __iomem *ctrl = fsl_qdma->ctrl_base; -+ void __iomem *block; -+ int i, count = 5; -+ int j; -+ u32 reg; -+ -+ /* Disable the command queue and wait for idle state. */ -+ reg = qdma_readl(fsl_qdma, ctrl + FSL_QDMA_DMR); -+ reg |= FSL_QDMA_DMR_DQD; -+ qdma_writel(fsl_qdma, reg, ctrl + FSL_QDMA_DMR); -+ for (j = 0; j < fsl_qdma->block_number; j++) { -+ block = fsl_qdma->block_base + -+ FSL_QDMA_BLOCK_BASE_OFFSET(fsl_qdma, j); -+ for (i = 0; i < FSL_QDMA_QUEUE_NUM_MAX; i++) -+ qdma_writel(fsl_qdma, 0, block + FSL_QDMA_BCQMR(i)); -+ } -+ while (1) { -+ reg = qdma_readl(fsl_qdma, ctrl + FSL_QDMA_DSR); -+ if (!(reg & FSL_QDMA_DSR_DB)) -+ break; -+ if (count-- < 0) -+ return -EBUSY; -+ udelay(100); -+ } -+ -+ for (j = 0; j < fsl_qdma->block_number; j++) { -+ -+ block = fsl_qdma->block_base + -+ FSL_QDMA_BLOCK_BASE_OFFSET(fsl_qdma, j); -+ -+ /* Disable status queue. */ -+ qdma_writel(fsl_qdma, 0, block + FSL_QDMA_BSQMR); -+ -+ /* -+ * clear the command queue interrupt detect register for -+ * all queues. -+ */ -+ qdma_writel(fsl_qdma, 0xffffffff, block + FSL_QDMA_BCQIDR(0)); -+ } -+ -+ return 0; -+} -+ -+static int fsl_qdma_queue_transfer_complete( -+ struct fsl_qdma_engine *fsl_qdma, -+ void *block, -+ int id) -+{ -+ struct fsl_qdma_queue *fsl_queue = fsl_qdma->queue; -+ struct fsl_qdma_queue *fsl_status = fsl_qdma->status[id]; -+ struct fsl_qdma_queue *temp_queue; -+ struct fsl_qdma_format *status_addr; -+ struct fsl_qdma_comp *fsl_comp = NULL; -+ u32 reg, i; -+ bool duplicate, duplicate_handle; -+ -+ while (1) { -+ duplicate = 0; -+ duplicate_handle = 0; -+ reg = qdma_readl(fsl_qdma, block + FSL_QDMA_BSQSR); -+ if (reg & FSL_QDMA_BSQSR_QE) -+ return 0; -+ -+ status_addr = fsl_status->virt_head; -+ -+ if (qdma_ccdf_get_queue(status_addr) == -+ __this_cpu_read(pre_queue) && -+ qdma_ccdf_addr_get64(status_addr) == -+ __this_cpu_read(pre_addr)) -+ duplicate = 1; -+ i = qdma_ccdf_get_queue(status_addr) + -+ id * fsl_qdma->n_queues; -+ __this_cpu_write(pre_addr, qdma_ccdf_addr_get64(status_addr)); -+ __this_cpu_write(pre_queue, qdma_ccdf_get_queue(status_addr)); -+ temp_queue = fsl_queue + i; -+ -+ spin_lock(&temp_queue->queue_lock); -+ if (list_empty(&temp_queue->comp_used)) { -+ if (duplicate) -+ duplicate_handle = 1; -+ else { -+ spin_unlock(&temp_queue->queue_lock); -+ return -1; -+ } -+ } else { -+ fsl_comp = list_first_entry(&temp_queue->comp_used, -+ struct fsl_qdma_comp, -+ list); -+ if (fsl_comp->bus_addr + 16 != -+ __this_cpu_read(pre_addr)) { -+ if (duplicate) -+ duplicate_handle = 1; -+ else { -+ spin_unlock(&temp_queue->queue_lock); -+ return -1; -+ } -+ } -+ -+ } -+ -+ if (duplicate_handle) { -+ reg = qdma_readl(fsl_qdma, block + FSL_QDMA_BSQMR); -+ reg |= FSL_QDMA_BSQMR_DI; -+ qdma_desc_addr_set64(status_addr, 0x0); -+ fsl_status->virt_head++; -+ if (fsl_status->virt_head == fsl_status->cq -+ + fsl_status->n_cq) -+ fsl_status->virt_head = fsl_status->cq; -+ qdma_writel(fsl_qdma, reg, block + FSL_QDMA_BSQMR); -+ spin_unlock(&temp_queue->queue_lock); -+ continue; -+ } -+ list_del(&fsl_comp->list); -+ -+ reg = qdma_readl(fsl_qdma, block + FSL_QDMA_BSQMR); -+ reg |= FSL_QDMA_BSQMR_DI; -+ qdma_desc_addr_set64(status_addr, 0x0); -+ fsl_status->virt_head++; -+ if (fsl_status->virt_head == fsl_status->cq + fsl_status->n_cq) -+ fsl_status->virt_head = fsl_status->cq; -+ qdma_writel(fsl_qdma, reg, block + FSL_QDMA_BSQMR); -+ spin_unlock(&temp_queue->queue_lock); -+ -+ spin_lock(&fsl_comp->qchan->vchan.lock); -+ vchan_cookie_complete(&fsl_comp->vdesc); -+ fsl_comp->qchan->status = DMA_COMPLETE; -+ spin_unlock(&fsl_comp->qchan->vchan.lock); -+ } -+ return 0; -+} -+ -+static irqreturn_t fsl_qdma_error_handler(int irq, void *dev_id) -+{ -+ struct fsl_qdma_engine *fsl_qdma = dev_id; -+ unsigned int intr; -+ void __iomem *status = fsl_qdma->status_base; -+ -+ intr = qdma_readl(fsl_qdma, status + FSL_QDMA_DEDR); -+ -+ if (intr) -+ dev_err(fsl_qdma->dma_dev.dev, "DMA transaction error!\n"); -+ -+ qdma_writel(fsl_qdma, 0xffffffff, status + FSL_QDMA_DEDR); -+ return IRQ_HANDLED; -+} -+ -+static irqreturn_t fsl_qdma_queue_handler(int irq, void *dev_id) -+{ -+ struct fsl_qdma_engine *fsl_qdma = dev_id; -+ unsigned int intr, reg; -+ void __iomem *ctrl = fsl_qdma->ctrl_base; -+ void __iomem *block; -+ int id; -+ -+ id = irq - fsl_qdma->irq_base; -+ if (id < 0 && id > fsl_qdma->block_number) { -+ dev_err(fsl_qdma->dma_dev.dev, -+ "irq %d is wrong irq_base is %d\n", -+ irq, fsl_qdma->irq_base); -+ } -+ -+ block = fsl_qdma->block_base + -+ FSL_QDMA_BLOCK_BASE_OFFSET(fsl_qdma, id); -+ -+ intr = qdma_readl(fsl_qdma, block + FSL_QDMA_BCQIDR(0)); -+ -+ if ((intr & FSL_QDMA_CQIDR_SQT) != 0) -+ intr = fsl_qdma_queue_transfer_complete(fsl_qdma, block, id); -+ -+ if (intr != 0) { -+ reg = qdma_readl(fsl_qdma, ctrl + FSL_QDMA_DMR); -+ reg |= FSL_QDMA_DMR_DQD; -+ qdma_writel(fsl_qdma, reg, ctrl + FSL_QDMA_DMR); -+ qdma_writel(fsl_qdma, 0, block + FSL_QDMA_BCQIER(0)); -+ dev_err(fsl_qdma->dma_dev.dev, "QDMA: status err!\n"); -+ } -+ -+ qdma_writel(fsl_qdma, 0xffffffff, block + FSL_QDMA_BCQIDR(0)); -+ -+ return IRQ_HANDLED; -+} -+ -+static int -+fsl_qdma_irq_init(struct platform_device *pdev, -+ struct fsl_qdma_engine *fsl_qdma) -+{ -+ char irq_name[20]; -+ int i; -+ int cpu; -+ int ret; -+ -+ fsl_qdma->error_irq = platform_get_irq_byname(pdev, -+ "qdma-error"); -+ if (fsl_qdma->error_irq < 0) { -+ dev_err(&pdev->dev, "Can't get qdma controller irq.\n"); -+ return fsl_qdma->error_irq; -+ } -+ -+ ret = devm_request_irq(&pdev->dev, fsl_qdma->error_irq, -+ fsl_qdma_error_handler, 0, "qDMA error", fsl_qdma); -+ if (ret) { -+ dev_err(&pdev->dev, "Can't register qDMA controller IRQ.\n"); -+ return ret; -+ } -+ -+ for (i = 0; i < fsl_qdma->block_number; i++) { -+ sprintf(irq_name, "qdma-queue%d", i); -+ fsl_qdma->queue_irq[i] = platform_get_irq_byname(pdev, -+ irq_name); -+ -+ if (fsl_qdma->queue_irq[i] < 0) { -+ dev_err(&pdev->dev, -+ "Can't get qdma queue %d irq.\n", -+ i); -+ return fsl_qdma->queue_irq[i]; -+ } -+ -+ ret = devm_request_irq(&pdev->dev, -+ fsl_qdma->queue_irq[i], -+ fsl_qdma_queue_handler, -+ 0, -+ "qDMA queue", -+ fsl_qdma); -+ if (ret) { -+ dev_err(&pdev->dev, -+ "Can't register qDMA queue IRQ.\n"); -+ return ret; -+ } -+ -+ cpu = i % num_online_cpus(); -+ ret = irq_set_affinity_hint(fsl_qdma->queue_irq[i], -+ get_cpu_mask(cpu)); -+ if (ret) { -+ dev_err(&pdev->dev, -+ "Can't set cpu %d affinity to IRQ %d.\n", -+ cpu, -+ fsl_qdma->queue_irq[i]); -+ return ret; -+ } -+ -+ } -+ -+ return 0; -+} -+ -+static void fsl_qdma_irq_exit( -+ struct platform_device *pdev, struct fsl_qdma_engine *fsl_qdma) -+{ -+ if (fsl_qdma->queue_irq[0] == fsl_qdma->error_irq) { -+ devm_free_irq(&pdev->dev, fsl_qdma->queue_irq[0], fsl_qdma); -+ } else { -+ devm_free_irq(&pdev->dev, fsl_qdma->queue_irq[0], fsl_qdma); -+ devm_free_irq(&pdev->dev, fsl_qdma->error_irq, fsl_qdma); -+ } -+} -+ -+static int fsl_qdma_reg_init(struct fsl_qdma_engine *fsl_qdma) -+{ -+ struct fsl_qdma_queue *fsl_queue = fsl_qdma->queue; -+ struct fsl_qdma_queue *temp; -+ void __iomem *ctrl = fsl_qdma->ctrl_base; -+ void __iomem *status = fsl_qdma->status_base; -+ void __iomem *block; -+ int i, j, ret; -+ u32 reg; -+ -+ /* Try to halt the qDMA engine first. */ -+ ret = fsl_qdma_halt(fsl_qdma); -+ if (ret) { -+ dev_err(fsl_qdma->dma_dev.dev, "DMA halt failed!"); -+ return ret; -+ } -+ -+ for (i = 0; i < fsl_qdma->block_number; i++) { -+ /* -+ * Clear the command queue interrupt detect register for -+ * all queues. -+ */ -+ -+ block = fsl_qdma->block_base + -+ FSL_QDMA_BLOCK_BASE_OFFSET(fsl_qdma, i); -+ qdma_writel(fsl_qdma, 0xffffffff, block + FSL_QDMA_BCQIDR(0)); -+ } -+ -+ for (j = 0; j < fsl_qdma->block_number; j++) { -+ block = fsl_qdma->block_base + -+ FSL_QDMA_BLOCK_BASE_OFFSET(fsl_qdma, j); -+ for (i = 0; i < fsl_qdma->n_queues; i++) { -+ temp = fsl_queue + i + (j * fsl_qdma->n_queues); -+ /* -+ * Initialize Command Queue registers to -+ * point to the first -+ * command descriptor in memory. -+ * Dequeue Pointer Address Registers -+ * Enqueue Pointer Address Registers -+ */ -+ -+ qdma_writel(fsl_qdma, temp->bus_addr, -+ block + FSL_QDMA_BCQDPA_SADDR(i)); -+ qdma_writel(fsl_qdma, temp->bus_addr, -+ block + FSL_QDMA_BCQEPA_SADDR(i)); -+ -+ /* Initialize the queue mode. */ -+ reg = FSL_QDMA_BCQMR_EN; -+ reg |= FSL_QDMA_BCQMR_CD_THLD(ilog2(temp->n_cq) - 4); -+ reg |= FSL_QDMA_BCQMR_CQ_SIZE(ilog2(temp->n_cq) - 6); -+ qdma_writel(fsl_qdma, reg, block + FSL_QDMA_BCQMR(i)); -+ } -+ -+ /* -+ * Workaround for erratum: ERR010812. -+ * We must enable XOFF to avoid the enqueue rejection occurs. -+ * Setting SQCCMR ENTER_WM to 0x20. -+ */ -+ -+ qdma_writel(fsl_qdma, FSL_QDMA_SQCCMR_ENTER_WM, -+ block + FSL_QDMA_SQCCMR); -+ -+ /* -+ * Initialize status queue registers to point to the first -+ * command descriptor in memory. -+ * Dequeue Pointer Address Registers -+ * Enqueue Pointer Address Registers -+ */ -+ -+ qdma_writel(fsl_qdma, fsl_qdma->status[j]->bus_addr, -+ block + FSL_QDMA_SQEPAR); -+ qdma_writel(fsl_qdma, fsl_qdma->status[j]->bus_addr, -+ block + FSL_QDMA_SQDPAR); -+ /* Initialize status queue interrupt. */ -+ qdma_writel(fsl_qdma, FSL_QDMA_BCQIER_CQTIE, -+ block + FSL_QDMA_BCQIER(0)); -+ qdma_writel(fsl_qdma, FSL_QDMA_BSQICR_ICEN | -+ FSL_QDMA_BSQICR_ICST(5) | 0x8000, -+ block + FSL_QDMA_BSQICR); -+ qdma_writel(fsl_qdma, FSL_QDMA_CQIER_MEIE | -+ FSL_QDMA_CQIER_TEIE, -+ block + FSL_QDMA_CQIER); -+ -+ /* Initialize the status queue mode. */ -+ reg = FSL_QDMA_BSQMR_EN; -+ reg |= FSL_QDMA_BSQMR_CQ_SIZE(ilog2( -+ fsl_qdma->status[j]->n_cq) - 6); -+ -+ qdma_writel(fsl_qdma, reg, block + FSL_QDMA_BSQMR); -+ reg = qdma_readl(fsl_qdma, block + FSL_QDMA_BSQMR); -+ -+ } -+ -+ /* Initialize controller interrupt register. */ -+ qdma_writel(fsl_qdma, 0xffffffff, status + FSL_QDMA_DEDR); -+ qdma_writel(fsl_qdma, 0xffffffff, status + FSL_QDMA_DEIER); -+ -+ reg = qdma_readl(fsl_qdma, ctrl + FSL_QDMA_DMR); -+ reg &= ~FSL_QDMA_DMR_DQD; -+ qdma_writel(fsl_qdma, reg, ctrl + FSL_QDMA_DMR); -+ -+ return 0; -+} -+ -+static struct dma_async_tx_descriptor * -+fsl_qdma_prep_memcpy(struct dma_chan *chan, dma_addr_t dst, -+ dma_addr_t src, size_t len, unsigned long flags) -+{ -+ struct fsl_qdma_chan *fsl_chan = to_fsl_qdma_chan(chan); -+ struct fsl_qdma_comp *fsl_comp; -+ -+ fsl_comp = fsl_qdma_request_enqueue_desc(fsl_chan); -+ -+ if (!fsl_comp) -+ return NULL; -+ -+ fsl_qdma_comp_fill_memcpy(fsl_comp, dst, src, len); -+ -+ return vchan_tx_prep(&fsl_chan->vchan, &fsl_comp->vdesc, flags); -+} -+ -+static void fsl_qdma_enqueue_desc(struct fsl_qdma_chan *fsl_chan) -+{ -+ struct fsl_qdma_queue *fsl_queue = fsl_chan->queue; -+ struct fsl_qdma_comp *fsl_comp; -+ struct virt_dma_desc *vdesc; -+ void __iomem *block = fsl_queue->block_base; -+ u32 reg; -+ -+ reg = qdma_readl(fsl_chan->qdma, block + FSL_QDMA_BCQSR(fsl_queue->id)); -+ if (reg & (FSL_QDMA_BCQSR_QF | FSL_QDMA_BCQSR_XOFF)) -+ return; -+ vdesc = vchan_next_desc(&fsl_chan->vchan); -+ if (!vdesc) -+ return; -+ list_del(&vdesc->node); -+ fsl_comp = to_fsl_qdma_comp(vdesc); -+ -+ memcpy(fsl_queue->virt_head++, fsl_comp->virt_addr, 16); -+ if (fsl_queue->virt_head == fsl_queue->cq + fsl_queue->n_cq) -+ fsl_queue->virt_head = fsl_queue->cq; -+ -+ list_add_tail(&fsl_comp->list, &fsl_queue->comp_used); -+ barrier(); -+ reg = qdma_readl(fsl_chan->qdma, block + FSL_QDMA_BCQMR(fsl_queue->id)); -+ reg |= FSL_QDMA_BCQMR_EI; -+ qdma_writel(fsl_chan->qdma, reg, block + FSL_QDMA_BCQMR(fsl_queue->id)); -+ fsl_chan->status = DMA_IN_PROGRESS; -+} -+ -+static enum dma_status fsl_qdma_tx_status(struct dma_chan *chan, -+ dma_cookie_t cookie, struct dma_tx_state *txstate) -+{ -+ return dma_cookie_status(chan, cookie, txstate); -+} -+ -+static void fsl_qdma_free_desc(struct virt_dma_desc *vdesc) -+{ -+ struct fsl_qdma_comp *fsl_comp; -+ struct fsl_qdma_queue *fsl_queue; -+ unsigned long flags; -+ -+ fsl_comp = to_fsl_qdma_comp(vdesc); -+ fsl_queue = fsl_comp->qchan->queue; -+ -+ spin_lock_irqsave(&fsl_queue->queue_lock, flags); -+ list_add_tail(&fsl_comp->list, &fsl_queue->comp_free); -+ spin_unlock_irqrestore(&fsl_queue->queue_lock, flags); -+} -+ -+static void fsl_qdma_issue_pending(struct dma_chan *chan) -+{ -+ struct fsl_qdma_chan *fsl_chan = to_fsl_qdma_chan(chan); -+ struct fsl_qdma_queue *fsl_queue = fsl_chan->queue; -+ unsigned long flags; -+ -+ spin_lock_irqsave(&fsl_queue->queue_lock, flags); -+ spin_lock(&fsl_chan->vchan.lock); -+ if (vchan_issue_pending(&fsl_chan->vchan)) -+ fsl_qdma_enqueue_desc(fsl_chan); -+ spin_unlock(&fsl_chan->vchan.lock); -+ spin_unlock_irqrestore(&fsl_queue->queue_lock, flags); -+} -+ -+static void fsl_qdma_synchronize(struct dma_chan *chan) -+{ -+ struct fsl_qdma_chan *fsl_chan = to_fsl_qdma_chan(chan); -+ -+ vchan_synchronize(&fsl_chan->vchan); -+} -+ -+static int fsl_qdma_terminate_all(struct dma_chan *chan) -+{ -+ struct fsl_qdma_chan *fsl_chan = to_fsl_qdma_chan(chan); -+ unsigned long flags; -+ LIST_HEAD(head); -+ -+ spin_lock_irqsave(&fsl_chan->vchan.lock, flags); -+ vchan_get_all_descriptors(&fsl_chan->vchan, &head); -+ spin_unlock_irqrestore(&fsl_chan->vchan.lock, flags); -+ vchan_dma_desc_free_list(&fsl_chan->vchan, &head); -+ return 0; -+} -+ -+static int fsl_qdma_alloc_chan_resources(struct dma_chan *chan) -+{ -+ struct fsl_qdma_chan *fsl_chan = to_fsl_qdma_chan(chan); -+ struct fsl_qdma_queue *fsl_queue = fsl_chan->queue; -+ struct fsl_qdma_engine *fsl_qdma = fsl_chan->qdma; -+ int ret; -+ -+ if (fsl_queue->comp_pool && fsl_queue->desc_pool) -+ return fsl_qdma->desc_allocated; -+ -+ INIT_LIST_HEAD(&fsl_queue->comp_free); -+ -+ /* -+ * The dma pool for queue command buffer -+ */ -+ fsl_queue->comp_pool = -+ dma_pool_create("comp_pool", -+ chan->device->dev, -+ FSL_QDMA_COMMAND_BUFFER_SIZE, -+ 64, 0); -+ if (!fsl_queue->comp_pool) -+ return -ENOMEM; -+ -+ /* -+ * The dma pool for Descriptor(SD/DD) buffer -+ */ -+ fsl_queue->desc_pool = -+ dma_pool_create("desc_pool", -+ chan->device->dev, -+ FSL_QDMA_DESCRIPTOR_BUFFER_SIZE, -+ 32, 0); -+ if (!fsl_queue->desc_pool) -+ goto err_desc_pool; -+ -+ ret = fsl_qdma_pre_request_enqueue_comp_desc(fsl_queue); -+ if (ret) { -+ dev_err(chan->device->dev, "failed to alloc dma buffer for " -+ "comp S/G descriptor\n"); -+ goto err_mem; -+ } -+ -+ ret = fsl_qdma_pre_request_enqueue_sd_desc(fsl_queue); -+ if (ret) { -+ dev_err(chan->device->dev, "failed to alloc dma buffer for " -+ "S/D descriptor\n"); -+ goto err_mem; -+ } -+ -+ fsl_qdma->desc_allocated++; -+ return fsl_qdma->desc_allocated; -+ -+err_mem: -+ dma_pool_destroy(fsl_queue->desc_pool); -+err_desc_pool: -+ dma_pool_destroy(fsl_queue->comp_pool); -+ return -ENOMEM; -+} -+ -+static int fsl_qdma_probe(struct platform_device *pdev) -+{ -+ struct device_node *np = pdev->dev.of_node; -+ struct fsl_qdma_engine *fsl_qdma; -+ struct fsl_qdma_chan *fsl_chan; -+ struct resource *res; -+ unsigned int len, chans, queues; -+ int ret, i; -+ int blk_num; -+ int blk_off; -+ -+ ret = of_property_read_u32(np, "channels", &chans); -+ if (ret) { -+ dev_err(&pdev->dev, "Can't get channels.\n"); -+ return ret; -+ } -+ -+ ret = of_property_read_u32(np, "block-offset", &blk_off); -+ if (ret) { -+ dev_err(&pdev->dev, "Can't get block-offset.\n"); -+ return ret; -+ } -+ -+ ret = of_property_read_u32(np, "block-number", &blk_num); -+ if (ret) { -+ dev_err(&pdev->dev, "Can't get block-number.\n"); -+ return ret; -+ } -+ -+ blk_num = min_t(int, blk_num, num_online_cpus()); -+ -+ len = sizeof(*fsl_qdma); -+ fsl_qdma = devm_kzalloc(&pdev->dev, len, GFP_KERNEL); -+ if (!fsl_qdma) -+ return -ENOMEM; -+ -+ len = sizeof(*fsl_chan) * chans; -+ fsl_qdma->chans = devm_kzalloc(&pdev->dev, len, GFP_KERNEL); -+ if (!fsl_qdma->chans) -+ return -ENOMEM; -+ -+ len = sizeof(struct fsl_qdma_queue *) * blk_num; -+ fsl_qdma->status = devm_kzalloc(&pdev->dev, len, GFP_KERNEL); -+ if (!fsl_qdma->status) -+ return -ENOMEM; -+ -+ len = sizeof(int) * blk_num; -+ fsl_qdma->queue_irq = devm_kzalloc(&pdev->dev, len, GFP_KERNEL); -+ if (!fsl_qdma->queue_irq) -+ return -ENOMEM; -+ -+ ret = of_property_read_u32(np, "queues", &queues); -+ if (ret) { -+ dev_err(&pdev->dev, "Can't get queues.\n"); -+ return ret; -+ } -+ -+ fsl_qdma->desc_allocated = 0; -+ fsl_qdma->n_chans = chans; -+ fsl_qdma->n_queues = queues; -+ fsl_qdma->block_number = blk_num; -+ fsl_qdma->block_offset = blk_off; -+ -+ mutex_init(&fsl_qdma->fsl_qdma_mutex); -+ -+ for (i = 0; i < fsl_qdma->block_number; i++) { -+ fsl_qdma->status[i] = fsl_qdma_prep_status_queue(pdev); -+ if (!fsl_qdma->status[i]) -+ return -ENOMEM; -+ } -+ res = platform_get_resource(pdev, IORESOURCE_MEM, 0); -+ fsl_qdma->ctrl_base = devm_ioremap_resource(&pdev->dev, res); -+ if (IS_ERR(fsl_qdma->ctrl_base)) -+ return PTR_ERR(fsl_qdma->ctrl_base); -+ -+ res = platform_get_resource(pdev, IORESOURCE_MEM, 1); -+ fsl_qdma->status_base = devm_ioremap_resource(&pdev->dev, res); -+ if (IS_ERR(fsl_qdma->status_base)) -+ return PTR_ERR(fsl_qdma->status_base); -+ -+ res = platform_get_resource(pdev, IORESOURCE_MEM, 2); -+ fsl_qdma->block_base = devm_ioremap_resource(&pdev->dev, res); -+ if (IS_ERR(fsl_qdma->block_base)) -+ return PTR_ERR(fsl_qdma->block_base); -+ fsl_qdma->queue = fsl_qdma_alloc_queue_resources(pdev, fsl_qdma); -+ if (!fsl_qdma->queue) -+ return -ENOMEM; -+ -+ ret = fsl_qdma_irq_init(pdev, fsl_qdma); -+ if (ret) -+ return ret; -+ -+ fsl_qdma->irq_base = platform_get_irq_byname(pdev, "qdma-queue0"); -+ fsl_qdma->big_endian = of_property_read_bool(np, "big-endian"); -+ INIT_LIST_HEAD(&fsl_qdma->dma_dev.channels); -+ -+ for (i = 0; i < fsl_qdma->n_chans; i++) { -+ struct fsl_qdma_chan *fsl_chan = &fsl_qdma->chans[i]; -+ -+ fsl_chan->qdma = fsl_qdma; -+ fsl_chan->queue = fsl_qdma->queue + i % (fsl_qdma->n_queues * -+ fsl_qdma->block_number); -+ fsl_chan->vchan.desc_free = fsl_qdma_free_desc; -+ vchan_init(&fsl_chan->vchan, &fsl_qdma->dma_dev); -+ } -+ -+ dma_cap_set(DMA_MEMCPY, fsl_qdma->dma_dev.cap_mask); -+ -+ fsl_qdma->dma_dev.dev = &pdev->dev; -+ fsl_qdma->dma_dev.device_free_chan_resources -+ = fsl_qdma_free_chan_resources; -+ fsl_qdma->dma_dev.device_alloc_chan_resources -+ = fsl_qdma_alloc_chan_resources; -+ fsl_qdma->dma_dev.device_tx_status = fsl_qdma_tx_status; -+ fsl_qdma->dma_dev.device_prep_dma_memcpy = fsl_qdma_prep_memcpy; -+ fsl_qdma->dma_dev.device_issue_pending = fsl_qdma_issue_pending; -+ fsl_qdma->dma_dev.device_synchronize = fsl_qdma_synchronize; -+ fsl_qdma->dma_dev.device_terminate_all = fsl_qdma_terminate_all; -+ -+ dma_set_mask(&pdev->dev, DMA_BIT_MASK(40)); -+ -+ platform_set_drvdata(pdev, fsl_qdma); -+ -+ ret = dma_async_device_register(&fsl_qdma->dma_dev); -+ if (ret) { -+ dev_err(&pdev->dev, -+ "Can't register NXP Layerscape qDMA engine.\n"); -+ return ret; -+ } -+ -+ ret = fsl_qdma_reg_init(fsl_qdma); -+ if (ret) { -+ dev_err(&pdev->dev, "Can't Initialize the qDMA engine.\n"); -+ return ret; -+ } -+ -+ return 0; -+} -+ -+static void fsl_qdma_cleanup_vchan(struct dma_device *dmadev) -+{ -+ struct fsl_qdma_chan *chan, *_chan; -+ -+ list_for_each_entry_safe(chan, _chan, -+ &dmadev->channels, vchan.chan.device_node) { -+ list_del(&chan->vchan.chan.device_node); -+ tasklet_kill(&chan->vchan.task); -+ } -+} -+ -+static int fsl_qdma_remove(struct platform_device *pdev) -+{ -+ struct device_node *np = pdev->dev.of_node; -+ struct fsl_qdma_engine *fsl_qdma = platform_get_drvdata(pdev); -+ struct fsl_qdma_queue *status; -+ int i; -+ -+ fsl_qdma_irq_exit(pdev, fsl_qdma); -+ fsl_qdma_cleanup_vchan(&fsl_qdma->dma_dev); -+ of_dma_controller_free(np); -+ dma_async_device_unregister(&fsl_qdma->dma_dev); -+ -+ for (i = 0; i < fsl_qdma->block_number; i++) { -+ status = fsl_qdma->status[i]; -+ dma_free_coherent(&pdev->dev, sizeof(struct fsl_qdma_format) * -+ status->n_cq, status->cq, status->bus_addr); -+ } -+ return 0; -+} -+ -+static const struct of_device_id fsl_qdma_dt_ids[] = { -+ { .compatible = "fsl,ls1021a-qdma", }, -+ { /* sentinel */ } -+}; -+MODULE_DEVICE_TABLE(of, fsl_qdma_dt_ids); -+ -+static struct platform_driver fsl_qdma_driver = { -+ .driver = { -+ .name = "fsl-qdma", -+ .of_match_table = fsl_qdma_dt_ids, -+ }, -+ .probe = fsl_qdma_probe, -+ .remove = fsl_qdma_remove, -+}; -+ -+module_platform_driver(fsl_qdma_driver); -+ -+MODULE_ALIAS("platform:fsl-qdma"); -+MODULE_DESCRIPTION("NXP Layerscape qDMA engine driver"); -+MODULE_LICENSE("GPL v2"); |