mediatek: backport upstream mediatek patches

Signed-off-by: John Crispin <john@phrozen.org>

1 files changed, 1144 insertions, 0 deletions

diff --git a/target/linux/mediatek/patches-4.14/0205-dmaengine-mediatek-Add-MediaTek-High-Speed-DMA-contr.patch b/target/linux/mediatek/patches-4.14/0205-dmaengine-mediatek-Add-MediaTek-High-Speed-DMA-contr.patch
new file mode 100644
index 0000000000..cd6e1004fb
--- /dev/null
+++ b/target/linux/mediatek/patches-4.14/0205-dmaengine-mediatek-Add-MediaTek-High-Speed-DMA-contr.patch
@@ -0,0 +1,1144 @@
+From 2b97c5d7886a920adc8f7c32c2a60583475654f2 Mon Sep 17 00:00:00 2001
+From: Sean Wang <sean.wang@mediatek.com>
+Date: Fri, 12 May 2017 17:05:12 +0800
+Subject: [PATCH 205/224] dmaengine: mediatek: Add MediaTek High-Speed DMA
+ controller for MT7622 and MT7623 SoC
+
+MediaTek High-Speed DMA controller (HSDMA) on MT7622 and MT7623 SoC has
+a single ring is dedicated to memory-to-memory transfer through ring based
+descriptor management.
+
+Even though there is only one physical ring available inside HSDMA, the
+driver can be easily extended to the support of multiple virtual channels
+processing simultaneously by means of DMA_VIRTUAL_CHANNELS effort.
+
+Signed-off-by: Sean Wang <sean.wang@mediatek.com>
+Cc: Randy Dunlap <rdunlap@infradead.org>
+Cc: Fengguang Wu <fengguang.wu@intel.com>
+Cc: Julia Lawall <julia.lawall@lip6.fr>
+---
+ drivers/dma/Kconfig              |    2 +
+ drivers/dma/Makefile             |    1 +
+ drivers/dma/mediatek/Kconfig     |   13 +
+ drivers/dma/mediatek/Makefile    |    1 +
+ drivers/dma/mediatek/mtk-hsdma.c | 1056 ++++++++++++++++++++++++++++++++++++++
+ 5 files changed, 1073 insertions(+)
+ create mode 100644 drivers/dma/mediatek/Kconfig
+ create mode 100644 drivers/dma/mediatek/Makefile
+ create mode 100644 drivers/dma/mediatek/mtk-hsdma.c
+
+diff --git a/drivers/dma/Kconfig b/drivers/dma/Kconfig
+index fadc4d8783bd..3100b6dfa6e8 100644
+--- a/drivers/dma/Kconfig
++++ b/drivers/dma/Kconfig
+@@ -604,6 +604,8 @@ config ZX_DMA
+ # driver files
+ source "drivers/dma/bestcomm/Kconfig"
+ 
++source "drivers/dma/mediatek/Kconfig"
++
+ source "drivers/dma/qcom/Kconfig"
+ 
+ source "drivers/dma/dw/Kconfig"
+diff --git a/drivers/dma/Makefile b/drivers/dma/Makefile
+index f08f8de1b567..26b0ef43a8f9 100644
+--- a/drivers/dma/Makefile
++++ b/drivers/dma/Makefile
+@@ -71,5 +71,6 @@ obj-$(CONFIG_XGENE_DMA) += xgene-dma.o
+ obj-$(CONFIG_ZX_DMA) += zx_dma.o
+ obj-$(CONFIG_ST_FDMA) += st_fdma.o
+ 
++obj-y += mediatek/
+ obj-y += qcom/
+ obj-y += xilinx/
+diff --git a/drivers/dma/mediatek/Kconfig b/drivers/dma/mediatek/Kconfig
+new file mode 100644
+index 000000000000..27bac0bba09e
+--- /dev/null
++++ b/drivers/dma/mediatek/Kconfig
+@@ -0,0 +1,13 @@
++
++config MTK_HSDMA
++	tristate "MediaTek High-Speed DMA controller support"
++	depends on ARCH_MEDIATEK || COMPILE_TEST
++	select DMA_ENGINE
++	select DMA_VIRTUAL_CHANNELS
++	---help---
++	  Enable support for High-Speed DMA controller on MediaTek
++	  SoCs.
++
++	  This controller provides the channels which is dedicated to
++	  memory-to-memory transfer to offload from CPU through ring-
++	  based descriptor management.
+diff --git a/drivers/dma/mediatek/Makefile b/drivers/dma/mediatek/Makefile
+new file mode 100644
+index 000000000000..6e778f842f01
+--- /dev/null
++++ b/drivers/dma/mediatek/Makefile
+@@ -0,0 +1 @@
++obj-$(CONFIG_MTK_HSDMA) += mtk-hsdma.o
+diff --git a/drivers/dma/mediatek/mtk-hsdma.c b/drivers/dma/mediatek/mtk-hsdma.c
+new file mode 100644
+index 000000000000..b7ec56ae02a6
+--- /dev/null
++++ b/drivers/dma/mediatek/mtk-hsdma.c
+@@ -0,0 +1,1056 @@
++// SPDX-License-Identifier: GPL-2.0
++// Copyright (c) 2017-2018 MediaTek Inc.
++
++/*
++ * Driver for MediaTek High-Speed DMA Controller
++ *
++ * Author: Sean Wang <sean.wang@mediatek.com>
++ *
++ */
++
++#include <linux/bitops.h>
++#include <linux/clk.h>
++#include <linux/dmaengine.h>
++#include <linux/dma-mapping.h>
++#include <linux/err.h>
++#include <linux/iopoll.h>
++#include <linux/list.h>
++#include <linux/module.h>
++#include <linux/of.h>
++#include <linux/of_device.h>
++#include <linux/of_dma.h>
++#include <linux/platform_device.h>
++#include <linux/pm_runtime.h>
++#include <linux/refcount.h>
++#include <linux/slab.h>
++
++#include "../virt-dma.h"
++
++#define MTK_HSDMA_USEC_POLL		20
++#define MTK_HSDMA_TIMEOUT_POLL		200000
++#define MTK_HSDMA_DMA_BUSWIDTHS		BIT(DMA_SLAVE_BUSWIDTH_4_BYTES)
++
++/* The default number of virtual channel */
++#define MTK_HSDMA_NR_VCHANS		3
++
++/* Only one physical channel supported */
++#define MTK_HSDMA_NR_MAX_PCHANS		1
++
++/* Macro for physical descriptor (PD) manipulation */
++/* The number of PD which must be 2 of power */
++#define MTK_DMA_SIZE			64
++#define MTK_HSDMA_NEXT_DESP_IDX(x, y)	(((x) + 1) & ((y) - 1))
++#define MTK_HSDMA_LAST_DESP_IDX(x, y)	(((x) - 1) & ((y) - 1))
++#define MTK_HSDMA_MAX_LEN		0x3f80
++#define MTK_HSDMA_ALIGN_SIZE		4
++#define MTK_HSDMA_PLEN_MASK		0x3fff
++#define MTK_HSDMA_DESC_PLEN(x)		(((x) & MTK_HSDMA_PLEN_MASK) << 16)
++#define MTK_HSDMA_DESC_PLEN_GET(x)	(((x) >> 16) & MTK_HSDMA_PLEN_MASK)
++
++/* Registers for underlying ring manipulation */
++#define MTK_HSDMA_TX_BASE		0x0
++#define MTK_HSDMA_TX_CNT		0x4
++#define MTK_HSDMA_TX_CPU		0x8
++#define MTK_HSDMA_TX_DMA		0xc
++#define MTK_HSDMA_RX_BASE		0x100
++#define MTK_HSDMA_RX_CNT		0x104
++#define MTK_HSDMA_RX_CPU		0x108
++#define MTK_HSDMA_RX_DMA		0x10c
++
++/* Registers for global setup */
++#define MTK_HSDMA_GLO			0x204
++#define MTK_HSDMA_GLO_MULTI_DMA		BIT(10)
++#define MTK_HSDMA_TX_WB_DDONE		BIT(6)
++#define MTK_HSDMA_BURST_64BYTES		(0x2 << 4)
++#define MTK_HSDMA_GLO_RX_BUSY		BIT(3)
++#define MTK_HSDMA_GLO_RX_DMA		BIT(2)
++#define MTK_HSDMA_GLO_TX_BUSY		BIT(1)
++#define MTK_HSDMA_GLO_TX_DMA		BIT(0)
++#define MTK_HSDMA_GLO_DMA		(MTK_HSDMA_GLO_TX_DMA |	\
++					 MTK_HSDMA_GLO_RX_DMA)
++#define MTK_HSDMA_GLO_BUSY		(MTK_HSDMA_GLO_RX_BUSY | \
++					 MTK_HSDMA_GLO_TX_BUSY)
++#define MTK_HSDMA_GLO_DEFAULT		(MTK_HSDMA_GLO_TX_DMA | \
++					 MTK_HSDMA_GLO_RX_DMA | \
++					 MTK_HSDMA_TX_WB_DDONE | \
++					 MTK_HSDMA_BURST_64BYTES | \
++					 MTK_HSDMA_GLO_MULTI_DMA)
++
++/* Registers for reset */
++#define MTK_HSDMA_RESET			0x208
++#define MTK_HSDMA_RST_TX		BIT(0)
++#define MTK_HSDMA_RST_RX		BIT(16)
++
++/* Registers for interrupt control */
++#define MTK_HSDMA_DLYINT		0x20c
++#define MTK_HSDMA_RXDLY_INT_EN		BIT(15)
++
++/* Interrupt fires when the pending number's more than the specified */
++#define MTK_HSDMA_RXMAX_PINT(x)		(((x) & 0x7f) << 8)
++
++/* Interrupt fires when the pending time's more than the specified in 20 us */
++#define MTK_HSDMA_RXMAX_PTIME(x)	((x) & 0x7f)
++#define MTK_HSDMA_DLYINT_DEFAULT	(MTK_HSDMA_RXDLY_INT_EN | \
++					 MTK_HSDMA_RXMAX_PINT(20) | \
++					 MTK_HSDMA_RXMAX_PTIME(20))
++#define MTK_HSDMA_INT_STATUS		0x220
++#define MTK_HSDMA_INT_ENABLE		0x228
++#define MTK_HSDMA_INT_RXDONE		BIT(16)
++
++enum mtk_hsdma_vdesc_flag {
++	MTK_HSDMA_VDESC_FINISHED	= 0x01,
++};
++
++#define IS_MTK_HSDMA_VDESC_FINISHED(x) ((x) == MTK_HSDMA_VDESC_FINISHED)
++
++/**
++ * struct mtk_hsdma_pdesc - This is the struct holding info describing physical
++ *			    descriptor (PD) and its placement must be kept at
++ *			    4-bytes alignment in little endian order.
++ * @desc[1-4]:		    The control pad used to indicate hardware how to
++ *			    deal with the descriptor such as source and
++ *			    destination address and data length. The maximum
++ *			    data length each pdesc can handle is 0x3f80 bytes
++ */
++struct mtk_hsdma_pdesc {
++	__le32 desc1;
++	__le32 desc2;
++	__le32 desc3;
++	__le32 desc4;
++} __packed __aligned(4);
++
++/**
++ * struct mtk_hsdma_vdesc - This is the struct holding info describing virtual
++ *			    descriptor (VD)
++ * @vd:			    An instance for struct virt_dma_desc
++ * @len:		    The total data size device wants to move
++ * @residue:		    The remaining data size device will move
++ * @dest:		    The destination address device wants to move to
++ * @src:		    The source address device wants to move from
++ */
++struct mtk_hsdma_vdesc {
++	struct virt_dma_desc vd;
++	size_t len;
++	size_t residue;
++	dma_addr_t dest;
++	dma_addr_t src;
++};
++
++/**
++ * struct mtk_hsdma_cb - This is the struct holding extra info required for RX
++ *			 ring to know what relevant VD the the PD is being
++ *			 mapped to.
++ * @vd:			 Pointer to the relevant VD.
++ * @flag:		 Flag indicating what action should be taken when VD
++ *			 is completed.
++ */
++struct mtk_hsdma_cb {
++	struct virt_dma_desc *vd;
++	enum mtk_hsdma_vdesc_flag flag;
++};
++
++/**
++ * struct mtk_hsdma_ring - This struct holds info describing underlying ring
++ *			   space
++ * @txd:		   The descriptor TX ring which describes DMA source
++ *			   information
++ * @rxd:		   The descriptor RX ring which describes DMA
++ *			   destination information
++ * @cb:			   The extra information pointed at by RX ring
++ * @tphys:		   The physical addr of TX ring
++ * @rphys:		   The physical addr of RX ring
++ * @cur_tptr:		   Pointer to the next free descriptor used by the host
++ * @cur_rptr:		   Pointer to the last done descriptor by the device
++ */
++struct mtk_hsdma_ring {
++	struct mtk_hsdma_pdesc *txd;
++	struct mtk_hsdma_pdesc *rxd;
++	struct mtk_hsdma_cb *cb;
++	dma_addr_t tphys;
++	dma_addr_t rphys;
++	u16 cur_tptr;
++	u16 cur_rptr;
++};
++
++/**
++ * struct mtk_hsdma_pchan - This is the struct holding info describing physical
++ *			   channel (PC)
++ * @ring:		   An instance for the underlying ring
++ * @sz_ring:		   Total size allocated for the ring
++ * @nr_free:		   Total number of free rooms in the ring. It would
++ *			   be accessed and updated frequently between IRQ
++ *			   context and user context to reflect whether ring
++ *			   can accept requests from VD.
++ */
++struct mtk_hsdma_pchan {
++	struct mtk_hsdma_ring ring;
++	size_t sz_ring;
++	atomic_t nr_free;
++};
++
++/**
++ * struct mtk_hsdma_vchan - This is the struct holding info describing virtual
++ *			   channel (VC)
++ * @vc:			   An instance for struct virt_dma_chan
++ * @issue_completion:	   The wait for all issued descriptors completited
++ * @issue_synchronize:	   Bool indicating channel synchronization starts
++ * @desc_hw_processing:	   List those descriptors the hardware is processing,
++ *			   which is protected by vc.lock
++ */
++struct mtk_hsdma_vchan {
++	struct virt_dma_chan vc;
++	struct completion issue_completion;
++	bool issue_synchronize;
++	struct list_head desc_hw_processing;
++};
++
++/**
++ * struct mtk_hsdma_soc - This is the struct holding differences among SoCs
++ * @ddone:		  Bit mask for DDONE
++ * @ls0:		  Bit mask for LS0
++ */
++struct mtk_hsdma_soc {
++	__le32 ddone;
++	__le32 ls0;
++};
++
++/**
++ * struct mtk_hsdma_device - This is the struct holding info describing HSDMA
++ *			     device
++ * @ddev:		     An instance for struct dma_device
++ * @base:		     The mapped register I/O base
++ * @clk:		     The clock that device internal is using
++ * @irq:		     The IRQ that device are using
++ * @dma_requests:	     The number of VCs the device supports to
++ * @vc:			     The pointer to all available VCs
++ * @pc:			     The pointer to the underlying PC
++ * @pc_refcnt:		     Track how many VCs are using the PC
++ * @lock:		     Lock protect agaisting multiple VCs access PC
++ * @soc:		     The pointer to area holding differences among
++ *			     vaious platform
++ */
++struct mtk_hsdma_device {
++	struct dma_device ddev;
++	void __iomem *base;
++	struct clk *clk;
++	u32 irq;
++
++	u32 dma_requests;
++	struct mtk_hsdma_vchan *vc;
++	struct mtk_hsdma_pchan *pc;
++	refcount_t pc_refcnt;
++
++	/* Lock used to protect against multiple VCs access PC */
++	spinlock_t lock;
++
++	const struct mtk_hsdma_soc *soc;
++};
++
++static struct mtk_hsdma_device *to_hsdma_dev(struct dma_chan *chan)
++{
++	return container_of(chan->device, struct mtk_hsdma_device, ddev);
++}
++
++static inline struct mtk_hsdma_vchan *to_hsdma_vchan(struct dma_chan *chan)
++{
++	return container_of(chan, struct mtk_hsdma_vchan, vc.chan);
++}
++
++static struct mtk_hsdma_vdesc *to_hsdma_vdesc(struct virt_dma_desc *vd)
++{
++	return container_of(vd, struct mtk_hsdma_vdesc, vd);
++}
++
++static struct device *hsdma2dev(struct mtk_hsdma_device *hsdma)
++{
++	return hsdma->ddev.dev;
++}
++
++static u32 mtk_dma_read(struct mtk_hsdma_device *hsdma, u32 reg)
++{
++	return readl(hsdma->base + reg);
++}
++
++static void mtk_dma_write(struct mtk_hsdma_device *hsdma, u32 reg, u32 val)
++{
++	writel(val, hsdma->base + reg);
++}
++
++static void mtk_dma_rmw(struct mtk_hsdma_device *hsdma, u32 reg,
++			u32 mask, u32 set)
++{
++	u32 val;
++
++	val = mtk_dma_read(hsdma, reg);
++	val &= ~mask;
++	val |= set;
++	mtk_dma_write(hsdma, reg, val);
++}
++
++static void mtk_dma_set(struct mtk_hsdma_device *hsdma, u32 reg, u32 val)
++{
++	mtk_dma_rmw(hsdma, reg, 0, val);
++}
++
++static void mtk_dma_clr(struct mtk_hsdma_device *hsdma, u32 reg, u32 val)
++{
++	mtk_dma_rmw(hsdma, reg, val, 0);
++}
++
++static void mtk_hsdma_vdesc_free(struct virt_dma_desc *vd)
++{
++	kfree(container_of(vd, struct mtk_hsdma_vdesc, vd));
++}
++
++static int mtk_hsdma_busy_wait(struct mtk_hsdma_device *hsdma)
++{
++	u32 status = 0;
++
++	return readl_poll_timeout(hsdma->base + MTK_HSDMA_GLO, status,
++				  !(status & MTK_HSDMA_GLO_BUSY),
++				  MTK_HSDMA_USEC_POLL,
++				  MTK_HSDMA_TIMEOUT_POLL);
++}
++
++static int mtk_hsdma_alloc_pchan(struct mtk_hsdma_device *hsdma,
++				 struct mtk_hsdma_pchan *pc)
++{
++	struct mtk_hsdma_ring *ring = &pc->ring;
++	int err;
++
++	memset(pc, 0, sizeof(*pc));
++
++	/*
++	 * Allocate ring space where [0 ... MTK_DMA_SIZE - 1] is for TX ring
++	 * and [MTK_DMA_SIZE ... 2 * MTK_DMA_SIZE - 1] is for RX ring.
++	 */
++	pc->sz_ring = 2 * MTK_DMA_SIZE * sizeof(*ring->txd);
++	ring->txd = dma_zalloc_coherent(hsdma2dev(hsdma), pc->sz_ring,
++					&ring->tphys, GFP_NOWAIT);
++	if (!ring->txd)
++		return -ENOMEM;
++
++	ring->rxd = &ring->txd[MTK_DMA_SIZE];
++	ring->rphys = ring->tphys + MTK_DMA_SIZE * sizeof(*ring->txd);
++	ring->cur_tptr = 0;
++	ring->cur_rptr = MTK_DMA_SIZE - 1;
++
++	ring->cb = kcalloc(MTK_DMA_SIZE, sizeof(*ring->cb), GFP_NOWAIT);
++	if (!ring->cb) {
++		err = -ENOMEM;
++		goto err_free_dma;
++	}
++
++	atomic_set(&pc->nr_free, MTK_DMA_SIZE - 1);
++
++	/* Disable HSDMA and wait for the completion */
++	mtk_dma_clr(hsdma, MTK_HSDMA_GLO, MTK_HSDMA_GLO_DMA);
++	err = mtk_hsdma_busy_wait(hsdma);
++	if (err)
++		goto err_free_cb;
++
++	/* Reset */
++	mtk_dma_set(hsdma, MTK_HSDMA_RESET,
++		    MTK_HSDMA_RST_TX | MTK_HSDMA_RST_RX);
++	mtk_dma_clr(hsdma, MTK_HSDMA_RESET,
++		    MTK_HSDMA_RST_TX | MTK_HSDMA_RST_RX);
++
++	/* Setup HSDMA initial pointer in the ring */
++	mtk_dma_write(hsdma, MTK_HSDMA_TX_BASE, ring->tphys);
++	mtk_dma_write(hsdma, MTK_HSDMA_TX_CNT, MTK_DMA_SIZE);
++	mtk_dma_write(hsdma, MTK_HSDMA_TX_CPU, ring->cur_tptr);
++	mtk_dma_write(hsdma, MTK_HSDMA_TX_DMA, 0);
++	mtk_dma_write(hsdma, MTK_HSDMA_RX_BASE, ring->rphys);
++	mtk_dma_write(hsdma, MTK_HSDMA_RX_CNT, MTK_DMA_SIZE);
++	mtk_dma_write(hsdma, MTK_HSDMA_RX_CPU, ring->cur_rptr);
++	mtk_dma_write(hsdma, MTK_HSDMA_RX_DMA, 0);
++
++	/* Enable HSDMA */
++	mtk_dma_set(hsdma, MTK_HSDMA_GLO, MTK_HSDMA_GLO_DMA);
++
++	/* Setup delayed interrupt */
++	mtk_dma_write(hsdma, MTK_HSDMA_DLYINT, MTK_HSDMA_DLYINT_DEFAULT);
++
++	/* Enable interrupt */
++	mtk_dma_set(hsdma, MTK_HSDMA_INT_ENABLE, MTK_HSDMA_INT_RXDONE);
++
++	return 0;
++
++err_free_cb:
++	kfree(ring->cb);
++
++err_free_dma:
++	dma_free_coherent(hsdma2dev(hsdma),
++			  pc->sz_ring, ring->txd, ring->tphys);
++	return err;
++}
++
++static void mtk_hsdma_free_pchan(struct mtk_hsdma_device *hsdma,
++				 struct mtk_hsdma_pchan *pc)
++{
++	struct mtk_hsdma_ring *ring = &pc->ring;
++
++	/* Disable HSDMA and then wait for the completion */
++	mtk_dma_clr(hsdma, MTK_HSDMA_GLO, MTK_HSDMA_GLO_DMA);
++	mtk_hsdma_busy_wait(hsdma);
++
++	/* Reset pointer in the ring */
++	mtk_dma_clr(hsdma, MTK_HSDMA_INT_ENABLE, MTK_HSDMA_INT_RXDONE);
++	mtk_dma_write(hsdma, MTK_HSDMA_TX_BASE, 0);
++	mtk_dma_write(hsdma, MTK_HSDMA_TX_CNT, 0);
++	mtk_dma_write(hsdma, MTK_HSDMA_TX_CPU, 0);
++	mtk_dma_write(hsdma, MTK_HSDMA_RX_BASE, 0);
++	mtk_dma_write(hsdma, MTK_HSDMA_RX_CNT, 0);
++	mtk_dma_write(hsdma, MTK_HSDMA_RX_CPU, MTK_DMA_SIZE - 1);
++
++	kfree(ring->cb);
++
++	dma_free_coherent(hsdma2dev(hsdma),
++			  pc->sz_ring, ring->txd, ring->tphys);
++}
++
++static int mtk_hsdma_issue_pending_vdesc(struct mtk_hsdma_device *hsdma,
++					 struct mtk_hsdma_pchan *pc,
++					 struct mtk_hsdma_vdesc *hvd)
++{
++	struct mtk_hsdma_ring *ring = &pc->ring;
++	struct mtk_hsdma_pdesc *txd, *rxd;
++	u16 reserved, prev, tlen, num_sgs;
++	unsigned long flags;
++
++	/* Protect against PC is accessed by multiple VCs simultaneously */
++	spin_lock_irqsave(&hsdma->lock, flags);
++
++	/*
++	 * Reserve rooms, where pc->nr_free is used to track how many free
++	 * rooms in the ring being updated in user and IRQ context.
++	 */
++	num_sgs = DIV_ROUND_UP(hvd->len, MTK_HSDMA_MAX_LEN);
++	reserved = min_t(u16, num_sgs, atomic_read(&pc->nr_free));
++
++	if (!reserved) {
++		spin_unlock_irqrestore(&hsdma->lock, flags);
++		return -ENOSPC;
++	}
++
++	atomic_sub(reserved, &pc->nr_free);
++
++	while (reserved--) {
++		/* Limit size by PD capability for valid data moving */
++		tlen = (hvd->len > MTK_HSDMA_MAX_LEN) ?
++		       MTK_HSDMA_MAX_LEN : hvd->len;
++
++		/*
++		 * Setup PDs using the remaining VD info mapped on those
++		 * reserved rooms. And since RXD is shared memory between the
++		 * host and the device allocated by dma_alloc_coherent call,
++		 * the helper macro WRITE_ONCE can ensure the data written to
++		 * RAM would really happens.
++		 */
++		txd = &ring->txd[ring->cur_tptr];
++		WRITE_ONCE(txd->desc1, hvd->src);
++		WRITE_ONCE(txd->desc2,
++			   hsdma->soc->ls0 | MTK_HSDMA_DESC_PLEN(tlen));
++
++		rxd = &ring->rxd[ring->cur_tptr];
++		WRITE_ONCE(rxd->desc1, hvd->dest);
++		WRITE_ONCE(rxd->desc2, MTK_HSDMA_DESC_PLEN(tlen));
++
++		/* Associate VD, the PD belonged to */
++		ring->cb[ring->cur_tptr].vd = &hvd->vd;
++
++		/* Move forward the pointer of TX ring */
++		ring->cur_tptr = MTK_HSDMA_NEXT_DESP_IDX(ring->cur_tptr,
++							 MTK_DMA_SIZE);
++
++		/* Update VD with remaining data */
++		hvd->src  += tlen;
++		hvd->dest += tlen;
++		hvd->len  -= tlen;
++	}
++
++	/*
++	 * Tagging flag for the last PD for VD will be responsible for
++	 * completing VD.
++	 */
++	if (!hvd->len) {
++		prev = MTK_HSDMA_LAST_DESP_IDX(ring->cur_tptr, MTK_DMA_SIZE);
++		ring->cb[prev].flag = MTK_HSDMA_VDESC_FINISHED;
++	}
++
++	/* Ensure all changes indeed done before we're going on */
++	wmb();
++
++	/*
++	 * Updating into hardware the pointer of TX ring lets HSDMA to take
++	 * action for those pending PDs.
++	 */
++	mtk_dma_write(hsdma, MTK_HSDMA_TX_CPU, ring->cur_tptr);
++
++	spin_unlock_irqrestore(&hsdma->lock, flags);
++
++	return 0;
++}
++
++static void mtk_hsdma_issue_vchan_pending(struct mtk_hsdma_device *hsdma,
++					  struct mtk_hsdma_vchan *hvc)
++{
++	struct virt_dma_desc *vd, *vd2;
++	int err;
++
++	lockdep_assert_held(&hvc->vc.lock);
++
++	list_for_each_entry_safe(vd, vd2, &hvc->vc.desc_issued, node) {
++		struct mtk_hsdma_vdesc *hvd;
++
++		hvd = to_hsdma_vdesc(vd);
++
++		/* Map VD into PC and all VCs shares a single PC */
++		err = mtk_hsdma_issue_pending_vdesc(hsdma, hsdma->pc, hvd);
++
++		/*
++		 * Move VD from desc_issued to desc_hw_processing when entire
++		 * VD is fit into available PDs. Otherwise, the uncompleted
++		 * VDs would stay in list desc_issued and then restart the
++		 * processing as soon as possible once underlying ring space
++		 * got freed.
++		 */
++		if (err == -ENOSPC || hvd->len > 0)
++			break;
++
++		/*
++		 * The extra list desc_hw_processing is used because
++		 * hardware can't provide sufficient information allowing us
++		 * to know what VDs are still working on the underlying ring.
++		 * Through the additional list, it can help us to implement
++		 * terminate_all, residue calculation and such thing needed
++		 * to know detail descriptor status on the hardware.
++		 */
++		list_move_tail(&vd->node, &hvc->desc_hw_processing);
++	}
++}
++
++static void mtk_hsdma_free_rooms_in_ring(struct mtk_hsdma_device *hsdma)
++{
++	struct mtk_hsdma_vchan *hvc;
++	struct mtk_hsdma_pdesc *rxd;
++	struct mtk_hsdma_vdesc *hvd;
++	struct mtk_hsdma_pchan *pc;
++	struct mtk_hsdma_cb *cb;
++	int i = MTK_DMA_SIZE;
++	__le32 desc2;
++	u32 status;
++	u16 next;
++
++	/* Read IRQ status */
++	status = mtk_dma_read(hsdma, MTK_HSDMA_INT_STATUS);
++	if (unlikely(!(status & MTK_HSDMA_INT_RXDONE)))
++		goto rx_done;
++
++	pc = hsdma->pc;
++
++	/*
++	 * Using a fail-safe loop with iterations of up to MTK_DMA_SIZE to
++	 * reclaim these finished descriptors: The most number of PDs the ISR
++	 * can handle at one time shouldn't be more than MTK_DMA_SIZE so we
++	 * take it as limited count instead of just using a dangerous infinite
++	 * poll.
++	 */
++	while (i--) {
++		next = MTK_HSDMA_NEXT_DESP_IDX(pc->ring.cur_rptr,
++					       MTK_DMA_SIZE);
++		rxd = &pc->ring.rxd[next];
++
++		/*
++		 * If MTK_HSDMA_DESC_DDONE is no specified, that means data
++		 * moving for the PD is still under going.
++		 */
++		desc2 = READ_ONCE(rxd->desc2);
++		if (!(desc2 & hsdma->soc->ddone))
++			break;
++
++		cb = &pc->ring.cb[next];
++		if (unlikely(!cb->vd)) {
++			dev_err(hsdma2dev(hsdma), "cb->vd cannot be null\n");
++			break;
++		}
++
++		/* Update residue of VD the associated PD belonged to */
++		hvd = to_hsdma_vdesc(cb->vd);
++		hvd->residue -= MTK_HSDMA_DESC_PLEN_GET(rxd->desc2);
++
++		/* Complete VD until the relevant last PD is finished */
++		if (IS_MTK_HSDMA_VDESC_FINISHED(cb->flag)) {
++			hvc = to_hsdma_vchan(cb->vd->tx.chan);
++
++			spin_lock(&hvc->vc.lock);
++
++			/* Remove VD from list desc_hw_processing */
++			list_del(&cb->vd->node);
++
++			/* Add VD into list desc_completed */
++			vchan_cookie_complete(cb->vd);
++
++			if (hvc->issue_synchronize &&
++			    list_empty(&hvc->desc_hw_processing)) {
++				complete(&hvc->issue_completion);
++				hvc->issue_synchronize = false;
++			}
++			spin_unlock(&hvc->vc.lock);
++
++			cb->flag = 0;
++		}
++
++		cb->vd = 0;
++
++		/*
++		 * Recycle the RXD with the helper WRITE_ONCE that can ensure
++		 * data written into RAM would really happens.
++		 */
++		WRITE_ONCE(rxd->desc1, 0);
++		WRITE_ONCE(rxd->desc2, 0);
++		pc->ring.cur_rptr = next;
++
++		/* Release rooms */
++		atomic_inc(&pc->nr_free);
++	}
++
++	/* Ensure all changes indeed done before we're going on */
++	wmb();
++
++	/* Update CPU pointer for those completed PDs */
++	mtk_dma_write(hsdma, MTK_HSDMA_RX_CPU, pc->ring.cur_rptr);
++
++	/*
++	 * Acking the pending IRQ allows hardware no longer to keep the used
++	 * IRQ line in certain trigger state when software has completed all
++	 * the finished physical descriptors.
++	 */
++	if (atomic_read(&pc->nr_free) >= MTK_DMA_SIZE - 1)
++		mtk_dma_write(hsdma, MTK_HSDMA_INT_STATUS, status);
++
++	/* ASAP handles pending VDs in all VCs after freeing some rooms */
++	for (i = 0; i < hsdma->dma_requests; i++) {
++		hvc = &hsdma->vc[i];
++		spin_lock(&hvc->vc.lock);
++		mtk_hsdma_issue_vchan_pending(hsdma, hvc);
++		spin_unlock(&hvc->vc.lock);
++	}
++
++rx_done:
++	/* All completed PDs are cleaned up, so enable interrupt again */
++	mtk_dma_set(hsdma, MTK_HSDMA_INT_ENABLE, MTK_HSDMA_INT_RXDONE);
++}
++
++static irqreturn_t mtk_hsdma_irq(int irq, void *devid)
++{
++	struct mtk_hsdma_device *hsdma = devid;
++
++	/*
++	 * Disable interrupt until all completed PDs are cleaned up in
++	 * mtk_hsdma_free_rooms call.
++	 */
++	mtk_dma_clr(hsdma, MTK_HSDMA_INT_ENABLE, MTK_HSDMA_INT_RXDONE);
++
++	mtk_hsdma_free_rooms_in_ring(hsdma);
++
++	return IRQ_HANDLED;
++}
++
++static struct virt_dma_desc *mtk_hsdma_find_active_desc(struct dma_chan *c,
++							dma_cookie_t cookie)
++{
++	struct mtk_hsdma_vchan *hvc = to_hsdma_vchan(c);
++	struct virt_dma_desc *vd;
++
++	list_for_each_entry(vd, &hvc->desc_hw_processing, node)
++		if (vd->tx.cookie == cookie)
++			return vd;
++
++	list_for_each_entry(vd, &hvc->vc.desc_issued, node)
++		if (vd->tx.cookie == cookie)
++			return vd;
++
++	return NULL;
++}
++
++static enum dma_status mtk_hsdma_tx_status(struct dma_chan *c,
++					   dma_cookie_t cookie,
++					   struct dma_tx_state *txstate)
++{
++	struct mtk_hsdma_vchan *hvc = to_hsdma_vchan(c);
++	struct mtk_hsdma_vdesc *hvd;
++	struct virt_dma_desc *vd;
++	enum dma_status ret;
++	unsigned long flags;
++	size_t bytes = 0;
++
++	ret = dma_cookie_status(c, cookie, txstate);
++	if (ret == DMA_COMPLETE || !txstate)
++		return ret;
++
++	spin_lock_irqsave(&hvc->vc.lock, flags);
++	vd = mtk_hsdma_find_active_desc(c, cookie);
++	spin_unlock_irqrestore(&hvc->vc.lock, flags);
++
++	if (vd) {
++		hvd = to_hsdma_vdesc(vd);
++		bytes = hvd->residue;
++	}
++
++	dma_set_residue(txstate, bytes);
++
++	return ret;
++}
++
++static void mtk_hsdma_issue_pending(struct dma_chan *c)
++{
++	struct mtk_hsdma_device *hsdma = to_hsdma_dev(c);
++	struct mtk_hsdma_vchan *hvc = to_hsdma_vchan(c);
++	unsigned long flags;
++
++	spin_lock_irqsave(&hvc->vc.lock, flags);
++
++	if (vchan_issue_pending(&hvc->vc))
++		mtk_hsdma_issue_vchan_pending(hsdma, hvc);
++
++	spin_unlock_irqrestore(&hvc->vc.lock, flags);
++}
++
++static struct dma_async_tx_descriptor *
++mtk_hsdma_prep_dma_memcpy(struct dma_chan *c, dma_addr_t dest,
++			  dma_addr_t src, size_t len, unsigned long flags)
++{
++	struct mtk_hsdma_vdesc *hvd;
++
++	hvd = kzalloc(sizeof(*hvd), GFP_NOWAIT);
++	if (!hvd)
++		return NULL;
++
++	hvd->len = len;
++	hvd->residue = len;
++	hvd->src = src;
++	hvd->dest = dest;
++
++	return vchan_tx_prep(to_virt_chan(c), &hvd->vd, flags);
++}
++
++static int mtk_hsdma_free_inactive_desc(struct dma_chan *c)
++{
++	struct virt_dma_chan *vc = to_virt_chan(c);
++	unsigned long flags;
++	LIST_HEAD(head);
++
++	spin_lock_irqsave(&vc->lock, flags);
++	list_splice_tail_init(&vc->desc_allocated, &head);
++	list_splice_tail_init(&vc->desc_submitted, &head);
++	list_splice_tail_init(&vc->desc_issued, &head);
++	spin_unlock_irqrestore(&vc->lock, flags);
++
++	/* At the point, we don't expect users put descriptor into VC again */
++	vchan_dma_desc_free_list(vc, &head);
++
++	return 0;
++}
++
++static void mtk_hsdma_free_active_desc(struct dma_chan *c)
++{
++	struct mtk_hsdma_vchan *hvc = to_hsdma_vchan(c);
++	bool sync_needed = false;
++
++	/*
++	 * Once issue_synchronize is being set, which means once the hardware
++	 * consumes all descriptors for the channel in the ring, the
++	 * synchronization must be be notified immediately it is completed.
++	 */
++	spin_lock(&hvc->vc.lock);
++	if (!list_empty(&hvc->desc_hw_processing)) {
++		hvc->issue_synchronize = true;
++		sync_needed = true;
++	}
++	spin_unlock(&hvc->vc.lock);
++
++	if (sync_needed)
++		wait_for_completion(&hvc->issue_completion);
++	/*
++	 * At the point, we expect that all remaining descriptors in the ring
++	 * for the channel should be all processing done.
++	 */
++	WARN_ONCE(!list_empty(&hvc->desc_hw_processing),
++		  "Desc pending still in list desc_hw_processing\n");
++
++	/* Free all descriptors in list desc_completed */
++	vchan_synchronize(&hvc->vc);
++
++	WARN_ONCE(!list_empty(&hvc->vc.desc_completed),
++		  "Desc pending still in list desc_completed\n");
++}
++
++static int mtk_hsdma_terminate_all(struct dma_chan *c)
++{
++	/*
++	 * Free pending descriptors not processed yet by hardware that have
++	 * previously been submitted to the channel.
++	 */
++	mtk_hsdma_free_inactive_desc(c);
++
++	/*
++	 * However, the DMA engine doesn't provide any way to stop these
++	 * descriptors being processed currently by hardware. The only way is
++	 * to just waiting until these descriptors are all processed completely
++	 * through mtk_hsdma_free_active_desc call.
++	 */
++	mtk_hsdma_free_active_desc(c);
++
++	return 0;
++}
++
++static int mtk_hsdma_alloc_chan_resources(struct dma_chan *c)
++{
++	struct mtk_hsdma_device *hsdma = to_hsdma_dev(c);
++	int err;
++
++	/*
++	 * Since HSDMA has only one PC, the resource for PC is being allocated
++	 * when the first VC is being created and the other VCs would run on
++	 * the same PC.
++	 */
++	if (!refcount_read(&hsdma->pc_refcnt)) {
++		err = mtk_hsdma_alloc_pchan(hsdma, hsdma->pc);
++		if (err)
++			return err;
++		/*
++		 * refcount_inc would complain increment on 0; use-after-free.
++		 * Thus, we need to explicitly set it as 1 initially.
++		 */
++		refcount_set(&hsdma->pc_refcnt, 1);
++	} else {
++		refcount_inc(&hsdma->pc_refcnt);
++	}
++
++	return 0;
++}
++
++static void mtk_hsdma_free_chan_resources(struct dma_chan *c)
++{
++	struct mtk_hsdma_device *hsdma = to_hsdma_dev(c);
++
++	/* Free all descriptors in all lists on the VC */
++	mtk_hsdma_terminate_all(c);
++
++	/* The resource for PC is not freed until all the VCs are destroyed */
++	if (!refcount_dec_and_test(&hsdma->pc_refcnt))
++		return;
++
++	mtk_hsdma_free_pchan(hsdma, hsdma->pc);
++}
++
++static int mtk_hsdma_hw_init(struct mtk_hsdma_device *hsdma)
++{
++	int err;
++
++	pm_runtime_enable(hsdma2dev(hsdma));
++	pm_runtime_get_sync(hsdma2dev(hsdma));
++
++	err = clk_prepare_enable(hsdma->clk);
++	if (err)
++		return err;
++
++	mtk_dma_write(hsdma, MTK_HSDMA_INT_ENABLE, 0);
++	mtk_dma_write(hsdma, MTK_HSDMA_GLO, MTK_HSDMA_GLO_DEFAULT);
++
++	return 0;
++}
++
++static int mtk_hsdma_hw_deinit(struct mtk_hsdma_device *hsdma)
++{
++	mtk_dma_write(hsdma, MTK_HSDMA_GLO, 0);
++
++	clk_disable_unprepare(hsdma->clk);
++
++	pm_runtime_put_sync(hsdma2dev(hsdma));
++	pm_runtime_disable(hsdma2dev(hsdma));
++
++	return 0;
++}
++
++static const struct mtk_hsdma_soc mt7623_soc = {
++	.ddone = BIT(31),
++	.ls0 = BIT(30),
++};
++
++static const struct mtk_hsdma_soc mt7622_soc = {
++	.ddone = BIT(15),
++	.ls0 = BIT(14),
++};
++
++static const struct of_device_id mtk_hsdma_match[] = {
++	{ .compatible = "mediatek,mt7623-hsdma", .data = &mt7623_soc},
++	{ .compatible = "mediatek,mt7622-hsdma", .data = &mt7622_soc},
++	{ /* sentinel */ }
++};
++MODULE_DEVICE_TABLE(of, mtk_hsdma_match);
++
++static int mtk_hsdma_probe(struct platform_device *pdev)
++{
++	struct mtk_hsdma_device *hsdma;
++	struct mtk_hsdma_vchan *vc;
++	struct dma_device *dd;
++	struct resource *res;
++	int i, err;
++
++	hsdma = devm_kzalloc(&pdev->dev, sizeof(*hsdma), GFP_KERNEL);
++	if (!hsdma)
++		return -ENOMEM;
++
++	dd = &hsdma->ddev;
++
++	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
++	hsdma->base = devm_ioremap_resource(&pdev->dev, res);
++	if (IS_ERR(hsdma->base))
++		return PTR_ERR(hsdma->base);
++
++	hsdma->soc = of_device_get_match_data(&pdev->dev);
++	if (!hsdma->soc) {
++		dev_err(&pdev->dev, "No device match found\n");
++		return -ENODEV;
++	}
++
++	hsdma->clk = devm_clk_get(&pdev->dev, "hsdma");
++	if (IS_ERR(hsdma->clk)) {
++		dev_err(&pdev->dev, "No clock for %s\n",
++			dev_name(&pdev->dev));
++		return PTR_ERR(hsdma->clk);
++	}
++
++	res = platform_get_resource(pdev, IORESOURCE_IRQ, 0);
++	if (!res) {
++		dev_err(&pdev->dev, "No irq resource for %s\n",
++			dev_name(&pdev->dev));
++		return -EINVAL;
++	}
++	hsdma->irq = res->start;
++
++	refcount_set(&hsdma->pc_refcnt, 0);
++	spin_lock_init(&hsdma->lock);
++
++	dma_cap_set(DMA_MEMCPY, dd->cap_mask);
++
++	dd->copy_align = MTK_HSDMA_ALIGN_SIZE;
++	dd->device_alloc_chan_resources = mtk_hsdma_alloc_chan_resources;
++	dd->device_free_chan_resources = mtk_hsdma_free_chan_resources;
++	dd->device_tx_status = mtk_hsdma_tx_status;
++	dd->device_issue_pending = mtk_hsdma_issue_pending;
++	dd->device_prep_dma_memcpy = mtk_hsdma_prep_dma_memcpy;
++	dd->device_terminate_all = mtk_hsdma_terminate_all;
++	dd->src_addr_widths = MTK_HSDMA_DMA_BUSWIDTHS;
++	dd->dst_addr_widths = MTK_HSDMA_DMA_BUSWIDTHS;
++	dd->directions = BIT(DMA_MEM_TO_MEM);
++	dd->residue_granularity = DMA_RESIDUE_GRANULARITY_SEGMENT;
++	dd->dev = &pdev->dev;
++	INIT_LIST_HEAD(&dd->channels);
++
++	hsdma->dma_requests = MTK_HSDMA_NR_VCHANS;
++	if (pdev->dev.of_node && of_property_read_u32(pdev->dev.of_node,
++						      "dma-requests",
++						      &hsdma->dma_requests)) {
++		dev_info(&pdev->dev,
++			 "Using %u as missing dma-requests property\n",
++			 MTK_HSDMA_NR_VCHANS);
++	}
++
++	hsdma->pc = devm_kcalloc(&pdev->dev, MTK_HSDMA_NR_MAX_PCHANS,
++				 sizeof(*hsdma->pc), GFP_KERNEL);
++	if (!hsdma->pc)
++		return -ENOMEM;
++
++	hsdma->vc = devm_kcalloc(&pdev->dev, hsdma->dma_requests,
++				 sizeof(*hsdma->vc), GFP_KERNEL);
++	if (!hsdma->vc)
++		return -ENOMEM;
++
++	for (i = 0; i < hsdma->dma_requests; i++) {
++		vc = &hsdma->vc[i];
++		vc->vc.desc_free = mtk_hsdma_vdesc_free;
++		vchan_init(&vc->vc, dd);
++		init_completion(&vc->issue_completion);
++		INIT_LIST_HEAD(&vc->desc_hw_processing);
++	}
++
++	err = dma_async_device_register(dd);
++	if (err)
++		return err;
++
++	err = of_dma_controller_register(pdev->dev.of_node,
++					 of_dma_xlate_by_chan_id, hsdma);
++	if (err) {
++		dev_err(&pdev->dev,
++			"MediaTek HSDMA OF registration failed %d\n", err);
++		goto err_unregister;
++	}
++
++	mtk_hsdma_hw_init(hsdma);
++
++	err = devm_request_irq(&pdev->dev, hsdma->irq,
++			       mtk_hsdma_irq, 0,
++			       dev_name(&pdev->dev), hsdma);
++	if (err) {
++		dev_err(&pdev->dev,
++			"request_irq failed with err %d\n", err);
++		goto err_unregister;
++	}
++
++	platform_set_drvdata(pdev, hsdma);
++
++	dev_info(&pdev->dev, "MediaTek HSDMA driver registered\n");
++
++	return 0;
++
++err_unregister:
++	dma_async_device_unregister(dd);
++
++	return err;
++}
++
++static int mtk_hsdma_remove(struct platform_device *pdev)
++{
++	struct mtk_hsdma_device *hsdma = platform_get_drvdata(pdev);
++	struct mtk_hsdma_vchan *vc;
++	int i;
++
++	/* Kill VC task */
++	for (i = 0; i < hsdma->dma_requests; i++) {
++		vc = &hsdma->vc[i];
++
++		list_del(&vc->vc.chan.device_node);
++		tasklet_kill(&vc->vc.task);
++	}
++
++	/* Disable DMA interrupt */
++	mtk_dma_write(hsdma, MTK_HSDMA_INT_ENABLE, 0);
++
++	/* Waits for any pending IRQ handlers to complete */
++	synchronize_irq(hsdma->irq);
++
++	/* Disable hardware */
++	mtk_hsdma_hw_deinit(hsdma);
++
++	dma_async_device_unregister(&hsdma->ddev);
++	of_dma_controller_free(pdev->dev.of_node);
++
++	return 0;
++}
++
++static struct platform_driver mtk_hsdma_driver = {
++	.probe		= mtk_hsdma_probe,
++	.remove		= mtk_hsdma_remove,
++	.driver = {
++		.name		= KBUILD_MODNAME,
++		.of_match_table	= mtk_hsdma_match,
++	},
++};
++module_platform_driver(mtk_hsdma_driver);
++
++MODULE_DESCRIPTION("MediaTek High-Speed DMA Controller Driver");
++MODULE_AUTHOR("Sean Wang <sean.wang@mediatek.com>");
++MODULE_LICENSE("GPL v2");
+-- 
+2.11.0
+