From afb17d8bb983eb4e1530300319565a30f4067f8c Mon Sep 17 00:00:00 2001
From: Phil Elwell <phil@raspberrypi.com>
Date: Wed, 28 Apr 2021 17:46:01 +0100
Subject: [PATCH] dmaengine: dw-axi-dmac: Fixes for RP1

Don't assume that DMA addresses of devices are the same as their
physical addresses - convert correctly.

The CFG2 register layout is used when there are more than 8 channels,
but also when configured for more than 16 target peripheral devices
because the index of the handshake signal has to be made wider.

Reset the DMAC on probe

The driver goes to the trouble of tracking when transfers have been
paused, but then doesn't report that state when queried.

Not having APB registers is not an error - for most use cases it's
not even of interest, it's expected. Demote the message to debug level,
which is disabled by default.

Each channel has a descriptor pool, which is shared between transfers.
It is unsafe to treat the total number of descriptors allocated from a
pool as the number allocated to a specific transfer; doing so leads
to releasing buffers that shouldn't be released and walking off the
ends of descriptor lists. Instead, give each transfer descriptor its
own count.

Support partial transfers:
Some use cases involve streaming from a device where the transfer only
proceeds when the device's FIFO occupancy exceeds a certain threshold.
In such cases (e.g. when pulling data from a UART) it is important to
know how much data has been transferred so far, in order that remaining
bytes can be read from the FIFO directly by software.

Add the necessary code to provide this "residue" value with a finer,
sub-transfer granularity.

In order to prevent the occasional byte getting stuck in the DMA
controller's internal buffers, restrict the destination memory width
to the source register width.

Signed-off-by: Phil Elwell <phil@raspberrypi.com>

dmaengine: dw-axi-dmac: Fix a non-atomic update

dw_axi_dma_interrupt disables interrupts for the duration of the channel
handling. It does so by clearing a bit in the DMA_CFG register - an
action that involves a read-modify-write. That in itself would be safe
because there will be no further interrupts, hence no reentrancy, were
it the only bit of code accessing that register.

The only neighbour of INT_EN is DMAC_EN - the main enable for the block.
That's not the sort of thing you would expect to be modified during the
normal course of operation, but bizarrely it is set at the start of the
transfer of every block, in axi_chan_block_xfer_star, by a call to
axi_dma_enable. This can lead to INT_EN being accidentally cleared,
which causes all DMA transfers to time out.

One might think that the enabling was being delayed until the first
transfer, but the probe function calls axi_dma_resume which in turn
calls axi_dma_enable, so that isn't the case.

Fix the atomicity problem by removing the spurious call to
axi_dma_enable.

Signed-off-by: Phil Elwell <phil@raspberrypi.com>
---
 .../dma/dw-axi-dmac/dw-axi-dmac-platform.c    | 135 +++++++++++++++---
 1 file changed, 114 insertions(+), 21 deletions(-)

--- a/drivers/dma/dw-axi-dmac/dw-axi-dmac-platform.c
+++ b/drivers/dma/dw-axi-dmac/dw-axi-dmac-platform.c
@@ -12,6 +12,7 @@
 #include <linux/device.h>
 #include <linux/dmaengine.h>
 #include <linux/dmapool.h>
+#include <linux/dma-direct.h>
 #include <linux/dma-mapping.h>
 #include <linux/err.h>
 #include <linux/interrupt.h>
@@ -95,6 +96,17 @@ axi_chan_iowrite64(struct axi_dma_chan *
 	iowrite32(upper_32_bits(val), chan->chan_regs + reg + 4);
 }
 
+static inline u64
+axi_chan_ioread64(struct axi_dma_chan *chan, u32 reg)
+{
+	/*
+	 * We split one 64 bit read into two 32 bit reads as some HW doesn't
+	 * support 64 bit access.
+	 */
+	return ((u64)ioread32(chan->chan_regs + reg + 4) << 32) +
+		ioread32(chan->chan_regs + reg);
+}
+
 static inline void axi_chan_config_write(struct axi_dma_chan *chan,
 					 struct axi_dma_chan_config *config)
 {
@@ -266,7 +278,18 @@ static void axi_dma_hw_init(struct axi_d
 {
 	int ret;
 	u32 i;
+	int retries = 1000;
 
+	axi_dma_iowrite32(chip, DMAC_RESET, 1);
+	while (axi_dma_ioread32(chip, DMAC_RESET)) {
+		retries--;
+		if (!retries) {
+			dev_err(chip->dev, "%s: DMAC failed to reset\n",
+				__func__);
+			return;
+		}
+		cpu_relax();
+	}
 	for (i = 0; i < chip->dw->hdata->nr_channels; i++) {
 		axi_chan_irq_disable(&chip->dw->chan[i], DWAXIDMAC_IRQ_ALL);
 		axi_chan_disable(&chip->dw->chan[i]);
@@ -351,6 +374,48 @@ static void vchan_desc_put(struct virt_d
 	axi_desc_put(vd_to_axi_desc(vdesc));
 }
 
+static u32 axi_dma_desc_src_pos(struct axi_dma_desc *desc, dma_addr_t addr)
+{
+	unsigned int idx = 0;
+	u32 pos = 0;
+
+	while (pos < desc->length) {
+		struct axi_dma_hw_desc *hw_desc = &desc->hw_desc[idx++];
+		u32 len = hw_desc->len;
+		dma_addr_t start = le64_to_cpu(hw_desc->lli->sar);
+
+		if (addr >= start && addr <= (start + len)) {
+			pos += addr - start;
+			break;
+		}
+
+		pos += len;
+	}
+
+	return pos;
+}
+
+static u32 axi_dma_desc_dst_pos(struct axi_dma_desc *desc, dma_addr_t addr)
+{
+	unsigned int idx = 0;
+	u32 pos = 0;
+
+	while (pos < desc->length) {
+		struct axi_dma_hw_desc *hw_desc = &desc->hw_desc[idx++];
+		u32 len = hw_desc->len;
+		dma_addr_t start = le64_to_cpu(hw_desc->lli->dar);
+
+		if (addr >= start && addr <= (start + len)) {
+			pos += addr - start;
+			break;
+		}
+
+		pos += len;
+	}
+
+	return pos;
+}
+
 static enum dma_status
 dma_chan_tx_status(struct dma_chan *dchan, dma_cookie_t cookie,
 		  struct dma_tx_state *txstate)
@@ -360,10 +425,7 @@ dma_chan_tx_status(struct dma_chan *dcha
 	enum dma_status status;
 	u32 completed_length;
 	unsigned long flags;
-	u32 completed_blocks;
 	size_t bytes = 0;
-	u32 length;
-	u32 len;
 
 	status = dma_cookie_status(dchan, cookie, txstate);
 	if (status == DMA_COMPLETE || !txstate)
@@ -372,16 +434,31 @@ dma_chan_tx_status(struct dma_chan *dcha
 	spin_lock_irqsave(&chan->vc.lock, flags);
 
 	vdesc = vchan_find_desc(&chan->vc, cookie);
-	if (vdesc) {
-		length = vd_to_axi_desc(vdesc)->length;
-		completed_blocks = vd_to_axi_desc(vdesc)->completed_blocks;
-		len = vd_to_axi_desc(vdesc)->hw_desc[0].len;
-		completed_length = completed_blocks * len;
-		bytes = length - completed_length;
+	if (vdesc && vdesc == vchan_next_desc(&chan->vc)) {
+		/* This descriptor is in-progress */
+		struct axi_dma_desc *desc = vd_to_axi_desc(vdesc);
+		dma_addr_t addr;
+
+		if (chan->direction == DMA_MEM_TO_DEV) {
+			addr = axi_chan_ioread64(chan, CH_SAR);
+			completed_length = axi_dma_desc_src_pos(desc, addr);
+		} else if (chan->direction == DMA_DEV_TO_MEM) {
+			addr = axi_chan_ioread64(chan, CH_DAR);
+			completed_length = axi_dma_desc_dst_pos(desc, addr);
+		} else {
+			completed_length = 0;
+		}
+		bytes = desc->length - completed_length;
+	} else if (vdesc) {
+		/* Still in the queue so not started */
+		bytes = vd_to_axi_desc(vdesc)->length;
 	}
 
-	spin_unlock_irqrestore(&chan->vc.lock, flags);
+	if (chan->is_paused && status == DMA_IN_PROGRESS)
+		status = DMA_PAUSED;
+
 	dma_set_residue(txstate, bytes);
+	spin_unlock_irqrestore(&chan->vc.lock, flags);
 
 	return status;
 }
@@ -435,8 +512,6 @@ static void axi_chan_block_xfer_start(st
 		return;
 	}
 
-	axi_dma_enable(chan->chip);
-
 	config.dst_multblk_type = DWAXIDMAC_MBLK_TYPE_LL;
 	config.src_multblk_type = DWAXIDMAC_MBLK_TYPE_LL;
 	config.tt_fc = DWAXIDMAC_TT_FC_MEM_TO_MEM_DMAC;
@@ -569,7 +644,7 @@ static void dw_axi_dma_set_hw_channel(st
 	unsigned long reg_value, val;
 
 	if (!chip->apb_regs) {
-		dev_err(chip->dev, "apb_regs not initialized\n");
+		dev_dbg(chip->dev, "apb_regs not initialized\n");
 		return;
 	}
 
@@ -673,18 +748,25 @@ static int dw_axi_dma_set_hw_desc(struct
 	switch (chan->direction) {
 	case DMA_MEM_TO_DEV:
 		reg_width = __ffs(chan->config.dst_addr_width);
-		device_addr = chan->config.dst_addr;
+		device_addr = phys_to_dma(chan->chip->dev, chan->config.dst_addr);
 		ctllo = reg_width << CH_CTL_L_DST_WIDTH_POS |
 			mem_width << CH_CTL_L_SRC_WIDTH_POS |
+			DWAXIDMAC_BURST_TRANS_LEN_1 << CH_CTL_L_DST_MSIZE_POS |
+			DWAXIDMAC_BURST_TRANS_LEN_4 << CH_CTL_L_SRC_MSIZE_POS |
 			DWAXIDMAC_CH_CTL_L_NOINC << CH_CTL_L_DST_INC_POS |
 			DWAXIDMAC_CH_CTL_L_INC << CH_CTL_L_SRC_INC_POS;
 		block_ts = len >> mem_width;
 		break;
 	case DMA_DEV_TO_MEM:
 		reg_width = __ffs(chan->config.src_addr_width);
-		device_addr = chan->config.src_addr;
+		/* Prevent partial access units getting lost */
+		if (mem_width > reg_width)
+			mem_width = reg_width;
+		device_addr = phys_to_dma(chan->chip->dev, chan->config.src_addr);
 		ctllo = reg_width << CH_CTL_L_SRC_WIDTH_POS |
 			mem_width << CH_CTL_L_DST_WIDTH_POS |
+			DWAXIDMAC_BURST_TRANS_LEN_4 << CH_CTL_L_DST_MSIZE_POS |
+			DWAXIDMAC_BURST_TRANS_LEN_1 << CH_CTL_L_SRC_MSIZE_POS |
 			DWAXIDMAC_CH_CTL_L_INC << CH_CTL_L_DST_INC_POS |
 			DWAXIDMAC_CH_CTL_L_NOINC << CH_CTL_L_SRC_INC_POS;
 		block_ts = len >> reg_width;
@@ -720,9 +802,6 @@ static int dw_axi_dma_set_hw_desc(struct
 	}
 
 	hw_desc->lli->block_ts_lo = cpu_to_le32(block_ts - 1);
-
-	ctllo |= DWAXIDMAC_BURST_TRANS_LEN_4 << CH_CTL_L_DST_MSIZE_POS |
-		 DWAXIDMAC_BURST_TRANS_LEN_4 << CH_CTL_L_SRC_MSIZE_POS;
 	hw_desc->lli->ctl_lo = cpu_to_le32(ctllo);
 
 	set_desc_src_master(hw_desc);
@@ -817,6 +896,8 @@ dw_axi_dma_chan_prep_cyclic(struct dma_c
 		src_addr += segment_len;
 	}
 
+	desc->nr_hw_descs = total_segments;
+
 	llp = desc->hw_desc[0].llp;
 
 	/* Managed transfer list */
@@ -896,6 +977,8 @@ dw_axi_dma_chan_prep_slave_sg(struct dma
 		} while (len >= segment_len);
 	}
 
+	desc->nr_hw_descs = loop;
+
 	/* Set end-of-link to the last link descriptor of list */
 	set_desc_last(&desc->hw_desc[num_sgs - 1]);
 
@@ -1003,6 +1086,8 @@ dma_chan_prep_dma_memcpy(struct dma_chan
 		num++;
 	}
 
+	desc->nr_hw_descs = num;
+
 	/* Set end-of-link to the last link descriptor of list */
 	set_desc_last(&desc->hw_desc[num - 1]);
 	/* Managed transfer list */
@@ -1051,7 +1136,7 @@ static void axi_chan_dump_lli(struct axi
 static void axi_chan_list_dump_lli(struct axi_dma_chan *chan,
 				   struct axi_dma_desc *desc_head)
 {
-	int count = atomic_read(&chan->descs_allocated);
+	int count = desc_head->nr_hw_descs;
 	int i;
 
 	for (i = 0; i < count; i++)
@@ -1094,11 +1179,11 @@ out:
 
 static void axi_chan_block_xfer_complete(struct axi_dma_chan *chan)
 {
-	int count = atomic_read(&chan->descs_allocated);
 	struct axi_dma_hw_desc *hw_desc;
 	struct axi_dma_desc *desc;
 	struct virt_dma_desc *vd;
 	unsigned long flags;
+	int count;
 	u64 llp;
 	int i;
 
@@ -1120,6 +1205,7 @@ static void axi_chan_block_xfer_complete
 	if (chan->cyclic) {
 		desc = vd_to_axi_desc(vd);
 		if (desc) {
+			count = desc->nr_hw_descs;
 			llp = lo_hi_readq(chan->chan_regs + CH_LLP);
 			for (i = 0; i < count; i++) {
 				hw_desc = &desc->hw_desc[i];
@@ -1140,6 +1226,9 @@ static void axi_chan_block_xfer_complete
 		/* Remove the completed descriptor from issued list before completing */
 		list_del(&vd->node);
 		vchan_cookie_complete(vd);
+
+		/* Submit queued descriptors after processing the completed ones */
+		axi_chan_start_first_queued(chan);
 	}
 
 out:
@@ -1397,6 +1486,10 @@ static int parse_device_properties(struc
 
 	chip->dw->hdata->nr_masters = tmp;
 
+	ret = device_property_read_u32(dev, "snps,dma-targets", &tmp);
+	if (!ret && tmp > 16)
+		chip->dw->hdata->use_cfg2 = true;
+
 	ret = device_property_read_u32(dev, "snps,data-width", &tmp);
 	if (ret)
 		return ret;
