efinix-linux/drivers/dma/efx_dma.c

#include <linux/platform_device.h>
#include <linux/module.h>
#include <linux/interrupt.h>
#include <linux/io.h>
#include <linux/dmaengine.h>
#include <linux/dma-mapping.h>
#include <linux/of.h>
#include <linux/of_dma.h>
#include <linux/of_irq.h>
#include <linux/list.h>
#include <linux/spinlock.h>

#include "dmaengine.h"
#include "virt-dma.h"

#define EFX_DMA_CHANNEL_INPUT_ADDRESS				0x00
#define EFX_DMA_CHANNEL_INPUT_STREAM				0x08
#define EFX_DMA_CHANNEL_INPUT_CONFIG				0x0c
#define EFX_DMA_CHANNEL_INPUT_CONFIG_MEMORY			(1 << 12)
#define EFX_DMA_CHANNEL_INPUT_CONFIG_STREAM			0x0
#define EFX_DMA_CHANNEL_INPUT_CONFIG_COMPLETION_ON_PACKET	(1 << 13)
#define EFX_DMA_CHANNEL_INPUT_CONFIG_WAIT_ON_PACKET		(1 << 14)

#define EFX_DMA_CHANNEL_OUTPUT_ADDRESS				0x10
#define EFX_DMA_CHANNEL_OUTPUT_STREAM				0x18
#define EFX_DMA_CHANNEL_OUTPUT_CONFIG				0x1c
#define EFX_DMA_CHANNEL_OUTPUT_CONFIG_MEMORY			(1 << 12)
#define EFX_DMA_CHANNEL_OUTPUT_CONFIG_STREAM			0x0
#define EFX_DMA_CHANNEL_OUTPUT_CONFIG_LAST			(1 << 13)

#define EFX_DMA_CHANNEL_DIRECT_BYTES				0x20
#define EFX_DMA_CHANNEL_STATUS					0x2c
#define EFX_DMA_CHANNEL_STATUS_DIRECT_START			(1 << 0)
#define EFX_DMA_CHANNEL_STATUS_BUSY				(1 << 0)
#define EFX_DMA_CHANNEL_STATUS_SELF_RESTART			(1 << 1)
#define EFX_DMA_CHANNEL_STATUS_STOP				(1 << 2)
#define EFX_DMA_CHANNEL_STATUS_LINKED_LIST_START		(1 << 4)

#define EFX_DMA_CHANNEL_FIFO					0x40
#define EFX_DMA_CHANNEL_PRIORITY				0x44
#define EFX_DMA_CHANNEL_INTERRUPT_ENABLE			0x50
#define EFX_DMA_CHANNEL_INTERRUPT_PENDING			0x54

// Interrupt at the end of each descriptor
#define EFX_DMA_CHANNEL_INTERRUPT_DESCRIPTOR_COMPLETION_MASK	(1 << 0)
// Interrupt at the middle of each descriptor, require the half_completion_interrpt
// option to be enabled for the channel
#define EFX_DMA_CHANNEL_INTERRUPT_DESCRIPTOR_COMPLETION_HALF_MASK	(1 << 1)
// Interrupt when the channel is going off (not busy anymore)
#define EFX_DMA_CHANNEL_INTERRUPT_CHANNEL_COMPLETION_MASK	(1 << 2)
// Interrupt each time that a linked list's descriptor stats field is updated
#define EFX_DMA_CHANNEL_INTERRUPT_LINKED_LIST_UPDATE_MASK	(1 << 3)
// Interrupt each time a S -> M channel has done transferring a packet into the memory
#define EFX_DMA_CHANNEL_INTERRUPT_INPUT_PACKET_MASK		(1 << 4)

#define EFX_DMA_CHANNEL_PROGRESS_BYTES				0x60
#define EFX_DMA_CHANNEL_LINKED_LIST_HEAD			0x70
#define EFX_DMA_CHANNEL_LINKED_LIST_FROM_SG_BUS			0x78

#define EFX_DMA_DESCRIPTOR_CONTROL_BYTES			0x7FFFFFF
#define EFX_DMA_DESCRIPTOR_CONTROL_END_OF_PACKET		(1 << 30)
#define EFX_DMA_DESCRIPTOR_NO_COMPLETION			(1 << 31)

#define EFX_DMA_DESCRIPTOR_STATUS_BYTES				0x7FFFFFF
#define EFX_DMA_DESCRIPTOR_STATUS_END_OF_PACKET			(1 << 30)
#define EFX_DMA_DESCRIPTOR_STATUS_COMPLETED			(1 << 31)

// DMA Hardware descriptor
struct efx_dma_hw_desc {
	u32 status;
	u32 control;
	u64 src_addr;
	u64 dst_addr;
	u64 next;	// physical address of next descriptor
} __aligned(64);

// Per transfer descriptor
struct efx_dma_desc {
	struct virt_dma_desc vdesc;
	struct efx_dma_hw_desc *hw_desc; // Pointer to the struct efx_dma_hw_desc
	dma_addr_t dma_handle; // DMA address of struct efx_dma_hw_desc
	struct list_head node; // List of struct efx_dma_hw_desc
	size_t segments; // Number of DMA descriptor
	bool cyclic; // True for cyclic transfer
	enum dma_transfer_direction direction;
	struct scatterlist *sg;
        struct page **pages;
};

// DMA channel specific data
struct efx_dma_chan {
	const char *name; // Channel name
	void __iomem *reg; // based address of DMA channel
	size_t chan_id; // Channel ID
	struct virt_dma_chan vchan;
	struct efx_dma_priv *priv; // Pointer to the DMA controller
	struct dma_slave_config *cfg; // Channel specific configuration
	u32 priority; // Priority number of DMA channel
	size_t irq;  // IRQ number used by the DMA channel
	struct efx_dma_desc *head_desc; // First DMA descriptor
	struct list_head pending_list; // List of all DMA descriptor
	spinlock_t lock; // Lock for manipulating pending_list
};

// DMA controller private data
struct efx_dma_priv {
	struct device *dev;
	void __iomem *base; // DMA controller based address
	struct dma_device dma_dev;
	struct efx_dma_chan *dchan; // Pointer to the DMA channel
	u32 chan_count; // Number of DMA channel
};

static inline struct efx_dma_chan *to_efx_dma_chan(struct dma_chan *chan)
{
	return container_of(chan, struct efx_dma_chan, vchan.chan);
}

static inline struct efx_dma_priv *to_efx_dma_priv(struct dma_chan *chan)
{
	struct efx_dma_chan *dchan = to_efx_dma_chan(chan);
	return dchan->priv;
}

static void efx_dma_input_memory(struct dma_chan *chan)
{
	struct efx_dma_chan *dchan = to_efx_dma_chan(chan);
	struct efx_dma_hw_desc *hw_desc = dchan->head_desc->hw_desc;
	u32 byte_per_burst = 0;

	if (dchan->cfg)
		byte_per_burst = dchan->cfg->src_maxburst;
	else
		byte_per_burst = chan->device->max_burst;

	iowrite32(hw_desc->src_addr, dchan->reg + EFX_DMA_CHANNEL_INPUT_ADDRESS);
	iowrite32(EFX_DMA_CHANNEL_INPUT_CONFIG_MEMORY | ((byte_per_burst - 1) & 0xFFF),
			dchan->reg + EFX_DMA_CHANNEL_INPUT_CONFIG);
}

static void efx_dma_output_memory(struct dma_chan *chan)
{
	struct efx_dma_chan *dchan = to_efx_dma_chan(chan);
	struct efx_dma_hw_desc *hw_desc = dchan->head_desc->hw_desc;
	u32 len = hw_desc->control & 0x1FFFFFF;

	iowrite32(hw_desc->dst_addr, dchan->reg + EFX_DMA_CHANNEL_OUTPUT_ADDRESS);
	iowrite32(EFX_DMA_CHANNEL_OUTPUT_CONFIG_MEMORY | ((len - 1) & 0xFFF),
			dchan->reg + EFX_DMA_CHANNEL_OUTPUT_CONFIG);
}

static void efx_dma_input_stream(struct dma_chan *chan, u32 wait_on_packet, u32 completion_on_packet)
{
	struct efx_dma_chan *dchan = to_efx_dma_chan(chan);
	completion_on_packet = completion_on_packet ? EFX_DMA_CHANNEL_INPUT_CONFIG_COMPLETION_ON_PACKET : 0;
	wait_on_packet = wait_on_packet ? EFX_DMA_CHANNEL_INPUT_CONFIG_WAIT_ON_PACKET : 0;

	iowrite32(0, dchan->reg + EFX_DMA_CHANNEL_INPUT_STREAM);
	iowrite32(EFX_DMA_CHANNEL_INPUT_CONFIG_STREAM
			| completion_on_packet | wait_on_packet,
			dchan->reg + EFX_DMA_CHANNEL_INPUT_CONFIG);
}

static void efx_dma_output_stream(struct dma_chan *chan)
{
	struct efx_dma_chan *dchan = to_efx_dma_chan(chan);

	iowrite32(0, dchan->reg + EFX_DMA_CHANNEL_OUTPUT_STREAM);
	iowrite32(EFX_DMA_CHANNEL_OUTPUT_CONFIG_LAST |  EFX_DMA_CHANNEL_OUTPUT_CONFIG_STREAM,
			dchan->reg + EFX_DMA_CHANNEL_OUTPUT_CONFIG);
}

static void efx_dma_linked_list_start(struct dma_chan *chan)
{
	struct efx_dma_chan *dchan = to_efx_dma_chan(chan);

	iowrite32(dchan->head_desc->dma_handle, dchan->reg + EFX_DMA_CHANNEL_LINKED_LIST_HEAD);
	iowrite32(0, dchan->reg + EFX_DMA_CHANNEL_LINKED_LIST_FROM_SG_BUS);
	iowrite32(EFX_DMA_CHANNEL_STATUS_LINKED_LIST_START, dchan->reg + EFX_DMA_CHANNEL_STATUS);
}

static void efx_dma_stop_channel(struct dma_chan *chan)
{
	struct efx_dma_chan *dchan = to_efx_dma_chan(chan);

	iowrite32(EFX_DMA_CHANNEL_STATUS_STOP, dchan->reg + EFX_DMA_CHANNEL_STATUS);
}

static bool efx_dma_busy(struct dma_chan *chan)
{
	struct efx_dma_chan *dchan = to_efx_dma_chan(chan);
	u32 busy;

	busy = readl(dchan->reg + EFX_DMA_CHANNEL_STATUS) & EFX_DMA_CHANNEL_STATUS_BUSY;
	return busy ? true : false;
}

static inline void efx_dma_interrupt_pending_clear(struct dma_chan *chan, u32 mask)
{
	struct efx_dma_chan *dchan = to_efx_dma_chan(chan);

	iowrite32(mask, dchan->reg + EFX_DMA_CHANNEL_INTERRUPT_PENDING);
}

static void efx_dma_interrupt_config(struct dma_chan *chan, u32 mask)
{
	struct efx_dma_chan *dchan = to_efx_dma_chan(chan);

	efx_dma_interrupt_pending_clear(chan, 0xFFFFFFFF);
	iowrite32(mask, dchan->reg + EFX_DMA_CHANNEL_INTERRUPT_ENABLE);
}

static void efx_dma_set_channel_priority(struct efx_dma_chan *dchan)
{
	iowrite32(dchan->priority, dchan->reg + EFX_DMA_CHANNEL_PRIORITY);
}

static void efx_dma_configure_registers(struct dma_chan *chan)
{
	struct efx_dma_chan *dchan = to_efx_dma_chan(chan);
	struct efx_dma_desc *desc = dchan->head_desc;

	if (desc->direction == DMA_DEV_TO_MEM) {
		efx_dma_output_memory(chan);
		efx_dma_input_stream(chan, 1, 0);
	} else if (desc->direction == DMA_MEM_TO_DEV) {
		efx_dma_input_memory(chan);
		efx_dma_output_stream(chan);
	} else if (desc->direction == DMA_MEM_TO_MEM) {
		efx_dma_input_memory(chan);
		efx_dma_output_memory(chan);
	}
}

static void efx_dma_start_transfer(struct dma_chan *chan)
{
	struct efx_dma_chan *dchan = to_efx_dma_chan(chan);
	struct virt_dma_desc *vdesc;

	vdesc = vchan_next_desc(&dchan->vchan);
	if (!vdesc)
		return;

	efx_dma_configure_registers(chan);
	efx_dma_linked_list_start(chan);
}

static irqreturn_t efx_dma_interrupt_handler(int irq, void *dev_id)
{
	struct efx_dma_chan *dchan = (struct efx_dma_chan *)dev_id;
	struct efx_dma_desc *desc = dchan->head_desc;
	struct dma_chan *chan = &dchan->vchan.chan;
	unsigned long flags;
	u32 pending;

	spin_lock_irqsave(&dchan->vchan.lock, flags);
	pending = readl(dchan->reg + EFX_DMA_CHANNEL_INTERRUPT_PENDING);

	if (pending & EFX_DMA_CHANNEL_INTERRUPT_DESCRIPTOR_COMPLETION_MASK)
		efx_dma_interrupt_pending_clear(chan, EFX_DMA_CHANNEL_INTERRUPT_DESCRIPTOR_COMPLETION_MASK);

	if (pending & EFX_DMA_CHANNEL_INTERRUPT_CHANNEL_COMPLETION_MASK)
		efx_dma_interrupt_pending_clear(chan, EFX_DMA_CHANNEL_INTERRUPT_CHANNEL_COMPLETION_MASK);

	vchan_cookie_complete(&desc->vdesc);

	// Unmask the interrupt
	efx_dma_interrupt_pending_clear(chan, 0x0);

	pending = readl(dchan->reg + EFX_DMA_CHANNEL_INTERRUPT_PENDING);
	spin_unlock_irqrestore(&dchan->vchan.lock, flags);

	return IRQ_HANDLED;
}

static int efx_dma_alloc_chan_resources(struct dma_chan *chan)
{
	return 0;
}

static void efx_dma_free_chan_resources(struct dma_chan *chan)
{

}

static void efx_dma_dump_pending_list(struct dma_chan *chan)
{
	struct efx_dma_priv *priv = to_efx_dma_priv(chan);
	struct efx_dma_chan *dchan = to_efx_dma_chan(chan);
	struct efx_dma_desc *desc;
	struct device *dev = priv->dev;
	int i = 0, count = 0;
	int limit = 5;

	// Count number of descriptor in the pending_list
	spin_lock(&dchan->lock);
	list_for_each_entry(desc, &dchan->pending_list, node)
		count++;
	spin_unlock(&dchan->lock);

	if (count == 0) {
		dev_dbg(dev, "Pending list is empty\n");
		return;
	}

	dev_dbg(dev, "Pending list has %d descriptors\n", count);

	// Print first 5 and last 5 descriptors
	spin_lock(&dchan->lock);
	list_for_each_entry(desc, &dchan->pending_list, node) {
		if (i < limit || i >= count - limit) {
			pr_debug("%s: Descriptor %d:\n", __func__, i);
			pr_debug("  desc=%p, hw_desc=%p, dma_handle=0x%llx\n",
					desc, desc->hw_desc, (unsigned long long)desc->dma_handle);

			if (desc->hw_desc) {
				pr_debug("  status=0x%x, control=0x%x\n",
					desc->hw_desc->status, desc->hw_desc->control);
				pr_debug("  src_addr=0x%llx, dst_addr=0x%llx, next=0x%llx\n",
						(unsigned long long)desc->hw_desc->src_addr,
						(unsigned long long)desc->hw_desc->dst_addr,
						(unsigned long long)desc->hw_desc->next);
			}

			pr_debug("  cyclic=%d, direction=%d, segments=%zu\n",
					desc->cyclic, desc->direction, desc->segments);
		}
		i++;
	}
	spin_unlock(&dchan->lock);
}

static void efx_dma_free_hw_desc_chain(struct dma_chan *chan)
{
	struct efx_dma_priv *priv = to_efx_dma_priv(chan);
	struct efx_dma_chan *dchan = to_efx_dma_chan(chan);
	struct efx_dma_desc *desc, *tmp;
	struct device *dev = priv->dev;
	int count = 0;
	int limit = 5, i = 0;

	// Count number of descriptor in the pending_list
	spin_lock(&dchan->lock);
	list_for_each_entry(desc, &dchan->pending_list, node)
		count++;
	spin_unlock(&dchan->lock);

	if (count == 0) {
		dev_info(dev, "Pending list is empty\n");
		return;
	}

	dev_info(dev, "Freeing %d descriptors\n", count);

	spin_lock(&dchan->lock);
	list_for_each_entry_safe(desc, tmp, &dchan->pending_list, node) {
		list_del(&desc->node);

		if (desc->hw_desc) {
			dma_free_coherent(dev, sizeof(struct efx_dma_hw_desc),
					desc->hw_desc, desc->dma_handle);
			if (i < limit || i >= count - limit) {
				pr_debug("  Freeing descriptor: %d\n", i);
				pr_debug("    free hw_desc=%p, dma_handle=0x%llx\n",desc->hw_desc,
						(unsigned long long)desc->dma_handle);
			}
		}

		kfree(desc);
		if (i < limit || i >= count - limit)
			pr_debug("    desc=%p\n",desc);
		i++;
	}

	dchan->head_desc = NULL;
	spin_unlock(&dchan->lock);
}

static void efx_dma_desc_free(struct virt_dma_desc *vd)
{
	struct dma_chan *chan = vd->tx.chan;

	efx_dma_free_hw_desc_chain(chan);
}

static void efx_dma_hw_desc_address_control(struct efx_dma_chan *dchan,
			struct efx_dma_desc *desc, dma_addr_t src_addr, dma_addr_t dst_addr,
			size_t period_len, struct scatterlist *sg, size_t i)
{
	struct efx_dma_hw_desc *hw_desc = desc->hw_desc;
	dma_addr_t segment_addr;
	u32 buf_len;
	u32 control = desc->cyclic ? EFX_DMA_DESCRIPTOR_NO_COMPLETION
				   : EFX_DMA_DESCRIPTOR_CONTROL_END_OF_PACKET;

	hw_desc->status = 0;
	if (sg) {
		buf_len = sg_dma_len(sg);
		segment_addr = sg_dma_address(sg);
	} else {
		buf_len = period_len - 1;
		segment_addr = src_addr + i * period_len;
	}

	hw_desc->control = (u32)(control | buf_len);

	// Configure last hw_desc
	if (i == (desc->segments - 1)) {
		if (desc->cyclic) {
			hw_desc->next = dchan->head_desc->dma_handle;
		} else {
			hw_desc->status = EFX_DMA_DESCRIPTOR_STATUS_COMPLETED;
		}
	}

	if (desc->direction == DMA_DEV_TO_MEM) {
		hw_desc->src_addr = 0;
		hw_desc->dst_addr = segment_addr;
	} else if (desc->direction == DMA_MEM_TO_DEV) {
		hw_desc->src_addr = segment_addr;
		hw_desc->dst_addr = 0;
	} else if (desc->direction == DMA_MEM_TO_MEM) {
		hw_desc->src_addr = src_addr + i * period_len;
		hw_desc->dst_addr = dst_addr + i * period_len;
	}
}

static int efx_dma_hw_desc_init(struct dma_chan *chan, dma_addr_t src_addr, dma_addr_t dst_addr,
				   struct scatterlist *sg, size_t len, size_t period_len,
				   enum dma_transfer_direction direction, bool cyclic)
{
	struct efx_dma_priv *priv = to_efx_dma_priv(chan);
        struct efx_dma_chan *dchan = to_efx_dma_chan(chan);
        struct efx_dma_desc *desc = NULL, *prev_desc = NULL;
        struct efx_dma_hw_desc *hw_desc = NULL;
        struct device *dev = priv->dev;
	struct scatterlist *sgl;
        size_t i;
        int ret = 0;

	for (i = 0; i < len; i++) {
                // Allocate memory for desc
                desc = kzalloc(sizeof(*desc), GFP_KERNEL);
                if (!desc) {
                        ret = -ENOMEM;
                        dev_err(dev, "Failed to allocate memory for descriptor\n");
                        goto err_free_desc_init;
                }

                desc->cyclic = cyclic;
                desc->segments = len;
                desc->direction = direction;
                INIT_LIST_HEAD(&desc->node);

                // Allocate memory for hw_desc
                hw_desc = dma_alloc_coherent(dev, sizeof(*hw_desc),
                                             &desc->dma_handle, GFP_KERNEL);
                if (!hw_desc) {
                    dev_err(dev, "Failed to allocate memory for hw descriptor\n");
                        kfree(desc);
                        ret = -ENOMEM;
                        goto err_free_desc_init;
                }
                desc->hw_desc = hw_desc;

                // Save head descriptor
                if (i == 0)
                        dchan->head_desc = desc;

		// Set the scatterlist pointer for sg mode. For cylic, sgl is NULL
		sgl = sg ? &sg[i] : NULL;
		// Configure the hw descriptor addresses and control
		efx_dma_hw_desc_address_control(dchan, desc, src_addr, dst_addr, period_len, sgl, i);

                // Chain descriptor
                if (prev_desc)
                        prev_desc->hw_desc->next = desc->dma_handle;

                prev_desc = desc;

                // Add to pending list
                spin_lock(&dchan->lock);
                list_add_tail(&desc->node, &dchan->pending_list);
                spin_unlock(&dchan->lock);
        }

	// Debugging
	efx_dma_dump_pending_list(chan);
        return 0;

err_free_desc_init:
        efx_dma_free_hw_desc_chain(chan);
        return ret;

}

static struct dma_async_tx_descriptor *efx_dma_prep_slave_sg(
			struct dma_chan *chan,
			struct scatterlist *sg, unsigned int sg_len,
			enum dma_transfer_direction direction,
			unsigned long flags, void *context)
{
	struct efx_dma_chan *dchan = to_efx_dma_chan(chan);
	struct device *dev = dchan->priv->dev;
	struct efx_dma_desc *desc = NULL;
	int ret;
	bool cyclic = false;

	// Validate input
	if (unlikely(!chan || !sg || sg_len <= 0))
		return NULL;

	// sg_len +1 to inlcude the last status of the descriptor
	sg_len += 1;

	if (flags & DMA_CTRL_REUSE) {
		cyclic = true;
		sg_len -= 1; // Cyclic transfer does not require additional descriptor
		flags &= ~DMA_CTRL_REUSE;
	}

	// Initialize hardware descriptor. Set cyclic to false
	ret = efx_dma_hw_desc_init(chan, 0, 0, sg, sg_len, 0, direction, cyclic);
	if (ret) {
		dev_err(dev, "Failed to initialize hardware descriptor\n");
		return NULL;
	}

	// Get head descriptor
	desc = dchan->head_desc;
	if (!desc) {
		dev_err(dev, "No head descriptor found\n");
		return NULL;
	}

	// Configure interrupt
	if (flags & DMA_PREP_INTERRUPT)
		efx_dma_interrupt_config(chan, EFX_DMA_CHANNEL_INTERRUPT_CHANNEL_COMPLETION_MASK);

	// Prepare virtual DMA descriptor
	dchan->vchan.cyclic = &desc->vdesc;
	return vchan_tx_prep(&dchan->vchan, &desc->vdesc, flags);
}

static struct dma_async_tx_descriptor *efx_dma_prep_cyclic(
		struct dma_chan *chan, dma_addr_t buf_addr, size_t buf_len,
		size_t period_len, enum dma_transfer_direction direction,
		unsigned long flags)
{
	struct efx_dma_chan *dchan = to_efx_dma_chan(chan);
	struct device *dev = dchan->priv->dev;
	struct efx_dma_desc *desc = NULL;
	size_t segments = buf_len / period_len;
	int ret;

	if (unlikely(!chan || !buf_addr || buf_len == 0 || period_len > buf_len))
		return NULL;

	ret = efx_dma_hw_desc_init(chan, buf_addr, 0, NULL, segments, period_len, direction, true);
	if (ret)
		return NULL;

	// Get head descriptor
        desc = dchan->head_desc;
        if (!desc) {
                dev_err(dev, "No head descriptor found\n");
                return NULL;
        }

	dchan->vchan.cyclic = &desc->vdesc;
	return vchan_tx_prep(&dchan->vchan, &desc->vdesc, flags);
}

static struct dma_async_tx_descriptor *efx_dma_prep_dma_memcpy(struct dma_chan *chan,
				dma_addr_t dst_addr, dma_addr_t src_addr, size_t len,
				unsigned long flags)
{
	struct efx_dma_chan *dchan = to_efx_dma_chan(chan);
	struct efx_dma_desc *desc = NULL;
	struct device *dev = dchan->priv->dev;
	size_t segments;
	int ret;

	// Validate input
	if (unlikely(!chan || !dst_addr || !src_addr || !len))
		return NULL;

	if (len < PAGE_SIZE)
		segments = 1;
	else
		segments = (len / PAGE_SIZE);

	// Increase segments by 1 to include the last status of the descriptor
	segments += 1;

	ret = efx_dma_hw_desc_init(chan, src_addr, dst_addr, NULL, len, segments, DMA_MEM_TO_MEM, false);
	if (ret) {
		return NULL;
	}

	// Get head descriptor
	desc = dchan->head_desc;
	if (!desc) {
		dev_err(dev, "No head descriptor found\n");
		return NULL;
	}

	dchan->vchan.cyclic = &desc->vdesc;

	return vchan_tx_prep(&dchan->vchan, &desc->vdesc, flags);
}


static void efx_dma_issue_pending(struct dma_chan *chan)
{
	struct efx_dma_chan *dchan = to_efx_dma_chan(chan);
	unsigned long flags;

	spin_lock_irqsave(&dchan->vchan.lock, flags);
	if (vchan_issue_pending(&dchan->vchan))
		efx_dma_start_transfer(chan);

	spin_unlock_irqrestore(&dchan->vchan.lock, flags);
}

static int efx_dma_device_config(struct dma_chan *chan,
				 struct dma_slave_config *config)
{
	struct efx_dma_chan *dchan = to_efx_dma_chan(chan);

	dchan->cfg = config;
	return 0;
}

static enum dma_status efx_dma_tx_status(struct dma_chan *chan, dma_cookie_t cookie,
					 struct dma_tx_state *txstate)
{
	if (efx_dma_busy(chan))
		return DMA_IN_PROGRESS;

	if (cookie < DMA_MIN_COOKIE)
		return DMA_ERROR;

	if (chan->completed_cookie == cookie)
		return DMA_COMPLETE;

	return DMA_COMPLETE;
}

static int efx_dma_terminate_all(struct dma_chan *chan)
{
	struct efx_dma_chan *dchan = to_efx_dma_chan(chan);
	struct virt_dma_chan *vchan = &dchan->vchan;
	struct device *dev = dchan->priv->dev;

	if (dchan) {
		// Stop all transfer of the DMA channels
		efx_dma_stop_channel(&dchan->vchan.chan);

		// Disabled interrupt
		efx_dma_interrupt_config(chan, 0);

		// Terminate all virtual DMA descriptors
		//vchan_free_chan_resources(vchan);

		// Free all pending descriptors
		if (dchan->head_desc)
			efx_dma_free_hw_desc_chain(&dchan->vchan.chan);

		// Clear cyclic pointer
		vchan->cyclic = NULL;
		dchan->head_desc = NULL;

		dev_info(dev, "DMA channel %s terminated and cleaned up\n", dchan->name);
	}

	return 0;
}

static void efx_dma_release(struct dma_device *dev)
{
}

static int efx_dma_chan_probe(struct platform_device *pdev)
{
	struct efx_dma_priv *priv = platform_get_drvdata(pdev);
	struct efx_dma_chan *dchan;
	struct device_node *node = pdev->dev.of_node;
	struct device_node *child = pdev->dev.of_node;
	size_t i = 0;
	int ret;

	priv->dchan = devm_kzalloc(&pdev->dev, sizeof(struct efx_dma_chan) * priv->chan_count, GFP_KERNEL);
	if (!priv->dchan) {
		dev_err(&pdev->dev, "Failed to allocated memory for DMA channels\n");
		return -ENOMEM;
	}

	for_each_child_of_node(node, child) {
		dchan = &priv->dchan[i];
		if (of_property_read_string(child, "dma-names", &dchan->name)) {
			dev_warn(&pdev->dev, "Failed to get dma-names for channel %zu\n", i);
			return -EINVAL;
		}

		dev_info(&pdev->dev, "Initialize DMA channel %zu for %s\n", i, dchan->name);
		dchan->priv = priv;
		vchan_init(&dchan->vchan, &priv->dma_dev);
		dchan->chan_id = i;
		dchan->reg = priv->base + i * 0x80;
		dchan->cfg = NULL;
		dchan->vchan.desc_free = &efx_dma_desc_free;

		dchan->irq = irq_of_parse_and_map(node, i);
		if (dchan->irq <= 0) {
			dev_warn(&pdev->dev, "Failed to get interrupt number\n");
		}

		ret = devm_request_irq(&pdev->dev, dchan->irq, efx_dma_interrupt_handler,
					IRQF_SHARED, dchan->name, dchan);

		if (ret) {
			dev_warn(&pdev->dev, "Warning: Failed to register interrupt handler\n");
		}

		efx_dma_stop_channel(&dchan->vchan.chan);
		efx_dma_interrupt_pending_clear(&dchan->vchan.chan, 0xFFFFFFFF);

		ret = of_property_read_u32(child, "chan-priority", &dchan->priority);
		if (ret) {
			dev_warn(&pdev->dev, "'chan-priority' is not found in the DMA device tree node. Use the default priority\n");
			dchan->priority = 0;
		}
		dev_info(&pdev->dev, "  channel address %px, priority %u\n", dchan->reg, dchan->priority);
		efx_dma_set_channel_priority(dchan);

		// initialize dchan->lock
		spin_lock_init(&dchan->lock);

		INIT_LIST_HEAD(&dchan->pending_list);
		i++;
	}

	return 0;
}

static struct dma_chan *of_dma_efx_dma_xlate(struct of_phandle_args *dma_spec,
					     struct of_dma *ofdma)
{
	struct efx_dma_priv *priv = ofdma->of_dma_data;
	struct efx_dma_chan *dchan;
	struct dma_chan *chan = NULL;
	size_t chan_id;

	chan_id = dma_spec->args[0];
	if (chan_id < 0 || chan_id > priv->chan_count)
		return NULL;

	dchan = &priv->dchan[chan_id];
	chan = dma_get_slave_channel(&dchan->vchan.chan);
	if (!chan) {
		dev_err(priv->dev, "Failed to get DMA slave channel %zu\n", chan_id);
		return NULL;
	}

	dev_info(priv->dev, "Found DMA slave '%s' at channel %zu\n", dchan->name, chan_id);
	return chan;
}

static int efx_dma_probe(struct platform_device *pdev)
{
	struct efx_dma_priv *priv;
	struct resource *res;
	struct device_node *node = pdev->dev.of_node;
	struct device_node *child = pdev->dev.of_node;
	int ret;

	priv = devm_kzalloc(&pdev->dev, sizeof(*priv), GFP_KERNEL);
	if (!priv)
		return -ENOMEM;

	priv->dev = &pdev->dev;
	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
	priv->base = devm_ioremap_resource(&pdev->dev, res);
	if (IS_ERR(priv->base))
		return PTR_ERR(priv->base);

	dma_cap_zero(priv->dma_dev.cap_mask);
	dma_cap_set(DMA_SLAVE, priv->dma_dev.cap_mask);
	dma_cap_set(DMA_CYCLIC, priv->dma_dev.cap_mask);
	dma_cap_set(DMA_MEMCPY, priv->dma_dev.cap_mask);

	ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
	if (ret) {
		dev_warn(&pdev->dev, "Failed to set 64-bit DMA mask, falling back to 32-bit\n");
		ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
		if (ret) {
			dev_err(&pdev->dev, "Failed to set 32-bit DMA mask\n");
			return ret;
		}
	}

	priv->dma_dev.src_addr_widths = DMA_SLAVE_BUSWIDTH_4_BYTES;
	priv->dma_dev.dst_addr_widths = DMA_SLAVE_BUSWIDTH_4_BYTES;
	priv->dma_dev.max_burst = 16;
	priv->dma_dev.directions = BIT(DMA_MEM_TO_DEV) | BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_MEM);

	INIT_LIST_HEAD(&priv->dma_dev.channels);

	// Initialize and register the DMA engine
	priv->dma_dev.dev = &pdev->dev;
	priv->dma_dev.device_alloc_chan_resources = efx_dma_alloc_chan_resources;
	priv->dma_dev.device_free_chan_resources = efx_dma_free_chan_resources;
	priv->dma_dev.device_prep_slave_sg = efx_dma_prep_slave_sg;
	priv->dma_dev.device_prep_dma_cyclic = efx_dma_prep_cyclic;
	priv->dma_dev.device_issue_pending = efx_dma_issue_pending;
	priv->dma_dev.device_config = efx_dma_device_config;
	priv->dma_dev.device_tx_status = efx_dma_tx_status;
	priv->dma_dev.device_prep_dma_memcpy = efx_dma_prep_dma_memcpy;
	priv->dma_dev.device_release = efx_dma_release;
	priv->dma_dev.device_terminate_all = efx_dma_terminate_all;

	platform_set_drvdata(pdev, priv);

	priv->chan_count = 0;
	ret = of_property_read_u32(node, "dma-channels", &priv->chan_count);
	if (ret) {
		dev_warn(&pdev->dev, "`dma-channels` not found in the DMA device tree node. Try to auto detect the number of channels\n");
		for_each_child_of_node(node, child) {
			priv->chan_count++;
		}
	}

	dev_info(&pdev->dev, "Found %u DMA channels\n", priv->chan_count);

	// Initialize DMA channels
	ret = efx_dma_chan_probe(pdev);
	if (ret)
		return ret;

	dev_info(&pdev->dev, "Register DMA controller\n");
	ret = dma_async_device_register(&priv->dma_dev);
	if (ret) {
		dev_err(&pdev->dev, "Failed to register DMA controller\n");
		return ret;
	}

	// Register DMA controller with device tree framework
	ret = of_dma_controller_register(node, of_dma_efx_dma_xlate, priv);
	if (ret < 0) {
		dev_err(&pdev->dev, "Unable to register DMA controller to DT\n");
		dma_async_device_unregister(&priv->dma_dev);
		return ret;
	}

	dev_info(&pdev->dev, "DMA controller registered\n");

	return 0;
}

static int efx_dma_remove(struct platform_device *pdev)
{
	struct efx_dma_priv *priv = platform_get_drvdata(pdev);

	dma_async_device_unregister(&priv->dma_dev);

	return 0;
}

static const struct of_device_id efx_dma_of_match[] = {
	{ .compatible = "efx,dma-controller" },
	{ },
};
MODULE_DEVICE_TABLE(of, efx_dma_of_match);

static struct platform_driver efx_dma_driver = {
	.probe = efx_dma_probe,
	.remove = efx_dma_remove,
	.driver = {
		.name = "efx-dma",
		.of_match_table = efx_dma_of_match,
	},
};

module_platform_driver(efx_dma_driver);

MODULE_AUTHOR("Alim Hussin <mnalim@efinixinc.com>");
MODULE_DESCRIPTION("Efinix DMA driver");
MODULE_LICENSE("GPL v2");