diff --git a/drivers/dma/Kconfig b/drivers/dma/Kconfig index 094c75357..55120725b 100644 --- a/drivers/dma/Kconfig +++ b/drivers/dma/Kconfig @@ -187,6 +187,13 @@ config DW_AXI_DMAC NOTE: This driver wasn't tested on 64 bit platform because of lack 64 bit platform with Synopsys DW AXI DMAC. +config EFINIX_DMA + bool "Efinix DMA support" + select DMA_ENGINE + select DMA_VIRTUAL_CHANNELS + help + Enable support for Efinix DMA controller + config EP93XX_DMA bool "Cirrus Logic EP93xx DMA support" depends on ARCH_EP93XX || COMPILE_TEST diff --git a/drivers/dma/Makefile b/drivers/dma/Makefile index 4477fec3f..1d9c1dbe5 100644 --- a/drivers/dma/Makefile +++ b/drivers/dma/Makefile @@ -29,6 +29,7 @@ obj-$(CONFIG_DMA_SUN6I) += sun6i-dma.o obj-$(CONFIG_DW_AXI_DMAC) += dw-axi-dmac/ obj-$(CONFIG_DW_DMAC_CORE) += dw/ obj-$(CONFIG_DW_EDMA) += dw-edma/ +obj-$(CONFIG_EFINIX_DMA) += efx_dma.o obj-$(CONFIG_EP93XX_DMA) += ep93xx_dma.o obj-$(CONFIG_FSL_DMA) += fsldma.o obj-$(CONFIG_FSL_EDMA) += fsl-edma.o fsl-edma-common.o diff --git a/drivers/dma/efx_dma.c b/drivers/dma/efx_dma.c new file mode 100644 index 000000000..06d56714f --- /dev/null +++ b/drivers/dma/efx_dma.c @@ -0,0 +1,889 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "dmaengine.h" +#include "virt-dma.h" + +#define EFX_DMA_CHANNEL_INPUT_ADDRESS 0x00 +#define EFX_DMA_CHANNEL_INPUT_STREAM 0x08 +#define EFX_DMA_CHANNEL_INPUT_CONFIG 0x0c +#define EFX_DMA_CHANNEL_INPUT_CONFIG_MEMORY (1 << 12) +#define EFX_DMA_CHANNEL_INPUT_CONFIG_STREAM 0x0 +#define EFX_DMA_CHANNEL_INPUT_CONFIG_COMPLETION_ON_PACKET (1 << 13) +#define EFX_DMA_CHANNEL_INPUT_CONFIG_WAIT_ON_PACKET (1 << 14) + +#define EFX_DMA_CHANNEL_OUTPUT_ADDRESS 0x10 +#define EFX_DMA_CHANNEL_OUTPUT_STREAM 0x18 +#define EFX_DMA_CHANNEL_OUTPUT_CONFIG 0x1c +#define EFX_DMA_CHANNEL_OUTPUT_CONFIG_MEMORY (1 << 12) +#define EFX_DMA_CHANNEL_OUTPUT_CONFIG_STREAM 0x0 +#define EFX_DMA_CHANNEL_OUTPUT_CONFIG_LAST (1 << 13) + +#define EFX_DMA_CHANNEL_DIRECT_BYTES 0x20 +#define EFX_DMA_CHANNEL_STATUS 0x2c +#define EFX_DMA_CHANNEL_STATUS_DIRECT_START (1 << 0) +#define EFX_DMA_CHANNEL_STATUS_BUSY (1 << 0) +#define EFX_DMA_CHANNEL_STATUS_SELF_RESTART (1 << 1) +#define EFX_DMA_CHANNEL_STATUS_STOP (1 << 2) +#define EFX_DMA_CHANNEL_STATUS_LINKED_LIST_START (1 << 4) + +#define EFX_DMA_CHANNEL_FIFO 0x40 +#define EFX_DMA_CHANNEL_PRIORITY 0x44 +#define EFX_DMA_CHANNEL_INTERRUPT_ENABLE 0x50 +#define EFX_DMA_CHANNEL_INTERRUPT_PENDING 0x54 + +// Interrupt at the end of each descriptor +#define EFX_DMA_CHANNEL_INTERRUPT_DESCRIPTOR_COMPLETION_MASK (1 << 0) +// Interrupt at the middle of each descriptor, require the half_completion_interrpt +// option to be enabled for the channel +#define EFX_DMA_CHANNEL_INTERRUPT_DESCRIPTOR_COMPLETION_HALF_MASK (1 << 1) +// Interrupt when the channel is going off (not busy anymore) +#define EFX_DMA_CHANNEL_INTERRUPT_CHANNEL_COMPLETION_MASK (1 << 2) +// Interrupt each time that a linked list's descriptor stats field is updated +#define EFX_DMA_CHANNEL_INTERRUPT_LINKED_LIST_UPDATE_MASK (1 << 3) +// Interrupt each time a S -> M channel has done transferring a packet into the memory +#define EFX_DMA_CHANNEL_INTERRUPT_INPUT_PACKET_MASK (1 << 4) + +#define EFX_DMA_CHANNEL_PROGRESS_BYTES 0x60 +#define EFX_DMA_CHANNEL_LINKED_LIST_HEAD 0x70 +#define EFX_DMA_CHANNEL_LINKED_LIST_FROM_SG_BUS 0x78 + +#define EFX_DMA_DESCRIPTOR_CONTROL_BYTES 0x7FFFFFF +#define EFX_DMA_DESCRIPTOR_CONTROL_END_OF_PACKET (1 << 30) +#define EFX_DMA_DESCRIPTOR_NO_COMPLETION (1 << 31) + +#define EFX_DMA_DESCRIPTOR_STATUS_BYTES 0x7FFFFFF +#define EFX_DMA_DESCRIPTOR_STATUS_END_OF_PACKET (1 << 30) +#define EFX_DMA_DESCRIPTOR_STATUS_COMPLETED (1 << 31) + +// DMA Hardware descriptor +struct efx_dma_hw_desc { + u32 status; + u32 control; + u64 src_addr; + u64 dst_addr; + u64 next; // physical address of next descriptor +} __aligned(64); + +// Per transfer descriptor +struct efx_dma_desc { + struct virt_dma_desc vdesc; + struct efx_dma_hw_desc *hw_desc; // Pointer to the struct efx_dma_hw_desc + dma_addr_t dma_handle; // DMA address of struct efx_dma_hw_desc + struct list_head node; // List of struct efx_dma_hw_desc + size_t segments; // Number of DMA descriptor + bool cyclic; // True for cyclic transfer + enum dma_transfer_direction direction; + struct scatterlist *sg; + struct page **pages; +}; + +// DMA channel specific data +struct efx_dma_chan { + const char *name; // Channel name + void __iomem *reg; // based address of DMA channel + size_t chan_id; // Channel ID + struct virt_dma_chan vchan; + struct efx_dma_priv *priv; // Pointer to the DMA controller + struct dma_slave_config *cfg; // Channel specific configuration + u32 priority; // Priority number of DMA channel + size_t irq; // IRQ number used by the DMA channel + struct efx_dma_desc *head_desc; // First DMA descriptor + struct list_head pending_list; // List of all DMA descriptor + spinlock_t lock; // Lock for manipulating pending_list +}; + +// DMA controller private data +struct efx_dma_priv { + struct device *dev; + void __iomem *base; // DMA controller based address + struct dma_device dma_dev; + struct efx_dma_chan *dchan; // Pointer to the DMA channel + u32 chan_count; // Number of DMA channel +}; + +static inline struct efx_dma_chan *to_efx_dma_chan(struct dma_chan *chan) +{ + return container_of(chan, struct efx_dma_chan, vchan.chan); +} + +static inline struct efx_dma_priv *to_efx_dma_priv(struct dma_chan *chan) +{ + struct efx_dma_chan *dchan = to_efx_dma_chan(chan); + return dchan->priv; +} + +static void efx_dma_input_memory(struct dma_chan *chan) +{ + struct efx_dma_chan *dchan = to_efx_dma_chan(chan); + struct efx_dma_hw_desc *hw_desc = dchan->head_desc->hw_desc; + u32 byte_per_burst = 0; + + if (dchan->cfg) + byte_per_burst = dchan->cfg->src_maxburst; + else + byte_per_burst = chan->device->max_burst; + + iowrite32(hw_desc->src_addr, dchan->reg + EFX_DMA_CHANNEL_INPUT_ADDRESS); + iowrite32(EFX_DMA_CHANNEL_INPUT_CONFIG_MEMORY | ((byte_per_burst - 1) & 0xFFF), + dchan->reg + EFX_DMA_CHANNEL_INPUT_CONFIG); +} + +static void efx_dma_output_memory(struct dma_chan *chan) +{ + struct efx_dma_chan *dchan = to_efx_dma_chan(chan); + struct efx_dma_hw_desc *hw_desc = dchan->head_desc->hw_desc; + u32 len = hw_desc->control & 0x1FFFFFF; + + iowrite32(hw_desc->dst_addr, dchan->reg + EFX_DMA_CHANNEL_OUTPUT_ADDRESS); + iowrite32(EFX_DMA_CHANNEL_OUTPUT_CONFIG_MEMORY | ((len - 1) & 0xFFF), + dchan->reg + EFX_DMA_CHANNEL_OUTPUT_CONFIG); +} + +static void efx_dma_input_stream(struct dma_chan *chan, u32 wait_on_packet, u32 completion_on_packet) +{ + struct efx_dma_chan *dchan = to_efx_dma_chan(chan); + completion_on_packet = completion_on_packet ? EFX_DMA_CHANNEL_INPUT_CONFIG_COMPLETION_ON_PACKET : 0; + wait_on_packet = wait_on_packet ? EFX_DMA_CHANNEL_INPUT_CONFIG_WAIT_ON_PACKET : 0; + + iowrite32(0, dchan->reg + EFX_DMA_CHANNEL_INPUT_STREAM); + iowrite32(EFX_DMA_CHANNEL_INPUT_CONFIG_STREAM + | completion_on_packet | wait_on_packet, + dchan->reg + EFX_DMA_CHANNEL_INPUT_CONFIG); +} + +static void efx_dma_output_stream(struct dma_chan *chan) +{ + struct efx_dma_chan *dchan = to_efx_dma_chan(chan); + + iowrite32(0, dchan->reg + EFX_DMA_CHANNEL_OUTPUT_STREAM); + iowrite32(EFX_DMA_CHANNEL_OUTPUT_CONFIG_LAST | EFX_DMA_CHANNEL_OUTPUT_CONFIG_STREAM, + dchan->reg + EFX_DMA_CHANNEL_OUTPUT_CONFIG); +} + +static void efx_dma_linked_list_start(struct dma_chan *chan) +{ + struct efx_dma_chan *dchan = to_efx_dma_chan(chan); + + iowrite32(dchan->head_desc->dma_handle, dchan->reg + EFX_DMA_CHANNEL_LINKED_LIST_HEAD); + iowrite32(0, dchan->reg + EFX_DMA_CHANNEL_LINKED_LIST_FROM_SG_BUS); + iowrite32(EFX_DMA_CHANNEL_STATUS_LINKED_LIST_START, dchan->reg + EFX_DMA_CHANNEL_STATUS); +} + +static void efx_dma_stop_channel(struct dma_chan *chan) +{ + struct efx_dma_chan *dchan = to_efx_dma_chan(chan); + + iowrite32(EFX_DMA_CHANNEL_STATUS_STOP, dchan->reg + EFX_DMA_CHANNEL_STATUS); +} + +static bool efx_dma_busy(struct dma_chan *chan) +{ + struct efx_dma_chan *dchan = to_efx_dma_chan(chan); + u32 busy; + + busy = readl(dchan->reg + EFX_DMA_CHANNEL_STATUS) & EFX_DMA_CHANNEL_STATUS_BUSY; + return busy ? true : false; +} + +static inline void efx_dma_interrupt_pending_clear(struct dma_chan *chan, u32 mask) +{ + struct efx_dma_chan *dchan = to_efx_dma_chan(chan); + + iowrite32(mask, dchan->reg + EFX_DMA_CHANNEL_INTERRUPT_PENDING); +} + +static void efx_dma_interrupt_config(struct dma_chan *chan, u32 mask) +{ + struct efx_dma_chan *dchan = to_efx_dma_chan(chan); + + efx_dma_interrupt_pending_clear(chan, 0xFFFFFFFF); + iowrite32(mask, dchan->reg + EFX_DMA_CHANNEL_INTERRUPT_ENABLE); +} + +static void efx_dma_set_channel_priority(struct efx_dma_chan *dchan) +{ + iowrite32(dchan->priority, dchan->reg + EFX_DMA_CHANNEL_PRIORITY); +} + +static void efx_dma_configure_registers(struct dma_chan *chan) +{ + struct efx_dma_chan *dchan = to_efx_dma_chan(chan); + struct efx_dma_desc *desc = dchan->head_desc; + + if (desc->direction == DMA_DEV_TO_MEM) { + efx_dma_output_memory(chan); + efx_dma_input_stream(chan, 1, 0); + } else if (desc->direction == DMA_MEM_TO_DEV) { + efx_dma_input_memory(chan); + efx_dma_output_stream(chan); + } else if (desc->direction == DMA_MEM_TO_MEM) { + efx_dma_input_memory(chan); + efx_dma_output_memory(chan); + } +} + +static void efx_dma_start_transfer(struct dma_chan *chan) +{ + struct efx_dma_chan *dchan = to_efx_dma_chan(chan); + struct virt_dma_desc *vdesc; + + vdesc = vchan_next_desc(&dchan->vchan); + if (!vdesc) + return; + + efx_dma_configure_registers(chan); + efx_dma_linked_list_start(chan); +} + +static irqreturn_t efx_dma_interrupt_handler(int irq, void *dev_id) +{ + struct efx_dma_chan *dchan = (struct efx_dma_chan *)dev_id; + struct efx_dma_desc *desc = dchan->head_desc; + struct dma_chan *chan = &dchan->vchan.chan; + unsigned long flags; + u32 pending; + + spin_lock_irqsave(&dchan->vchan.lock, flags); + pending = readl(dchan->reg + EFX_DMA_CHANNEL_INTERRUPT_PENDING); + + if (pending & EFX_DMA_CHANNEL_INTERRUPT_DESCRIPTOR_COMPLETION_MASK) + efx_dma_interrupt_pending_clear(chan, EFX_DMA_CHANNEL_INTERRUPT_DESCRIPTOR_COMPLETION_MASK); + + if (pending & EFX_DMA_CHANNEL_INTERRUPT_CHANNEL_COMPLETION_MASK) + efx_dma_interrupt_pending_clear(chan, EFX_DMA_CHANNEL_INTERRUPT_CHANNEL_COMPLETION_MASK); + + vchan_cookie_complete(&desc->vdesc); + + // Unmask the interrupt + efx_dma_interrupt_pending_clear(chan, 0x0); + + pending = readl(dchan->reg + EFX_DMA_CHANNEL_INTERRUPT_PENDING); + spin_unlock_irqrestore(&dchan->vchan.lock, flags); + + return IRQ_HANDLED; +} + +static int efx_dma_alloc_chan_resources(struct dma_chan *chan) +{ + return 0; +} + +static void efx_dma_free_chan_resources(struct dma_chan *chan) +{ + +} + +static void efx_dma_dump_pending_list(struct dma_chan *chan) +{ + struct efx_dma_priv *priv = to_efx_dma_priv(chan); + struct efx_dma_chan *dchan = to_efx_dma_chan(chan); + struct efx_dma_desc *desc; + struct device *dev = priv->dev; + int i = 0, count = 0; + int limit = 5; + + // Count number of descriptor in the pending_list + spin_lock(&dchan->lock); + list_for_each_entry(desc, &dchan->pending_list, node) + count++; + spin_unlock(&dchan->lock); + + if (count == 0) { + dev_dbg(dev, "Pending list is empty\n"); + return; + } + + dev_dbg(dev, "Pending list has %d descriptors\n", count); + + // Print first 5 and last 5 descriptors + spin_lock(&dchan->lock); + list_for_each_entry(desc, &dchan->pending_list, node) { + if (i < limit || i >= count - limit) { + pr_debug("%s: Descriptor %d:\n", __func__, i); + pr_debug(" desc=%p, hw_desc=%p, dma_handle=0x%llx\n", + desc, desc->hw_desc, (unsigned long long)desc->dma_handle); + + if (desc->hw_desc) { + pr_debug(" status=0x%x, control=0x%x\n", + desc->hw_desc->status, desc->hw_desc->control); + pr_debug(" src_addr=0x%llx, dst_addr=0x%llx, next=0x%llx\n", + (unsigned long long)desc->hw_desc->src_addr, + (unsigned long long)desc->hw_desc->dst_addr, + (unsigned long long)desc->hw_desc->next); + } + + pr_debug(" cyclic=%d, direction=%d, segments=%zu\n", + desc->cyclic, desc->direction, desc->segments); + } + i++; + } + spin_unlock(&dchan->lock); +} + +static void efx_dma_free_hw_desc_chain(struct dma_chan *chan) +{ + struct efx_dma_priv *priv = to_efx_dma_priv(chan); + struct efx_dma_chan *dchan = to_efx_dma_chan(chan); + struct efx_dma_desc *desc, *tmp; + struct device *dev = priv->dev; + int count = 0; + int limit = 5, i = 0; + + // Count number of descriptor in the pending_list + spin_lock(&dchan->lock); + list_for_each_entry(desc, &dchan->pending_list, node) + count++; + spin_unlock(&dchan->lock); + + if (count == 0) { + dev_info(dev, "Pending list is empty\n"); + return; + } + + dev_info(dev, "Freeing %d descriptors\n", count); + + spin_lock(&dchan->lock); + list_for_each_entry_safe(desc, tmp, &dchan->pending_list, node) { + list_del(&desc->node); + + if (desc->hw_desc) { + dma_free_coherent(dev, sizeof(struct efx_dma_hw_desc), + desc->hw_desc, desc->dma_handle); + if (i < limit || i >= count - limit) { + pr_debug(" Freeing descriptor: %d\n", i); + pr_debug(" free hw_desc=%p, dma_handle=0x%llx\n",desc->hw_desc, + (unsigned long long)desc->dma_handle); + } + } + + kfree(desc); + if (i < limit || i >= count - limit) + pr_debug(" desc=%p\n",desc); + i++; + } + + dchan->head_desc = NULL; + spin_unlock(&dchan->lock); +} + +static void efx_dma_desc_free(struct virt_dma_desc *vd) +{ + struct dma_chan *chan = vd->tx.chan; + + efx_dma_free_hw_desc_chain(chan); +} + +static void efx_dma_hw_desc_address_control(struct efx_dma_chan *dchan, + struct efx_dma_desc *desc, dma_addr_t src_addr, dma_addr_t dst_addr, + size_t period_len, struct scatterlist *sg, size_t i) +{ + struct efx_dma_hw_desc *hw_desc = desc->hw_desc; + dma_addr_t segment_addr; + u32 buf_len; + u32 control = desc->cyclic ? EFX_DMA_DESCRIPTOR_NO_COMPLETION + : EFX_DMA_DESCRIPTOR_CONTROL_END_OF_PACKET; + + hw_desc->status = 0; + if (sg) { + buf_len = sg_dma_len(sg); + segment_addr = sg_dma_address(sg); + } else { + buf_len = period_len - 1; + segment_addr = src_addr + i * period_len; + } + + hw_desc->control = (u32)(control | buf_len); + + // Configure last hw_desc + if (i == (desc->segments - 1)) { + if (desc->cyclic) { + hw_desc->next = dchan->head_desc->dma_handle; + } else { + hw_desc->status = EFX_DMA_DESCRIPTOR_STATUS_COMPLETED; + } + } + + if (desc->direction == DMA_DEV_TO_MEM) { + hw_desc->src_addr = 0; + hw_desc->dst_addr = segment_addr; + } else if (desc->direction == DMA_MEM_TO_DEV) { + hw_desc->src_addr = segment_addr; + hw_desc->dst_addr = 0; + } else if (desc->direction == DMA_MEM_TO_MEM) { + hw_desc->src_addr = src_addr + i * period_len; + hw_desc->dst_addr = dst_addr + i * period_len; + } +} + +static int efx_dma_hw_desc_init(struct dma_chan *chan, dma_addr_t src_addr, dma_addr_t dst_addr, + struct scatterlist *sg, size_t len, size_t period_len, + enum dma_transfer_direction direction, bool cyclic) +{ + struct efx_dma_priv *priv = to_efx_dma_priv(chan); + struct efx_dma_chan *dchan = to_efx_dma_chan(chan); + struct efx_dma_desc *desc = NULL, *prev_desc = NULL; + struct efx_dma_hw_desc *hw_desc = NULL; + struct device *dev = priv->dev; + struct scatterlist *sgl; + size_t i; + int ret = 0; + + for (i = 0; i < len; i++) { + // Allocate memory for desc + desc = kzalloc(sizeof(*desc), GFP_KERNEL); + if (!desc) { + ret = -ENOMEM; + dev_err(dev, "Failed to allocate memory for descriptor\n"); + goto err_free_desc_init; + } + + desc->cyclic = cyclic; + desc->segments = len; + desc->direction = direction; + INIT_LIST_HEAD(&desc->node); + + // Allocate memory for hw_desc + hw_desc = dma_alloc_coherent(dev, sizeof(*hw_desc), + &desc->dma_handle, GFP_KERNEL); + if (!hw_desc) { + dev_err(dev, "Failed to allocate memory for hw descriptor\n"); + kfree(desc); + ret = -ENOMEM; + goto err_free_desc_init; + } + desc->hw_desc = hw_desc; + + // Save head descriptor + if (i == 0) + dchan->head_desc = desc; + + // Set the scatterlist pointer for sg mode. For cylic, sgl is NULL + sgl = sg ? &sg[i] : NULL; + // Configure the hw descriptor addresses and control + efx_dma_hw_desc_address_control(dchan, desc, src_addr, dst_addr, period_len, sgl, i); + + // Chain descriptor + if (prev_desc) + prev_desc->hw_desc->next = desc->dma_handle; + + prev_desc = desc; + + // Add to pending list + spin_lock(&dchan->lock); + list_add_tail(&desc->node, &dchan->pending_list); + spin_unlock(&dchan->lock); + } + + // Debugging + efx_dma_dump_pending_list(chan); + return 0; + +err_free_desc_init: + efx_dma_free_hw_desc_chain(chan); + return ret; + +} + +static struct dma_async_tx_descriptor *efx_dma_prep_slave_sg( + struct dma_chan *chan, + struct scatterlist *sg, unsigned int sg_len, + enum dma_transfer_direction direction, + unsigned long flags, void *context) +{ + struct efx_dma_chan *dchan = to_efx_dma_chan(chan); + struct device *dev = dchan->priv->dev; + struct efx_dma_desc *desc = NULL; + int ret; + bool cyclic = false; + + // Validate input + if (unlikely(!chan || !sg || sg_len <= 0)) + return NULL; + + // sg_len +1 to inlcude the last status of the descriptor + sg_len += 1; + + if (flags & DMA_CTRL_REUSE) { + cyclic = true; + sg_len -= 1; // Cyclic transfer does not require additional descriptor + flags &= ~DMA_CTRL_REUSE; + } + + // Initialize hardware descriptor. Set cyclic to false + ret = efx_dma_hw_desc_init(chan, 0, 0, sg, sg_len, 0, direction, cyclic); + if (ret) { + dev_err(dev, "Failed to initialize hardware descriptor\n"); + return NULL; + } + + // Get head descriptor + desc = dchan->head_desc; + if (!desc) { + dev_err(dev, "No head descriptor found\n"); + return NULL; + } + + // Configure interrupt + if (flags & DMA_PREP_INTERRUPT) + efx_dma_interrupt_config(chan, EFX_DMA_CHANNEL_INTERRUPT_CHANNEL_COMPLETION_MASK); + + // Prepare virtual DMA descriptor + dchan->vchan.cyclic = &desc->vdesc; + return vchan_tx_prep(&dchan->vchan, &desc->vdesc, flags); +} + +static struct dma_async_tx_descriptor *efx_dma_prep_cyclic( + struct dma_chan *chan, dma_addr_t buf_addr, size_t buf_len, + size_t period_len, enum dma_transfer_direction direction, + unsigned long flags) +{ + struct efx_dma_chan *dchan = to_efx_dma_chan(chan); + struct device *dev = dchan->priv->dev; + struct efx_dma_desc *desc = NULL; + size_t segments = buf_len / period_len; + int ret; + + if (unlikely(!chan || !buf_addr || buf_len == 0 || period_len > buf_len)) + return NULL; + + ret = efx_dma_hw_desc_init(chan, buf_addr, 0, NULL, segments, period_len, direction, true); + if (ret) + return NULL; + + // Get head descriptor + desc = dchan->head_desc; + if (!desc) { + dev_err(dev, "No head descriptor found\n"); + return NULL; + } + + dchan->vchan.cyclic = &desc->vdesc; + return vchan_tx_prep(&dchan->vchan, &desc->vdesc, flags); +} + +static struct dma_async_tx_descriptor *efx_dma_prep_dma_memcpy(struct dma_chan *chan, + dma_addr_t dst_addr, dma_addr_t src_addr, size_t len, + unsigned long flags) +{ + struct efx_dma_chan *dchan = to_efx_dma_chan(chan); + struct efx_dma_desc *desc = NULL; + struct device *dev = dchan->priv->dev; + size_t segments; + int ret; + + // Validate input + if (unlikely(!chan || !dst_addr || !src_addr || !len)) + return NULL; + + if (len < PAGE_SIZE) + segments = 1; + else + segments = (len / PAGE_SIZE); + + // Increase segments by 1 to include the last status of the descriptor + segments += 1; + + ret = efx_dma_hw_desc_init(chan, src_addr, dst_addr, NULL, len, segments, DMA_MEM_TO_MEM, false); + if (ret) { + return NULL; + } + + // Get head descriptor + desc = dchan->head_desc; + if (!desc) { + dev_err(dev, "No head descriptor found\n"); + return NULL; + } + + dchan->vchan.cyclic = &desc->vdesc; + + return vchan_tx_prep(&dchan->vchan, &desc->vdesc, flags); +} + + +static void efx_dma_issue_pending(struct dma_chan *chan) +{ + struct efx_dma_chan *dchan = to_efx_dma_chan(chan); + unsigned long flags; + + spin_lock_irqsave(&dchan->vchan.lock, flags); + if (vchan_issue_pending(&dchan->vchan)) + efx_dma_start_transfer(chan); + + spin_unlock_irqrestore(&dchan->vchan.lock, flags); +} + +static int efx_dma_device_config(struct dma_chan *chan, + struct dma_slave_config *config) +{ + struct efx_dma_chan *dchan = to_efx_dma_chan(chan); + + dchan->cfg = config; + return 0; +} + +static enum dma_status efx_dma_tx_status(struct dma_chan *chan, dma_cookie_t cookie, + struct dma_tx_state *txstate) +{ + if (efx_dma_busy(chan)) + return DMA_IN_PROGRESS; + + if (cookie < DMA_MIN_COOKIE) + return DMA_ERROR; + + if (chan->completed_cookie == cookie) + return DMA_COMPLETE; + + return DMA_COMPLETE; +} + +static int efx_dma_terminate_all(struct dma_chan *chan) +{ + struct efx_dma_chan *dchan = to_efx_dma_chan(chan); + struct virt_dma_chan *vchan = &dchan->vchan; + struct device *dev = dchan->priv->dev; + + if (dchan) { + // Stop all transfer of the DMA channels + efx_dma_stop_channel(&dchan->vchan.chan); + + // Disabled interrupt + efx_dma_interrupt_config(chan, 0); + + // Terminate all virtual DMA descriptors + //vchan_free_chan_resources(vchan); + + // Free all pending descriptors + if (dchan->head_desc) + efx_dma_free_hw_desc_chain(&dchan->vchan.chan); + + // Clear cyclic pointer + vchan->cyclic = NULL; + dchan->head_desc = NULL; + + dev_info(dev, "DMA channel %s terminated and cleaned up\n", dchan->name); + } + + return 0; +} + +static void efx_dma_release(struct dma_device *dev) +{ +} + +static int efx_dma_chan_probe(struct platform_device *pdev) +{ + struct efx_dma_priv *priv = platform_get_drvdata(pdev); + struct efx_dma_chan *dchan; + struct device_node *node = pdev->dev.of_node; + struct device_node *child = pdev->dev.of_node; + size_t i = 0; + int ret; + + priv->dchan = devm_kzalloc(&pdev->dev, sizeof(struct efx_dma_chan) * priv->chan_count, GFP_KERNEL); + if (!priv->dchan) { + dev_err(&pdev->dev, "Failed to allocated memory for DMA channels\n"); + return -ENOMEM; + } + + for_each_child_of_node(node, child) { + dchan = &priv->dchan[i]; + if (of_property_read_string(child, "dma-names", &dchan->name)) { + dev_warn(&pdev->dev, "Failed to get dma-names for channel %zu\n", i); + return -EINVAL; + } + + dev_info(&pdev->dev, "Initialize DMA channel %zu for %s\n", i, dchan->name); + dchan->priv = priv; + vchan_init(&dchan->vchan, &priv->dma_dev); + dchan->chan_id = i; + dchan->reg = priv->base + i * 0x80; + dchan->cfg = NULL; + dchan->vchan.desc_free = &efx_dma_desc_free; + + dchan->irq = irq_of_parse_and_map(node, i); + if (dchan->irq <= 0) { + dev_warn(&pdev->dev, "Failed to get interrupt number\n"); + } + + ret = devm_request_irq(&pdev->dev, dchan->irq, efx_dma_interrupt_handler, + IRQF_SHARED, dchan->name, dchan); + + if (ret) { + dev_warn(&pdev->dev, "Warning: Failed to register interrupt handler\n"); + } + + efx_dma_stop_channel(&dchan->vchan.chan); + efx_dma_interrupt_pending_clear(&dchan->vchan.chan, 0xFFFFFFFF); + + ret = of_property_read_u32(child, "chan-priority", &dchan->priority); + if (ret) { + dev_warn(&pdev->dev, "'chan-priority' is not found in the DMA device tree node. Use the default priority\n"); + dchan->priority = 0; + } + dev_info(&pdev->dev, " channel address %px, priority %u\n", dchan->reg, dchan->priority); + efx_dma_set_channel_priority(dchan); + + // initialize dchan->lock + spin_lock_init(&dchan->lock); + + INIT_LIST_HEAD(&dchan->pending_list); + i++; + } + + return 0; +} + +static struct dma_chan *of_dma_efx_dma_xlate(struct of_phandle_args *dma_spec, + struct of_dma *ofdma) +{ + struct efx_dma_priv *priv = ofdma->of_dma_data; + struct efx_dma_chan *dchan; + struct dma_chan *chan = NULL; + size_t chan_id; + + chan_id = dma_spec->args[0]; + if (chan_id < 0 || chan_id > priv->chan_count) + return NULL; + + dchan = &priv->dchan[chan_id]; + chan = dma_get_slave_channel(&dchan->vchan.chan); + if (!chan) { + dev_err(priv->dev, "Failed to get DMA slave channel %zu\n", chan_id); + return NULL; + } + + dev_info(priv->dev, "Found DMA slave '%s' at channel %zu\n", dchan->name, chan_id); + return chan; +} + +static int efx_dma_probe(struct platform_device *pdev) +{ + struct efx_dma_priv *priv; + struct resource *res; + struct device_node *node = pdev->dev.of_node; + struct device_node *child = pdev->dev.of_node; + int ret; + + priv = devm_kzalloc(&pdev->dev, sizeof(*priv), GFP_KERNEL); + if (!priv) + return -ENOMEM; + + priv->dev = &pdev->dev; + res = platform_get_resource(pdev, IORESOURCE_MEM, 0); + priv->base = devm_ioremap_resource(&pdev->dev, res); + if (IS_ERR(priv->base)) + return PTR_ERR(priv->base); + + dma_cap_zero(priv->dma_dev.cap_mask); + dma_cap_set(DMA_SLAVE, priv->dma_dev.cap_mask); + dma_cap_set(DMA_CYCLIC, priv->dma_dev.cap_mask); + dma_cap_set(DMA_MEMCPY, priv->dma_dev.cap_mask); + + ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); + if (ret) { + dev_warn(&pdev->dev, "Failed to set 64-bit DMA mask, falling back to 32-bit\n"); + ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)); + if (ret) { + dev_err(&pdev->dev, "Failed to set 32-bit DMA mask\n"); + return ret; + } + } + + priv->dma_dev.src_addr_widths = DMA_SLAVE_BUSWIDTH_4_BYTES; + priv->dma_dev.dst_addr_widths = DMA_SLAVE_BUSWIDTH_4_BYTES; + priv->dma_dev.max_burst = 16; + priv->dma_dev.directions = BIT(DMA_MEM_TO_DEV) | BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_MEM); + + INIT_LIST_HEAD(&priv->dma_dev.channels); + + // Initialize and register the DMA engine + priv->dma_dev.dev = &pdev->dev; + priv->dma_dev.device_alloc_chan_resources = efx_dma_alloc_chan_resources; + priv->dma_dev.device_free_chan_resources = efx_dma_free_chan_resources; + priv->dma_dev.device_prep_slave_sg = efx_dma_prep_slave_sg; + priv->dma_dev.device_prep_dma_cyclic = efx_dma_prep_cyclic; + priv->dma_dev.device_issue_pending = efx_dma_issue_pending; + priv->dma_dev.device_config = efx_dma_device_config; + priv->dma_dev.device_tx_status = efx_dma_tx_status; + priv->dma_dev.device_prep_dma_memcpy = efx_dma_prep_dma_memcpy; + priv->dma_dev.device_release = efx_dma_release; + priv->dma_dev.device_terminate_all = efx_dma_terminate_all; + + platform_set_drvdata(pdev, priv); + + priv->chan_count = 0; + ret = of_property_read_u32(node, "dma-channels", &priv->chan_count); + if (ret) { + dev_warn(&pdev->dev, "`dma-channels` not found in the DMA device tree node. Try to auto detect the number of channels\n"); + for_each_child_of_node(node, child) { + priv->chan_count++; + } + } + + dev_info(&pdev->dev, "Found %u DMA channels\n", priv->chan_count); + + // Initialize DMA channels + ret = efx_dma_chan_probe(pdev); + if (ret) + return ret; + + dev_info(&pdev->dev, "Register DMA controller\n"); + ret = dma_async_device_register(&priv->dma_dev); + if (ret) { + dev_err(&pdev->dev, "Failed to register DMA controller\n"); + return ret; + } + + // Register DMA controller with device tree framework + ret = of_dma_controller_register(node, of_dma_efx_dma_xlate, priv); + if (ret < 0) { + dev_err(&pdev->dev, "Unable to register DMA controller to DT\n"); + dma_async_device_unregister(&priv->dma_dev); + return ret; + } + + dev_info(&pdev->dev, "DMA controller registered\n"); + + return 0; +} + +static int efx_dma_remove(struct platform_device *pdev) +{ + struct efx_dma_priv *priv = platform_get_drvdata(pdev); + + dma_async_device_unregister(&priv->dma_dev); + + return 0; +} + +static const struct of_device_id efx_dma_of_match[] = { + { .compatible = "efx,dma-controller" }, + { }, +}; +MODULE_DEVICE_TABLE(of, efx_dma_of_match); + +static struct platform_driver efx_dma_driver = { + .probe = efx_dma_probe, + .remove = efx_dma_remove, + .driver = { + .name = "efx-dma", + .of_match_table = efx_dma_of_match, + }, +}; + +module_platform_driver(efx_dma_driver); + +MODULE_AUTHOR("Alim Hussin "); +MODULE_DESCRIPTION("Efinix DMA driver"); +MODULE_LICENSE("GPL v2");