dma: add Efinix DMA controller driver
This commit is contained in:
Mohamad Noor Alim Hussin
@@ -187,6 +187,13 @@ config DW_AXI_DMAC
|
||||
NOTE: This driver wasn't tested on 64 bit platform because
|
||||
of lack 64 bit platform with Synopsys DW AXI DMAC.
|
||||
|
||||
config EFINIX_DMA
|
||||
bool "Efinix DMA support"
|
||||
select DMA_ENGINE
|
||||
select DMA_VIRTUAL_CHANNELS
|
||||
help
|
||||
Enable support for Efinix DMA controller
|
||||
|
||||
config EP93XX_DMA
|
||||
bool "Cirrus Logic EP93xx DMA support"
|
||||
depends on ARCH_EP93XX || COMPILE_TEST
|
||||
|
||||
@@ -29,6 +29,7 @@ obj-$(CONFIG_DMA_SUN6I) += sun6i-dma.o
|
||||
obj-$(CONFIG_DW_AXI_DMAC) += dw-axi-dmac/
|
||||
obj-$(CONFIG_DW_DMAC_CORE) += dw/
|
||||
obj-$(CONFIG_DW_EDMA) += dw-edma/
|
||||
obj-$(CONFIG_EFINIX_DMA) += efx_dma.o
|
||||
obj-$(CONFIG_EP93XX_DMA) += ep93xx_dma.o
|
||||
obj-$(CONFIG_FSL_DMA) += fsldma.o
|
||||
obj-$(CONFIG_FSL_EDMA) += fsl-edma.o fsl-edma-common.o
|
||||
|
||||
889
drivers/dma/efx_dma.c
Normal file
889
drivers/dma/efx_dma.c
Normal file
@@ -0,0 +1,889 @@
|
||||
#include <linux/platform_device.h>
|
||||
#include <linux/module.h>
|
||||
#include <linux/interrupt.h>
|
||||
#include <linux/io.h>
|
||||
#include <linux/dmaengine.h>
|
||||
#include <linux/dma-mapping.h>
|
||||
#include <linux/of.h>
|
||||
#include <linux/of_dma.h>
|
||||
#include <linux/of_irq.h>
|
||||
#include <linux/list.h>
|
||||
#include <linux/spinlock.h>
|
||||
|
||||
#include "dmaengine.h"
|
||||
#include "virt-dma.h"
|
||||
|
||||
#define EFX_DMA_CHANNEL_INPUT_ADDRESS 0x00
|
||||
#define EFX_DMA_CHANNEL_INPUT_STREAM 0x08
|
||||
#define EFX_DMA_CHANNEL_INPUT_CONFIG 0x0c
|
||||
#define EFX_DMA_CHANNEL_INPUT_CONFIG_MEMORY (1 << 12)
|
||||
#define EFX_DMA_CHANNEL_INPUT_CONFIG_STREAM 0x0
|
||||
#define EFX_DMA_CHANNEL_INPUT_CONFIG_COMPLETION_ON_PACKET (1 << 13)
|
||||
#define EFX_DMA_CHANNEL_INPUT_CONFIG_WAIT_ON_PACKET (1 << 14)
|
||||
|
||||
#define EFX_DMA_CHANNEL_OUTPUT_ADDRESS 0x10
|
||||
#define EFX_DMA_CHANNEL_OUTPUT_STREAM 0x18
|
||||
#define EFX_DMA_CHANNEL_OUTPUT_CONFIG 0x1c
|
||||
#define EFX_DMA_CHANNEL_OUTPUT_CONFIG_MEMORY (1 << 12)
|
||||
#define EFX_DMA_CHANNEL_OUTPUT_CONFIG_STREAM 0x0
|
||||
#define EFX_DMA_CHANNEL_OUTPUT_CONFIG_LAST (1 << 13)
|
||||
|
||||
#define EFX_DMA_CHANNEL_DIRECT_BYTES 0x20
|
||||
#define EFX_DMA_CHANNEL_STATUS 0x2c
|
||||
#define EFX_DMA_CHANNEL_STATUS_DIRECT_START (1 << 0)
|
||||
#define EFX_DMA_CHANNEL_STATUS_BUSY (1 << 0)
|
||||
#define EFX_DMA_CHANNEL_STATUS_SELF_RESTART (1 << 1)
|
||||
#define EFX_DMA_CHANNEL_STATUS_STOP (1 << 2)
|
||||
#define EFX_DMA_CHANNEL_STATUS_LINKED_LIST_START (1 << 4)
|
||||
|
||||
#define EFX_DMA_CHANNEL_FIFO 0x40
|
||||
#define EFX_DMA_CHANNEL_PRIORITY 0x44
|
||||
#define EFX_DMA_CHANNEL_INTERRUPT_ENABLE 0x50
|
||||
#define EFX_DMA_CHANNEL_INTERRUPT_PENDING 0x54
|
||||
|
||||
// Interrupt at the end of each descriptor
|
||||
#define EFX_DMA_CHANNEL_INTERRUPT_DESCRIPTOR_COMPLETION_MASK (1 << 0)
|
||||
// Interrupt at the middle of each descriptor, require the half_completion_interrpt
|
||||
// option to be enabled for the channel
|
||||
#define EFX_DMA_CHANNEL_INTERRUPT_DESCRIPTOR_COMPLETION_HALF_MASK (1 << 1)
|
||||
// Interrupt when the channel is going off (not busy anymore)
|
||||
#define EFX_DMA_CHANNEL_INTERRUPT_CHANNEL_COMPLETION_MASK (1 << 2)
|
||||
// Interrupt each time that a linked list's descriptor stats field is updated
|
||||
#define EFX_DMA_CHANNEL_INTERRUPT_LINKED_LIST_UPDATE_MASK (1 << 3)
|
||||
// Interrupt each time a S -> M channel has done transferring a packet into the memory
|
||||
#define EFX_DMA_CHANNEL_INTERRUPT_INPUT_PACKET_MASK (1 << 4)
|
||||
|
||||
#define EFX_DMA_CHANNEL_PROGRESS_BYTES 0x60
|
||||
#define EFX_DMA_CHANNEL_LINKED_LIST_HEAD 0x70
|
||||
#define EFX_DMA_CHANNEL_LINKED_LIST_FROM_SG_BUS 0x78
|
||||
|
||||
#define EFX_DMA_DESCRIPTOR_CONTROL_BYTES 0x7FFFFFF
|
||||
#define EFX_DMA_DESCRIPTOR_CONTROL_END_OF_PACKET (1 << 30)
|
||||
#define EFX_DMA_DESCRIPTOR_NO_COMPLETION (1 << 31)
|
||||
|
||||
#define EFX_DMA_DESCRIPTOR_STATUS_BYTES 0x7FFFFFF
|
||||
#define EFX_DMA_DESCRIPTOR_STATUS_END_OF_PACKET (1 << 30)
|
||||
#define EFX_DMA_DESCRIPTOR_STATUS_COMPLETED (1 << 31)
|
||||
|
||||
// DMA Hardware descriptor
|
||||
struct efx_dma_hw_desc {
|
||||
u32 status;
|
||||
u32 control;
|
||||
u64 src_addr;
|
||||
u64 dst_addr;
|
||||
u64 next; // physical address of next descriptor
|
||||
} __aligned(64);
|
||||
|
||||
// Per transfer descriptor
|
||||
struct efx_dma_desc {
|
||||
struct virt_dma_desc vdesc;
|
||||
struct efx_dma_hw_desc *hw_desc; // Pointer to the struct efx_dma_hw_desc
|
||||
dma_addr_t dma_handle; // DMA address of struct efx_dma_hw_desc
|
||||
struct list_head node; // List of struct efx_dma_hw_desc
|
||||
size_t segments; // Number of DMA descriptor
|
||||
bool cyclic; // True for cyclic transfer
|
||||
enum dma_transfer_direction direction;
|
||||
struct scatterlist *sg;
|
||||
struct page **pages;
|
||||
};
|
||||
|
||||
// DMA channel specific data
|
||||
struct efx_dma_chan {
|
||||
const char *name; // Channel name
|
||||
void __iomem *reg; // based address of DMA channel
|
||||
size_t chan_id; // Channel ID
|
||||
struct virt_dma_chan vchan;
|
||||
struct efx_dma_priv *priv; // Pointer to the DMA controller
|
||||
struct dma_slave_config *cfg; // Channel specific configuration
|
||||
u32 priority; // Priority number of DMA channel
|
||||
size_t irq; // IRQ number used by the DMA channel
|
||||
struct efx_dma_desc *head_desc; // First DMA descriptor
|
||||
struct list_head pending_list; // List of all DMA descriptor
|
||||
spinlock_t lock; // Lock for manipulating pending_list
|
||||
};
|
||||
|
||||
// DMA controller private data
|
||||
struct efx_dma_priv {
|
||||
struct device *dev;
|
||||
void __iomem *base; // DMA controller based address
|
||||
struct dma_device dma_dev;
|
||||
struct efx_dma_chan *dchan; // Pointer to the DMA channel
|
||||
u32 chan_count; // Number of DMA channel
|
||||
};
|
||||
|
||||
static inline struct efx_dma_chan *to_efx_dma_chan(struct dma_chan *chan)
|
||||
{
|
||||
return container_of(chan, struct efx_dma_chan, vchan.chan);
|
||||
}
|
||||
|
||||
static inline struct efx_dma_priv *to_efx_dma_priv(struct dma_chan *chan)
|
||||
{
|
||||
struct efx_dma_chan *dchan = to_efx_dma_chan(chan);
|
||||
return dchan->priv;
|
||||
}
|
||||
|
||||
static void efx_dma_input_memory(struct dma_chan *chan)
|
||||
{
|
||||
struct efx_dma_chan *dchan = to_efx_dma_chan(chan);
|
||||
struct efx_dma_hw_desc *hw_desc = dchan->head_desc->hw_desc;
|
||||
u32 byte_per_burst = 0;
|
||||
|
||||
if (dchan->cfg)
|
||||
byte_per_burst = dchan->cfg->src_maxburst;
|
||||
else
|
||||
byte_per_burst = chan->device->max_burst;
|
||||
|
||||
iowrite32(hw_desc->src_addr, dchan->reg + EFX_DMA_CHANNEL_INPUT_ADDRESS);
|
||||
iowrite32(EFX_DMA_CHANNEL_INPUT_CONFIG_MEMORY | ((byte_per_burst - 1) & 0xFFF),
|
||||
dchan->reg + EFX_DMA_CHANNEL_INPUT_CONFIG);
|
||||
}
|
||||
|
||||
static void efx_dma_output_memory(struct dma_chan *chan)
|
||||
{
|
||||
struct efx_dma_chan *dchan = to_efx_dma_chan(chan);
|
||||
struct efx_dma_hw_desc *hw_desc = dchan->head_desc->hw_desc;
|
||||
u32 len = hw_desc->control & 0x1FFFFFF;
|
||||
|
||||
iowrite32(hw_desc->dst_addr, dchan->reg + EFX_DMA_CHANNEL_OUTPUT_ADDRESS);
|
||||
iowrite32(EFX_DMA_CHANNEL_OUTPUT_CONFIG_MEMORY | ((len - 1) & 0xFFF),
|
||||
dchan->reg + EFX_DMA_CHANNEL_OUTPUT_CONFIG);
|
||||
}
|
||||
|
||||
static void efx_dma_input_stream(struct dma_chan *chan, u32 wait_on_packet, u32 completion_on_packet)
|
||||
{
|
||||
struct efx_dma_chan *dchan = to_efx_dma_chan(chan);
|
||||
completion_on_packet = completion_on_packet ? EFX_DMA_CHANNEL_INPUT_CONFIG_COMPLETION_ON_PACKET : 0;
|
||||
wait_on_packet = wait_on_packet ? EFX_DMA_CHANNEL_INPUT_CONFIG_WAIT_ON_PACKET : 0;
|
||||
|
||||
iowrite32(0, dchan->reg + EFX_DMA_CHANNEL_INPUT_STREAM);
|
||||
iowrite32(EFX_DMA_CHANNEL_INPUT_CONFIG_STREAM
|
||||
| completion_on_packet | wait_on_packet,
|
||||
dchan->reg + EFX_DMA_CHANNEL_INPUT_CONFIG);
|
||||
}
|
||||
|
||||
static void efx_dma_output_stream(struct dma_chan *chan)
|
||||
{
|
||||
struct efx_dma_chan *dchan = to_efx_dma_chan(chan);
|
||||
|
||||
iowrite32(0, dchan->reg + EFX_DMA_CHANNEL_OUTPUT_STREAM);
|
||||
iowrite32(EFX_DMA_CHANNEL_OUTPUT_CONFIG_LAST | EFX_DMA_CHANNEL_OUTPUT_CONFIG_STREAM,
|
||||
dchan->reg + EFX_DMA_CHANNEL_OUTPUT_CONFIG);
|
||||
}
|
||||
|
||||
static void efx_dma_linked_list_start(struct dma_chan *chan)
|
||||
{
|
||||
struct efx_dma_chan *dchan = to_efx_dma_chan(chan);
|
||||
|
||||
iowrite32(dchan->head_desc->dma_handle, dchan->reg + EFX_DMA_CHANNEL_LINKED_LIST_HEAD);
|
||||
iowrite32(0, dchan->reg + EFX_DMA_CHANNEL_LINKED_LIST_FROM_SG_BUS);
|
||||
iowrite32(EFX_DMA_CHANNEL_STATUS_LINKED_LIST_START, dchan->reg + EFX_DMA_CHANNEL_STATUS);
|
||||
}
|
||||
|
||||
static void efx_dma_stop_channel(struct dma_chan *chan)
|
||||
{
|
||||
struct efx_dma_chan *dchan = to_efx_dma_chan(chan);
|
||||
|
||||
iowrite32(EFX_DMA_CHANNEL_STATUS_STOP, dchan->reg + EFX_DMA_CHANNEL_STATUS);
|
||||
}
|
||||
|
||||
static bool efx_dma_busy(struct dma_chan *chan)
|
||||
{
|
||||
struct efx_dma_chan *dchan = to_efx_dma_chan(chan);
|
||||
u32 busy;
|
||||
|
||||
busy = readl(dchan->reg + EFX_DMA_CHANNEL_STATUS) & EFX_DMA_CHANNEL_STATUS_BUSY;
|
||||
return busy ? true : false;
|
||||
}
|
||||
|
||||
static inline void efx_dma_interrupt_pending_clear(struct dma_chan *chan, u32 mask)
|
||||
{
|
||||
struct efx_dma_chan *dchan = to_efx_dma_chan(chan);
|
||||
|
||||
iowrite32(mask, dchan->reg + EFX_DMA_CHANNEL_INTERRUPT_PENDING);
|
||||
}
|
||||
|
||||
static void efx_dma_interrupt_config(struct dma_chan *chan, u32 mask)
|
||||
{
|
||||
struct efx_dma_chan *dchan = to_efx_dma_chan(chan);
|
||||
|
||||
efx_dma_interrupt_pending_clear(chan, 0xFFFFFFFF);
|
||||
iowrite32(mask, dchan->reg + EFX_DMA_CHANNEL_INTERRUPT_ENABLE);
|
||||
}
|
||||
|
||||
static void efx_dma_set_channel_priority(struct efx_dma_chan *dchan)
|
||||
{
|
||||
iowrite32(dchan->priority, dchan->reg + EFX_DMA_CHANNEL_PRIORITY);
|
||||
}
|
||||
|
||||
static void efx_dma_configure_registers(struct dma_chan *chan)
|
||||
{
|
||||
struct efx_dma_chan *dchan = to_efx_dma_chan(chan);
|
||||
struct efx_dma_desc *desc = dchan->head_desc;
|
||||
|
||||
if (desc->direction == DMA_DEV_TO_MEM) {
|
||||
efx_dma_output_memory(chan);
|
||||
efx_dma_input_stream(chan, 1, 0);
|
||||
} else if (desc->direction == DMA_MEM_TO_DEV) {
|
||||
efx_dma_input_memory(chan);
|
||||
efx_dma_output_stream(chan);
|
||||
} else if (desc->direction == DMA_MEM_TO_MEM) {
|
||||
efx_dma_input_memory(chan);
|
||||
efx_dma_output_memory(chan);
|
||||
}
|
||||
}
|
||||
|
||||
static void efx_dma_start_transfer(struct dma_chan *chan)
|
||||
{
|
||||
struct efx_dma_chan *dchan = to_efx_dma_chan(chan);
|
||||
struct virt_dma_desc *vdesc;
|
||||
|
||||
vdesc = vchan_next_desc(&dchan->vchan);
|
||||
if (!vdesc)
|
||||
return;
|
||||
|
||||
efx_dma_configure_registers(chan);
|
||||
efx_dma_linked_list_start(chan);
|
||||
}
|
||||
|
||||
static irqreturn_t efx_dma_interrupt_handler(int irq, void *dev_id)
|
||||
{
|
||||
struct efx_dma_chan *dchan = (struct efx_dma_chan *)dev_id;
|
||||
struct efx_dma_desc *desc = dchan->head_desc;
|
||||
struct dma_chan *chan = &dchan->vchan.chan;
|
||||
unsigned long flags;
|
||||
u32 pending;
|
||||
|
||||
spin_lock_irqsave(&dchan->vchan.lock, flags);
|
||||
pending = readl(dchan->reg + EFX_DMA_CHANNEL_INTERRUPT_PENDING);
|
||||
|
||||
if (pending & EFX_DMA_CHANNEL_INTERRUPT_DESCRIPTOR_COMPLETION_MASK)
|
||||
efx_dma_interrupt_pending_clear(chan, EFX_DMA_CHANNEL_INTERRUPT_DESCRIPTOR_COMPLETION_MASK);
|
||||
|
||||
if (pending & EFX_DMA_CHANNEL_INTERRUPT_CHANNEL_COMPLETION_MASK)
|
||||
efx_dma_interrupt_pending_clear(chan, EFX_DMA_CHANNEL_INTERRUPT_CHANNEL_COMPLETION_MASK);
|
||||
|
||||
vchan_cookie_complete(&desc->vdesc);
|
||||
|
||||
// Unmask the interrupt
|
||||
efx_dma_interrupt_pending_clear(chan, 0x0);
|
||||
|
||||
pending = readl(dchan->reg + EFX_DMA_CHANNEL_INTERRUPT_PENDING);
|
||||
spin_unlock_irqrestore(&dchan->vchan.lock, flags);
|
||||
|
||||
return IRQ_HANDLED;
|
||||
}
|
||||
|
||||
static int efx_dma_alloc_chan_resources(struct dma_chan *chan)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
static void efx_dma_free_chan_resources(struct dma_chan *chan)
|
||||
{
|
||||
|
||||
}
|
||||
|
||||
static void efx_dma_dump_pending_list(struct dma_chan *chan)
|
||||
{
|
||||
struct efx_dma_priv *priv = to_efx_dma_priv(chan);
|
||||
struct efx_dma_chan *dchan = to_efx_dma_chan(chan);
|
||||
struct efx_dma_desc *desc;
|
||||
struct device *dev = priv->dev;
|
||||
int i = 0, count = 0;
|
||||
int limit = 5;
|
||||
|
||||
// Count number of descriptor in the pending_list
|
||||
spin_lock(&dchan->lock);
|
||||
list_for_each_entry(desc, &dchan->pending_list, node)
|
||||
count++;
|
||||
spin_unlock(&dchan->lock);
|
||||
|
||||
if (count == 0) {
|
||||
dev_dbg(dev, "Pending list is empty\n");
|
||||
return;
|
||||
}
|
||||
|
||||
dev_dbg(dev, "Pending list has %d descriptors\n", count);
|
||||
|
||||
// Print first 5 and last 5 descriptors
|
||||
spin_lock(&dchan->lock);
|
||||
list_for_each_entry(desc, &dchan->pending_list, node) {
|
||||
if (i < limit || i >= count - limit) {
|
||||
pr_debug("%s: Descriptor %d:\n", __func__, i);
|
||||
pr_debug(" desc=%p, hw_desc=%p, dma_handle=0x%llx\n",
|
||||
desc, desc->hw_desc, (unsigned long long)desc->dma_handle);
|
||||
|
||||
if (desc->hw_desc) {
|
||||
pr_debug(" status=0x%x, control=0x%x\n",
|
||||
desc->hw_desc->status, desc->hw_desc->control);
|
||||
pr_debug(" src_addr=0x%llx, dst_addr=0x%llx, next=0x%llx\n",
|
||||
(unsigned long long)desc->hw_desc->src_addr,
|
||||
(unsigned long long)desc->hw_desc->dst_addr,
|
||||
(unsigned long long)desc->hw_desc->next);
|
||||
}
|
||||
|
||||
pr_debug(" cyclic=%d, direction=%d, segments=%zu\n",
|
||||
desc->cyclic, desc->direction, desc->segments);
|
||||
}
|
||||
i++;
|
||||
}
|
||||
spin_unlock(&dchan->lock);
|
||||
}
|
||||
|
||||
static void efx_dma_free_hw_desc_chain(struct dma_chan *chan)
|
||||
{
|
||||
struct efx_dma_priv *priv = to_efx_dma_priv(chan);
|
||||
struct efx_dma_chan *dchan = to_efx_dma_chan(chan);
|
||||
struct efx_dma_desc *desc, *tmp;
|
||||
struct device *dev = priv->dev;
|
||||
int count = 0;
|
||||
int limit = 5, i = 0;
|
||||
|
||||
// Count number of descriptor in the pending_list
|
||||
spin_lock(&dchan->lock);
|
||||
list_for_each_entry(desc, &dchan->pending_list, node)
|
||||
count++;
|
||||
spin_unlock(&dchan->lock);
|
||||
|
||||
if (count == 0) {
|
||||
dev_info(dev, "Pending list is empty\n");
|
||||
return;
|
||||
}
|
||||
|
||||
dev_info(dev, "Freeing %d descriptors\n", count);
|
||||
|
||||
spin_lock(&dchan->lock);
|
||||
list_for_each_entry_safe(desc, tmp, &dchan->pending_list, node) {
|
||||
list_del(&desc->node);
|
||||
|
||||
if (desc->hw_desc) {
|
||||
dma_free_coherent(dev, sizeof(struct efx_dma_hw_desc),
|
||||
desc->hw_desc, desc->dma_handle);
|
||||
if (i < limit || i >= count - limit) {
|
||||
pr_debug(" Freeing descriptor: %d\n", i);
|
||||
pr_debug(" free hw_desc=%p, dma_handle=0x%llx\n",desc->hw_desc,
|
||||
(unsigned long long)desc->dma_handle);
|
||||
}
|
||||
}
|
||||
|
||||
kfree(desc);
|
||||
if (i < limit || i >= count - limit)
|
||||
pr_debug(" desc=%p\n",desc);
|
||||
i++;
|
||||
}
|
||||
|
||||
dchan->head_desc = NULL;
|
||||
spin_unlock(&dchan->lock);
|
||||
}
|
||||
|
||||
static void efx_dma_desc_free(struct virt_dma_desc *vd)
|
||||
{
|
||||
struct dma_chan *chan = vd->tx.chan;
|
||||
|
||||
efx_dma_free_hw_desc_chain(chan);
|
||||
}
|
||||
|
||||
static void efx_dma_hw_desc_address_control(struct efx_dma_chan *dchan,
|
||||
struct efx_dma_desc *desc, dma_addr_t src_addr, dma_addr_t dst_addr,
|
||||
size_t period_len, struct scatterlist *sg, size_t i)
|
||||
{
|
||||
struct efx_dma_hw_desc *hw_desc = desc->hw_desc;
|
||||
dma_addr_t segment_addr;
|
||||
u32 buf_len;
|
||||
u32 control = desc->cyclic ? EFX_DMA_DESCRIPTOR_NO_COMPLETION
|
||||
: EFX_DMA_DESCRIPTOR_CONTROL_END_OF_PACKET;
|
||||
|
||||
hw_desc->status = 0;
|
||||
if (sg) {
|
||||
buf_len = sg_dma_len(sg);
|
||||
segment_addr = sg_dma_address(sg);
|
||||
} else {
|
||||
buf_len = period_len - 1;
|
||||
segment_addr = src_addr + i * period_len;
|
||||
}
|
||||
|
||||
hw_desc->control = (u32)(control | buf_len);
|
||||
|
||||
// Configure last hw_desc
|
||||
if (i == (desc->segments - 1)) {
|
||||
if (desc->cyclic) {
|
||||
hw_desc->next = dchan->head_desc->dma_handle;
|
||||
} else {
|
||||
hw_desc->status = EFX_DMA_DESCRIPTOR_STATUS_COMPLETED;
|
||||
}
|
||||
}
|
||||
|
||||
if (desc->direction == DMA_DEV_TO_MEM) {
|
||||
hw_desc->src_addr = 0;
|
||||
hw_desc->dst_addr = segment_addr;
|
||||
} else if (desc->direction == DMA_MEM_TO_DEV) {
|
||||
hw_desc->src_addr = segment_addr;
|
||||
hw_desc->dst_addr = 0;
|
||||
} else if (desc->direction == DMA_MEM_TO_MEM) {
|
||||
hw_desc->src_addr = src_addr + i * period_len;
|
||||
hw_desc->dst_addr = dst_addr + i * period_len;
|
||||
}
|
||||
}
|
||||
|
||||
static int efx_dma_hw_desc_init(struct dma_chan *chan, dma_addr_t src_addr, dma_addr_t dst_addr,
|
||||
struct scatterlist *sg, size_t len, size_t period_len,
|
||||
enum dma_transfer_direction direction, bool cyclic)
|
||||
{
|
||||
struct efx_dma_priv *priv = to_efx_dma_priv(chan);
|
||||
struct efx_dma_chan *dchan = to_efx_dma_chan(chan);
|
||||
struct efx_dma_desc *desc = NULL, *prev_desc = NULL;
|
||||
struct efx_dma_hw_desc *hw_desc = NULL;
|
||||
struct device *dev = priv->dev;
|
||||
struct scatterlist *sgl;
|
||||
size_t i;
|
||||
int ret = 0;
|
||||
|
||||
for (i = 0; i < len; i++) {
|
||||
// Allocate memory for desc
|
||||
desc = kzalloc(sizeof(*desc), GFP_KERNEL);
|
||||
if (!desc) {
|
||||
ret = -ENOMEM;
|
||||
dev_err(dev, "Failed to allocate memory for descriptor\n");
|
||||
goto err_free_desc_init;
|
||||
}
|
||||
|
||||
desc->cyclic = cyclic;
|
||||
desc->segments = len;
|
||||
desc->direction = direction;
|
||||
INIT_LIST_HEAD(&desc->node);
|
||||
|
||||
// Allocate memory for hw_desc
|
||||
hw_desc = dma_alloc_coherent(dev, sizeof(*hw_desc),
|
||||
&desc->dma_handle, GFP_KERNEL);
|
||||
if (!hw_desc) {
|
||||
dev_err(dev, "Failed to allocate memory for hw descriptor\n");
|
||||
kfree(desc);
|
||||
ret = -ENOMEM;
|
||||
goto err_free_desc_init;
|
||||
}
|
||||
desc->hw_desc = hw_desc;
|
||||
|
||||
// Save head descriptor
|
||||
if (i == 0)
|
||||
dchan->head_desc = desc;
|
||||
|
||||
// Set the scatterlist pointer for sg mode. For cylic, sgl is NULL
|
||||
sgl = sg ? &sg[i] : NULL;
|
||||
// Configure the hw descriptor addresses and control
|
||||
efx_dma_hw_desc_address_control(dchan, desc, src_addr, dst_addr, period_len, sgl, i);
|
||||
|
||||
// Chain descriptor
|
||||
if (prev_desc)
|
||||
prev_desc->hw_desc->next = desc->dma_handle;
|
||||
|
||||
prev_desc = desc;
|
||||
|
||||
// Add to pending list
|
||||
spin_lock(&dchan->lock);
|
||||
list_add_tail(&desc->node, &dchan->pending_list);
|
||||
spin_unlock(&dchan->lock);
|
||||
}
|
||||
|
||||
// Debugging
|
||||
efx_dma_dump_pending_list(chan);
|
||||
return 0;
|
||||
|
||||
err_free_desc_init:
|
||||
efx_dma_free_hw_desc_chain(chan);
|
||||
return ret;
|
||||
|
||||
}
|
||||
|
||||
static struct dma_async_tx_descriptor *efx_dma_prep_slave_sg(
|
||||
struct dma_chan *chan,
|
||||
struct scatterlist *sg, unsigned int sg_len,
|
||||
enum dma_transfer_direction direction,
|
||||
unsigned long flags, void *context)
|
||||
{
|
||||
struct efx_dma_chan *dchan = to_efx_dma_chan(chan);
|
||||
struct device *dev = dchan->priv->dev;
|
||||
struct efx_dma_desc *desc = NULL;
|
||||
int ret;
|
||||
bool cyclic = false;
|
||||
|
||||
// Validate input
|
||||
if (unlikely(!chan || !sg || sg_len <= 0))
|
||||
return NULL;
|
||||
|
||||
// sg_len +1 to inlcude the last status of the descriptor
|
||||
sg_len += 1;
|
||||
|
||||
if (flags & DMA_CTRL_REUSE) {
|
||||
cyclic = true;
|
||||
sg_len -= 1; // Cyclic transfer does not require additional descriptor
|
||||
flags &= ~DMA_CTRL_REUSE;
|
||||
}
|
||||
|
||||
// Initialize hardware descriptor. Set cyclic to false
|
||||
ret = efx_dma_hw_desc_init(chan, 0, 0, sg, sg_len, 0, direction, cyclic);
|
||||
if (ret) {
|
||||
dev_err(dev, "Failed to initialize hardware descriptor\n");
|
||||
return NULL;
|
||||
}
|
||||
|
||||
// Get head descriptor
|
||||
desc = dchan->head_desc;
|
||||
if (!desc) {
|
||||
dev_err(dev, "No head descriptor found\n");
|
||||
return NULL;
|
||||
}
|
||||
|
||||
// Configure interrupt
|
||||
if (flags & DMA_PREP_INTERRUPT)
|
||||
efx_dma_interrupt_config(chan, EFX_DMA_CHANNEL_INTERRUPT_CHANNEL_COMPLETION_MASK);
|
||||
|
||||
// Prepare virtual DMA descriptor
|
||||
dchan->vchan.cyclic = &desc->vdesc;
|
||||
return vchan_tx_prep(&dchan->vchan, &desc->vdesc, flags);
|
||||
}
|
||||
|
||||
static struct dma_async_tx_descriptor *efx_dma_prep_cyclic(
|
||||
struct dma_chan *chan, dma_addr_t buf_addr, size_t buf_len,
|
||||
size_t period_len, enum dma_transfer_direction direction,
|
||||
unsigned long flags)
|
||||
{
|
||||
struct efx_dma_chan *dchan = to_efx_dma_chan(chan);
|
||||
struct device *dev = dchan->priv->dev;
|
||||
struct efx_dma_desc *desc = NULL;
|
||||
size_t segments = buf_len / period_len;
|
||||
int ret;
|
||||
|
||||
if (unlikely(!chan || !buf_addr || buf_len == 0 || period_len > buf_len))
|
||||
return NULL;
|
||||
|
||||
ret = efx_dma_hw_desc_init(chan, buf_addr, 0, NULL, segments, period_len, direction, true);
|
||||
if (ret)
|
||||
return NULL;
|
||||
|
||||
// Get head descriptor
|
||||
desc = dchan->head_desc;
|
||||
if (!desc) {
|
||||
dev_err(dev, "No head descriptor found\n");
|
||||
return NULL;
|
||||
}
|
||||
|
||||
dchan->vchan.cyclic = &desc->vdesc;
|
||||
return vchan_tx_prep(&dchan->vchan, &desc->vdesc, flags);
|
||||
}
|
||||
|
||||
static struct dma_async_tx_descriptor *efx_dma_prep_dma_memcpy(struct dma_chan *chan,
|
||||
dma_addr_t dst_addr, dma_addr_t src_addr, size_t len,
|
||||
unsigned long flags)
|
||||
{
|
||||
struct efx_dma_chan *dchan = to_efx_dma_chan(chan);
|
||||
struct efx_dma_desc *desc = NULL;
|
||||
struct device *dev = dchan->priv->dev;
|
||||
size_t segments;
|
||||
int ret;
|
||||
|
||||
// Validate input
|
||||
if (unlikely(!chan || !dst_addr || !src_addr || !len))
|
||||
return NULL;
|
||||
|
||||
if (len < PAGE_SIZE)
|
||||
segments = 1;
|
||||
else
|
||||
segments = (len / PAGE_SIZE);
|
||||
|
||||
// Increase segments by 1 to include the last status of the descriptor
|
||||
segments += 1;
|
||||
|
||||
ret = efx_dma_hw_desc_init(chan, src_addr, dst_addr, NULL, len, segments, DMA_MEM_TO_MEM, false);
|
||||
if (ret) {
|
||||
return NULL;
|
||||
}
|
||||
|
||||
// Get head descriptor
|
||||
desc = dchan->head_desc;
|
||||
if (!desc) {
|
||||
dev_err(dev, "No head descriptor found\n");
|
||||
return NULL;
|
||||
}
|
||||
|
||||
dchan->vchan.cyclic = &desc->vdesc;
|
||||
|
||||
return vchan_tx_prep(&dchan->vchan, &desc->vdesc, flags);
|
||||
}
|
||||
|
||||
|
||||
static void efx_dma_issue_pending(struct dma_chan *chan)
|
||||
{
|
||||
struct efx_dma_chan *dchan = to_efx_dma_chan(chan);
|
||||
unsigned long flags;
|
||||
|
||||
spin_lock_irqsave(&dchan->vchan.lock, flags);
|
||||
if (vchan_issue_pending(&dchan->vchan))
|
||||
efx_dma_start_transfer(chan);
|
||||
|
||||
spin_unlock_irqrestore(&dchan->vchan.lock, flags);
|
||||
}
|
||||
|
||||
static int efx_dma_device_config(struct dma_chan *chan,
|
||||
struct dma_slave_config *config)
|
||||
{
|
||||
struct efx_dma_chan *dchan = to_efx_dma_chan(chan);
|
||||
|
||||
dchan->cfg = config;
|
||||
return 0;
|
||||
}
|
||||
|
||||
static enum dma_status efx_dma_tx_status(struct dma_chan *chan, dma_cookie_t cookie,
|
||||
struct dma_tx_state *txstate)
|
||||
{
|
||||
if (efx_dma_busy(chan))
|
||||
return DMA_IN_PROGRESS;
|
||||
|
||||
if (cookie < DMA_MIN_COOKIE)
|
||||
return DMA_ERROR;
|
||||
|
||||
if (chan->completed_cookie == cookie)
|
||||
return DMA_COMPLETE;
|
||||
|
||||
return DMA_COMPLETE;
|
||||
}
|
||||
|
||||
static int efx_dma_terminate_all(struct dma_chan *chan)
|
||||
{
|
||||
struct efx_dma_chan *dchan = to_efx_dma_chan(chan);
|
||||
struct virt_dma_chan *vchan = &dchan->vchan;
|
||||
struct device *dev = dchan->priv->dev;
|
||||
|
||||
if (dchan) {
|
||||
// Stop all transfer of the DMA channels
|
||||
efx_dma_stop_channel(&dchan->vchan.chan);
|
||||
|
||||
// Disabled interrupt
|
||||
efx_dma_interrupt_config(chan, 0);
|
||||
|
||||
// Terminate all virtual DMA descriptors
|
||||
//vchan_free_chan_resources(vchan);
|
||||
|
||||
// Free all pending descriptors
|
||||
if (dchan->head_desc)
|
||||
efx_dma_free_hw_desc_chain(&dchan->vchan.chan);
|
||||
|
||||
// Clear cyclic pointer
|
||||
vchan->cyclic = NULL;
|
||||
dchan->head_desc = NULL;
|
||||
|
||||
dev_info(dev, "DMA channel %s terminated and cleaned up\n", dchan->name);
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static void efx_dma_release(struct dma_device *dev)
|
||||
{
|
||||
}
|
||||
|
||||
static int efx_dma_chan_probe(struct platform_device *pdev)
|
||||
{
|
||||
struct efx_dma_priv *priv = platform_get_drvdata(pdev);
|
||||
struct efx_dma_chan *dchan;
|
||||
struct device_node *node = pdev->dev.of_node;
|
||||
struct device_node *child = pdev->dev.of_node;
|
||||
size_t i = 0;
|
||||
int ret;
|
||||
|
||||
priv->dchan = devm_kzalloc(&pdev->dev, sizeof(struct efx_dma_chan) * priv->chan_count, GFP_KERNEL);
|
||||
if (!priv->dchan) {
|
||||
dev_err(&pdev->dev, "Failed to allocated memory for DMA channels\n");
|
||||
return -ENOMEM;
|
||||
}
|
||||
|
||||
for_each_child_of_node(node, child) {
|
||||
dchan = &priv->dchan[i];
|
||||
if (of_property_read_string(child, "dma-names", &dchan->name)) {
|
||||
dev_warn(&pdev->dev, "Failed to get dma-names for channel %zu\n", i);
|
||||
return -EINVAL;
|
||||
}
|
||||
|
||||
dev_info(&pdev->dev, "Initialize DMA channel %zu for %s\n", i, dchan->name);
|
||||
dchan->priv = priv;
|
||||
vchan_init(&dchan->vchan, &priv->dma_dev);
|
||||
dchan->chan_id = i;
|
||||
dchan->reg = priv->base + i * 0x80;
|
||||
dchan->cfg = NULL;
|
||||
dchan->vchan.desc_free = &efx_dma_desc_free;
|
||||
|
||||
dchan->irq = irq_of_parse_and_map(node, i);
|
||||
if (dchan->irq <= 0) {
|
||||
dev_warn(&pdev->dev, "Failed to get interrupt number\n");
|
||||
}
|
||||
|
||||
ret = devm_request_irq(&pdev->dev, dchan->irq, efx_dma_interrupt_handler,
|
||||
IRQF_SHARED, dchan->name, dchan);
|
||||
|
||||
if (ret) {
|
||||
dev_warn(&pdev->dev, "Warning: Failed to register interrupt handler\n");
|
||||
}
|
||||
|
||||
efx_dma_stop_channel(&dchan->vchan.chan);
|
||||
efx_dma_interrupt_pending_clear(&dchan->vchan.chan, 0xFFFFFFFF);
|
||||
|
||||
ret = of_property_read_u32(child, "chan-priority", &dchan->priority);
|
||||
if (ret) {
|
||||
dev_warn(&pdev->dev, "'chan-priority' is not found in the DMA device tree node. Use the default priority\n");
|
||||
dchan->priority = 0;
|
||||
}
|
||||
dev_info(&pdev->dev, " channel address %px, priority %u\n", dchan->reg, dchan->priority);
|
||||
efx_dma_set_channel_priority(dchan);
|
||||
|
||||
// initialize dchan->lock
|
||||
spin_lock_init(&dchan->lock);
|
||||
|
||||
INIT_LIST_HEAD(&dchan->pending_list);
|
||||
i++;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static struct dma_chan *of_dma_efx_dma_xlate(struct of_phandle_args *dma_spec,
|
||||
struct of_dma *ofdma)
|
||||
{
|
||||
struct efx_dma_priv *priv = ofdma->of_dma_data;
|
||||
struct efx_dma_chan *dchan;
|
||||
struct dma_chan *chan = NULL;
|
||||
size_t chan_id;
|
||||
|
||||
chan_id = dma_spec->args[0];
|
||||
if (chan_id < 0 || chan_id > priv->chan_count)
|
||||
return NULL;
|
||||
|
||||
dchan = &priv->dchan[chan_id];
|
||||
chan = dma_get_slave_channel(&dchan->vchan.chan);
|
||||
if (!chan) {
|
||||
dev_err(priv->dev, "Failed to get DMA slave channel %zu\n", chan_id);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
dev_info(priv->dev, "Found DMA slave '%s' at channel %zu\n", dchan->name, chan_id);
|
||||
return chan;
|
||||
}
|
||||
|
||||
static int efx_dma_probe(struct platform_device *pdev)
|
||||
{
|
||||
struct efx_dma_priv *priv;
|
||||
struct resource *res;
|
||||
struct device_node *node = pdev->dev.of_node;
|
||||
struct device_node *child = pdev->dev.of_node;
|
||||
int ret;
|
||||
|
||||
priv = devm_kzalloc(&pdev->dev, sizeof(*priv), GFP_KERNEL);
|
||||
if (!priv)
|
||||
return -ENOMEM;
|
||||
|
||||
priv->dev = &pdev->dev;
|
||||
res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
|
||||
priv->base = devm_ioremap_resource(&pdev->dev, res);
|
||||
if (IS_ERR(priv->base))
|
||||
return PTR_ERR(priv->base);
|
||||
|
||||
dma_cap_zero(priv->dma_dev.cap_mask);
|
||||
dma_cap_set(DMA_SLAVE, priv->dma_dev.cap_mask);
|
||||
dma_cap_set(DMA_CYCLIC, priv->dma_dev.cap_mask);
|
||||
dma_cap_set(DMA_MEMCPY, priv->dma_dev.cap_mask);
|
||||
|
||||
ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
|
||||
if (ret) {
|
||||
dev_warn(&pdev->dev, "Failed to set 64-bit DMA mask, falling back to 32-bit\n");
|
||||
ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
|
||||
if (ret) {
|
||||
dev_err(&pdev->dev, "Failed to set 32-bit DMA mask\n");
|
||||
return ret;
|
||||
}
|
||||
}
|
||||
|
||||
priv->dma_dev.src_addr_widths = DMA_SLAVE_BUSWIDTH_4_BYTES;
|
||||
priv->dma_dev.dst_addr_widths = DMA_SLAVE_BUSWIDTH_4_BYTES;
|
||||
priv->dma_dev.max_burst = 16;
|
||||
priv->dma_dev.directions = BIT(DMA_MEM_TO_DEV) | BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_MEM);
|
||||
|
||||
INIT_LIST_HEAD(&priv->dma_dev.channels);
|
||||
|
||||
// Initialize and register the DMA engine
|
||||
priv->dma_dev.dev = &pdev->dev;
|
||||
priv->dma_dev.device_alloc_chan_resources = efx_dma_alloc_chan_resources;
|
||||
priv->dma_dev.device_free_chan_resources = efx_dma_free_chan_resources;
|
||||
priv->dma_dev.device_prep_slave_sg = efx_dma_prep_slave_sg;
|
||||
priv->dma_dev.device_prep_dma_cyclic = efx_dma_prep_cyclic;
|
||||
priv->dma_dev.device_issue_pending = efx_dma_issue_pending;
|
||||
priv->dma_dev.device_config = efx_dma_device_config;
|
||||
priv->dma_dev.device_tx_status = efx_dma_tx_status;
|
||||
priv->dma_dev.device_prep_dma_memcpy = efx_dma_prep_dma_memcpy;
|
||||
priv->dma_dev.device_release = efx_dma_release;
|
||||
priv->dma_dev.device_terminate_all = efx_dma_terminate_all;
|
||||
|
||||
platform_set_drvdata(pdev, priv);
|
||||
|
||||
priv->chan_count = 0;
|
||||
ret = of_property_read_u32(node, "dma-channels", &priv->chan_count);
|
||||
if (ret) {
|
||||
dev_warn(&pdev->dev, "`dma-channels` not found in the DMA device tree node. Try to auto detect the number of channels\n");
|
||||
for_each_child_of_node(node, child) {
|
||||
priv->chan_count++;
|
||||
}
|
||||
}
|
||||
|
||||
dev_info(&pdev->dev, "Found %u DMA channels\n", priv->chan_count);
|
||||
|
||||
// Initialize DMA channels
|
||||
ret = efx_dma_chan_probe(pdev);
|
||||
if (ret)
|
||||
return ret;
|
||||
|
||||
dev_info(&pdev->dev, "Register DMA controller\n");
|
||||
ret = dma_async_device_register(&priv->dma_dev);
|
||||
if (ret) {
|
||||
dev_err(&pdev->dev, "Failed to register DMA controller\n");
|
||||
return ret;
|
||||
}
|
||||
|
||||
// Register DMA controller with device tree framework
|
||||
ret = of_dma_controller_register(node, of_dma_efx_dma_xlate, priv);
|
||||
if (ret < 0) {
|
||||
dev_err(&pdev->dev, "Unable to register DMA controller to DT\n");
|
||||
dma_async_device_unregister(&priv->dma_dev);
|
||||
return ret;
|
||||
}
|
||||
|
||||
dev_info(&pdev->dev, "DMA controller registered\n");
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int efx_dma_remove(struct platform_device *pdev)
|
||||
{
|
||||
struct efx_dma_priv *priv = platform_get_drvdata(pdev);
|
||||
|
||||
dma_async_device_unregister(&priv->dma_dev);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static const struct of_device_id efx_dma_of_match[] = {
|
||||
{ .compatible = "efx,dma-controller" },
|
||||
{ },
|
||||
};
|
||||
MODULE_DEVICE_TABLE(of, efx_dma_of_match);
|
||||
|
||||
static struct platform_driver efx_dma_driver = {
|
||||
.probe = efx_dma_probe,
|
||||
.remove = efx_dma_remove,
|
||||
.driver = {
|
||||
.name = "efx-dma",
|
||||
.of_match_table = efx_dma_of_match,
|
||||
},
|
||||
};
|
||||
|
||||
module_platform_driver(efx_dma_driver);
|
||||
|
||||
MODULE_AUTHOR("Alim Hussin <mnalim@efinixinc.com>");
|
||||
MODULE_DESCRIPTION("Efinix DMA driver");
|
||||
MODULE_LICENSE("GPL v2");
|
||||
Reference in New Issue
Block a user