#include #include #include #include #include #include #include #include #include #include #include #include "dmaengine.h" #include "virt-dma.h" #define EFX_DMA_CHANNEL_INPUT_ADDRESS 0x00 #define EFX_DMA_CHANNEL_INPUT_STREAM 0x08 #define EFX_DMA_CHANNEL_INPUT_CONFIG 0x0c #define EFX_DMA_CHANNEL_INPUT_CONFIG_MEMORY (1 << 12) #define EFX_DMA_CHANNEL_INPUT_CONFIG_STREAM 0x0 #define EFX_DMA_CHANNEL_INPUT_CONFIG_COMPLETION_ON_PACKET (1 << 13) #define EFX_DMA_CHANNEL_INPUT_CONFIG_WAIT_ON_PACKET (1 << 14) #define EFX_DMA_CHANNEL_OUTPUT_ADDRESS 0x10 #define EFX_DMA_CHANNEL_OUTPUT_STREAM 0x18 #define EFX_DMA_CHANNEL_OUTPUT_CONFIG 0x1c #define EFX_DMA_CHANNEL_OUTPUT_CONFIG_MEMORY (1 << 12) #define EFX_DMA_CHANNEL_OUTPUT_CONFIG_STREAM 0x0 #define EFX_DMA_CHANNEL_OUTPUT_CONFIG_LAST (1 << 13) #define EFX_DMA_CHANNEL_DIRECT_BYTES 0x20 #define EFX_DMA_CHANNEL_STATUS 0x2c #define EFX_DMA_CHANNEL_STATUS_DIRECT_START (1 << 0) #define EFX_DMA_CHANNEL_STATUS_BUSY (1 << 0) #define EFX_DMA_CHANNEL_STATUS_SELF_RESTART (1 << 1) #define EFX_DMA_CHANNEL_STATUS_STOP (1 << 2) #define EFX_DMA_CHANNEL_STATUS_LINKED_LIST_START (1 << 4) #define EFX_DMA_CHANNEL_FIFO 0x40 #define EFX_DMA_CHANNEL_PRIORITY 0x44 #define EFX_DMA_CHANNEL_INTERRUPT_ENABLE 0x50 #define EFX_DMA_CHANNEL_INTERRUPT_PENDING 0x54 // Interrupt at the end of each descriptor #define EFX_DMA_CHANNEL_INTERRUPT_DESCRIPTOR_COMPLETION_MASK (1 << 0) // Interrupt at the middle of each descriptor, require the half_completion_interrpt // option to be enabled for the channel #define EFX_DMA_CHANNEL_INTERRUPT_DESCRIPTOR_COMPLETION_HALF_MASK (1 << 1) // Interrupt when the channel is going off (not busy anymore) #define EFX_DMA_CHANNEL_INTERRUPT_CHANNEL_COMPLETION_MASK (1 << 2) // Interrupt each time that a linked list's descriptor stats field is updated #define EFX_DMA_CHANNEL_INTERRUPT_LINKED_LIST_UPDATE_MASK (1 << 3) // Interrupt each time a S -> M channel has done transferring a packet into the memory #define EFX_DMA_CHANNEL_INTERRUPT_INPUT_PACKET_MASK (1 << 4) #define EFX_DMA_CHANNEL_PROGRESS_BYTES 0x60 #define EFX_DMA_CHANNEL_LINKED_LIST_HEAD 0x70 #define EFX_DMA_CHANNEL_LINKED_LIST_FROM_SG_BUS 0x78 #define EFX_DMA_DESCRIPTOR_CONTROL_BYTES 0x7FFFFFF #define EFX_DMA_DESCRIPTOR_CONTROL_END_OF_PACKET (1 << 30) #define EFX_DMA_DESCRIPTOR_NO_COMPLETION (1 << 31) #define EFX_DMA_DESCRIPTOR_STATUS_BYTES 0x7FFFFFF #define EFX_DMA_DESCRIPTOR_STATUS_END_OF_PACKET (1 << 30) #define EFX_DMA_DESCRIPTOR_STATUS_COMPLETED (1 << 31) // DMA Hardware descriptor struct efx_dma_hw_desc { u32 status; u32 control; u64 src_addr; u64 dst_addr; u64 next; // physical address of next descriptor } __aligned(64); // Per transfer descriptor struct efx_dma_desc { struct virt_dma_desc vdesc; struct efx_dma_hw_desc *hw_desc; // Pointer to the struct efx_dma_hw_desc dma_addr_t dma_handle; // DMA address of struct efx_dma_hw_desc struct list_head node; // List of struct efx_dma_hw_desc size_t segments; // Number of DMA descriptor bool cyclic; // True for cyclic transfer enum dma_transfer_direction direction; struct scatterlist *sg; struct page **pages; }; // DMA channel specific data struct efx_dma_chan { const char *name; // Channel name void __iomem *reg; // based address of DMA channel size_t chan_id; // Channel ID struct virt_dma_chan vchan; struct efx_dma_priv *priv; // Pointer to the DMA controller struct dma_slave_config *cfg; // Channel specific configuration u32 priority; // Priority number of DMA channel size_t irq; // IRQ number used by the DMA channel struct efx_dma_desc *head_desc; // First DMA descriptor struct list_head pending_list; // List of all DMA descriptor spinlock_t lock; // Lock for manipulating pending_list }; // DMA controller private data struct efx_dma_priv { struct device *dev; void __iomem *base; // DMA controller based address struct dma_device dma_dev; struct efx_dma_chan *dchan; // Pointer to the DMA channel u32 chan_count; // Number of DMA channel }; static inline struct efx_dma_chan *to_efx_dma_chan(struct dma_chan *chan) { return container_of(chan, struct efx_dma_chan, vchan.chan); } static inline struct efx_dma_priv *to_efx_dma_priv(struct dma_chan *chan) { struct efx_dma_chan *dchan = to_efx_dma_chan(chan); return dchan->priv; } static void efx_dma_input_memory(struct dma_chan *chan) { struct efx_dma_chan *dchan = to_efx_dma_chan(chan); struct efx_dma_hw_desc *hw_desc = dchan->head_desc->hw_desc; u32 byte_per_burst = 0; if (dchan->cfg) byte_per_burst = dchan->cfg->src_maxburst; else byte_per_burst = chan->device->max_burst; iowrite32(hw_desc->src_addr, dchan->reg + EFX_DMA_CHANNEL_INPUT_ADDRESS); iowrite32(EFX_DMA_CHANNEL_INPUT_CONFIG_MEMORY | ((byte_per_burst - 1) & 0xFFF), dchan->reg + EFX_DMA_CHANNEL_INPUT_CONFIG); } static void efx_dma_output_memory(struct dma_chan *chan) { struct efx_dma_chan *dchan = to_efx_dma_chan(chan); struct efx_dma_hw_desc *hw_desc = dchan->head_desc->hw_desc; u32 len = hw_desc->control & 0x1FFFFFF; iowrite32(hw_desc->dst_addr, dchan->reg + EFX_DMA_CHANNEL_OUTPUT_ADDRESS); iowrite32(EFX_DMA_CHANNEL_OUTPUT_CONFIG_MEMORY | ((len - 1) & 0xFFF), dchan->reg + EFX_DMA_CHANNEL_OUTPUT_CONFIG); } static void efx_dma_input_stream(struct dma_chan *chan, u32 wait_on_packet, u32 completion_on_packet) { struct efx_dma_chan *dchan = to_efx_dma_chan(chan); completion_on_packet = completion_on_packet ? EFX_DMA_CHANNEL_INPUT_CONFIG_COMPLETION_ON_PACKET : 0; wait_on_packet = wait_on_packet ? EFX_DMA_CHANNEL_INPUT_CONFIG_WAIT_ON_PACKET : 0; iowrite32(0, dchan->reg + EFX_DMA_CHANNEL_INPUT_STREAM); iowrite32(EFX_DMA_CHANNEL_INPUT_CONFIG_STREAM | completion_on_packet | wait_on_packet, dchan->reg + EFX_DMA_CHANNEL_INPUT_CONFIG); } static void efx_dma_output_stream(struct dma_chan *chan) { struct efx_dma_chan *dchan = to_efx_dma_chan(chan); iowrite32(0, dchan->reg + EFX_DMA_CHANNEL_OUTPUT_STREAM); iowrite32(EFX_DMA_CHANNEL_OUTPUT_CONFIG_LAST | EFX_DMA_CHANNEL_OUTPUT_CONFIG_STREAM, dchan->reg + EFX_DMA_CHANNEL_OUTPUT_CONFIG); } static void efx_dma_linked_list_start(struct dma_chan *chan) { struct efx_dma_chan *dchan = to_efx_dma_chan(chan); iowrite32(dchan->head_desc->dma_handle, dchan->reg + EFX_DMA_CHANNEL_LINKED_LIST_HEAD); iowrite32(0, dchan->reg + EFX_DMA_CHANNEL_LINKED_LIST_FROM_SG_BUS); iowrite32(EFX_DMA_CHANNEL_STATUS_LINKED_LIST_START, dchan->reg + EFX_DMA_CHANNEL_STATUS); } static void efx_dma_stop_channel(struct dma_chan *chan) { struct efx_dma_chan *dchan = to_efx_dma_chan(chan); iowrite32(EFX_DMA_CHANNEL_STATUS_STOP, dchan->reg + EFX_DMA_CHANNEL_STATUS); } static bool efx_dma_busy(struct dma_chan *chan) { struct efx_dma_chan *dchan = to_efx_dma_chan(chan); u32 busy; busy = readl(dchan->reg + EFX_DMA_CHANNEL_STATUS) & EFX_DMA_CHANNEL_STATUS_BUSY; return busy ? true : false; } static inline void efx_dma_interrupt_pending_clear(struct dma_chan *chan, u32 mask) { struct efx_dma_chan *dchan = to_efx_dma_chan(chan); iowrite32(mask, dchan->reg + EFX_DMA_CHANNEL_INTERRUPT_PENDING); } static void efx_dma_interrupt_config(struct dma_chan *chan, u32 mask) { struct efx_dma_chan *dchan = to_efx_dma_chan(chan); efx_dma_interrupt_pending_clear(chan, 0xFFFFFFFF); iowrite32(mask, dchan->reg + EFX_DMA_CHANNEL_INTERRUPT_ENABLE); } static void efx_dma_set_channel_priority(struct efx_dma_chan *dchan) { iowrite32(dchan->priority, dchan->reg + EFX_DMA_CHANNEL_PRIORITY); } static void efx_dma_configure_registers(struct dma_chan *chan) { struct efx_dma_chan *dchan = to_efx_dma_chan(chan); struct efx_dma_desc *desc = dchan->head_desc; if (desc->direction == DMA_DEV_TO_MEM) { efx_dma_output_memory(chan); efx_dma_input_stream(chan, 1, 0); } else if (desc->direction == DMA_MEM_TO_DEV) { efx_dma_input_memory(chan); efx_dma_output_stream(chan); } else if (desc->direction == DMA_MEM_TO_MEM) { efx_dma_input_memory(chan); efx_dma_output_memory(chan); } } static void efx_dma_start_transfer(struct dma_chan *chan) { struct efx_dma_chan *dchan = to_efx_dma_chan(chan); struct virt_dma_desc *vdesc; vdesc = vchan_next_desc(&dchan->vchan); if (!vdesc) return; efx_dma_configure_registers(chan); efx_dma_linked_list_start(chan); } static irqreturn_t efx_dma_interrupt_handler(int irq, void *dev_id) { struct efx_dma_chan *dchan = (struct efx_dma_chan *)dev_id; struct efx_dma_desc *desc = dchan->head_desc; struct dma_chan *chan = &dchan->vchan.chan; unsigned long flags; u32 pending; spin_lock_irqsave(&dchan->vchan.lock, flags); pending = readl(dchan->reg + EFX_DMA_CHANNEL_INTERRUPT_PENDING); if (pending & EFX_DMA_CHANNEL_INTERRUPT_DESCRIPTOR_COMPLETION_MASK) efx_dma_interrupt_pending_clear(chan, EFX_DMA_CHANNEL_INTERRUPT_DESCRIPTOR_COMPLETION_MASK); if (pending & EFX_DMA_CHANNEL_INTERRUPT_CHANNEL_COMPLETION_MASK) efx_dma_interrupt_pending_clear(chan, EFX_DMA_CHANNEL_INTERRUPT_CHANNEL_COMPLETION_MASK); vchan_cookie_complete(&desc->vdesc); // Unmask the interrupt efx_dma_interrupt_pending_clear(chan, 0x0); pending = readl(dchan->reg + EFX_DMA_CHANNEL_INTERRUPT_PENDING); spin_unlock_irqrestore(&dchan->vchan.lock, flags); return IRQ_HANDLED; } static int efx_dma_alloc_chan_resources(struct dma_chan *chan) { return 0; } static void efx_dma_free_chan_resources(struct dma_chan *chan) { } static void efx_dma_dump_pending_list(struct dma_chan *chan) { struct efx_dma_priv *priv = to_efx_dma_priv(chan); struct efx_dma_chan *dchan = to_efx_dma_chan(chan); struct efx_dma_desc *desc; struct device *dev = priv->dev; int i = 0, count = 0; int limit = 5; // Count number of descriptor in the pending_list spin_lock(&dchan->lock); list_for_each_entry(desc, &dchan->pending_list, node) count++; spin_unlock(&dchan->lock); if (count == 0) { dev_dbg(dev, "Pending list is empty\n"); return; } dev_dbg(dev, "Pending list has %d descriptors\n", count); // Print first 5 and last 5 descriptors spin_lock(&dchan->lock); list_for_each_entry(desc, &dchan->pending_list, node) { if (i < limit || i >= count - limit) { pr_debug("%s: Descriptor %d:\n", __func__, i); pr_debug(" desc=%p, hw_desc=%p, dma_handle=0x%llx\n", desc, desc->hw_desc, (unsigned long long)desc->dma_handle); if (desc->hw_desc) { pr_debug(" status=0x%x, control=0x%x\n", desc->hw_desc->status, desc->hw_desc->control); pr_debug(" src_addr=0x%llx, dst_addr=0x%llx, next=0x%llx\n", (unsigned long long)desc->hw_desc->src_addr, (unsigned long long)desc->hw_desc->dst_addr, (unsigned long long)desc->hw_desc->next); } pr_debug(" cyclic=%d, direction=%d, segments=%zu\n", desc->cyclic, desc->direction, desc->segments); } i++; } spin_unlock(&dchan->lock); } static void efx_dma_free_hw_desc_chain(struct dma_chan *chan) { struct efx_dma_priv *priv = to_efx_dma_priv(chan); struct efx_dma_chan *dchan = to_efx_dma_chan(chan); struct efx_dma_desc *desc, *tmp; struct device *dev = priv->dev; int count = 0; int limit = 5, i = 0; // Count number of descriptor in the pending_list spin_lock(&dchan->lock); list_for_each_entry(desc, &dchan->pending_list, node) count++; spin_unlock(&dchan->lock); if (count == 0) { dev_info(dev, "Pending list is empty\n"); return; } dev_info(dev, "Freeing %d descriptors\n", count); spin_lock(&dchan->lock); list_for_each_entry_safe(desc, tmp, &dchan->pending_list, node) { list_del(&desc->node); if (desc->hw_desc) { dma_free_coherent(dev, sizeof(struct efx_dma_hw_desc), desc->hw_desc, desc->dma_handle); if (i < limit || i >= count - limit) { pr_debug(" Freeing descriptor: %d\n", i); pr_debug(" free hw_desc=%p, dma_handle=0x%llx\n",desc->hw_desc, (unsigned long long)desc->dma_handle); } } kfree(desc); if (i < limit || i >= count - limit) pr_debug(" desc=%p\n",desc); i++; } dchan->head_desc = NULL; spin_unlock(&dchan->lock); } static void efx_dma_desc_free(struct virt_dma_desc *vd) { struct dma_chan *chan = vd->tx.chan; efx_dma_free_hw_desc_chain(chan); } static void efx_dma_hw_desc_address_control(struct efx_dma_chan *dchan, struct efx_dma_desc *desc, dma_addr_t src_addr, dma_addr_t dst_addr, size_t period_len, struct scatterlist *sg, size_t i) { struct efx_dma_hw_desc *hw_desc = desc->hw_desc; dma_addr_t segment_addr; u32 buf_len; u32 control = desc->cyclic ? EFX_DMA_DESCRIPTOR_NO_COMPLETION : EFX_DMA_DESCRIPTOR_CONTROL_END_OF_PACKET; hw_desc->status = 0; if (sg) { buf_len = sg_dma_len(sg); segment_addr = sg_dma_address(sg); } else { buf_len = period_len - 1; segment_addr = src_addr + i * period_len; } hw_desc->control = (u32)(control | buf_len); // Configure last hw_desc if (i == (desc->segments - 1)) { if (desc->cyclic) { hw_desc->next = dchan->head_desc->dma_handle; } else { hw_desc->status = EFX_DMA_DESCRIPTOR_STATUS_COMPLETED; } } if (desc->direction == DMA_DEV_TO_MEM) { hw_desc->src_addr = 0; hw_desc->dst_addr = segment_addr; } else if (desc->direction == DMA_MEM_TO_DEV) { hw_desc->src_addr = segment_addr; hw_desc->dst_addr = 0; } else if (desc->direction == DMA_MEM_TO_MEM) { hw_desc->src_addr = src_addr + i * period_len; hw_desc->dst_addr = dst_addr + i * period_len; } } static int efx_dma_hw_desc_init(struct dma_chan *chan, dma_addr_t src_addr, dma_addr_t dst_addr, struct scatterlist *sg, size_t len, size_t period_len, enum dma_transfer_direction direction, bool cyclic) { struct efx_dma_priv *priv = to_efx_dma_priv(chan); struct efx_dma_chan *dchan = to_efx_dma_chan(chan); struct efx_dma_desc *desc = NULL, *prev_desc = NULL; struct efx_dma_hw_desc *hw_desc = NULL; struct device *dev = priv->dev; struct scatterlist *sgl; size_t i; int ret = 0; for (i = 0; i < len; i++) { // Allocate memory for desc desc = kzalloc(sizeof(*desc), GFP_KERNEL); if (!desc) { ret = -ENOMEM; dev_err(dev, "Failed to allocate memory for descriptor\n"); goto err_free_desc_init; } desc->cyclic = cyclic; desc->segments = len; desc->direction = direction; INIT_LIST_HEAD(&desc->node); // Allocate memory for hw_desc hw_desc = dma_alloc_coherent(dev, sizeof(*hw_desc), &desc->dma_handle, GFP_KERNEL); if (!hw_desc) { dev_err(dev, "Failed to allocate memory for hw descriptor\n"); kfree(desc); ret = -ENOMEM; goto err_free_desc_init; } desc->hw_desc = hw_desc; // Save head descriptor if (i == 0) dchan->head_desc = desc; // Set the scatterlist pointer for sg mode. For cylic, sgl is NULL sgl = sg ? &sg[i] : NULL; // Configure the hw descriptor addresses and control efx_dma_hw_desc_address_control(dchan, desc, src_addr, dst_addr, period_len, sgl, i); // Chain descriptor if (prev_desc) prev_desc->hw_desc->next = desc->dma_handle; prev_desc = desc; // Add to pending list spin_lock(&dchan->lock); list_add_tail(&desc->node, &dchan->pending_list); spin_unlock(&dchan->lock); } // Debugging efx_dma_dump_pending_list(chan); return 0; err_free_desc_init: efx_dma_free_hw_desc_chain(chan); return ret; } static struct dma_async_tx_descriptor *efx_dma_prep_slave_sg( struct dma_chan *chan, struct scatterlist *sg, unsigned int sg_len, enum dma_transfer_direction direction, unsigned long flags, void *context) { struct efx_dma_chan *dchan = to_efx_dma_chan(chan); struct device *dev = dchan->priv->dev; struct efx_dma_desc *desc = NULL; int ret; bool cyclic = false; // Validate input if (unlikely(!chan || !sg || sg_len <= 0)) return NULL; // sg_len +1 to inlcude the last status of the descriptor sg_len += 1; if (flags & DMA_CTRL_REUSE) { cyclic = true; sg_len -= 1; // Cyclic transfer does not require additional descriptor flags &= ~DMA_CTRL_REUSE; } // Initialize hardware descriptor. Set cyclic to false ret = efx_dma_hw_desc_init(chan, 0, 0, sg, sg_len, 0, direction, cyclic); if (ret) { dev_err(dev, "Failed to initialize hardware descriptor\n"); return NULL; } // Get head descriptor desc = dchan->head_desc; if (!desc) { dev_err(dev, "No head descriptor found\n"); return NULL; } // Configure interrupt if (flags & DMA_PREP_INTERRUPT) efx_dma_interrupt_config(chan, EFX_DMA_CHANNEL_INTERRUPT_CHANNEL_COMPLETION_MASK); // Prepare virtual DMA descriptor dchan->vchan.cyclic = &desc->vdesc; return vchan_tx_prep(&dchan->vchan, &desc->vdesc, flags); } static struct dma_async_tx_descriptor *efx_dma_prep_cyclic( struct dma_chan *chan, dma_addr_t buf_addr, size_t buf_len, size_t period_len, enum dma_transfer_direction direction, unsigned long flags) { struct efx_dma_chan *dchan = to_efx_dma_chan(chan); struct device *dev = dchan->priv->dev; struct efx_dma_desc *desc = NULL; size_t segments = buf_len / period_len; int ret; if (unlikely(!chan || !buf_addr || buf_len == 0 || period_len > buf_len)) return NULL; ret = efx_dma_hw_desc_init(chan, buf_addr, 0, NULL, segments, period_len, direction, true); if (ret) return NULL; // Get head descriptor desc = dchan->head_desc; if (!desc) { dev_err(dev, "No head descriptor found\n"); return NULL; } dchan->vchan.cyclic = &desc->vdesc; return vchan_tx_prep(&dchan->vchan, &desc->vdesc, flags); } static struct dma_async_tx_descriptor *efx_dma_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dst_addr, dma_addr_t src_addr, size_t len, unsigned long flags) { struct efx_dma_chan *dchan = to_efx_dma_chan(chan); struct efx_dma_desc *desc = NULL; struct device *dev = dchan->priv->dev; size_t segments; int ret; // Validate input if (unlikely(!chan || !dst_addr || !src_addr || !len)) return NULL; if (len < PAGE_SIZE) segments = 1; else segments = (len / PAGE_SIZE); // Increase segments by 1 to include the last status of the descriptor segments += 1; ret = efx_dma_hw_desc_init(chan, src_addr, dst_addr, NULL, len, segments, DMA_MEM_TO_MEM, false); if (ret) { return NULL; } // Get head descriptor desc = dchan->head_desc; if (!desc) { dev_err(dev, "No head descriptor found\n"); return NULL; } dchan->vchan.cyclic = &desc->vdesc; return vchan_tx_prep(&dchan->vchan, &desc->vdesc, flags); } static void efx_dma_issue_pending(struct dma_chan *chan) { struct efx_dma_chan *dchan = to_efx_dma_chan(chan); unsigned long flags; spin_lock_irqsave(&dchan->vchan.lock, flags); if (vchan_issue_pending(&dchan->vchan)) efx_dma_start_transfer(chan); spin_unlock_irqrestore(&dchan->vchan.lock, flags); } static int efx_dma_device_config(struct dma_chan *chan, struct dma_slave_config *config) { struct efx_dma_chan *dchan = to_efx_dma_chan(chan); dchan->cfg = config; return 0; } static enum dma_status efx_dma_tx_status(struct dma_chan *chan, dma_cookie_t cookie, struct dma_tx_state *txstate) { if (efx_dma_busy(chan)) return DMA_IN_PROGRESS; if (cookie < DMA_MIN_COOKIE) return DMA_ERROR; if (chan->completed_cookie == cookie) return DMA_COMPLETE; return DMA_COMPLETE; } static int efx_dma_terminate_all(struct dma_chan *chan) { struct efx_dma_chan *dchan = to_efx_dma_chan(chan); struct virt_dma_chan *vchan = &dchan->vchan; struct device *dev = dchan->priv->dev; if (dchan) { // Stop all transfer of the DMA channels efx_dma_stop_channel(&dchan->vchan.chan); // Disabled interrupt efx_dma_interrupt_config(chan, 0); // Terminate all virtual DMA descriptors //vchan_free_chan_resources(vchan); // Free all pending descriptors if (dchan->head_desc) efx_dma_free_hw_desc_chain(&dchan->vchan.chan); // Clear cyclic pointer vchan->cyclic = NULL; dchan->head_desc = NULL; dev_info(dev, "DMA channel %s terminated and cleaned up\n", dchan->name); } return 0; } static void efx_dma_release(struct dma_device *dev) { } static int efx_dma_chan_probe(struct platform_device *pdev) { struct efx_dma_priv *priv = platform_get_drvdata(pdev); struct efx_dma_chan *dchan; struct device_node *node = pdev->dev.of_node; struct device_node *child = pdev->dev.of_node; size_t i = 0; int ret; priv->dchan = devm_kzalloc(&pdev->dev, sizeof(struct efx_dma_chan) * priv->chan_count, GFP_KERNEL); if (!priv->dchan) { dev_err(&pdev->dev, "Failed to allocated memory for DMA channels\n"); return -ENOMEM; } for_each_child_of_node(node, child) { dchan = &priv->dchan[i]; if (of_property_read_string(child, "dma-names", &dchan->name)) { dev_warn(&pdev->dev, "Failed to get dma-names for channel %zu\n", i); return -EINVAL; } dev_info(&pdev->dev, "Initialize DMA channel %zu for %s\n", i, dchan->name); dchan->priv = priv; vchan_init(&dchan->vchan, &priv->dma_dev); dchan->chan_id = i; dchan->reg = priv->base + i * 0x80; dchan->cfg = NULL; dchan->vchan.desc_free = &efx_dma_desc_free; dchan->irq = irq_of_parse_and_map(node, i); if (dchan->irq <= 0) { dev_warn(&pdev->dev, "Failed to get interrupt number\n"); } ret = devm_request_irq(&pdev->dev, dchan->irq, efx_dma_interrupt_handler, IRQF_SHARED, dchan->name, dchan); if (ret) { dev_warn(&pdev->dev, "Warning: Failed to register interrupt handler\n"); } efx_dma_stop_channel(&dchan->vchan.chan); efx_dma_interrupt_pending_clear(&dchan->vchan.chan, 0xFFFFFFFF); ret = of_property_read_u32(child, "chan-priority", &dchan->priority); if (ret) { dev_warn(&pdev->dev, "'chan-priority' is not found in the DMA device tree node. Use the default priority\n"); dchan->priority = 0; } dev_info(&pdev->dev, " channel address %px, priority %u\n", dchan->reg, dchan->priority); efx_dma_set_channel_priority(dchan); // initialize dchan->lock spin_lock_init(&dchan->lock); INIT_LIST_HEAD(&dchan->pending_list); i++; } return 0; } static struct dma_chan *of_dma_efx_dma_xlate(struct of_phandle_args *dma_spec, struct of_dma *ofdma) { struct efx_dma_priv *priv = ofdma->of_dma_data; struct efx_dma_chan *dchan; struct dma_chan *chan = NULL; size_t chan_id; chan_id = dma_spec->args[0]; if (chan_id < 0 || chan_id > priv->chan_count) return NULL; dchan = &priv->dchan[chan_id]; chan = dma_get_slave_channel(&dchan->vchan.chan); if (!chan) { dev_err(priv->dev, "Failed to get DMA slave channel %zu\n", chan_id); return NULL; } dev_info(priv->dev, "Found DMA slave '%s' at channel %zu\n", dchan->name, chan_id); return chan; } static int efx_dma_probe(struct platform_device *pdev) { struct efx_dma_priv *priv; struct resource *res; struct device_node *node = pdev->dev.of_node; struct device_node *child = pdev->dev.of_node; int ret; priv = devm_kzalloc(&pdev->dev, sizeof(*priv), GFP_KERNEL); if (!priv) return -ENOMEM; priv->dev = &pdev->dev; res = platform_get_resource(pdev, IORESOURCE_MEM, 0); priv->base = devm_ioremap_resource(&pdev->dev, res); if (IS_ERR(priv->base)) return PTR_ERR(priv->base); dma_cap_zero(priv->dma_dev.cap_mask); dma_cap_set(DMA_SLAVE, priv->dma_dev.cap_mask); dma_cap_set(DMA_CYCLIC, priv->dma_dev.cap_mask); dma_cap_set(DMA_MEMCPY, priv->dma_dev.cap_mask); ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); if (ret) { dev_warn(&pdev->dev, "Failed to set 64-bit DMA mask, falling back to 32-bit\n"); ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)); if (ret) { dev_err(&pdev->dev, "Failed to set 32-bit DMA mask\n"); return ret; } } priv->dma_dev.src_addr_widths = DMA_SLAVE_BUSWIDTH_4_BYTES; priv->dma_dev.dst_addr_widths = DMA_SLAVE_BUSWIDTH_4_BYTES; priv->dma_dev.max_burst = 16; priv->dma_dev.directions = BIT(DMA_MEM_TO_DEV) | BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_MEM); INIT_LIST_HEAD(&priv->dma_dev.channels); // Initialize and register the DMA engine priv->dma_dev.dev = &pdev->dev; priv->dma_dev.device_alloc_chan_resources = efx_dma_alloc_chan_resources; priv->dma_dev.device_free_chan_resources = efx_dma_free_chan_resources; priv->dma_dev.device_prep_slave_sg = efx_dma_prep_slave_sg; priv->dma_dev.device_prep_dma_cyclic = efx_dma_prep_cyclic; priv->dma_dev.device_issue_pending = efx_dma_issue_pending; priv->dma_dev.device_config = efx_dma_device_config; priv->dma_dev.device_tx_status = efx_dma_tx_status; priv->dma_dev.device_prep_dma_memcpy = efx_dma_prep_dma_memcpy; priv->dma_dev.device_release = efx_dma_release; priv->dma_dev.device_terminate_all = efx_dma_terminate_all; platform_set_drvdata(pdev, priv); priv->chan_count = 0; ret = of_property_read_u32(node, "dma-channels", &priv->chan_count); if (ret) { dev_warn(&pdev->dev, "`dma-channels` not found in the DMA device tree node. Try to auto detect the number of channels\n"); for_each_child_of_node(node, child) { priv->chan_count++; } } dev_info(&pdev->dev, "Found %u DMA channels\n", priv->chan_count); // Initialize DMA channels ret = efx_dma_chan_probe(pdev); if (ret) return ret; dev_info(&pdev->dev, "Register DMA controller\n"); ret = dma_async_device_register(&priv->dma_dev); if (ret) { dev_err(&pdev->dev, "Failed to register DMA controller\n"); return ret; } // Register DMA controller with device tree framework ret = of_dma_controller_register(node, of_dma_efx_dma_xlate, priv); if (ret < 0) { dev_err(&pdev->dev, "Unable to register DMA controller to DT\n"); dma_async_device_unregister(&priv->dma_dev); return ret; } dev_info(&pdev->dev, "DMA controller registered\n"); return 0; } static int efx_dma_remove(struct platform_device *pdev) { struct efx_dma_priv *priv = platform_get_drvdata(pdev); dma_async_device_unregister(&priv->dma_dev); return 0; } static const struct of_device_id efx_dma_of_match[] = { { .compatible = "efx,dma-controller" }, { }, }; MODULE_DEVICE_TABLE(of, efx_dma_of_match); static struct platform_driver efx_dma_driver = { .probe = efx_dma_probe, .remove = efx_dma_remove, .driver = { .name = "efx-dma", .of_match_table = efx_dma_of_match, }, }; module_platform_driver(efx_dma_driver); MODULE_AUTHOR("Alim Hussin "); MODULE_DESCRIPTION("Efinix DMA driver"); MODULE_LICENSE("GPL v2");