diff --git a/src/cndm/dpdk/README.md b/src/cndm/dpdk/README.md new file mode 100644 index 0000000..0165445 --- /dev/null +++ b/src/cndm/dpdk/README.md @@ -0,0 +1,39 @@ +# DPDK PMD for Corundum + +## Introduction + +This is an out-of-tree DPDK PMD for corundum-micro and corundum-lite. It has feature parity with the main Linux kernel driver, at least in terms of the core datapath. Some features may be limited by the architecture and API of DPDK. This PMD has been tested with DPDK 26.03, but will likely work with other versions as well. + +## How to build + +- Download DPDK in a separate directory +- Create a symbolic link from `taxi/src/cndm/dpdk/cndm` to `dpdk/drivers/net/cndm` +- Edit `dpdk/drivers/net/meson.build` to add `cndm` to the drivers list +- In the root of the DPDK repo, run meson + - Default: `meson setup build` + - Build all examples: `meson setup -Dexamples=all build` + - Forced reconfigure: `meson setup --wipe build` + - Check the meson summary - some system packages may need to be installed +- In the root of the DPDK repo, run `ninja -C build` + +## How to test + +- It may be necessary to enable huge pages on the kernel command line + - For example, `default_hugepagesz=1G hugepagesz=1G hugepages=16` + - This must be done in the bootloader configuration (grub, systemd-boot, etc.) +- Run `sudo ./usertools/dpdk-devbind.py -s` to determine NIC PCI ID and NUMA node +- Run `sudo ./usertools/dpdk-devbind.py -b vfio-pci ` to bind the vfio-pci driver to the NIC + - It may be necessary to add `--noimmu-mode` if the IOMMU is disabled +- Run `numactl -H` to determine which CPU cores are on the same NUMA node as the NIC +- Run `dpdk-testpmd` + - `sudo ./build/app/dpdk-testpmd -l -- -i --portlist=` + - cpulist specifies the CPU cores to use (e.g. `-l 0-3`) + - portlist specifies the ports to use (e.g. `--portlist=0,2`) +- At the `dpdk-testpmd` command line: + - Run `start tx_first` to start a simple forwarding test, with the first port in the list acting as the transmitter + - Run `stop` to stop the test and print statistics + - Run `show port info 0` to display port configuration information + - Run `show port 0 eeprom` to dump the board EEPROM + - Run `show port 0 module_eeprom` to dump the module EEPROM + - Use the `help` command to get more information about the testpmd commands + - Run `quit` to quit diff --git a/src/cndm/dpdk/cndm/cndm.h b/src/cndm/dpdk/cndm/cndm.h new file mode 100644 index 0000000..2a6bece --- /dev/null +++ b/src/cndm/dpdk/cndm/cndm.h @@ -0,0 +1,249 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2025-2026 FPGA Ninja, LLC + * + * Authors: + * - Alex Forencich + */ + +#ifndef CNDM_H +#define CNDM_H + +#include +#include + +#include "cndm_hw.h" + +#define ARRAY_SIZE(x) RTE_DIM(x) +#define ETH_ALEN RTE_ETHER_ADDR_LEN + +#ifndef RTE_PCI_EXP_LNKCAP +#define RTE_PCI_EXP_LNKCAP 12 +#endif + +#ifndef RTE_PCI_EXP_LNKCTL_RCB +#define RTE_PCI_EXP_LNKCTL_RCB 0x0008 +#endif + +#ifndef RTE_PCI_EXP_LNKCAP_SLS +#define RTE_PCI_EXP_LNKCAP_SLS 0x0000000f +#endif + +#ifndef RTE_PCI_EXP_LNKCAP_MLW +#define RTE_PCI_EXP_LNKCAP_MLW 0x000003f0 +#endif + +#ifndef RTE_PCI_EXP_DEVCTL_RELAX_EN +#define RTE_PCI_EXP_DEVCTL_RELAX_EN 0x0010 +#endif + +#ifndef RTE_PCI_EXP_DEVCTL_PHANTOM +#define RTE_PCI_EXP_DEVCTL_PHANTOM 0x0200 +#endif + +#ifndef RTE_PCI_EXP_DEVCTL_NOSNOOP_EN +#define RTE_PCI_EXP_DEVCTL_NOSNOOP_EN 0x0800 +#endif + +#define DRIVER_VERSION "0.1" + +struct cndm_dev { + struct rte_pci_device *pdev; + + struct rte_eth_dev *eth_dev[32]; + + uint64_t hw_regs_size; + phys_addr_t hw_regs_phys; + __u8 *hw_addr; + + rte_spinlock_t mbox_lock; + + __u32 port_count; + + // config + __u16 cfg_page_max; + __u32 cmd_ver; + + // FW ID + __u32 fpga_id; + __u32 fw_id; + __u32 fw_ver; + __u32 board_id; + __u32 board_ver; + __u32 build_date; + __u32 git_hash; + __u32 release_info; + char build_date_str[32]; + + // HW config + __u16 sys_clk_per_ns_num; + __u16 sys_clk_per_ns_den; + __u16 ptp_clk_per_ns_num; + __u16 ptp_clk_per_ns_den; + + // Resources + __u8 log_max_eq; + __u8 log_max_eq_sz; + __u8 eq_pool; + __u8 eqe_ver; + __u8 log_max_cq; + __u8 log_max_cq_sz; + __u8 cq_pool; + __u8 cqe_ver; + __u8 log_max_sq; + __u8 log_max_sq_sz; + __u8 sq_pool; + __u8 sqe_ver; + __u8 log_max_rq; + __u8 log_max_rq_sz; + __u8 rq_pool; + __u8 rqe_ver; + + // HW IDs + char sn_str[32]; + struct rte_ether_addr base_mac; + int mac_cnt; +}; + +struct cndm_tx_info { + struct rte_mbuf *mbuf; +}; + +struct cndm_rx_info { + struct rte_mbuf *mbuf; +}; + +struct __rte_cache_aligned cndm_ring { + // written on enqueue + __u32 prod_ptr; + __u64 bytes; + __u64 packet; + __u64 dropped_packets; + struct netdev_queue *tx_queue; + + // written from completion + __u32 cons_ptr __rte_cache_aligned; + __u64 ts_s; + __u8 ts_valid; + + // mostly constant + __u32 size; + __u32 full_size; + __u32 size_mask; + __u32 stride; + + __u32 mtu; + + size_t buf_size; + __u8 *buf; + rte_iova_t buf_dma_addr; + + union { + struct cndm_tx_info *tx_info; + struct cndm_rx_info *rx_info; + }; + + struct cndm_dev *cdev; + struct cndm_priv *priv; + unsigned int socket_id; + int index; + int enabled; + + struct rte_mempool *mp; + + struct cndm_cq *cq; + + __u32 db_offset; + __u8 *db_addr; +}; + +struct cndm_cq { + __u32 cons_ptr; + + __u32 size; + __u32 size_mask; + __u32 stride; + + size_t buf_size; + __u8 *buf; + rte_iova_t buf_dma_addr; + + struct cndm_dev *cdev; + struct cndm_priv *priv; + unsigned int socket_id; + int cqn; + int enabled; + + struct cndm_ring *src_ring; + + void (*handler)(struct cndm_cq *cq); + + __u32 db_offset; + __u8 *db_addr; +}; + +struct cndm_priv { + struct rte_eth_dev *eth_dev; + struct cndm_dev *cdev; + + int dev_port; + int port_id; + + bool registered; + bool port_up; + + __u8 *hw_addr; + + int rxq_count; + int txq_count; + + struct cndm_ring *txq; + struct cndm_ring *rxq; +}; + +extern int cndm_logtype_driver; +#define RTE_LOGTYPE_CNDM_DRIVER cndm_logtype_driver + +#define DRV_LOG(level, ...) \ + RTE_LOG_LINE_PREFIX(level, CNDM_DRIVER, "%s(): ", __func__, __VA_ARGS__) + +// cndm_cmd.c +int cndm_exec_mbox_cmd(struct cndm_dev *cdev, void *cmd, void *rsp); +int cndm_exec_cmd(struct cndm_dev *cdev, void *cmd, void *rsp); +int cndm_access_reg(struct cndm_dev *cdev, __u32 reg, int raw, int write, __u64 *data); +int cndm_hwid_sn_rd(struct cndm_dev *cdev, int *len, void *data); +int cndm_hwid_mac_rd(struct cndm_dev *cdev, __u16 index, int *cnt, void *data); + +// cndm_ethdev.c +struct rte_eth_dev *cndm_create_eth_dev(struct cndm_dev *cdev, int port); +void cndm_destroy_eth_dev(struct rte_eth_dev *eth_dev); + +// cndm_cq.c +struct cndm_cq *cndm_create_cq(struct cndm_priv *priv, unsigned int socket_id); +void cndm_destroy_cq(struct cndm_cq *cq); +int cndm_open_cq(struct cndm_cq *cq, int size); +void cndm_close_cq(struct cndm_cq *cq); +void cndm_cq_write_cons_ptr(const struct cndm_cq *cq); +void cndm_cq_write_cons_ptr_arm(const struct cndm_cq *cq); + +// cndm_sq.c +struct cndm_ring *cndm_create_sq(struct cndm_priv *priv, unsigned int socket_id); +void cndm_destroy_sq(struct cndm_ring *sq); +int cndm_open_sq(struct cndm_ring *sq, struct cndm_priv *priv, struct cndm_cq *cq, int size); +void cndm_close_sq(struct cndm_ring *sq); +bool cndm_is_sq_ring_empty(const struct cndm_ring *sq); +bool cndm_is_sq_ring_full(const struct cndm_ring *sq); +void cndm_sq_write_prod_ptr(const struct cndm_ring *sq); +uint16_t cndm_xmit_pkt_burst(void *queue, struct rte_mbuf **pkts, uint16_t nb_pkts); + +// cndm_rq.c +struct cndm_ring *cndm_create_rq(struct cndm_priv *priv, unsigned int socket_id); +void cndm_destroy_rq(struct cndm_ring *rq); +int cndm_open_rq(struct cndm_ring *rq, struct cndm_priv *priv, struct cndm_cq *cq, int size); +void cndm_close_rq(struct cndm_ring *rq); +bool cndm_is_rq_ring_empty(const struct cndm_ring *rq); +bool cndm_is_rq_ring_full(const struct cndm_ring *rq); +void cndm_rq_write_prod_ptr(const struct cndm_ring *rq); +int cndm_refill_rx_buffers(struct cndm_ring *rq); +uint16_t cndm_recv_pkt_burst(void *queue, struct rte_mbuf **pkts, uint16_t nb_pkts); + +#endif diff --git a/src/cndm/dpdk/cndm/cndm_cmd.c b/src/cndm/dpdk/cndm/cndm_cmd.c new file mode 100644 index 0000000..450657a --- /dev/null +++ b/src/cndm/dpdk/cndm/cndm_cmd.c @@ -0,0 +1,164 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2026 FPGA Ninja, LLC + * + * Authors: + * - Alex Forencich + */ + +#include "cndm.h" + +#include + +#include +#include + +int cndm_exec_mbox_cmd(struct cndm_dev *cdev, void *cmd, void *rsp) +{ + bool done = false; + int ret = 0; + int k; + + if (!cmd || !rsp) + return -EINVAL; + + rte_spinlock_lock(&cdev->mbox_lock); + + // write command to mailbox + for (k = 0; k < 16; k++) { + rte_write32(*((__u32 *)((__u8 *)cmd + k*4)), cdev->hw_addr + 0x10000 + k*4); + } + + // ensure the command is completely written + rte_wmb(); + + // execute it + rte_write32(0x00000001, cdev->hw_addr + 0x0200); + + // wait for completion + for (k = 0; k < 100; k++) { + done = (rte_read32(cdev->hw_addr + 0x0200) & 0x00000001) == 0; + if (done) + break; + + rte_delay_us(100); + } + + if (done) { + // read response from mailbox + for (k = 0; k < 16; k++) { + *((__u32 *)((__u8 *)rsp + k*4)) = rte_read32(cdev->hw_addr + 0x10000 + 0x40 + k*4); + } + } else { + DRV_LOG(ERR, "Command timed out"); + rte_hexdump(stderr, "cmd", cmd, sizeof(struct cndm_cmd_cfg)); + ret = -ETIMEDOUT; + } + + rte_spinlock_unlock(&cdev->mbox_lock); + return ret; +} + +int cndm_exec_cmd(struct cndm_dev *cdev, void *cmd, void *rsp) +{ + return cndm_exec_mbox_cmd(cdev, cmd, rsp); +} + +int cndm_access_reg(struct cndm_dev *cdev, __u32 reg, int raw, int write, __u64 *data) +{ + struct cndm_cmd_reg cmd; + struct cndm_cmd_reg rsp; + int ret = 0; + + cmd.opcode = CNDM_CMD_OP_ACCESS_REG; + cmd.flags = 0x00000000; + cmd.reg_addr = reg; + cmd.write_val = *data; + cmd.read_val = 0; + + if (write) + cmd.flags |= CNDM_CMD_REG_FLG_WRITE; + if (raw) + cmd.flags |= CNDM_CMD_REG_FLG_RAW; + + ret = cndm_exec_cmd(cdev, &cmd, &rsp); + if (ret) + return ret; + + if (rsp.status) + return rsp.status; + + if (!write) + *data = rsp.read_val; + + return 0; +} + +int cndm_hwid_sn_rd(struct cndm_dev *cdev, int *len, void *data) +{ + struct cndm_cmd_hwid cmd; + struct cndm_cmd_hwid rsp; + int k = 0; + int ret = 0; + char buf[64]; + const char *ptr; + + cmd.opcode = CNDM_CMD_OP_HWID; + cmd.flags = 0x00000000; + cmd.index = 0; + cmd.brd_opcode = CNDM_CMD_BRD_OP_HWID_SN_RD; + cmd.brd_flags = 0x00000000; + + ret = cndm_exec_cmd(cdev, &cmd, &rsp); + if (ret) + return ret; + + if (rsp.status || rsp.brd_status) + return rsp.status ? rsp.status : rsp.brd_status; + + // memcpy(&buf, &rsp.data, min(cmd.len, 32)); // TODO + memcpy(&buf, &rsp.data, 32); + buf[32] = 0; + + for (k = 0; k < 32; k++) { + if (!isascii(buf[k]) || !isprint(buf[k])) { + buf[k] = 0; + break; + } + } + + ptr = rte_str_skip_leading_spaces(buf); + + if (len) + *len = strlen(ptr); + if (data) + rte_strscpy(data, ptr, 32); + + return 0; +} + +int cndm_hwid_mac_rd(struct cndm_dev *cdev, __u16 index, int *cnt, void *data) +{ + struct cndm_cmd_hwid cmd; + struct cndm_cmd_hwid rsp; + int ret = 0; + + cmd.opcode = CNDM_CMD_OP_HWID; + cmd.flags = 0x00000000; + cmd.index = index; + cmd.brd_opcode = CNDM_CMD_BRD_OP_HWID_MAC_RD; + cmd.brd_flags = 0x00000000; + + ret = cndm_exec_cmd(cdev, &cmd, &rsp); + if (ret) + return ret; + + if (rsp.status || rsp.brd_status) + return rsp.status ? rsp.status : rsp.brd_status; + + if (cnt) + *cnt = 1; // *((__u16 *)&rsp.data); // TODO + if (data) + memcpy(data, ((__u8 *)&rsp.data)+2, ETH_ALEN); + + return 0; +} diff --git a/src/cndm/dpdk/cndm/cndm_cq.c b/src/cndm/dpdk/cndm/cndm_cq.c new file mode 100644 index 0000000..acbe5d1 --- /dev/null +++ b/src/cndm/dpdk/cndm/cndm_cq.c @@ -0,0 +1,144 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2025-2026 FPGA Ninja, LLC + * + * Authors: + * - Alex Forencich + */ + +#include "cndm.h" + +#include +#include + +struct cndm_cq *cndm_create_cq(struct cndm_priv *priv, unsigned int socket_id) +{ + struct cndm_cq *cq; + + cq = rte_zmalloc_socket("cndm_cq", sizeof(*cq), 0, socket_id); + if (!cq) + return NULL; + + cq->cdev = priv->cdev; + cq->priv = priv; + cq->socket_id = socket_id; + + cq->cqn = -1; + cq->enabled = 0; + + cq->cons_ptr = 0; + + cq->db_offset = 0; + cq->db_addr = NULL; + + return cq; +} + +void cndm_destroy_cq(struct cndm_cq *cq) +{ + cndm_close_cq(cq); + + rte_free(cq); +} + +int cndm_open_cq(struct cndm_cq *cq, int size) +{ + __u32 dqn = 0xC0000000; + int ret = 0; + + struct cndm_cmd_queue cmd; + struct cndm_cmd_queue rsp; + + if (cq->enabled || cq->buf) + return -EINVAL; + + cq->size = rte_align32pow2(size); + cq->size_mask = cq->size - 1; + cq->stride = 16; + + cq->buf_size = cq->size * cq->stride; + cq->buf = rte_zmalloc_socket("cndm_cq_ring", cq->buf_size, + RTE_CACHE_LINE_SIZE, cq->socket_id); + if (!cq->buf) + return -ENOMEM; + cq->buf_dma_addr = rte_malloc_virt2iova(cq->buf); + + cq->cons_ptr = 0; + + // clear all phase tag bits + memset(cq->buf, 0, cq->buf_size); + + cmd.opcode = CNDM_CMD_OP_CREATE_CQ; + cmd.flags = 0x00000000; + cmd.port = cq->priv->dev_port; + cmd.qn = 0; + cmd.qn2 = dqn; + cmd.pd = 0; + cmd.size = rte_log2_u32(cq->size); + cmd.dboffs = 0; + cmd.ptr1 = cq->buf_dma_addr; + cmd.ptr2 = 0; + + ret = cndm_exec_cmd(cq->cdev, &cmd, &rsp); + if (ret) { + DRV_LOG(ERR, "Failed to execute command"); + goto fail; + } + + if (rsp.status || rsp.dboffs == 0) { + DRV_LOG(ERR, "Failed to allocate CQ"); + ret = rsp.status; + goto fail; + } + + cq->cqn = rsp.qn; + cq->db_offset = rsp.dboffs; + cq->db_addr = cq->cdev->hw_addr + rsp.dboffs; + + cq->enabled = 1; + + DRV_LOG(DEBUG, "Opened CQ %d", cq->cqn); + + return 0; + +fail: + cndm_close_cq(cq); + return ret; +} + +void cndm_close_cq(struct cndm_cq *cq) +{ + struct cndm_dev *cdev = cq->cdev; + struct cndm_cmd_queue cmd; + struct cndm_cmd_queue rsp; + + cq->enabled = 0; + + if (cq->cqn != -1) { + cmd.opcode = CNDM_CMD_OP_DESTROY_CQ; + cmd.flags = 0x00000000; + cmd.port = cq->priv->dev_port; + cmd.qn = cq->cqn; + + cndm_exec_cmd(cdev, &cmd, &rsp); + + cq->cqn = -1; + cq->db_offset = 0; + cq->db_addr = NULL; + } + + if (cq->buf) { + rte_free(cq->buf); + cq->buf = NULL; + cq->buf_dma_addr = 0; + } +} + +void cndm_cq_write_cons_ptr(const struct cndm_cq *cq) +{ + rte_write32(cq->cons_ptr & 0xffff, cq->db_addr); +} + +void cndm_cq_write_cons_ptr_arm(const struct cndm_cq *cq) +{ + rte_write32((cq->cons_ptr & 0xffff) | 0x80000000, cq->db_addr); +} diff --git a/src/cndm/dpdk/cndm/cndm_ethdev.c b/src/cndm/dpdk/cndm/cndm_ethdev.c new file mode 100644 index 0000000..bfb7726 --- /dev/null +++ b/src/cndm/dpdk/cndm/cndm_ethdev.c @@ -0,0 +1,943 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2026 FPGA Ninja, LLC + * + * Authors: + * - Alex Forencich + */ + +#include "cndm.h" + +#include +#include +#include +#include + +static int cndm_link_update(struct rte_eth_dev *eth_dev, int wait_to_complete __rte_unused) +{ + struct rte_eth_link link; + + memset(&link, 0, sizeof(link)); + + link.link_duplex = RTE_ETH_LINK_FULL_DUPLEX; + link.link_autoneg = RTE_ETH_LINK_SPEED_FIXED; + link.link_speed = RTE_ETH_SPEED_NUM_100G; + link.link_status = RTE_ETH_LINK_UP; + + if (!eth_dev->data->dev_started) + link.link_status = RTE_ETH_LINK_DOWN; + + return rte_eth_linkstatus_set(eth_dev, &link); +} + +static int cndm_promiscuous_mode_enable(struct rte_eth_dev *eth_dev __rte_unused) +{ + return 0; +} + +static int cndm_promiscuous_mode_disable(struct rte_eth_dev *eth_dev __rte_unused) +{ + return 0; +} + +static int cndm_mac_addr_set(struct rte_eth_dev *eth_dev, struct rte_ether_addr *addr) +{ + rte_ether_addr_copy(addr, eth_dev->data->mac_addrs); + + return 0; +} + +static int cndm_mtu_set(struct rte_eth_dev *eth_dev, uint16_t mtu) +{ + // struct cndm_priv *priv = eth_dev->data->dev_private; + + // TODO limits + + eth_dev->data->mtu = mtu; + + return 0; +} + + +static int cndm_stats_get(struct rte_eth_dev *eth_dev __rte_unused, + struct rte_eth_stats *stats __rte_unused, struct eth_queue_stats *qstats __rte_unused) +{ + // TODO + + return 0; +} + +static int cndm_stats_reset(struct rte_eth_dev *eth_dev __rte_unused) +{ + // TODO + + return 0; +} + +static int cndm_dev_info_get(struct rte_eth_dev *eth_dev, + struct rte_eth_dev_info *dev_info) +{ + struct cndm_priv *priv = eth_dev->data->dev_private; + + DRV_LOG(DEBUG, "Info get for eth_dev %s", eth_dev->data->name); + + dev_info->min_mtu = RTE_ETHER_MIN_MTU; + dev_info->max_mtu = 1500; // TODO + + dev_info->min_rx_bufsize = 64; + dev_info->max_rx_pktlen = dev_info->max_mtu + RTE_ETHER_HDR_LEN; + + dev_info->max_rx_queues = 1; //RTE_MIN(1 << priv->log_max_rq, UINT16_MAX); + dev_info->max_tx_queues = 1; //RTE_MIN(1 << priv->log_max_sq, UINT16_MAX); + + dev_info->max_mac_addrs = 1; + dev_info->max_hash_mac_addrs = 0; + + dev_info->max_vfs = 0; + + dev_info->rx_offload_capa = 0; + + dev_info->tx_offload_capa = 0; + + dev_info->reta_size = 0; + dev_info->hash_key_size = 0; + dev_info->flow_type_rss_offloads = 0; + + dev_info->default_rxconf = (struct rte_eth_rxconf){ + .rx_thresh = { + .pthresh = 8, + .hthresh = 8, + .wthresh = 0, + }, + .rx_free_thresh = 32, + .rx_drop_en = 1, + }; + + dev_info->default_txconf = (struct rte_eth_txconf){ + .tx_thresh = { + .pthresh = 32, + .hthresh = 0, + .wthresh = 0, + }, + .tx_rs_thresh = 32, + .tx_free_thresh = 32, + }; + + dev_info->rx_desc_lim.nb_min = 4; + dev_info->rx_desc_lim.nb_max = RTE_MIN(1 << priv->cdev->log_max_rq_sz, UINT16_MAX); + dev_info->rx_desc_lim.nb_align = 4; + dev_info->rx_desc_lim.nb_seg_max = 1; + dev_info->rx_desc_lim.nb_mtu_seg_max = 1; + + dev_info->tx_desc_lim.nb_min = 4; + dev_info->tx_desc_lim.nb_max = RTE_MIN(1 << priv->cdev->log_max_sq_sz, UINT16_MAX); + dev_info->tx_desc_lim.nb_align = 4; + dev_info->tx_desc_lim.nb_seg_max = 1; + dev_info->tx_desc_lim.nb_mtu_seg_max = 1; + + dev_info->speed_capa = RTE_ETH_LINK_SPEED_100G; + + dev_info->default_rxportconf.burst_size = 1; + dev_info->default_rxportconf.ring_size = 128; + dev_info->default_rxportconf.nb_queues = 1; + + dev_info->default_txportconf.burst_size = 1; + dev_info->default_txportconf.ring_size = 128; + dev_info->default_txportconf.nb_queues = 1; + + return 0; +} + +static void cndm_dev_rxq_info_get(struct rte_eth_dev *eth_dev, + uint16_t rx_queue_id, struct rte_eth_rxq_info *qinfo) +{ + struct cndm_ring *rq; + + rq = eth_dev->data->rx_queues[rx_queue_id]; + + if (!rq) + return; + + qinfo->mp = rq->mp; + qinfo->scattered_rx = eth_dev->data->scattered_rx; + qinfo->nb_desc = rq->size; + // qinfo->rx_buf_size; + qinfo->avail_thresh = 0; + // qinfo->conf; +} + +static void cndm_dev_txq_info_get(struct rte_eth_dev *eth_dev, + uint16_t tx_queue_id, struct rte_eth_txq_info *qinfo) +{ + struct cndm_ring *sq; + + sq = eth_dev->data->tx_queues[tx_queue_id]; + + if (!sq) + return; + + qinfo->nb_desc = sq->size; + // qinfo->conf; +} + +static int cndm_dev_fw_version_get(struct rte_eth_dev *eth_dev, + char *fw_version, size_t fw_size) +{ + struct cndm_priv *priv = eth_dev->data->dev_private; + struct cndm_dev *cdev = priv->cdev; + int ret; + + ret = snprintf(fw_version, fw_size, "%d.%d.%d", cdev->fw_ver >> 20, + (cdev->fw_ver >> 12) & 0xff, cdev->fw_ver & 0xfff); + if (ret < 0) + return -EINVAL; + + ret += 1; + + if (fw_size < (size_t)ret) + return ret; + + return 0; +} + +static int cndm_dev_rx_queue_stop(struct rte_eth_dev *eth_dev, + uint16_t rx_queue_id) +{ + struct cndm_ring *rq; + struct cndm_cq *cq; + + DRV_LOG(DEBUG, "RX queue stop for eth_dev %s queue %d", eth_dev->data->name, rx_queue_id); + + rq = eth_dev->data->rx_queues[rx_queue_id]; + + if (!rq) + goto done; + + cq = rq->cq; + + cndm_close_rq(rq); + + if (cq) { + cndm_close_cq(cq); + cndm_destroy_cq(cq); + } + +done: + eth_dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED; + return 0; +} + +static int cndm_dev_rx_queue_start(struct rte_eth_dev *eth_dev, + uint16_t rx_queue_id) +{ + struct cndm_priv *priv = eth_dev->data->dev_private; + struct cndm_ring *rq; + struct cndm_cq *cq; + int ret = 0; + + DRV_LOG(DEBUG, "RX queue start for eth_dev %s queue %d", eth_dev->data->name, rx_queue_id); + + rq = eth_dev->data->rx_queues[rx_queue_id]; + + if (!rq) + return -EINVAL; + + cq = cndm_create_cq(priv, rq->socket_id); + if (!cq) { + goto fail; + } + + ret = cndm_open_cq(cq, rq->size); + if (ret) { + cndm_destroy_cq(cq); + goto fail; + } + + ret = cndm_open_rq(rq, priv, cq, rq->size); + if (ret) { + cndm_destroy_cq(cq); + goto fail; + } + + eth_dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED; + + return 0; +fail: + cndm_dev_rx_queue_stop(eth_dev, rx_queue_id); + return ret; +} + +static void cndm_dev_rx_queue_release(struct rte_eth_dev *eth_dev, + uint16_t rx_queue_id) +{ + struct cndm_ring *rq; + + DRV_LOG(DEBUG, "RX queue release for eth_dev %s queue %d", eth_dev->data->name, rx_queue_id); + + cndm_dev_rx_queue_stop(eth_dev, rx_queue_id); + + rq = eth_dev->data->rx_queues[rx_queue_id]; + + cndm_destroy_rq(rq); + + eth_dev->data->rx_queues[rx_queue_id] = NULL; +} + +static int cndm_dev_rx_queue_setup(struct rte_eth_dev *eth_dev, + uint16_t rx_queue_id, uint16_t nb_rx_desc, + unsigned int socket_id, + const struct rte_eth_rxconf *rx_conf __rte_unused, + struct rte_mempool *mp) +{ + struct cndm_priv *priv = eth_dev->data->dev_private; + struct cndm_ring *rq; + + DRV_LOG(DEBUG, "RX queue setup for eth_dev %s queue %d", eth_dev->data->name, rx_queue_id); + + rq = cndm_create_rq(priv, socket_id); + + if (!rq) + return -ENOMEM; + + rq->size = rte_align32pow2(nb_rx_desc); + rq->mp = mp; + + eth_dev->data->rx_queues[rx_queue_id] = rq; + + return 0; +} + +static int cndm_dev_tx_queue_stop(struct rte_eth_dev *eth_dev, + uint16_t tx_queue_id) +{ + struct cndm_ring *sq; + struct cndm_cq *cq; + + DRV_LOG(DEBUG, "TX queue stop for eth_dev %s queue %d", eth_dev->data->name, tx_queue_id); + + sq = eth_dev->data->tx_queues[tx_queue_id]; + + if (!sq) + goto done; + + cq = sq->cq; + + cndm_close_sq(sq); + + if (cq) { + cndm_close_cq(cq); + cndm_destroy_cq(cq); + } + +done: + eth_dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED; + return 0; +} + +static int cndm_dev_tx_queue_start(struct rte_eth_dev *eth_dev, + uint16_t tx_queue_id) +{ + struct cndm_priv *priv = eth_dev->data->dev_private; + struct cndm_ring *sq; + struct cndm_cq *cq; + int ret = 0; + + DRV_LOG(DEBUG, "TX queue start for eth_dev %s queue %d", eth_dev->data->name, tx_queue_id); + + sq = eth_dev->data->tx_queues[tx_queue_id]; + + if (!sq) + return -EINVAL; + + cq = cndm_create_cq(priv, sq->socket_id); + if (!cq) { + goto fail; + } + + ret = cndm_open_cq(cq, sq->size); + if (ret) { + cndm_destroy_cq(cq); + goto fail; + } + + ret = cndm_open_sq(sq, priv, cq, sq->size); + if (ret) { + cndm_destroy_cq(cq); + goto fail; + } + + eth_dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED; + + return 0; +fail: + cndm_dev_tx_queue_stop(eth_dev, tx_queue_id); + return ret; +} + +static void cndm_dev_tx_queue_release(struct rte_eth_dev *eth_dev, + uint16_t tx_queue_id) +{ + struct cndm_ring *sq; + + DRV_LOG(DEBUG, "TX queue release for eth_dev %s queue %d", eth_dev->data->name, tx_queue_id); + + cndm_dev_tx_queue_stop(eth_dev, tx_queue_id); + + sq = eth_dev->data->tx_queues[tx_queue_id]; + + cndm_destroy_sq(sq); + + eth_dev->data->tx_queues[tx_queue_id] = NULL; +} + +static int cndm_dev_tx_queue_setup(struct rte_eth_dev *eth_dev, + uint16_t tx_queue_id, uint16_t nb_tx_desc, + unsigned int socket_id, + const struct rte_eth_txconf *tx_conf __rte_unused) +{ + struct cndm_priv *priv = eth_dev->data->dev_private; + struct cndm_ring *sq; + + DRV_LOG(DEBUG, "TX queue setup for eth_dev %s queue %d", eth_dev->data->name, tx_queue_id); + + sq = cndm_create_sq(priv, socket_id); + + if (!sq) + return -ENOMEM; + + sq->size = rte_align32pow2(nb_tx_desc); + + eth_dev->data->tx_queues[tx_queue_id] = sq; + + return 0; +} + +static int cndm_dev_configure(struct rte_eth_dev *eth_dev) +{ + DRV_LOG(DEBUG, "Dev configure for eth_dev %s", eth_dev->data->name); + + return 0; +} + +static int cndm_dev_stop(struct rte_eth_dev *eth_dev) +{ + DRV_LOG(DEBUG, "Dev stop for eth_dev %s", eth_dev->data->name); + + eth_dev->data->dev_started = 0; + + eth_dev->rx_pkt_burst = rte_eth_pkt_burst_dummy; + eth_dev->tx_pkt_burst = rte_eth_pkt_burst_dummy; + + // Stop all queues + for (int i = 0; i < eth_dev->data->nb_rx_queues; i++) { + cndm_dev_rx_queue_stop(eth_dev, i); + } + + for (int i = 0; i < eth_dev->data->nb_tx_queues; i++) { + cndm_dev_tx_queue_stop(eth_dev, i); + } + + return 0; +} + +static int cndm_dev_start(struct rte_eth_dev *eth_dev) +{ + int ret = 0; + + DRV_LOG(DEBUG, "Dev start for eth_dev %s", eth_dev->data->name); + + if (eth_dev->data->dev_started) + return -EINVAL; + + // Start all queues + for (int i = 0; i < eth_dev->data->nb_rx_queues; i++) { + ret = cndm_dev_rx_queue_start(eth_dev, i); + if (ret) + goto fail; + } + + for (int i = 0; i < eth_dev->data->nb_tx_queues; i++) { + ret = cndm_dev_tx_queue_start(eth_dev, i); + if (ret) + goto fail; + } + + eth_dev->rx_pkt_burst = cndm_recv_pkt_burst; + eth_dev->tx_pkt_burst = cndm_xmit_pkt_burst; + + eth_dev->data->dev_started = 1; + + return 0; +fail: + cndm_dev_stop(eth_dev); + return ret; +} + +static int cndm_dev_close(struct rte_eth_dev *eth_dev) +{ + DRV_LOG(DEBUG, "Dev close for eth_dev %s", eth_dev->data->name); + + cndm_dev_stop(eth_dev); + + // TODO + + return 0; +} + +static int cndm_read_eeprom(struct cndm_priv *priv, __u16 offset, __u16 len, __u8 *data) +{ + int ret = 0; + + struct cndm_cmd_hwid cmd; + struct cndm_cmd_hwid rsp; + + if (len > 32) + len = 32; + + cmd.opcode = CNDM_CMD_OP_HWID; + cmd.flags = 0x00000000; + cmd.index = 0; + cmd.brd_opcode = CNDM_CMD_BRD_OP_EEPROM_RD; + cmd.brd_flags = 0x00000000; + cmd.dev_addr_offset = 0; + cmd.page = 0; + cmd.bank = 0; + cmd.addr = offset; + cmd.len = len; + + ret = cndm_exec_cmd(priv->cdev, &cmd, &rsp); + if (ret) { + DRV_LOG(ERR, "Failed to execute command"); + return -ret; + } + + if (rsp.status || rsp.brd_status) { + DRV_LOG(WARNING, "Failed to read EEPROM"); + return rsp.status ? -rsp.status : -rsp.brd_status; + } + + if (data) + memcpy(data, ((void *)&rsp.data), len); + + return len; +} + +static int cndm_write_eeprom(struct cndm_priv *priv, __u16 offset, __u16 len, __u8 *data) +{ + int ret = 0; + + struct cndm_cmd_hwid cmd; + struct cndm_cmd_hwid rsp; + + // limit length to 32 + if (len > 32) + len = 32; + + // do not cross 32-byte boundaries + if (len > 32 - (offset & 31)) + len = 32 - (offset & 31); + + cmd.opcode = CNDM_CMD_OP_HWID; + cmd.flags = 0x00000000; + cmd.index = 0; + cmd.brd_opcode = CNDM_CMD_BRD_OP_EEPROM_WR; + cmd.brd_flags = 0x00000000; + cmd.dev_addr_offset = 0; + cmd.page = 0; + cmd.bank = 0; + cmd.addr = offset; + cmd.len = len; + + memcpy(((void *)&cmd.data), data, len); + + ret = cndm_exec_cmd(priv->cdev, &cmd, &rsp); + if (ret) { + DRV_LOG(ERR, "Failed to execute command"); + return -ret; + } + + if (rsp.status || rsp.brd_status) { + DRV_LOG(WARNING, "Failed to write EEPROM"); + return rsp.status ? -rsp.status : -rsp.brd_status; + } + + return len; +} + +static int cndm_get_eeprom_length(struct rte_eth_dev *eth_dev __rte_unused) +{ + return 256; // TODO +} + +static int cndm_get_eeprom(struct rte_eth_dev *eth_dev, struct rte_dev_eeprom_info *eeprom) +{ + struct cndm_priv *priv = eth_dev->data->dev_private; + unsigned int i = 0; + int read_len; + + if (eeprom->length == 0) + return -EINVAL; + + eeprom->magic = 0x4d444e43; + + memset(eeprom->data, 0, eeprom->length); + + while (i < eeprom->length) { + read_len = cndm_read_eeprom(priv, eeprom->offset + i, + eeprom->length - i, (__u8 *)eeprom->data + i); + + if (read_len == 0) + return 0; + + if (read_len < 0) { + DRV_LOG(ERR, "Failed to read EEPROM (%d)", read_len); + return read_len; + } + + i += read_len; + } + + return 0; +} + +static int cndm_set_eeprom(struct rte_eth_dev *eth_dev, struct rte_dev_eeprom_info *eeprom) +{ + struct cndm_priv *priv = eth_dev->data->dev_private; + unsigned int i = 0; + int write_len; + + if (eeprom->length == 0) + return -EINVAL; + + if (eeprom->magic != 0x4d444e43) + return -EFAULT; + + while (i < eeprom->length) { + write_len = cndm_write_eeprom(priv, eeprom->offset + i, + eeprom->length - i, (__u8 *)eeprom->data + i); + + if (write_len == 0) + return 0; + + if (write_len < 0) { + DRV_LOG(ERR, "Failed to write EEPROM (%d)", write_len); + return write_len; + } + + i += write_len; + } + + return 0; +} + +#define SFF_MODULE_ID_SFP 0x03 +#define SFF_MODULE_ID_QSFP 0x0c +#define SFF_MODULE_ID_QSFP_PLUS 0x0d +#define SFF_MODULE_ID_QSFP28 0x11 + +static int cndm_read_module_eeprom(struct cndm_priv *priv, + unsigned short i2c_addr, __u16 page, __u16 bank, __u16 offset, __u16 len, __u8 *data) +{ + int ret = 0; + + struct cndm_cmd_hwid cmd; + struct cndm_cmd_hwid rsp; + + if (len > 32) + len = 32; + + cmd.opcode = CNDM_CMD_OP_HWMON; + cmd.flags = 0x00000000; + cmd.index = 0; // TODO + cmd.brd_opcode = CNDM_CMD_BRD_OP_OPTIC_RD; + cmd.brd_flags = 0x00000000; + cmd.dev_addr_offset = i2c_addr - 0x50; + cmd.page = page; + cmd.bank = bank; + cmd.addr = offset; + cmd.len = len; + + ret = cndm_exec_cmd(priv->cdev, &cmd, &rsp); + if (ret) { + DRV_LOG(ERR, "Failed to execute command"); + return -ret; + } + + if (rsp.status || rsp.brd_status) { + DRV_LOG(WARNING, "Failed to read module EEPROM"); + return rsp.status ? -rsp.status : -rsp.brd_status; + } + + if (data) + memcpy(data, ((void *)&rsp.data), len); + + return len; +} + +static int cndm_query_module_id(struct cndm_priv *priv) +{ + int ret; + __u8 data = 0; + + ret = cndm_read_module_eeprom(priv, 0x50, 0, 0, 0, 1, &data); + + if (ret < 0) + return ret; + + return data; +} + +static int cndm_query_module_eeprom_by_page(struct cndm_priv *priv, + unsigned short i2c_addr, __u16 page, __u16 bank, __u16 offset, __u16 len, __u8 *data) +{ + int module_id; + int ret; + + module_id = cndm_query_module_id(priv); + + if (module_id < 0) { + DRV_LOG(ERR, "Failed to read module ID (%d)", module_id); + return module_id; + } + + switch (module_id) { + case SFF_MODULE_ID_SFP: + if (page > 0 || bank > 0) + return -EINVAL; + if (i2c_addr != 0x50 && i2c_addr != 0x51) + return -EINVAL; + break; + case SFF_MODULE_ID_QSFP: + case SFF_MODULE_ID_QSFP_PLUS: + case SFF_MODULE_ID_QSFP28: + if (page > 3 || bank > 0) + return -EINVAL; + if (i2c_addr != 0x50) + return -EINVAL; + break; + default: + DRV_LOG(ERR, "Unknown module ID (0x%x)", module_id); + return -EINVAL; + } + + // read data + ret = cndm_read_module_eeprom(priv, i2c_addr, page, bank, offset, len, data); + + return ret; +} + +static int cndm_query_module_eeprom(struct cndm_priv *priv, + __u16 offset, __u16 len, __u8 *data) +{ + int module_id; + unsigned short i2c_addr = 0x50; + __u16 page = 0; + __u16 bank = 0; + + module_id = cndm_query_module_id(priv); + + if (module_id < 0) { + DRV_LOG(ERR, "Failed to read module ID (%d)", module_id); + return module_id; + } + + switch (module_id) { + case SFF_MODULE_ID_SFP: + i2c_addr = 0x50; + page = 0; + if (offset >= 256) { + offset -= 256; + i2c_addr = 0x51; + } + break; + case SFF_MODULE_ID_QSFP: + case SFF_MODULE_ID_QSFP_PLUS: + case SFF_MODULE_ID_QSFP28: + i2c_addr = 0x50; + if (offset < 256) { + page = 0; + } else { + page = 1 + ((offset - 256) / 128); + offset -= page * 128; + } + break; + default: + DRV_LOG(ERR, "Unknown module ID (0x%x)", module_id); + return -EINVAL; + } + + // clip request to end of page + if (offset + len > 256) + len = 256 - offset; + + return cndm_query_module_eeprom_by_page(priv, i2c_addr, + page, bank, offset, len, data); +} + +static int cndm_get_module_info(struct rte_eth_dev *eth_dev, struct rte_eth_dev_module_info *modinfo) +{ + struct cndm_priv *priv = eth_dev->data->dev_private; + int read_len = 0; + __u8 data[16]; + + // read module ID and revision + read_len = cndm_read_module_eeprom(priv, 0x50, 0, 0, 0, 2, data); + + if (read_len < 0) + return read_len; + + if (read_len < 2) + return -EIO; + + // check identifier byte at address 0 + switch (data[0]) { + case SFF_MODULE_ID_SFP: + modinfo->type = RTE_ETH_MODULE_SFF_8472; + modinfo->eeprom_len = RTE_ETH_MODULE_SFF_8472_LEN; + break; + case SFF_MODULE_ID_QSFP: + modinfo->type = RTE_ETH_MODULE_SFF_8436; + modinfo->eeprom_len = RTE_ETH_MODULE_SFF_8436_MAX_LEN; + break; + case SFF_MODULE_ID_QSFP_PLUS: + // check revision at address 1 + if (data[1] >= 0x03) { + modinfo->type = RTE_ETH_MODULE_SFF_8636; + modinfo->eeprom_len = RTE_ETH_MODULE_SFF_8636_MAX_LEN; + } else { + modinfo->type = RTE_ETH_MODULE_SFF_8436; + modinfo->eeprom_len = RTE_ETH_MODULE_SFF_8436_MAX_LEN; + } + break; + case SFF_MODULE_ID_QSFP28: + modinfo->type = RTE_ETH_MODULE_SFF_8636; + modinfo->eeprom_len = RTE_ETH_MODULE_SFF_8636_MAX_LEN; + break; + default: + DRV_LOG(ERR, "Unknown module ID (0x%x)", data[0]); + return -EINVAL; + } + + return 0; +} + +static int cndm_get_module_eeprom(struct rte_eth_dev *eth_dev, struct rte_dev_eeprom_info *eeprom) +{ + struct cndm_priv *priv = eth_dev->data->dev_private; + unsigned int i = 0; + int read_len; + + if (eeprom->length == 0) + return -EINVAL; + + eeprom->magic = 0x4d444e43; + + memset(eeprom->data, 0, eeprom->length); + + while (i < eeprom->length) { + read_len = cndm_query_module_eeprom(priv, eeprom->offset + i, + eeprom->length - i, (__u8 *)eeprom->data + i); + + if (read_len == 0) + return 0; + + if (read_len < 0) { + DRV_LOG(ERR, "Failed to read module EEPROM (%d)", read_len); + return read_len; + } + + i += read_len; + } + + return 0; +} + +static const struct eth_dev_ops cndm_eth_dev_ops = { + .dev_configure = cndm_dev_configure, + .dev_start = cndm_dev_start, + .dev_stop = cndm_dev_stop, + .dev_close = cndm_dev_close, + .link_update = cndm_link_update, + .promiscuous_enable = cndm_promiscuous_mode_enable, + .promiscuous_disable = cndm_promiscuous_mode_disable, + .mac_addr_set = cndm_mac_addr_set, + .mtu_set = cndm_mtu_set, + .stats_get = cndm_stats_get, + .stats_reset = cndm_stats_reset, + .dev_infos_get = cndm_dev_info_get, + .rxq_info_get = cndm_dev_rxq_info_get, + .txq_info_get = cndm_dev_txq_info_get, + .fw_version_get = cndm_dev_fw_version_get, + .rx_queue_start = cndm_dev_rx_queue_start, + .rx_queue_stop = cndm_dev_rx_queue_stop, + .tx_queue_start = cndm_dev_tx_queue_start, + .tx_queue_stop = cndm_dev_tx_queue_stop, + .rx_queue_setup = cndm_dev_rx_queue_setup, + .rx_queue_release = cndm_dev_rx_queue_release, + .tx_queue_setup = cndm_dev_tx_queue_setup, + .tx_queue_release = cndm_dev_tx_queue_release, + .get_eeprom_length = cndm_get_eeprom_length, + .get_eeprom = cndm_get_eeprom, + .set_eeprom = cndm_set_eeprom, + .get_module_info = cndm_get_module_info, + .get_module_eeprom = cndm_get_module_eeprom, +}; + +struct rte_eth_dev *cndm_create_eth_dev(struct cndm_dev *cdev, int port) +{ + struct rte_eth_dev *eth_dev = NULL; + struct cndm_priv *priv; + char name[RTE_ETH_NAME_MAX_LEN]; + + snprintf(name, sizeof(name), "%s_p%d", cdev->pdev->device.name, port); + + DRV_LOG(DEBUG, "Create eth_dev %s (port %d)", name, port); + + eth_dev = rte_eth_dev_allocate(name); + if (!eth_dev) + return NULL; + + priv = rte_zmalloc_socket(NULL, sizeof(*priv), RTE_CACHE_LINE_SIZE, + cdev->pdev->device.numa_node); + if (!priv) + goto fail; + + eth_dev->data->dev_private = priv; + eth_dev->device = &cdev->pdev->device; + priv->eth_dev = eth_dev; + priv->cdev = cdev; + + priv->port_id = eth_dev->data->port_id; + priv->dev_port = port; + + rte_eth_copy_pci_info(eth_dev, cdev->pdev); + + priv->hw_addr = cdev->hw_addr; + + priv->rxq_count = 1; + priv->txq_count = 1; + + eth_dev->data->mac_addrs = rte_calloc("cndm_mac", 1, + sizeof(struct rte_ether_addr), 0); + rte_eth_random_addr((void *)eth_dev->data->mac_addrs); + + eth_dev->dev_ops = &cndm_eth_dev_ops; + eth_dev->rx_pkt_burst = rte_eth_pkt_burst_dummy; + eth_dev->tx_pkt_burst = rte_eth_pkt_burst_dummy; + + rte_eth_dev_probing_finish(eth_dev); + + priv->registered = 1; + + return eth_dev; + +fail: + cndm_destroy_eth_dev(eth_dev); + return NULL; +} + +void cndm_destroy_eth_dev(struct rte_eth_dev *eth_dev) +{ + DRV_LOG(DEBUG, "Destroy eth_dev %s", eth_dev->data->name); + + cndm_dev_stop(eth_dev); +} diff --git a/src/cndm/dpdk/cndm/cndm_hw.h b/src/cndm/dpdk/cndm/cndm_hw.h new file mode 100644 index 0000000..593daec --- /dev/null +++ b/src/cndm/dpdk/cndm/cndm_hw.h @@ -0,0 +1,260 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2025-2026 FPGA Ninja, LLC + * + * Authors: + * - Alex Forencich + */ + +#ifndef CNDM_HW_H +#define CNDM_HW_H + +#include + +#define CNDM_CMD_OP_NOP 0x0000 + +#define CNDM_CMD_OP_CFG 0x0100 + +#define CNDM_CMD_OP_ACCESS_REG 0x0180 +#define CNDM_CMD_OP_PTP 0x0190 +#define CNDM_CMD_OP_HWID 0x01A0 +#define CNDM_CMD_OP_HWMON 0x01B0 +#define CNDM_CMD_OP_PLL 0x01C0 + +#define CNDM_CMD_OP_CREATE_EQ 0x0200 +#define CNDM_CMD_OP_MODIFY_EQ 0x0201 +#define CNDM_CMD_OP_QUERY_EQ 0x0202 +#define CNDM_CMD_OP_DESTROY_EQ 0x0203 + +#define CNDM_CMD_OP_CREATE_CQ 0x0210 +#define CNDM_CMD_OP_MODIFY_CQ 0x0211 +#define CNDM_CMD_OP_QUERY_CQ 0x0212 +#define CNDM_CMD_OP_DESTROY_CQ 0x0213 + +#define CNDM_CMD_OP_CREATE_SQ 0x0220 +#define CNDM_CMD_OP_MODIFY_SQ 0x0221 +#define CNDM_CMD_OP_QUERY_SQ 0x0222 +#define CNDM_CMD_OP_DESTROY_SQ 0x0223 + +#define CNDM_CMD_OP_CREATE_RQ 0x0230 +#define CNDM_CMD_OP_MODIFY_RQ 0x0231 +#define CNDM_CMD_OP_QUERY_RQ 0x0232 +#define CNDM_CMD_OP_DESTROY_RQ 0x0233 + +#define CNDM_CMD_OP_CREATE_QP 0x0240 +#define CNDM_CMD_OP_MODIFY_QP 0x0241 +#define CNDM_CMD_OP_QUERY_QP 0x0242 +#define CNDM_CMD_OP_DESTROY_QP 0x0243 + +#define CNDM_CMD_BRD_OP_NOP 0x0000 + +#define CNDM_CMD_BRD_OP_FLASH_RD 0x0100 +#define CNDM_CMD_BRD_OP_FLASH_WR 0x0101 +#define CNDM_CMD_BRD_OP_FLASH_CMD 0x0108 + +#define CNDM_CMD_BRD_OP_EEPROM_RD 0x0200 +#define CNDM_CMD_BRD_OP_EEPROM_WR 0x0201 + +#define CNDM_CMD_BRD_OP_OPTIC_RD 0x0300 +#define CNDM_CMD_BRD_OP_OPTIC_WR 0x0301 + +#define CNDM_CMD_BRD_OP_HWID_SN_RD 0x0400 +#define CNDM_CMD_BRD_OP_HWID_VPD_RD 0x0410 +#define CNDM_CMD_BRD_OP_HWID_MAC_RD 0x0480 + +#define CNDM_CMD_BRD_OP_PLL_STATUS_RD 0x0500 +#define CNDM_CMD_BRD_OP_PLL_TUNE_RAW_RD 0x0502 +#define CNDM_CMD_BRD_OP_PLL_TUNE_RAW_WR 0x0503 +#define CNDM_CMD_BRD_OP_PLL_TUNE_PPT_RD 0x0504 +#define CNDM_CMD_BRD_OP_PLL_TUNE_PPT_WR 0x0505 + +#define CNDM_CMD_BRD_OP_I2C_RD 0x8100 +#define CNDM_CMD_BRD_OP_I2C_WR 0x8101 + +struct cndm_cmd_cfg { + __le16 rsvd; + union { + __le16 opcode; + __le16 status; + }; + __le32 flags; + struct { + __le16 cfg_page; + __le16 cfg_page_max; + }; + __le32 cmd_ver; + + __le32 fw_ver; + __u8 port_count; + __u8 rsvd2[3]; + __le32 rsvd3[2]; + + union { + struct { + // Page 0: FW ID + __le32 fpga_id; + __le32 fw_id; + __le32 fw_ver; + __le32 board_id; + __le32 board_ver; + __le32 build_date; + __le32 git_hash; + __le32 release_info; + } p0; + struct { + // Page 1: HW config + __le16 port_count; + __le16 rsvd1; + __le32 rsvd2[3]; + __le16 sys_clk_per_ns_den; + __le16 sys_clk_per_ns_num; + __le16 ptp_clk_per_ns_den; + __le16 ptp_clk_per_ns_num; + __le32 rsvd3[2]; + } p1; + struct { + // Page 2: Resources + __u8 log_max_eq; + __u8 log_max_eq_sz; + __u8 eq_pool; + __u8 eqe_ver; + __u8 log_max_cq; + __u8 log_max_cq_sz; + __u8 cq_pool; + __u8 cqe_ver; + __u8 log_max_sq; + __u8 log_max_sq_sz; + __u8 sq_pool; + __u8 sqe_ver; + __u8 log_max_rq; + __u8 log_max_rq_sz; + __u8 rq_pool; + __u8 rqe_ver; + __le32 rsvd[4]; + } p2; + }; +}; + +#define CNDM_CMD_REG_FLG_WRITE 0x00000001 +#define CNDM_CMD_REG_FLG_RAW 0x00000100 + +struct cndm_cmd_reg { + __le16 rsvd; + union { + __le16 opcode; + __le16 status; + }; + __le32 flags; + __le32 rsvd1[5]; + __le32 reg_addr; + __le64 write_val; + __le64 read_val; + __le32 rsvd2[4]; +}; + +#define CNDM_CMD_PTP_FLG_SET_TOD 0x00000001 +#define CNDM_CMD_PTP_FLG_OFFSET_TOD 0x00000002 +#define CNDM_CMD_PTP_FLG_SET_REL 0x00000004 +#define CNDM_CMD_PTP_FLG_OFFSET_REL 0x00000008 +#define CNDM_CMD_PTP_FLG_OFFSET_FNS 0x00000010 +#define CNDM_CMD_PTP_FLG_SET_PERIOD 0x00000080 + +struct cndm_cmd_ptp { + __le16 rsvd; + union { + __le16 opcode; + __le16 status; + }; + __le32 flags; + __le32 fns; + __le32 tod_ns; + + __le64 tod_sec; + __le64 rel_ns; + + __le64 ptm; + __le64 nom_period; + + __le64 period; + __le32 rsvd2[2]; +}; + +struct cndm_cmd_hwid { + __le16 rsvd; + union { + __le16 opcode; + __le16 status; + }; + __le32 flags; + __le16 index; + union { + __le16 brd_opcode; + __le16 brd_status; + }; + __le32 brd_flags; + __u8 page; + __u8 bank; + __u8 dev_addr_offset; + __u8 rsvd2; + __le32 addr; + __le32 len; + __le32 rsvd3; + __le32 data[8]; +}; + +struct cndm_cmd_queue { + __le16 rsvd; + union { + __le16 opcode; + __le16 status; + }; + __le32 flags; + __le32 port; + __le32 qn; + + __le32 qn2; + __le32 pd; + __le32 size; + __le32 dboffs; + + __le64 ptr1; + __le64 ptr2; + + __le32 prod; + __le32 cons; + __le32 dw14; + __le32 dw15; +}; + +struct cndm_desc { + __le16 rsvd0; + union { + struct { + __le16 csum_cmd; + } tx; + struct { + __le16 rsvd0; + } rx; + }; + + __le32 len; + __le64 addr; +}; + +struct cndm_cpl { + __u8 rsvd[4]; + __le32 len; + __le32 ts_ns; + __le16 ts_fns; + __u8 ts_s; + __u8 phase; +}; + +struct cndm_event { + __le16 rsvd0; + __le16 type; + __le32 source; + __le32 rsvd1; + __le32 phase; +}; + +#endif diff --git a/src/cndm/dpdk/cndm/cndm_main.c b/src/cndm/dpdk/cndm/cndm_main.c new file mode 100644 index 0000000..74a2a1c --- /dev/null +++ b/src/cndm/dpdk/cndm/cndm_main.c @@ -0,0 +1,340 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2025-2026 FPGA Ninja, LLC + * + * Authors: + * - Alex Forencich + */ + +#include "cndm.h" + +#include + +#include +#include +#include +#include + +static void cndm_common_remove(struct cndm_dev *cdev); + +static int cndm_common_probe(struct cndm_dev *cdev) +{ + int ret = 0; + + struct cndm_cmd_cfg cmd; + struct cndm_cmd_cfg rsp; + + rte_spinlock_init(&cdev->mbox_lock); + + // Read config page 0 + cmd.opcode = CNDM_CMD_OP_CFG; + cmd.flags = 0x00000000; + cmd.cfg_page = 0; + + ret = cndm_exec_cmd(cdev, &cmd, &rsp); + if (ret) { + DRV_LOG(NOTICE, "Failed to execute command"); + goto fail; + } + + if (rsp.status) { + DRV_LOG(NOTICE, "Command failed"); + ret = rsp.status; + goto fail; + } + + cdev->cfg_page_max = rsp.cfg_page_max; + cdev->cmd_ver = rsp.cmd_ver; + + DRV_LOG(NOTICE, "Config pages: %d", cdev->cfg_page_max+1); + DRV_LOG(NOTICE, "Command version: %d.%d.%d", cdev->cmd_ver >> 20, + (cdev->cmd_ver >> 12) & 0xff, + cdev->cmd_ver & 0xfff); + + // FW ID + cdev->fpga_id = rsp.p0.fpga_id; + cdev->fw_id = rsp.p0.fw_id; + cdev->fw_ver = rsp.p0.fw_ver; + cdev->board_id = rsp.p0.board_id; + cdev->board_ver = rsp.p0.board_ver; + cdev->build_date = rsp.p0.build_date; + cdev->git_hash = rsp.p0.git_hash; + cdev->release_info = rsp.p0.release_info; + + DRV_LOG(NOTICE, "FPGA ID: 0x%08x", cdev->fpga_id); + DRV_LOG(NOTICE, "FW ID: 0x%08x", cdev->fw_id); + DRV_LOG(NOTICE, "FW version: %d.%d.%d", cdev->fw_ver >> 20, + (cdev->fw_ver >> 12) & 0xff, + cdev->fw_ver & 0xfff); + DRV_LOG(NOTICE, "Board ID: 0x%08x", cdev->board_id); + DRV_LOG(NOTICE, "Board version: %d.%d.%d", cdev->board_ver >> 20, + (cdev->board_ver >> 12) & 0xff, + cdev->board_ver & 0xfff); + + time_t build_date = cdev->build_date; + struct tm *tm_info = gmtime(&build_date); + strftime(cdev->build_date_str, sizeof(cdev->build_date_str), "%F %T", tm_info); + + DRV_LOG(NOTICE, "Build date: %s UTC (raw: 0x%08x)", cdev->build_date_str, cdev->build_date); + DRV_LOG(NOTICE, "Git hash: %08x", cdev->git_hash); + DRV_LOG(NOTICE, "Release info: %08x", cdev->release_info); + + // Read config page 1 + cmd.opcode = CNDM_CMD_OP_CFG; + cmd.flags = 0x00000000; + cmd.cfg_page = 1; + + ret = cndm_exec_cmd(cdev, &cmd, &rsp); + if (ret) { + DRV_LOG(NOTICE, "Failed to execute command"); + goto fail; + } + + if (rsp.status) { + DRV_LOG(NOTICE, "Command failed"); + ret = rsp.status; + goto fail; + } + + // HW config + cdev->port_count = rsp.p1.port_count; + cdev->sys_clk_per_ns_num = rsp.p1.sys_clk_per_ns_num; + cdev->sys_clk_per_ns_den = rsp.p1.sys_clk_per_ns_den; + cdev->ptp_clk_per_ns_num = rsp.p1.ptp_clk_per_ns_num; + cdev->ptp_clk_per_ns_den = rsp.p1.ptp_clk_per_ns_den; + + DRV_LOG(NOTICE, "Port count: %d", cdev->port_count); + if (cdev->sys_clk_per_ns_num != 0) { + __u64 a, b, c; + a = (__u64)cdev->sys_clk_per_ns_den * 1000; + b = a / cdev->sys_clk_per_ns_num; + c = a - (b * cdev->sys_clk_per_ns_num); + c = (c * 1000000000) / cdev->sys_clk_per_ns_num; + DRV_LOG(NOTICE, "Sys clock freq: %lld.%09lld MHz (raw %d/%d ns)", b, c, cdev->sys_clk_per_ns_num, cdev->sys_clk_per_ns_den); + } + if (cdev->ptp_clk_per_ns_num != 0) { + __u64 a, b, c; + a = (__u64)cdev->ptp_clk_per_ns_den * 1000; + b = a / cdev->ptp_clk_per_ns_num; + c = a - (b * cdev->ptp_clk_per_ns_num); + c = (c * 1000000000) / cdev->ptp_clk_per_ns_num; + DRV_LOG(NOTICE, "PTP clock freq: %lld.%09lld MHz (raw %d/%d ns)", b, c, cdev->ptp_clk_per_ns_num, cdev->ptp_clk_per_ns_den); + } + + // Read config page 2 + cmd.opcode = CNDM_CMD_OP_CFG; + cmd.flags = 0x00000000; + cmd.cfg_page = 2; + + ret = cndm_exec_cmd(cdev, &cmd, &rsp); + if (ret) { + DRV_LOG(NOTICE, "Failed to execute command"); + goto fail; + } + + if (rsp.status) { + DRV_LOG(NOTICE, "Command failed"); + ret = rsp.status; + goto fail; + } + + // Resources + cdev->log_max_eq = rsp.p2.log_max_eq; + cdev->log_max_eq_sz = rsp.p2.log_max_eq_sz; + cdev->eq_pool = rsp.p2.eq_pool; + cdev->eqe_ver = rsp.p2.eqe_ver; + cdev->log_max_cq = rsp.p2.log_max_cq; + cdev->log_max_cq_sz = rsp.p2.log_max_cq_sz; + cdev->cq_pool = rsp.p2.cq_pool; + cdev->cqe_ver = rsp.p2.cqe_ver; + cdev->log_max_sq = rsp.p2.log_max_sq; + cdev->log_max_sq_sz = rsp.p2.log_max_sq_sz; + cdev->sq_pool = rsp.p2.sq_pool; + cdev->sqe_ver = rsp.p2.sqe_ver; + cdev->log_max_rq = rsp.p2.log_max_rq; + cdev->log_max_rq_sz = rsp.p2.log_max_rq_sz; + cdev->rq_pool = rsp.p2.rq_pool; + cdev->rqe_ver = rsp.p2.rqe_ver; + + DRV_LOG(NOTICE, "Max EQ count: %d (log %d)", 1 << cdev->log_max_eq, cdev->log_max_eq); + DRV_LOG(NOTICE, "Max EQ size: %d (log %d)", 1 << cdev->log_max_eq_sz, cdev->log_max_eq_sz); + DRV_LOG(NOTICE, "EQ pool: %d", cdev->eq_pool); + DRV_LOG(NOTICE, "EQE version: %d", cdev->eqe_ver); + DRV_LOG(NOTICE, "Max CQ count: %d (log %d)", 1 << cdev->log_max_cq, cdev->log_max_cq); + DRV_LOG(NOTICE, "Max CQ size: %d (log %d)", 1 << cdev->log_max_cq_sz, cdev->log_max_cq_sz); + DRV_LOG(NOTICE, "CQ pool: %d", cdev->cq_pool); + DRV_LOG(NOTICE, "CQE version: %d", cdev->cqe_ver); + DRV_LOG(NOTICE, "Max SQ count: %d (log %d)", 1 << cdev->log_max_sq, cdev->log_max_sq); + DRV_LOG(NOTICE, "Max SQ size: %d (log %d)", 1 << cdev->log_max_sq_sz, cdev->log_max_sq_sz); + DRV_LOG(NOTICE, "SQ pool: %d", cdev->sq_pool); + DRV_LOG(NOTICE, "SQE version: %d", cdev->sqe_ver); + DRV_LOG(NOTICE, "Max RQ count: %d (log %d)", 1 << cdev->log_max_rq, cdev->log_max_rq); + DRV_LOG(NOTICE, "Max RQ size: %d (log %d)", 1 << cdev->log_max_rq_sz, cdev->log_max_rq_sz); + DRV_LOG(NOTICE, "RQ pool: %d", cdev->rq_pool); + DRV_LOG(NOTICE, "RQE version: %d", cdev->rqe_ver); + + DRV_LOG(NOTICE, "Read HW IDs"); + + ret = cndm_hwid_sn_rd(cdev, NULL, &cdev->sn_str); + if (ret || !strlen(cdev->sn_str)) { + DRV_LOG(NOTICE, "No readable serial number"); + } else { + DRV_LOG(NOTICE, "SN: %s", cdev->sn_str); + } + + ret = cndm_hwid_mac_rd(cdev, 0, &cdev->mac_cnt, &cdev->base_mac); + if (ret) { + DRV_LOG(NOTICE, "No readable MACs"); + cdev->mac_cnt = 0; + } else if (!rte_is_valid_assigned_ether_addr(&cdev->base_mac)) { + DRV_LOG(WARNING + , "Base MAC is invalid"); + cdev->mac_cnt = 0; + } else { + DRV_LOG(NOTICE, "MAC count: %d", cdev->mac_cnt); + DRV_LOG(NOTICE, "Base MAC: " RTE_ETHER_ADDR_PRT_FMT, + RTE_ETHER_ADDR_BYTES(&cdev->base_mac)); + } + + if (cdev->port_count > ARRAY_SIZE(cdev->eth_dev)) + cdev->port_count = ARRAY_SIZE(cdev->eth_dev); + + for (__u32 k = 0; k < cdev->port_count; k++) { + struct rte_eth_dev *eth_dev; + + eth_dev = cndm_create_eth_dev(cdev, k); + if (!eth_dev) { + ret = -1; + goto fail_eth_dev; + } + + cdev->eth_dev[k] = eth_dev; + } + +fail_eth_dev: + return 0; + +fail: + cndm_common_remove(cdev); + return ret; +} + +static void cndm_common_remove(struct cndm_dev *cdev) +{ + for (size_t k = 0; k < ARRAY_SIZE(cdev->eth_dev); k++) { + if (cdev->eth_dev[k]) { + cndm_destroy_eth_dev(cdev->eth_dev[k]); + cdev->eth_dev[k] = NULL; + } + } +} + +static int cndm_pci_probe(struct rte_pci_driver *pdrv __rte_unused, struct rte_pci_device *pdev) +{ + struct cndm_dev *cdev; + off_t pcie_cap; + int ret = 0; + + DRV_LOG(NOTICE, "PCI probe"); + DRV_LOG(NOTICE, "Corundum DPDK PMD"); + DRV_LOG(NOTICE, "Version " DRIVER_VERSION); + DRV_LOG(NOTICE, "Copyright (c) 2026 FPGA Ninja, LLC"); + DRV_LOG(NOTICE, "https://fpga.ninja/"); + DRV_LOG(NOTICE, "PCIe configuration summary:"); + + pcie_cap = rte_pci_find_capability(pdev, RTE_PCI_CAP_ID_EXP); + + if (pcie_cap) { + __u16 devctl; + __u32 lnkcap; + __u16 lnkctl; + __u16 lnksta; + + rte_pci_read_config(pdev, &devctl, 2, pcie_cap + RTE_PCI_EXP_DEVCTL); + rte_pci_read_config(pdev, &lnkcap, 4, pcie_cap + RTE_PCI_EXP_LNKCAP); + rte_pci_read_config(pdev, &lnkctl, 2, pcie_cap + RTE_PCI_EXP_LNKCTL); + rte_pci_read_config(pdev, &lnksta, 2, pcie_cap + RTE_PCI_EXP_LNKSTA); + + DRV_LOG(NOTICE, " Max payload size: %d bytes", + 128 << ((devctl & RTE_PCI_EXP_DEVCTL_PAYLOAD) >> 5)); + DRV_LOG(NOTICE, " Max read request size: %d bytes", + 128 << ((devctl & RTE_PCI_EXP_DEVCTL_READRQ) >> 12)); + DRV_LOG(NOTICE, " Read completion boundary: %d bytes", + lnkctl & RTE_PCI_EXP_LNKCTL_RCB ? 128 : 64); + DRV_LOG(NOTICE, " Link capability: gen %d x%d", + lnkcap & RTE_PCI_EXP_LNKCAP_SLS, (lnkcap & RTE_PCI_EXP_LNKCAP_MLW) >> 4); + DRV_LOG(NOTICE, " Link status: gen %d x%d", + lnksta & RTE_PCI_EXP_LNKSTA_CLS, (lnksta & RTE_PCI_EXP_LNKSTA_NLW) >> 4); + DRV_LOG(NOTICE, " Relaxed ordering: %s", + devctl & RTE_PCI_EXP_DEVCTL_RELAX_EN ? "enabled" : "disabled"); + DRV_LOG(NOTICE, " Phantom functions: %s", + devctl & RTE_PCI_EXP_DEVCTL_PHANTOM ? "enabled" : "disabled"); + DRV_LOG(NOTICE, " Extended tags: %s", + devctl & RTE_PCI_EXP_DEVCTL_EXT_TAG ? "enabled" : "disabled"); + DRV_LOG(NOTICE, " No snoop: %s", + devctl & RTE_PCI_EXP_DEVCTL_NOSNOOP_EN ? "enabled" : "disabled"); + } + + DRV_LOG(NOTICE, " NUMA node: %d", pdev->device.numa_node); + + cdev = rte_zmalloc_socket(pdev->device.name, + sizeof(struct cndm_dev), RTE_CACHE_LINE_SIZE, + pdev->device.numa_node); + + if (!cdev) + return -ENOMEM; + + cdev->pdev = pdev; + + // TODO store cdev reference + + cdev->hw_regs_size = pdev->mem_resource[0].len; + cdev->hw_regs_phys = pdev->mem_resource[0].phys_addr; + cdev->hw_addr = pdev->mem_resource[0].addr; + + DRV_LOG(NOTICE, "Control BAR size: %lu", cdev->hw_regs_size); + if (!cdev->hw_addr) { + ret = -ENOMEM; + DRV_LOG(ERR, "Failed to map control BAR"); + goto fail; + } + + if (rte_read32(pdev->mem_resource[0].addr) == 0xffffffff) { + ret = -EIO; + DRV_LOG(ERR, "Device needs to be reset"); + goto fail; + } + + ret = cndm_common_probe(cdev); + if (ret) + goto fail; + + return 0; + +fail: + rte_free(cdev); + return ret; +} + +static int cndm_pci_remove(struct rte_pci_device *pdev __rte_unused) +{ + DRV_LOG(NOTICE, "PCI remove"); + + return 0; +} + +static const struct rte_pci_id pci_id_cndm_map[] = { + {RTE_PCI_DEVICE(0x1234, 0xC001)}, + {0} +}; + +static struct rte_pci_driver rte_cndm_pmd = { + .id_table = pci_id_cndm_map, + .drv_flags = RTE_PCI_DRV_NEED_MAPPING, + .probe = cndm_pci_probe, + .remove = cndm_pci_remove, +}; + +RTE_PMD_REGISTER_PCI(net_cndm, rte_cndm_pmd); +RTE_PMD_REGISTER_PCI_TABLE(net_cndm, pci_id_cndm_map); +RTE_PMD_REGISTER_KMOD_DEP(net_cndm, "* vfio-pci"); +RTE_LOG_REGISTER_DEFAULT(cndm_logtype_driver, NOTICE); diff --git a/src/cndm/dpdk/cndm/cndm_rq.c b/src/cndm/dpdk/cndm/cndm_rq.c new file mode 100644 index 0000000..86c15ae --- /dev/null +++ b/src/cndm/dpdk/cndm/cndm_rq.c @@ -0,0 +1,329 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2025-2026 FPGA Ninja, LLC + * + * Authors: + * - Alex Forencich + */ + +#include "cndm.h" + +#include +#include + +struct cndm_ring *cndm_create_rq(struct cndm_priv *priv, unsigned int socket_id) +{ + struct cndm_ring *rq; + + rq = rte_zmalloc_socket("cndm_rq", sizeof(*rq), 0, socket_id); + if (!rq) + return NULL; + + rq->cdev = priv->cdev; + rq->priv = priv; + rq->socket_id = socket_id; + + rq->index = -1; + rq->enabled = 0; + + rq->prod_ptr = 0; + rq->cons_ptr = 0; + + rq->db_offset = 0; + rq->db_addr = NULL; + + return rq; +} + +void cndm_destroy_rq(struct cndm_ring *rq) +{ + cndm_close_rq(rq); + + rte_free(rq); +} + +int cndm_open_rq(struct cndm_ring *rq, struct cndm_priv *priv, struct cndm_cq *cq, int size) +{ + int ret = 0; + + struct cndm_cmd_queue cmd; + struct cndm_cmd_queue rsp; + + if (rq->enabled || rq->buf || !priv || !cq) + return -EINVAL; + + rq->size = rte_align32pow2(size); + rq->size_mask = rq->size - 1; + rq->stride = 16; + + rq->rx_info = rte_zmalloc_socket("cndm_rq_info_ring", + sizeof(*rq->rx_info) * rq->size, + RTE_CACHE_LINE_SIZE, rq->socket_id); + if (!rq->rx_info) + return -ENOMEM; + + rq->buf_size = rq->size * rq->stride; + rq->buf = rte_zmalloc_socket("cndm_rq_ring", rq->buf_size, + RTE_CACHE_LINE_SIZE, rq->socket_id); + if (!rq->buf) { + ret = -ENOMEM; + goto fail; + } + rq->buf_dma_addr = rte_malloc_virt2iova(rq->buf); + + rq->priv = priv; + rq->cq = cq; + cq->src_ring = rq; + + rq->prod_ptr = 0; + rq->cons_ptr = 0; + + cmd.opcode = CNDM_CMD_OP_CREATE_RQ; + cmd.flags = 0x00000000; + cmd.port = rq->priv->dev_port; + cmd.qn = 0; + cmd.qn2 = cq->cqn; + cmd.pd = 0; + cmd.size = rte_log2_u32(rq->size); + cmd.dboffs = 0; + cmd.ptr1 = rq->buf_dma_addr; + cmd.ptr2 = 0; + + ret = cndm_exec_cmd(rq->cdev, &cmd, &rsp); + if (ret) { + DRV_LOG(ERR, "Failed to execute command"); + goto fail; + } + + if (rsp.status || rsp.dboffs == 0) { + DRV_LOG(ERR, "Failed to allocate RQ"); + ret = rsp.status; + goto fail; + } + + rq->index = rsp.qn; + rq->db_offset = rsp.dboffs; + rq->db_addr = rq->cdev->hw_addr + rsp.dboffs; + + rq->enabled = 1; + + DRV_LOG(DEBUG, "Opened RQ %d (CQ %d)", rq->index, cq->cqn); + + ret = cndm_refill_rx_buffers(rq); + if (ret) { + DRV_LOG(ERR, "failed to allocate RX buffer for RX queue index %d (of %u total) entry index %u (of %u total)", + rq->index, priv->rxq_count, rq->prod_ptr, rq->size); + if (ret == -ENOMEM) + DRV_LOG(ERR, "machine might not have enough DMA-capable RAM; try to decrease number of RX channels (currently %u) and/or RX ring parameters (entries; currently %u)", + priv->rxq_count, rq->size); + + goto fail; + } + + return 0; + +fail: + cndm_close_rq(rq); + return ret; +} + +void cndm_close_rq(struct cndm_ring *rq) +{ + struct cndm_dev *cdev = rq->cdev; + struct cndm_cmd_queue cmd; + struct cndm_cmd_queue rsp; + + rq->enabled = 0; + + if (rq->cq) { + rq->cq->src_ring = NULL; + rq->cq->handler = NULL; + } + + rq->cq = NULL; + + if (rq->index != -1) { + cmd.opcode = CNDM_CMD_OP_DESTROY_RQ; + cmd.flags = 0x00000000; + cmd.port = rq->priv->dev_port; + cmd.qn = rq->index; + + cndm_exec_cmd(cdev, &cmd, &rsp); + + rq->index = -1; + rq->db_offset = 0; + rq->db_addr = NULL; + } + + if (rq->buf) { + // cndm_free_rx_buf(rq); // TODO!!! + + rte_free(rq->buf); + rq->buf = NULL; + rq->buf_dma_addr = 0; + } + + if (rq->rx_info) { + rte_free(rq->rx_info); + rq->rx_info = NULL; + } + + rq->priv = NULL; +} + +bool cndm_is_rq_ring_empty(const struct cndm_ring *rq) +{ + return rq->prod_ptr == rq->cons_ptr; +} + +bool cndm_is_rq_ring_full(const struct cndm_ring *rq) +{ + return (rq->prod_ptr - rq->cons_ptr) >= rq->size; +} + +void cndm_rq_write_prod_ptr(const struct cndm_ring *rq) +{ + rte_write32(rq->prod_ptr & 0xffff, rq->db_addr); +} + +// static void cndm_free_rx_desc(struct cndm_ring *rq, int index) +// { +// struct cndm_priv *priv = rq->priv; +// struct device *dev = priv->dev; +// struct cndm_rx_info *rx_info = &rq->rx_info[index]; + +// DRV_LOG(DEBUG, "Free RX desc index %d", index); + +// if (!rx_info->page) +// return; + +// dma_unmap_page(dev, rx_info->dma_addr, rx_info->len, DMA_FROM_DEVICE); +// rx_info->dma_addr = 0; +// __free_pages(rx_info->page, 0); +// rx_info->page = NULL; +// } + +// int cndm_free_rx_buf(struct cndm_ring *rq) +// { +// u32 index; +// int cnt = 0; + +// while (!cndm_is_rq_ring_empty(rq)) { +// index = rq->cons_ptr & rq->size_mask; +// cndm_free_rx_desc(rq, index); +// rq->cons_ptr++; +// cnt++; +// } + +// return cnt; +// } + +#define BATCH_SIZE 32 + +int cndm_refill_rx_buffers(struct cndm_ring *rq) +{ + __u32 missing = rq->size - (rq->prod_ptr - rq->cons_ptr); + int ret = 0; + struct rte_mbuf *mbufs[BATCH_SIZE]; + int batch_count; + __u32 index; + struct cndm_desc *rx_desc; + struct cndm_rx_info *rx_info; + struct rte_mbuf *mbuf; + + if (missing < 8) + return 0; + + while (missing > 0) { + batch_count = RTE_MIN((int)missing, BATCH_SIZE); + ret = rte_pktmbuf_alloc_bulk(rq->mp, mbufs, batch_count); + if (ret) { + DRV_LOG(ERR, "Failed to allocate mbufs"); + goto done; + } + + for (int i = 0; i < batch_count; i++) { + mbuf = mbufs[i]; + index = rq->prod_ptr & rq->size_mask; + + rx_desc = (struct cndm_desc *)(rq->buf + index*rq->stride); + rx_info = &rq->rx_info[index]; + + rx_desc->len = rte_cpu_to_le_32(rte_pktmbuf_data_room_size(rq->mp) - RTE_PKTMBUF_HEADROOM); + rx_desc->addr = rte_cpu_to_le_64(rte_pktmbuf_iova(mbuf)); + + rx_info->mbuf = mbuf; + + rq->prod_ptr++; + missing--; + } + } + +done: + rte_io_wmb(); + cndm_rq_write_prod_ptr(rq); + + return ret; +} + +uint16_t cndm_recv_pkt_burst(void *queue, struct rte_mbuf **pkts, uint16_t nb_pkts) +{ + struct cndm_ring *rq = queue; + struct cndm_cq *cq = rq->cq; + __u32 index; + uint16_t pkt_recv = 0; + struct cndm_cpl *cpl; + struct cndm_rx_info *rx_info; + struct rte_mbuf *mbuf; + + __u32 len; + __u32 cq_cons_ptr; + __u32 cq_index; + __u32 cons_ptr; + + cq_cons_ptr = cq->cons_ptr; + cons_ptr = rq->cons_ptr; + + while (pkt_recv < nb_pkts) { + cq_index = cq_cons_ptr & cq->size_mask; + cpl = (struct cndm_cpl *)(cq->buf + cq_index*cq->stride); + + if (!!(cpl->phase & 0x80) == !!(cq_cons_ptr & cq->size)) + break; + + rte_io_rmb(); + + index = cons_ptr & rq->size_mask; + rx_info = &rq->rx_info[index]; + mbuf = rx_info->mbuf; + + len = RTE_MIN(rte_le_to_cpu_32(cpl->len), rte_pktmbuf_data_room_size(rq->mp) - RTE_PKTMBUF_HEADROOM); + + if (!mbuf) { + DRV_LOG(ERR, "null mbuf at index %d", index); + } else { + mbuf->nb_segs = 1; + mbuf->next = NULL; + mbuf->data_len = len; + mbuf->pkt_len = len; + mbuf->port = rq->priv->port_id; + + pkts[pkt_recv] = mbuf; + rx_info->mbuf = NULL; + } + + cq_cons_ptr++; + cons_ptr++; + pkt_recv++; + } + + cq->cons_ptr = cq_cons_ptr; + rq->cons_ptr = cons_ptr; + + cndm_cq_write_cons_ptr(cq); + + if (pkt_recv) { + cndm_refill_rx_buffers(rq); + } + + return pkt_recv; +} diff --git a/src/cndm/dpdk/cndm/cndm_sq.c b/src/cndm/dpdk/cndm/cndm_sq.c new file mode 100644 index 0000000..8e6ffbd --- /dev/null +++ b/src/cndm/dpdk/cndm/cndm_sq.c @@ -0,0 +1,285 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2025-2026 FPGA Ninja, LLC + * + * Authors: + * - Alex Forencich + */ + +#include "cndm.h" + +#include +#include + +struct cndm_ring *cndm_create_sq(struct cndm_priv *priv, unsigned int socket_id) +{ + struct cndm_ring *sq; + + sq = rte_zmalloc_socket("cndm_sq", sizeof(*sq), 0, socket_id); + if (!sq) + return NULL; + + sq->cdev = priv->cdev; + sq->priv = priv; + sq->socket_id = socket_id; + + sq->index = -1; + sq->enabled = 0; + + sq->prod_ptr = 0; + sq->cons_ptr = 0; + + sq->db_offset = 0; + sq->db_addr = NULL; + + return sq; +} + +void cndm_destroy_sq(struct cndm_ring *sq) +{ + cndm_close_sq(sq); + + rte_free(sq); +} + +int cndm_open_sq(struct cndm_ring *sq, struct cndm_priv *priv, struct cndm_cq *cq, int size) +{ + int ret = 0; + + struct cndm_cmd_queue cmd; + struct cndm_cmd_queue rsp; + + if (sq->enabled || sq->buf || !priv || !cq) + return -EINVAL; + + sq->size = rte_align32pow2(size); + sq->size_mask = sq->size - 1; + sq->stride = 16; + + sq->tx_info = rte_zmalloc_socket("cndm_sq_info_ring", + sizeof(*sq->tx_info) * sq->size, + RTE_CACHE_LINE_SIZE, sq->socket_id); + if (!sq->tx_info) + return -ENOMEM; + + sq->buf_size = sq->size * sq->stride; + sq->buf = rte_zmalloc_socket("cndm_sq_ring", sq->buf_size, + RTE_CACHE_LINE_SIZE, sq->socket_id); + if (!sq->buf) { + ret = -ENOMEM; + goto fail; + } + sq->buf_dma_addr = rte_malloc_virt2iova(sq->buf); + + sq->priv = priv; + sq->cq = cq; + cq->src_ring = sq; + + sq->prod_ptr = 0; + sq->cons_ptr = 0; + + cmd.opcode = CNDM_CMD_OP_CREATE_SQ; + cmd.flags = 0x00000000; + cmd.port = sq->priv->dev_port; + cmd.qn = 0; + cmd.qn2 = cq->cqn; + cmd.pd = 0; + cmd.size = rte_log2_u32(sq->size); + cmd.dboffs = 0; + cmd.ptr1 = sq->buf_dma_addr; + cmd.ptr2 = 0; + + ret = cndm_exec_cmd(sq->cdev, &cmd, &rsp); + if (ret) { + DRV_LOG(ERR, "Failed to execute command"); + goto fail; + } + + if (rsp.status || rsp.dboffs == 0) { + DRV_LOG(ERR, "Failed to allocate SQ"); + ret = rsp.status; + goto fail; + } + + sq->index = rsp.qn; + sq->db_offset = rsp.dboffs; + sq->db_addr = sq->cdev->hw_addr + rsp.dboffs; + + sq->enabled = 1; + + DRV_LOG(DEBUG, "Opened SQ %d (CQ %d)", sq->index, cq->cqn); + + return 0; + +fail: + cndm_close_sq(sq); + return ret; +} + +void cndm_close_sq(struct cndm_ring *sq) +{ + struct cndm_dev *cdev = sq->cdev; + struct cndm_cmd_queue cmd; + struct cndm_cmd_queue rsp; + + sq->enabled = 0; + + if (sq->cq) { + sq->cq->src_ring = NULL; + sq->cq->handler = NULL; + } + + sq->cq = NULL; + + if (sq->index != -1) { + cmd.opcode = CNDM_CMD_OP_DESTROY_SQ; + cmd.flags = 0x00000000; + cmd.port = sq->priv->dev_port; + cmd.qn = sq->index; + + cndm_exec_cmd(cdev, &cmd, &rsp); + + sq->index = -1; + sq->db_offset = 0; + sq->db_addr = NULL; + } + + if (sq->buf) { + // cndm_free_tx_buf(sq); // TODO!!! + + rte_free(sq->buf); + sq->buf = NULL; + sq->buf_dma_addr = 0; + } + + if (sq->tx_info) { + rte_free(sq->tx_info); + sq->tx_info = NULL; + } + + sq->priv = NULL; +} + +bool cndm_is_sq_ring_empty(const struct cndm_ring *sq) +{ + return sq->prod_ptr == sq->cons_ptr; +} + +bool cndm_is_sq_ring_full(const struct cndm_ring *sq) +{ + return (sq->prod_ptr - sq->cons_ptr) >= sq->size; +} + +void cndm_sq_write_prod_ptr(const struct cndm_ring *sq) +{ + rte_write32(sq->prod_ptr & 0xffff, sq->db_addr); +} + +// static void cndm_free_tx_desc(struct cndm_ring *sq, int index, int napi_budget) +// { +// struct cndm_priv *priv = sq->priv; +// struct cndm_tx_info *tx_info = &sq->tx_info[index]; +// struct sk_buff *skb = tx_info->skb; + +// DRV_LOG(DEBUG, "Free TX desc index %d", index); + +// dma_unmap_single(dev, tx_info->dma_addr, tx_info->len, DMA_TO_DEVICE); +// tx_info->dma_addr = 0; + +// napi_consume_skb(skb, napi_budget); +// tx_info->skb = NULL; +// } + +// int cndm_free_tx_buf(struct cndm_ring *sq) +// { +// __u32 index; +// int cnt = 0; + +// while (!cndm_is_sq_ring_empty(sq)) { +// index = sq->cons_ptr & sq->size_mask; +// cndm_free_tx_desc(sq, index, 0); +// sq->cons_ptr++; +// cnt++; +// } + +// return cnt; +// } + +uint16_t cndm_xmit_pkt_burst(void *queue, struct rte_mbuf **pkts, uint16_t nb_pkts) +{ + struct cndm_ring *sq = queue; + struct cndm_cq *cq = sq->cq; + __u32 index; + uint16_t pkt_sent = 0; + struct cndm_desc *tx_desc; + struct cndm_cpl *cpl; + struct cndm_tx_info *tx_info; + + __u32 cq_cons_ptr; + __u32 cq_index; + __u32 cons_ptr; + + for (uint16_t pkt_idx = 0; pkt_idx < nb_pkts; pkt_idx++) { + struct rte_mbuf *pkt_mbuf = pkts[pkt_idx]; + + if (sq->prod_ptr - sq->cons_ptr >= sq->size) { + DRV_LOG(DEBUG, "TX ring full"); + break; + } + + if (pkt_mbuf->nb_segs > 1) { + DRV_LOG(ERR, "too many segments (%d > 1)", pkt_mbuf->nb_segs); + // TODO free mbuf, perhaps? + continue; + } + + index = sq->prod_ptr & sq->size_mask; + + tx_desc = (struct cndm_desc *)(sq->buf + index*sq->stride); + tx_info = &sq->tx_info[index]; + + tx_desc->len = rte_cpu_to_le_32(pkt_mbuf->data_len); + tx_desc->addr = rte_cpu_to_le_64(rte_pktmbuf_iova(pkt_mbuf)); + + tx_info->mbuf = pkt_mbuf; + + sq->prod_ptr++; + pkt_sent++; + } + + if (pkt_sent) { + rte_io_wmb(); + cndm_sq_write_prod_ptr(sq); + } + + cq_cons_ptr = cq->cons_ptr; + cons_ptr = sq->cons_ptr; + + while (true) { + cq_index = cq_cons_ptr & cq->size_mask; + cpl = (struct cndm_cpl *)(cq->buf + cq_index*cq->stride); + + if (!!(cpl->phase & 0x80) == !!(cq_cons_ptr & cq->size)) + break; + + rte_io_rmb(); + + index = cons_ptr & sq->size_mask; + tx_info = &sq->tx_info[index]; + + if (!tx_info->mbuf) { + DRV_LOG(ERR, "null mbuf at index %d", index); + } else { + rte_pktmbuf_free(tx_info->mbuf); + } + + cq_cons_ptr++; + cons_ptr++; + } + + cq->cons_ptr = cq_cons_ptr; + sq->cons_ptr = cons_ptr; + + cndm_cq_write_cons_ptr(cq); + + return pkt_sent; +} diff --git a/src/cndm/dpdk/cndm/meson.build b/src/cndm/dpdk/cndm/meson.build new file mode 100644 index 0000000..c6d97fb --- /dev/null +++ b/src/cndm/dpdk/cndm/meson.build @@ -0,0 +1,17 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright 2026 FPGA Ninja, LLC + +if not is_linux + build = false + reason = 'only supported on Linux' + subdir_done() +endif + +sources = files( + 'cndm_main.c', + 'cndm_cmd.c', + 'cndm_ethdev.c', + 'cndm_cq.c', + 'cndm_sq.c', + 'cndm_rq.c', +)