cndm: Make IRQ assignments configurable, add IRQ rate limiter

Signed-off-by: Alex Forencich <alex@alexforencich.com>

src/cndm/rtl/cndm_lite_core.f

@@ -20,3 +20,4 @@ cndm_micro_cpl_wr.sv
 ../lib/taxi/src/axis/rtl/taxi_axis_demux.sv
 ../lib/taxi/src/ptp/rtl/taxi_ptp_td_phc_apb.f
 ../lib/taxi/src/ptp/rtl/taxi_ptp_td_rel2tod.sv
+../lib/taxi/src/pcie/rtl/taxi_irq_rate_limit.sv

src/cndm/rtl/cndm_lite_core.sv

@@ -68,7 +68,10 @@ module cndm_lite_core #(
     taxi_dma_ram_if.wr_slv         dma_ram_wr,
     taxi_dma_ram_if.rd_slv         dma_ram_rd,
 
-    output wire logic [PORTS-1:0]  irq,
+    /*
+     * Interrupts
+     */
+    taxi_axis_if.src               m_axis_irq,
 
     /*
      * PTP
@@ -117,6 +120,8 @@ localparam PORT_OFFSET_HOST = 2;
 localparam PORT_BASE_ADDR_DP = PTP_TS_EN ? 32'h00010000 : 32'h00000000;
 localparam PORT_BASE_ADDR_HOST = 32'h00020000;
 
+localparam SYS_CLK_CYC_PER_US = (1000*SYS_CLK_PER_NS_DEN+SYS_CLK_PER_NS_NUM-1)/SYS_CLK_PER_NS_NUM;
+
 taxi_axil_if #(
     .DATA_W(s_axil_ctrl_wr.DATA_W),
     .ADDR_W(16),
@@ -517,6 +522,54 @@ dma_mux_inst (
     .client_ram_rd(dma_ram_int)
 );
 
+taxi_axis_if #(
+    .DATA_W(m_axis_irq.DATA_W),
+    .KEEP_EN(0),
+    .KEEP_W(1)
+) axis_irq_int(), axis_irq_port[PORTS]();
+
+taxi_axis_arb_mux #(
+    .S_COUNT($size(axis_irq_port)),
+    .ARB_ROUND_ROBIN(1),
+    .ARB_LSB_HIGH_PRIO(1)
+)
+irq_mux_inst (
+    .clk(clk),
+    .rst(rst),
+
+    /*
+     * AXI4-Stream input (sink)
+     */
+    .s_axis(axis_irq_port),
+
+    /*
+     * AXI4-Stream output (source)
+     */
+    .m_axis(axis_irq_int)
+);
+
+taxi_irq_rate_limit
+irq_rate_limit_inst (
+    .clk(clk),
+    .rst(rst),
+
+    /*
+     * Interrupt request input
+     */
+    .s_axis_irq(axis_irq_int),
+
+    /*
+     * Interrupt request output
+     */
+    .m_axis_irq(m_axis_irq),
+
+    /*
+     * Configuration
+     */
+    .prescale(16'(SYS_CLK_CYC_PER_US)),
+    .min_interval(10) // TODO make configurable
+);
+
 for (genvar p = 0; p < PORTS; p = p + 1) begin : port
 
     cndm_micro_port #(
@@ -553,7 +606,10 @@ for (genvar p = 0; p < PORTS; p = p + 1) begin : port
         .dma_ram_wr(dma_ram_int[p]),
         .dma_ram_rd(dma_ram_int[p]),
 
-        .irq(irq[p]),
+        /*
+         * Interrupts
+         */
+        .m_axis_irq(axis_irq_port[p]),
 
         /*
          * PTP

src/cndm/rtl/cndm_lite_pcie_us.sv

@@ -459,8 +459,25 @@ cfg_inst (
     .cfg_mgmt_read_write_done(cfg_mgmt_read_write_done)
 );
 
-wire [PORTS-1:0] irq;
-wire [31:0] msi_irq = 32'(irq);
+localparam IRQN_W = $clog2(32);
+
+taxi_axis_if #(
+    .DATA_W(IRQN_W),
+    .KEEP_EN(0),
+    .KEEP_W(1)
+) axis_irq();
+
+logic [31:0] msi_irq_reg = '0;
+
+assign axis_irq.tready = 1'b1;
+
+always @(posedge pcie_clk) begin
+    msi_irq_reg <= '0;
+
+    if (axis_irq.tvalid) begin
+        msi_irq_reg[axis_irq.tdata] <= 1'b1;
+    end
+end
 
 taxi_pcie_us_msi #(
     .MSI_CNT(32)
@@ -472,7 +489,7 @@ msi_inst (
     /*
      * Interrupt request inputs
      */
-    .msi_irq(msi_irq),
+    .msi_irq(msi_irq_reg),
 
     /*
      * Interface to UltraScale PCIe IP core
@@ -547,7 +564,10 @@ core_inst (
     .dma_ram_wr(dma_ram),
     .dma_ram_rd(dma_ram),
 
-    .irq(irq),
+    /*
+     * Interrupts
+     */
+    .m_axis_irq(axis_irq),
 
     /*
      * PTP

src/cndm/rtl/cndm_micro_core.f

@@ -20,3 +20,4 @@ cndm_micro_cpl_wr.sv
 ../lib/taxi/src/axis/rtl/taxi_axis_demux.sv
 ../lib/taxi/src/ptp/rtl/taxi_ptp_td_phc_apb.f
 ../lib/taxi/src/ptp/rtl/taxi_ptp_td_rel2tod.sv
+../lib/taxi/src/pcie/rtl/taxi_irq_rate_limit.sv

src/cndm/rtl/cndm_micro_core.sv

@@ -68,7 +68,10 @@ module cndm_micro_core #(
     taxi_dma_ram_if.wr_slv         dma_ram_wr,
     taxi_dma_ram_if.rd_slv         dma_ram_rd,
 
-    output wire logic [PORTS-1:0]  irq,
+    /*
+     * Interrupts
+     */
+    taxi_axis_if.src               m_axis_irq,
 
     /*
      * PTP
@@ -117,6 +120,8 @@ localparam PORT_OFFSET_HOST = 2;
 localparam PORT_BASE_ADDR_DP = PTP_TS_EN ? 32'h00010000 : 32'h00000000;
 localparam PORT_BASE_ADDR_HOST = 32'h00020000;
 
+localparam SYS_CLK_CYC_PER_US = (1000*SYS_CLK_PER_NS_DEN+SYS_CLK_PER_NS_NUM-1)/SYS_CLK_PER_NS_NUM;
+
 taxi_axil_if #(
     .DATA_W(s_axil_ctrl_wr.DATA_W),
     .ADDR_W(16),
@@ -517,6 +522,54 @@ dma_mux_inst (
     .client_ram_rd(dma_ram_int)
 );
 
+taxi_axis_if #(
+    .DATA_W(m_axis_irq.DATA_W),
+    .KEEP_EN(0),
+    .KEEP_W(1)
+) axis_irq_int(), axis_irq_port[PORTS]();
+
+taxi_axis_arb_mux #(
+    .S_COUNT($size(axis_irq_port)),
+    .ARB_ROUND_ROBIN(1),
+    .ARB_LSB_HIGH_PRIO(1)
+)
+irq_mux_inst (
+    .clk(clk),
+    .rst(rst),
+
+    /*
+     * AXI4-Stream input (sink)
+     */
+    .s_axis(axis_irq_port),
+
+    /*
+     * AXI4-Stream output (source)
+     */
+    .m_axis(axis_irq_int)
+);
+
+taxi_irq_rate_limit
+irq_rate_limit_inst (
+    .clk(clk),
+    .rst(rst),
+
+    /*
+     * Interrupt request input
+     */
+    .s_axis_irq(axis_irq_int),
+
+    /*
+     * Interrupt request output
+     */
+    .m_axis_irq(m_axis_irq),
+
+    /*
+     * Configuration
+     */
+    .prescale(16'(SYS_CLK_CYC_PER_US)),
+    .min_interval(10) // TODO make configurable
+);
+
 for (genvar p = 0; p < PORTS; p = p + 1) begin : port
 
     cndm_micro_port #(
@@ -553,7 +606,10 @@ for (genvar p = 0; p < PORTS; p = p + 1) begin : port
         .dma_ram_wr(dma_ram_int[p]),
         .dma_ram_rd(dma_ram_int[p]),
 
-        .irq(irq[p]),
+        /*
+         * Interrupts
+         */
+        .m_axis_irq(axis_irq_port[p]),
 
         /*
          * PTP

src/cndm/rtl/cndm_micro_cpl_wr.sv

@@ -40,15 +40,22 @@ module cndm_micro_cpl_wr #(
     taxi_dma_desc_if.sts_snk  dma_wr_desc_sts,
     taxi_dma_ram_if.rd_slv    dma_ram_rd,
 
-    taxi_axis_if.snk          s_axis_cpl,
-    output wire logic         irq
+    /*
+     * Interrupts
+     */
+    taxi_axis_if.src          m_axis_irq,
+
+    taxi_axis_if.snk          s_axis_cpl
 );
 
 localparam DMA_ADDR_W = dma_wr_desc_req.DST_ADDR_W;
 
+localparam IRQN_W = m_axis_irq.DATA_W;
+
 logic [CQN_W-1:0]       cq_req_cqn_reg = '0;
 logic                   cq_req_valid_reg = 1'b0;
 logic                   cq_req_ready;
+logic [IRQN_W-1:0]      cq_rsp_irqn;
 logic [DMA_ADDR_W-1:0]  cq_rsp_addr;
 logic                   cq_rsp_phase_tag;
 logic                   cq_rsp_error;
@@ -57,7 +64,7 @@ logic                   cq_rsp_ready_reg = 1'b0;
 
 cndm_micro_queue_state #(
     .QN_W(CQN_W),
-    .DQN_W(CQN_W), // TODO
+    .DQN_W(IRQN_W),
     .IS_CQ(1),
     .QTYPE_EN(0),
     .QE_SIZE(16),
@@ -86,7 +93,7 @@ cq_mgr_inst (
     .req_valid(cq_req_valid_reg),
     .req_ready(cq_req_ready),
     .rsp_qn(),
-    .rsp_dqn(),
+    .rsp_dqn(cq_rsp_irqn),
     .rsp_addr(cq_rsp_addr),
     .rsp_phase_tag(cq_rsp_phase_tag),
     .rsp_error(cq_rsp_error),
@@ -104,9 +111,17 @@ state_t state_reg = STATE_IDLE;
 
 logic phase_tag_reg = 1'b0;
 
-logic irq_reg = 1'b0;
+logic [IRQN_W-1:0] m_axis_irq_irqn_reg = '0;
+logic m_axis_irq_tvalid_reg = 1'b0;
 
-assign irq = irq_reg;
+assign m_axis_irq.tdata  = m_axis_irq_irqn_reg;
+assign m_axis_irq.tkeep  = '1;
+assign m_axis_irq.tstrb  = m_axis_irq.tkeep;
+assign m_axis_irq.tvalid = m_axis_irq_tvalid_reg;
+assign m_axis_irq.tlast  = 1'b1;
+assign m_axis_irq.tid    = '0;
+assign m_axis_irq.tdest  = '0;
+assign m_axis_irq.tuser  = '0;
 
 always_ff @(posedge clk) begin
     s_axis_cpl.tready <= 1'b0;
@@ -127,7 +142,7 @@ always_ff @(posedge clk) begin
     cq_req_valid_reg <= cq_req_valid_reg && !cq_req_ready;
     cq_rsp_ready_reg <= 1'b0;
 
-    irq_reg <= 1'b0;
+    m_axis_irq_tvalid_reg <= m_axis_irq_tvalid_reg && !m_axis_irq.tready;
 
     case (state_reg)
         STATE_IDLE: begin
@@ -149,6 +164,7 @@ always_ff @(posedge clk) begin
             if (cq_rsp_valid && cq_rsp_ready_reg) begin
                 cq_rsp_ready_reg <= 1'b0;
 
+                m_axis_irq_irqn_reg <= cq_rsp_irqn;
                 dma_wr_desc_req.req_dst_addr <= cq_rsp_addr;
                 phase_tag_reg <= cq_rsp_phase_tag;
 
@@ -165,7 +181,7 @@ always_ff @(posedge clk) begin
         STATE_WRITE_DATA: begin
             if (dma_wr_desc_sts.sts_valid) begin
                 s_axis_cpl.tready <= 1'b1;
-                irq_reg <= 1'b1;
+                m_axis_irq_tvalid_reg <= 1'b1;
                 state_reg <= STATE_IDLE;
             end
         end
@@ -178,7 +194,7 @@ always_ff @(posedge clk) begin
         state_reg <= STATE_IDLE;
         cq_req_valid_reg <= 1'b0;
         cq_rsp_ready_reg <= 1'b0;
-        irq_reg <= 1'b0;
+        m_axis_irq_tvalid_reg <= 1'b0;
     end
 end
 

src/cndm/rtl/cndm_micro_pcie_us.sv

@@ -459,8 +459,25 @@ cfg_inst (
     .cfg_mgmt_read_write_done(cfg_mgmt_read_write_done)
 );
 
-wire [PORTS-1:0] irq;
-wire [31:0] msi_irq = 32'(irq);
+localparam IRQN_W = $clog2(32);
+
+taxi_axis_if #(
+    .DATA_W(IRQN_W),
+    .KEEP_EN(0),
+    .KEEP_W(1)
+) axis_irq();
+
+logic [31:0] msi_irq_reg = '0;
+
+assign axis_irq.tready = 1'b1;
+
+always @(posedge pcie_clk) begin
+    msi_irq_reg <= '0;
+
+    if (axis_irq.tvalid) begin
+        msi_irq_reg[axis_irq.tdata] <= 1'b1;
+    end
+end
 
 taxi_pcie_us_msi #(
     .MSI_CNT(32)
@@ -472,7 +489,7 @@ msi_inst (
     /*
      * Interrupt request inputs
      */
-    .msi_irq(msi_irq),
+    .msi_irq(msi_irq_reg),
 
     /*
      * Interface to UltraScale PCIe IP core
@@ -548,7 +565,10 @@ core_inst (
     .dma_ram_wr(dma_ram),
     .dma_ram_rd(dma_ram),
 
-    .irq(irq),
+    /*
+     * Interrupts
+     */
+    .m_axis_irq(axis_irq),
 
     /*
      * PTP

src/cndm/rtl/cndm_micro_port.sv

@@ -49,7 +49,10 @@ module cndm_micro_port #(
     taxi_dma_ram_if.wr_slv    dma_ram_wr,
     taxi_dma_ram_if.rd_slv    dma_ram_rd,
 
-    output wire logic         irq,
+    /*
+     * Interrupts
+     */
+    taxi_axis_if.src          m_axis_irq,
 
     /*
      * PTP
@@ -505,8 +508,12 @@ cpl_wr_inst (
     .dma_wr_desc_sts(dma_wr_desc_int[0]),
     .dma_ram_rd(dma_ram_rd_int[0]),
 
-    .s_axis_cpl(axis_cpl),
-    .irq(irq)
+    /*
+     * Interrupts
+     */
+    .m_axis_irq(m_axis_irq),
+
+    .s_axis_cpl(axis_cpl)
 );
 
 // TX path