cndm: Add board control logic to RK-XCKU5P-F

Signed-off-by: Alex Forencich <alex@alexforencich.com>

src/cndm/board/RK_XCKU5P_F/fpga/fpga.xdc

@@ -142,8 +142,8 @@ set_property -dict {LOC W12  IOSTANDARD LVCMOS33 SLEW SLOW DRIVE 8} [get_ports q
 set_property -dict {LOC AA13 IOSTANDARD LVCMOS33 PULLUP true} [get_ports qsfp_modprsl]
 set_property -dict {LOC Y13  IOSTANDARD LVCMOS33 PULLUP true} [get_ports qsfp_intl]
 set_property -dict {LOC W14  IOSTANDARD LVCMOS33 SLEW SLOW DRIVE 8} [get_ports qsfp_lpmode]
-#set_property -dict {LOC AE15 IOSTANDARD LVCMOS33 SLEW SLOW DRIVE 12 PULLUP true} [get_ports {qsfp_i2c_scl}]
+set_property -dict {LOC AE15 IOSTANDARD LVCMOS33 SLEW SLOW DRIVE 12 PULLUP true} [get_ports {qsfp_i2c_scl}]
-#set_property -dict {LOC AE13 IOSTANDARD LVCMOS33 SLEW SLOW DRIVE 12 PULLUP true} [get_ports {qsfp_i2c_sda}]
+set_property -dict {LOC AE13 IOSTANDARD LVCMOS33 SLEW SLOW DRIVE 12 PULLUP true} [get_ports {qsfp_i2c_sda}]
 
 # 156.25 MHz MGT reference clock
 create_clock -period 6.4 -name qsfp_mgt_refclk [get_ports {qsfp_mgt_refclk_p}]

src/cndm/board/RK_XCKU5P_F/fpga/fpga/Makefile

@@ -19,10 +19,12 @@ TAXI_SRC_DIR = $(LIB_DIR)/taxi/src
 SYN_FILES = $(RTL_DIR)/fpga.sv
 SYN_FILES += $(RTL_DIR)/fpga_core.sv
 SYN_FILES += $(TAXI_SRC_DIR)/cndm/rtl/cndm_micro_pcie_us.f
+SYN_FILES += $(TAXI_SRC_DIR)/cndm/rtl/cndm_brd_ctrl_i2c.f
 SYN_FILES += $(TAXI_SRC_DIR)/eth/rtl/us/taxi_eth_mac_25g_us.f
 SYN_FILES += $(TAXI_SRC_DIR)/axis/rtl/taxi_axis_async_fifo.f
 SYN_FILES += $(TAXI_SRC_DIR)/xfcp/rtl/taxi_xfcp_if_uart.f
 SYN_FILES += $(TAXI_SRC_DIR)/xfcp/rtl/taxi_xfcp_switch.sv
+SYN_FILES += $(TAXI_SRC_DIR)/xfcp/rtl/taxi_xfcp_mod_i2c_master.f
 SYN_FILES += $(TAXI_SRC_DIR)/xfcp/rtl/taxi_xfcp_mod_apb.f
 SYN_FILES += $(TAXI_SRC_DIR)/xfcp/rtl/taxi_xfcp_mod_stats.f
 SYN_FILES += $(TAXI_SRC_DIR)/sync/rtl/taxi_sync_reset.sv

src/cndm/board/RK_XCKU5P_F/fpga/fpga_10g/Makefile

@@ -19,10 +19,12 @@ TAXI_SRC_DIR = $(LIB_DIR)/taxi/src
 SYN_FILES = $(RTL_DIR)/fpga.sv
 SYN_FILES += $(RTL_DIR)/fpga_core.sv
 SYN_FILES += $(TAXI_SRC_DIR)/cndm/rtl/cndm_micro_pcie_us.f
+SYN_FILES += $(TAXI_SRC_DIR)/cndm/rtl/cndm_brd_ctrl_i2c.f
 SYN_FILES += $(TAXI_SRC_DIR)/eth/rtl/us/taxi_eth_mac_25g_us.f
 SYN_FILES += $(TAXI_SRC_DIR)/axis/rtl/taxi_axis_async_fifo.f
 SYN_FILES += $(TAXI_SRC_DIR)/xfcp/rtl/taxi_xfcp_if_uart.f
 SYN_FILES += $(TAXI_SRC_DIR)/xfcp/rtl/taxi_xfcp_switch.sv
+SYN_FILES += $(TAXI_SRC_DIR)/xfcp/rtl/taxi_xfcp_mod_i2c_master.f
 SYN_FILES += $(TAXI_SRC_DIR)/xfcp/rtl/taxi_xfcp_mod_apb.f
 SYN_FILES += $(TAXI_SRC_DIR)/xfcp/rtl/taxi_xfcp_mod_stats.f
 SYN_FILES += $(TAXI_SRC_DIR)/sync/rtl/taxi_sync_reset.sv

src/cndm/board/RK_XCKU5P_F/fpga/rtl/fpga.sv

@@ -91,8 +91,8 @@ module fpga #
     input  wire logic        qsfp_modprsl,
     input  wire logic        qsfp_intl,
     output wire logic        qsfp_lpmode,
-    // inout  wire logic        qsfp_i2c_scl,
+    inout  wire logic        qsfp_i2c_scl,
-    // inout  wire logic        qsfp_i2c_sda.
+    inout  wire logic        qsfp_i2c_sda,
 
     /*
      * PCIe
@@ -251,6 +251,36 @@ sync_signal_inst (
     .out({uart_rxd_int})
 );
 
+wire qsfp_modprsl_int;
+wire qsfp_intl_int;
+wire qsfp_i2c_scl_i;
+wire qsfp_i2c_scl_o;
+wire qsfp_i2c_sda_i;
+wire qsfp_i2c_sda_o;
+
+reg qsfp_i2c_scl_o_reg;
+reg qsfp_i2c_sda_o_reg;
+
+always @(posedge pcie_user_clk) begin
+    qsfp_i2c_scl_o_reg <= qsfp_i2c_scl_o;
+    qsfp_i2c_sda_o_reg <= qsfp_i2c_sda_o;
+end
+
+taxi_sync_signal #(
+    .WIDTH(4),
+    .N(2)
+)
+qsfp_sync_inst (
+    .clk(pcie_user_clk),
+    .in({qsfp_modprsl, qsfp_intl,
+        qsfp_i2c_scl, qsfp_i2c_sda}),
+    .out({qsfp_modprsl_int, qsfp_intl_int,
+        qsfp_i2c_scl_i, qsfp_i2c_sda_i})
+);
+
+assign qsfp_i2c_scl = qsfp_i2c_scl_o_reg ? 1'bz : qsfp_i2c_scl_o_reg;
+assign qsfp_i2c_sda = qsfp_i2c_sda_o_reg ? 1'bz : qsfp_i2c_sda_o_reg;
+
 // Flash
 wire qspi_clk_int;
 wire [3:0] qspi_dq_int;
@@ -806,10 +836,15 @@ core_inst (
     .qsfp_mgt_refclk_n(qsfp_mgt_refclk_n),
     .qsfp_modsell(qsfp_modsell),
     .qsfp_resetl(qsfp_resetl),
-    .qsfp_modprsl(qsfp_modprsl),
+    .qsfp_modprsl(qsfp_modprsl_int),
-    .qsfp_intl(qsfp_intl),
+    .qsfp_intl(qsfp_intl_int),
     .qsfp_lpmode(qsfp_lpmode),
 
+    .qsfp_i2c_scl_i(qsfp_i2c_scl_i),
+    .qsfp_i2c_scl_o(qsfp_i2c_scl_o),
+    .qsfp_i2c_sda_i(qsfp_i2c_sda_i),
+    .qsfp_i2c_sda_o(qsfp_i2c_sda_o),
+
     /*
      * PCIe
      */

src/cndm/board/RK_XCKU5P_F/fpga/rtl/fpga_core.sv

@@ -94,6 +94,11 @@ module fpga_core #
     input  wire logic                     qsfp_intl,
     output wire logic                     qsfp_lpmode,
 
+    input  wire logic                     qsfp_i2c_scl_i,
+    output wire logic                     qsfp_i2c_scl_o,
+    input  wire logic                     qsfp_i2c_sda_i,
+    output wire logic                     qsfp_i2c_sda_o,
+
     /*
      * PCIe
      */
@@ -254,7 +259,7 @@ xfcp_if_uart_inst (
     .prescale(16'(125000000/3000000))
 );
 
-localparam XFCP_PORTS = 2;
+localparam XFCP_PORTS = 3;
 
 taxi_axis_if #(.DATA_W(8), .USER_EN(1), .USER_W(1)) xfcp_sw_ds[XFCP_PORTS](), xfcp_sw_us[XFCP_PORTS]();
 
@@ -306,6 +311,70 @@ xfcp_stats_inst (
     .s_axis_stat(axis_stat)
 );
 
+// I2C
+wire [1:0] qsfp_i2c_scl_o_int;
+wire [1:0] qsfp_i2c_sda_o_int;
+
+assign qsfp_i2c_scl_o = &qsfp_i2c_scl_o_int;
+assign qsfp_i2c_sda_o = &qsfp_i2c_sda_o_int;
+wire qsfp_i2c_scl_i_int = qsfp_i2c_scl_i & qsfp_i2c_scl_o;
+wire qsfp_i2c_sda_i_int = qsfp_i2c_sda_i & qsfp_i2c_sda_o;
+
+taxi_xfcp_mod_i2c_master #(
+    .XFCP_EXT_ID_STR("I2C"),
+    .DEFAULT_PRESCALE(16'(125000000/200000/4))
+)
+xfcp_mod_i2c_inst (
+    .clk(clk_125mhz),
+    .rst(rst_125mhz),
+
+    /*
+     * XFCP upstream port
+     */
+    .xfcp_usp_ds(xfcp_sw_ds[1]),
+    .xfcp_usp_us(xfcp_sw_us[1]),
+
+    /*
+     * I2C interface
+     */
+    .i2c_scl_i(qsfp_i2c_scl_i_int),
+    .i2c_scl_o(qsfp_i2c_scl_o_int[0]),
+    .i2c_sda_i(qsfp_i2c_sda_i_int),
+    .i2c_sda_o(qsfp_i2c_sda_o_int[0])
+);
+
+localparam logic OPTIC_EN = 1'b1;
+localparam OPTIC_CNT = 1;
+
+localparam logic EEPROM_EN = 1'b0;
+localparam EEPROM_IDX = OPTIC_EN ? OPTIC_CNT : 0;
+
+localparam logic MAC_EEPROM_EN = EEPROM_EN;
+localparam MAC_EEPROM_IDX = EEPROM_IDX;
+localparam MAC_EEPROM_OFFSET = 32;
+localparam MAC_COUNT = OPTIC_CNT;
+localparam logic MAC_FROM_BASE = 1'b1;
+
+localparam logic SN_EEPROM_EN = EEPROM_EN;
+localparam SN_EEPROM_IDX = EEPROM_IDX;
+localparam SN_EEPROM_OFFSET = 0;
+localparam SN_LEN = 32;
+
+localparam logic PLL_EN = 1'b0;
+localparam PLL_IDX = EEPROM_IDX + (EEPROM_EN ? 1 : 0);
+
+localparam logic MUX_EN = 1'b0;
+localparam MUX_CNT = 2;
+localparam logic [MUX_CNT-1:0][6:0] MUX_I2C_ADDR = '0;
+
+localparam DEV_CNT = PLL_IDX + (PLL_EN ? 1 : 0);
+localparam logic [DEV_CNT-1:0][6:0] DEV_I2C_ADDR = {7'h50};
+localparam logic [DEV_CNT-1:0][31:0] DEV_ADDR_CFG = {32'h7e_7f_0070};
+localparam logic [DEV_CNT-1:0][MUX_CNT-1:0][7:0] DEV_MUX_MASK = '0;
+
+localparam I2C_PRESCALE = SIM ? 2 : 250000/(400*4);
+localparam I2C_TBUF_CYC = SIM ? 10 : 1000;
+
 taxi_axis_if #(
     .DATA_W(32),
     .KEEP_EN(1),
@@ -315,6 +384,61 @@ taxi_axis_if #(
     .USER_W(1)
 ) axis_brd_ctrl_cmd(), axis_brd_ctrl_rsp();
 
+cndm_brd_ctrl_i2c #(
+    .OPTIC_EN(OPTIC_EN),
+    .OPTIC_CNT(OPTIC_CNT),
+
+    .EEPROM_EN(EEPROM_EN),
+    .EEPROM_IDX(EEPROM_IDX),
+
+    .MAC_EEPROM_EN(MAC_EEPROM_EN),
+    .MAC_EEPROM_IDX(MAC_EEPROM_IDX),
+    .MAC_EEPROM_OFFSET(MAC_EEPROM_OFFSET),
+    .MAC_COUNT(MAC_COUNT),
+    .MAC_FROM_BASE(MAC_FROM_BASE),
+
+    .SN_EEPROM_EN(SN_EEPROM_EN),
+    .SN_EEPROM_IDX(SN_EEPROM_IDX),
+    .SN_EEPROM_OFFSET(SN_EEPROM_OFFSET),
+    .SN_LEN(SN_LEN),
+
+    .PLL_EN(PLL_EN),
+    .PLL_IDX(PLL_IDX),
+
+    .MUX_EN(MUX_EN),
+    .MUX_CNT(MUX_CNT),
+    .MUX_I2C_ADDR(MUX_I2C_ADDR),
+
+    .DEV_CNT(DEV_CNT),
+    .DEV_I2C_ADDR(DEV_I2C_ADDR),
+    .DEV_ADDR_CFG(DEV_ADDR_CFG),
+    .DEV_MUX_MASK(DEV_MUX_MASK),
+
+    .I2C_PRESCALE(I2C_PRESCALE),
+    .I2C_TBUF_CYC(I2C_TBUF_CYC)
+)
+board_ctrl_i2c_ch_inst (
+    .clk(pcie_clk),
+    .rst(pcie_rst),
+
+    /*
+     * Board control command interface
+     */
+    .s_axis_cmd(axis_brd_ctrl_cmd),
+    .m_axis_rsp(axis_brd_ctrl_rsp),
+
+    /*
+     * I2C interface
+     */
+    .i2c_scl_i(qsfp_i2c_scl_i_int),
+    .i2c_scl_o(qsfp_i2c_scl_o_int[1]),
+    .i2c_sda_i(qsfp_i2c_sda_i_int),
+    .i2c_sda_o(qsfp_i2c_sda_o_int[1]),
+
+    .dev_sel(),
+    .dev_rst()
+);
+
 // QSFP28
 assign qsfp_modsell = 1'b0;
 assign qsfp_resetl = 1'b1;
@@ -397,8 +521,8 @@ xfcp_mod_apb_inst (
     /*
      * XFCP upstream port
      */
-    .xfcp_usp_ds(xfcp_sw_ds[1]),
+    .xfcp_usp_ds(xfcp_sw_ds[2]),
-    .xfcp_usp_us(xfcp_sw_us[1]),
+    .xfcp_usp_us(xfcp_sw_us[2]),
 
     /*
      * APB master interface
@@ -669,7 +793,7 @@ cndm_micro_pcie_us #(
 
     // Structural configuration
     .PORTS($size(axis_qsfp_tx)),
-    .BRD_CTRL_EN(1'b0),
+    .BRD_CTRL_EN(1'b1),
     .SYS_CLK_PER_NS_NUM(4),
     .SYS_CLK_PER_NS_DEN(1),
 

src/cndm/board/RK_XCKU5P_F/fpga/tb/fpga_core/Makefile

@@ -25,10 +25,12 @@ TOPLEVEL = $(COCOTB_TOPLEVEL)
 VERILOG_SOURCES += $(COCOTB_TOPLEVEL).sv
 VERILOG_SOURCES += $(RTL_DIR)/$(DUT).sv
 VERILOG_SOURCES += $(TAXI_SRC_DIR)/cndm/rtl/cndm_micro_pcie_us.f
+VERILOG_SOURCES += $(TAXI_SRC_DIR)/cndm/rtl/cndm_brd_ctrl_i2c.f
 VERILOG_SOURCES += $(TAXI_SRC_DIR)/eth/rtl/us/taxi_eth_mac_25g_us.f
 VERILOG_SOURCES += $(TAXI_SRC_DIR)/axis/rtl/taxi_axis_async_fifo.f
 VERILOG_SOURCES += $(TAXI_SRC_DIR)/xfcp/rtl/taxi_xfcp_if_uart.f
 VERILOG_SOURCES += $(TAXI_SRC_DIR)/xfcp/rtl/taxi_xfcp_switch.sv
+VERILOG_SOURCES += $(TAXI_SRC_DIR)/xfcp/rtl/taxi_xfcp_mod_i2c_master.f
 VERILOG_SOURCES += $(TAXI_SRC_DIR)/xfcp/rtl/taxi_xfcp_mod_apb.f
 VERILOG_SOURCES += $(TAXI_SRC_DIR)/xfcp/rtl/taxi_xfcp_mod_stats.f
 VERILOG_SOURCES += $(TAXI_SRC_DIR)/sync/rtl/taxi_sync_reset.sv

src/cndm/board/RK_XCKU5P_F/fpga/tb/fpga_core/test_fpga_core.py

@@ -11,6 +11,7 @@ Authors:
 
 import logging
 import os
+import struct
 import sys
 
 import pytest
@@ -22,6 +23,8 @@ from cocotb.triggers import RisingEdge, FallingEdge, Timer
 
 from cocotbext.axi import AxiStreamBus
 from cocotbext.eth import XgmiiFrame
+from cocotbext.uart import UartSource, UartSink
+from cocotbext.i2c import I2cMemory
 from cocotbext.pcie.core import RootComplex
 from cocotbext.pcie.xilinx.us import UltraScalePlusPcieDevice
 
@@ -324,6 +327,33 @@ class TB:
                 gbx_cfg=gbx_cfg
             ))
 
+        # UART
+        self.uart_source = UartSource(dut.uart_rxd, baud=3000000, bits=8, stop_bits=1)
+        self.uart_sink = UartSink(dut.uart_txd, baud=3000000, bits=8, stop_bits=1)
+
+        # I2C
+        self.qsfp_i2c = I2cMemory(sda=dut.qsfp_i2c_sda_o, sda_o=dut.qsfp_i2c_sda_i,
+            scl=dut.qsfp_i2c_scl_o, scl_o=dut.qsfp_i2c_scl_i, addr=0x50, size=256)
+
+        self.qsfp_i2c.write_mem(0, bytes.fromhex("""
+            11 07 00 2f 00 af 00 00 00 55 55 00 00 00 00 00
+            00 00 00 00 00 00 1a 87 00 00 80 5c 00 00 00 00
+            00 00 00 00 00 00 00 2e 00 00 5a 0f 00 00 5a 0f
+            5a 0f 33 6d 00 00 30 a7 33 4a 00 00 00 00 00 00
+            00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
+            00 00 00 00 00 00 00 aa aa 00 00 00 00 01 00 00
+            00 00 ff 00 00 00 00 00 00 00 00 00 00 0a 00 00
+            00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
+            11 cc 07 80 00 00 00 00 00 00 00 05 ff 02 01 00
+            00 00 00 40 49 4e 54 45 4c 20 43 4f 52 50 20 20
+            20 20 20 20 00 00 02 b3 53 50 54 53 42 50 32 43
+            4c 43 4b 53 20 20 20 20 30 32 66 58 05 14 37 5e
+            06 07 ff be 43 52 43 4c 32 30 31 37 31 41 35 35
+            50 20 20 20 32 30 30 35 30 38 30 30 0c 08 68 4f
+            00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
+            00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
+        """))
+
         dut.btn.setimmediatevalue(0)
 
         dut.qsfp_modprsl.setimmediatevalue(0)
@@ -376,6 +406,28 @@ async def run_test(dut):
 
     tb.log.info("Init complete")
 
+    tb.log.info("Read QSFP")
+
+    rsp = await driver.exec_cmd(struct.pack("<HHLHHLbbbbLLL",
+        0, # rsvd
+        cndm.CNDM_CMD_OP_HWMON, # opcode
+        0x00000000, # flags
+        0, # index
+        cndm.CNDM_CMD_BRD_OP_OPTIC_RD, # board op
+        0, # flags
+        0, # rsvd
+        0, # dev addr offset
+        0, # bank
+        0, # page
+        0x00, # addr
+        32, # len
+        0, # rsvd
+    ))
+
+    print(rsp)
+
+    tb.log.info("Data: %s", rsp[32:32+32].hex())
+
     tb.log.info("Wait for block lock")
     for k in range(1200):
         await RisingEdge(tb.dut.clk_125mhz)
@@ -476,10 +528,12 @@ def test_fpga_core(request, mac_data_w):
         os.path.join(tests_dir, f"{toplevel}.sv"),
         os.path.join(rtl_dir, f"{dut}.sv"),
         os.path.join(taxi_src_dir, "cndm", "rtl", "cndm_micro_pcie_us.f"),
+        os.path.join(taxi_src_dir, "cndm", "rtl", "cndm_brd_ctrl_i2c.f"),
         os.path.join(taxi_src_dir, "eth", "rtl", "us", "taxi_eth_mac_25g_us.f"),
         os.path.join(taxi_src_dir, "axis", "rtl", "taxi_axis_async_fifo.f"),
         os.path.join(taxi_src_dir, "xfcp", "rtl", "taxi_xfcp_if_uart.f"),
         os.path.join(taxi_src_dir, "xfcp", "rtl", "taxi_xfcp_switch.sv"),
+        os.path.join(taxi_src_dir, "xfcp", "rtl", "taxi_xfcp_mod_i2c_master.f"),
         os.path.join(taxi_src_dir, "xfcp", "rtl", "taxi_xfcp_mod_apb.f"),
         os.path.join(taxi_src_dir, "xfcp", "rtl", "taxi_xfcp_mod_stats.f"),
         os.path.join(taxi_src_dir, "sync", "rtl", "taxi_sync_reset.sv"),

src/cndm/board/RK_XCKU5P_F/fpga/tb/fpga_core/test_fpga_core.sv

@@ -76,6 +76,11 @@ logic qsfp_modprsl;
 logic qsfp_intl;
 logic qsfp_lpmode;
 
+logic qsfp_i2c_scl_i;
+logic qsfp_i2c_scl_o;
+logic qsfp_i2c_sda_i;
+logic qsfp_i2c_sda_o;
+
 logic pcie_clk;
 logic pcie_rst;
 
@@ -235,6 +240,11 @@ uut (
     .qsfp_intl(qsfp_intl),
     .qsfp_lpmode(qsfp_lpmode),
 
+    .qsfp_i2c_scl_i(qsfp_i2c_scl_i),
+    .qsfp_i2c_scl_o(qsfp_i2c_scl_o),
+    .qsfp_i2c_sda_i(qsfp_i2c_sda_i),
+    .qsfp_i2c_sda_o(qsfp_i2c_sda_o),
+
     /*
      * PCIe
      */