From db8b1fc27efbdab9b663c22b3403a87f8baad1dd Mon Sep 17 00:00:00 2001 From: Alex Forencich Date: Sat, 22 Feb 2025 22:33:54 -0800 Subject: [PATCH] example/VCU108: Add 25G MACs on QSFP28 port on VCU108 Signed-off-by: Alex Forencich --- example/VCU108/fpga/README.md | 5 +- example/VCU108/fpga/fpga.xdc | 56 +-- example/VCU108/fpga/fpga/Makefile | 2 + example/VCU108/fpga/fpga_10g/Makefile | 84 +++++ example/VCU108/fpga/rtl/fpga.sv | 55 ++- example/VCU108/fpga/rtl/fpga_core.sv | 324 +++++++++++++++++- example/VCU108/fpga/tb/fpga_core/Makefile | 5 +- example/VCU108/fpga/tb/fpga_core/baser.py | 1 + .../fpga/tb/fpga_core/test_fpga_core.py | 84 ++++- 9 files changed, 573 insertions(+), 43 deletions(-) create mode 100644 example/VCU108/fpga/fpga_10g/Makefile create mode 120000 example/VCU108/fpga/tb/fpga_core/baser.py diff --git a/example/VCU108/fpga/README.md b/example/VCU108/fpga/README.md index 921fbf0..6afda44 100644 --- a/example/VCU108/fpga/README.md +++ b/example/VCU108/fpga/README.md @@ -4,17 +4,20 @@ This example design targets the Xilinx VCU108 FPGA board. -The design places a looped-back MAC on the BASE-T port as well as a looped-back UART on on the USB UART connection. +The design places looped-back MACs on the BASE-T and QSFP28 ports as well as a looped-back UART on on the USB UART connection. * USB UART * Looped-back UART * RJ-45 Ethernet port with Marvell 88E1111 PHY * Looped-back MAC via SGMII via Xilinx PCS/PMA core and LVDS IOSERDES +* QSFP28 + * Looped-back 10G or 25G MACs via GTY transceivers ## Board details * FPGA: xcvu095-ffva2104-2-e * 1000BASE-T PHY: Marvell 88E1111 via SGMII +* 25GBASE-R PHY: Soft PCS with GTY transceivers ## Licensing diff --git a/example/VCU108/fpga/fpga.xdc b/example/VCU108/fpga/fpga.xdc index ee6ba4f..fa13b6d 100644 --- a/example/VCU108/fpga/fpga.xdc +++ b/example/VCU108/fpga/fpga.xdc @@ -177,41 +177,41 @@ set_input_delay 0 [get_ports {phy_int_n}] #create_clock -period 6.4 -name bullseye_mgt_refclk [get_ports bullseye_mgt_refclk_1_p] # QSFP28 Interface -#set_property -dict {LOC AG45} [get_ports {qsfp_rx_p[0]}] ;# MGTYRXP0_127 GTYE3_CHANNEL_X0Y12 / GTYE3_COMMON_X0Y3 -#set_property -dict {LOC AG46} [get_ports {qsfp_rx_n[0]}] ;# MGTYRXN0_127 GTYE3_CHANNEL_X0Y12 / GTYE3_COMMON_X0Y3 -#set_property -dict {LOC AK42} [get_ports {qsfp_tx_p[0]}] ;# MGTYTXP0_127 GTYE3_CHANNEL_X0Y12 / GTYE3_COMMON_X0Y3 -#set_property -dict {LOC AK43} [get_ports {qsfp_tx_n[0]}] ;# MGTYTXN0_127 GTYE3_CHANNEL_X0Y12 / GTYE3_COMMON_X0Y3 -#set_property -dict {LOC AF43} [get_ports {qsfp_rx_p[1]}] ;# MGTYRXP1_127 GTYE3_CHANNEL_X0Y13 / GTYE3_COMMON_X0Y3 -#set_property -dict {LOC AF44} [get_ports {qsfp_rx_n[1]}] ;# MGTYRXN1_127 GTYE3_CHANNEL_X0Y13 / GTYE3_COMMON_X0Y3 -#set_property -dict {LOC AJ40} [get_ports {qsfp_tx_p[1]}] ;# MGTYTXP1_127 GTYE3_CHANNEL_X0Y13 / GTYE3_COMMON_X0Y3 -#set_property -dict {LOC AJ41} [get_ports {qsfp_tx_n[1]}] ;# MGTYTXN1_127 GTYE3_CHANNEL_X0Y13 / GTYE3_COMMON_X0Y3 -#set_property -dict {LOC AE45} [get_ports {qsfp_rx_p[2]}] ;# MGTYRXP2_127 GTYE3_CHANNEL_X0Y14 / GTYE3_COMMON_X0Y3 -#set_property -dict {LOC AE46} [get_ports {qsfp_rx_n[2]}] ;# MGTYRXN2_127 GTYE3_CHANNEL_X0Y14 / GTYE3_COMMON_X0Y3 -#set_property -dict {LOC AG40} [get_ports {qsfp_tx_p[2]}] ;# MGTYTXP2_127 GTYE3_CHANNEL_X0Y14 / GTYE3_COMMON_X0Y3 -#set_property -dict {LOC AG41} [get_ports {qsfp_tx_n[2]}] ;# MGTYTXN2_127 GTYE3_CHANNEL_X0Y14 / GTYE3_COMMON_X0Y3 -#set_property -dict {LOC AD43} [get_ports {qsfp_rx_p[3]}] ;# MGTYRXP3_127 GTYE3_CHANNEL_X0Y15 / GTYE3_COMMON_X0Y3 -#set_property -dict {LOC AD44} [get_ports {qsfp_rx_n[3]}] ;# MGTYRXN3_127 GTYE3_CHANNEL_X0Y15 / GTYE3_COMMON_X0Y3 -#set_property -dict {LOC AE40} [get_ports {qsfp_tx_p[3]}] ;# MGTYTXP3_127 GTYE3_CHANNEL_X0Y15 / GTYE3_COMMON_X0Y3 -#set_property -dict {LOC AE41} [get_ports {qsfp_tx_n[3]}] ;# MGTYTXN3_127 GTYE3_CHANNEL_X0Y15 / GTYE3_COMMON_X0Y3 -#set_property -dict {LOC AF38} [get_ports qsfp_mgt_refclk_0_p] ;# MGTREFCLK0P_127 from U32 SI570 via U102 SI53340 -#set_property -dict {LOC AF39} [get_ports qsfp_mgt_refclk_0_n] ;# MGTREFCLK0N_127 from U32 SI570 via U102 SI53340 +set_property -dict {LOC AG45} [get_ports {qsfp_rx_p[0]}] ;# MGTYRXP0_127 GTYE3_CHANNEL_X0Y12 / GTYE3_COMMON_X0Y3 +set_property -dict {LOC AG46} [get_ports {qsfp_rx_n[0]}] ;# MGTYRXN0_127 GTYE3_CHANNEL_X0Y12 / GTYE3_COMMON_X0Y3 +set_property -dict {LOC AK42} [get_ports {qsfp_tx_p[0]}] ;# MGTYTXP0_127 GTYE3_CHANNEL_X0Y12 / GTYE3_COMMON_X0Y3 +set_property -dict {LOC AK43} [get_ports {qsfp_tx_n[0]}] ;# MGTYTXN0_127 GTYE3_CHANNEL_X0Y12 / GTYE3_COMMON_X0Y3 +set_property -dict {LOC AF43} [get_ports {qsfp_rx_p[1]}] ;# MGTYRXP1_127 GTYE3_CHANNEL_X0Y13 / GTYE3_COMMON_X0Y3 +set_property -dict {LOC AF44} [get_ports {qsfp_rx_n[1]}] ;# MGTYRXN1_127 GTYE3_CHANNEL_X0Y13 / GTYE3_COMMON_X0Y3 +set_property -dict {LOC AJ40} [get_ports {qsfp_tx_p[1]}] ;# MGTYTXP1_127 GTYE3_CHANNEL_X0Y13 / GTYE3_COMMON_X0Y3 +set_property -dict {LOC AJ41} [get_ports {qsfp_tx_n[1]}] ;# MGTYTXN1_127 GTYE3_CHANNEL_X0Y13 / GTYE3_COMMON_X0Y3 +set_property -dict {LOC AE45} [get_ports {qsfp_rx_p[2]}] ;# MGTYRXP2_127 GTYE3_CHANNEL_X0Y14 / GTYE3_COMMON_X0Y3 +set_property -dict {LOC AE46} [get_ports {qsfp_rx_n[2]}] ;# MGTYRXN2_127 GTYE3_CHANNEL_X0Y14 / GTYE3_COMMON_X0Y3 +set_property -dict {LOC AG40} [get_ports {qsfp_tx_p[2]}] ;# MGTYTXP2_127 GTYE3_CHANNEL_X0Y14 / GTYE3_COMMON_X0Y3 +set_property -dict {LOC AG41} [get_ports {qsfp_tx_n[2]}] ;# MGTYTXN2_127 GTYE3_CHANNEL_X0Y14 / GTYE3_COMMON_X0Y3 +set_property -dict {LOC AD43} [get_ports {qsfp_rx_p[3]}] ;# MGTYRXP3_127 GTYE3_CHANNEL_X0Y15 / GTYE3_COMMON_X0Y3 +set_property -dict {LOC AD44} [get_ports {qsfp_rx_n[3]}] ;# MGTYRXN3_127 GTYE3_CHANNEL_X0Y15 / GTYE3_COMMON_X0Y3 +set_property -dict {LOC AE40} [get_ports {qsfp_tx_p[3]}] ;# MGTYTXP3_127 GTYE3_CHANNEL_X0Y15 / GTYE3_COMMON_X0Y3 +set_property -dict {LOC AE41} [get_ports {qsfp_tx_n[3]}] ;# MGTYTXN3_127 GTYE3_CHANNEL_X0Y15 / GTYE3_COMMON_X0Y3 +set_property -dict {LOC AF38} [get_ports qsfp_mgt_refclk_0_p] ;# MGTREFCLK0P_127 from U32 SI570 via U102 SI53340 +set_property -dict {LOC AF39} [get_ports qsfp_mgt_refclk_0_n] ;# MGTREFCLK0N_127 from U32 SI570 via U102 SI53340 #set_property -dict {LOC AD38} [get_ports qsfp_mgt_refclk_1_p] ;# MGTREFCLK1P_127 from U57 CKOUT2 SI5328 #set_property -dict {LOC AD39} [get_ports qsfp_mgt_refclk_1_n] ;# MGTREFCLK1N_127 from U57 CKOUT2 SI5328 #set_property -dict {LOC AG34 IOSTANDARD LVDS} [get_ports qsfp_recclk_p] ;# to U57 CKIN1 SI5328 #set_property -dict {LOC AH35 IOSTANDARD LVDS} [get_ports qsfp_recclk_n] ;# to U57 CKIN1 SI5328 -#set_property -dict {LOC AL24 IOSTANDARD LVCMOS18 SLEW SLOW DRIVE 8} [get_ports qsfp_modsell] -#set_property -dict {LOC AM24 IOSTANDARD LVCMOS18 SLEW SLOW DRIVE 8} [get_ports qsfp_resetl] -#set_property -dict {LOC AL25 IOSTANDARD LVCMOS18 PULLUP true} [get_ports qsfp_modprsl] -#set_property -dict {LOC AL21 IOSTANDARD LVCMOS18 PULLUP true} [get_ports qsfp_intl] -#set_property -dict {LOC AM21 IOSTANDARD LVCMOS18 SLEW SLOW DRIVE 8} [get_ports qsfp_lpmode] +set_property -dict {LOC AL24 IOSTANDARD LVCMOS18 SLEW SLOW DRIVE 8} [get_ports qsfp_modsell] +set_property -dict {LOC AM24 IOSTANDARD LVCMOS18 SLEW SLOW DRIVE 8} [get_ports qsfp_resetl] +set_property -dict {LOC AL25 IOSTANDARD LVCMOS18 PULLUP true} [get_ports qsfp_modprsl] +set_property -dict {LOC AL21 IOSTANDARD LVCMOS18 PULLUP true} [get_ports qsfp_intl] +set_property -dict {LOC AM21 IOSTANDARD LVCMOS18 SLEW SLOW DRIVE 8} [get_ports qsfp_lpmode] # 156.25 MHz MGT reference clock -#create_clock -period 6.400 -name qsfp_mgt_refclk_0 [get_ports qsfp_mgt_refclk_0_p] +create_clock -period 6.400 -name qsfp_mgt_refclk_0 [get_ports qsfp_mgt_refclk_0_p] -#set_false_path -to [get_ports {qsfp_modsell qsfp_resetl qsfp_lpmode}] -#set_output_delay 0 [get_ports {qsfp_modsell qsfp_resetl qsfp_lpmode}] -#set_false_path -from [get_ports {qsfp_modprsl qsfp_intl}] -#set_input_delay 0 [get_ports {qsfp_modprsl qsfp_intl}] +set_false_path -to [get_ports {qsfp_modsell qsfp_resetl qsfp_lpmode}] +set_output_delay 0 [get_ports {qsfp_modsell qsfp_resetl qsfp_lpmode}] +set_false_path -from [get_ports {qsfp_modprsl qsfp_intl}] +set_input_delay 0 [get_ports {qsfp_modprsl qsfp_intl}] # CFP2 GTY #set_property -dict {LOC J45 } [get_ports {cfp2_rx_p[0]}] ;# MGTYRXP1_130 GTYE3_CHANNEL_X0Y25 / GTYE3_COMMON_X0Y6 diff --git a/example/VCU108/fpga/fpga/Makefile b/example/VCU108/fpga/fpga/Makefile index 56a1f08..75cab3d 100644 --- a/example/VCU108/fpga/fpga/Makefile +++ b/example/VCU108/fpga/fpga/Makefile @@ -15,6 +15,7 @@ FPGA_ARCH = virtexu SYN_FILES = ../rtl/fpga.sv SYN_FILES += ../rtl/fpga_core.sv SYN_FILES += ../lib/taxi/rtl/eth/taxi_eth_mac_1g_fifo.f +SYN_FILES += ../lib/taxi/rtl/eth/us/taxi_eth_mac_25g_us.f SYN_FILES += ../lib/taxi/rtl/lss/taxi_uart.f SYN_FILES += ../lib/taxi/rtl/sync/taxi_sync_reset.sv SYN_FILES += ../lib/taxi/rtl/sync/taxi_sync_signal.sv @@ -28,6 +29,7 @@ XDC_FILES += ../lib/taxi/syn/vivado/taxi_sync_reset.tcl # IP IP_TCL_FILES = ../ip/sgmii_pcs_pma_0.tcl +IP_TCL_FILES += ../lib/taxi/rtl/eth/us/taxi_eth_mac_25g_us_gty_25g_156.tcl include ../common/vivado.mk diff --git a/example/VCU108/fpga/fpga_10g/Makefile b/example/VCU108/fpga/fpga_10g/Makefile new file mode 100644 index 0000000..4a80743 --- /dev/null +++ b/example/VCU108/fpga/fpga_10g/Makefile @@ -0,0 +1,84 @@ +# SPDX-License-Identifier: MIT +# +# Copyright (c) 2025 FPGA Ninja, LLC +# +# Authors: +# - Alex Forencich +# + +# FPGA settings +FPGA_PART = xcvu095-ffva2104-2-e +FPGA_TOP = fpga +FPGA_ARCH = virtexu + +# Files for synthesis +SYN_FILES = ../rtl/fpga.sv +SYN_FILES += ../rtl/fpga_core.sv +SYN_FILES += ../lib/taxi/rtl/eth/taxi_eth_mac_1g_fifo.f +SYN_FILES += ../lib/taxi/rtl/eth/us/taxi_eth_mac_25g_us.f +SYN_FILES += ../lib/taxi/rtl/lss/taxi_uart.f +SYN_FILES += ../lib/taxi/rtl/sync/taxi_sync_reset.sv +SYN_FILES += ../lib/taxi/rtl/sync/taxi_sync_signal.sv +SYN_FILES += ../lib/taxi/rtl/io/taxi_debounce_switch.sv + +# XDC files +XDC_FILES = ../fpga.xdc +XDC_FILES += ../lib/taxi/syn/vivado/taxi_eth_mac_fifo.tcl +XDC_FILES += ../lib/taxi/syn/vivado/taxi_axis_async_fifo.tcl +XDC_FILES += ../lib/taxi/syn/vivado/taxi_sync_reset.tcl + +# IP +IP_TCL_FILES = ../ip/sgmii_pcs_pma_0.tcl +IP_TCL_FILES += ../lib/taxi/rtl/eth/us/taxi_eth_mac_25g_us_gty_10g_156.tcl + +include ../common/vivado.mk + +program: $(PROJECT).bit + echo "open_hw_manager" > program.tcl + echo "connect_hw_server" >> program.tcl + echo "open_hw_target" >> program.tcl + echo "current_hw_device [lindex [get_hw_devices] 0]" >> program.tcl + echo "refresh_hw_device -update_hw_probes false [current_hw_device]" >> program.tcl + echo "set_property PROGRAM.FILE {$(PROJECT).bit} [current_hw_device]" >> program.tcl + echo "program_hw_devices [current_hw_device]" >> program.tcl + echo "exit" >> program.tcl + vivado -nojournal -nolog -mode batch -source program.tcl + +$(PROJECT).mcs $(PROJECT).prm: $(PROJECT).bit + echo "write_cfgmem -force -format mcs -size 128 -interface BPIx16 -loadbit {up 0x0000000 $*.bit} -checksum -file $*.mcs" > generate_mcs.tcl + echo "exit" >> generate_mcs.tcl + vivado -nojournal -nolog -mode batch -source generate_mcs.tcl + mkdir -p rev + COUNT=100; \ + while [ -e rev/$*_rev$$COUNT.bit ]; \ + do COUNT=$$((COUNT+1)); done; \ + COUNT=$$((COUNT-1)); \ + for x in .mcs .prm; \ + do cp $*$$x rev/$*_rev$$COUNT$$x; \ + echo "Output: rev/$*_rev$$COUNT$$x"; done; + +flash: $(PROJECT).mcs $(PROJECT).prm + echo "open_hw_manager" > flash.tcl + echo "connect_hw_server" >> flash.tcl + echo "open_hw_target" >> flash.tcl + echo "current_hw_device [lindex [get_hw_devices] 0]" >> flash.tcl + echo "refresh_hw_device -update_hw_probes false [current_hw_device]" >> flash.tcl + echo "create_hw_cfgmem -hw_device [current_hw_device] [lindex [get_cfgmem_parts {mt28gu01gaax1e-bpi-x16}] 0]" >> flash.tcl + echo "current_hw_cfgmem -hw_device [current_hw_device] [get_property PROGRAM.HW_CFGMEM [current_hw_device]]" >> flash.tcl + echo "set_property PROGRAM.FILES [list \"$(PROJECT).mcs\"] [current_hw_cfgmem]" >> flash.tcl + echo "set_property PROGRAM.PRM_FILES [list \"$(PROJECT).prm\"] [current_hw_cfgmem]" >> flash.tcl + echo "set_property PROGRAM.ERASE 1 [current_hw_cfgmem]" >> flash.tcl + echo "set_property PROGRAM.CFG_PROGRAM 1 [current_hw_cfgmem]" >> flash.tcl + echo "set_property PROGRAM.VERIFY 1 [current_hw_cfgmem]" >> flash.tcl + echo "set_property PROGRAM.CHECKSUM 0 [current_hw_cfgmem]" >> flash.tcl + echo "set_property PROGRAM.ADDRESS_RANGE {use_file} [current_hw_cfgmem]" >> flash.tcl + echo "set_property PROGRAM.BPI_RS_PINS {25:24} [current_hw_cfgmem]" >> flash.tcl + echo "set_property PROGRAM.UNUSED_PIN_TERMINATION {pull-none} [current_hw_cfgmem]" >> flash.tcl + echo "create_hw_bitstream -hw_device [current_hw_device] [get_property PROGRAM.HW_CFGMEM_BITFILE [current_hw_device]]" >> flash.tcl + echo "program_hw_devices [current_hw_device]" >> flash.tcl + echo "refresh_hw_device [current_hw_device]" >> flash.tcl + echo "program_hw_cfgmem -hw_cfgmem [current_hw_cfgmem]" >> flash.tcl + echo "boot_hw_device [current_hw_device]" >> flash.tcl + echo "exit" >> flash.tcl + vivado -nojournal -nolog -mode batch -source flash.tcl + diff --git a/example/VCU108/fpga/rtl/fpga.sv b/example/VCU108/fpga/rtl/fpga.sv index 12f5f6f..18ddb0c 100644 --- a/example/VCU108/fpga/rtl/fpga.sv +++ b/example/VCU108/fpga/rtl/fpga.sv @@ -15,7 +15,12 @@ Authors: /* * FPGA top-level module */ -module fpga +module fpga # +( + parameter logic SIM = 1'b0, + parameter string VENDOR = "XILINX", + parameter string FAMILY = "virtexu" +) ( /* * Clock: 125MHz LVDS @@ -54,7 +59,26 @@ module fpga input wire logic phy_sgmii_clk_p, input wire logic phy_sgmii_clk_n, output wire logic phy_reset_n, - input wire logic phy_int_n + input wire logic phy_int_n, + + /* + * Ethernet: QSFP28 + */ + input wire logic [3:0] qsfp_rx_p, + input wire logic [3:0] qsfp_rx_n, + output wire logic [3:0] qsfp_tx_p, + output wire logic [3:0] qsfp_tx_n, + input wire logic qsfp_mgt_refclk_0_p, + input wire logic qsfp_mgt_refclk_0_n, + // input wire logic qsfp_mgt_refclk_1_p, + // input wire logic qsfp_mgt_refclk_1_n, + // output wire logic qsfp_recclk_p, + // output wire logic qsfp_recclk_n, + output wire logic qsfp_modsell, + output wire logic qsfp_resetl, + input wire logic qsfp_modprsl, + input wire logic qsfp_intl, + output wire logic qsfp_lpmode ); // Clock and reset @@ -324,7 +348,11 @@ wire [7:0] led_int; // SW12:4 (sw[0]) off for LSB of status vector, on for MSB assign led = sw[3] ? (sw[0] ? pcspma_status_vector[15:8] : pcspma_status_vector[7:0]) : led_int; -fpga_core +fpga_core #( + .SIM(SIM), + .VENDOR(VENDOR), + .FAMILY(FAMILY) +) core_inst ( /* * Clock: 125MHz @@ -365,7 +393,26 @@ core_inst ( .phy_gmii_tx_en(phy_gmii_tx_en_int), .phy_gmii_tx_er(phy_gmii_tx_er_int), .phy_reset_n(phy_reset_n), - .phy_int_n(phy_int_n) + .phy_int_n(phy_int_n), + + /* + * Ethernet: QSFP28 + */ + .qsfp_rx_p(qsfp_rx_p), + .qsfp_rx_n(qsfp_rx_n), + .qsfp_tx_p(qsfp_tx_p), + .qsfp_tx_n(qsfp_tx_n), + .qsfp_mgt_refclk_0_p(qsfp_mgt_refclk_0_p), + .qsfp_mgt_refclk_0_n(qsfp_mgt_refclk_0_n), + // .qsfp_mgt_refclk_1_p(qsfp_mgt_refclk_1_p), + // .qsfp_mgt_refclk_1_n(qsfp_mgt_refclk_1_n), + // .qsfp_recclk_p(qsfp_recclk_p), + // .qsfp_recclk_n(qsfp_recclk_n), + .qsfp_modsell(qsfp_modsell), + .qsfp_resetl(qsfp_resetl), + .qsfp_modprsl(qsfp_modprsl), + .qsfp_intl(qsfp_intl), + .qsfp_lpmode(qsfp_lpmode) ); endmodule diff --git a/example/VCU108/fpga/rtl/fpga_core.sv b/example/VCU108/fpga/rtl/fpga_core.sv index c396085..a09275b 100644 --- a/example/VCU108/fpga/rtl/fpga_core.sv +++ b/example/VCU108/fpga/rtl/fpga_core.sv @@ -15,7 +15,12 @@ Authors: /* * FPGA core logic */ -module fpga_core +module fpga_core # +( + parameter logic SIM = 1'b0, + parameter string VENDOR = "XILINX", + parameter string FAMILY = "virtexu" +) ( /* * Clock: 125MHz @@ -56,10 +61,29 @@ module fpga_core output wire logic phy_gmii_tx_en, output wire logic phy_gmii_tx_er, output wire logic phy_reset_n, - input wire logic phy_int_n + input wire logic phy_int_n, + + /* + * Ethernet: QSFP28 + */ + input wire logic [3:0] qsfp_rx_p, + input wire logic [3:0] qsfp_rx_n, + output wire logic [3:0] qsfp_tx_p, + output wire logic [3:0] qsfp_tx_n, + input wire logic qsfp_mgt_refclk_0_p, + input wire logic qsfp_mgt_refclk_0_n, + // input wire logic qsfp_mgt_refclk_1_p, + // input wire logic qsfp_mgt_refclk_1_n, + // output wire logic qsfp_recclk_p, + // output wire logic qsfp_recclk_n, + output wire logic qsfp_modsell, + output wire logic qsfp_resetl, + input wire logic qsfp_modprsl, + input wire logic qsfp_intl, + output wire logic qsfp_lpmode ); -assign led = 8'(sw); +// assign led = 8'(sw); // UART assign uart_rts = 0; @@ -110,9 +134,9 @@ taxi_axis_if #(.DATA_W(96), .KEEP_W(1), .ID_W(8)) axis_tx_cpl(); taxi_eth_mac_1g_fifo #( .PADDING_EN(1), .MIN_FRAME_LEN(64), - .TX_FIFO_DEPTH(4096), + .TX_FIFO_DEPTH(16384), .TX_FRAME_FIFO(1), - .RX_FIFO_DEPTH(4096), + .RX_FIFO_DEPTH(16384), .RX_FRAME_FIFO(1) ) eth_mac_inst ( @@ -173,6 +197,296 @@ eth_mac_inst ( .cfg_rx_enable(1'b1) ); +// QSFP28 +assign qsfp_modsell = 1'b0; +assign qsfp_resetl = 1'b1; +assign qsfp_lpmode = 1'b0; + +wire [3:0] qsfp_tx_clk; +wire [3:0] qsfp_tx_rst; +wire [3:0] qsfp_rx_clk; +wire [3:0] qsfp_rx_rst; + +wire [3:0] qsfp_rx_status; + +assign led = {qsfp_rx_status, qsfp_rx_rst}; + +wire qsfp_gtpowergood; + +wire qsfp_mgt_refclk_0; +wire qsfp_mgt_refclk_0_int; +wire qsfp_mgt_refclk_0_bufg; + +wire qsfp_rst; + +taxi_axis_if #(.DATA_W(64), .ID_W(8)) axis_qsfp_tx[3:0](); +taxi_axis_if #(.DATA_W(96), .KEEP_W(1), .ID_W(8)) axis_qsfp_tx_cpl[3:0](); +taxi_axis_if #(.DATA_W(64), .ID_W(8)) axis_qsfp_rx[3:0](); + +if (SIM) begin + + assign qsfp_gtpowergood = 1'b1; + + assign qsfp_mgt_refclk_0 = qsfp_mgt_refclk_0_p; + assign qsfp_mgt_refclk_0_int = qsfp_mgt_refclk_0_p; + assign qsfp_mgt_refclk_0_bufg = qsfp_mgt_refclk_0_int; + +end else begin + + IBUFDS_GTE3 ibufds_gte3_qsfp_mgt_refclk_0_inst ( + .I (qsfp_mgt_refclk_0_p), + .IB (qsfp_mgt_refclk_0_n), + .CEB (1'b0), + .O (qsfp_mgt_refclk_0), + .ODIV2 (qsfp_mgt_refclk_0_int) + ); + + BUFG_GT bufg_gt_qsfp_mgt_refclk_0_inst ( + .CE (qsfp_gtpowergood), + .CEMASK (1'b1), + .CLR (1'b0), + .CLRMASK (1'b1), + .DIV (3'd0), + .I (qsfp_mgt_refclk_0_int), + .O (qsfp_mgt_refclk_0_bufg) + ); + +end + +taxi_sync_reset #( + .N(4) +) +qsfp_sync_reset_inst ( + .clk(qsfp_mgt_refclk_0_bufg), + .rst(rst), + .out(qsfp_rst) +); + +taxi_eth_mac_25g_us #( + .SIM(SIM), + .VENDOR(VENDOR), + .FAMILY(FAMILY), + + .CNT(4), + + // GT type + .GT_TYPE("GTY"), + + // PHY parameters + .PADDING_EN(1'b1), + .DIC_EN(1'b1), + .MIN_FRAME_LEN(64), + .PTP_TS_EN(1'b0), + .PTP_TS_FMT_TOD(1'b1), + .PTP_TS_W(96), + .PRBS31_EN(1'b0), + .TX_SERDES_PIPELINE(1), + .RX_SERDES_PIPELINE(1), + .COUNT_125US(125000/6.4) +) +qsfp_mac_inst ( + .xcvr_ctrl_clk(clk), + .xcvr_ctrl_rst(qsfp_rst), + + /* + * Common + */ + .xcvr_gtpowergood_out(qsfp_gtpowergood), + .xcvr_gtrefclk00_in(qsfp_mgt_refclk_0), + .xcvr_qpll0lock_out(), + .xcvr_qpll0clk_out(), + .xcvr_qpll0refclk_out(), + + /* + * Serial data + */ + .xcvr_txp(qsfp_tx_p), + .xcvr_txn(qsfp_tx_n), + .xcvr_rxp(qsfp_rx_p), + .xcvr_rxn(qsfp_rx_n), + + /* + * MAC clocks + */ + .rx_clk(qsfp_rx_clk), + .rx_rst_in('0), + .rx_rst_out(qsfp_rx_rst), + .tx_clk(qsfp_tx_clk), + .tx_rst_in('0), + .tx_rst_out(qsfp_tx_rst), + .ptp_sample_clk('0), + + /* + * Transmit interface (AXI stream) + */ + .s_axis_tx(axis_qsfp_tx), + .m_axis_tx_cpl(axis_qsfp_tx_cpl), + + /* + * Receive interface (AXI stream) + */ + .m_axis_rx(axis_qsfp_rx), + + /* + * PTP clock + */ + .tx_ptp_ts('0), + .tx_ptp_ts_step('0), + .rx_ptp_ts('0), + .rx_ptp_ts_step('0), + + + /* + * Link-level Flow Control (LFC) (IEEE 802.3 annex 31B PAUSE) + */ + .tx_lfc_req('0), + .tx_lfc_resend('0), + .rx_lfc_en('0), + .rx_lfc_req(), + .rx_lfc_ack('0), + + /* + * Priority Flow Control (PFC) (IEEE 802.3 annex 31D PFC) + */ + .tx_pfc_req('0), + .tx_pfc_resend('0), + .rx_pfc_en('0), + .rx_pfc_req(), + .rx_pfc_ack('0), + + /* + * Pause interface + */ + .tx_lfc_pause_en('0), + .tx_pause_req('0), + .tx_pause_ack(), + + /* + * Status + */ + .tx_start_packet(), + .tx_error_underflow(), + .rx_start_packet(), + .rx_error_count(), + .rx_error_bad_frame(), + .rx_error_bad_fcs(), + .rx_bad_block(), + .rx_sequence_error(), + .rx_block_lock(), + .rx_high_ber(), + .rx_status(qsfp_rx_status), + .stat_tx_mcf(), + .stat_rx_mcf(), + .stat_tx_lfc_pkt(), + .stat_tx_lfc_xon(), + .stat_tx_lfc_xoff(), + .stat_tx_lfc_paused(), + .stat_tx_pfc_pkt(), + .stat_tx_pfc_xon(), + .stat_tx_pfc_xoff(), + .stat_tx_pfc_paused(), + .stat_rx_lfc_pkt(), + .stat_rx_lfc_xon(), + .stat_rx_lfc_xoff(), + .stat_rx_lfc_paused(), + .stat_rx_pfc_pkt(), + .stat_rx_pfc_xon(), + .stat_rx_pfc_xoff(), + .stat_rx_pfc_paused(), + + /* + * Configuration + */ + .cfg_ifg('{4{8'd12}}), + .cfg_tx_enable('1), + .cfg_rx_enable('1), + .cfg_tx_prbs31_enable('0), + .cfg_rx_prbs31_enable('0), + .cfg_mcf_rx_eth_dst_mcast('{4{48'h01_80_C2_00_00_01}}), + .cfg_mcf_rx_check_eth_dst_mcast('1), + .cfg_mcf_rx_eth_dst_ucast('{4{48'd0}}), + .cfg_mcf_rx_check_eth_dst_ucast('0), + .cfg_mcf_rx_eth_src('{4{48'd0}}), + .cfg_mcf_rx_check_eth_src('0), + .cfg_mcf_rx_eth_type('{4{16'h8808}}), + .cfg_mcf_rx_opcode_lfc('{4{16'h0001}}), + .cfg_mcf_rx_check_opcode_lfc('1), + .cfg_mcf_rx_opcode_pfc('{4{16'h0101}}), + .cfg_mcf_rx_check_opcode_pfc('1), + .cfg_mcf_rx_forward('0), + .cfg_mcf_rx_enable('0), + .cfg_tx_lfc_eth_dst('{4{48'h01_80_C2_00_00_01}}), + .cfg_tx_lfc_eth_src('{4{48'h80_23_31_43_54_4C}}), + .cfg_tx_lfc_eth_type('{4{16'h8808}}), + .cfg_tx_lfc_opcode('{4{16'h0001}}), + .cfg_tx_lfc_en('0), + .cfg_tx_lfc_quanta('{4{16'hffff}}), + .cfg_tx_lfc_refresh('{4{16'h7fff}}), + .cfg_tx_pfc_eth_dst('{4{48'h01_80_C2_00_00_01}}), + .cfg_tx_pfc_eth_src('{4{48'h80_23_31_43_54_4C}}), + .cfg_tx_pfc_eth_type('{4{16'h8808}}), + .cfg_tx_pfc_opcode('{4{16'h0101}}), + .cfg_tx_pfc_en('0), + .cfg_tx_pfc_quanta('{4{'{8{16'hffff}}}}), + .cfg_tx_pfc_refresh('{4{'{8{16'h7fff}}}}), + .cfg_rx_lfc_opcode('{4{16'h0001}}), + .cfg_rx_lfc_en('0), + .cfg_rx_pfc_opcode('{4{16'h0101}}), + .cfg_rx_pfc_en('0) +); + +for (genvar n = 0; n < 4; n = n + 1) begin : qsfp_ch + + taxi_axis_async_fifo #( + .DEPTH(16384), + .RAM_PIPELINE(2), + .FRAME_FIFO(1), + .USER_BAD_FRAME_VALUE(1'b1), + .USER_BAD_FRAME_MASK(1'b1), + .DROP_OVERSIZE_FRAME(1), + .DROP_BAD_FRAME(1), + .DROP_WHEN_FULL(1) + ) + ch_fifo ( + /* + * AXI4-Stream input (sink) + */ + .s_clk(qsfp_rx_clk[n]), + .s_rst(qsfp_rx_rst[n]), + .s_axis(axis_qsfp_rx[n]), + + /* + * AXI4-Stream output (source) + */ + .m_clk(qsfp_tx_clk[n]), + .m_rst(qsfp_tx_rst[n]), + .m_axis(axis_qsfp_tx[n]), + + /* + * Pause + */ + .s_pause_req(1'b0), + .s_pause_ack(), + .m_pause_req(1'b0), + .m_pause_ack(), + + /* + * Status + */ + .s_status_depth(), + .s_status_depth_commit(), + .s_status_overflow(), + .s_status_bad_frame(), + .s_status_good_frame(), + .m_status_depth(), + .m_status_depth_commit(), + .m_status_overflow(), + .m_status_bad_frame(), + .m_status_good_frame() + ); + +end + endmodule `resetall diff --git a/example/VCU108/fpga/tb/fpga_core/Makefile b/example/VCU108/fpga/tb/fpga_core/Makefile index a9ae733..0ea3c0d 100644 --- a/example/VCU108/fpga/tb/fpga_core/Makefile +++ b/example/VCU108/fpga/tb/fpga_core/Makefile @@ -20,6 +20,7 @@ MODULE = $(COCOTB_TEST_MODULES) TOPLEVEL = $(COCOTB_TOPLEVEL) VERILOG_SOURCES += ../../rtl/$(DUT).sv VERILOG_SOURCES += ../../lib/taxi/rtl/eth/taxi_eth_mac_1g_fifo.f +VERILOG_SOURCES += ../../lib/taxi/rtl/eth/us/taxi_eth_mac_25g_us.f VERILOG_SOURCES += ../../lib/taxi/rtl/lss/taxi_uart.f VERILOG_SOURCES += ../../lib/taxi/rtl/sync/taxi_sync_reset.sv VERILOG_SOURCES += ../../lib/taxi/rtl/sync/taxi_sync_signal.sv @@ -32,7 +33,9 @@ uniq_base = $(if $1,$(call uniq_base,$(foreach f,$1,$(if $(filter-out $(notdir $ VERILOG_SOURCES := $(call uniq_base,$(call process_f_files,$(VERILOG_SOURCES))) # module parameters -#export PARAM_A := value +export PARAM_SIM := "1'b1" +export PARAM_VENDOR := "\"XILINX\"" +export PARAM_FAMILY := "\"virtexu\"" ifeq ($(SIM), icarus) PLUSARGS += -fst diff --git a/example/VCU108/fpga/tb/fpga_core/baser.py b/example/VCU108/fpga/tb/fpga_core/baser.py new file mode 120000 index 0000000..ac1737a --- /dev/null +++ b/example/VCU108/fpga/tb/fpga_core/baser.py @@ -0,0 +1 @@ +../../lib/taxi/tb/eth/baser.py \ No newline at end of file diff --git a/example/VCU108/fpga/tb/fpga_core/test_fpga_core.py b/example/VCU108/fpga/tb/fpga_core/test_fpga_core.py index ebecd3d..99342d4 100644 --- a/example/VCU108/fpga/tb/fpga_core/test_fpga_core.py +++ b/example/VCU108/fpga/tb/fpga_core/test_fpga_core.py @@ -11,6 +11,7 @@ Authors: import logging import os +import sys import cocotb_test.simulator @@ -20,8 +21,19 @@ from cocotb.clock import Clock from cocotb.triggers import RisingEdge, Combine from cocotbext.eth import GmiiFrame, GmiiSource, GmiiSink +from cocotbext.eth import XgmiiFrame from cocotbext.uart import UartSource, UartSink +try: + from baser import BaseRSerdesSource, BaseRSerdesSink +except ImportError: + # attempt import from current directory + sys.path.insert(0, os.path.join(os.path.dirname(__file__))) + try: + from baser import BaseRSerdesSource, BaseRSerdesSink + finally: + del sys.path[0] + class TB: def __init__(self, dut, speed=1000e6): @@ -32,6 +44,7 @@ class TB: cocotb.start_soon(Clock(dut.clk, 8, units="ns").start()) cocotb.start_soon(Clock(dut.phy_gmii_clk, 8, units="ns").start()) + cocotb.start_soon(Clock(dut.qsfp_mgt_refclk_0_p, 6.4, units="ns").start()) self.gmii_source = GmiiSource(dut.phy_gmii_rxd, dut.phy_gmii_rx_er, dut.phy_gmii_rx_dv, dut.phy_gmii_clk, dut.phy_gmii_rst, dut.phy_gmii_clk_en) @@ -41,6 +54,16 @@ class TB: self.uart_source = UartSource(dut.uart_rxd, baud=115200, bits=8, stop_bits=1) self.uart_sink = UartSink(dut.uart_txd, baud=115200, bits=8, stop_bits=1) + self.qsfp_sources = [] + self.qsfp_sinks = [] + + for ch in dut.qsfp_mac_inst.ch: + cocotb.start_soon(Clock(ch.ch_inst.tx_clk, 2.56, units="ns").start()) + cocotb.start_soon(Clock(ch.ch_inst.rx_clk, 2.56, units="ns").start()) + + self.qsfp_sources.append(BaseRSerdesSource(ch.ch_inst.serdes_rx_data, ch.ch_inst.serdes_rx_hdr, ch.ch_inst.rx_clk, slip=ch.ch_inst.serdes_rx_bitslip, reverse=True)) + self.qsfp_sinks.append(BaseRSerdesSink(ch.ch_inst.serdes_tx_data, ch.ch_inst.serdes_tx_hdr, ch.ch_inst.tx_clk, reverse=True)) + dut.phy_gmii_clk_en.setimmediatevalue(1) dut.btnu.setimmediatevalue(0) @@ -68,6 +91,9 @@ class TB: self.dut.rst.value = 0 self.dut.phy_gmii_rst.value = 0 + for k in range(10): + await RisingEdge(self.dut.clk) + async def uart_test(tb, source, sink): tb.log.info("Test UART") @@ -130,6 +156,46 @@ async def mac_test(tb, source, sink): tb.log.info("MAC test done") +async def mac_test_25g(tb, source, sink): + tb.log.info("Test MAC") + + tb.log.info("Multiple small packets") + + count = 64 + + pkts = [bytearray([(x+k) % 256 for x in range(60)]) for k in range(count)] + + for p in pkts: + await source.send(XgmiiFrame.from_payload(p)) + + for k in range(count): + rx_frame = await sink.recv() + + tb.log.info("RX frame: %s", rx_frame) + + assert rx_frame.get_payload() == pkts[k] + assert rx_frame.check_fcs() + + tb.log.info("Multiple large packets") + + count = 32 + + pkts = [bytearray([(x+k) % 256 for x in range(1514)]) for k in range(count)] + + for p in pkts: + await source.send(XgmiiFrame.from_payload(p)) + + for k in range(count): + rx_frame = await sink.recv() + + tb.log.info("RX frame: %s", rx_frame) + + assert rx_frame.get_payload() == pkts[k] + assert rx_frame.check_fcs() + + tb.log.info("MAC test done") + + @cocotb.test() async def run_test(dut): @@ -137,15 +203,22 @@ async def run_test(dut): await tb.init() + tests = [] + tb.log.info("Start UART test") - uart_test_cr = cocotb.start_soon(uart_test(tb, tb.uart_source, tb.uart_sink)) + tests.append(cocotb.start_soon(uart_test(tb, tb.uart_source, tb.uart_sink))) tb.log.info("Start BASE-T MAC loopback test") - baset_test_cr = cocotb.start_soon(mac_test(tb, tb.gmii_source, tb.gmii_sink)) + tests.append(cocotb.start_soon(mac_test(tb, tb.gmii_source, tb.gmii_sink))) - await Combine(uart_test_cr, baset_test_cr) + for k in range(len(tb.qsfp_sources)): + tb.log.info("Start QSFP %d MAC loopback test", k) + + tests.append(cocotb.start_soon(mac_test_25g(tb, tb.qsfp_sources[k], tb.qsfp_sinks[k]))) + + await Combine(*tests) await RisingEdge(dut.clk) await RisingEdge(dut.clk) @@ -179,6 +252,7 @@ def test_fpga_core(request): verilog_sources = [ os.path.join(rtl_dir, f"{dut}.sv"), os.path.join(lib_dir, "taxi", "rtl", "eth", "taxi_eth_mac_1g_fifo.f"), + os.path.join(lib_dir, "taxi", "rtl", "eth", "us", "taxi_eth_mac_25g_us.f"), os.path.join(lib_dir, "taxi", "rtl", "lss", "taxi_uart.f"), os.path.join(lib_dir, "taxi", "rtl", "sync", "taxi_sync_reset.sv"), os.path.join(lib_dir, "taxi", "rtl", "sync", "taxi_sync_signal.sv"), @@ -189,7 +263,9 @@ def test_fpga_core(request): parameters = {} - # parameters['A'] = val + parameters['SIM'] = "1'b1" + parameters['VENDOR'] = "\"XILINX\"" + parameters['FAMILY'] = "\"virtexu\"" extra_env = {f'PARAM_{k}': str(v) for k, v in parameters.items()}