alibaba_pcie/sw/test/pcie_dma_test.c

#define _XOPEN_SOURCE 700
#include <fcntl.h> /* open */
#include <stdint.h> /* uint64_t  */
#include <stdio.h> /* printf */
#include <stdlib.h> /* size_t */
#include <unistd.h> /* pread, sysconf */
#include <sys/mman.h>
#include <string.h>

#include "alibaba_pcie_top_regs.h"

typedef struct {
    uint64_t pfn : 55;
    unsigned int soft_dirty : 1;
    unsigned int file_page : 1;
    unsigned int swapped : 1;
    unsigned int present : 1;
} PagemapEntry;

/* Parse the pagemap entry for the given virtual address.
 *
 * @param[out] entry      the parsed entry
 * @param[in]  pagemap_fd file descriptor to an open /proc/pid/pagemap file
 * @param[in]  vaddr      virtual address to get entry for
 * @return 0 for success, 1 for failure
 */
int pagemap_get_entry(PagemapEntry *entry, int pagemap_fd, uintptr_t vaddr)
{
    size_t nread;
    ssize_t ret;
    uint64_t data;
    uintptr_t vpn;

    vpn = vaddr / sysconf(_SC_PAGE_SIZE);
    nread = 0;
    while (nread < sizeof(data)) {
        ret = pread(pagemap_fd, ((uint8_t*)&data) + nread, sizeof(data) - nread,
                vpn * sizeof(data) + nread);
        nread += ret;
        if (ret <= 0) {
            return 1;
        }
    }
    entry->pfn = data & (((uint64_t)1 << 55) - 1);
    entry->soft_dirty = (data >> 55) & 1;
    entry->file_page = (data >> 61) & 1;
    entry->swapped = (data >> 62) & 1;
    entry->present = (data >> 63) & 1;
    return 0;
}

/* Convert the given virtual address to physical using /proc/PID/pagemap.
 *
 * @param[out] paddr physical address
 * @param[in]  pid   process to convert for
 * @param[in] vaddr virtual address to get entry for
 * @return 0 for success, 1 for failure
 */
int virt_to_phys_user(uintptr_t *paddr, uintptr_t vaddr)
{
    char pagemap_file[BUFSIZ];
    int pagemap_fd;

    snprintf(pagemap_file, sizeof(pagemap_file), "/proc/self/pagemap");
    pagemap_fd = open(pagemap_file, O_RDONLY);
    if (pagemap_fd < 0) {
        return 1;
    }
    PagemapEntry entry;
    if (pagemap_get_entry(&entry, pagemap_fd, vaddr)) {
        return 1;
    }
    close(pagemap_fd);
    *paddr = (entry.pfn * sysconf(_SC_PAGE_SIZE)) + (vaddr % sysconf(_SC_PAGE_SIZE));
    return 0;
}

int main(void)
{

    uint32_t dma_mem_addr = 0x00;

    //char* src_1 = "Hello, world! This is some data that is stored in system memory!";
    char* src_1 = "AAAA";
    //char* src_2 = "This is many different words, in a different order";
    char* src_2 = "BBBB";

    //printf("Allocating 1024 bytes as source\n");
    char* src = (char*)malloc(1024);
    printf("Virtual address: %lx\n", src);
    uintptr_t src_phys;
    virt_to_phys_user(&src_phys, (uintptr_t)src);
    printf("Physical address: %lx\n", src_phys);
    printf("\n\n");


    printf("Allocating 1024 bytes as destination\n");
    char* dst = (char*)malloc(1024);
    printf("Virtual address: %p\n", dst);
    uintptr_t dst_phys;
    virt_to_phys_user(&dst_phys, (uintptr_t)dst);
    printf("Physical address: %lx\n", dst_phys);
    printf("\n\n");


    printf("mmaping PCIe space\n");
    // this is hardcoded, seems to be deterministic.
    uint32_t pcie_physical_base_offset = 0xfe800000;
    int fd = open("/dev/mem", O_RDWR|O_SYNC);
    alibaba_pcie_top_regs_t* top_regs = (alibaba_pcie_top_regs_t*)mmap(0, 64, PROT_READ|PROT_WRITE, MAP_SHARED, fd, pcie_physical_base_offset);
    printf("Virtual PCIe Base: %p\n", top_regs);


    for (int i = 0; i < 2; i++) { 
        if (i == 0) {
            strncpy((void*)src, src_1, 1024);
        } else {
            strncpy((void*)src, src_2, 1024);
        }

        memset((void*)dst, 0, 1024);

        printf("Sending read DMA\n");
        top_regs->pcie_top_regs.pcie_dma_regs.dma_rd.src_addr_low = (uint32_t)src_phys;
        top_regs->pcie_top_regs.pcie_dma_regs.dma_rd.src_addr_high = (uint32_t)(src_phys >> 32);
        top_regs->pcie_top_regs.pcie_dma_regs.dma_rd.dst_addr = dma_mem_addr;
        top_regs->pcie_top_regs.pcie_dma_regs.dma_rd.length = strlen(src);

        for (int i = 0; i < 4; i++) {
            printf("top_regs[%d] = %x\n", i, (&top_regs->pcie_top_regs.pcie_dma_regs.dma_rd)[i]);
        }

        top_regs->pcie_top_regs.pcie_dma_regs.dma_rd.trigger = 1;
        printf("%d\n", top_regs->pcie_top_regs.pcie_dma_regs.dma_rd.trigger);
        printf("\n\n");

        printf("Sending write DMA\n");

        // we use dma_mem_addr twice, but these are actually 2 separate memories.

        // Set up stream to memory DMA to store ethernet frame
        printf("Setting up stream to memory DMA");
        top_regs->eth_dma_wrapper_regs.pcie_dma_regs.dma_wr.src_addr = 0;
        top_regs->eth_dma_wrapper_regs.pcie_dma_regs.dma_wr.dst_addr_low = dma_mem_addr;
        top_regs->eth_dma_wrapper_regs.pcie_dma_regs.dma_wr.dst_addr_high =  0;
        top_regs->eth_dma_wrapper_regs.pcie_dma_regs.dma_wr.length = strlen(src);
        top_regs->eth_dma_wrapper_regs.pcie_dma_regs.dma_wr.trigger = 1;

        // Trigger memory to stream dma to send ethernet frame
        printf("Sending memory to stream DMA");
        top_regs->eth_dma_wrapper_regs.pcie_dma_regs.dma_rd.src_addr_low = dma_mem_addr;
        top_regs->eth_dma_wrapper_regs.pcie_dma_regs.dma_rd.src_addr_high = 0;
        top_regs->eth_dma_wrapper_regs.pcie_dma_regs.dma_rd.dst_addr = 0;
        top_regs->eth_dma_wrapper_regs.pcie_dma_regs.dma_rd.length = strlen(src);
        top_regs->eth_dma_wrapper_regs.pcie_dma_regs.dma_rd.trigger = 1;


        printf("Sending read DMA\n");
        top_regs->pcie_top_regs.pcie_dma_regs.dma_wr.dst_addr_low = (uint32_t)dst_phys;
        top_regs->pcie_top_regs.pcie_dma_regs.dma_wr.dst_addr_high = (uint32_t)(dst_phys >> 32);
        top_regs->pcie_top_regs.pcie_dma_regs.dma_wr.src_addr = dma_mem_addr;
        top_regs->pcie_top_regs.pcie_dma_regs.dma_wr.length = strlen(src);

        for (int i = 8; i < 12; i++) {
            printf("pcie_base[%d] = %x\n", i, (&top_regs->pcie_top_regs.pcie_dma_regs.dma_wr)[i]);
        }

        top_regs->pcie_top_regs.pcie_dma_regs.dma_wr.trigger = 1;
        printf("%d\n", top_regs->pcie_top_regs.pcie_dma_regs.dma_wr.trigger);
        printf("\n\n");

        printf("strlen(dst)=%d\n", strlen(dst));
        printf("%s\n", dst);

    }
}