#define _XOPEN_SOURCE 700 #include /* open */ #include /* uint64_t */ #include /* printf */ #include /* size_t */ #include /* pread, sysconf */ #include #include #include "alibaba_pcie_top_regs.h" typedef struct { uint64_t pfn : 55; unsigned int soft_dirty : 1; unsigned int file_page : 1; unsigned int swapped : 1; unsigned int present : 1; } PagemapEntry; /* Parse the pagemap entry for the given virtual address. * * @param[out] entry the parsed entry * @param[in] pagemap_fd file descriptor to an open /proc/pid/pagemap file * @param[in] vaddr virtual address to get entry for * @return 0 for success, 1 for failure */ int pagemap_get_entry(PagemapEntry *entry, int pagemap_fd, uintptr_t vaddr) { size_t nread; ssize_t ret; uint64_t data; uintptr_t vpn; vpn = vaddr / sysconf(_SC_PAGE_SIZE); nread = 0; while (nread < sizeof(data)) { ret = pread(pagemap_fd, ((uint8_t*)&data) + nread, sizeof(data) - nread, vpn * sizeof(data) + nread); nread += ret; if (ret <= 0) { return 1; } } entry->pfn = data & (((uint64_t)1 << 55) - 1); entry->soft_dirty = (data >> 55) & 1; entry->file_page = (data >> 61) & 1; entry->swapped = (data >> 62) & 1; entry->present = (data >> 63) & 1; return 0; } /* Convert the given virtual address to physical using /proc/PID/pagemap. * * @param[out] paddr physical address * @param[in] pid process to convert for * @param[in] vaddr virtual address to get entry for * @return 0 for success, 1 for failure */ int virt_to_phys_user(uintptr_t *paddr, uintptr_t vaddr) { char pagemap_file[BUFSIZ]; int pagemap_fd; snprintf(pagemap_file, sizeof(pagemap_file), "/proc/self/pagemap"); pagemap_fd = open(pagemap_file, O_RDONLY); if (pagemap_fd < 0) { return 1; } PagemapEntry entry; if (pagemap_get_entry(&entry, pagemap_fd, vaddr)) { return 1; } close(pagemap_fd); *paddr = (entry.pfn * sysconf(_SC_PAGE_SIZE)) + (vaddr % sysconf(_SC_PAGE_SIZE)); return 0; } int main(void) { uint32_t dma_mem_addr = 0x00; //char* src_1 = "Hello, world! This is some data that is stored in system memory!"; char* src_1 = "AAAA"; //char* src_2 = "This is many different words, in a different order"; char* src_2 = "BBBB"; //printf("Allocating 1024 bytes as source\n"); char* src = (char*)malloc(1024); printf("Virtual address: %lx\n", src); uintptr_t src_phys; virt_to_phys_user(&src_phys, (uintptr_t)src); printf("Physical address: %lx\n", src_phys); printf("\n\n"); printf("Allocating 1024 bytes as destination\n"); char* dst = (char*)malloc(1024); printf("Virtual address: %p\n", dst); uintptr_t dst_phys; virt_to_phys_user(&dst_phys, (uintptr_t)dst); printf("Physical address: %lx\n", dst_phys); printf("\n\n"); printf("mmaping PCIe space\n"); // this is hardcoded, seems to be deterministic. uint32_t pcie_physical_base_offset = 0xfe800000; int fd = open("/dev/mem", O_RDWR|O_SYNC); alibaba_pcie_top_regs_t* top_regs = (alibaba_pcie_top_regs_t*)mmap(0, 64, PROT_READ|PROT_WRITE, MAP_SHARED, fd, pcie_physical_base_offset); printf("Virtual PCIe Base: %p\n", top_regs); for (int i = 0; i < 2; i++) { if (i == 0) { strncpy((void*)src, src_1, 1024); } else { strncpy((void*)src, src_2, 1024); } memset((void*)dst, 0, 1024); printf("Sending read DMA\n"); top_regs->pcie_top_regs.pcie_dma_regs.dma_rd.src_addr_low = (uint32_t)src_phys; top_regs->pcie_top_regs.pcie_dma_regs.dma_rd.src_addr_high = (uint32_t)(src_phys >> 32); top_regs->pcie_top_regs.pcie_dma_regs.dma_rd.dst_addr = dma_mem_addr; top_regs->pcie_top_regs.pcie_dma_regs.dma_rd.length = strlen(src); for (int i = 0; i < 4; i++) { printf("pcie_rd_dma[%d] = %x\n", i, (&top_regs->pcie_top_regs.pcie_dma_regs.dma_rd.src_addr_low)[i]); } top_regs->pcie_top_regs.pcie_dma_regs.dma_rd.trigger = 1; printf("%d\n", top_regs->pcie_top_regs.pcie_dma_regs.dma_rd.trigger); printf("\n\n"); printf("Sending write DMA\n"); // we use dma_mem_addr twice, but these are actually 2 separate memories. // Set up stream to memory DMA to store ethernet frame printf("Setting up stream to memory DMA\n"); top_regs->eth_dma_wrapper_regs.pcie_dma_regs.dma_wr.src_addr = 0; top_regs->eth_dma_wrapper_regs.pcie_dma_regs.dma_wr.dst_addr_low = dma_mem_addr; top_regs->eth_dma_wrapper_regs.pcie_dma_regs.dma_wr.dst_addr_high = 0; top_regs->eth_dma_wrapper_regs.pcie_dma_regs.dma_wr.length = strlen(src); top_regs->eth_dma_wrapper_regs.pcie_dma_regs.dma_wr.trigger = 1; for (int i = 0; i < 4; i++) { printf("eth_wr_dma[%d] = %x\n", i, (&top_regs->eth_dma_wrapper_regs.pcie_dma_regs.dma_wr.dst_addr_low )[i]); } // Trigger memory to stream dma to send ethernet frame printf("Sending memory to stream DMA\n"); top_regs->eth_dma_wrapper_regs.pcie_dma_regs.dma_rd.src_addr_low = dma_mem_addr; top_regs->eth_dma_wrapper_regs.pcie_dma_regs.dma_rd.src_addr_high = 0; top_regs->eth_dma_wrapper_regs.pcie_dma_regs.dma_rd.dst_addr = 0; top_regs->eth_dma_wrapper_regs.pcie_dma_regs.dma_rd.length = strlen(src); top_regs->eth_dma_wrapper_regs.pcie_dma_regs.dma_rd.trigger = 1; for (int i = 0; i < 4; i++) { printf("eth_rd_dma[%d] = %x\n", i, (&top_regs->eth_dma_wrapper_regs.pcie_dma_regs.dma_rd.src_addr_low)[i]); } printf("Sending read DMA\n"); top_regs->pcie_top_regs.pcie_dma_regs.dma_wr.dst_addr_low = (uint32_t)dst_phys; top_regs->pcie_top_regs.pcie_dma_regs.dma_wr.dst_addr_high = (uint32_t)(dst_phys >> 32); top_regs->pcie_top_regs.pcie_dma_regs.dma_wr.src_addr = dma_mem_addr; top_regs->pcie_top_regs.pcie_dma_regs.dma_wr.length = strlen(src); for (int i = 8; i < 12; i++) { printf("pcie_wr_dma[%d] = %x\n", i, (&top_regs->pcie_top_regs.pcie_dma_regs.dma_wr.dst_addr_low)[i]); } top_regs->pcie_top_regs.pcie_dma_regs.dma_wr.trigger = 1; printf("%d\n", top_regs->pcie_top_regs.pcie_dma_regs.dma_wr.trigger); printf("\n\n"); printf("strlen(dst)=%d\n", strlen(dst)); printf("%s\n", dst); } }