使用vfio进行映射时不能写入FPGA寄存器。

Question

我试图用vfio驱动程序在PCI-e总线上映射一个FPGA .我可以正确地读取寄存器，但是，当我试图写寄存器时，似乎写了一个寄存器，并且我可以将它读回来，但是，当我停止用于读/写和重新启动它的进程时，我发现寄存器回滚回原来的值。如果我用、uio映射FPGA，一切都很好。

这是加载vfio驱动程序的脚本：

modprobe vfio-pci enable_sriov=1 disable_idle_d3=1
VID="10ee"
DID="903f"
DBDF="0000:"`lspci -n | grep -E ${VID}:${DID} | cut -d ' ' -f1`
echo ${DBDF}
ROOT_DBDF="0000:3a:00.0"
readlink /sys/bus/pci/devices/${DBDF}/iommu_group
GROUP=`readlink /sys/bus/pci/devices/${DBDF}/iommu_group | rev | cut -d '/' -f1 | rev`
echo "GROUP " ${GROUP}
# unbind the ROOT device makes the group viable
echo ${ROOT_DBDF}  > /sys/bus/pci/devices/${ROOT_DBDF}/driver/unbind; sleep 1
echo ${DBDF}  > /sys/bus/pci/devices/${DBDF}/driver/unbind; sleep 1
echo ${VID} ${DID} > /sys/bus/pci/drivers/vfio-pci/remove_id; sleep 1
echo ${VID} ${DID} > /sys/bus/pci/drivers/vfio-pci/new_id; sleep 1
echo 8086 2030 > /sys/bus/pci/drivers/vfio-pci/new_id; sleep 1
ls -l /sys/bus/pci/devices/${DBDF}/iommu_group/devices; sleep 1
chmod 660 /dev/vfio/vfio

然后，我使用以下程序来读取和写入寄存器。

#include <stdio.h>
#include <stdint.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <sys/mman.h>
#include <sys/ioctl.h>

#include <linux/vfio.h>

int main(int argc, char** argv)
{

        int container, group, device, i;
        struct vfio_group_status group_status =
                                        { .argsz = sizeof(group_status) };
        struct vfio_iommu_type1_info iommu_info = { .argsz = sizeof(iommu_info) };
        struct vfio_iommu_type1_dma_map dma_map = { .argsz = sizeof(dma_map) };
        struct vfio_device_info device_info = { .argsz = sizeof(device_info) };

        /* Create a new container */
        container = open("/dev/vfio/vfio", O_RDWR);

        if (ioctl(container, VFIO_GET_API_VERSION) != VFIO_API_VERSION)
                printf("Unknown API version\n");
                /* Unknown API version */

        if (!ioctl(container, VFIO_CHECK_EXTENSION, VFIO_TYPE1_IOMMU))
                printf("Doesn't support IOMMU driver we want\n");

        /* Open the group */
        group = open("/dev/vfio/69", O_RDWR);

        /* Test the group is viable and available */
        ioctl(group, VFIO_GROUP_GET_STATUS, &group_status);

        if (!(group_status.flags & VFIO_GROUP_FLAGS_VIABLE))
                printf("Group is not viable\n");
                /* Group is not viable (ie, not all devices bound for vfio) */

        /* Add the group to the container */
        ioctl(group, VFIO_GROUP_SET_CONTAINER, &container);

        /* Enable the IOMMU model we want */
        ioctl(container, VFIO_SET_IOMMU, VFIO_TYPE1_IOMMU);

        /* Get addition IOMMU info */
        ioctl(container, VFIO_IOMMU_GET_INFO, &iommu_info);

        /* Allocate some space and setup a DMA mapping */
/*
        dma_map.vaddr = mmap(0, 1024 * 1024, PROT_READ | PROT_WRITE,
                        MAP_PRIVATE | MAP_ANONYMOUS, 0, 0);
        dma_map.size = 1024 * 1024;
        dma_map.iova = 0;
        dma_map.flags = VFIO_DMA_MAP_FLAG_READ | VFIO_DMA_MAP_FLAG_WRITE;

        ioctl(container, VFIO_IOMMU_MAP_DMA, &dma_map);
*/
        /* Get a file descriptor for the device */
        device = ioctl(group, VFIO_GROUP_GET_DEVICE_FD, "0000:3b:00.0");

        /* Test and setup the device */
        ioctl(device, VFIO_DEVICE_GET_INFO, &device_info);

        printf("NUM REGIONS %d\n", device_info.num_regions);

        struct vfio_region_info regs[64];
        for (i = 0; i < device_info.num_regions; i++) {
                regs[i].argsz = sizeof(struct vfio_region_info);
                regs[i].index = i;

                ioctl(device, VFIO_DEVICE_GET_REGION_INFO, &regs[i]);

                printf("region %d flags %08x offset %lld size %lld\n", i, regs[i].flags, regs[i].offset, regs[i].size);
                /* Setup mappings... read/write offsets, mmaps
                * For PCI devices, config space is a region */
        }

        volatile uint8_t* ptr = mmap(0, regs[0].size, PROT_READ | PROT_WRITE, MAP_SHARED, device, 0);

        printf("addr %p\n", ptr);

        printf("reg 0x38000 %08x\n", *(uint32_t*)(ptr + 0x38000));

        {
                uint32_t ival = *(volatile uint32_t*) (ptr + 0x38008);
                *(volatile uint32_t*) (ptr + 0x38008) = ival + 0x1000;
                printf("%08x\n", *(volatile uint32_t*) (ptr + 0x38008));
        }
        printf("reg 0x38008 %08x\n", *(volatile uint32_t*)(ptr + 0x38008));

}

Answer 1

mmap调用中使用了MAP_SHARED，但程序的行为类似于使用了MAP_PRIVATE。