PCIe 软件开发

/*
 * pci_dev - 表示一个PCI设备
 * 包含设备的所有信息：配置空间、资源、驱动等
 */
struct pci_dev {
    struct list_head bus_list;      /* 总线链表 */
    struct pci_bus *bus;            /* 所属总线 */
    struct pci_bus *subordinate;    /* 桥接设备的下游总线 */
    
    u16 vendor;                     /* 厂商ID */
    u16 device;                     /* 设备ID */
    u16 subsystem_vendor;           /* 子系统厂商ID */
    u16 subsystem_device;           /* 子系统设备ID */
    u8 revision;                    /* 修订版本 */
    u8 hdr_type;                    /* 头部类型 */
    
    struct resource resource[DEVICE_COUNT_RESOURCE]; /* BAR资源 */
    
    u8 irq;                         /* 中断号 */
    struct msix_entry *msix_entries; /* MSI-X条目 */
    
    struct pci_driver *driver;      /* 绑定的驱动 */
    void *driver_data;              /* 驱动私有数据 */
};

/*
 * pci_driver - PCI驱动结构
 * 定义驱动标识、探测/移除回调等
 */
struct pci_driver {
    struct list_head node;
    const char *name;
    const struct pci_device_id *id_table;
    
    int (*probe)(struct pci_dev *dev, const struct pci_device_id *id);
    void (*remove)(struct pci_dev *dev);
    int (*suspend)(struct pci_dev *dev, pm_message_t state);
    int (*resume)(struct pci_dev *dev);
    void (*shutdown)(struct pci_dev *dev);
    
    struct device_driver driver;
    const struct pci_error_handlers *err_handler;
};

/*
 * 定义设备ID表 - 驱动匹配规则
 */
static const struct pci_device_id my_pci_ids[] = {
    /* vendor, device, subvendor, subdevice, class, class_mask, driver_data */
    { PCI_DEVICE(0x1234, 0x5678) },  /* 匹配厂商1234, 设备5678 */
    { PCI_DEVICE_CLASS(PCI_CLASS_STORAGE_NVME, 0xffffff) }, /* 匹配NVMe类 */
    { 0, }
};
MODULE_DEVICE_TABLE(pci, my_pci_ids);

/*
 * 驱动结构定义
 */
static struct pci_driver my_pci_driver = {
    .name = "my_pci_driver",
    .id_table = my_pci_ids,
    .probe = my_probe,
    .remove = my_remove,
};

/*
 * 模块注册
 */
module_pci_driver(my_pci_driver);

MODULE_LICENSE("GPL");
MODULE_AUTHOR("Your Name");
MODULE_DESCRIPTION("PCIe Device Driver Example");

static int my_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
{
    int ret;
    void __iomem *mmio_base;
    struct my_device *dev;
    
    /* 1. 启用PCI设备 */
    ret = pci_enable_device(pdev);
    if (ret) {
        dev_err(&pdev->dev, "无法启用设备\n");
        return ret;
    }
    
    /* 2. 请求MMIO区域 (BAR0) */
    ret = pci_request_region(pdev, 0, "my_driver");
    if (ret) {
        dev_err(&pdev->dev, "无法请求BAR0区域\n");
        goto err_disable;
    }
    
    /* 3. 映射MMIO到内核虚拟地址空间 */
    mmio_base = pci_iomap(pdev, 0, 0);
    if (!mmio_base) {
        dev_err(&pdev->dev, "无法映射MMIO\n");
        ret = -ENOMEM;
        goto err_region;
    }
    
    /* 4. 设置DMA掩码 */
    ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
    if (ret) {
        ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
        if (ret) {
            dev_err(&pdev->dev, "不支持DMA\n");
            goto err_iomap;
        }
    }
    
    /* 5. 启用总线主控 (DMA) */
    pci_set_master(pdev);
    
    /* 6. 分配设备私有结构 */
    dev = kzalloc(sizeof(*dev), GFP_KERNEL);
    if (!dev) {
        ret = -ENOMEM;
        goto err_iomap;
    }
    
    dev->pdev = pdev;
    dev->mmio_base = mmio_base;
    pci_set_drvdata(pdev, dev);
    
    /* 7. 初始化设备硬件 */
    my_hw_init(dev);
    
    /* 8. 注册中断 */
    ret = request_irq(pdev->irq, my_interrupt, IRQF_SHARED,
                      "my_device", dev);
    if (ret) {
        dev_err(&pdev->dev, "无法注册中断\n");
        goto err_free;
    }
    
    dev_info(&pdev->dev, "设备初始化成功\n");
    return 0;

err_free:
    kfree(dev);
err_iomap:
    pci_iounmap(pdev, mmio_base);
err_region:
    pci_release_region(pdev, 0);
err_disable:
    pci_disable_device(pdev);
    return ret;
}

static void my_remove(struct pci_dev *pdev)
{
    struct my_device *dev = pci_get_drvdata(pdev);
    
    if (!dev)
        return;
    
    /* 禁用中断 */
    free_irq(pdev->irq, dev);
    
    /* 停止设备 */
    my_hw_stop(dev);
    
    /* 清理资源 */
    pci_iounmap(pdev, dev->mmio_base);
    pci_release_region(pdev, 0);
    pci_disable_device(pdev);
    
    kfree(dev);
    pci_set_drvdata(pdev, NULL);
}

/*
 * MSI-X中断实现
 */

#define NUM_MSIX_VECTORS 16

struct msix_entry msix_entries[NUM_MSIX_VECTORS];

static irqreturn_t my_msix_handler(int irq, void *data)
{
    struct my_queue *q = data;
    u32 status;
    
    /* 读取中断状态 */
    status = readl(q->mmio + QUEUE_INT_STATUS);
    writel(status, q->mmio + QUEUE_INT_STATUS); /* 清除中断 */
    
    /* 处理完成队列 */
    my_process_completions(q);
    
    return IRQ_HANDLED;
}

static int setup_msix(struct pci_dev *pdev, struct my_device *dev)
{
    int ret, i;
    
    /* 启用MSI-X */
    ret = pci_alloc_irq_vectors(pdev, NUM_MSIX_VECTORS, NUM_MSIX_VECTORS,
                                PCI_IRQ_MSIX);
    if (ret < 0) {
        dev_err(&pdev->dev, "无法分配MSI-X向量: %d\n", ret);
        return ret;
    }
    
    /* 为每个向量注册中断处理函数 */
    for (i = 0; i < NUM_MSIX_VECTORS; i++) {
        int vector = pci_irq_vector(pdev, i);
        
        ret = request_irq(vector, my_msix_handler, 0,
                          "my_device", &dev->queues[i]);
        if (ret) {
            dev_err(&pdev->dev, "无法请求IRQ %d: %d\n", vector, ret);
            goto err_free_irqs;
        }
        
        dev->queues[i].vector = vector;
    }
    
    return 0;

err_free_irqs:
    while (--i >= 0) {
        free_irq(dev->queues[i].vector, &dev->queues[i]);
    }
    pci_free_irq_vectors(pdev);
    return ret;
}

/*
 * pci_host_bridge - 表示一个PCI主机桥
 * 是RC控制器在内核中的抽象
 */
struct pci_host_bridge {
    struct device dev;
    struct pci_bus *bus;           /* 根总线 */
    struct list_head windows;      /* 资源窗口 */
    void *sysdata;                 /* 平台私有数据 */
    
    /* 回调函数 */
    int (*map_irq)(const struct pci_dev *, u8, u8);
    void (*swizzle_irq)(struct pci_dev *, u8 *, u8 *);
};

/*
 * pci_ops - 配置空间访问操作
 * RC驱动必须实现这些操作
 */
struct pci_ops {
    void __iomem *(*map_bus)(struct pci_bus *bus, unsigned int devfn, int where);
    int (*read)(struct pci_bus *bus, unsigned int devfn, int where, int size, u32 *val);
    int (*write)(struct pci_bus *bus, unsigned int devfn, int where, int size, u32 *val);
};

/*
 * ECAM配置空间映射
 * 地址计算：Base + (Bus << 20) + (Device << 15) + (Function << 12) + Offset
 */
static void __iomem *ecam_map_bus(struct pci_bus *bus, unsigned int devfn, int where)
{
    struct my_rc_data *data = bus->sysdata;
    void __iomem *base = data->ecam_base;
    unsigned int busn = bus->number;
    unsigned int dev = PCI_SLOT(devfn);
    unsigned int fn = PCI_FUNC(devfn);
    
    u32 offset = (busn << 20) | (dev << 15) | (fn << 12) | (where & ~3);
    
    return base + offset;
}

static int ecam_read(struct pci_bus *bus, unsigned int devfn, 
                     int where, int size, u32 *val)
{
    void __iomem *addr = ecam_map_bus(bus, devfn, where);
    
    if (!addr) {
        *val = ~0;
        return PCIBIOS_DEVICE_NOT_FOUND;
    }
    
    *val = readl(addr);
    
    /* 根据size调整返回值 */
    if (size == 1)
        *val = (*val >> ((where & 3) * 8)) & 0xff;
    else if (size == 2)
        *val = (*val >> ((where & 3) * 8)) & 0xffff;
    
    return PCIBIOS_SUCCESSFUL;
}

static struct pci_ops ecam_ops = {
    .map_bus = ecam_map_bus,
    .read = ecam_read,
    .write = ecam_write,  /* 类似实现 */
};

/*
 * RC驱动probe函数示例
 */
static int my_rc_probe(struct platform_device *pdev)
{
    struct device *dev = &pdev->dev;
    struct pci_host_bridge *bridge;
    struct resource *cfg_res, *io_res, *mem_res;
    struct my_rc_data *data;
    int ret;
    
    /* 1. 分配host bridge结构 */
    bridge = devm_pci_alloc_host_bridge(dev, sizeof(*data));
    if (!bridge)
        return -ENOMEM;
    
    data = pci_host_bridge_priv(bridge);
    
    /* 2. 获取资源（从设备树或ACPI） */
    cfg_res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "config");
    data->ecam_base = devm_ioremap_resource(dev, cfg_res);
    if (IS_ERR(data->ecam_base))
        return PTR_ERR(data->ecam_base);
    
    /* 3. 初始化硬件 */
    ret = my_rc_hw_init(pdev, data);
    if (ret)
        return ret;
    
    /* 4. 设置配置空间操作 */
    bridge->ops = &ecam_ops;
    bridge->sysdata = data;
    
    /* 5. 设置资源窗口 */
    list_splice_init(&bridge->windows, &bridge->dma_ranges);
    
    /* 6. 注册host bridge */
    ret = devm_pci_scan_host_bridge(dev, bridge);
    if (ret)
        return ret;
    
    /* 7. 为每个根总线创建sysfs */
    pci_bus_assign_resources(bridge->bus);
    pci_bus_add_devices(bridge->bus);
    
    return 0;
}

static const struct of_device_id my_rc_of_match[] = {
    { .compatible = "vendor,pcie-rc", },
    { }
};

static struct platform_driver my_rc_driver = {
    .probe = my_rc_probe,
    .driver = {
        .name = "my-pcie-rc",
        .of_match_table = my_rc_of_match,
    },
};
builtin_platform_driver(my_rc_driver);

/* ARM平台设备树示例 */
pcie@40000000 {
    compatible = "vendor,pcie-rc";
    reg = <0x40000000 0x01000000>,    /* 配置空间 */
          <0x41000000 0x00100000>;    /* RC寄存器 */
    reg-names = "config", "ctrl";
    
    #address-cells = <3>;
    #size-cells = <2>;
    ranges = <0x02000000 0x0 0x80000000 0x80000000 0x0 0x20000000>;
    
    bus-range = <0x00 0xff>;
    
    interrupts = ,
                 ;
    interrupt-names = "intx", "msi";
    
    #interrupt-cells = <1>;
    interrupt-map-mask = <0 0 0 7>;
    interrupt-map = <0 0 0 1 &gic GIC_SPI 120 ...>;
    
    status = "okay";
};

/*
 * EP控制器初始化示例（伪代码）
 * 以Xilinx PCIe IP核为例
 */

/* 1. 设置设备ID和厂商ID */
Xil_Out32(PCIE_EP_BASE + PCIE_CFG_VENDOR_ID, 0x1234);
Xil_Out32(PCIE_EP_BASE + PCIE_CFG_DEVICE_ID, 0x5678);

/* 2. 配置BAR */
/* BAR0: 16KB Memory空间，可预取 */
Xil_Out32(PCIE_EP_BASE + PCIE_CFG_BAR0, 0x00004004); /* 16KB, Prefetchable, 64-bit */
Xil_Out32(PCIE_EP_BASE + PCIE_CFG_BAR0_HIGH, 0x00000000);

/* BAR2: I/O空间 */
Xil_Out32(PCIE_EP_BASE + PCIE_CFG_BAR2, 0x00000101); /* 256B I/O */

/* 3. 配置能力结构 */
/* 设置MSI能力 */
Xil_Out32(PCIE_EP_BASE + PCIE_CAP_MSI_ADDR, msi_address);
Xil_Out32(PCIE_EP_BASE + PCIE_CAP_MSI_DATA, msi_data);

/* 4. 启用EP控制器 */
Xil_Out32(PCIE_EP_BASE + PCIE_CTRL, PCIE_CTRL_ENABLE);

/* 5. 等待链路训练完成 */
while (!(Xil_In32(PCIE_EP_BASE + PCIE_STATUS) & PCIE_STATUS_LINK_UP))
    ;

/*
 * 处理来自RC的Memory Write请求
 */
void handle_rc_write(u32 bar_index, u32 offset, u32 data)
{
    switch (bar_index) {
    case 0: /* BAR0 - 控制寄存器区域 */
        if (offset == REG_CTRL_START) {
            /* 处理启动命令 */
            start_device();
        } else if (offset == REG_CTRL_RESET) {
            /* 处理复位命令 */
            reset_device();
        }
        break;
        
    case 2: /* BAR2 - 数据缓冲区 */
        /* 处理数据写入 */
        data_buffer[offset/4] = data;
        break;
    }
}

/*
 * 发送MSI中断到RC
 */
void send_msi_interrupt(u32 vector)
{
    /* 设置MSI数据 */
    Xil_Out32(PCIE_EP_BASE + PCIE_MSI_DATA, vector);
    
    /* 触发MSI */
    Xil_Out32(PCIE_EP_BASE + PCIE_MSI_TRIGGER, 1);
}

/*
 * Linux PCIe EP驱动示例
 * 需要启用CONFIG_PCI_ENDPOINT
 */
#include 
#include 

static int my_epf_bind(struct pci_epf *epf)
{
    struct pci_epc *epc = epf->epc;
    struct pci_epf_bar *epf_bar = &epf->bar[0];
    int ret;
    
    /* 设置配置空间 */
    ret = pci_epc_write_header(epc, epf->func_no, &epf->header);
    
    /* 配置BAR */
    ret = pci_epc_set_bar(epc, epf->func_no, epf_bar);
    
    /* 映射地址 */
    epf->epc_mem = pci_epc_mem_alloc_addr(epc, &epf->phys_addr, 
                                           epf_bar->size);
    
    return 0;
}

static struct pci_epf_ops my_epf_ops = {
    .bind = my_epf_bind,
    .unbind = my_epf_unbind,
    .linkup = my_epf_linkup,
};

static struct pci_epf_driver my_epf_driver = {
    .driver.name = "my-pcie-epf",
    .ops = &my_epf_ops,
};