PCI 设备详解三

上篇文章已经分析了探测PCI总线的部分代码，碍于篇幅，这里另启一篇。重点分析下pci_scan_root_bus函数

2016-10-24

---

pci_scan_root_bus函数

struct pci_bus *pci_scan_root_bus(struct device *parent, int bus,
        struct pci_ops *ops, void *sysdata, struct list_head *resources)
{
    struct pci_host_bridge_window *window;
    bool found = false;
    struct pci_bus *b;
    int max;
    /*寻找bus的资源*/
    list_for_each_entry(window, resources, list)
        if (window->res->flags & IORESOURCE_BUS) {
            found = true;
            break;
        }
    /*创建bus对应的结构*/
    b = pci_create_root_bus(parent, bus, ops, sysdata, resources);
    if (!b)
        return NULL;
 
    if (!found) {
        dev_info(&b->dev,
         "No busn resource found for root bus, will use [bus %02x-ff]\n",
            bus);
        pci_bus_insert_busn_res(b, bus, );
    }
    /*遍历子总线*/
    max = pci_scan_child_bus(b);
 
    if (!found)
        pci_bus_update_busn_res_end(b, max);
 
    pci_bus_add_devices(b);
    return b;
}

这里首先寻找bus总线号资源，前面在x86_pci_root_bus_resources函数中已经分配了，所以这里理论上是已经分配好了，不过还是验证下！！内核中总是精益求精。接着调用了pci_create_root_bus函数创建了对应的bus结构，然后调用pci_scan_child_bus函数遍历该总线下所有的子总线。最后就调用pci_bus_add_devices添加设备。总体上就是这么几步，但是要弄清楚，还真是不小的工作量。我们一步步来：

1、pci_create_root_bus函数

 struct pci_bus *pci_create_root_bus(struct device *parent, int bus,
         struct pci_ops *ops, void *sysdata, struct list_head *resources)
 {
     int error;
     struct pci_host_bridge *bridge;
     struct pci_bus *b, *b2;
     struct pci_host_bridge_window *window, *n;
     struct resource *res;
     resource_size_t offset;
     char bus_addr[];
     char *fmt;
     /*创建一个pci_bus结构*/
     b = pci_alloc_bus();
     if (!b)
         return NULL;
     /*基本的初始化*/
     b->sysdata = sysdata;
     b->ops = ops;
     /*0号总线的总线号正是该条根总线下的总线号资源的起始号*/
     b->number = b->busn_res.start = bus;
     /**/
     b2 = pci_find_bus(pci_domain_nr(b), bus);
     if (b2) {
         /* If we already got to this bus through a different bridge, ignore it */
         dev_dbg(&b2->dev, "bus already known\n");
         goto err_out;
     }
 
     bridge = pci_alloc_host_bridge(b);
     if (!bridge)
         goto err_out;
 
     bridge->dev.parent = parent;
     bridge->dev.release = pci_release_host_bridge_dev;
     dev_set_name(&bridge->dev, "pci%04x:%02x", pci_domain_nr(b), bus);
     error = pcibios_root_bridge_prepare(bridge);
     if (error) {
         kfree(bridge);
         goto err_out;
     }
     /*桥也是作为一个设备存在*/
     error = device_register(&bridge->dev);
     if (error) {
         put_device(&bridge->dev);
         goto err_out;
     }
     /*建立总线到桥的指向*/
     b->bridge = get_device(&bridge->dev);
     device_enable_async_suspend(b->bridge);
     pci_set_bus_of_node(b);
 
     if (!parent)
         set_dev_node(b->bridge, pcibus_to_node(b));
 
     b->dev.class = &pcibus_class;
     b->dev.parent = b->bridge;
     dev_set_name(&b->dev, "%04x:%02x", pci_domain_nr(b), bus);
     error = device_register(&b->dev);
     if (error)
         goto class_dev_reg_err;
 
     pcibios_add_bus(b);
 
     /* Create legacy_io and legacy_mem files for this bus */
     pci_create_legacy_files(b);
 
     if (parent)
         dev_info(parent, "PCI host bridge to bus %s\n", dev_name(&b->dev));
     else
         printk(KERN_INFO "PCI host bridge to bus %s\n", dev_name(&b->dev));
 
     /* Add initial resources to the bus */
     list_for_each_entry_safe(window, n, resources, list) {
         /*从全局的资源链表摘下，加入到特定桥的windows链表中*/
         list_move_tail(&window->list, &bridge->windows);
 
         res = window->res;
         offset = window->offset;
         /*如果资源是总线号资源*/
         if (res->flags & IORESOURCE_BUS)
             pci_bus_insert_busn_res(b, bus, res->end);
         else
             pci_bus_add_resource(b, res, );
         /*看总线地址到物理地址的偏移*/
         if (offset) {
             if (resource_type(res) == IORESOURCE_IO)
                 fmt = " (bus address [%#06llx-%#06llx])";
             else
                 fmt = " (bus address [%#010llx-%#010llx])";
             snprintf(bus_addr, sizeof(bus_addr), fmt,
                  (unsigned long long) (res->start - offset),
                  (unsigned long long) (res->end - offset));
         } else
             bus_addr[] = '\0';
         dev_info(&b->dev, "root bus resource %pR%s\n", res, bus_addr);
     }
 
     down_write(&pci_bus_sem);
     /*加入根总线链表*/
     list_add_tail(&b->node, &pci_root_buses);
     up_write(&pci_bus_sem);
 
     return b;
 
 class_dev_reg_err:
     put_device(&bridge->dev);
     device_unregister(&bridge->dev);
 err_out:
     kfree(b);
     return NULL;
 }

该函数和之前的相比就略显庞大了。不过也难怪，到了最后的阶段一般都挺复杂。哈哈！这里调用pci_alloc_bus函数分配了一个pci_bus结构，然后做基本的初始化。注意一个就是

 b->number = b->busn_res.start = bus;

总线号资源时预分配好的，且一个总线的总线号就是其对应总线号区间的起始号。

然后调用pci_find_bus检测下本次总线号是否已经存在对应的总线结构，如果存在，则表明有错误，当然一般是不会存在的。

然后调用pci_alloc_host_bridge函数分配了一个pci_host_bridge结构作为主桥。然后在主桥和总线之间建立关系。因为桥也是一种设备，所以需要注册。

所以一直到这里，代码虽然繁琐却不难理解。

到下面需要给总线分配资源了，之前我们是初始化了资源，并没有在总线和资源之间建立关系，需要分清楚。看下面的list_for_each_entry_safe

这里实现的功能就是把window从resources链表中取下，然后加入到刚才创建host-bridge的window链表中，这样就算把资源分配给了主桥，回想下前面提到桥设备的窗口就可以明白了。只是这里的意思貌似只考虑了一个主桥，虽然大部分都是一个主桥。然后把资源一个个资源都和总线相关联。这样总线的资源是有了。

最后调用list_add_tail把总线加入到全局的根总线链表。

下面看第二个函数pci_scan_child_bus，总线的递归遍历就是在这里做的。

 unsigned int pci_scan_child_bus(struct pci_bus *bus)
 {
     unsigned int devfn, pass, max = bus->busn_res.start;
     struct pci_dev *dev;
 
     dev_dbg(&bus->dev, "scanning bus\n");
 
     /* Go find them, Rover! 遍历一条总线上的所有子总线，一条总线有32个接口，一个接口有8个子功能，所以这里只能以8递增*/
     for (devfn = ; devfn < 0x100; devfn += )
         /*在遍历每一个接口，这里一个接口最多有八个function*/
     /*在这里，就把总线上的每一个设备都探测过了并加入到了bus对应的设备链表中，后面遍历还要用到*/
         pci_scan_slot(bus, devfn);
 
     /* Reserve buses for SR-IOV capability. 加上预留的总线号的数量*/
     max += pci_iov_bus_range(bus);
 
     /*
      * After performing arch-dependent fixup of the bus, look behind
      * all PCI-to-PCI bridges on this bus.
      */
      /*查找PCI桥*/
     if (!bus->is_added) {
         dev_dbg(&bus->dev, "fixups for bus\n");
         pcibios_fixup_bus(bus);
         bus->is_added = ;
     }
     /*据说是需要调用两次pci_scan_bridge，第一次配置，第二次遍历*/
     for (pass=; pass < ; pass++)
         list_for_each_entry(dev, &bus->devices, bus_list) {
             if (dev->hdr_type == PCI_HEADER_TYPE_BRIDGE ||
                 dev->hdr_type == PCI_HEADER_TYPE_CARDBUS)
                 /*遍历PCI桥*/
                 max = pci_scan_bridge(bus, dev, max, pass);
         }
 
     /*
      * We've scanned the bus and so we know all about what's on
      * the other side of any bridges that may be on this bus plus
      * any devices.
      *
      * Return how far we've got finding sub-buses.
      */
     dev_dbg(&bus->dev, "bus scan returning with max=%02x\n", max);
     return max;
 }

这里做的工作也不难理解，先注意有个max变量，初始值是当前总线的总线号，表示已经探测的总线的数量，后续会用到。

一条总线上有32个插槽，而每一个插槽都可以包含八个功能即逻辑设备，所以这里以8递进。在循环中每次调用一下pci_scan_slot函数探测下具体的插槽。

 int pci_scan_slot(struct pci_bus *bus, int devfn)
 {
     unsigned fn, nr = ;
     struct pci_dev *dev;
 
     if (only_one_child(bus) && (devfn > ))
         return ; /* Already scanned the entire slot */
     /*遍历了第一个功能号，即fn=0*/
     dev = pci_scan_single_device(bus, devfn);
     if (!dev)
         return ;
     if (!dev->is_added)
         nr++;
     /*fn=1开始，遍历其他的功能*/
     for (fn = next_fn(bus, dev, ); fn > ; fn = next_fn(bus, dev, fn)) {
         dev = pci_scan_single_device(bus, devfn + fn);
         if (dev) {
             /**/
             if (!dev->is_added)
                 nr++;
             /*如果找到第二个设备就说明这是个多功能的设备*/
             dev->multifunction = ;
         }
     }
 
     /* only one slot has pcie device */
     if (bus->self && nr)
         pcie_aspm_init_link_state(bus->self);
 
     return nr;
 }

最先开始仍然是判断，如果这里该插槽只有一个逻辑设备即不是多功能的，且devfn=0，那么就表示在寻找一个不存在的设备，直接return 0，否则就调用pci_scan_single_device函数探测该插槽各个逻辑设备。接着调动了pci_scan_single_device函数，该函数检查下对应设备号的设备是否已经存在于总线的设备链表中，不存在才会往下调用pci_scan_device函数探测。

 static struct pci_dev *pci_scan_device(struct pci_bus *bus, int devfn)
 {
     struct pci_dev *dev;
     u32 l;
     /*获取设备厂商*/
     if (!pci_bus_read_dev_vendor_id(bus, devfn, &l, *))
         return NULL;
     /*分配一个dev结构*/
     dev = pci_alloc_dev(bus);
     if (!dev)
         return NULL;
 
     dev->devfn = devfn;
     dev->vendor = l & 0xffff;
     dev->device = (l >> ) & 0xffff;
 
     pci_set_of_node(dev);
     /*初始化设备*/
     if (pci_setup_device(dev)) {
         pci_bus_put(dev->bus);
         kfree(dev);
         return NULL;
     }
 
     return dev;
 }

这里就要做实质性的工作了，创建了一个设备结构并设置相关的信息如设备号，厂商等，然后调用pci_setup_device函数对设备进行全面的初始化，比较重要是地址空间的映射。这里先不说这些，后面再提。最后会调用pci_device_add函数把设备注册进系统，主要还是在设备和总线之间建立联系。回到pci_scan_child_bus函数中，经过这一步就把当前总线上的各个逻辑设备遍历了一遍，也就是都凡是存在的逻辑设备都有了对应的结构，且都存在于总线的设备链表中。然后开始组个检测这些设备，其目的在于寻找PCI-PCI 桥的存在也即1型设备。这里如果找到一个桥设备就会调用pci_scan_bridge函数遍历桥设备：

 int pci_scan_bridge(struct pci_bus *bus, struct pci_dev *dev, int max, int pass)
 {
     struct pci_bus *child;
     int is_cardbus = (dev->hdr_type == PCI_HEADER_TYPE_CARDBUS);
     u32 buses, i, j = ;
     u16 bctl;
     u8 primary, secondary, subordinate;
     int broken = ;
     /*这里是先读设备配置空间的总线号*/
     pci_read_config_dword(dev, PCI_PRIMARY_BUS, &buses);
     primary = buses & 0xFF;//父总线号
     secondary = (buses >> ) & 0xFF;//子总线号
     subordinate = (buses >> ) & 0xFF;//桥下最大的总线号
 
     dev_dbg(&dev->dev, "scanning [bus %02x-%02x] behind bridge, pass %d\n",
         secondary, subordinate, pass);
     /*!primary为真两种情况，1为空 2为0(代表根总线)，加上后面的&&才表示为空*/
     if (!primary && (primary != bus->number) && secondary && subordinate) {
         /*Primary bus硬件实现为0，当是root总线时，正好总线号也是0就不需要修改，而其他子总线就需要重新设置*/
         dev_warn(&dev->dev, "Primary bus is hard wired to 0\n");
         /*手动设置*/
         primary = bus->number;
     }
 
     /* Check if setup is sensible at all 监测配置是否合法*/
     if (!pass &&
         (primary != bus->number || secondary <= bus->number ||
          secondary > subordinate)) {
         dev_info(&dev->dev, "bridge configuration invalid ([bus %02x-%02x]), reconfiguring\n",
              secondary, subordinate);
         broken = ;
     }
 
     /* Disable MasterAbortMode during probing to avoid reporting
        of bus errors (in some architectures) */
     pci_read_config_word(dev, PCI_BRIDGE_CONTROL, &bctl);
     pci_write_config_word(dev, PCI_BRIDGE_CONTROL,
                   bctl & ~PCI_BRIDGE_CTL_MASTER_ABORT);
 
     if ((secondary || subordinate) && !pcibios_assign_all_busses() &&
         !is_cardbus && !broken) {
         unsigned int cmax;
         /*
          * Bus already configured by firmware, process it in the first
          * pass and just note the configuration.
          */
         if (pass)
             goto out;
 
         /*
          * If we already got to this bus through a different bridge,
          * don't re-add it. This can happen with the i450NX chipset.
          *
          * However, we continue to descend down the hierarchy and
          * scan remaining child buses.
          */
          /*得到子总线结构*/
         child = pci_find_bus(pci_domain_nr(bus), secondary);
         if (!child) {
             child = pci_add_new_bus(bus, dev, secondary);
             if (!child)
                 goto out;
             /*设置子总线的primary指针*/
             child->primary = primary;
             /*给子总线也分配总线号资源*/
             pci_bus_insert_busn_res(child, secondary, subordinate);
             child->bridge_ctl = bctl;
         }
         /*递归遍历子总线*/
         cmax = pci_scan_child_bus(child);
         if (cmax > max)
             max = cmax;
         if (child->busn_res.end > max)
             max = child->busn_res.end;
     } else {
         /*
          * We need to assign a number to this bus which we always
          * do in the second pass.
          */
         if (!pass) {
             if (pcibios_assign_all_busses() || broken)
                 /* Temporarily disable forwarding of the
                    configuration cycles on all bridges in
                    this bus segment to avoid possible
                    conflicts in the second pass between two
                    bridges programmed with overlapping
                    bus ranges. */
                 pci_write_config_dword(dev, PCI_PRIMARY_BUS,
                                buses & ~0xffffff);
             goto out;
         }
 
         /* Clear errors */
         pci_write_config_word(dev, PCI_STATUS, 0xffff);
 
         /* Prevent assigning a bus number that already exists.
          * This can happen when a bridge is hot-plugged, so in
          * this case we only re-scan this bus. */
         child = pci_find_bus(pci_domain_nr(bus), max+);
         if (!child) {
             child = pci_add_new_bus(bus, dev, ++max);
             if (!child)
                 goto out;
             pci_bus_insert_busn_res(child, max, 0xff);
         }
         buses = (buses & 0xff000000)
               | ((unsigned int)(child->primary)     <<  )
               | ((unsigned int)(child->busn_res.start)   <<  )
               | ((unsigned int)(child->busn_res.end) << );
 
         /*
          * yenta.c forces a secondary latency timer of 176.
          * Copy that behaviour here.
          */
         if (is_cardbus) {
             buses &= ~0xff000000;
             buses |= CARDBUS_LATENCY_TIMER << ;
         }
 
         /*
          * We need to blast all three values with a single write.
          */
         pci_write_config_dword(dev, PCI_PRIMARY_BUS, buses);
 
         if (!is_cardbus) {
             child->bridge_ctl = bctl;
             /*
              * Adjust subordinate busnr in parent buses.
              * We do this before scanning for children because
              * some devices may not be detected if the bios
              * was lazy.
              */
              /*修正父总线的总线号资源范围*/
             pci_fixup_parent_subordinate_busnr(child, max);
             /* Now we can scan all subordinate buses... */
             max = pci_scan_child_bus(child);
             /*
              * now fix it up again since we have found
              * the real value of max.
              */
             pci_fixup_parent_subordinate_busnr(child, max);
         } else {
             /*
              * For CardBus bridges, we leave 4 bus numbers
              * as cards with a PCI-to-PCI bridge can be
              * inserted later.
              */
             for (i=; i<CARDBUS_RESERVE_BUSNR; i++) {
                 struct pci_bus *parent = bus;
                 if (pci_find_bus(pci_domain_nr(bus),
                             max+i+))
                     break;
                 while (parent->parent) {
                     if ((!pcibios_assign_all_busses()) &&
                         (parent->busn_res.end > max) &&
                         (parent->busn_res.end <= max+i)) {
                         j = ;
                     }
                     parent = parent->parent;
                 }
                 if (j) {
                     /*
                      * Often, there are two cardbus bridges
                      * -- try to leave one valid bus number
                      * for each one.
                      */
                     i /= ;
                     break;
                 }
             }
             max += i;
             pci_fixup_parent_subordinate_busnr(child, max);
         }
         /*
          * Set the subordinate bus number to its real value.
          */
         pci_bus_update_busn_res_end(child, max);
         pci_write_config_byte(dev, PCI_SUBORDINATE_BUS, max);
     }
 
     sprintf(child->name,
         (is_cardbus ? "PCI CardBus %04x:%02x" : "PCI Bus %04x:%02x"),
         pci_domain_nr(bus), child->number);
 
     /* Has only triggered on CardBus, fixup is in yenta_socket */
     while (bus->parent) {
         if ((child->busn_res.end > bus->busn_res.end) ||
             (child->number > bus->busn_res.end) ||
             (child->number < bus->number) ||
             (child->busn_res.end < bus->number)) {
             dev_info(&child->dev, "%pR %s "
                 "hidden behind%s bridge %s %pR\n",
                 &child->busn_res,
                 (bus->number > child->busn_res.end &&
                  bus->busn_res.end < child->number) ?
                     "wholly" : "partially",
                 bus->self->transparent ? " transparent" : "",
                 dev_name(&bus->dev),
                 &bus->busn_res);
         }
         bus = bus->parent;
     }
 
 out:
     pci_write_config_word(dev, PCI_BRIDGE_CONTROL, bctl);
 
     return max;
 }

该函数通过递归的方式完成了所有总线以及设备的遍历。每一递归都执行两次该函数，第一次探测是否被BIOS处理，第二次才做真正的探测工作。

首先是先读取桥设备的配置空间，获得桥设备的primary bus,secondary bus,subordinate bus号，然后进行判断，如果secondary bus和subordinate bus均不为0则说明配置有效，因为初始primary bus号被硬件初始化为0，所以这里如果传递进来的bus number不是0，就需要重新设置。

然后检查这些号码是否合法。合法情况下就在首次执行pci_scan_bridge函数的时候进行子总线的遍历。可以看到这里同样先是调用pci_find_bus函数查找下secondary号总线是否已经存在，不存在才调用pci_add_new_bus函数new一个新的bus结构，同时在该函数中也对总线的部分变量做了初始化。接着设置总线的primary指针。随后需要给总线分配总线号资源了。根据已有的配置，这里secondary是子总线的号，而subordinate就是总线下最大的总线号，所以这正是总线的总线号区间。然后继续调用pci_scan_child_bus函数继续遍历当前子总线。就这么层一层的递归下去。知道最后没有桥了，就从pci_scan_child_bus函数返回探测到的总线的数量即max.而如果配置空间没有被配置，那么就需要重新配置，这里首次执行pci_scan_bridge函数就只是把配置空间总线号区域清零。到了第二次，大题上根前面类似，不过这里因为没有secondary 号，所以只能按照max+1来寻找或者创建子总线结构，同时对于子总线的总线区间设置成0xff即255最大值。然后写入到桥配置空间中。这个时候已经探测了一个新的总线，那么需要对父总线的总线号区间进行更新，然后执行pci_scan_child_bus函数探测当前子总线的其他总线，在递归返回的时候，需要再次执行更新。并且需要把总线的总线号资源设置成正确的区间。因为开始分配的时候设置默认总线号区间最大为255.

整个递归流程完毕，就知道了一共存在多少总线，且总线上的设备都已经正确配置并都已经加入到了设备链表中。

总结：

本次分析可谓是困难重重，对于很多大牛来说，这或许根本不是事，但是笔者平时的研究没哟涉及到PCI设备这一层面，仅仅是为了分析qemu中的virtIO才着手分析PCI设备。其中可能不乏错误之处，还望老师们看到多多指正。笔者也正是发现只记录不分享，久而久之就越发懒散，好的东西信手沾来虽然容易，然是后续基本不会再看。而写下来给别人分享就不同了，因为担心写错，好多模糊的地方自己需要再三斟酌，同时也是对自己基础的强化，利人利己！