OpenStack 虚拟机启动流程 UML 分析（内含 UML 源码）

目录

文章目录

• 目录
• 前言
• API 请求
• Nova API 阶段
• Nova Conductor 阶段
• Nova Scheduler 阶段
• Nova Compute 阶段（计算节点资源分配部分）
• Nova Compute 阶段（NUMA、CPU 资源分配部分）
• Nova Compute 阶段（虚拟机资源构建部分）

前言

上图可见 虚拟机启动流程 之于 OpenStack 的含义，本文秉承一图抵前言的原则，通过 UML 图展现虚拟机启动的详细流程，并不断扩充，这是一篇持续更新的文章。

API 请求

一切缘起于一个请求：POST /servers

https://developer.openstack.org/api-ref/compute/#create-server

Creates a server.

The progress of this operation depends on the location of the requested image, network I/O, host load, selected flavor, and other factors.

To check the progress of the request, make a GET /servers/{id} request. This call returns a progress attribute, which is a percentage value from 0 to 100.

The Location header returns the full URL to the newly created server and is available as a self and bookmark link in the server representation.

When you create a server, the response shows only the server ID, its links, and the admin password. You can get additional attributes through subsequent GET requests on the server.

Include the block_device_mapping_v2 parameter in the create request body to boot a server from a volume.

Include the key_name parameter in the create request body to add a keypair to the server when you create it. To create a keypair, make a create keypair request.

Nova API 阶段

@startuml
actor User

entity "api/openstack/compute/server.py"
entity "compute/api.py"
entity "conductor/api.py"
entity "conductor/rpcapi.py"
entity "conductor/manager.py"

autonumber

User -> "api/openstack/compute/server.py": POST /servers

activate "api/openstack/compute/server.py"
"api/openstack/compute/server.py" -> "api/openstack/compute/server.py": ServersController.create
note left: 组装用于创建虚拟机的参数 create_kwargs

"api/openstack/compute/server.py" -> "compute/api.py": API.create
activate "compute/api.py"
"compute/api.py" -> "compute/api.py": scheduler_utils.build_filter_properties
note left: 组装用于调度的数据结构 filter_properties

"compute/api.py" -> "compute/api.py": _create_instance
note left: 验证输入的参数，并转换为标准数据结构对象

activate "compute/api.py"
"compute/api.py" -> "compute/api.py": _validate_and_build_base_options
"compute/api.py" -> "compute/api.py": _check_and_transform_bdm
note left: 组装 block_device_mapping 数据结构
"compute/api.py" -> "compute/api.py": _provision_instances
activate "compute/api.py"
"compute/api.py" -> "compute/api.py": objects.RequestSpec.from_components
note left: 实例化 objects.RequestSpec 对象，包含了调度时所考虑的调度因子
"compute/api.py" -> "compute/api.py": objects.Instance
note left: 实例化 objects.Instance 对象
"compute/api.py" -> "compute/api.py": objects.InstanceMapping
note left: 实例化 objects.InstanceMapping 对象
"compute/api.py" -> "compute/api.py": objects.BuildRequest
note left: 实例化 objects.BuildRequest 对象
deactivate "compute/api.py"

"compute/api.py" -> "conductor/api.py": self.compute_task_api.schedule_and_build_instances
note left:

deactivate "compute/api.py"

"conductor/api.py" -> "conductor/rpcapi.py": schedule_and_build_instances
"conductor/rpcapi.py" -> "conductor/manager.py": CAST schedule_and_build_instances
"conductor/rpcapi.py" -> "conductor/api.py": return None
"conductor/api.py" -> "compute/api.py": return None
"compute/api.py" -> "api/openstack/compute/server.py": return instances, reservation_id
"api/openstack/compute/server.py" -> User: RESP HTTP 202
note left: instances, reservation_id
deactivate "api/openstack/compute/server.py"
@enduml

Nova Conductor 阶段

@startuml
entity "conductor/rpcapi.py"
entity "conductor/manager.py"
entity "scheduler/rpcapi.py"
entity "scheduler/manager.py"
entity "compute/rpcapi.py"
entity "compute/manager.py"
autonumber

"conductor/rpcapi.py" -> "conductor/manager.py": CAST schedule_and_build_instances
activate "conductor/manager.py"
"conductor/manager.py" -> "conductor/manager.py": _schedule_instances
activate "conductor/manager.py"
"conductor/manager.py" -> "scheduler/rpcapi.py": self.scheduler_client.select_destinations
"scheduler/rpcapi.py" -> "scheduler/manager.py": CALL select_destinations
note left: 调度创建虚拟机的计算节点
"scheduler/manager.py" -> "scheduler/rpcapi.py": return
"scheduler/rpcapi.py" -> "conductor/manager.py": return host_lists
"conductor/manager.py" -> "conductor/manager.py": 获取主机调度的 cell
note left: Convert host from the scheduler into a cell record
"conductor/manager.py" -> "conductor/manager.py": 创建 Instance 数据库记录
"conductor/manager.py" -> "conductor/manager.py": 创建 BlockDeviceMapping 数据库记录
"conductor/manager.py" -> "conductor/manager.py": 创建 Tag 数据库记录
"conductor/manager.py" -> "compute/rpcapi.py": self.compute_rpcapi.build_and_run_instance
"compute/rpcapi.py" -> "compute/manager.py": CAST build_and_run_instance
"compute/rpcapi.py" -> "conductor/manager.py": return

deactivate "conductor/manager.py"
deactivate "conductor/manager.py"
@enduml

Nova Scheduler 阶段

@startuml
entity "scheduler/rpcapi.py"
entity "scheduler/manager.py"
entity "scheduler/client/report.py"
entity "scheduler/filter_scheduler.py"
entity Placement
entity "scheduler/host_manager.py"

autonumber

"scheduler/rpcapi.py" -> "scheduler/manager.py": CALL select_destinations
activate "scheduler/manager.py"
"scheduler/manager.py" -> "scheduler/manager.py": objects.RequestSpec.from_primitives
note left: 获取 Request Spec Object，用于调度
"scheduler/manager.py" -> "scheduler/manager.py": self.placement_client.get_allocation_candidates
note left: 从 Placement 获取调度候选人
"scheduler/manager.py" -> "scheduler/client/report.py": get_allocation_candidates
"scheduler/client/report.py" -> Placement: GET /allocation_candidates?%s
"scheduler/client/report.py" -> "scheduler/manager.py": return allocation_candidates
note left: alloc_reqs_by_rp_uuid, provider_summaries, allocation_request_version
"scheduler/manager.py" -> "scheduler/filter_scheduler.py": self.driver.select_destinations
activate "scheduler/filter_scheduler.py"
"scheduler/filter_scheduler.py" -> "scheduler/filter_scheduler.py": _schedule
activate "scheduler/filter_scheduler.py"
"scheduler/filter_scheduler.py" -> "scheduler/filter_scheduler.py": _get_all_host_states
note left: 获取调度候选人对应的计算节点
"scheduler/filter_scheduler.py" -> "scheduler/filter_scheduler.py": _get_sorted_hosts
activate "scheduler/filter_scheduler.py"
"scheduler/filter_scheduler.py" -> "scheduler/host_manager.py": self.host_manager.get_filtered_hosts
note left: 进行 Filters 过滤
"scheduler/filter_scheduler.py" -> "scheduler/host_manager.py": self.host_manager.get_weighed_hosts
note left: 进行 Weighed 排序
deactivate "scheduler/filter_scheduler.py"
"scheduler/filter_scheduler.py" -> "scheduler/client/report.py": utils.claim_resources
note left: 扣除 Placement resources 数量
"scheduler/filter_scheduler.py" -> "scheduler/client/report.py": _consume_selected_host
note left: 根据 Request Spec Object 临时扣除资源
"scheduler/filter_scheduler.py" -> "scheduler/host_manager.py": consume_from_request
note left: 临时扣除 Nova Scheduler 维护的资源数据，存储在内存中的 HostState 实例对象
"scheduler/filter_scheduler.py" -> "scheduler/client/report.py": return selections_to_return
"scheduler/client/report.py" -> "scheduler/manager.py": return
deactivate "scheduler/filter_scheduler.py"
deactivate "scheduler/filter_scheduler.py"
"scheduler/manager.py" -> "scheduler/rpcapi.py": return selections
deactivate "scheduler/manager.py"
@enduml

Nova Compute 阶段（计算节点资源分配部分）

@startuml
entity "compute/rpcapi.py"
entity "compute/manager.py"
entity "compute/resource_tracker.py"
entity "compute/claims.py"
autonumber

"compute/rpcapi.py" -> "compute/manager.py": CAST build_and_run_instance

activate "compute/manager.py"
"compute/manager.py" -> "compute/manager.py": build_and_run_instance
activate "compute/manager.py"
"compute/manager.py" -> "compute/manager.py": _do_build_and_run_instance
activate "compute/manager.py"
"compute/manager.py" -> "compute/manager.py": _build_and_run_instance
activate "compute/manager.py"
"compute/manager.py" -> "compute/resource_tracker.py": instance_claim
activate "compute/resource_tracker.py"
"compute/resource_tracker.py" -> "compute/resource_tracker.py": 获取虚拟机所需要的开销
note left
 1. RAM
 2. CPU
 3. Disk
end note
"compute/resource_tracker.py" -> "compute/resource_tracker.py": 获取 ComputeNode Object
"compute/resource_tracker.py" -> "compute/resource_tracker.py": 获取虚拟机所需要的 PCI 设备
"compute/resource_tracker.py" -> "compute/resource_tracker.py": claims.Claim 宣告创建虚拟机所需要的资源
"compute/resource_tracker.py" -> "compute/claims.py": claims.Claim
activate "compute/claims.py"
"compute/claims.py" -> "compute/claims.py": _claim_test
note left
尝试在目的节点上声明虚拟机所需要的资源，判断是否能够满足
NOTE：这里只是宣告资源需求，并非直接使用资源
end note

activate "compute/claims.py"
"compute/claims.py" -> "compute/claims.py": _test_memory
"compute/claims.py" -> "compute/claims.py": _test_disk
"compute/claims.py" -> "compute/claims.py": _test_vcpus
"compute/claims.py" -> "compute/claims.py": _test_numa_topology（展开）
note left: 为具有 NUMA 亲和、CPU 绑定需求的虚拟机构建 GuestOS NUMA Topo 并分配 NUMA、CPU 资源
"compute/claims.py" -> "compute/claims.py": _test_pci
"compute/claims.py" -> "compute/resource_tracker.py": return claims.MoveClaim 实例对象
deactivate "compute/claims.py"
deactivate "compute/claims.py"

"compute/resource_tracker.py" -> "compute/resource_tracker.py": pci_tracker.claim_instance （展开）
note left: 为具有 PCI 需求的虚拟机在 PCI Device Pool 中选取具体的 PCI 设备

"compute/resource_tracker.py" -> "compute/resource_tracker.py": _update_usage_from_instance
note left
Mark resources in-use and update stats
Update usage for a single instance.
end note
"compute/resource_tracker.py" -> "compute/resource_tracker.py": _update
note left: 将上述宣告的资源刷新到数据库记录
activate "compute/resource_tracker.py"
"compute/resource_tracker.py" -> "compute/resource_tracker.py": compute_node.save()
"compute/resource_tracker.py" -> "compute/resource_tracker.py": _update_to_placement
note left: Send resource and inventory changes to placement.
"compute/resource_tracker.py" -> "compute/resource_tracker.py": self.pci_tracker.save()
deactivate "compute/resource_tracker.py"
"compute/resource_tracker.py" -> "compute/manager.py": Claim 实例化对象
deactivate "compute/resource_tracker.py"
deactivate "compute/manager.py"
deactivate "compute/manager.py"
deactivate "compute/manager.py"
"compute/manager.py" -> "compute/manager.py": _build_resources（后续）
deactivate "compute/manager.py"
@enduml

Nova Compute 阶段（NUMA、CPU 资源分配部分）

@startuml
entity "compute/claims.py"
entity "virt/hardware.py"
autonumber

activate "compute/claims.py"
"compute/claims.py" -> "compute/claims.py": _test_numa_topology
note left
为具有 NUMA 亲和、CPU 绑定需求的虚拟机
构建 GuestOS NUMA Topo 并分配 NUMA、CPU 资源
end note
activate "compute/claims.py"
"compute/claims.py" -> "compute/claims.py": 获取 instance requested_topology
"compute/claims.py" -> "compute/claims.py": 获取 host_topology
"compute/claims.py" -> "compute/claims.py": 获取 instance pci_requests
"compute/claims.py" -> "virt/hardware.py": numa_fit_instance_to_host
note left
Fit the instance topology onto the host topology.
将 GuestOS NUMA Topo 根据既定算法嵌入到 Host NUMA Topo
end note
activate "virt/hardware.py"
"virt/hardware.py" -> "virt/hardware.py": 首先准备 NUMA Node 调度因子
note left
以下因子都会被考虑到 NUMA 节点的分配调度上：
  1. emulator_threads_policy: QEMU 模拟器线程策略
  2. network_metadata: 符合网络 NUMA 要求
  3. pci_requests: 与 SR-IOV 网卡同 NUMA
end note

"virt/hardware.py" -> "virt/hardware.py": itertools.permutations(host_cells, len(instance_topology))
note left: * 根据 GuestOS NUMA Node 的数量对 Host NUMA Node 进行全排列组合

"virt/hardware.py" -> "virt/hardware.py": _numa_fit_instance_cell
note left
* Ensure an instance cell can fit onto a host cell
轮询 Host NUMA Node 全排列，
将 GuestOS NUMA Node 分别尝试放置，
直到 GuestOS NUMA Topo 被合理的放置到 Host NUMA Topo 中，
并满足上述 NUMA 调度因子。
end note

activate "virt/hardware.py"
"virt/hardware.py" -> "virt/hardware.py": Fit pagesize
"virt/hardware.py" -> "virt/hardware.py": Fit memory
"virt/hardware.py" -> "virt/hardware.py": Fit CPU amount
"virt/hardware.py" -> "virt/hardware.py": _numa_fit_instance_cell_with_pinning
note left: Fit CPU Pin
activate "virt/hardware.py"
"virt/hardware.py" -> "virt/hardware.py": _pack_instance_onto_cores
note left
Try to pack the instance cell onto cores
Pack an instance onto a set of siblings.
end note
deactivate "virt/hardware.py"

"virt/hardware.py" -> "virt/hardware.py": instance_cell.id = host_cell.id
note left: 将 Host NUMA 映射到 GuestOS NUMA
deactivate "virt/hardware.py"
"virt/hardware.py" -> "compute/claims.py": return objects.InstanceNUMATopology
note left: 返回 GuestOS NUMA Topo 资源实例对象
deactivate "compute/claims.py"
"compute/claims.py" -> "compute/claims.py": Claimed GuestOS NUMA Topo
note left: 声明要使用的 GuestOS NUMA Topo 资源，在整个虚拟机启动流程中有效
deactivate "compute/claims.py"
@enduml

Nova Compute 阶段（虚拟机资源构建部分）

@startuml
entity "compute/manager.py"
entity "virt/libvirt/driver.py"
autonumber

"compute/manager.py" -> "compute/manager.py": _build_and_run_instance
activate "compute/manager.py"
"compute/manager.py" -> "compute/manager.py": 计算节点资源分配部分
"compute/manager.py" -> "compute/manager.py": _build_resources
activate "compute/manager.py"
"compute/manager.py" -> "compute/manager.py": _build_networks_for_instance
note left: 异步构建网络资源
"compute/manager.py" -> "compute/manager.py": _default_block_device_names
note left: 保证所有虚拟机磁盘都设置了名称，e.g. sda/sdb
"compute/manager.py" -> "compute/manager.py": _prep_block_device
note left: 将块设备挂载到计算节点
deactivate "compute/manager.py"

"compute/manager.py" -> "virt/libvirt/driver.py": self.driver.spawn
note left: 孵化一个虚拟机创建任何，并在合适的时机执行它
activate "virt/libvirt/driver.py"
"virt/libvirt/driver.py" -> "virt/libvirt/driver.py": _create_image
note left
 1. Created local Root disk file from glance image file
 2. Created Swap disk file
 3. Created Ephemeral disk file
end note
"virt/libvirt/driver.py" -> "virt/libvirt/driver.py": _ensure_console_log_for_instance
"virt/libvirt/driver.py" -> "virt/libvirt/driver.py": _get_guest_xml
note left: 生成 Libvirt 虚拟机 XML 文件
activate "virt/libvirt/driver.py"
"virt/libvirt/driver.py" -> "virt/libvirt/driver.py": _get_guest_config
note left
 1. guest.virt_type
 2. guest.name
 3. guest.uuid
 4. guest.memory
 5. guest.vcpus
 6. guest.cpuset
 7. guest.cputune
 8. guest.numatune
 9. guest.membacking
 10. ...
end note
activate "virt/libvirt/driver.py"
"virt/libvirt/driver.py" -> "virt/libvirt/driver.py": _get_guest_numa_config
note left: Returns the config objects for the guest NUMA specs.
activate "virt/libvirt/driver.py"
"virt/libvirt/driver.py" -> "virt/libvirt/driver.py": _get_host_numa_topology
note left: 获取 Host NUMA Topo
"virt/libvirt/driver.py" -> "virt/libvirt/driver.py": _get_cpu_numa_config_from_instance
note left: 获取 Guest NUMA Topo from Instance Object
"virt/libvirt/driver.py" -> "virt/libvirt/driver.py": Init GuestOS CPUTune configuration
"virt/libvirt/driver.py" -> "virt/libvirt/driver.py": Init GuestOS NUMATune configuration

"virt/libvirt/driver.py" -> "virt/libvirt/driver.py": Set realtime scheduler for CPUTune
note left
1. _get_cell_pairs: 根据 Claims 的 GuestOS NUMA Topo 和 Host NUMA Topo 再次进行匹配
2. set NUMATune for the cell
3. set CPUTune for the cell
end note
deactivate "virt/libvirt/driver.py"
"virt/libvirt/driver.py" -> "virt/libvirt/driver.py": _get_guest_memory_backing_config
"virt/libvirt/driver.py" -> "virt/libvirt/driver.py": _get_guest_config_meta
"virt/libvirt/driver.py" -> "virt/libvirt/driver.py": _get_guest_idmaps
"virt/libvirt/driver.py" -> "virt/libvirt/driver.py": _get_guest_cpu_config
"virt/libvirt/driver.py" -> "virt/libvirt/driver.py": _get_guest_os_type
"virt/libvirt/driver.py" -> "virt/libvirt/driver.py": _get_guest_storage_config
note left: 配置存储
"virt/libvirt/driver.py" -> "virt/libvirt/driver.py": self.vif_driver.get_config
note left: 配置网络
"virt/libvirt/driver.py" -> "virt/libvirt/driver.py": _create_consoles
note left: 配置 Console
"virt/libvirt/driver.py" -> "virt/libvirt/driver.py": _get_guest_pointer_model
"virt/libvirt/driver.py" -> "virt/libvirt/driver.py": _guest_add_spice_channel
"virt/libvirt/driver.py" -> "virt/libvirt/driver.py": _guest_add_video_device
"virt/libvirt/driver.py" -> "virt/libvirt/driver.py": _set_qemu_guest_agent
"virt/libvirt/driver.py" -> "virt/libvirt/driver.py": _guest_add_pcie_root_ports
note left: 配置 PCIe 设备
"virt/libvirt/driver.py" -> "virt/libvirt/driver.py": _guest_add_pci_devices
note left: 配置 PCI 设备
"virt/libvirt/driver.py" -> "virt/libvirt/driver.py": _guest_add_watchdog_action
"virt/libvirt/driver.py" -> "virt/libvirt/driver.py": _guest_add_memory_balloon
"virt/libvirt/driver.py" -> "virt/libvirt/driver.py": _guest_add_mdevs
note left: 配置 GPU 设备
"virt/libvirt/driver.py" -> "compute/manager.py": return GuestOS XML
deactivate "virt/libvirt/driver.py"
"compute/manager.py" -> "compute/manager.py": _create_domain_and_network
note left
1. 执行一些 firewall 相关的配置
2. 创建 Libvirt Guest Domain 实例
end note

deactivate "virt/libvirt/driver.py"
deactivate "virt/libvirt/driver.py"

deactivate "compute/manager.py"
@enduml