Openstack Nova 源码分析 — 使用 VCDriver 创建 VMware Instance

目录

• 目录
• 前言
• 流程图
• nova-compute vCenter

前言

在上一篇Openstack Nova 源码分析 — Create instances (nova-conductor阶段)中，记录了 nova-api 接收到创建虚拟机的请求后，在 nova-conductor 中的执行流程。最终 nova-comductor 通过调用 nova-compute 的 RPC 接口函数 compute_rpcapi.build_and_run_instance() 将创建虚拟机的请求，通过 Queue 传递给 nova-compute 。本篇继续往下看看 Openstack 创建一个虚拟机时，程序流在 nova-compute 和 Virt Driver 阶段的执行过程。而且本篇使用 VCDirver 作为Virt Driver Type 。

NOTE:下面的代码块大多为节选。

流程图

nova-compute & vCenter

在之前的文章VMware 接入 Openstack — 使用 Openstack 创建 vCenter 虚拟机已经记录过如何将 VMware 接入 Openstack ，本质是通过 nova-compute 和 vCenter 中的 Cluster 一一对应来进行管理。

紧接着，当 nova-conductor 调用了 nova-compute 的 RPC 接口后，相应接口的具体操作函数在 nova.compute.manager 中实现。

# nova/compute/manager.py

1841     def build_and_run_instance(self, context, instance, image, request_spec,
   1                      filter_properties, admin_password=None,
   2                      injected_files=None, requested_networks=None,
   3                      security_groups=None, block_device_mapping=None,
   4                      node=None, limits=None):
   5
   6         @utils.synchronized(instance.uuid)
   7         def _locked_do_build_and_run_instance(*args, **kwargs):
   8             # NOTE(danms): We grab the semaphore with the instance uuid
   9             # locked because we could wait in line to build this instance
  10             # for a while and we want to make sure that nothing else tries
  11             # to do anything with this instance while we wait.
  12             with self._build_semaphore:
  13                 self._do_build_and_run_instance(*args, **kwargs)
  14
  15         # NOTE(danms): We spawn here to return the RPC worker thread back to
  16         # the pool. Since what follows could take a really long time, we don't
  17         # want to tie up RPC workers.
  18         utils.spawn_n(_locked_do_build_and_run_instance,
  19                       context, instance, image, request_spec,
  20                       filter_properties, admin_password, injected_files,
  21                       requested_networks, security_groups,
  22                       block_device_mapping, node, limits)

在上述的 nova.compute.manager.ComputeManager:build_and_run_instance()中调用了_do_build_and_run_instance() 函数。

# nova/compute/manager.py

1870     def _do_build_and_run_instance(self, context, instance, image,
   1             request_spec, filter_properties, admin_password, injected_files,
   2             requested_networks, security_groups, block_device_mapping,
   3             node=None, limits=None):
    ...
1901         try:
   1             self._build_and_run_instance(context, instance, image,
   2                     decoded_files, admin_password, requested_networks,
   3                     security_groups, block_device_mapping, node, limits,
   4                     filter_properties)
   5             return build_results.ACTIVE

再跳转到 _build_and_run_instance()，这个函数非常重要。

# nova/compute/manager.py

93   from nova.virt import driver

667  class ComputeManager(manager.Manager): 

677      def __init__(self, compute_driver=None, *args, **kwargs): 

679          self.virtapi = ComputeVirtAPI(self)

718          self.driver = driver.load_compute_driver(self.virtapi, compute_driver)
             # 加载 Driver, 过程如下：
             # nova.virt.driver:load_compute_driver()
             #     ==> oslo_utils.importutils:import_object_ns()
             #     ==> nova.utils:check_isinstance()
             # Return： 一个由 (compute_driver = CONF.compute_driver) 决定的 ComputeDriver 实例化对象 driver

1979     def _build_and_run_instance(self, context, instance, image, injected_files,
   1             admin_password, requested_networks, security_groups,
   2             block_device_mapping, node, limits, filter_properties):

########## Get image_id
1983         image_name = image.get('name')
             # ==> nova.image.__init__
             #     ==> nova.image.api.API:get()
             #         ==> nova.image.api.API:_get_session_and_image_id()
             #             ==> nova.image.glance:get_remote_image_service()
             #                 ==> nova.image.glance.GlanceImageService:show()
             #                     ==> nova.image.glance._tnslate_from_glanceranslate_from_glance
             # Return:ClanceImageService.show() 返回一个包含了 Image_Mete 信息的 Dict['name'] == uri_or_id

   1         self._notify_about_instance_usage(context, instance, 'create.start',
   2                 extra_usage_info={'image_name': image_name})
             # 向外部发出一个 start to create instance 的通知

########## 应用 claim 机制 Update table:compute_nodes
1986         try:
   1             rt = self._get_resource_tracker(node)
   2             with rt.instance_claim(context, instance, limits):

                     # 通过传参来创建资源Dict (EG. network_info/block_device_info)
1994                 with self._build_resources(context, instance,
   1                         requested_networks, security_groups, image,
   2                         block_device_mapping) as resources:

                         # Change the Instance status
   3                     instance.vm_state = vm_states.BUILDING
   4                     instance.task_state = task_states.SPAWNING
   2                     # NOTE(JoshNang) This also saves the changes to the
   3                     # instance from _allocate_network_async, as they aren't
   4                     # saved in that function to prevent races.
   5                     instance.save(expected_task_state=
   6                             task_states.BLOCK_DEVICE_MAPPING)

                         # Block Storage(cinder)
2004                     block_device_info = resources['block_device_info']

                         # Network
2005                     network_info = resources['network_info'] 

########## Create Instance
                         # 由 nova.virt.driver.ComputeDriver 实例化对象 driver 调用 spawn（） 函数来进行虚拟机的创建
2006                     self.driver.spawn(context, instance, image,
   1                                       injected_files, admin_password,
   2                                       network_info=network_info,
   3                                       block_device_info=block_device_info)

NOTE：task states 执行 Task 时的过渡状态，在状态发生改变时会向外部发出通知。前提是配置项notify_on_state_change要配置为vm_state或vm_and_task_state(有待验证)。

因为我希望使用 VCDriver 驱动类型， 所以在 Nova 的配置文件 /etc/nova.conf 中设置选项compute_driver=vmwareapi.VMwareVCDriver。

这样的话，通过执行代码 compute_driver = CONF.compute_driver 就可以获得 VCDriver 的 driver 对象。当我们使用这个 driver 对象来创建虚拟机时，程序流会进入到nova/virt/vmwareapi/ 再通过调用 VMware 提供的 API 接口 (nova.virt.vmwareapi.driver.VMwareVCDriver:spawn())来最终实现虚拟机的创建 。

# nova/virt/vmwareapi/driver.py

46  from nova.virt.vmwareapi import vmops 

125 class VMwareVCDriver(driver.ComputeDriver):
1     """The VC host connection object."""

148     def __init__(self, virtapi, scheme="https"): 

166         self._session = VMwareAPISession(scheme=scheme)                                 

            # 实例化了一个 vmops.VMwareVMOps 对象
186         self._vmops = vmops.VMwareVMOps(self._session,
  1                                         virtapi,
  2                                         self._volumeops,
  3                                         self._cluster_ref,
  4                                         datastore_regex=self._datastore_regex)

401     def spawn(self, context, instance, image_meta, injected_files,
  1               admin_password, network_info=None, block_device_info=None):
  2         """Create VM instance."""

  3         image_meta = objects.ImageMeta.from_dict(image_meta)
            # nova.object.image_meta.ImageMeta:from_dict()
            #     ==> nova.object.image_meta.ImageMetaProps:from_dict()
            # Return： obj = cls() = image_meta.get("properties", {})[0]

            # _vmops 为 nova.virt.vmmwareapi.vmops.VMwareVMOps 的实例化对象
  4         self._vmops.spawn(context, instance, image_meta, injected_files,
  5                           admin_password, network_info, block_device_info)

跳转到 nova.virt.vmwareapi.vmops.VMwareVMOps:spawn()

# nova/virt/vmwareapi/vmops.py

62   from nova.virt.vmwareapi import vm_util 

# 这一个类封装了对 vCenter 的虚拟机的操作函数(EG. _get_base_folder/_extend_virtual_disk/_delete_datastore_file/build_virtual_machine)
149  class VMwareVMOps(object):
   1     """Management class for VM-related tasks."""

152      def __init__(self, session, virtapi, volumeops, cluster=None,
   1                  datastore_regex=None):

677      def spawn(self, context, instance, image_meta, injected_files,
   1               admin_password, network_info, block_device_info=None):
   2
   3         client_factory = self._session.vim.client.factory
   4         image_info = images.VMwareImage.from_image(instance.image_ref,
   5                                                    image_meta)
             # nova.virt.vmwareapi.images.VMwareImage:from_image()
             # """Returns VMwareImage, the subset of properties the driver uses."""
             # Return vmware image object
             # 在这里类中，实现了一系列关于 Image 属性参数的设定和克隆 。

685          vi = self._get_vm_config_info(instance, image_info,
   1                                       extra_specs)
             # 获取创建虚拟机所需要的参数信息(EG. ds/dc)
             # nova.virt.vmwareapi.vmops._get_vm_config_info()
             # Return： VirtualMachineInstanceConfigInfo()

691          vm_ref = self.build_virtual_machine(instance,
   1                                             image_info,
   2                                             vi.dc_info,
   3                                             vi.datastore,
   4                                             network_info,
   5                                             extra_specs,
   6                                             metadata)

继续跳转到 nova.virt.vmwareapi.vmopss.VMwareVMOps:build_virtual_machine() 。

277      def build_virtual_machine(self, instance, image_info,
   1                               dc_info, datastore, network_info, extra_specs,
   2                               metadata):
   3         vif_infos = vmwarevif.get_vif_info(self._session,
   4                                            self._cluster,
   5                                            utils.is_neutron(),
   6                                            image_info.vif_model,
   7                                            network_info)
             # 获取 Virtual Interface info
             # nova.virt.vmwareapi.vif:get_vif_info()
             #     ==> nova.virt.vmwareapi.vif:get_vif_dict()
             #Return: vif_info 包含了(netowork_name/mac_address/network_ref/iface_id/cif_model)等信息
   8
   9         if extra_specs.storage_policy:
  10             profile_spec = vm_util.get_storage_profile_spec(
  11                 self._session, extra_specs.storage_policy)
  12         else:
  13             profile_spec = None
  14         # Get the create vm config spec
  15         client_factory = self._session.vim.client.factory
  16         config_spec = vm_util.get_vm_create_spec(client_factory,
  17                                                  instance,
  18                                                  datastore.name,
  19                                                  vif_infos,
  20                                                  extra_specs,
  21                                                  image_info.os_type,
  22                                                  profile_spec=profile_spec,
  23                                                  metadata=metadata)
             # 获取虚拟机的 config_spec (profile/cpu/memory/Neutron 等)
             # nova.virt.vmwareapi.vmm_util.get_vm_create_spec（）

  24         # Create the VM
             # create_vm() 会返回 Task 的 Result ，并附值给 vm_ref。 vm_ref 被用于创建虚拟机后的一切列数据更新 。
  25         vm_ref = vm_util.create_vm(self._session, instance, dc_info.vmFolder,
  26                                    config_spec, self._root_resource_pool)
  27         return vm_ref

在 VMwareVMOps:build_virtual_machine() 函数中又调用了 nova.virt.vmwareapi.vm_util:get_vm_create_spec() 函数来获取创建虚拟机所需要的参数信息。同时也调用了nova.virt.vmwareapi.vm_util:create_vm() 来 Create the VM 。所以我们先转到 nova.virt.vmwareapi.vm_util Module 去看看具体的 Return 。

# nova/virt/vmwareapi/vm_util.py
"""
The VMware API VM utility module to build SOAP object specs.
"""

1287 def create_vm(session, instance, vm_folder, config_spec, res_pool_ref):
   1     """Create VM on ESX host."""
   2     LOG.debug("Creating VM on the ESX host", instance=instance)

         # session 是 nova.virt.vmwareapi.driver.VMwareAPISession 的实例化对象
   3     vm_create_task = session._call_method(
   4         session.vim,
   5         "CreateVM_Task", vm_folder,
   6         config=config_spec, pool=res_pool_ref)
   7     try:
   8         task_info = session._wait_for_task(vm_create_task)
   9     except vexc.VMwareDriverException:
  10         # An invalid guestId will result in an error with no specific fault
  11         # type and the generic error 'A specified parameter was not correct'.
  12         # As guestId is user-editable, we try to help the user out with some
  13         # additional information if we notice that guestId isn't in our list of
  14         # known-good values.
  15         # We don't check this in advance or do anything more than warn because
  16         # we can't guarantee that our list of known-good guestIds is complete.
  17         # Consequently, a value which we don't recognise may in fact be valid.
  18         with excutils.save_and_reraise_exception():
  19             if config_spec.guestId not in constants.VALID_OS_TYPES:
  20                 LOG.warning(_LW('vmware_ostype from image is not recognised: '
  21                                 '\'%(ostype)s\'. An invalid os type may be '
  22                                 'one cause of this instance creation failure'),
  23                          {'ostype': config_spec.guestId})
  24     LOG.debug("Created VM on the ESX host", instance=instance)
  25     return task_info.result
         # 这个函数的最终返回 Task 的执行结果

到此为止关于虚拟机的创建就完成了，需要注意的是：在我们创建完虚拟机之后其实还有许多的事情是需要做的，EG. 更新数据库/开启虚拟机

所以，在nova.virt.vmwareapi.vm_util:create_vm() 中得到了创建虚拟机的 Task Result task_info.result 之后，需要使用这一 Return 来进行一系列的操作。当然，这一系列的操作会在 VM 相关任务管理类： nova.virt.vmwareapi.wmops.VMwareVMOps 中实现。

# nova/virt/vmmwareapi/vmops.py
# 下面的操作，都是在成功创建了虚拟机并接收 vm_ref (vm_ref = vm_util.create_vm())返回值之后执行。
   2         # Cache the vm_ref. This saves a remote call to the VC. This uses the
   1         # instance uuid.
701          vm_util.vm_ref_cache_update(instance.uuid, vm_ref) 

   2         # Set the machine.id parameter of the instance to inject
   1         # the NIC configuration inside the VM
708          if CONF.flat_injected:
   1             self._set_machine_id(client_factory, instance, network_info,
   2                                 vm_ref=vm_ref)

   2         # Set the vnc configuration of the instance, vnc port starts from 5900
   1         if CONF.vnc.enabled:
714              self._get_and_set_vnc_config(client_factory, instance, vm_ref)     

   6         if instance.image_ref:
   5             self._imagecache.enlist_image(
   4                     image_info.image_id, vi.datastore, vi.dc_info.ref)
   3             self._fetch_image_if_missing(context, vi)
   2
   1             if image_info.is_iso:
727                  self._use_iso_image(vm_ref, vi)
   1             elif image_info.linked_clone:
   2                 self._use_disk_image_as_linked_clone(vm_ref, vi)
   3             else:
   4                 self._use_disk_image_as_full_clone(vm_ref, vi)

   1         # Create ephemeral disks
758          self._create_ephemeral(block_device_info, instance, vm_ref,
   1                                vi.dc_info, vi.datastore, instance.uuid,
   2                                vi.ii.adapter_type)
   3         self._create_swap(block_device_info, instance, vm_ref, vi.dc_info,
   4                           vi.datastore, instance.uuid, vi.ii.adapter_type)

764          if configdrive.required_by(instance):
   1             self._configure_config_drive(
   2                     instance, vm_ref, vi.dc_info, vi.datastore,
   3                     injected_files, admin_password, network_info)

             # 将虚拟机起电
769          vm_util.power_on_instance(self._session, instance, vm_ref=vm_ref)