neutron_tempest_plugin.scenario.admin.test_floatingip.FloatingIpTestCasesAdmin.test_two_vms_fips [22.075122s] ... FAILED
https://logserver.rdoproject.org/openstack-periodic-integration-stable3/opendev.org/openstack/tripleo-ci/master/periodic-tripleo-ci-centos-8-ovb-1ctlr_2comp-featureset020-train/5adb21f/logs/undercloud/var/log/tempest/tempest_run.log.txt.gz
Captured traceback:
~~~~~~~~~~~~~~~~~~~
Traceback (most recent call last):
File "/usr/lib/python3.6/site-packages/neutron_tempest_plugin/scenario/admin/test_floatingip.py", line 102, in test_two_vms_fips
server_ssh_clients, fips = self._create_vms(hyper, avail_zone)
File "/usr/lib/python3.6/site-packages/neutron_tempest_plugin/scenario/admin/test_floatingip.py", line 77, in _create_vms
const.SERVER_STATUS_ACTIVE)
File "/usr/lib/python3.6/site-packages/tempest/common/waiters.py", line 76, in wait_for_server_status
server_id=server_id)
tempest.exceptions.BuildErrorException: Server 25ddccbf-706d-41fa-a6a6-f5f9f70c5b2c failed to build and is in ERROR status
Details: {'code': 500, 'created': '2021-03-30T07:29:25Z', 'message': 'Connection to the hypervisor is broken on host', 'details': 'Traceback (most recent call last):\n File "/usr/lib/python3.6/site-packages/nova/compute/manager.py", line 2670, in _build_resources\n yield resources\n File "/usr/lib/python3.6/site-packages/nova/compute/manager.py", line 2444, in _build_and_run_instance\n block_device_info=block_device_info)\n File "/usr/lib/python3.6/site-packages/nova/virt/libvirt/driver.py", line 3557, in spawn\n cleanup_instance_disks=created_disks)\n File "/usr/lib/python3.6/site-packages/nova/virt/libvirt/driver.py", line 6388, in _create_domain_and_network\n guest.resume()\n File "/usr/lib/python3.6/site-packages/nova/virt/libvirt/guest.py", line 195, in resume\n self._domain.resume()\n File "/usr/lib/python3.6/site-packages/eventlet/tpool.py", line 190, in doit\n result = proxy_call(self._autowrap, f, *args, **kwargs)\n File "/usr/lib/python3.6/site-packages/eventlet/tpool.py", line 148, in proxy_call\n rv = execute(f, *args, **kwargs)\n File "/usr/lib/python3.6/site-packages/eventlet/tpool.py", line 129, in execute\n six.reraise(c, e, tb)\n File "/usr/lib/python3.6/site-packages/six.py", line 703, in reraise\n raise value\n File "/usr/lib/python3.6/site-packages/eventlet/tpool.py", line 83, in tworker\n rv = meth(*args, **kwargs)\n File "/usr/lib64/python3.6/site-packages/libvirt.py", line 2180, in resume\n if ret == -1: raise libvirtError (\'virDomainResume() failed\', dom=self)\nlibvirt.libvirtError: internal error: client socket is closed\n\nDuring handling of the above exception, another exception occurred:\n\nTraceback (most recent call last):\n File "/usr/lib/python3.6/site-packages/nova/virt/libvirt/host.py", line 503, in get_connection\n conn = self._get_connection()\n File "/usr/lib/python3.6/site-packages/nova/virt/libvirt/host.py", line 486, in _get_connection\n {\'msg\': ex})\n File "/usr/lib/python3.6/site-packages/oslo_utils/excutils.py", line 220, in __exit__\n self.force_reraise()\n File "/usr/lib/python3.6/site-packages/oslo_utils/excutils.py", line 196, in force_reraise\n six.reraise(self.type_, self.value, self.tb)\n File "/usr/lib/python3.6/site-packages/six.py", line 703, in reraise\n raise value\n File "/usr/lib/python3.6/site-packages/nova/virt/libvirt/host.py", line 475, in _get_connection\n self._wrapped_conn = self._get_new_connection()\n File "/usr/lib/python3.6/site-packages/nova/virt/libvirt/host.py", line 429, in _get_new_connection\n wrapped_conn = self._connect(self._uri, self._read_only)\n File "/usr/lib/python3.6/site-packages/nova/virt/libvirt/host.py", line 285, in _connect\n return self._libvirt_proxy.openAuth(uri, auth, flags)\n File "/usr/lib/python3.6/site-packages/eventlet/tpool.py", line 190, in doit\n result = proxy_call(self._autowrap, f, *args, **kwargs)\n File "/usr/lib/python3.6/site-packages/eventlet/tpool.py", line 148, in proxy_call\n rv = execute(f, *args, **kwargs)\n File "/usr/lib/python3.6/site-packages/eventlet/tpool.py", line 129, in execute\n six.reraise(c, e, tb)\n File "/usr/lib/python3.6/site-packages/six.py", line 703, in reraise\n raise value\n File "/usr/lib/python3.6/site-packages/eventlet/tpool.py", line 83, in tworker\n rv = meth(*args, **kwargs)\n File "/usr/lib64/python3.6/site-packages/libvirt.py", line 104, in openAuth\n if ret is None:raise libvirtError(\'virConnectOpenAuth() failed\')\nlibvirt.libvirtError: Failed to connect socket to \'/var/run/libvirt/libvirt-sock\': Connection refused\n\nDuring handling of the above exception, another exception occurred:\n\nTraceback (most recent call last):\n File "/usr/lib/python3.6/site-packages/nova/compute/manager.py", line 2689, in _build_resources\n try_deallocate_networks=deallocate_networks)\n File "/usr/lib/python3.6/site-packages/nova/compute/manager.py", line 2868, in _shutdown_instance\n requested_networks)\n File "/usr/lib/python3.6/site-packages/oslo_utils/excutils.py", line 220, in __exit__\n self.force_reraise()\n File "/usr/lib/python3.6/site-packages/oslo_utils/excutils.py", line 196, in force_reraise\n six.reraise(self.type_, self.value, self.tb)\n File "/usr/lib/python3.6/site-packages/six.py", line 703, in reraise\n raise value\n File "/usr/lib/python3.6/site-packages/nova/compute/manager.py", line 2855, in _shutdown_instance\n block_device_info)\n File "/usr/lib/python3.6/site-packages/nova/virt/libvirt/driver.py", line 1328, in destroy\n self._destroy(instance)\n File "/usr/lib/python3.6/site-packages/nova/virt/libvirt/driver.py", line 1182, in _destroy\n guest = self._host.get_guest(instance)\n File "/usr/lib/python3.6/site-packages/nova/virt/libvirt/host.py", line 589, in get_guest\n return libvirt_guest.Guest(self._get_domain(instance))\n File "/usr/lib/python3.6/site-packages/nova/virt/libvirt/host.py", line 604, in _get_domain\n conn = self.get_connection()\n File "/usr/lib/python3.6/site-packages/nova/virt/libvirt/host.py", line 515, in get_connection\n raise exception.HypervisorUnavailable()\nnova.exception.HypervisorUnavailable: Connection to the hypervisor is broken on host\n\nDuring handling of the above exception, another exception occurred:\n\nTraceback (most recent call last):\n File "/usr/lib/python3.6/site-packages/nova/compute/manager.py", line 2447, in _build_and_run_instance\n instance=instance)\n File "/usr/lib64/python3.6/contextlib.py", line 99, in __exit__\n self.gen.throw(type, value, traceback)\n File "/usr/lib/python3.6/site-packages/nova/compute/manager.py", line 2697, in _build_resources\n reason=six.text_type(exc))\nnova.exception.BuildAbortException: Build of instance 25ddccbf-706d-41fa-a6a6-f5f9f70c5b2c aborted: internal error: client socket is closed\n\nDuring handling of the above exception, another exception occurred:\n\nTraceback (most recent call last):\n File "/usr/lib/python3.6/site-packages/nova/compute/manager.py", line 2447, in _build_and_run_instance\n instance=instance)\n File "/usr/lib/python3.6/site-packages/nova/compute/claims.py", line 43, in __exit__\n self.abort()\n File "/usr/lib/python3.6/site-packages/nova/compute/claims.py", line 84, in abort\n self.nodename)\n File "/usr/lib/python3.6/site-packages/oslo_concurrency/lockutils.py", line 328, in inner\n return f(*args, **kwargs)\n File "/usr/lib/python3.6/site-packages/nova/compute/resource_tracker.py", line 571, in abort_instance_claim\n self._update(context.elevated(), self.compute_nodes[nodename])\n File "/usr/lib/python3.6/site-packages/nova/compute/resource_tracker.py", line 1237, in _update\n self._update_to_placement(context, compute_node, startup)\n File "/usr/lib/python3.6/site-packages/retrying.py", line 68, in wrapped_f\n return Retrying(*dargs, **dkw).call(f, *args, **kw)\n File "/usr/lib/python3.6/site-packages/retrying.py", line 223, in call\n return attempt.get(self._wrap_exception)\n File "/usr/lib/python3.6/site-packages/retrying.py", line 261, in get\n six.reraise(self.value[0], self.value[1], self.value[2])\n File "/usr/lib/python3.6/site-packages/six.py", line 703, in reraise\n raise value\n File "/usr/lib/python3.6/site-packages/retrying.py", line 217, in call\n attempt = Attempt(fn(*args, **kwargs), attempt_number, False)\n File "/usr/lib/python3.6/site-packages/nova/compute/resource_tracker.py", line 1157, in _update_to_placement\n self.driver.update_provider_tree(prov_tree, nodename)\n File "/usr/lib/python3.6/site-packages/nova/virt/libvirt/driver.py", line 7362, in update_provider_tree\n memory_mb = int(self._host.get_memory_mb_total())\n File "/usr/lib/python3.6/site-packages/nova/virt/libvirt/host.py", line 1061, in get_memory_mb_total\n return self._get_hardware_info()[1]\n File "/usr/lib/python3.6/site-packages/nova/virt/libvirt/host.py", line 1051, in _get_hardware_info\n return self.get_connection().getInfo()\n File "/usr/lib/python3.6/site-packages/nova/virt/libvirt/host.py", line 515, in get_connection\n raise exception.HypervisorUnavailable()\nnova.exception.HypervisorUnavailable: Connection to the hypervisor is broken on host\n\nDuring handling of the above exception, another exception occurred:\n\nTraceback (most recent call last):\n File "/usr/lib/python3.6/site-packages/nova/compute/manager.py", line 2168, in _do_build_and_run_instance\n filter_properties, request_spec)\n File "/usr/lib/python3.6/site-packages/nova/compute/manager.py", line 2544, in _build_and_run_instance\n instance_uuid=instance.uuid, reason=six.text_type(e))\nnova.exception.RescheduledException: Build of instance 25ddccbf-706d-41fa-a6a6-f5f9f70c5b2c was re-scheduled: Connection to the hypervisor is broken on host\n'}
According to the log from nova-compute https:/
2021-03-30 07:29:22.751 7 ERROR nova.compute.
2021-03-30 07:29:22.751 7 ERROR nova.compute.
2021-03-30 07:29:22.751 7 ERROR nova.compute.
2021-03-30 07:29:22.751 7 ERROR nova.compute.
2021-03-30 07:29:22.751 7 ERROR nova.compute.
2021-03-30 07:29:22.751 7 ERROR nova.compute.
2021-03-30 07:29:22.751 7 ERROR nova.compute.
2021-03-30 07:29:22.751 7 ERROR nova.compute.
2021-03-30 07:29:22.751 7 ERROR nova.compute.
2021-03-30 07:29:22.751 7 ERROR nova.compute.
2021-03-30 07:29:22.751 7 ERROR nova.compute.
2021-03-30 07:29:22.751 7 ERROR nova.compute.
2021-03-30 07:29:22.751 7 ERROR nova.compute.
2021-03-30 07:29:22.751 7 ERROR nova.compute.
2021-03-30 07:29:22.751 7 ERROR nova.compute.
2021-03-30 07:29:22.751 7 ERROR nova.compute.
2021-03-30 07:29:22.751 7 ERROR nova.compute.