I saw something similar today in another test - it hits novalidhost and then we get a 404 while waiting for the instance to go to ERROR state: http://logs.openstack.org/34/607934/15/gate/nova-tox-functional-py35/25b8065/testr_results.html.gz ft1.2: nova.tests.functional.wsgi.test_servers.EnforceVolumeBackedForZeroDiskFlavorTestCase.test_create_volume_backed_server_with_zero_disk_allowed_StringException: pythonlogging:'': {{{ 2018-11-02 01:11:15,309 INFO [nova.api.openstack.requestlog] 127.0.0.1 "GET /v2.1" status: 204 len: 0 microversion: - time: 0.000513 2018-11-02 01:11:15,345 INFO [nova.api.openstack.requestlog] 127.0.0.1 "POST /v2.1/6f70656e737461636b20342065766572/flavors" status: 200 len: 577 microversion: 2.1 time: 0.031808 2018-11-02 01:11:15,358 INFO [nova.api.openstack.placement.objects.resource_provider] Synced traits from os_traits into API DB: {'HW_NIC_OFFLOAD_TXVLAN', 'HW_CPU_AARCH64_JSCVT', 'HW_CPU_X86_SSSE3', 'HW_NIC_OFFLOAD_RXHASH', 'HW_GPU_CUDA_COMPUTE_CAPABILITY_V3_0', 'HW_CPU_AARCH64_FP', 'HW_CPU_AARCH64_SM4', 'HW_CPU_AARCH64_EVTSTRM', 'HW_CPU_HYPERTHREADING', 'COMPUTE_DEVICE_TAGGING', 'HW_GPU_API_DIRECTX_V10', 'HW_CPU_X86_MMX', 'HW_GPU_API_OPENGL_V1_2', 'HW_CPU_X86_SSE', 'HW_CPU_AARCH64_CPUID', 'HW_CPU_X86_SGX', 'HW_GPU_API_OPENGL_V2_1', 'HW_NIC_OFFLOAD_TCS', 'HW_CPU_AARCH64_SHA512', 'HW_GPU_RESOLUTION_W1920H1080', 'HW_NIC_OFFLOAD_GSO', 'HW_GPU_API_OPENGL_V4_3', 'HW_NIC_OFFLOAD_TX', 'HW_NIC_OFFLOAD_RX', 'HW_NIC_SRIOV_TRUSTED', 'HW_GPU_API_VULKAN', 'HW_NIC_OFFLOAD_RXVLAN', 'HW_CPU_X86_ABM', 'HW_NIC_OFFLOAD_TXUDP', 'HW_CPU_X86_CLMUL', 'HW_GPU_API_OPENGL_V4_4', 'HW_NIC_DCB_QCN', 'HW_CPU_X86_FMA3', 'HW_GPU_RESOLUTION_W1366H768', 'HW_CPU_AARCH64_LRCPC', 'HW_GPU_CUDA_SDK_V6_5', 'HW_GPU_CUDA_COMPUTE_CAPABILITY_V6_0', 'HW_GPU_MAX_DISPLAY_HEADS_1', 'HW_GPU_CUDA_SDK_V9_2', 'HW_GPU_API_OPENGL_V4_2', 'HW_GPU_API_DIRECT2D', 'HW_GPU_CUDA_COMPUTE_CAPABILITY_V2_0', 'HW_GPU_CUDA_COMPUTE_CAPABILITY_V7_1', 'HW_GPU_RESOLUTION_W640H480', 'HW_GPU_RESOLUTION_W800H600', 'HW_CPU_X86_SSE3', 'HW_CPU_AARCH64_SHA2', 'HW_CPU_X86_AVX', 'HW_GPU_CUDA_SDK_V7_5', 'HW_GPU_RESOLUTION_W2560H1440', 'HW_GPU_CUDA_COMPUTE_CAPABILITY_V5_2', 'HW_NIC_OFFLOAD_RDMA', 'HW_CPU_X86_MPX', 'HW_GPU_API_DIRECT3D_V8_1', 'HW_CPU_X86_AVX2', 'HW_GPU_API_OPENCL_V1_0', 'HW_GPU_API_DIRECT3D_V10_0', 'HW_CPU_X86_SSE4A', 'HW_GPU_RESOLUTION_W1280H800', 'HW_CPU_AARCH64_ASIMDDP', 'HW_CPU_X86_SSE2', 'HW_GPU_API_DIRECT3D_V10_1', 'HW_GPU_CUDA_COMPUTE_CAPABILITY_V6_2', 'HW_CPU_X86_TBM', 'HW_NIC_ACCEL_IPSEC', 'HW_GPU_API_DIRECT3D_V6_0', 'HW_GPU_RESOLUTION_W1440H900', 'COMPUTE_NET_ATTACH_INTERFACE_WITH_TAG', 'HW_CPU_X86_AESNI', 'HW_GPU_API_DIRECTX_V11', 'HW_CPU_AARCH64_SHA3', 'HW_GPU_API_DIRECT3D_V11_0', 'HW_GPU_CUDA_COMPUTE_CAPABILITY_V5_3', 'HW_GPU_MAX_DISPLAY_HEADS_2', 'HW_GPU_API_DIRECTX_V12', 'HW_NIC_OFFLOAD_FDF', 'HW_NIC_ACCEL_SSL', 'HW_GPU_CUDA_COMPUTE_CAPABILITY_V5_0', 'HW_CPU_X86_VMX', 'HW_GPU_RESOLUTION_W1600H900', 'HW_CPU_X86_XOP', 'HW_NIC_VMDQ', 'HW_GPU_RESOLUTION_W1600H1200', 'HW_GPU_API_DIRECT3D_V12_0', 'HW_GPU_API_OPENGL_V3_2', 'HW_NIC_ACCEL_RSA', 'HW_GPU_API_OPENGL_V3_0', 'HW_NIC_OFFLOAD_UFO', 'HW_CPU_X86_AVX512DQ', 'HW_GPU_RESOLUTION_W7680H4320', 'HW_CPU_AARCH64_ATOMICS', 'HW_NIC_OFFLOAD_VXLAN', 'HW_GPU_API_OPENGL_V2_0', 'HW_NIC_OFFLOAD_L2CRC', 'HW_GPU_RESOLUTION_W1680H1050', 'HW_CPU_AARCH64_CRC32', 'HW_CPU_X86_ASF', 'HW_GPU_RESOLUTION_W1360H768', 'HW_GPU_CUDA_COMPUTE_CAPABILITY_V1_3', 'HW_GPU_API_OPENGL_V4_1', 'HW_GPU_API_DIRECT3D_V11_1', 'HW_NIC_OFFLOAD_GRO', 'HW_CPU_X86_F16C', 'HW_GPU_RESOLUTION_W320H240', 'COMPUTE_TRUSTED_CERTS', 'HW_GPU_CUDA_COMPUTE_CAPABILITY_V1_0', 'HW_NIC_OFFLOAD_QINQ', 'HW_NIC_OFFLOAD_TSO', 'HW_GPU_API_DIRECT3D_V11_3', 'HW_NIC_OFFLOAD_SCS', 'HW_GPU_API_OPENGL_V4_0', 'HW_GPU_RESOLUTION_W1024H600', 'HW_CPU_AARCH64_ASIMD', 'HW_GPU_API_DIRECT3D_V9_0B', 'HW_GPU_CUDA_COMPUTE_CAPABILITY_V2_1', 'HW_CPU_X86_AVX512CD', 'MISC_SHARES_VIA_AGGREGATE', 'HW_NIC_OFFLOAD_GRE', 'HW_NIC_OFFLOAD_LRO', 'HW_CPU_X86_AVX512VL', 'HW_CPU_X86_SSE41', 'HW_GPU_RESOLUTION_W3840H2160', 'HW_GPU_API_OPENGL_V3_1', 'HW_NIC_MULTIQUEUE', 'HW_CPU_X86_BMI', 'HW_GPU_CUDA_COMPUTE_CAPABILITY_V3_2', 'HW_GPU_API_OPENGL_V3_3', 'HW_GPU_API_DIRECT3D_V9_0', 'HW_CPU_X86_FMA4', 'HW_CPU_AARCH64_PMULL', 'HW_CPU_X86_AVX512F', 'HW_GPU_API_OPENGL_V1_4', 'HW_GPU_MAX_DISPLAY_HEADS_6', 'HW_GPU_API_OPENCL_V1_2', 'HW_CPU_X86_AVX512PF', 'HW_GPU_API_DIRECT3D_V9_0C', 'HW_GPU_CUDA_COMPUTE_CAPABILITY_V1_1', 'HW_GPU_RESOLUTION_W1280H1024', 'HW_NIC_PROGRAMMABLE_PIPELINE', 'HW_NIC_SRIOV', 'HW_CPU_AARCH64_SHA1', 'HW_CPU_AARCH64_ASIMDRDM', 'HW_GPU_RESOLUTION_W2560H1600', 'HW_CPU_AARCH64_AES', 'COMPUTE_VOLUME_EXTEND', 'HW_CPU_AARCH64_DCPOP', 'HW_NIC_SRIOV_MULTIQUEUE', 'HW_GPU_CUDA_SDK_V10_0', 'COMPUTE_VOLUME_ATTACH_WITH_TAG', 'HW_GPU_CUDA_SDK_V9_0', 'HW_NIC_OFFLOAD_LSO', 'HW_CPU_X86_SVM', 'HW_NIC_ACCEL_ECC', 'HW_CPU_AARCH64_ASIMDHP', 'HW_CPU_AARCH64_SM3', 'HW_CPU_X86_SSE42', 'HW_GPU_API_OPENGL_V1_3', 'HW_NIC_SRIOV_QOS_TX', 'HW_GPU_RESOLUTION_W1280H720', 'HW_GPU_API_OPENCL_V2_2', 'HW_NIC_OFFLOAD_GENEVE', 'HW_GPU_CUDA_COMPUTE_CAPABILITY_V6_1', 'COMPUTE_NET_ATTACH_INTERFACE', 'HW_CPU_AARCH64_FPHP', 'HW_GPU_CUDA_COMPUTE_CAPABILITY_V7_0', 'STORAGE_DISK_SSD', 'HW_GPU_API_DXVA', 'HW_GPU_API_DIRECT3D_V8_0', 'HW_GPU_API_OPENGL_V1_1', 'HW_GPU_CUDA_COMPUTE_CAPABILITY_V1_2', 'HW_NIC_ACCEL_DIFFIEH', 'HW_GPU_CUDA_SDK_V8_0', 'HW_GPU_CUDA_COMPUTE_CAPABILITY_V3_7', 'HW_GPU_API_OPENGL_V4_5', 'HW_CPU_X86_BMI2', 'HW_NIC_DCB_ETS', 'HW_GPU_API_DIRECT3D_V7_0', 'HW_CPU_X86_SHA', 'STORAGE_DISK_HDD', 'HW_NIC_ACCEL_TLS', 'HW_NIC_OFFLOAD_SG', 'HW_CPU_X86_TSX', 'HW_NIC_ACCEL_LZS', 'HW_NIC_DCB_PFC', 'HW_GPU_API_DIRECT3D_V9_0L', 'COMPUTE_VOLUME_ATTACH', 'HW_GPU_MAX_DISPLAY_HEADS_8', 'HW_GPU_CUDA_SDK_V9_1', 'HW_GPU_API_OPENCL_V2_1', 'HW_CPU_AARCH64_FCMA', 'HW_GPU_RESOLUTION_W1024H768', 'HW_NIC_OFFLOAD_UCS', 'HW_GPU_API_OPENCL_V1_1', 'COMPUTE_VOLUME_MULTI_ATTACH', 'HW_CPU_X86_3DNOW', 'HW_GPU_API_DIRECT3D_V11_2', 'HW_NIC_SRIOV_QOS_RX', 'HW_GPU_API_OPENGL_V1_5', 'HW_CPU_X86_AVX512ER', 'HW_CPU_X86_AVX512BW', 'HW_GPU_RESOLUTION_W1280H768', 'HW_GPU_CUDA_COMPUTE_CAPABILITY_V3_5', 'HW_CPU_AARCH64_SVE', 'HW_GPU_CUDA_COMPUTE_CAPABILITY_V7_2', 'HW_GPU_RESOLUTION_W1152H864', 'HW_GPU_RESOLUTION_W1920H1200', 'HW_GPU_MAX_DISPLAY_HEADS_4', 'HW_GPU_API_OPENCL_V2_0', 'HW_NIC_ACCEL_DEFLATE', 'HW_NIC_OFFLOAD_SWITCHDEV'} 2018-11-02 01:11:15,364 INFO [nova.service] Starting conductor node (version 18.1.0) 2018-11-02 01:11:15,389 INFO [nova.service] Starting scheduler node (version 18.1.0) 2018-11-02 01:11:15,416 INFO [nova.api.openstack.requestlog] 127.0.0.1 "GET /v2.1" status: 204 len: 0 microversion: - time: 0.000450 2018-11-02 01:11:15,423 INFO [nova.api.openstack.requestlog] 127.0.0.1 "GET /v2.1/6f70656e737461636b20342065766572/images/detail" status: 200 len: 5609 microversion: 2.1 time: 0.003905 2018-11-02 01:11:15,580 INFO [nova.api.openstack.requestlog] 127.0.0.1 "POST /v2.1/6f70656e737461636b20342065766572/servers" status: 202 len: 480 microversion: 2.1 time: 0.152285 2018-11-02 01:11:15,622 INFO [nova.api.openstack.requestlog] 127.0.0.1 "GET /v2.1/6f70656e737461636b20342065766572/servers/d8c1248e-5406-4159-960b-b54f652d65df" status: 200 len: 1254 microversion: 2.1 time: 0.036957 2018-11-02 01:11:15,657 INFO [nova.api.openstack.placement.requestlog] 127.0.0.1 "GET /placement/allocation_candidates?limit=1000&resources=MEMORY_MB%3A1024%2CVCPU%3A2" status: 200 len: 53 microversion: 1.29 2018-11-02 01:11:15,658 INFO [nova.scheduler.manager] Got no allocation candidates from the Placement API. This could be due to insufficient resources or a temporary occurrence as compute nodes start up. 2018-11-02 01:11:15,659 ERROR [nova.conductor.manager] Failed to schedule instances Traceback (most recent call last): File "/home/zuul/src/git.openstack.org/openstack/nova/nova/conductor/manager.py", line 1255, in schedule_and_build_instances instance_uuids, return_alternates=True) File "/home/zuul/src/git.openstack.org/openstack/nova/nova/conductor/manager.py", line 750, in _schedule_instances return_alternates=return_alternates) File "/home/zuul/src/git.openstack.org/openstack/nova/nova/scheduler/utils.py", line 952, in wrapped return func(*args, **kwargs) File "/home/zuul/src/git.openstack.org/openstack/nova/nova/scheduler/client/__init__.py", line 53, in select_destinations instance_uuids, return_objects, return_alternates) File "/home/zuul/src/git.openstack.org/openstack/nova/nova/scheduler/client/__init__.py", line 37, in __run_method return getattr(self.instance, __name)(*args, **kwargs) File "/home/zuul/src/git.openstack.org/openstack/nova/nova/scheduler/client/query.py", line 42, in select_destinations instance_uuids, return_objects, return_alternates) File "/home/zuul/src/git.openstack.org/openstack/nova/nova/scheduler/rpcapi.py", line 158, in select_destinations return cctxt.call(ctxt, 'select_destinations', **msg_args) File "/home/zuul/src/git.openstack.org/openstack/nova/.tox/py35/lib/python3.5/site-packages/oslo_messaging/rpc/client.py", line 179, in call retry=self.retry) File "/home/zuul/src/git.openstack.org/openstack/nova/.tox/py35/lib/python3.5/site-packages/oslo_messaging/transport.py", line 128, in _send retry=retry) File "/home/zuul/src/git.openstack.org/openstack/nova/.tox/py35/lib/python3.5/site-packages/oslo_messaging/_drivers/impl_fake.py", line 222, in send return self._send(target, ctxt, message, wait_for_reply, timeout) File "/home/zuul/src/git.openstack.org/openstack/nova/.tox/py35/lib/python3.5/site-packages/oslo_messaging/_drivers/impl_fake.py", line 209, in _send raise failure File "/home/zuul/src/git.openstack.org/openstack/nova/.tox/py35/lib/python3.5/site-packages/oslo_messaging/rpc/server.py", line 229, in inner return func(*args, **kwargs) File "/home/zuul/src/git.openstack.org/openstack/nova/nova/scheduler/manager.py", line 154, in select_destinations raise exception.NoValidHost(reason="") nova.exception.NoValidHost: No valid host was found. 2018-11-02 01:11:15,701 WARNING [nova.scheduler.utils] Failed to compute_task_build_instances: No valid host was found. 2018-11-02 01:11:15,702 WARNING [nova.scheduler.utils] Setting instance to ERROR state. 2018-11-02 01:11:16,254 WARNING [nova.compute.api] The nova-osapi_compute service version is from before Ocata and may cause problems looking up instances in a cells v2 setup. Check your nova-api service configuration and cell mappings. You may need to remove stale nova-osapi_compute service records from the cell database. 2018-11-02 01:11:16,306 INFO [nova.api.openstack.wsgi] HTTP exception thrown: Instance d8c1248e-5406-4159-960b-b54f652d65df could not be found. 2018-11-02 01:11:16,307 INFO [nova.api.openstack.requestlog] 127.0.0.1 "GET /v2.1/6f70656e737461636b20342065766572/servers/d8c1248e-5406-4159-960b-b54f652d65df" status: 404 len: 111 microversion: 2.1 time: 0.063434 }}} stderr: {{{ /home/zuul/src/git.openstack.org/openstack/nova/nova/test.py:293: DeprecationWarning: Using class 'MoxStubout' (either directly or via inheritance) is deprecated in version '3.5.0' mox_fixture = self.useFixture(moxstubout.MoxStubout()) /home/zuul/src/git.openstack.org/openstack/nova/.tox/py35/lib/python3.5/site-packages/paste/deploy/loadwsgi.py:22: DeprecationWarning: Parameters to load are deprecated. Call .resolve and .require separately. return pkg_resources.EntryPoint.parse("x=" + s).load(False) }}} Traceback (most recent call last): File "/home/zuul/src/git.openstack.org/openstack/nova/nova/tests/functional/wsgi/test_servers.py", line 427, in test_create_volume_backed_server_with_zero_disk_allowed server = self._wait_for_state_change(self.api, server, 'ERROR') File "/home/zuul/src/git.openstack.org/openstack/nova/nova/tests/functional/integrated_helpers.py", line 259, in _wait_for_state_change admin_api, server, {'status': expected_status}, max_retries) File "/home/zuul/src/git.openstack.org/openstack/nova/nova/tests/functional/integrated_helpers.py", line 243, in _wait_for_server_parameter server = admin_api.get_server(server['id']) File "/home/zuul/src/git.openstack.org/openstack/nova/nova/tests/functional/api/client.py", line 268, in get_server return self.api_get('/servers/%s' % server_id).body['server'] File "/home/zuul/src/git.openstack.org/openstack/nova/nova/tests/functional/api/client.py", line 225, in api_get return APIResponse(self.api_request(relative_uri, **kwargs)) File "/home/zuul/src/git.openstack.org/openstack/nova/nova/tests/functional/api/client.py", line 207, in api_request raise OpenStackApiNotFoundException(response=response) nova.tests.functional.api.client.OpenStackApiNotFoundException: Item not found There must be a race when we create the instance in cell0 in error state and when the build request is deleted.