The problem with live migration et al seems to occur at the time of upgrade from newton to ocata. Not necessarily with the cinder-ceph topology change: For example an instance created at newton, then the cloud is upgraded to ocata. Attempting to live migrate: Source host: 2019-01-14 19:18:01.258 4742 ERROR nova.compute.manager [instance: f4ff161f-7c51-4d2a-a97d-c8cff12d5651] 2019-01-14 19:18:01.775 4742 ERROR root [req-5e9c7fd1-7072-4878-8a97-6751581fbba4 16d150ce78794d3eba5cafa0f6e83b36 dbdbac7a5d09477b891321ef11690f03 - - -] Original exception being dropped: ['Traceback (most recent call last): ', ' File "/usr/lib/python2.7/dist-packages/nova/compute/manager.py", line 5440, in _do_live_migration block_migration, disk, dest, migrate_data) ', ' File "/usr/lib/python2.7/dist-packages/nova/compute/rpcapi.py", line 723, in pre_live_migration disk=disk, migrate_data=migrate_data) ', ' File "/usr/lib/python2.7/dist-packages/oslo_messaging/rpc/client.py", line 169, in call retry=self.retry) ', ' File "/usr/lib/python2.7/dist-packages/oslo_messaging/transport.py", line 97, in _send timeout=timeout, retry=retry) ', ' File "/usr/lib/python2.7/dist-packages/oslo_messaging/_drivers/amqpdriver.py", line 458, in send retry=retry) ', ' File "/usr/lib/python2.7/dist-packages/oslo_messaging/_drivers/amqpdriver.py", line 449, in _send raise result ', 'RemoteError: Remote error: ClientException Internal Server Error (HTTP 500) [u\'Traceback (most recent call last):\ \', u\' File "/usr/lib/python2.7/dist-packages/oslo_messaging/rpc/server.py", line 155, in _process_incoming\ res = self.dispatcher.dispatch(message)\ \', u\' File "/usr/lib/python2.7/dist-packages/oslo_messaging/rpc/dispatcher.py", line 222, in dispatch\ return self._do_dispatch(endpoint, method, ctxt, args)\ \', u\' File "/usr/lib/python2.7/dist-packages/oslo_messaging/rpc/dispatcher.py", line 192, in _do_dispatch\ result = func(ctxt, **new_args)\ \', u\' File "/usr/lib/python2.7/dist-packages/nova/exception_wrapper.py", line 75, in wrapped\ function_name, call_dict, binary)\ \', u\' File "/usr/lib/python2.7/dist-packages/oslo_utils/excutils.py", line 220, in __exit__\ self.force_reraise()\ \', u\' File "/usr/lib/python2.7/dist-packages/oslo_utils/excutils.py", line 196, in force_reraise\ six.reraise(self.type_, self.value, self.tb)\ \', u\' File "/usr/lib/python2.7/dist-packages/nova/exception_wrapper.py", line 66, in wrapped\ return f(self, context, *args, **kw)\ \', u\' File "/usr/lib/python2.7/dist-packages/nova/compute/utils.py", line 686, in decorated_function\ return function(self, context, *args, **kwargs)\ \', u\' File "/usr/lib/python2.7/dist-packages/nova/compute/manager.py", line 216, in decorated_function\ kwargs[\\\'instance\\\'], e, sys.exc_info())\ \', u\' File "/usr/lib/python2.7/dist-packages/oslo_utils/excutils.py", line 220, in __exit__\ self.force_reraise()\ \', u\' File "/usr/lib/python2.7/dist-packages/oslo_utils/excutils.py", line 196, in force_reraise\ six.reraise(self.type_, self.value, self.tb)\ \', u\' File "/usr/lib/python2.7/dist-packages/nova/compute/manager.py", line 204, in decorated_function\ return function(self, context, *args, **kwargs)\ \', u\' File "/usr/lib/python2.7/dist-packages/nova/compute/manager.py", line 5373, in pre_live_migration\ context, instance, refresh_conn_info=True)\ \', u\' File "/usr/lib/python2.7/dist-packages/nova/compute/manager.py", line 1696, in _get_instance_block_device_info\ context, instance, self.volume_api, self.driver)\ \', u\' File "/usr/lib/python2.7/dist-packages/nova/virt/block_device.py", line 521, in refresh_conn_infos\ device.refresh_connection_info(*refresh_args, **refresh_kwargs)\ \', u\' File "/usr/lib/python2.7/dist-packages/nova/virt/block_device.py", line 48, in wrapped\ ret_val = method(obj, context, *args, **kwargs)\ \', u\' File "/usr/lib/python2.7/dist-packages/nova/virt/block_device.py", line 331, in refresh_connection_info\ connector)\ \', u\' File "/usr/lib/python2.7/dist-packages/nova/volume/cinder.py", line 168, in wrapper\ res = method(self, ctx, *args, **kwargs)\ \', u\' File "/usr/lib/python2.7/dist-packages/nova/volume/cinder.py", line 190, in wrapper\ res = method(self, ctx, volume_id, *args, **kwargs)\ \', u\' File "/usr/lib/python2.7/dist-packages/nova/volume/cinder.py", line 391, in initialize_connection\ exc.code if hasattr(exc, \\\'code\\\') else None)})\ \', u\' File "/usr/lib/python2.7/dist-packages/oslo_utils/excutils.py", line 220, in __exit__\ self.force_reraise()\ \', u\' File "/usr/lib/python2.7/dist-packages/oslo_utils/excutils.py", line 196, in force_reraise\ six.reraise(self.type_, self.value, self.tb)\ \', u\' File "/usr/lib/python2.7/dist-packages/nova/volume/cinder.py", line 365, in initialize_connection\ context).volumes.initialize_connection(volume_id, connector)\ \', u\' File "/usr/lib/python2.7/dist-packages/cinderclient/v2/volumes.py", line 404, in initialize_connection\ {\\\'connector\\\': connector})\ \', u\' File "/usr/lib/python2.7/dist-packages/cinderclient/v2/volumes.py", line 334, in _action\ resp, body = self.api.client.post(url, body=body)\ \', u\' File "/usr/lib/python2.7/dist-packages/cinderclient/client.py", line 167, in post\ return self._cs_request(url, \\\'POST\\\', **kwargs)\ \', u\' File "/usr/lib/python2.7/dist-packages/cinderclient/client.py", line 155, in _cs_request\ return self.request(url, method, **kwargs)\ \', u\' File "/usr/lib/python2.7/dist-packages/cinderclient/client.py", line 144, in request\ raise exceptions.from_response(resp, body)\ \', u\'ClientException: Internal Server Error (HTTP 500)\ \']. '] Destination host: 2019-01-14 19:18:01.660 8643 WARNING nova.compute.manager [req-5e9c7fd1-7072-4878-8a97-6751581fbba4 16d150ce78794d3eba5cafa0f6e83b36 dbdbac7a5d09477b891321ef11690f03 - - -] [instance: f4ff161f-7c51-4d2a-a97d-c8cff12d5651] Detaching volume from unknown instance 2019-01-14 19:18:01.662 8643 WARNING nova.virt.libvirt.driver [req-5e9c7fd1-7072-4878-8a97-6751581fbba4 16d150ce78794d3eba5cafa0f6e83b36 dbdbac7a5d09477b891321ef11690f03 - - -] [instance: f4ff161f-7c51-4d2a-a97d-c8cff12d5651] During detach_volume, instance disappeared. 2019-01-14 19:18:01.725 8643 ERROR oslo_messaging.rpc.server [req-5e9c7fd1-7072-4878-8a97-6751581fbba4 16d150ce78794d3eba5cafa0f6e83b36 dbdbac7a5d09477b891321ef11690f03 - - -] Exception during message handling 2019-01-14 19:18:01.725 8643 ERROR oslo_messaging.rpc.server Traceback (most recent call last): 2019-01-14 19:18:01.725 8643 ERROR oslo_messaging.rpc.server File "/usr/lib/python2.7/dist-packages/oslo_messaging/rpc/server.py", line 155, in _process_incoming 2019-01-14 19:18:01.725 8643 ERROR oslo_messaging.rpc.server res = self.dispatcher.dispatch(message) 2019-01-14 19:18:01.725 8643 ERROR oslo_messaging.rpc.server File "/usr/lib/python2.7/dist-packages/oslo_messaging/rpc/dispatcher.py", line 222, in dispatch 2019-01-14 19:18:01.725 8643 ERROR oslo_messaging.rpc.server return self._do_dispatch(endpoint, method, ctxt, args) 2019-01-14 19:18:01.725 8643 ERROR oslo_messaging.rpc.server File "/usr/lib/python2.7/dist-packages/oslo_messaging/rpc/dispatcher.py", line 192, in _do_dispatch 2019-01-14 19:18:01.725 8643 ERROR oslo_messaging.rpc.server result = func(ctxt, **new_args) 2019-01-14 19:18:01.725 8643 ERROR oslo_messaging.rpc.server File "/usr/lib/python2.7/dist-packages/nova/exception_wrapper.py", line 75, in wrapped 2019-01-14 19:18:01.725 8643 ERROR oslo_messaging.rpc.server function_name, call_dict, binary) 2019-01-14 19:18:01.725 8643 ERROR oslo_messaging.rpc.server File "/usr/lib/python2.7/dist-packages/oslo_utils/excutils.py", line 220, in __exit__ 2019-01-14 19:18:01.725 8643 ERROR oslo_messaging.rpc.server self.force_reraise() 2019-01-14 19:18:01.725 8643 ERROR oslo_messaging.rpc.server File "/usr/lib/python2.7/dist-packages/oslo_utils/excutils.py", line 196, in force_reraise 2019-01-14 19:18:01.725 8643 ERROR oslo_messaging.rpc.server six.reraise(self.type_, self.value, self.tb) 2019-01-14 19:18:01.725 8643 ERROR oslo_messaging.rpc.server File "/usr/lib/python2.7/dist-packages/nova/exception_wrapper.py", line 66, in wrapped 2019-01-14 19:18:01.725 8643 ERROR oslo_messaging.rpc.server return f(self, context, *args, **kw) 2019-01-14 19:18:01.725 8643 ERROR oslo_messaging.rpc.server File "/usr/lib/python2.7/dist-packages/nova/compute/manager.py", line 5178, in remove_volume_connection 2019-01-14 19:18:01.725 8643 ERROR oslo_messaging.rpc.server self.volume_api.terminate_connection(context, volume_id, connector) 2019-01-14 19:18:01.725 8643 ERROR oslo_messaging.rpc.server File "/usr/lib/python2.7/dist-packages/nova/volume/cinder.py", line 168, in wrapper 2019-01-14 19:18:01.725 8643 ERROR oslo_messaging.rpc.server res = method(self, ctx, *args, **kwargs) 2019-01-14 19:18:01.725 8643 ERROR oslo_messaging.rpc.server File "/usr/lib/python2.7/dist-packages/nova/volume/cinder.py", line 190, in wrapper 2019-01-14 19:18:01.725 8643 ERROR oslo_messaging.rpc.server res = method(self, ctx, volume_id, *args, **kwargs) 2019-01-14 19:18:01.725 8643 ERROR oslo_messaging.rpc.server File "/usr/lib/python2.7/dist-packages/nova/volume/cinder.py", line 396, in terminate_connection 2019-01-14 19:18:01.725 8643 ERROR oslo_messaging.rpc.server connector) 2019-01-14 19:18:01.725 8643 ERROR oslo_messaging.rpc.server File "/usr/lib/python2.7/dist-packages/cinderclient/v2/volumes.py", line 414, in terminate_connection 2019-01-14 19:18:01.725 8643 ERROR oslo_messaging.rpc.server {'connector': connector}) 2019-01-14 19:18:01.725 8643 ERROR oslo_messaging.rpc.server File "/usr/lib/python2.7/dist-packages/cinderclient/v2/volumes.py", line 334, in _action 2019-01-14 19:18:01.725 8643 ERROR oslo_messaging.rpc.server resp, body = self.api.client.post(url, body=body) 2019-01-14 19:18:01.725 8643 ERROR oslo_messaging.rpc.server File "/usr/lib/python2.7/dist-packages/cinderclient/client.py", line 167, in post 2019-01-14 19:18:01.725 8643 ERROR oslo_messaging.rpc.server return self._cs_request(url, 'POST', **kwargs) 2019-01-14 19:18:01.725 8643 ERROR oslo_messaging.rpc.server File "/usr/lib/python2.7/dist-packages/cinderclient/client.py", line 155, in _cs_request 2019-01-14 19:18:01.725 8643 ERROR oslo_messaging.rpc.server return self.request(url, method, **kwargs) 2019-01-14 19:18:01.725 8643 ERROR oslo_messaging.rpc.server File "/usr/lib/python2.7/dist-packages/cinderclient/client.py", line 144, in request 2019-01-14 19:18:01.725 8643 ERROR oslo_messaging.rpc.server raise exceptions.from_response(resp, body) 2019-01-14 19:18:01.725 8643 ERROR oslo_messaging.rpc.server ClientException: Internal Server Error (HTTP 500) 2019-01-14 19:18:01.725 8643 ERROR oslo_messaging.rpc.server