Intermittent volume decoding issues occur when live migrating instances using Powerflex backend volumes.
storage: powerflex 4.5
openstack release: zed
os: rocky 9
Live migration of instances becomes impossible intermittently with the log output below.
nova_compute log
2024-04-18 11:47:35.007 7 ERROR oslo_messaging.rpc.server [None req-931b1ac1-49b9-4f18-bde8-5cc6a584e12c 5aedc8032351d28dff0440bd409e9a51a7fedc3281c810e2faaee013e5d18559 da817c7a5a8d47ce8c8b37ac2059a88a - - b10cce22cf2143db8385daee01754faa default] Exception during message handling: os_brick.exception.BrickException: Error unmapping volume erARePVqQzmcdq0Wu5ztcA==: The volume is not mapped to the Host
2024-04-18 11:47:35.007 7 ERROR oslo_messaging.rpc.server Traceback (most recent call last):
2024-04-18 11:47:35.007 7 ERROR oslo_messaging.rpc.server File "/var/lib/kolla/venv/lib/python3.9/site-packages/oslo_messaging/rpc/server.py", line 165, in _process_incoming
2024-04-18 11:47:35.007 7 ERROR oslo_messaging.rpc.server res = self.dispatcher.dispatch(message)
2024-04-18 11:47:35.007 7 ERROR oslo_messaging.rpc.server File "/var/lib/kolla/venv/lib/python3.9/site-packages/oslo_messaging/rpc/dispatcher.py", line 309, in dispatch
2024-04-18 11:47:35.007 7 ERROR oslo_messaging.rpc.server return self._do_dispatch(endpoint, method, ctxt, args)
2024-04-18 11:47:35.007 7 ERROR oslo_messaging.rpc.server File "/var/lib/kolla/venv/lib/python3.9/site-packages/oslo_messaging/rpc/dispatcher.py", line 229, in _do_dispatch
2024-04-18 11:47:35.007 7 ERROR oslo_messaging.rpc.server result = func(ctxt, **new_args)
2024-04-18 11:47:35.007 7 ERROR oslo_messaging.rpc.server File "/var/lib/kolla/venv/lib/python3.9/site-packages/nova/exception_wrapper.py", line 71, in wrapped
2024-04-18 11:47:35.007 7 ERROR oslo_messaging.rpc.server _emit_versioned_exception_notification(
2024-04-18 11:47:35.007 7 ERROR oslo_messaging.rpc.server File "/var/lib/kolla/venv/lib/python3.9/site-packages/oslo_utils/excutils.py", line 227, in __exit__
2024-04-18 11:47:35.007 7 ERROR oslo_messaging.rpc.server self.force_reraise()
2024-04-18 11:47:35.007 7 ERROR oslo_messaging.rpc.server File "/var/lib/kolla/venv/lib/python3.9/site-packages/oslo_utils/excutils.py", line 200, in force_reraise
2024-04-18 11:47:35.007 7 ERROR oslo_messaging.rpc.server raise self.value
2024-04-18 11:47:35.007 7 ERROR oslo_messaging.rpc.server File "/var/lib/kolla/venv/lib/python3.9/site-packages/nova/exception_wrapper.py", line 63, in wrapped
2024-04-18 11:47:35.007 7 ERROR oslo_messaging.rpc.server return f(self, context, *args, **kw)
2024-04-18 11:47:35.007 7 ERROR oslo_messaging.rpc.server File "/var/lib/kolla/venv/lib/python3.9/site-packages/nova/compute/manager.py", line 7782, in remove_volume_connection
2024-04-18 11:47:35.007 7 ERROR oslo_messaging.rpc.server self._remove_volume_connection(context, bdm, instance)
2024-04-18 11:47:35.007 7 ERROR oslo_messaging.rpc.server File "/var/lib/kolla/venv/lib/python3.9/site-packages/nova/compute/manager.py", line 7803, in _remove_volume_connection
2024-04-18 11:47:35.007 7 ERROR oslo_messaging.rpc.server driver_bdm.driver_detach(context, instance,
2024-04-18 11:47:35.007 7 ERROR oslo_messaging.rpc.server File "/var/lib/kolla/venv/lib/python3.9/site-packages/nova/virt/block_device.py", line 398, in driver_detach
2024-04-18 11:47:35.007 7 ERROR oslo_messaging.rpc.server volume_api.roll_detaching(context, volume_id)
2024-04-18 11:47:35.007 7 ERROR oslo_messaging.rpc.server File "/var/lib/kolla/venv/lib/python3.9/site-packages/oslo_utils/excutils.py", line 227, in __exit__
2024-04-18 11:47:35.007 7 ERROR oslo_messaging.rpc.server self.force_reraise()
2024-04-18 11:47:35.007 7 ERROR oslo_messaging.rpc.server File "/var/lib/kolla/venv/lib/python3.9/site-packages/oslo_utils/excutils.py", line 200, in force_reraise
2024-04-18 11:47:35.007 7 ERROR oslo_messaging.rpc.server raise self.value
2024-04-18 11:47:35.007 7 ERROR oslo_messaging.rpc.server File "/var/lib/kolla/venv/lib/python3.9/site-packages/nova/virt/block_device.py", line 379, in driver_detach
2024-04-18 11:47:35.007 7 ERROR oslo_messaging.rpc.server virt_driver.detach_volume(context, connection_info, instance, mp,
2024-04-18 11:47:35.007 7 ERROR oslo_messaging.rpc.server File "/var/lib/kolla/venv/lib/python3.9/site-packages/nova/virt/libvirt/driver.py", line 2711, in detach_volume
2024-04-18 11:47:35.007 7 ERROR oslo_messaging.rpc.server self._disconnect_volume(context, connection_info, instance,
2024-04-18 11:47:35.007 7 ERROR oslo_messaging.rpc.server File "/var/lib/kolla/venv/lib/python3.9/site-packages/nova/virt/libvirt/driver.py", line 1976, in _disconnect_volume
2024-04-18 11:47:35.007 7 ERROR oslo_messaging.rpc.server vol_driver.disconnect_volume(
2024-04-18 11:47:35.007 7 ERROR oslo_messaging.rpc.server File "/var/lib/kolla/venv/lib/python3.9/site-packages/nova/virt/libvirt/volume/scaleio.py", line 61, in disconnect_volume
2024-04-18 11:47:35.007 7 ERROR oslo_messaging.rpc.server self.connector.disconnect_volume(
2024-04-18 11:47:35.007 7 ERROR oslo_messaging.rpc.server File "/var/lib/kolla/venv/lib/python3.9/site-packages/os_brick/utils.py", line 171, in trace_logging_wrapper
2024-04-18 11:47:35.007 7 ERROR oslo_messaging.rpc.server return f(*args, **kwargs)
2024-04-18 11:47:35.007 7 ERROR oslo_messaging.rpc.server File "/var/lib/kolla/venv/lib/python3.9/site-packages/os_brick/initiator/connectors/base.py", line 78, in inner
2024-04-18 11:47:35.007 7 ERROR oslo_messaging.rpc.server return f(*args, **kwargs)
2024-04-18 11:47:35.007 7 ERROR oslo_messaging.rpc.server File "/var/lib/kolla/venv/lib/python3.9/site-packages/os_brick/initiator/connectors/base.py", line 78, in inner
2024-04-18 11:47:35.007 7 ERROR oslo_messaging.rpc.server return f(*args, **kwargs)
2024-04-18 11:47:35.007 7 ERROR oslo_messaging.rpc.server File "/var/lib/kolla/venv/lib/python3.9/site-packages/os_brick/utils.py", line 386, in change_encrypted
2024-04-18 11:47:35.007 7 ERROR oslo_messaging.rpc.server res = func(**call_args)
2024-04-18 11:47:35.007 7 ERROR oslo_messaging.rpc.server File "/var/lib/kolla/venv/lib/python3.9/site-packages/os_brick/initiator/connectors/scaleio.py", line 527, in disconnect_volume
2024-04-18 11:47:35.007 7 ERROR oslo_messaging.rpc.server raise exception.BrickException(message=msg)
2024-04-18 11:47:35.007 7 ERROR oslo_messaging.rpc.server os_brick.exception.BrickException: Error unmapping volume erARePVqQzmcdq0Wu5ztcA==: The volume is not mapped to the Host
2024-04-18 11:47:35.007 7 ERROR oslo_messaging.rpc.server
2024-04-18 11:47:35.894 7 ERROR oslo_messaging.rpc.server [None req-931b1ac1-49b9-4f18-bde8-5cc6a584e12c 5aedc8032351d28dff0440bd409e9a51a7fedc3281c810e2faaee013e5d18559 da817c7a5a8d47ce8c8b37ac2059a88a - - b10cce22cf2143db8385daee01754faa default] Exception during message handling: requests.exceptions.JSONDecodeError: Expecting value: line 1 column 1 (char 0)
2024-04-18 11:47:35.894 7 ERROR oslo_messaging.rpc.server [None req-931b1ac1-49b9-4f18-bde8-5cc6a584e12c 5aedc8032351d28dff0440bd409e9a51a7fedc3281c810e2faaee013e5d18559 da817c7a5a8d47ce8c8b37ac2059a88a - - b10cce22cf2143db8385daee01754faa default] Exception during message handling: requests.exceptions.JSONDecodeError: Expecting value: line 1 column 1 (char 0)
2024-04-18 11:47:35.894 7 ERROR oslo_messaging.rpc.server Traceback (most recent call last):
2024-04-18 11:47:35.894 7 ERROR oslo_messaging.rpc.server File "/var/lib/kolla/venv/lib/python3.9/site-packages/requests/models.py", line 971, in json
2024-04-18 11:47:35.894 7 ERROR oslo_messaging.rpc.server return complexjson.loads(self.text, **kwargs)
2024-04-18 11:47:35.894 7 ERROR oslo_messaging.rpc.server File "/var/lib/kolla/venv/lib64/python3.9/site-packages/simplejson/__init__.py", line 525, in loads
2024-04-18 11:47:35.894 7 ERROR oslo_messaging.rpc.server return _default_decoder.decode(s)
2024-04-18 11:47:35.894 7 ERROR oslo_messaging.rpc.server File "/var/lib/kolla/venv/lib64/python3.9/site-packages/simplejson/decoder.py", line 370, in decode
2024-04-18 11:47:35.894 7 ERROR oslo_messaging.rpc.server obj, end = self.raw_decode(s)
2024-04-18 11:47:35.894 7 ERROR oslo_messaging.rpc.server [None req-931b1ac1-49b9-4f18-bde8-5cc6a584e12c 5aedc8032351d28dff0440bd409e9a51a7fedc3281c810e2faaee013e5d18559 da817c7a5a8d47ce8c8b37ac2059a88a - - b10cce22cf2143db8385daee01754faa default] Exception during message handling: requests.exceptions.JSONDecodeError: Expecting value: line 1 column 1 (char 0)
2024-04-18 11:47:35.894 7 ERROR oslo_messaging.rpc.server Traceback (most recent call last):
2024-04-18 11:47:35.894 7 ERROR oslo_messaging.rpc.server File "/var/lib/kolla/venv/lib/python3.9/site-packages/requests/models.py", line 971, in json
2024-04-18 11:47:35.894 7 ERROR oslo_messaging.rpc.server return complexjson.loads(self.text, **kwargs)
2024-04-18 11:47:35.894 7 ERROR oslo_messaging.rpc.server File "/var/lib/kolla/venv/lib64/python3.9/site-packages/simplejson/__init__.py", line 525, in loads
2024-04-18 11:47:35.894 7 ERROR oslo_messaging.rpc.server return _default_decoder.decode(s)
2024-04-18 11:47:35.894 7 ERROR oslo_messaging.rpc.server File "/var/lib/kolla/venv/lib64/python3.9/site-packages/simplejson/decoder.py", line 370, in decode
2024-04-18 11:47:35.894 7 ERROR oslo_messaging.rpc.server obj, end = self.raw_decode(s)
2024-04-18 11:47:35.894 7 ERROR oslo_messaging.rpc.server File "/var/lib/kolla/venv/lib64/python3.9/site-packages/simplejson/decoder.py", line 400, in raw_decode
2024-04-18 11:47:35.894 7 ERROR oslo_messaging.rpc.server return self.scan_once(s, idx=_w(s, idx).end())
2024-04-18 11:47:35.894 7 ERROR oslo_messaging.rpc.server simplejson.errors.JSONDecodeError: Expecting value: line 1 column 1 (char 0)
2024-04-18 11:47:35.894 7 ERROR oslo_messaging.rpc.server
2024-04-18 11:47:35.894 7 ERROR oslo_messaging.rpc.server During handling of the above exception, another exception occurred:
2024-04-18 11:47:35.894 7 ERROR oslo_messaging.rpc.server
2024-04-18 11:47:35.894 7 ERROR oslo_messaging.rpc.server Traceback (most recent call last):
2024-04-18 11:47:35.894 7 ERROR oslo_messaging.rpc.server File "/var/lib/kolla/venv/lib/python3.9/site-packages/oslo_messaging/rpc/server.py", line 165, in _process_incoming
2024-04-18 11:47:35.894 7 ERROR oslo_messaging.rpc.server res = self.dispatcher.dispatch(message)
2024-04-18 11:47:35.894 7 ERROR oslo_messaging.rpc.server File "/var/lib/kolla/venv/lib/python3.9/site-packages/oslo_messaging/rpc/dispatcher.py", line 309, in dispatch
2024-04-18 11:47:35.894 7 ERROR oslo_messaging.rpc.server return self._do_dispatch(endpoint, method, ctxt, args)
2024-04-18 11:47:35.894 7 ERROR oslo_messaging.rpc.server File "/var/lib/kolla/venv/lib/python3.9/site-packages/oslo_messaging/rpc/dispatcher.py", line 229, in _do_dispatch
2024-04-18 11:47:35.894 7 ERROR oslo_messaging.rpc.server result = func(ctxt, **new_args)
2024-04-18 11:47:35.894 7 ERROR oslo_messaging.rpc.server File "/var/lib/kolla/venv/lib/python3.9/site-packages/nova/exception_wrapper.py", line 71, in wrapped
2024-04-18 11:47:35.894 7 ERROR oslo_messaging.rpc.server _emit_versioned_exception_notification(
2024-04-18 11:47:35.894 7 ERROR oslo_messaging.rpc.server File "/var/lib/kolla/venv/lib/python3.9/site-packages/oslo_utils/excutils.py", line 227, in __exit__
2024-04-18 11:47:35.894 7 ERROR oslo_messaging.rpc.server self.force_reraise()
2024-04-18 11:47:35.894 7 ERROR oslo_messaging.rpc.server File "/var/lib/kolla/venv/lib/python3.9/site-packages/oslo_utils/excutils.py", line 200, in force_reraise
2024-04-18 11:47:35.894 7 ERROR oslo_messaging.rpc.server raise self.value
2024-04-18 11:47:35.894 7 ERROR oslo_messaging.rpc.server File "/var/lib/kolla/venv/lib/python3.9/site-packages/nova/exception_wrapper.py", line 63, in wrapped
2024-04-18 11:47:35.894 7 ERROR oslo_messaging.rpc.server return f(self, context, *args, **kw)
2024-04-18 11:47:35.894 7 ERROR oslo_messaging.rpc.server File "/var/lib/kolla/venv/lib/python3.9/site-packages/nova/compute/utils.py", line 1439, in decorated_function
2024-04-18 11:47:35.894 7 ERROR oslo_messaging.rpc.server return function(self, context, *args, **kwargs)
2024-04-18 11:47:35.894 7 ERROR oslo_messaging.rpc.server File "/var/lib/kolla/venv/lib/python3.9/site-packages/nova/compute/manager.py", line 212, in decorated_function
2024-04-18 11:47:35.894 7 ERROR oslo_messaging.rpc.server compute_utils.add_instance_fault_from_exc(context,
2024-04-18 11:47:35.894 7 ERROR oslo_messaging.rpc.server File "/var/lib/kolla/venv/lib/python3.9/site-packages/oslo_utils/excutils.py", line 227, in __exit__
2024-04-18 11:47:35.894 7 ERROR oslo_messaging.rpc.server self.force_reraise()
2024-04-18 11:47:35.894 7 ERROR oslo_messaging.rpc.server File "/var/lib/kolla/venv/lib/python3.9/site-packages/oslo_utils/excutils.py", line 200, in force_reraise
2024-04-18 11:47:35.894 7 ERROR oslo_messaging.rpc.server raise self.value
2024-04-18 11:47:35.894 7 ERROR oslo_messaging.rpc.server File "/var/lib/kolla/venv/lib/python3.9/site-packages/nova/compute/manager.py", line 201, in decorated_function
2024-04-18 11:47:35.894 7 ERROR oslo_messaging.rpc.server return function(self, context, *args, **kwargs)
2024-04-18 11:47:35.894 7 ERROR oslo_messaging.rpc.server File "/var/lib/kolla/venv/lib/python3.9/site-packages/nova/compute/manager.py", line 9627, in rollback_live_migration_at_destination
2024-04-18 11:47:35.894 7 ERROR oslo_messaging.rpc.server self.driver.rollback_live_migration_at_destination(
2024-04-18 11:47:35.894 7 ERROR oslo_messaging.rpc.server File "/var/lib/kolla/venv/lib/python3.9/site-packages/nova/virt/libvirt/driver.py", line 10697, in rollback_live_migration_at_destination
2024-04-18 11:47:35.894 7 ERROR oslo_messaging.rpc.server self.destroy(context, instance, network_info, block_device_info,
2024-04-18 11:47:35.894 7 ERROR oslo_messaging.rpc.server File "/var/lib/kolla/venv/lib/python3.9/site-packages/nova/virt/libvirt/driver.py", line 1524, in destroy
2024-04-18 11:47:35.894 7 ERROR oslo_messaging.rpc.server self.cleanup(context, instance, network_info, block_device_info,
2024-04-18 11:47:35.894 7 ERROR oslo_messaging.rpc.server File "/var/lib/kolla/venv/lib/python3.9/site-packages/nova/virt/libvirt/driver.py", line 1594, in cleanup
2024-04-18 11:47:35.894 7 ERROR oslo_messaging.rpc.server return self._cleanup(
2024-04-18 11:47:35.894 7 ERROR oslo_messaging.rpc.server File "/var/lib/kolla/venv/lib/python3.9/site-packages/nova/virt/libvirt/driver.py", line 1655, in _cleanup
2024-04-18 11:47:35.894 7 ERROR oslo_messaging.rpc.server LOG.warning(
2024-04-18 11:47:35.894 7 ERROR oslo_messaging.rpc.server File "/var/lib/kolla/venv/lib/python3.9/site-packages/oslo_utils/excutils.py", line 227, in __exit__
2024-04-18 11:47:35.894 7 ERROR oslo_messaging.rpc.server self.force_reraise()
2024-04-18 11:47:35.894 7 ERROR oslo_messaging.rpc.server File "/var/lib/kolla/venv/lib/python3.9/site-packages/oslo_utils/excutils.py", line 200, in force_reraise
2024-04-18 11:47:35.894 7 ERROR oslo_messaging.rpc.server raise self.value
2024-04-18 11:47:35.894 7 ERROR oslo_messaging.rpc.server File "/var/lib/kolla/venv/lib/python3.9/site-packages/nova/virt/libvirt/driver.py", line 1645, in _cleanup
2024-04-18 11:47:35.894 7 ERROR oslo_messaging.rpc.server self._disconnect_volume(
2024-04-18 11:47:35.894 7 ERROR oslo_messaging.rpc.server File "/var/lib/kolla/venv/lib/python3.9/site-packages/nova/virt/libvirt/driver.py", line 1976, in _disconnect_volume
2024-04-18 11:47:35.894 7 ERROR oslo_messaging.rpc.server vol_driver.disconnect_volume(
2024-04-18 11:47:35.894 7 ERROR oslo_messaging.rpc.server File "/var/lib/kolla/venv/lib/python3.9/site-packages/nova/virt/libvirt/volume/scaleio.py", line 61, in disconnect_volume
2024-04-18 11:47:35.894 7 ERROR oslo_messaging.rpc.server self.connector.disconnect_volume(
2024-04-18 11:47:35.894 7 ERROR oslo_messaging.rpc.server File "/var/lib/kolla/venv/lib/python3.9/site-packages/os_brick/utils.py", line 171, in trace_logging_wrapper
2024-04-18 11:47:35.894 7 ERROR oslo_messaging.rpc.server return f(*args, **kwargs)
2024-04-18 11:47:35.894 7 ERROR oslo_messaging.rpc.server File "/var/lib/kolla/venv/lib/python3.9/site-packages/os_brick/initiator/connectors/base.py", line 78, in inner
2024-04-18 11:47:35.894 7 ERROR oslo_messaging.rpc.server return f(*args, **kwargs)
2024-04-18 11:47:35.894 7 ERROR oslo_messaging.rpc.server File "/var/lib/kolla/venv/lib/python3.9/site-packages/os_brick/utils.py", line 386, in change_encrypted
2024-04-18 11:47:35.894 7 ERROR oslo_messaging.rpc.server res = func(**call_args)
2024-04-18 11:47:35.894 7 ERROR oslo_messaging.rpc.server File "/var/lib/kolla/venv/lib/python3.9/site-packages/os_brick/initiator/connectors/scaleio.py", line 515, in disconnect_volume
2024-04-18 11:47:35.894 7 ERROR oslo_messaging.rpc.server response = r.json()
2024-04-18 11:47:35.894 7 ERROR oslo_messaging.rpc.server File "/var/lib/kolla/venv/lib/python3.9/site-packages/requests/models.py", line 975, in json
2024-04-18 11:47:35.894 7 ERROR oslo_messaging.rpc.server raise RequestsJSONDecodeError(e.msg, e.doc, e.pos)
2024-04-18 11:47:35.894 7 ERROR oslo_messaging.rpc.server requests.exceptions.JSONDecodeError: Expecting value: line 1 column 1 (char 0)
2024-04-18 11:47:35.894 7 ERROR oslo_messaging.rpc.server
If you look at the error details, a JSONDecodeError occurs and the process of decoding the unique ID used in powerflex into openstack uuid format fails.
