we have a similar bug downstream against queens in the other direction alos https://bugzilla.redhat.com/show_bug.cgi?id=1838786 in the downstream bug the failure is on attach libvirtError: Requested operation is not valid: target vdf already exists where as upstream we have "nova.exception.DeviceDetachFailed: Device detach failed for vdb: Unable to detach the device from the live config." in both caes its as if the live domain view and nova's/cinder'd view or the domain are getting out of sync with each other the call trace upstream traceback for detach looks like this Traceback (most recent call last): File "/opt/stack/nova/nova/virt/block_device.py", line 328, in driver_detach virt_driver.detach_volume(context, connection_info, instance, mp, File "/opt/stack/nova/nova/virt/libvirt/driver.py", line 2004, in detach_volume wait_for_detach = guest.detach_device_with_retry(guest.get_disk, File "/opt/stack/nova/nova/virt/libvirt/guest.py", line 425, in detach_device_with_retry _try_detach_device(conf, persistent, live) File "/opt/stack/nova/nova/virt/libvirt/guest.py", line 414, in _try_detach_device ctx.reraise = True File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 220, in __exit__ self.force_reraise() File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 196, in force_reraise six.reraise(self.type_, self.value, self.tb) File "/usr/local/lib/python3.8/dist-packages/six.py", line 703, in reraise raise value File "/opt/stack/nova/nova/virt/libvirt/guest.py", line 387, in _try_detach_device self.detach_device(conf, persistent=persistent, live=live) File "/opt/stack/nova/nova/virt/libvirt/guest.py", line 475, in detach_device self._domain.detachDeviceFlags(device_xml, flags=flags) File "/usr/local/lib/python3.8/dist-packages/eventlet/tpool.py", line 190, in doit result = proxy_call(self._autowrap, f, *args, **kwargs) File "/usr/local/lib/python3.8/dist-packages/eventlet/tpool.py", line 148, in proxy_call rv = execute(f, *args, **kwargs) File "/usr/local/lib/python3.8/dist-packages/eventlet/tpool.py", line 129, in execute six.reraise(c, e, tb) File "/usr/local/lib/python3.8/dist-packages/six.py", line 703, in reraise raise value File "/usr/local/lib/python3.8/dist-packages/eventlet/tpool.py", line 83, in tworker rv = meth(*args, **kwargs) File "/usr/local/lib/python3.8/dist-packages/libvirt.py", line 1408, in detachDeviceFlags if ret == -1: raise libvirtError ('virDomainDetachDeviceFlags() failed', dom=self) libvirt.libvirtError: device not found: no target device vdb and the down stream traceback for attach looks like Traceback (most recent call last): File "/usr/lib/python2.7/site-packages/nova/compute/manager.py", line 5433, in _attach_volume do_driver_attach=True) File "/usr/lib/python2.7/site-packages/nova/virt/block_device.py", line 46, in wrapped ret_val = method(obj, context, *args, **kwargs) File "/usr/lib/python2.7/site-packages/nova/virt/block_device.py", line 624, in attach virt_driver, do_driver_attach) File "/usr/lib/python2.7/site-packages/nova/virt/block_device.py", line 602, in _do_attach do_driver_attach) File "/usr/lib/python2.7/site-packages/nova/virt/block_device.py", line 549, in _volume_attach attachment_id) File "/usr/lib/python2.7/site-packages/oslo_utils/excutils.py", line 220, in __exit__ self.force_reraise() File "/usr/lib/python2.7/site-packages/oslo_utils/excutils.py", line 196, in force_reraise six.reraise(self.type_, self.value, self.tb) File "/usr/lib/python2.7/site-packages/nova/virt/block_device.py", line 540, in _volume_attach device_type=self['device_type'], encryption=encryption) File "/usr/lib/python2.7/site-packages/nova/virt/libvirt/driver.py", line 1520, in attach_volume encryption=encryption) File "/usr/lib/python2.7/site-packages/oslo_utils/excutils.py", line 220, in __exit__ self.force_reraise() File "/usr/lib/python2.7/site-packages/oslo_utils/excutils.py", line 196, in force_reraise six.reraise(self.type_, self.value, self.tb) File "/usr/lib/python2.7/site-packages/nova/virt/libvirt/driver.py", line 1493, in attach_volume guest.attach_device(conf, persistent=True, live=live) File "/usr/lib/python2.7/site-packages/nova/virt/libvirt/guest.py", line 306, in attach_device self._domain.attachDeviceFlags(device_xml, flags=flags) File "/usr/lib/python2.7/site-packages/eventlet/tpool.py", line 186, in doit result = proxy_call(self._autowrap, f, *args, **kwargs) File "/usr/lib/python2.7/site-packages/eventlet/tpool.py", line 144, in proxy_call rv = execute(f, *args, **kwargs) File "/usr/lib/python2.7/site-packages/eventlet/tpool.py", line 125, in execute six.reraise(c, e, tb) File "/usr/lib/python2.7/site-packages/eventlet/tpool.py", line 83, in tworker rv = meth(*args, **kwargs) File "/usr/lib64/python2.7/site-packages/libvirt.py", line 605, in attachDeviceFlags if ret == -1: raise libvirtError ('virDomainAttachDeviceFlags() failed', dom=self) libvirtError: Requested operation is not valid: target vdf already exists these bug may be unrelated but i find it interesting that we are seeint issue with both attach and detach at the same time even if they are with two master vs queens and both seam to be realted to nova's view of the live domain mismatching with livbvirts.