Currently, seems cinder will always rename rbd image after volume migration. https://github.com/openstack/cinder/blob/c29285e7dcda7f5300c5a3375294dfa4aeb56430/cinder/volume/drivers/rbd.py#L1774 as long as it could do so.
But I think it brought up problem when i try to retype an in-used volume. After retyping succeeds, cinder volume will rename it back to previous name, this works fine until I shutdown and start vm. since block device mapping always remembers previous image name, it will try to connect to a non-existent volume which is problematic and causes boot failed.
Cinder version:
stable/rocky on ubuntu 16.04. the reported issue seems still exists in mainline.
Cinder configuration:
[ceph]
image_volume_cache_enabled = True
volume_clear = zero
rbd_max_clone_depth = 5
rbd_flatten_volume_from_snapshot = False
rbd_secret_uuid = 7de9ebf7-d0b6-4b08-97df-1a0eb3721c5a
rbd_user = cinder
rbd_pool = volumes
rbd_ceph_conf = /etc/ceph/ceph.conf
volume_driver = cinder.volume.drivers.rbd.RBDDriver
volume_backend_name = ceph
[test]
image_volume_cache_enabled = True
volume_clear = zero
rbd_max_clone_depth = 5
rbd_flatten_volume_from_snapshot = False
rbd_secret_uuid = 7de9ebf7-d0b6-4b08-97df-1a0eb3721c5a
rbd_user = cinder
rbd_pool = test
rbd_ceph_conf = /etc/ceph/ceph.conf
volume_driver = cinder.volume.drivers.rbd.RBDDriver
volume_backend_name = test
during retyping
root@sjtt-test:~# openstack volume list
+--------------------------------------+------+-----------+------+-----------------------------+
| ID | Name | Status | Size | Attached to |
+--------------------------------------+------+-----------+------+-----------------------------+
| fbb17466-46d6-4a8d-bd74-44bb259d6a4c | v3 | retyping | 1 | Attached to s1 on /dev/vdb |
| 49f43fe8-b514-4964-a153-850d7030f7b9 | v3 | attaching | 1 | |--------------+
after retype
root@sjtt-test:~# openstack volume list
+--------------------------------------+------+-----------+------+-----------------------------+
| ID | Name | Status | Size | Attached to |
+--------------------------------------+------+-----------+------+-----------------------------+
| fbb17466-46d6-4a8d-bd74-44bb259d6a4c | v3 | in-use | 1 | Attached to s1 on /dev/vdb |
root@sjtt-test:/etc/cinder# rbd ls test
volume-fbb17466-46d6-4a8d-bd74-44bb259d6a4c
mysql> select * from block_device_mapping where deleted = 0 and device_name='/dev/vdb' \G
*************************** 1. row ***************************
created_at: 2020-03-11 01:10:17
updated_at: 2020-03-11 01:11:38
deleted_at: NULL
id: 78
device_name: /dev/vdb
delete_on_termination: 0
snapshot_id: NULL
volume_id: fbb17466-46d6-4a8d-bd74-44bb259d6a4c
volume_size: 1
no_device: 0
connection_info: {"status": "reserved", "detached_at": "", "volume_id": "49f43fe8-b514-4964-a153-850d7030f7b9", "attach_mode": "null", "driver_volume_type": "rbd", "instance": "ca9b9311-17d2-4dbc-9054-a26296424543", "attached_at": "", "serial": "fbb17466-46d6-4a8d-bd74-44bb259d6a4c", "data": {"secret_type": "ceph", "name": "test/volume-49f43fe8-b514-4964-a153-850d7030f7b9", "encrypted": false, "discard": true, "keyring": null, "cluster_name": "ceph", "secret_uuid": "7de9ebf7-d0b6-4b08-97df-1a0eb3721c5a", "qos_specs": null, "auth_enabled": true, "volume_id": "49f43fe8-b514-4964-a153-850d7030f7b9", "hosts": ["10.110.31.164"], "access_mode": "rw", "auth_username": "cinder", "ports": ["6789"]}}
instance_uuid: ca9b9311-17d2-4dbc-9054-a26296424543
deleted: 0
source_type: volume
destination_type: volume
guest_format: NULL
device_type: NULL
disk_bus: NULL
boot_index: NULL
image_id: NULL
tag: NULL
attachment_id: 340eee36-d01e-48ab-ba9c-af72c9de6b5c
uuid: 8ad634a8-5437-4855-b8e4-984dcd92e449
1 row in set (0.00 sec)
apparently bdm does not change to new volume name.
and thus, shutdown and boot a vm failed with follow exception
Mar 11 09:17:29 sjtt-test nova-compute[31220]: ERROR oslo_messaging.rpc.server if ret == -1: raise libvirtError ('virDomainCreateWithFlags() failed', dom=self)
Mar 11 09:17:29 sjtt-test nova-compute[31220]: ERROR oslo_messaging.rpc.server libvirtError: internal error: process exited while connecting to monitor: 2020-03-11T01:17:28.423223Z qemu-system-x86_64: -drive file=rbd:test/volume-49f43fe8-b514-4964-a153-850d7030f7b9:id=cinder:auth_supported=cephx\;none:mon_host=10.110.31.164\:6789,file.password-secret=virtio-disk1-secret0,format=raw,if=none,id=drive-virtio-disk1,serial=fbb17466-46d6-4a8d-bd74-44bb259d6a4c,cache=writeback,discard=unmap: 'serial' is deprecated, please use the corresponding option of '-device' instead
...skipping...
Mar 11 09:17:29 sjtt-test nova-compute[31220]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/manager.py", line 206, in decorated_function
Mar 11 09:17:29 sjtt-test nova-compute[31220]: ERROR oslo_messaging.rpc.server return function(self, context, *args, **kwargs)
Mar 11 09:17:29 sjtt-test nova-compute[31220]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/manager.py", line 2908, in start_instance
Mar 11 09:17:29 sjtt-test nova-compute[31220]: ERROR oslo_messaging.rpc.server self._power_on(context, instance)
Mar 11 09:17:29 sjtt-test nova-compute[31220]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/manager.py", line 2878, in _power_on
Mar 11 09:17:29 sjtt-test nova-compute[31220]: ERROR oslo_messaging.rpc.server block_device_info)
Mar 11 09:17:29 sjtt-test nova-compute[31220]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/virt/libvirt/driver.py", line 3199, in power_on
Mar 11 09:17:29 sjtt-test nova-compute[31220]: ERROR oslo_messaging.rpc.server self._hard_reboot(context, instance, network_info, block_device_info)
Mar 11 09:17:29 sjtt-test nova-compute[31220]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/virt/libvirt/driver.py", line 3089, in _hard_reboot
Mar 11 09:17:29 sjtt-test nova-compute[31220]: ERROR oslo_messaging.rpc.server vifs_already_plugged=True)
Mar 11 09:17:29 sjtt-test nova-compute[31220]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/virt/libvirt/driver.py", line 5948, in _create_domain_and_network
Mar 11 09:17:29 sjtt-test nova-compute[31220]: ERROR oslo_messaging.rpc.server destroy_disks_on_failure)
Mar 11 09:17:29 sjtt-test nova-compute[31220]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python2.7/dist-packages/oslo_utils/excutils.py", line 220, in __exit__
Mar 11 09:17:29 sjtt-test nova-compute[31220]: ERROR oslo_messaging.rpc.server self.force_reraise()
Mar 11 09:17:29 sjtt-test nova-compute[31220]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python2.7/dist-packages/oslo_utils/excutils.py", line 196, in force_reraise
Mar 11 09:17:29 sjtt-test nova-compute[31220]: ERROR oslo_messaging.rpc.server six.reraise(self.type_, self.value, self.tb)
Mar 11 09:17:29 sjtt-test nova-compute[31220]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/virt/libvirt/driver.py", line 5917, in _create_domain_and_network
Mar 11 09:17:29 sjtt-test nova-compute[31220]: ERROR oslo_messaging.rpc.server post_xml_callback=post_xml_callback)
Mar 11 09:17:29 sjtt-test nova-compute[31220]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/virt/libvirt/driver.py", line 5851, in _create_domain
Mar 11 09:17:29 sjtt-test nova-compute[31220]: ERROR oslo_messaging.rpc.server guest.launch(pause=pause)
Mar 11 09:17:29 sjtt-test nova-compute[31220]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/virt/libvirt/guest.py", line 144, in launch
Mar 11 09:17:29 sjtt-test nova-compute[31220]: ERROR oslo_messaging.rpc.server self._encoded_xml, errors='ignore')
Mar 11 09:17:29 sjtt-test nova-compute[31220]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python2.7/dist-packages/oslo_utils/excutils.py", line 220, in __exit__
Mar 11 09:17:29 sjtt-test nova-compute[31220]: ERROR oslo_messaging.rpc.server self.force_reraise()
Mar 11 09:17:29 sjtt-test nova-compute[31220]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python2.7/dist-packages/oslo_utils/excutils.py", line 196, in force_reraise
Mar 11 09:17:29 sjtt-test nova-compute[31220]: ERROR oslo_messaging.rpc.server six.reraise(self.type_, self.value, self.tb)
Mar 11 09:17:29 sjtt-test nova-compute[31220]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/virt/libvirt/guest.py", line 139, in launch
Mar 11 09:17:29 sjtt-test nova-compute[31220]: ERROR oslo_messaging.rpc.server return self._domain.createWithFlags(flags)
Mar 11 09:17:29 sjtt-test nova-compute[31220]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python2.7/dist-packages/eventlet/tpool.py", line 186, in doit
Mar 11 09:17:29 sjtt-test nova-compute[31220]: ERROR oslo_messaging.rpc.server result = proxy_call(self._autowrap, f, *args, **kwargs)
Mar 11 09:17:29 sjtt-test nova-compute[31220]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python2.7/dist-packages/eventlet/tpool.py", line 144, in proxy_call
Mar 11 09:17:29 sjtt-test nova-compute[31220]: ERROR oslo_messaging.rpc.server rv = execute(f, *args, **kwargs)
Mar 11 09:17:29 sjtt-test nova-compute[31220]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python2.7/dist-packages/eventlet/tpool.py", line 125, in execute
Mar 11 09:17:29 sjtt-test nova-compute[31220]: ERROR oslo_messaging.rpc.server six.reraise(c, e, tb)
Mar 11 09:17:29 sjtt-test nova-compute[31220]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python2.7/dist-packages/eventlet/tpool.py", line 83, in tworker
Mar 11 09:17:29 sjtt-test nova-compute[31220]: ERROR oslo_messaging.rpc.server rv = meth(*args, **kwargs)
Mar 11 09:17:29 sjtt-test nova-compute[31220]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python2.7/dist-packages/libvirt.py", line 1098, in createWithFlags
Mar 11 09:17:29 sjtt-test nova-compute[31220]: ERROR oslo_messaging.rpc.server if ret == -1: raise libvirtError ('virDomainCreateWithFlags() failed', dom=self)
Mar 11 09:17:29 sjtt-test nova-compute[31220]: ERROR oslo_messaging.rpc.server libvirtError: internal error: process exited while connecting to monitor: 2020-03-11T01:17:28.423223Z qemu-system-x86_64: -drive file=rbd:test/volume-49f43fe8-b514-4964-a153-850d7030f7b9:id=cinder:auth_supported=cephx\;none:mon_host=10.110.31.164\:6789,file.password-secret=virtio-disk1-secret0,format=raw,if=none,id=drive-virtio-disk1,serial=fbb17466-46d6-4a8d-bd74-44bb259d6a4c,cache=writeback,discard=unmap: 'serial' is deprecated, please use the corresponding option of '-device' instead
Mar 11 09:17:29 sjtt-test nova-compute[31220]: ERROR oslo_messaging.rpc.server 2020-03-11T01:17:28.444643Z qemu-system-x86_64: -drive file=rbd:test/volume-49f43fe8-b514-4964-a153-850d7030f7b9:id=cinder:auth_supported=cephx\;none:mon_host=10.110.31.164\:6789,file.password-secret=virtio-disk1-secret0,format=raw,if=none,id=drive-virtio-disk1,serial=fbb17466-46d6-4a8d-bd74-44bb259d6a4c,cache=writeback,discard=unmap: error reading header from volume-49f43fe8-b514-4964-a153-850d7030f7b9: No such file or directory
Reviewed: https:/
Committed: https:/
Submitter: Zuul
Branch: master
commit 662677280beedeb
Author: ushen <email address hidden>
Date: Mon Mar 9 19:34:24 2020 +0800
Do not rename rbd based volume after migration
After rbd based volume migrated (retyped), it will be
renamed to the original name unless name collision or
some errors happen. But the problem is this information
seems not reflect on connection info, especially this volume
is already in-use.
This patch mimics behavior of lvm driver by not renaming
in-used volume after migration(retype)
Close-Bug: #1866935
Change-Id: I4db0dc978d55d4