Nova Pike,
I have nfs for cinder volumes and ephemeral disks, /var/lib/nova/instances is a share mounted on compute hosts
I created a boot-able volume, ran an instance (no config-drives, etc). Initiated a resize from micro1 to micro2:
+--------------------------------------+-----------+-------+------+-----------+-------+-----------+
| ID | Name | RAM | Disk | Ephemeral | VCPUs | Is Public |
+--------------------------------------+-----------+-------+------+-----------+-------+-----------+
| 80bd2ab5-d99d-4e0f-8ae9-26ee31534210 | m1.micro1 | 64 | 1 | 0 | 1 | True |
| 23a09d59-a182-4863-9ecc-935de6c01170 | m1.micro2 | 65 | 2 | 1 | 1 | True |
Before resize:
[root@operator ~]# ls -ltrah /var/nfs/*/*
-rw-r--r-- 1 root root 93 Oct 26 12:30 /var/nfs/nova/compute_nodes
-rw-rw-rw- 1 42427 42427 1.0G Oct 26 12:47 /var/nfs/cinder/volume-a7b6fb81-8143-4bb3-9216-eba1e5dc0433
/var/nfs/nova/locks:
total 8.0K
-rw-r--r-- 1 root root 0 Oct 25 18:35 nova-storage-registry-lock
-rw-r--r-- 1 root root 0 Oct 26 11:03 nova-2dc09a364cbe47628fc413a564bc87425971543f
-rw-r--r-- 1 root root 0 Oct 26 12:18 nova-ephemeral_1_0706d66
-rw-r--r-- 1 root root 0 Oct 26 12:18 nova-swap_1
drwxr-xr-x 2 root root 4.0K Oct 26 12:18 .
drwxrwxrwx 5 root root 4.0K Oct 26 12:47 ..
/var/nfs/nova/_base:
total 20M
-rw-r--r-- 1 42427 42427 40M Oct 26 11:49 2dc09a364cbe47628fc413a564bc87425971543f
-rw-r--r-- 1 42427 42427 1.0G Oct 26 12:18 ephemeral_1_0706d66
drwxr-xr-x 2 root root 4.0K Oct 26 12:18 .
-rw-r--r-- 1 42427 42427 1.0M Oct 26 12:18 swap_1
drwxrwxrwx 5 root root 4.0K Oct 26 12:47 ..
/var/nfs/nova/14dba039-bc6a-4726-a582-a1c6cb7fddf4:
total 28K
drwxrwxrwx 5 root root 4.0K Oct 26 12:47 ..
drwxr-xr-x 2 root root 4.0K Oct 26 12:47 .
-rw-r--r-- 1 42427 42427 17K Oct 26 12:47 console.log
After resize:
[root@operator ~]# ls -ltrah /var/nfs/*/*
-rw-r--r-- 1 root root 93 Oct 26 12:30 /var/nfs/nova/compute_nodes
-rw-rw-rw- 1 42427 42427 1.0G Oct 26 12:50 /var/nfs/cinder/volume-a7b6fb81-8143-4bb3-9216-eba1e5dc0433
/var/nfs/nova/locks:
total 8.0K
-rw-r--r-- 1 root root 0 Oct 25 18:35 nova-storage-registry-lock
-rw-r--r-- 1 root root 0 Oct 26 11:03 nova-2dc09a364cbe47628fc413a564bc87425971543f
-rw-r--r-- 1 root root 0 Oct 26 12:18 nova-ephemeral_1_0706d66
-rw-r--r-- 1 root root 0 Oct 26 12:18 nova-swap_1
drwxr-xr-x 2 root root 4.0K Oct 26 12:18 .
drwxrwxrwx 5 root root 4.0K Oct 26 12:49 ..
/var/nfs/nova/_base:
total 20M
-rw-r--r-- 1 42427 42427 40M Oct 26 11:49 2dc09a364cbe47628fc413a564bc87425971543f
drwxr-xr-x 2 root root 4.0K Oct 26 12:18 .
-rw-r--r-- 1 42427 42427 1.0G Oct 26 12:49 ephemeral_1_0706d66
-rw-r--r-- 1 42427 42427 1.0M Oct 26 12:49 swap_1
drwxrwxrwx 5 root root 4.0K Oct 26 12:49 ..
/var/nfs/nova/14dba039-bc6a-4726-a582-a1c6cb7fddf4:
total 420K
-rw-r--r-- 1 42427 42427 193K Oct 26 12:49 .nfs000000000038000e00000006
-rw-r--r-- 1 42427 42427 193K Oct 26 12:49 .nfs000000000038000f00000005
drwxrwxrwx 5 root root 4.0K Oct 26 12:49 ..
-rw-r--r-- 1 42427 42427 20K Oct 26 12:49 .nfs000000000038001200000004
drwxr-xr-x 2 root root 4.0K Oct 26 12:49 .
Error: Failed to perform requested operation on instance "test1", the instance has an error status: Please try again later [Error: [Errno 39] Directory not empty: '/var/lib/nova/instances/14dba039-bc6a-4726-a582-a1c6cb7fddf4'].
nova compute log on original host:
2017-10-26 12:49:50.267 1 ERROR nova.compute.manager [req-fff5cc2b-82a5-441b-a123-f0384f8db7e7 f19e34c10c0148509af54be1540a627a 62743901dffd4739be97e2d3a2deead7 - default default] [instance: 14dba039-bc6a-4726-a582-a1c6cb7fddf4] Setting instance vm_state to ERROR: OSError: [Errno 39] Directory not empty: '/var/lib/nova/instances/14dba039-bc6a-4726-a582-a1c6cb7fddf4'
2017-10-26 12:49:50.267 1 ERROR nova.compute.manager [instance: 14dba039-bc6a-4726-a582-a1c6cb7fddf4] Traceback (most recent call last):
2017-10-26 12:49:50.267 1 ERROR nova.compute.manager [instance: 14dba039-bc6a-4726-a582-a1c6cb7fddf4] File "/var/lib/kolla/venv/lib/python2.7/site-packages/nova/compute/manager.py", line 6886, in _error_out_instance_on_exception
2017-10-26 12:49:50.267 1 ERROR nova.compute.manager [instance: 14dba039-bc6a-4726-a582-a1c6cb7fddf4] yield
2017-10-26 12:49:50.267 1 ERROR nova.compute.manager [instance: 14dba039-bc6a-4726-a582-a1c6cb7fddf4] File "/var/lib/kolla/venv/lib/python2.7/site-packages/nova/compute/manager.py", line 3609, in _confirm_resize
2017-10-26 12:49:50.267 1 ERROR nova.compute.manager [instance: 14dba039-bc6a-4726-a582-a1c6cb7fddf4] network_info)
2017-10-26 12:49:50.267 1 ERROR nova.compute.manager [instance: 14dba039-bc6a-4726-a582-a1c6cb7fddf4] File "/var/lib/kolla/venv/lib/python2.7/site-packages/nova/virt/libvirt/driver.py", line 7727, in confirm_migration
2017-10-26 12:49:50.267 1 ERROR nova.compute.manager [instance: 14dba039-bc6a-4726-a582-a1c6cb7fddf4] self._cleanup_resize(instance, network_info)
2017-10-26 12:49:50.267 1 ERROR nova.compute.manager [instance: 14dba039-bc6a-4726-a582-a1c6cb7fddf4] File "/var/lib/kolla/venv/lib/python2.7/site-packages/nova/virt/libvirt/driver.py", line 1149, in _cleanup_resize
2017-10-26 12:49:50.267 1 ERROR nova.compute.manager [instance: 14dba039-bc6a-4726-a582-a1c6cb7fddf4] shutil.rmtree(inst_base)
2017-10-26 12:49:50.267 1 ERROR nova.compute.manager [instance: 14dba039-bc6a-4726-a582-a1c6cb7fddf4] File "/usr/lib64/python2.7/shutil.py", line 256, in rmtree
2017-10-26 12:49:50.267 1 ERROR nova.compute.manager [instance: 14dba039-bc6a-4726-a582-a1c6cb7fddf4] onerror(os.rmdir, path, sys.exc_info())
2017-10-26 12:49:50.267 1 ERROR nova.compute.manager [instance: 14dba039-bc6a-4726-a582-a1c6cb7fddf4] File "/usr/lib64/python2.7/shutil.py", line 254, in rmtree
2017-10-26 12:49:50.267 1 ERROR nova.compute.manager [instance: 14dba039-bc6a-4726-a582-a1c6cb7fddf4] os.rmdir(path)
2017-10-26 12:49:50.267 1 ERROR nova.compute.manager [instance: 14dba039-bc6a-4726-a582-a1c6cb7fddf4] OSError: [Errno 39] Directory not empty: '/var/lib/nova/instances/14dba039-bc6a-4726-a582-a1c6cb7fddf4'
2017-10-26 12:49:50.267 1 ERROR nova.compute.manager [instance: 14dba039-bc6a-4726-a582-a1c6cb7fddf4]
2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server [req-fff5cc2b-82a5-441b-a123-f0384f8db7e7 f19e34c10c0148509af54be1540a627a 62743901dffd4739be97e2d3a2deead7 - default default] Exception during message handling: OSError: [Errno 39] Directory not empty: '/var/lib/nova/instances/14dba039-bc6a-4726-a582-a1c6cb7fddf4'
2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server Traceback (most recent call last):
2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server File "/var/lib/kolla/venv/lib/python2.7/site-packages/oslo_messaging/rpc/server.py", line 160, in _process_incoming
2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server res = self.dispatcher.dispatch(message)
2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server File "/var/lib/kolla/venv/lib/python2.7/site-packages/oslo_messaging/rpc/dispatcher.py", line 213, in dispatch
2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server return self._do_dispatch(endpoint, method, ctxt, args)
2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server File "/var/lib/kolla/venv/lib/python2.7/site-packages/oslo_messaging/rpc/dispatcher.py", line 183, in _do_dispatch
2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server result = func(ctxt, **new_args)
2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server File "/var/lib/kolla/venv/lib/python2.7/site-packages/nova/exception_wrapper.py", line 76, in wrapped
2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server function_name, call_dict, binary)
2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server File "/var/lib/kolla/venv/lib/python2.7/site-packages/oslo_utils/excutils.py", line 220, in __exit__
2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server self.force_reraise()
2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server File "/var/lib/kolla/venv/lib/python2.7/site-packages/oslo_utils/excutils.py", line 196, in force_reraise
2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server six.reraise(self.type_, self.value, self.tb)
2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server File "/var/lib/kolla/venv/lib/python2.7/site-packages/nova/exception_wrapper.py", line 67, in wrapped
2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server return f(self, context, *args, **kw)
2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server File "/var/lib/kolla/venv/lib/python2.7/site-packages/nova/compute/utils.py", line 874, in decorated_function
2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server return function(self, context, *args, **kwargs)
2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server File "/var/lib/kolla/venv/lib/python2.7/site-packages/nova/compute/manager.py", line 217, in decorated_function
2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server kwargs['instance'], e, sys.exc_info())
2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server File "/var/lib/kolla/venv/lib/python2.7/site-packages/oslo_utils/excutils.py", line 220, in __exit__
2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server self.force_reraise()
2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server File "/var/lib/kolla/venv/lib/python2.7/site-packages/oslo_utils/excutils.py", line 196, in force_reraise
2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server six.reraise(self.type_, self.value, self.tb)
2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server File "/var/lib/kolla/venv/lib/python2.7/site-packages/nova/compute/manager.py", line 205, in decorated_function
2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server return function(self, context, *args, **kwargs)
2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server File "/var/lib/kolla/venv/lib/python2.7/site-packages/nova/compute/manager.py", line 3587, in confirm_resize
2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server do_confirm_resize(context, instance, migration.id)
2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server File "/var/lib/kolla/venv/lib/python2.7/site-packages/oslo_concurrency/lockutils.py", line 271, in inner
2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server return f(*args, **kwargs)
2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server File "/var/lib/kolla/venv/lib/python2.7/site-packages/nova/compute/manager.py", line 3585, in do_confirm_resize
2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server self._confirm_resize(context, instance, migration=migration)
2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server File "/var/lib/kolla/venv/lib/python2.7/site-packages/nova/compute/manager.py", line 3609, in _confirm_resize
2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server network_info)
2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server File "/var/lib/kolla/venv/lib/python2.7/site-packages/nova/virt/libvirt/driver.py", line 7727, in confirm_migration
2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server self._cleanup_resize(instance, network_info)
2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server File "/var/lib/kolla/venv/lib/python2.7/site-packages/nova/virt/libvirt/driver.py", line 1149, in _cleanup_resize
2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server shutil.rmtree(inst_base)
2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server File "/usr/lib64/python2.7/shutil.py", line 256, in rmtree
2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server onerror(os.rmdir, path, sys.exc_info())
2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server File "/usr/lib64/python2.7/shutil.py", line 254, in rmtree
2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server os.rmdir(path)
2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server OSError: [Errno 39] Directory not empty: '/var/lib/nova/instances/14dba039-bc6a-4726-a582-a1c6cb7fddf4'
2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server
2017-10-26 12:50:27.547 1 WARNING nova.virt.libvirt.driver [req-b6000967-10d8-4ad2-939e-44509ea6f900 - - - - -] Periodic task is updating the host stat, it is trying to get disk instance-00000005, but disk file was removed by concurrent operations such as resize.: OSError: [Errno 2] No such file or directory: '/var/lib/nova/instances/14dba039-bc6a-4726-a582-a1c6cb7fddf4/disk.local'
2017-10-26 12:50:28.048 1 WARNING nova.compute.resource_tracker [req-b6000967-10d8-4ad2-939e-44509ea6f900 - - - - -] [instance: 14dba039-bc6a-4726-a582-a1c6cb7fddf4] Instance not resizing, skipping migration.
The same thing works with cinder lvm backend quite well, also ephemeral instances can be resized from micro1 to micro2.
i tried similar test with volume backed instances that just made a bigger volume but also had config drive - the same errors