LXC with LVM image backend: fail to delete instance

[Qiu Yu]

Steps to reproduce
--
libvirt_type=lxc to choose LXC as hypervisor type
libvirt_images_type=lvm to choose LVM as imagebackend store

nova boot container1
nova boot container2
nova delete container1

Expected result
--
Nicely destroy container and remove LVM disk

Actual result
--
libvirt driver fail to remove LVM disk due to device mapper error: "device-mapper: remove ioctl on failed: Device or resource busy"

Logs
--
2012-12-18 14:01:46 ERROR nova.openstack.common.rpc.amqp [-] Exception during message handling
2012-12-18 14:01:46 TRACE nova.openstack.common.rpc.amqp Traceback (most recent call last):
2012-12-18 14:01:46 TRACE nova.openstack.common.rpc.amqp File "/home/stack/nova/nova/openstack/common/rpc/amqp.py", line 276, in _process_data
2012-12-18 14:01:46 TRACE nova.openstack.common.rpc.amqp rval = self.proxy.dispatch(ctxt, version, method, **args)
2012-12-18 14:01:46 TRACE nova.openstack.common.rpc.amqp File "/home/stack/nova/nova/openstack/common/rpc/dispatcher.py", line 145, in dispatch
2012-12-18 14:01:46 TRACE nova.openstack.common.rpc.amqp return getattr(proxyobj, method)(ctxt, **kwargs)
2012-12-18 14:01:46 TRACE nova.openstack.common.rpc.amqp File "/home/stack/nova/nova/exception.py", line 117, in wrapped
2012-12-18 14:01:46 TRACE nova.openstack.common.rpc.amqp temp_level, payload)
2012-12-18 14:01:46 TRACE nova.openstack.common.rpc.amqp File "/usr/lib64/python2.6/contextlib.py", line 23, in __exit__
2012-12-18 14:01:46 TRACE nova.openstack.common.rpc.amqp self.gen.next()
2012-12-18 14:01:46 TRACE nova.openstack.common.rpc.amqp File "/home/stack/nova/nova/exception.py", line 92, in wrapped
2012-12-18 14:01:46 TRACE nova.openstack.common.rpc.amqp return f(*args, **kw)
2012-12-18 14:01:46 TRACE nova.openstack.common.rpc.amqp File "/home/stack/nova/nova/compute/manager.py", line 196, in decorated_function
2012-12-18 14:01:46 TRACE nova.openstack.common.rpc.amqp kwargs['instance']['uuid'], e, sys.exc_info())
2012-12-18 14:01:46 TRACE nova.openstack.common.rpc.amqp File "/usr/lib64/python2.6/contextlib.py", line 23, in __exit__
2012-12-18 14:01:46 TRACE nova.openstack.common.rpc.amqp self.gen.next()
2012-12-18 14:01:46 TRACE nova.openstack.common.rpc.amqp File "/home/stack/nova/nova/compute/manager.py", line 190, in decorated_function
2012-12-18 14:01:46 TRACE nova.openstack.common.rpc.amqp return function(self, context, *args, **kwargs)
2012-12-18 14:01:46 TRACE nova.openstack.common.rpc.amqp File "/home/stack/nova/nova/compute/manager.py", line 945, in terminate_instance
2012-12-18 14:01:46 TRACE nova.openstack.common.rpc.amqp do_terminate_instance(instance)
2012-12-18 14:01:46 TRACE nova.openstack.common.rpc.amqp File "/home/stack/nova/nova/utils.py", line 760, in inner
2012-12-18 14:01:46 TRACE nova.openstack.common.rpc.amqp retval = f(*args, **kwargs)
2012-12-18 14:01:46 TRACE nova.openstack.common.rpc.amqp File "/home/stack/nova/nova/compute/manager.py", line 937, in do_terminate_instance
2012-12-18 14:01:46 TRACE nova.openstack.common.rpc.amqp self._delete_instance(context, instance)
2012-12-18 14:01:46 TRACE nova.openstack.common.rpc.amqp File "/home/stack/nova/nova/compute/manager.py", line 890, in _delete_instance
2012-12-18 14:01:46 TRACE nova.openstack.common.rpc.amqp self._shutdown_instance(context, instance)
2012-12-18 14:01:46 TRACE nova.openstack.common.rpc.amqp File "/home/stack/nova/nova/compute/manager.py", line 854, in _shutdown_instance
2012-12-18 14:01:46 TRACE nova.openstack.common.rpc.amqp block_device_info)
2012-12-18 14:01:46 TRACE nova.openstack.common.rpc.amqp File "/home/stack/nova/nova/virt/libvirt/driver.py", line 498, in destroy
2012-12-18 14:01:46 TRACE nova.openstack.common.rpc.amqp self._cleanup(instance, network_info, block_device_info)
2012-12-18 14:01:46 TRACE nova.openstack.common.rpc.amqp File "/home/stack/nova/nova/virt/libvirt/driver.py", line 574, in _cleanup
2012-12-18 14:01:46 TRACE nova.openstack.common.rpc.amqp self._cleanup_lvm(instance)
2012-12-18 14:01:46 TRACE nova.openstack.common.rpc.amqp File "/home/stack/nova/nova/virt/libvirt/driver.py", line 580, in _cleanup_lvm
2012-12-18 14:01:46 TRACE nova.openstack.common.rpc.amqp libvirt_utils.remove_logical_volumes(*disks)
2012-12-18 14:01:46 TRACE nova.openstack.common.rpc.amqp File "/home/stack/nova/nova/virt/libvirt/utils.py", line 156, in remove_logical_volumes
2012-12-18 14:01:46 TRACE nova.openstack.common.rpc.amqp execute(*lvremove, attempts=3, run_as_root=True)
2012-12-18 14:01:46 TRACE nova.openstack.common.rpc.amqp File "/home/stack/nova/nova/virt/libvirt/utils.py", line 53, in execute
2012-12-18 14:01:46 TRACE nova.openstack.common.rpc.amqp return utils.execute(*args, **kwargs)
2012-12-18 14:01:46 TRACE nova.openstack.common.rpc.amqp File "/home/stack/nova/nova/utils.py", line 206, in execute
2012-12-18 14:01:46 TRACE nova.openstack.common.rpc.amqp cmd=' '.join(cmd))
2012-12-18 14:01:46 TRACE nova.openstack.common.rpc.amqp ProcessExecutionError: Unexpected error while running command.
2012-12-18 14:01:46 TRACE nova.openstack.common.rpc.amqp Command: sudo /usr/bin/nova-rootwrap /etc/nova/rootwrap.conf lvremove -f /dev/xenvg/instance-00000589_disk
2012-12-18 14:01:46 TRACE nova.openstack.common.rpc.amqp Exit code: 5
2012-12-18 14:01:46 TRACE nova.openstack.common.rpc.amqp Stdout: ''
2012-12-18 14:01:46 TRACE nova.openstack.common.rpc.amqp Stderr: ' device-mapper: remove ioctl on failed: Device or resource busy\n device-mapper: remove ioctl on failed: Device or resource busy\n device-mapper: remove ioctl on failed: Device or resource busy\n device-mapper: remove ioctl on failed: Device or resource busy\n device-mapper: remove ioctl on failed: Device or resource busy\n device-mapper: remove ioctl on failed: Device or resource busy\n device-mapper: remove ioctl on failed: Device or resource busy\n device-mapper: remove ioctl on failed: Device or resource busy\n device-mapper: remove ioctl on failed: Device or resource busy\n device-mapper: remove ioctl on failed: Device or resource busy\n device-mapper: remove ioctl on failed: Device or resource busy\n device-mapper: remove ioctl on failed: Device or resource busy\n device-mapper: remove ioctl on failed: Device or resource busy\n device-mapper: remove ioctl on failed: Device or resource busy\n device-mapper: remove ioctl on failed: Device or resource busy\n device-mapper: remove ioctl on failed: Device or resource busy\n device-mapper: remove ioctl on failed: Device or resource busy\n device-mapper: remove ioctl on failed: Device or resource busy\n device-mapper: remove ioctl on failed: Device or resource busy\n device-mapper: remove ioctl on failed: Device or resource busy\n device-mapper: remove ioctl on failed: Device or resource busy\n device-mapper: remove ioctl on failed: Device or resource busy\n device-mapper: remove ioctl on failed: Device or resource busy\n device-mapper: remove ioctl on failed: Device or resource busy\n device-mapper: remove ioctl on failed: Device or resource busy\n Unable to deactivate lxcvg-instance--00000589_disk (253:5)\n Unable to deactivate logical volume "instance-00000589_disk"\n'

[OpenStack Infra (hudson-openstack)]

Fix proposed to branch: master
Review: https://review.openstack.org/18309

[OpenStack Infra (hudson-openstack)]

Reviewed: https://review.openstack.org/18309
Committed: http://github.com/openstack/nova/commit/5f697f64e5c445ba1b62c82d9167fd6b9c7256d2
Submitter: Jenkins
Branch: master

commit 5f697f64e5c445ba1b62c82d9167fd6b9c7256d2
Author: unicell <email address hidden>
Date: Mon Dec 24 18:08:56 2012 +0800

    Teardown container rootfs in host namespace for lxc

    Unlike other hypervisor to directly using image for VM, rootfs need to
    be mounted in setup_container() for lxc before launching the domain.

    However, this leads to several problems:

    * previously spawned container polluted later spawned container, which
      in turn causing lvm device refused to be removed, as bug descirbes
    * container rootfs not mounted after nova stop/start sequence

    This commit addresses above two issues.

    Fixes bug 1091639

    Change-Id: I718baa6e46cf07a5458a5b80d42139dd353ae51c

[Lawrance (jing)]

hi, i‘am the bug report of bug1093379(https://bugs.launchpad.net/nova/+bug/1093379)

today i try your patch in the folsom release(via apt-get),and append the following configration in "nova-compute.conf"

==

[DEFAULT]
libvirt_type=lxc
libvirt_images_type=lvm
libvirt_images_volume_group=cinder-volumes

==

now i can boot lxc instance succesfully,but i can't ping the fixed ip and i can see nothing when connecting to lxc instance via "virsh --connect lxc:/// console"

when i stop lxc instance ,the nova-compute.log will report error message

==

2013-02-19 17:08:27 2630 ERROR nova.openstack.common.rpc.amqp [-] Exception during message handling
2013-02-19 17:08:27 2630 TRACE nova.openstack.common.rpc.amqp Traceback (most recent call last):
2013-02-19 17:08:27 2630 TRACE nova.openstack.common.rpc.amqp File "/usr/lib/python2.7/dist-packages/nova/openstack/common/rpc/amqp.py", line 276, in _process_data
2013-02-19 17:08:27 2630 TRACE nova.openstack.common.rpc.amqp rval = self.proxy.dispatch(ctxt, version, method, **args)
2013-02-19 17:08:27 2630 TRACE nova.openstack.common.rpc.amqp File "/usr/lib/python2.7/dist-packages/nova/openstack/common/rpc/dispatcher.py", line 145, in dispatch
2013-02-19 17:08:27 2630 TRACE nova.openstack.common.rpc.amqp return getattr(proxyobj, method)(ctxt, **kwargs)
2013-02-19 17:08:27 2630 TRACE nova.openstack.common.rpc.amqp File "/usr/lib/python2.7/dist-packages/nova/exception.py", line 117, in wrapped
2013-02-19 17:08:27 2630 TRACE nova.openstack.common.rpc.amqp temp_level, payload)
2013-02-19 17:08:27 2630 TRACE nova.openstack.common.rpc.amqp File "/usr/lib/python2.7/contextlib.py", line 24, in __exit__
2013-02-19 17:08:27 2630 TRACE nova.openstack.common.rpc.amqp self.gen.next()
2013-02-19 17:08:27 2630 TRACE nova.openstack.common.rpc.amqp File "/usr/lib/python2.7/dist-packages/nova/exception.py", line 92, in wrapped
2013-02-19 17:08:27 2630 TRACE nova.openstack.common.rpc.amqp return f(*args, **kw)
2013-02-19 17:08:27 2630 TRACE nova.openstack.common.rpc.amqp File "/usr/lib/python2.7/dist-packages/nova/compute/manager.py", line 175, in decorated_function
2013-02-19 17:08:27 2630 TRACE nova.openstack.common.rpc.amqp pass
2013-02-19 17:08:27 2630 TRACE nova.openstack.common.rpc.amqp File "/usr/lib/python2.7/contextlib.py", line 24, in __exit__
2013-02-19 17:08:27 2630 TRACE nova.openstack.common.rpc.amqp self.gen.next()
2013-02-19 17:08:27 2630 TRACE nova.openstack.common.rpc.amqp File "/usr/lib/python2.7/dist-packages/nova/compute/manager.py", line 161, in decorated_function
2013-02-19 17:08:27 2630 TRACE nova.openstack.common.rpc.amqp return function(self, context, *args, **kwargs)
2013-02-19 17:08:27 2630 TRACE nova.openstack.common.rpc.amqp File "/usr/lib/python2.7/dist-packages/nova/compute/manager.py", line 196, in decorated_function
2013-02-19 17:08:27 2630 TRACE nova.openstack.common.rpc.amqp kwargs['instance']['uuid'], e, sys.exc_info())
2013-02-19 17:08:27 2630 TRACE nova.openstack.common.rpc.amqp File "/usr/lib/python2.7/contextlib.py", line 24, in __exit__
2013-02-19 17:08:27 2630 TRACE nova.openstack.common.rpc.amqp self.gen.next()
2013-02-19 17:...