I added some code to cinder/cinder/volume/manager.py and reproduce it:
diff --git a/cinder/volume/manager.py b/cinder/volume/manager.py index d8e23c2..6331165 100644 --- a/cinder/volume/manager.py +++ b/cinder/volume/manager.py @@ -270,12 +270,16 @@ class VolumeManager(manager.SchedulerDependentManager): volume_ref['id'], {'status': 'error'}) return - except Exception: + except Exception as ex: # restore source volume status before reschedule if sourcevol_ref is not None: self.db.volume_update(context, sourcevol_ref['id'], {'status': sourcevol_ref['status']}) + print "==="*20 + print ex + print "==="*20 exc_info = sys.exc_info() + print exc_info # try to re-schedule volume: self._reschedule_or_reraise(context, volume_id, exc_info, snapshot_id, image_id,
error log:
============================================================ Unexpected error while running command. Command: sudo cinder-rootwrap /etc/cinder/rootwrap.conf dd if=/dev/zero of=/dev/mapper/stack--volumes-clone--snap--12b110c4--01f2--4221--a06f--3883cb3a3d0a count=1024 bs=1M conv=fdatasync Exit code: 1 Stdout: '' Stderr: "/bin/dd: fdatasync failed for `/dev/mapper/stack--volumes-clone--snap--12b110c4--01f2--4221--a06f--3883cb3a3d0a': Input/output error\n1024+0 records in\n1024+0 records out\n1073741824 bytes (1.1 GB) copied, 466.395 s, 2.3 MB/s\n" ============================================================ (None, None, None) 2013-07-04 21:07:23.759 ERROR cinder.volume.manager [req-781185b1-59ef-4d32-a3c5-1ff4c4f36769 d18561d834914ffba68fbb7a0c869ecf 827e49101c9b48e8998360d1e6abb346] volume volume-8419dd91-957b-4759-9ce4-78b08e5c0bd4: create failed 2013-07-04 21:07:23.760 ERROR cinder.openstack.common.rpc.amqp [req-781185b1-59ef-4d32-a3c5-1ff4c4f36769 d18561d834914ffba68fbb7a0c869ecf 827e49101c9b48e8998360d1e6abb346] Exception during message handling 2013-07-04 21:07:23.760 TRACE cinder.openstack.common.rpc.amqp Traceback (most recent call last): 2013-07-04 21:07:23.760 TRACE cinder.openstack.common.rpc.amqp File "/opt/stack/cinder/cinder/openstack/common/rpc/amqp.py", line 433, in _process_data 2013-07-04 21:07:23.760 TRACE cinder.openstack.common.rpc.amqp **args) 2013-07-04 21:07:23.760 TRACE cinder.openstack.common.rpc.amqp File "/opt/stack/cinder/cinder/openstack/common/rpc/dispatcher.py", line 148, in dispatch 2013-07-04 21:07:23.760 TRACE cinder.openstack.common.rpc.amqp return getattr(proxyobj, method)(ctxt, **kwargs) 2013-07-04 21:07:23.760 TRACE cinder.openstack.common.rpc.amqp File "/opt/stack/cinder/cinder/volume/manager.py", line 311, in create_volume 2013-07-04 21:07:23.760 TRACE cinder.openstack.common.rpc.amqp "create.end") 2013-07-04 21:07:23.760 TRACE cinder.openstack.common.rpc.amqp File "/usr/lib/python2.7/contextlib.py", line 24, in __exit__ 2013-07-04 21:07:23.760 TRACE cinder.openstack.common.rpc.amqp self.gen.next() 2013-07-04 21:07:23.760 TRACE cinder.openstack.common.rpc.amqp File "/opt/stack/cinder/cinder/volume/manager.py", line 287, in create_volume 2013-07-04 21:07:23.760 TRACE cinder.openstack.common.rpc.amqp allow_reschedule) 2013-07-04 21:07:23.760 TRACE cinder.openstack.common.rpc.amqp File "/opt/stack/cinder/cinder/volume/manager.py", line 370, in _reschedule_or_reraise 2013-07-04 21:07:23.760 TRACE cinder.openstack.common.rpc.amqp raise exc_info[0], exc_info[1], exc_info[2] 2013-07-04 21:07:23.760 TRACE cinder.openstack.common.rpc.amqp TypeError: exceptions must be old-style classes or derived from BaseException, not NoneType 2013-07-04 21:07:23.760 TRACE cinder.openstack.common.rpc.amqp 2013-07-04 21:08:06.381 DEBUG cinder.openstack.common.periodic_task [-] Running periodic task VolumeManager._publish_service_capabilities from (pid=5155) run_periodic_tasks /opt/stack/cinder/cinder/openstack/common/periodic_task.py:176
I added some code to cinder/ cinder/ volume/ manager. py and reproduce it:
diff --git a/cinder/ volume/ manager. py b/cinder/ volume/ manager. py volume/ manager. py volume/ manager. py manager. SchedulerDepend entManager) :
volume_ ref['id' ],
{'status' : 'error'})
return
self.db. volume_ update( context, sourcevol_ ref['id' ],
{'status' : sourcevol_ ref['status' ]})
exc_ info = sys.exc_info()
self. _reschedule_ or_reraise( context, volume_id, exc_info,
snapshot_ id, image_id,
index d8e23c2..6331165 100644
--- a/cinder/
+++ b/cinder/
@@ -270,12 +270,16 @@ class VolumeManager(
- except Exception:
+ except Exception as ex:
# restore source volume status before reschedule
if sourcevol_ref is not None:
+ print "==="*20
+ print ex
+ print "==="*20
+ print exc_info
# try to re-schedule volume:
error log:
======= ======= ======= ======= ======= ======= ======= ======= ==== rootwrap. conf dd if=/dev/zero of=/dev/ mapper/ stack-- volumes- clone-- snap--12b110c4- -01f2-- 4221--a06f- -3883cb3a3d0a count=1024 bs=1M conv=fdatasync stack-- volumes- clone-- snap--12b110c4- -01f2-- 4221--a06f- -3883cb3a3d0a' : Input/output error\n1024+0 records in\n1024+0 records out\n1073741824 bytes (1.1 GB) copied, 466.395 s, 2.3 MB/s\n" ======= ======= ======= ======= ======= ======= ======= ==== volume. manager [req-781185b1- 59ef-4d32- a3c5-1ff4c4f367 69 d18561d834914ff ba68fbb7a0c869e cf 827e49101c9b48e 8998360d1e6abb3 46] volume volume- 8419dd91- 957b-4759- 9ce4-78b08e5c0b d4: create failed openstack. common. rpc.amqp [req-781185b1- 59ef-4d32- a3c5-1ff4c4f367 69 d18561d834914ff ba68fbb7a0c869e cf 827e49101c9b48e 8998360d1e6abb3 46] Exception during message handling openstack. common. rpc.amqp Traceback (most recent call last): openstack. common. rpc.amqp File "/opt/stack/ cinder/ cinder/ openstack/ common/ rpc/amqp. py", line 433, in _process_data openstack. common. rpc.amqp **args) openstack. common. rpc.amqp File "/opt/stack/ cinder/ cinder/ openstack/ common/ rpc/dispatcher. py", line 148, in dispatch openstack. common. rpc.amqp return getattr(proxyobj, method)(ctxt, **kwargs) openstack. common. rpc.amqp File "/opt/stack/ cinder/ cinder/ volume/ manager. py", line 311, in create_volume openstack. common. rpc.amqp "create.end") openstack. common. rpc.amqp File "/usr/lib/ python2. 7/contextlib. py", line 24, in __exit__ openstack. common. rpc.amqp self.gen.next() openstack. common. rpc.amqp File "/opt/stack/ cinder/ cinder/ volume/ manager. py", line 287, in create_volume openstack. common. rpc.amqp allow_reschedule) openstack. common. rpc.amqp File "/opt/stack/ cinder/ cinder/ volume/ manager. py", line 370, in _reschedule_ or_reraise openstack. common. rpc.amqp raise exc_info[0], exc_info[1], exc_info[2] openstack. common. rpc.amqp TypeError: exceptions must be old-style classes or derived from BaseException, not NoneType openstack. common. rpc.amqp openstack. common. periodic_ task [-] Running periodic task VolumeManager. _publish_ service_ capabilities from (pid=5155) run_periodic_tasks /opt/stack/ cinder/ cinder/ openstack/ common/ periodic_ task.py: 176
Unexpected error while running command.
Command: sudo cinder-rootwrap /etc/cinder/
Exit code: 1
Stdout: ''
Stderr: "/bin/dd: fdatasync failed for `/dev/mapper/
=======
(None, None, None)
2013-07-04 21:07:23.759 ERROR cinder.
2013-07-04 21:07:23.760 ERROR cinder.
2013-07-04 21:07:23.760 TRACE cinder.
2013-07-04 21:07:23.760 TRACE cinder.
2013-07-04 21:07:23.760 TRACE cinder.
2013-07-04 21:07:23.760 TRACE cinder.
2013-07-04 21:07:23.760 TRACE cinder.
2013-07-04 21:07:23.760 TRACE cinder.
2013-07-04 21:07:23.760 TRACE cinder.
2013-07-04 21:07:23.760 TRACE cinder.
2013-07-04 21:07:23.760 TRACE cinder.
2013-07-04 21:07:23.760 TRACE cinder.
2013-07-04 21:07:23.760 TRACE cinder.
2013-07-04 21:07:23.760 TRACE cinder.
2013-07-04 21:07:23.760 TRACE cinder.
2013-07-04 21:07:23.760 TRACE cinder.
2013-07-04 21:07:23.760 TRACE cinder.
2013-07-04 21:08:06.381 DEBUG cinder.