After the following message in the compute log, the volume attachment was not cleaned up after attachment failure:
(req-b2ddac96-3e08-407d-803f-0c308943aa1d)
Traceback (most recent call last):, File "/usr/lib/python2.7/dist-packages/oslo_messaging/rpc/server.py", line 163, in _process_incoming, res = self.dispatcher.dispatch(message), File "/usr/lib/python2.7/dist-packages/oslo_messaging/rpc/dispatcher.py", line 220, in dispatch, return self._do_dispatch(endpoint, method, ctxt, args), File "/usr/lib/python2.7/dist-packages/oslo_messaging/rpc/dispatcher.py", line 190, in _do_dispatch, result = func(ctxt, **new_args), File "/usr/lib/python2.7/dist-packages/nova/exception_wrapper.py", line 76, in wrapped, function_name, call_dict, binary), File "/usr/lib/python2.7/dist-packages/oslo_utils/excutils.py", line 220, in __exit__, self.force_reraise(), File "/usr/lib/python2.7/dist-packages/oslo_utils/excutils.py", line 196, in force_reraise, six.reraise(self.type_, self.value, self.tb), File "/usr/lib/python2.7/dist-packages/nova/exception_wrapper.py", line 67, in wrapped, return f(self, context, *args, **kw), File "/usr/lib/python2.7/dist-packages/nova/compute/utils.py", line 976, in decorated_function, with EventReporter(context, event_name, instance_uuid):, File "/usr/lib/python2.7/dist-packages/nova/compute/utils.py", line 947, in __enter__, self.context, uuid, self.event_name, want_result=False), File "/usr/lib/python2.7/dist-packages/oslo_versionedobjects/base.py", line 177, in wrapper, args, kwargs), File "/usr/lib/python2.7/dist-packages/nova/conductor/rpcapi.py", line 240, in object_class_action_versions, args=args, kwargs=kwargs), File "/usr/lib/python2.7/dist-packages/oslo_messaging/rpc/client.py", line 174, in call, retry=self.retry), File "/usr/lib/python2.7/dist-packages/oslo_messaging/transport.py", line 131, in _send, timeout=timeout, retry=retry), File "/usr/lib/python2.7/dist-packages/oslo_messaging/_drivers/amqpdriver.py", line 559, in send, retry=retry), File "/usr/lib/python2.7/dist-packages/oslo_messaging/_drivers/amqpdriver.py", line 548, in _send, result = self._waiter.wait(msg_id, timeout), File "/usr/lib/python2.7/dist-packages/oslo_messaging/_drivers/amqpdriver.py", line 440, in wait, message = self.waiters.get(msg_id, timeout=timeout), File "/usr/lib/python2.7/dist-packages/oslo_messaging/_drivers/amqpdriver.py", line 328, in get, 'to message ID %s' % msg_id), MessagingTimeout: Timed out waiting for a reply to message ID 4f12e98ad4c4413887154a4b9101e52b
After the following message in the compute log, the volume attachment was not cleaned up after attachment failure:
(req-b2ddac96- 3e08-407d- 803f-0c308943aa 1d)
Traceback (most recent call last):, File "/usr/lib/ python2. 7/dist- packages/ oslo_messaging/ rpc/server. py", line 163, in _process_incoming, res = self.dispatcher .dispatch( message) , File "/usr/lib/ python2. 7/dist- packages/ oslo_messaging/ rpc/dispatcher. py", line 220, in dispatch, return self._do_ dispatch( endpoint, method, ctxt, args), File "/usr/lib/ python2. 7/dist- packages/ oslo_messaging/ rpc/dispatcher. py", line 190, in _do_dispatch, result = func(ctxt, **new_args), File "/usr/lib/ python2. 7/dist- packages/ nova/exception_ wrapper. py", line 76, in wrapped, function_name, call_dict, binary), File "/usr/lib/ python2. 7/dist- packages/ oslo_utils/ excutils. py", line 220, in __exit__, self.force_ reraise( ), File "/usr/lib/ python2. 7/dist- packages/ oslo_utils/ excutils. py", line 196, in force_reraise, six.reraise( self.type_ , self.value, self.tb), File "/usr/lib/ python2. 7/dist- packages/ nova/exception_ wrapper. py", line 67, in wrapped, return f(self, context, *args, **kw), File "/usr/lib/ python2. 7/dist- packages/ nova/compute/ utils.py" , line 976, in decorated_function, with EventReporter( context, event_name, instance_uuid):, File "/usr/lib/ python2. 7/dist- packages/ nova/compute/ utils.py" , line 947, in __enter__, self.context, uuid, self.event_name, want_result=False), File "/usr/lib/ python2. 7/dist- packages/ oslo_versionedo bjects/ base.py" , line 177, in wrapper, args, kwargs), File "/usr/lib/ python2. 7/dist- packages/ nova/conductor/ rpcapi. py", line 240, in object_ class_action_ versions, args=args, kwargs=kwargs), File "/usr/lib/ python2. 7/dist- packages/ oslo_messaging/ rpc/client. py", line 174, in call, retry=self.retry), File "/usr/lib/ python2. 7/dist- packages/ oslo_messaging/ transport. py", line 131, in _send, timeout=timeout, retry=retry), File "/usr/lib/ python2. 7/dist- packages/ oslo_messaging/ _drivers/ amqpdriver. py", line 559, in send, retry=retry), File "/usr/lib/ python2. 7/dist- packages/ oslo_messaging/ _drivers/ amqpdriver. py", line 548, in _send, result = self._waiter. wait(msg_ id, timeout), File "/usr/lib/ python2. 7/dist- packages/ oslo_messaging/ _drivers/ amqpdriver. py", line 440, in wait, message = self.waiters. get(msg_ id, timeout=timeout), File "/usr/lib/ python2. 7/dist- packages/ oslo_messaging/ _drivers/ amqpdriver. py", line 328, in get, 'to message ID %s' % msg_id), MessagingTimeout: Timed out waiting for a reply to message ID 4f12e98ad4c4413 887154a4b9101e5 2b
There was an issue with rabbit at that point, and it seems that it happened in the following decorator https:/ /github. com/openstack/ nova/blob/ b44b540fc70504f 3869ef230226420 95de0ea99e/ nova/compute/ manager. py#L6853 in its enter method. It might be worth moving attachment cleanup https:/ /github. com/openstack/ nova/blob/ b44b540fc70504f 3869ef230226420 95de0ea99e/ nova/compute/ manager. py#L6890- L6900 to API here https:/ /github. com/openstack/ nova/blob/ b44b540fc70504f 3869ef230226420 95de0ea99e/ nova/compute/ api.py# L4412