We are getting these in the nova-api logs for a few things so this could be related. We aren't getting anything in the cinder-api logs though and we are seeing the exact same issue there. (just less frequent as the api is used less) 2014-09-05 08:36:09.025 17883 TRACE nova.api.openstack Traceback (most recent call last): 2014-09-05 08:36:09.025 17883 TRACE nova.api.openstack File "/usr/lib/python2.7/dist-packages/nova/api/openstack/__init__.py", line 125, in __call__ 2014-09-05 08:36:09.025 17883 TRACE nova.api.openstack return req.get_response(self.application) 2014-09-05 08:36:09.025 17883 TRACE nova.api.openstack File "/usr/lib/python2.7/dist-packages/webob/request.py", line 1320, in send 2014-09-05 08:36:09.025 17883 TRACE nova.api.openstack application, catch_exc_info=False) 2014-09-05 08:36:09.025 17883 TRACE nova.api.openstack File "/usr/lib/python2.7/dist-packages/webob/request.py", line 1284, in call_application 2014-09-05 08:36:09.025 17883 TRACE nova.api.openstack app_iter = application(self.environ, start_response) 2014-09-05 08:36:09.025 17883 TRACE nova.api.openstack File "/usr/lib/python2.7/dist-packages/webob/dec.py", line 144, in __call__ 2014-09-05 08:36:09.025 17883 TRACE nova.api.openstack return resp(environ, start_response) 2014-09-05 08:36:09.025 17883 TRACE nova.api.openstack File "/usr/lib/python2.7/dist-packages/keystoneclient/middleware/auth_token.py", line 582, in __call__ 2014-09-05 08:36:09.025 17883 TRACE nova.api.openstack return self.app(env, start_response) 2014-09-05 08:36:09.025 17883 TRACE nova.api.openstack File "/usr/lib/python2.7/dist-packages/webob/dec.py", line 144, in __call__ 2014-09-05 08:36:09.025 17883 TRACE nova.api.openstack return resp(environ, start_response) 2014-09-05 08:36:09.025 17883 TRACE nova.api.openstack File "/usr/lib/python2.7/dist-packages/webob/dec.py", line 144, in __call__ 2014-09-05 08:36:09.025 17883 TRACE nova.api.openstack return resp(environ, start_response) 2014-09-05 08:36:09.025 17883 TRACE nova.api.openstack File "/usr/lib/python2.7/dist-packages/routes/middleware.py", line 131, in __call__ 2014-09-05 08:36:09.025 17883 TRACE nova.api.openstack response = self.app(environ, start_response) 2014-09-05 08:36:09.025 17883 TRACE nova.api.openstack File "/usr/lib/python2.7/dist-packages/webob/dec.py", line 144, in __call__ 2014-09-05 08:36:09.025 17883 TRACE nova.api.openstack return resp(environ, start_response) 2014-09-05 08:36:09.025 17883 TRACE nova.api.openstack File "/usr/lib/python2.7/dist-packages/webob/dec.py", line 130, in __call__ 2014-09-05 08:36:09.025 17883 TRACE nova.api.openstack resp = self.call_func(req, *args, **self.kwargs) 2014-09-05 08:36:09.025 17883 TRACE nova.api.openstack File "/usr/lib/python2.7/dist-packages/webob/dec.py", line 195, in call_func 2014-09-05 08:36:09.025 17883 TRACE nova.api.openstack return self.func(req, *args, **kwargs) 2014-09-05 08:36:09.025 17883 TRACE nova.api.openstack File "/usr/lib/python2.7/dist-packages/nova/api/openstack/wsgi.py", line 917, in __call__ 2014-09-05 08:36:09.025 17883 TRACE nova.api.openstack content_type, body, accept) 2014-09-05 08:36:09.025 17883 TRACE nova.api.openstack File "/usr/lib/python2.7/dist-packages/nova/api/openstack/wsgi.py", line 983, in _process_stack 2014-09-05 08:36:09.025 17883 TRACE nova.api.openstack action_result = self.dispatch(meth, request, action_args) 2014-09-05 08:36:09.025 17883 TRACE nova.api.openstack File "/usr/lib/python2.7/dist-packages/nova/api/openstack/wsgi.py", line 1070, in dispatch 2014-09-05 08:36:09.025 17883 TRACE nova.api.openstack return method(req=request, **action_args) 2014-09-05 08:36:09.025 17883 TRACE nova.api.openstack File "/usr/lib/python2.7/dist-packages/nova/api/openstack/compute/contrib/console_output.py", line 67, in get_console_output 2014-09-05 08:36:09.025 17883 TRACE nova.api.openstack length) 2014-09-05 08:36:09.025 17883 TRACE nova.api.openstack File "/usr/lib/python2.7/dist-packages/nova/compute/api.py", line 218, in _wrapped 2014-09-05 08:36:09.025 17883 TRACE nova.api.openstack return fn(self, context, instance, *args, **kwargs) 2014-09-05 08:36:09.025 17883 TRACE nova.api.openstack File "/usr/lib/python2.7/dist-packages/nova/compute/cells_api.py", line 430, in get_console_output 2014-09-05 08:36:09.025 17883 TRACE nova.api.openstack *args, **kwargs) 2014-09-05 08:36:09.025 17883 TRACE nova.api.openstack File "/usr/lib/python2.7/dist-packages/nova/compute/cells_api.py", line 201, in _call_to_cells 2014-09-05 08:36:09.025 17883 TRACE nova.api.openstack method, instance_uuid, *args, **kwargs) 2014-09-05 08:36:09.025 17883 TRACE nova.api.openstack File "/usr/lib/python2.7/dist-packages/nova/cells/rpcapi.py", line 130, in call_compute_api_method 2014-09-05 08:36:09.025 17883 TRACE nova.api.openstack call=True) 2014-09-05 08:36:09.025 17883 TRACE nova.api.openstack File "/usr/lib/python2.7/dist-packages/oslo/messaging/rpc/client.py", line 361, in call 2014-09-05 08:36:09.025 17883 TRACE nova.api.openstack return self.prepare().call(ctxt, method, **kwargs) 2014-09-05 08:36:09.025 17883 TRACE nova.api.openstack File "/usr/lib/python2.7/dist-packages/oslo/messaging/rpc/client.py", line 150, in call 2014-09-05 08:36:09.025 17883 TRACE nova.api.openstack wait_for_reply=True, timeout=timeout) 2014-09-05 08:36:09.025 17883 TRACE nova.api.openstack File "/usr/lib/python2.7/dist-packages/oslo/messaging/transport.py", line 90, in _send 2014-09-05 08:36:09.025 17883 TRACE nova.api.openstack timeout=timeout) 2014-09-05 08:36:09.025 17883 TRACE nova.api.openstack File "/usr/lib/python2.7/dist-packages/oslo/messaging/_drivers/amqpdriver.py", line 412, in send 2014-09-05 08:36:09.025 17883 TRACE nova.api.openstack return self._send(target, ctxt, message, wait_for_reply, timeout) 2014-09-05 08:36:09.025 17883 TRACE nova.api.openstack File "/usr/lib/python2.7/dist-packages/oslo/messaging/_drivers/amqpdriver.py", line 403, in _send 2014-09-05 08:36:09.025 17883 TRACE nova.api.openstack result = self._waiter.wait(msg_id, timeout) 2014-09-05 08:36:09.025 17883 TRACE nova.api.openstack File "/usr/lib/python2.7/dist-packages/oslo/messaging/_drivers/amqpdriver.py", line 267, in wait 2014-09-05 08:36:09.025 17883 TRACE nova.api.openstack reply, ending = self._poll_connection(msg_id, timeout) 2014-09-05 08:36:09.025 17883 TRACE nova.api.openstack File "/usr/lib/python2.7/dist-packages/oslo/messaging/_drivers/amqpdriver.py", line 217, in _poll_connection 2014-09-05 08:36:09.025 17883 TRACE nova.api.openstack % msg_id) 2014-09-05 08:36:09.025 17883 TRACE nova.api.openstack MessagingTimeout: Timed out waiting for a reply to message ID e913d74ca8dc43a29f1d3143e235f853 2014-09-05 08:36:09.025 17883 TRACE nova.api.openstack Strace of a cinder-api process that is stuck: Process 1577 attached gettimeofday({1409871217, 128631}, NULL) = 0 epoll_wait(7, {{EPOLLIN, {u32=8, u64=39432335262744584}}}, 1023, 60000) = 1 epoll_ctl(7, EPOLL_CTL_DEL, 8, {EPOLLRDNORM|EPOLLWRNORM|EPOLLHUP|EPOLLRDHUP|EPOLLET|0xfa74800, {u32=32669, u64=22396489217114013}}) = 0 accept(8, 0x7fffbfc161d0, [16]) = -1 EAGAIN (Resource temporarily unavailable) epoll_ctl(7, EPOLL_CTL_ADD, 8, {EPOLLIN|EPOLLPRI|EPOLLERR|EPOLLHUP, {u32=8, u64=39432335262744584}}) = 0 gettimeofday({1409871218, 462312}, NULL) = 0 epoll_wait(7, {{EPOLLIN, {u32=8, u64=39432335262744584}}}, 1023, 60000) = 1 epoll_ctl(7, EPOLL_CTL_DEL, 8, {EPOLLRDNORM|EPOLLWRNORM|EPOLLHUP|EPOLLRDHUP|EPOLLET|0xfa74800, {u32=32669, u64=22396489217114013}}) = 0 accept(8, 0x7fffbfc161d0, [16]) = -1 EAGAIN (Resource temporarily unavailable) epoll_ctl(7, EPOLL_CTL_ADD, 8, {EPOLLIN|EPOLLPRI|EPOLLERR|EPOLLHUP, {u32=8, u64=39432335262744584}}) = 0 gettimeofday({1409871218, 820144}, NULL) = 0 epoll_wait(7, {{EPOLLIN, {u32=8, u64=39432335262744584}}}, 1023, 60000) = 1 epoll_ctl(7, EPOLL_CTL_DEL, 8, {EPOLLRDNORM|EPOLLWRNORM|EPOLLHUP|EPOLLRDHUP|EPOLLET|0xfa74800, {u32=32669, u64=22396489217114013}}) = 0 accept(8, 0x7fffbfc161d0, [16]) = -1 EAGAIN (Resource temporarily unavailable) epoll_ctl(7, EPOLL_CTL_ADD, 8, {EPOLLIN|EPOLLPRI|EPOLLERR|EPOLLHUP, {u32=8, u64=39432335262744584}}) = 0 gettimeofday({1409871219, 592145}, NULL) = 0 epoll_wait(7, {{EPOLLIN, {u32=8, u64=39432335262744584}}}, 1023, 60000) = 1 epoll_ctl(7, EPOLL_CTL_DEL, 8, {EPOLLRDNORM|EPOLLWRNORM|EPOLLHUP|EPOLLRDHUP|EPOLLET|0xfa74800, {u32=32669, u64=22396489217114013}}) = 0 accept(8, 0x7fffbfc161d0, [16]) = -1 EAGAIN (Resource temporarily unavailable) epoll_ctl(7, EPOLL_CTL_ADD, 8, {EPOLLIN|EPOLLPRI|EPOLLERR|EPOLLHUP, {u32=8, u64=39432335262744584}}) = 0 gettimeofday({1409871221, 750442}, NULL) = 0 epoll_wait(7, {{EPOLLIN, {u32=8, u64=39432335262744584}}}, 1023, 60000) = 1 epoll_ctl(7, EPOLL_CTL_DEL, 8, {EPOLLRDNORM|EPOLLWRNORM|EPOLLHUP|EPOLLRDHUP|EPOLLET|0xfa74800, {u32=32669, u64=22396489217114013}}) = 0 accept(8, 0x7fffbfc161d0, [16]) = -1 EAGAIN (Resource temporarily unavailable) epoll_ctl(7, EPOLL_CTL_ADD, 8, {EPOLLIN|EPOLLPRI|EPOLLERR|EPOLLHUP, {u32=8, u64=39432335262744584}}) = 0 gettimeofday({1409871223, 508851}, NULL) = 0 epoll_wait(7, {{EPOLLIN, {u32=8, u64=39432335262744584}}}, 1023, 60000) = 1 epoll_ctl(7, EPOLL_CTL_DEL, 8, {EPOLLRDNORM|EPOLLWRNORM|EPOLLHUP|EPOLLRDHUP|EPOLLET|0xfa74800, {u32=32669, u64=22396489217114013}}) = 0 accept(8, 0x7fffbfc161d0, [16]) = -1 EAGAIN (Resource temporarily unavailable) epoll_ctl(7, EPOLL_CTL_ADD, 8, {EPOLLIN|EPOLLPRI|EPOLLERR|EPOLLHUP, {u32=8, u64=39432335262744584}}) = 0 gettimeofday({1409871223, 940830}, NULL) = 0 epoll_wait(7, {{EPOLLIN, {u32=8, u64=39432335262744584}}}, 1023, 60000) = 1 epoll_ctl(7, EPOLL_CTL_DEL, 8, {EPOLLRDNORM|EPOLLWRNORM|EPOLLHUP|EPOLLRDHUP|EPOLLET|0xfa74800, {u32=32669, u64=22396489217114013}}) = 0 accept(8, 0x7fffbfc161d0, [16]) = -1 EAGAIN (Resource temporarily unavailable) epoll_ctl(7, EPOLL_CTL_ADD, 8, {EPOLLIN|EPOLLPRI|EPOLLERR|EPOLLHUP, {u32=8, u64=39432335262744584}}) = 0 gettimeofday({1409871226, 755337}, NULL) = 0 epoll_wait(7, ^CProcess 1577 detached