ERROR nova.api.openstack [req-b249d867-efa9-4b69-bb00-bcd188d35d95 f6aafe318df1469498437c69e9f20206 677e5e66567a4d1ebe0b1946452a03ef] Ca
TRACE nova.api.openstack Traceback (most recent call last):
TRACE nova.api.openstack File "/opt/stack/venvs/nova/lib/python2.7/site-packages/nova/api/openstack/__init__.py", line 120, in __call_
TRACE nova.api.openstack return req.get_response(self.application)
TRACE nova.api.openstack File "/usr/lib/python2.7/site-packages/webob/request.py", line 1296, in send
TRACE nova.api.openstack application, catch_exc_info=False)
TRACE nova.api.openstack File "/usr/lib/python2.7/site-packages/webob/request.py", line 1260, in call_application
TRACE nova.api.openstack app_iter = application(self.environ, start_response)
TRACE nova.api.openstack File "/usr/lib/python2.7/site-packages/webob/dec.py", line 144, in __call__
TRACE nova.api.openstack return resp(environ, start_response)
TRACE nova.api.openstack File "/usr/lib/python2.7/site-packages/keystoneclient/middleware/auth_token.py", line 571, in __call__
TRACE nova.api.openstack return self.app(env, start_response)
TRACE nova.api.openstack File "/usr/lib/python2.7/site-packages/webob/dec.py", line 144, in __call__
TRACE nova.api.openstack return resp(environ, start_response)
TRACE nova.api.openstack File "/usr/lib/python2.7/site-packages/webob/dec.py", line 144, in __call__
TRACE nova.api.openstack return resp(environ, start_response)
TRACE nova.api.openstack File "/usr/lib/python2.7/site-packages/routes/middleware.py", line 131, in __call__
TRACE nova.api.openstack response = self.app(environ, start_response)
TRACE nova.api.openstack File "/usr/lib/python2.7/site-packages/webob/dec.py", line 144, in __call__
TRACE nova.api.openstack return resp(environ, start_response)
TRACE nova.api.openstack File "/usr/lib/python2.7/site-packages/webob/dec.py", line 130, in __call__
TRACE nova.api.openstack resp = self.call_func(req, *args, **self.kwargs)
TRACE nova.api.openstack File "/usr/lib/python2.7/site-packages/webob/dec.py", line 195, in call_func
TRACE nova.api.openstack return self.func(req, *args, **kwargs)
TRACE nova.api.openstack File "/opt/stack/venvs/nova/lib/python2.7/site-packages/nova/api/openstack/wsgi.py", line 940, in __call__
TRACE nova.api.openstack content_type, body, accept)
TRACE nova.api.openstack File "/opt/stack/venvs/nova/lib/python2.7/site-packages/nova/api/openstack/wsgi.py", line 1025, in _process_s
TRACE nova.api.openstack request, action_args)
TRACE nova.api.openstack File "/opt/stack/venvs/nova/lib/python2.7/site-packages/nova/api/openstack/wsgi.py", line 913, in post_proces
TRACE nova.api.openstack **action_args)
TRACE nova.api.openstack File "/opt/stack/venvs/nova/lib/python2.7/site-packages/nova/api/openstack/compute/contrib/security_groups.py
TRACE nova.api.openstack return self._show(req, resp_obj)
TRACE nova.api.openstack File "/opt/stack/venvs/nova/lib/python2.7/site-packages/nova/api/openstack/compute/contrib/security_groups.py
TRACE nova.api.openstack self._extend_servers(req, [resp_obj.obj['server']])
TRACE nova.api.openstack File "/opt/stack/venvs/nova/lib/python2.7/site-packages/nova/api/openstack/compute/contrib/security_groups.py
TRACE nova.api.openstack servers[0]['id']))
TRACE nova.api.openstack File "/opt/stack/venvs/nova/lib/python2.7/site-packages/nova/network/security_group/neutron_driver.py", line
TRACE nova.api.openstack ports = neutron.list_ports(**params)
TRACE nova.api.openstack File "/opt/stack/venvs/nova/lib/python2.7/site-packages/neutronclient/v2_0/client.py", line 108, in with_para
TRACE nova.api.openstack ret = self.function(instance, *args, **kwargs)
TRACE nova.api.openstack File "/opt/stack/venvs/nova/lib/python2.7/site-packages/neutronclient/v2_0/client.py", line 298, in list_port
TRACE nova.api.openstack **_params)
TRACE nova.api.openstack File "/opt/stack/venvs/nova/lib/python2.7/site-packages/neutronclient/v2_0/client.py", line 1197, in list
TRACE nova.api.openstack for r in self._pagination(collection, path, **params):
TRACE nova.api.openstack File "/opt/stack/venvs/nova/lib/python2.7/site-packages/neutronclient/v2_0/client.py", line 1210, in _paginat
TRACE nova.api.openstack res = self.get(path, params=params)
TRACE nova.api.openstack File "/opt/stack/venvs/nova/lib/python2.7/site-packages/neutronclient/v2_0/client.py", line 1183, in get
TRACE nova.api.openstack headers=headers, params=params)
TRACE nova.api.openstack File "/opt/stack/venvs/nova/lib/python2.7/site-packages/neutronclient/v2_0/client.py", line 1175, in retry_re
TRACE nova.api.openstack raise exceptions.ConnectionFailed(reason=_("Maximum attempts reached"))
TRACE nova.api.openstack ConnectionFailed: Connection to neutron failed: Maximum attempts reached
TRACE nova.api.openstack
I looks like http requests/responses over persistent tcp connections to neutron are happening out of order
See the first two http requests attached (from same TCP connection)
I'm seeing something similar on Tripleo/Fedora
ERROR nova.api.openstack [req-b249d867- efa9-4b69- bb00-bcd188d35d 95 f6aafe318df1469 498437c69e9f202 06 677e5e66567a4d1 ebe0b1946452a03 ef] Ca venvs/nova/ lib/python2. 7/site- packages/ nova/api/ openstack/ __init_ _.py", line 120, in __call_ response( self.applicatio n) python2. 7/site- packages/ webob/request. py", line 1296, in send info=False) python2. 7/site- packages/ webob/request. py", line 1260, in call_application self.environ, start_response) python2. 7/site- packages/ webob/dec. py", line 144, in __call__ python2. 7/site- packages/ keystoneclient/ middleware/ auth_token. py", line 571, in __call__ python2. 7/site- packages/ webob/dec. py", line 144, in __call__ python2. 7/site- packages/ webob/dec. py", line 144, in __call__ python2. 7/site- packages/ routes/ middleware. py", line 131, in __call__ python2. 7/site- packages/ webob/dec. py", line 144, in __call__ python2. 7/site- packages/ webob/dec. py", line 130, in __call__ python2. 7/site- packages/ webob/dec. py", line 195, in call_func venvs/nova/ lib/python2. 7/site- packages/ nova/api/ openstack/ wsgi.py" , line 940, in __call__ venvs/nova/ lib/python2. 7/site- packages/ nova/api/ openstack/ wsgi.py" , line 1025, in _process_s venvs/nova/ lib/python2. 7/site- packages/ nova/api/ openstack/ wsgi.py" , line 913, in post_proces venvs/nova/ lib/python2. 7/site- packages/ nova/api/ openstack/ compute/ contrib/ security_ groups. py venvs/nova/ lib/python2. 7/site- packages/ nova/api/ openstack/ compute/ contrib/ security_ groups. py servers( req, [resp_obj. obj['server' ]]) venvs/nova/ lib/python2. 7/site- packages/ nova/api/ openstack/ compute/ contrib/ security_ groups. py venvs/nova/ lib/python2. 7/site- packages/ nova/network/ security_ group/neutron_ driver. py", line list_ports( **params) venvs/nova/ lib/python2. 7/site- packages/ neutronclient/ v2_0/client. py", line 108, in with_para instance, *args, **kwargs) venvs/nova/ lib/python2. 7/site- packages/ neutronclient/ v2_0/client. py", line 298, in list_port venvs/nova/ lib/python2. 7/site- packages/ neutronclient/ v2_0/client. py", line 1197, in list n(collection, path, **params): venvs/nova/ lib/python2. 7/site- packages/ neutronclient/ v2_0/client. py", line 1210, in _paginat venvs/nova/ lib/python2. 7/site- packages/ neutronclient/ v2_0/client. py", line 1183, in get venvs/nova/ lib/python2. 7/site- packages/ neutronclient/ v2_0/client. py", line 1175, in retry_re ConnectionFaile d(reason= _("Maximum attempts reached"))
TRACE nova.api.openstack Traceback (most recent call last):
TRACE nova.api.openstack File "/opt/stack/
TRACE nova.api.openstack return req.get_
TRACE nova.api.openstack File "/usr/lib/
TRACE nova.api.openstack application, catch_exc_
TRACE nova.api.openstack File "/usr/lib/
TRACE nova.api.openstack app_iter = application(
TRACE nova.api.openstack File "/usr/lib/
TRACE nova.api.openstack return resp(environ, start_response)
TRACE nova.api.openstack File "/usr/lib/
TRACE nova.api.openstack return self.app(env, start_response)
TRACE nova.api.openstack File "/usr/lib/
TRACE nova.api.openstack return resp(environ, start_response)
TRACE nova.api.openstack File "/usr/lib/
TRACE nova.api.openstack return resp(environ, start_response)
TRACE nova.api.openstack File "/usr/lib/
TRACE nova.api.openstack response = self.app(environ, start_response)
TRACE nova.api.openstack File "/usr/lib/
TRACE nova.api.openstack return resp(environ, start_response)
TRACE nova.api.openstack File "/usr/lib/
TRACE nova.api.openstack resp = self.call_func(req, *args, **self.kwargs)
TRACE nova.api.openstack File "/usr/lib/
TRACE nova.api.openstack return self.func(req, *args, **kwargs)
TRACE nova.api.openstack File "/opt/stack/
TRACE nova.api.openstack content_type, body, accept)
TRACE nova.api.openstack File "/opt/stack/
TRACE nova.api.openstack request, action_args)
TRACE nova.api.openstack File "/opt/stack/
TRACE nova.api.openstack **action_args)
TRACE nova.api.openstack File "/opt/stack/
TRACE nova.api.openstack return self._show(req, resp_obj)
TRACE nova.api.openstack File "/opt/stack/
TRACE nova.api.openstack self._extend_
TRACE nova.api.openstack File "/opt/stack/
TRACE nova.api.openstack servers[0]['id']))
TRACE nova.api.openstack File "/opt/stack/
TRACE nova.api.openstack ports = neutron.
TRACE nova.api.openstack File "/opt/stack/
TRACE nova.api.openstack ret = self.function(
TRACE nova.api.openstack File "/opt/stack/
TRACE nova.api.openstack **_params)
TRACE nova.api.openstack File "/opt/stack/
TRACE nova.api.openstack for r in self._paginatio
TRACE nova.api.openstack File "/opt/stack/
TRACE nova.api.openstack res = self.get(path, params=params)
TRACE nova.api.openstack File "/opt/stack/
TRACE nova.api.openstack headers=headers, params=params)
TRACE nova.api.openstack File "/opt/stack/
TRACE nova.api.openstack raise exceptions.
TRACE nova.api.openstack ConnectionFailed: Connection to neutron failed: Maximum attempts reached
TRACE nova.api.openstack
I looks like http requests/responses over persistent tcp connections to neutron are happening out of order
See the first two http requests attached (from same TCP connection)