I don't think we have found root cause for this issue since I still hit it as well after confirming I have applied the patch https://review.openstack.org/#/c/49210/ # grep ".clear" -B 5 /usr/lib/python2.6/site-packages/neutronclient/client.py raise exceptions.ConnectionFailed(reason=e) finally: # Temporary Fix for gate failures. RPC calls and HTTP requests # seem to be stepping on each other resulting in bogus fd's being # picked up for making http requests self.connections.clear() the following error log I collected by applying my bellow patch shows that we are still hitting the error "Second simultaneous read on fileno" described by the link http://blog.eventlet.net/2010/03/18/safety/ diff --git a/neutronclient/v2_0/client.py b/neutronclient/v2_0/client.py index 733f339..4d95a84 100644 --- a/neutronclient/v2_0/client.py +++ b/neutronclient/v2_0/client.py @@ -1219,10 +1219,14 @@ class Client(object): try: return self.do_request(method, action, body=body, headers=headers, params=params) - except exceptions.ConnectionFailed: - # Exception has already been logged by do_request() + except exceptions.ConnectionFailed as e: + # Actually exception has not been logged by do_request(), + # we do it here temporarily to locate the problem + _logger.error("************ zhhuabj begin **************") + _logger.error(e) + _logger.error("************ zhhuabj end **************") if i < self.retries: - _logger.debug(_('Retrying connection to Neutron service')) + _logger.error(_('Retrying connection to Neutron service')) time.sleep(self.retry_interval) raise exceptions.ConnectionFailed(reason=_("Maximum attempts reached")) 2013-11-17 03:42:01.436 11335 ERROR neutronclient.v2_0.client [-] ************ zhhuabj begin ************** 2013-11-17 03:42:01.436 11335 ERROR neutronclient.v2_0.client [-] Connection to neutron failed: Second simultaneous read on fileno 42 detected. Unless you really know what you're doing, make sure that only one greenthread can read any particular socket. Consider using a pools.Pool. If you do know what you're doing and want to disable this error, call eventlet.debug.hub_prevent_multiple_readers(False) 2013-11-17 03:42:01.436 11335 ERROR neutronclient.v2_0.client [-] ************ zhhuabj end ************** 2013-11-17 03:42:01.437 11335 ERROR nova.api.openstack [req-fbe8ae49-9dc0-4c62-aea0-164e8a11160c 328702c5a15a4d14a49959f6ed5ec4b0 68ad27cec264496fbc0ddd2bde659216] NV-A68A08C Caught error: Connection to neutron failed: Maximum attempts reached 2013-11-17 03:42:01.437 11335 TRACE nova.api.openstack Traceback (most recent call last): 2013-11-17 03:42:01.437 11335 TRACE nova.api.openstack File "/usr/lib/python2.6/site-packages/nova/api/openstack/__init__.py", line 119, in __call__ 2013-11-17 03:42:01.437 11335 TRACE nova.api.openstack return req.get_response(self.application) 2013-11-17 03:42:01.437 11335 TRACE nova.api.openstack File "/usr/lib/python2.6/site-packages/webob/request.py", line 1296, in send 2013-11-17 03:42:01.437 11335 TRACE nova.api.openstack application, catch_exc_info=False) 2013-11-17 03:42:01.437 11335 TRACE nova.api.openstack File "/usr/lib/python2.6/site-packages/webob/request.py", line 1260, in call_application 2013-11-17 03:42:01.437 11335 TRACE nova.api.openstack app_iter = application(self.environ, start_response) 2013-11-17 03:42:01.437 11335 TRACE nova.api.openstack File "/usr/lib/python2.6/site-packages/webob/dec.py", line 144, in __call__ 2013-11-17 03:42:01.437 11335 TRACE nova.api.openstack return resp(environ, start_response) 2013-11-17 03:42:01.437 11335 TRACE nova.api.openstack File "/usr/lib/python2.6/site-packages/keystoneclient/middleware/auth_token.py", line 571, in __call__ 2013-11-17 03:42:01.437 11335 TRACE nova.api.openstack return self.app(env, start_response) 2013-11-17 03:42:01.437 11335 TRACE nova.api.openstack File "/usr/lib/python2.6/site-packages/webob/dec.py", line 144, in __call__ 2013-11-17 03:42:01.437 11335 TRACE nova.api.openstack return resp(environ, start_response) 2013-11-17 03:42:01.437 11335 TRACE nova.api.openstack File "/usr/lib/python2.6/site-packages/webob/dec.py", line 144, in __call__ 2013-11-17 03:42:01.437 11335 TRACE nova.api.openstack return resp(environ, start_response) 2013-11-17 03:42:01.437 11335 TRACE nova.api.openstack File "/usr/lib/python2.6/site-packages/routes/middleware.py", line 131, in __call__ 2013-11-17 03:42:01.437 11335 TRACE nova.api.openstack response = self.app(environ, start_response) 2013-11-17 03:42:01.437 11335 TRACE nova.api.openstack File "/usr/lib/python2.6/site-packages/webob/dec.py", line 144, in __call__ 2013-11-17 03:42:01.437 11335 TRACE nova.api.openstack return resp(environ, start_response) 2013-11-17 03:42:01.437 11335 TRACE nova.api.openstack File "/usr/lib/python2.6/site-packages/webob/dec.py", line 130, in __call__ 2013-11-17 03:42:01.437 11335 TRACE nova.api.openstack resp = self.call_func(req, *args, **self.kwargs) 2013-11-17 03:42:01.437 11335 TRACE nova.api.openstack File "/usr/lib/python2.6/site-packages/webob/dec.py", line 195, in call_func 2013-11-17 03:42:01.437 11335 TRACE nova.api.openstack return self.func(req, *args, **kwargs) 2013-11-17 03:42:01.437 11335 TRACE nova.api.openstack File "/usr/lib/python2.6/site-packages/nova/api/openstack/wsgi.py", line 917, in __call__ 2013-11-17 03:42:01.437 11335 TRACE nova.api.openstack content_type, body, accept) 2013-11-17 03:42:01.437 11335 TRACE nova.api.openstack File "/usr/lib/python2.6/site-packages/nova/api/openstack/wsgi.py", line 1002, in _process_stack 2013-11-17 03:42:01.437 11335 TRACE nova.api.openstack request, action_args) 2013-11-17 03:42:01.437 11335 TRACE nova.api.openstack File "/usr/lib/python2.6/site-packages/nova/api/openstack/wsgi.py", line 890, in post_process_extensions 2013-11-17 03:42:01.437 11335 TRACE nova.api.openstack **action_args) 2013-11-17 03:42:01.437 11335 TRACE nova.api.openstack File "/usr/lib/python2.6/site-packages/nova/api/openstack/compute/contrib/security_groups.py", line 572, in show 2013-11-17 03:42:01.437 11335 TRACE nova.api.openstack return self._show(req, resp_obj) 2013-11-17 03:42:01.437 11335 TRACE nova.api.openstack File "/usr/lib/python2.6/site-packages/nova/api/openstack/compute/contrib/security_groups.py", line 568, in _show 2013-11-17 03:42:01.437 11335 TRACE nova.api.openstack self._extend_servers(req, [resp_obj.obj['server']]) 2013-11-17 03:42:01.437 11335 TRACE nova.api.openstack File "/usr/lib/python2.6/site-packages/nova/api/openstack/compute/contrib/security_groups.py", line 527, in _extend_servers 2013-11-17 03:42:01.437 11335 TRACE nova.api.openstack servers[0]['id'])) 2013-11-17 03:42:01.437 11335 TRACE nova.api.openstack File "/usr/lib/python2.6/site-packages/nova/network/security_group/neutron_driver.py", line 321, in get_instance_security_groups 2013-11-17 03:42:01.437 11335 TRACE nova.api.openstack ports = neutron.list_ports(**params) 2013-11-17 03:42:01.437 11335 TRACE nova.api.openstack File "/usr/lib/python2.6/site-packages/neutronclient/v2_0/client.py", line 108, in with_params 2013-11-17 03:42:01.437 11335 TRACE nova.api.openstack ret = self.function(instance, *args, **kwargs) 2013-11-17 03:42:01.437 11335 TRACE nova.api.openstack File "/usr/lib/python2.6/site-packages/neutronclient/v2_0/client.py", line 298, in list_ports 2013-11-17 03:42:01.437 11335 TRACE nova.api.openstack **_params) 2013-11-17 03:42:01.437 11335 TRACE nova.api.openstack File "/usr/lib/python2.6/site-packages/neutronclient/v2_0/client.py", line 1201, in list 2013-11-17 03:42:01.437 11335 TRACE nova.api.openstack for r in self._pagination(collection, path, **params): 2013-11-17 03:42:01.437 11335 TRACE nova.api.openstack File "/usr/lib/python2.6/site-packages/neutronclient/v2_0/client.py", line 1214, in _pagination 2013-11-17 03:42:01.437 11335 TRACE nova.api.openstack res = self.get(path, params=params) 2013-11-17 03:42:01.437 11335 TRACE nova.api.openstack File "/usr/lib/python2.6/site-packages/neutronclient/v2_0/client.py", line 1187, in get 2013-11-17 03:42:01.437 11335 TRACE nova.api.openstack headers=headers, params=params) 2013-11-17 03:42:01.437 11335 TRACE nova.api.openstack File "/usr/lib/python2.6/site-packages/neutronclient/v2_0/client.py", line 1179, in retry_request 2013-11-17 03:42:01.437 11335 TRACE nova.api.openstack raise exceptions.ConnectionFailed(reason=_("Maximum attempts reached")) 2013-11-17 03:42:01.437 11335 TRACE nova.api.openstack ConnectionFailed: Connection to neutron failed: Maximum attempts reached 2013-11-17 03:42:01.437 11335 TRACE nova.api.openstack