we are hitting this testing all versions of openstack with the latest charms:
during rally tests we get several occurrences of this
+----------------------------------------------------------------------------------------------------------------------------+
| Response Times (sec) |
+-------------------------+-----------+--------------+--------------+--------------+-----------+-----------+---------+-------+
| Action | Min (sec) | Median (sec) | 90%ile (sec) | 95%ile (sec) | Max (sec) | Avg (sec) | Success | Count |
+-------------------------+-----------+--------------+--------------+--------------+-----------+-----------+---------+-------+
| cinder_v3.create_volume | 15.842 | 16.277 | 16.741 | 16.975 | 17.21 | 16.328 | 100.0% | 10 |
| nova.boot_server | 6.773 | 7.443 | 9.774 | 9.822 | 9.871 | 8.017 | 100.0% | 10 |
| nova.resize | 37.668 | 54.583 | 58.466 | 58.799 | 59.133 | 53.598 | 90.0% | 10 |
| nova.resize_confirm | 2.472 | 2.529 | 2.587 | 2.595 | 2.604 | 2.527 | 100.0% | 9 |
| nova.delete_server | 2.46 | 2.583 | 2.719 | 2.781 | 2.843 | 2.598 | 100.0% | 9 |
| total | 69.552 | 83.33 | 87.167 | 88.235 | 89.302 | 82.555 | 90.0% | 10 |
| -> duration | 68.552 | 82.33 | 86.167 | 87.235 | 88.302 | 81.555 | 90.0% | 10 |
| -> idle_duration | 1.0 | 1.0 | 1.0 | 1.0 | 1.0 | 1.0 | 90.0% | 10 |
+-------------------------+-----------+--------------+--------------+--------------+-----------+-----------+---------+-------+
GetResourceFailure: Failed to get the resource <Server: s_rally_33ab096b_DKnPoVNb>: Unable to establish connection to http://nova.production.solutionsqa:8774/v2.1/servers/0a0a423b-d748-4d84-ac88-6d3d85892df0: ('Connection aborted.', RemoteDisconnected('Remote end closed connection without response',))
Traceback (most recent call last):
File "/snap/fcbtest/14/lib/python3.6/site-packages/urllib3/connectionpool.py", line 677, in urlopen
chunked=chunked,
File "/snap/fcbtest/14/lib/python3.6/site-packages/urllib3/connectionpool.py", line 426, in _make_request
six.raise_from(e, None)
File "<string>", line 3, in raise_from
File "/snap/fcbtest/14/lib/python3.6/site-packages/urllib3/connectionpool.py", line 421, in _make_request
httplib_response = conn.getresponse()
File "/snap/fcbtest/14/usr/lib/python3.6/http/client.py", line 1356, in getresponse
response.begin()
File "/snap/fcbtest/14/usr/lib/python3.6/http/client.py", line 307, in begin
version, status, reason = self._read_status()
File "/snap/fcbtest/14/usr/lib/python3.6/http/client.py", line 276, in _read_status
raise RemoteDisconnected("Remote end closed connection without"
http.client.RemoteDisconnected: Remote end closed connection without response
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/snap/fcbtest/14/lib/python3.6/site-packages/requests/adapters.py", line 449, in send
timeout=timeout
File "/snap/fcbtest/14/lib/python3.6/site-packages/urllib3/connectionpool.py", line 725, in urlopen
method, url, error=e, _pool=self, _stacktrace=sys.exc_info()[2]
File "/snap/fcbtest/14/lib/python3.6/site-packages/urllib3/util/retry.py", line 403, in increment
raise six.reraise(type(error), error, _stacktrace)
File "/snap/fcbtest/14/lib/python3.6/site-packages/urllib3/packages/six.py", line 734, in reraise
raise value.with_traceback(tb)
File "/snap/fcbtest/14/lib/python3.6/site-packages/urllib3/connectionpool.py", line 677, in urlopen
chunked=chunked,
File "/snap/fcbtest/14/lib/python3.6/site-packages/urllib3/connectionpool.py", line 426, in _make_request
six.raise_from(e, None)
File "<string>", line 3, in raise_from
File "/snap/fcbtest/14/lib/python3.6/site-packages/urllib3/connectionpool.py", line 421, in _make_request
httplib_response = conn.getresponse()
File "/snap/fcbtest/14/usr/lib/python3.6/http/client.py", line 1356, in getresponse
response.begin()
File "/snap/fcbtest/14/usr/lib/python3.6/http/client.py", line 307, in begin
version, status, reason = self._read_status()
File "/snap/fcbtest/14/usr/lib/python3.6/http/client.py", line 276, in _read_status
raise RemoteDisconnected("Remote end closed connection without"
urllib3.exceptions.ProtocolError: ('Connection aborted.', RemoteDisconnected('Remote end closed connection without response',))
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/snap/fcbtest/14/lib/python3.6/site-packages/keystoneauth1/session.py", line 1004, in _send_request
resp = self.session.request(method, url, **kwargs)
File "/snap/fcbtest/14/lib/python3.6/site-packages/requests/sessions.py", line 530, in request
resp = self.send(prep, **send_kwargs)
File "/snap/fcbtest/14/lib/python3.6/site-packages/requests/sessions.py", line 643, in send
r = adapter.send(request, **kwargs)
File "/snap/fcbtest/14/lib/python3.6/site-packages/requests/adapters.py", line 498, in send
raise ConnectionError(err, request=request)
requests.exceptions.ConnectionError: ('Connection aborted.', RemoteDisconnected('Remote end closed connection without response',))
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/snap/fcbtest/14/lib/python3.6/site-packages/rally/task/utils.py", line 74, in _get_from_manager
res = resource.manager.get(getattr(resource, id_attr))
File "/snap/fcbtest/14/lib/python3.6/site-packages/novaclient/v2/servers.py", line 846, in get
return self._get("/servers/%s" % base.getid(server), "server")
File "/snap/fcbtest/14/lib/python3.6/site-packages/novaclient/base.py", line 352, in _get
resp, body = self.api.client.get(url)
File "/snap/fcbtest/14/lib/python3.6/site-packages/keystoneauth1/adapter.py", line 386, in get
return self.request(url, 'GET', **kwargs)
File "/snap/fcbtest/14/lib/python3.6/site-packages/novaclient/client.py", line 72, in request
**kwargs)
File "/snap/fcbtest/14/lib/python3.6/site-packages/keystoneauth1/adapter.py", line 545, in request
resp = super(LegacyJsonAdapter, self).request(*args, **kwargs)
File "/snap/fcbtest/14/lib/python3.6/site-packages/keystoneauth1/adapter.py", line 248, in request
return self.session.request(url, method, **kwargs)
File "/snap/fcbtest/14/lib/python3.6/site-packages/keystoneauth1/session.py", line 913, in request
resp = send(**kwargs)
File "/snap/fcbtest/14/lib/python3.6/site-packages/keystoneauth1/session.py", line 1020, in _send_request
raise exceptions.ConnectFailure(msg)
keystoneauth1.exceptions.connection.ConnectFailure: Unable to establish connection to http://nova.production.solutionsqa:8774/v2.1/servers/0a0a423b-d748-4d84-ac88-6d3d85892df0: ('Connection aborted.', RemoteDisconnected('Remote end closed connection without response',))
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/snap/fcbtest/14/lib/python3.6/site-packages/rally/task/runner.py", line 69, in _run_scenario_once
getattr(scenario_inst, method_name)(**scenario_kwargs)
File "/snap/fcbtest/14/lib/python3.6/site-packages/rally_openstack/task/scenarios/nova/servers.py", line 598, in run
self._resize(server, to_flavor)
File "/snap/fcbtest/14/lib/python3.6/site-packages/rally/task/atomic.py", line 91, in func_atomic_actions
f = func(self, *args, **kwargs)
File "/snap/fcbtest/14/lib/python3.6/site-packages/rally_openstack/task/scenarios/nova/utils.py", line 699, in _resize
check_interval=CONF.openstack.nova_server_resize_poll_interval
File "/snap/fcbtest/14/lib/python3.6/site-packages/rally/task/utils.py", line 213, in wait_for_status
resource = update_resource(resource)
File "/snap/fcbtest/14/lib/python3.6/site-packages/rally/task/utils.py", line 78, in _get_from_manager
raise exceptions.GetResourceFailure(resource=resource, err=e)
rally.exceptions.GetResourceFailure: Failed to get the resource <Server: s_rally_33ab096b_DKnPoVNb>: Unable to establish connection to http://nova.production.solutionsqa:8774/v2.1/servers/0a0a423b-d748-4d84-ac88-6d3d85892df0: ('Connection aborted.', RemoteDisconnected('Remote end closed connection without response',))
Example:
https://solutions.qa.canonical.com/qa/testRun/cfcd4c89-5819-4249-918f-057860f5259b
Sub'd to field high