seen in https://zuul.opendev.org/t/openstack/buildset/c3bb550c079341fd86f4c0791eca0232
https://zuul.opendev.org/t/openstack/build/a65a4112e4e44b84afd6a40b395e2b28
```
___________________ TestECObjController.test_GET_disconnect ____________________
self = <test.unit.proxy.controllers.test_obj.TestECObjController testMethod=test_GET_disconnect>
def test_GET_disconnect(self):
self.app.recoverable_node_timeout = 0.01
self.app.client_timeout = 0.1
# Before, we used the default 64k chunk size, so the entire ~16k test
# data would come in the first chunk, and the generator should
# cleanly exit by the time we reiterate() the response.
self.app.object_chunk_size = 10
segment_size = self.policy.ec_segment_size
test_data = (b'test' * segment_size)[:-743]
etag = md5(test_data, usedforsecurity=False).hexdigest()
ec_archive_bodies = self._make_ec_archive_bodies(test_data)
headers = {
'X-Object-Sysmeta-Ec-Etag': etag,
'X-Object-Sysmeta-Ec-Content-Length': len(test_data),
'X-Timestamp': Timestamp(self.ts()).normal,
}
num_slow = 4
responses = [
(200, SlowBody(body, 0.1 if i < num_slow else 0.0),
self._add_frag_index(i, headers))
for i, body in enumerate(ec_archive_bodies)
] * self.policy.ec_duplication_factor
status_codes, body_iter, headers = zip(*responses)
req = swift.common.swob.Request.blank('/v1/a/c/o')
> with mocked_http_conn(*status_codes, body_iter=body_iter,
headers=headers) as log:
/home/zuul/src/opendev.org/openstack/swift/test/unit/proxy/controllers/test_obj.py:2577:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
/usr/lib/python3.10/contextlib.py:142: in __exit__
next(self.gen)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
args = (200, 200, 200, 200, 200, 200, ...)
kwargs = {'body_iter': (<test.unit.SlowBody object at 0x7f07ebe3edd0>, <test.unit.SlowBody object at 0x7f07ebe3f7f0>, <test.uni...ag': '29392154c37580c51c1dc5b8131194d2', 'X-Object-Sysmeta-Ec-Frag-Index': 5, 'X-Timestamp': '1691513675.00000'}, ...)}
responses = [<test.unit.fake_http_connect.<locals>.FakeConn object at 0x7f07ebe3e590>, <test.unit.fake_http_connect.<locals>.FakeC...als>.FakeConn object at 0x7f0855532110>, <test.unit.fake_http_connect.<locals>.FakeConn object at 0x7f07ebf8d450>, ...]
capture_requests = <function mocked_http_conn.<locals>.capture_requests at 0x7f07e0ffb9a0>
fake_conn = <function fake_http_connect.<locals>.connect at 0x7f07e0ffb490>
left_over_status = []
@contextmanager
def mocked_http_conn(*args, **kwargs):
requests = []
responses = []
def capture_requests(ip, port, method, path, headers, qs, ssl):
if six.PY2 and not isinstance(ip, bytes):
ip = ip.encode('ascii')
req = {
'ip': ip,
'port': port,
'method': method,
'path': path,
'headers': headers,
'qs': qs,
'ssl': ssl,
}
requests.append(req)
kwargs.setdefault('give_connect', capture_requests)
kwargs['capture_connections'] = responses
fake_conn = fake_http_connect(*args, **kwargs)
fake_conn.requests = requests
fake_conn.responses = responses
with mocklib.patch('swift.common.bufferedhttp.http_connect_raw',
new=fake_conn):
yield fake_conn
left_over_status = list(fake_conn.code_iter)
if left_over_status:
raise AssertionError('left over status %r' % left_over_status)
if fake_conn.unexpected_requests:
> raise AssertionError('unexpected requests:\n%s' % '\n '.join(
'%r' % (req,) for req in fake_conn.unexpected_requests))
E AssertionError: unexpected requests:
E (('10.0.0.14', 1014, 'GET', '/sda/0/a/c/o', {'X-Backend-Storage-Policy-Index': '0', 'X-Backend-Etag-Is-At': 'X-Object-Sysmeta-Ec-Etag', 'Host': 'localhost:80', 'X-Trans-Id': 'txc00ea049dc0b478483148-0064d2734b', 'X-Timestamp': '1691513675.29225', 'Connection': 'close', 'User-Agent': 'proxy-server 8132', 'Referer': 'GET http://localhost/v1/a/c/o', 'Range': 'bytes=0-1885', 'X-Backend-Fragment-Preferences': '[{"timestamp": "0000000001.00000", "exclude": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]}]'}, '', False), {})
E (('10.0.0.15', 1015, 'GET', '/sda/0/a/c/o', {'X-Backend-Storage-Policy-Index': '0', 'X-Backend-Etag-Is-At': 'X-Object-Sysmeta-Ec-Etag', 'Host': 'localhost:80', 'X-Trans-Id': 'txc00ea049dc0b478483148-0064d2734b', 'X-Timestamp': '1691513675.29225', 'Connection': 'close', 'User-Agent': 'proxy-server 8132', 'Referer': 'GET http://localhost/v1/a/c/o', 'Range': 'bytes=0-1885', 'X-Backend-Fragment-Preferences': '[{"timestamp": "0000000001.00000", "exclude": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]}]'}, '', False), {})
E (('10.0.0.16', 1016, 'GET', '/sda/0/a/c/o', {'X-Backend-Storage-Policy-Index': '0', 'X-Backend-Etag-Is-At': 'X-Object-Sysmeta-Ec-Etag', 'Host': 'localhost:80', 'X-Trans-Id': 'txc00ea049dc0b478483148-0064d2734b', 'X-Timestamp': '1691513675.29225', 'Connection': 'close', 'User-Agent': 'proxy-server 8132', 'Referer': 'GET http://localhost/v1/a/c/o', 'Range': 'bytes=0-1885', 'X-Backend-Fragment-Preferences': '[{"timestamp": "0000000001.00000", "exclude": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]}]'}, '', False), {})
E (('10.0.0.17', 1017, 'GET', '/sda/0/a/c/o', {'X-Backend-Storage-Policy-Index': '0', 'X-Backend-Etag-Is-At': 'X-Object-Sysmeta-Ec-Etag', 'Host': 'localhost:80', 'X-Trans-Id': 'txc00ea049dc0b478483148-0064d2734b', 'X-Timestamp': '1691513675.29225', 'Connection': 'close', 'User-Agent': 'proxy-server 8132', 'Referer': 'GET http://localhost/v1/a/c/o', 'Range': 'bytes=0-1885', 'X-Backend-Fragment-Preferences': '[{"timestamp": "0000000001.00000", "exclude": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]}]'}, '', False), {})
E (('10.0.0.18', 1018, 'GET', '/sda/0/a/c/o', {'X-Backend-Storage-Policy-Index': '0', 'X-Backend-Etag-Is-At': 'X-Object-Sysmeta-Ec-Etag', 'Host': 'localhost:80', 'X-Trans-Id': 'txc00ea049dc0b478483148-0064d2734b', 'X-Timestamp': '1691513675.29225', 'Connection': 'close', 'User-Agent': 'proxy-server 8132', 'Referer': 'GET http://localhost/v1/a/c/o', 'Range': 'bytes=0-1885', 'X-Backend-Fragment-Preferences': '[{"timestamp": "0000000001.00000", "exclude": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]}]'}, '', False), {})
E (('10.0.0.19', 1019, 'GET', '/sda/0/a/c/o', {'X-Backend-Storage-Policy-Index': '0', 'X-Backend-Etag-Is-At': 'X-Object-Sysmeta-Ec-Etag', 'Host': 'localhost:80', 'X-Trans-Id': 'txc00ea049dc0b478483148-0064d2734b', 'X-Timestamp': '1691513675.29225', 'Connection': 'close', 'User-Agent': 'proxy-server 8132', 'Referer': 'GET http://localhost/v1/a/c/o', 'Range': 'bytes=0-1885', 'X-Backend-Fragment-Preferences': '[{"timestamp": "0000000001.00000", "exclude": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]}]'}, '', False), {})
E (('10.0.0.20', 1020, 'GET', '/sda/0/a/c/o', {'X-Backend-Storage-Policy-Index': '0', 'X-Backend-Etag-Is-At': 'X-Object-Sysmeta-Ec-Etag', 'Host': 'localhost:80', 'X-Trans-Id': 'txc00ea049dc0b478483148-0064d2734b', 'X-Timestamp': '1691513675.29225', 'Connection': 'close', 'User-Agent': 'proxy-server 8132', 'Referer': 'GET http://localhost/v1/a/c/o', 'Range': 'bytes=0-1885', 'X-Backend-Fragment-Preferences': '[{"timestamp": "0000000001.00000", "exclude": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]}]'}, '', False), {})
E (('10.0.0.21', 1021, 'GET', '/sda/0/a/c/o', {'X-Backend-Storage-Policy-Index': '0', 'X-Backend-Etag-Is-At': 'X-Object-Sysmeta-Ec-Etag', 'Host': 'localhost:80', 'X-Trans-Id': 'txc00ea049dc0b478483148-0064d2734b', 'X-Timestamp': '1691513675.29225', 'Connection': 'close', 'User-Agent': 'proxy-server 8132', 'Referer': 'GET http://localhost/v1/a/c/o', 'Range': 'bytes=0-1885', 'X-Backend-Fragment-Preferences': '[{"timestamp": "0000000001.00000", "exclude": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]}]'}, '', False), {})
E (('10.0.0.22', 1022, 'GET', '/sda/0/a/c/o', {'X-Backend-Storage-Policy-Index': '0', 'X-Backend-Etag-Is-At': 'X-Object-Sysmeta-Ec-Etag', 'Host': 'localhost:80', 'X-Trans-Id': 'txc00ea049dc0b478483148-0064d2734b', 'X-Timestamp': '1691513675.29225', 'Connection': 'close', 'User-Agent': 'proxy-server 8132', 'Referer': 'GET http://localhost/v1/a/c/o', 'Range': 'bytes=0-1885', 'X-Backend-Fragment-Preferences': '[{"timestamp": "0000000001.00000", "exclude": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]}]'}, '', False), {})
E (('10.0.0.23', 1023, 'GET', '/sda/0/a/c/o', {'X-Backend-Storage-Policy-Index': '0', 'X-Backend-Etag-Is-At': 'X-Object-Sysmeta-Ec-Etag', 'Host': 'localhost:80', 'X-Trans-Id': 'txc00ea049dc0b478483148-0064d2734b', 'X-Timestamp': '1691513675.29225', 'Connection': 'close', 'User-Agent': 'proxy-server 8132', 'Referer': 'GET http://localhost/v1/a/c/o', 'Range': 'bytes=0-1885', 'X-Backend-Fragment-Preferences': '[{"timestamp": "0000000001.00000", "exclude": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]}]'}, '', False), {})
E (('10.0.0.24', 1024, 'GET', '/sda/0/a/c/o', {'X-Backend-Storage-Policy-Index': '0', 'X-Backend-Etag-Is-At': 'X-Object-Sysmeta-Ec-Etag', 'Host': 'localhost:80', 'X-Trans-Id': 'txc00ea049dc0b478483148-0064d2734b', 'X-Timestamp': '1691513675.29225', 'Connection': 'close', 'User-Agent': 'proxy-server 8132', 'Referer': 'GET http://localhost/v1/a/c/o', 'Range': 'bytes=0-1885', 'X-Backend-Fragment-Preferences': '[{"timestamp": "0000000001.00000", "exclude": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]}]'}, '', False), {})
E (('10.0.0.25', 1025, 'GET', '/sda/0/a/c/o', {'X-Backend-Storage-Policy-Index': '0', 'X-Backend-Etag-Is-At': 'X-Object-Sysmeta-Ec-Etag', 'Host': 'localhost:80', 'X-Trans-Id': 'txc00ea049dc0b478483148-0064d2734b', 'X-Timestamp': '1691513675.29225', 'Connection': 'close', 'User-Agent': 'proxy-server 8132', 'Referer': 'GET http://localhost/v1/a/c/o', 'Range': 'bytes=0-1885', 'X-Backend-Fragment-Preferences': '[{"timestamp": "0000000001.00000", "exclude": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]}]'}, '', False), {})
E (('10.0.0.26', 1026, 'GET', '/sda/0/a/c/o', {'X-Backend-Storage-Policy-Index': '0', 'X-Backend-Etag-Is-At': 'X-Object-Sysmeta-Ec-Etag', 'Host': 'localhost:80', 'X-Trans-Id': 'txc00ea049dc0b478483148-0064d2734b', 'X-Timestamp': '1691513675.29225', 'Connection': 'close', 'User-Agent': 'proxy-server 8132', 'Referer': 'GET http://localhost/v1/a/c/o', 'Range': 'bytes=0-1885', 'X-Backend-Fragment-Preferences': '[{"timestamp": "0000000001.00000", "exclude": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]}]'}, '', False), {})
E (('10.0.0.27', 1027, 'GET', '/sda/0/a/c/o', {'X-Backend-Storage-Policy-Index': '0', 'X-Backend-Etag-Is-At': 'X-Object-Sysmeta-Ec-Etag', 'Host': 'localhost:80', 'X-Trans-Id': 'txc00ea049dc0b478483148-0064d2734b', 'X-Timestamp': '1691513675.29225', 'Connection': 'close', 'User-Agent': 'proxy-server 8132', 'Referer': 'GET http://localhost/v1/a/c/o', 'Range': 'bytes=0-1885', 'X-Backend-Fragment-Preferences': '[{"timestamp": "0000000001.00000", "exclude": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]}]'}, '', False), {})
/home/zuul/src/opendev.org/openstack/swift/test/unit/__init__.py:1048: AssertionError
----------------------------- Captured stdout call -----------------------------
proxy-server DEBUG: Loaded override config for (default): ProxyOverrideOptions({}, {'sorting_method': 'shuffle', 'read_affinity': '', 'write_affinity': '', 'write_affinity_node_count': '2 * replicas', 'write_affinity_handoff_delete_count': None, 'rebalance_missing_suppression_count': 1, 'concurrent_gets': False, 'concurrency_timeout': 1.0, 'concurrent_ec_extra_requests': 0}, app) (txn: txn1) (client_ip: 127.0.0.2)
proxy-server ERROR: Trying to read EC fragment during GET (retrying) 10.0.0.4:1004/sde (txn: txc00ea049dc0b478483148-0064d2734b)
proxy-server ERROR: Trying to read EC fragment during GET (retrying) 10.0.0.0:1000/sda (txn: txc00ea049dc0b478483148-0064d2734b)
proxy-server ERROR: ERROR with Object server 10.0.0.14:1014/sda re: Trying to GET /v1/a/c/o:
Traceback (most recent call last):
File "/home/zuul/src/opendev.org/openstack/swift/swift/proxy/controllers/obj.py", line 2534, in _iter_bytes_from_response_part
buf += chunk
File "/home/zuul/src/opendev.org/openstack/swift/test/unit/__init__.py", line 783, in __radd__
self.slowdown()
File "/home/zuul/src/opendev.org/openstack/swift/test/unit/__init__.py", line 774, in slowdown
eventlet.sleep(self.slowness)
File "/home/zuul/src/opendev.org/openstack/swift/.tox/py310/lib/python3.10/site-packages/eventlet/greenthread.py", line 36, in sleep
hub.switch()
File "/home/zuul/src/opendev.org/openstack/swift/.tox/py310/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 313, in switch
return self.greenlet.switch()
swift.common.exceptions.ChunkReadTimeout: 0.01 seconds
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/zuul/src/opendev.org/openstack/swift/swift/proxy/controllers/obj.py", line 2669, in _make_node_request
conn = http_connect(
File "/home/zuul/src/opendev.org/openstack/swift/swift/common/bufferedhttp.py", line 280, in http_connect
return http_connect_raw(
File "/home/zuul/src/opendev.org/openstack/swift/test/unit/__init__.py", line 973, in connect
i, status = next(conn_id_and_code_iter)
StopIteration (txn: txc00ea049dc0b478483148-0064d2734b)
proxy-server ERROR: ERROR with Object server 10.0.0.15:1015/sda re: Trying to GET /v1/a/c/o:
Traceback (most recent call last):
File "/home/zuul/src/opendev.org/openstack/swift/swift/proxy/controllers/obj.py", line 2534, in _iter_bytes_from_response_part
buf += chunk
File "/home/zuul/src/opendev.org/openstack/swift/test/unit/__init__.py", line 783, in __radd__
self.slowdown()
File "/home/zuul/src/opendev.org/openstack/swift/test/unit/__init__.py", line 774, in slowdown
eventlet.sleep(self.slowness)
File "/home/zuul/src/opendev.org/openstack/swift/.tox/py310/lib/python3.10/site-packages/eventlet/greenthread.py", line 36, in sleep
hub.switch()
File "/home/zuul/src/opendev.org/openstack/swift/.tox/py310/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 313, in switch
return self.greenlet.switch()
swift.common.exceptions.ChunkReadTimeout: 0.01 seconds
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/zuul/src/opendev.org/openstack/swift/swift/proxy/controllers/obj.py", line 2669, in _make_node_request
conn = http_connect(
File "/home/zuul/src/opendev.org/openstack/swift/swift/common/bufferedhttp.py", line 280, in http_connect
return http_connect_raw(
File "/home/zuul/src/opendev.org/openstack/swift/test/unit/__init__.py", line 973, in connect
i, status = next(conn_id_and_code_iter)
StopIteration (txn: txc00ea049dc0b478483148-0064d2734b)
proxy-server ERROR: ERROR with Object server 10.0.0.16:1016/sda re: Trying to GET /v1/a/c/o:
Traceback (most recent call last):
File "/home/zuul/src/opendev.org/openstack/swift/swift/proxy/controllers/obj.py", line 2534, in _iter_bytes_from_response_part
buf += chunk
File "/home/zuul/src/opendev.org/openstack/swift/test/unit/__init__.py", line 783, in __radd__
self.slowdown()
File "/home/zuul/src/opendev.org/openstack/swift/test/unit/__init__.py", line 774, in slowdown
eventlet.sleep(self.slowness)
File "/home/zuul/src/opendev.org/openstack/swift/.tox/py310/lib/python3.10/site-packages/eventlet/greenthread.py", line 36, in sleep
hub.switch()
File "/home/zuul/src/opendev.org/openstack/swift/.tox/py310/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 313, in switch
return self.greenlet.switch()
swift.common.exceptions.ChunkReadTimeout: 0.01 seconds
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/zuul/src/opendev.org/openstack/swift/swift/proxy/controllers/obj.py", line 2669, in _make_node_request
conn = http_connect(
File "/home/zuul/src/opendev.org/openstack/swift/swift/common/bufferedhttp.py", line 280, in http_connect
return http_connect_raw(
File "/home/zuul/src/opendev.org/openstack/swift/test/unit/__init__.py", line 973, in connect
i, status = next(conn_id_and_code_iter)
StopIteration (txn: txc00ea049dc0b478483148-0064d2734b)
proxy-server ERROR: ERROR with Object server 10.0.0.17:1017/sda re: Trying to GET /v1/a/c/o:
Traceback (most recent call last):
File "/home/zuul/src/opendev.org/openstack/swift/swift/proxy/controllers/obj.py", line 2534, in _iter_bytes_from_response_part
buf += chunk
File "/home/zuul/src/opendev.org/openstack/swift/test/unit/__init__.py", line 783, in __radd__
self.slowdown()
File "/home/zuul/src/opendev.org/openstack/swift/test/unit/__init__.py", line 774, in slowdown
eventlet.sleep(self.slowness)
File "/home/zuul/src/opendev.org/openstack/swift/.tox/py310/lib/python3.10/site-packages/eventlet/greenthread.py", line 36, in sleep
hub.switch()
File "/home/zuul/src/opendev.org/openstack/swift/.tox/py310/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 313, in switch
return self.greenlet.switch()
swift.common.exceptions.ChunkReadTimeout: 0.01 seconds
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/zuul/src/opendev.org/openstack/swift/swift/proxy/controllers/obj.py", line 2669, in _make_node_request
conn = http_connect(
File "/home/zuul/src/opendev.org/openstack/swift/swift/common/bufferedhttp.py", line 280, in http_connect
return http_connect_raw(
File "/home/zuul/src/opendev.org/openstack/swift/test/unit/__init__.py", line 973, in connect
i, status = next(conn_id_and_code_iter)
StopIteration (txn: txc00ea049dc0b478483148-0064d2734b)
proxy-server ERROR: ERROR with Object server 10.0.0.18:1018/sda re: Trying to GET /v1/a/c/o:
Traceback (most recent call last):
File "/home/zuul/src/opendev.org/openstack/swift/swift/proxy/controllers/obj.py", line 2534, in _iter_bytes_from_response_part
buf += chunk
File "/home/zuul/src/opendev.org/openstack/swift/test/unit/__init__.py", line 783, in __radd__
self.slowdown()
File "/home/zuul/src/opendev.org/openstack/swift/test/unit/__init__.py", line 774, in slowdown
eventlet.sleep(self.slowness)
File "/home/zuul/src/opendev.org/openstack/swift/.tox/py310/lib/python3.10/site-packages/eventlet/greenthread.py", line 36, in sleep
hub.switch()
File "/home/zuul/src/opendev.org/openstack/swift/.tox/py310/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 313, in switch
return self.greenlet.switch()
swift.common.exceptions.ChunkReadTimeout: 0.01 seconds
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/zuul/src/opendev.org/openstack/swift/swift/proxy/controllers/obj.py", line 2669, in _make_node_request
conn = http_connect(
File "/home/zuul/src/opendev.org/openstack/swift/swift/common/bufferedhttp.py", line 280, in http_connect
return http_connect_raw(
File "/home/zuul/src/opendev.org/openstack/swift/test/unit/__init__.py", line 973, in connect
i, status = next(conn_id_and_code_iter)
StopIteration (txn: txc00ea049dc0b478483148-0064d2734b)
proxy-server ERROR: ERROR with Object server 10.0.0.19:1019/sda re: Trying to GET /v1/a/c/o:
Traceback (most recent call last):
File "/home/zuul/src/opendev.org/openstack/swift/swift/proxy/controllers/obj.py", line 2534, in _iter_bytes_from_response_part
buf += chunk
File "/home/zuul/src/opendev.org/openstack/swift/test/unit/__init__.py", line 783, in __radd__
self.slowdown()
File "/home/zuul/src/opendev.org/openstack/swift/test/unit/__init__.py", line 774, in slowdown
eventlet.sleep(self.slowness)
File "/home/zuul/src/opendev.org/openstack/swift/.tox/py310/lib/python3.10/site-packages/eventlet/greenthread.py", line 36, in sleep
hub.switch()
File "/home/zuul/src/opendev.org/openstack/swift/.tox/py310/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 313, in switch
return self.greenlet.switch()
swift.common.exceptions.ChunkReadTimeout: 0.01 seconds
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/zuul/src/opendev.org/openstack/swift/swift/proxy/controllers/obj.py", line 2669, in _make_node_request
conn = http_connect(
File "/home/zuul/src/opendev.org/openstack/swift/swift/common/bufferedhttp.py", line 280, in http_connect
return http_connect_raw(
File "/home/zuul/src/opendev.org/openstack/swift/test/unit/__init__.py", line 973, in connect
i, status = next(conn_id_and_code_iter)
StopIteration (txn: txc00ea049dc0b478483148-0064d2734b)
proxy-server ERROR: ERROR with Object server 10.0.0.20:1020/sda re: Trying to GET /v1/a/c/o:
Traceback (most recent call last):
File "/home/zuul/src/opendev.org/openstack/swift/swift/proxy/controllers/obj.py", line 2534, in _iter_bytes_from_response_part
buf += chunk
File "/home/zuul/src/opendev.org/openstack/swift/test/unit/__init__.py", line 783, in __radd__
self.slowdown()
File "/home/zuul/src/opendev.org/openstack/swift/test/unit/__init__.py", line 774, in slowdown
eventlet.sleep(self.slowness)
File "/home/zuul/src/opendev.org/openstack/swift/.tox/py310/lib/python3.10/site-packages/eventlet/greenthread.py", line 36, in sleep
hub.switch()
File "/home/zuul/src/opendev.org/openstack/swift/.tox/py310/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 313, in switch
return self.greenlet.switch()
swift.common.exceptions.ChunkReadTimeout: 0.01 seconds
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/zuul/src/opendev.org/openstack/swift/swift/proxy/controllers/obj.py", line 2669, in _make_node_request
conn = http_connect(
File "/home/zuul/src/opendev.org/openstack/swift/swift/common/bufferedhttp.py", line 280, in http_connect
return http_connect_raw(
File "/home/zuul/src/opendev.org/openstack/swift/test/unit/__init__.py", line 973, in connect
i, status = next(conn_id_and_code_iter)
StopIteration (txn: txc00ea049dc0b478483148-0064d2734b)
proxy-server ERROR: ERROR with Object server 10.0.0.21:1021/sda re: Trying to GET /v1/a/c/o:
Traceback (most recent call last):
File "/home/zuul/src/opendev.org/openstack/swift/swift/proxy/controllers/obj.py", line 2534, in _iter_bytes_from_response_part
buf += chunk
File "/home/zuul/src/opendev.org/openstack/swift/test/unit/__init__.py", line 783, in __radd__
self.slowdown()
File "/home/zuul/src/opendev.org/openstack/swift/test/unit/__init__.py", line 774, in slowdown
eventlet.sleep(self.slowness)
File "/home/zuul/src/opendev.org/openstack/swift/.tox/py310/lib/python3.10/site-packages/eventlet/greenthread.py", line 36, in sleep
hub.switch()
File "/home/zuul/src/opendev.org/openstack/swift/.tox/py310/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 313, in switch
return self.greenlet.switch()
swift.common.exceptions.ChunkReadTimeout: 0.01 seconds
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/zuul/src/opendev.org/openstack/swift/swift/proxy/controllers/obj.py", line 2669, in _make_node_request
conn = http_connect(
File "/home/zuul/src/opendev.org/openstack/swift/swift/common/bufferedhttp.py", line 280, in http_connect
return http_connect_raw(
File "/home/zuul/src/opendev.org/openstack/swift/test/unit/__init__.py", line 973, in connect
i, status = next(conn_id_and_code_iter)
StopIteration (txn: txc00ea049dc0b478483148-0064d2734b)
proxy-server ERROR: ERROR with Object server 10.0.0.22:1022/sda re: Trying to GET /v1/a/c/o:
Traceback (most recent call last):
File "/home/zuul/src/opendev.org/openstack/swift/swift/proxy/controllers/obj.py", line 2534, in _iter_bytes_from_response_part
buf += chunk
File "/home/zuul/src/opendev.org/openstack/swift/test/unit/__init__.py", line 783, in __radd__
self.slowdown()
File "/home/zuul/src/opendev.org/openstack/swift/test/unit/__init__.py", line 774, in slowdown
eventlet.sleep(self.slowness)
File "/home/zuul/src/opendev.org/openstack/swift/.tox/py310/lib/python3.10/site-packages/eventlet/greenthread.py", line 36, in sleep
hub.switch()
File "/home/zuul/src/opendev.org/openstack/swift/.tox/py310/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 313, in switch
return self.greenlet.switch()
swift.common.exceptions.ChunkReadTimeout: 0.01 seconds
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/zuul/src/opendev.org/openstack/swift/swift/proxy/controllers/obj.py", line 2669, in _make_node_request
conn = http_connect(
File "/home/zuul/src/opendev.org/openstack/swift/swift/common/bufferedhttp.py", line 280, in http_connect
return http_connect_raw(
File "/home/zuul/src/opendev.org/openstack/swift/test/unit/__init__.py", line 973, in connect
i, status = next(conn_id_and_code_iter)
StopIteration (txn: txc00ea049dc0b478483148-0064d2734b)
proxy-server ERROR: ERROR with Object server 10.0.0.23:1023/sda re: Trying to GET /v1/a/c/o:
Traceback (most recent call last):
File "/home/zuul/src/opendev.org/openstack/swift/swift/proxy/controllers/obj.py", line 2534, in _iter_bytes_from_response_part
buf += chunk
File "/home/zuul/src/opendev.org/openstack/swift/test/unit/__init__.py", line 783, in __radd__
self.slowdown()
File "/home/zuul/src/opendev.org/openstack/swift/test/unit/__init__.py", line 774, in slowdown
eventlet.sleep(self.slowness)
File "/home/zuul/src/opendev.org/openstack/swift/.tox/py310/lib/python3.10/site-packages/eventlet/greenthread.py", line 36, in sleep
hub.switch()
File "/home/zuul/src/opendev.org/openstack/swift/.tox/py310/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 313, in switch
return self.greenlet.switch()
swift.common.exceptions.ChunkReadTimeout: 0.01 seconds
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/zuul/src/opendev.org/openstack/swift/swift/proxy/controllers/obj.py", line 2669, in _make_node_request
conn = http_connect(
File "/home/zuul/src/opendev.org/openstack/swift/swift/common/bufferedhttp.py", line 280, in http_connect
return http_connect_raw(
File "/home/zuul/src/opendev.org/openstack/swift/test/unit/__init__.py", line 973, in connect
i, status = next(conn_id_and_code_iter)
StopIteration (txn: txc00ea049dc0b478483148-0064d2734b)
proxy-server ERROR: ERROR with Object server 10.0.0.24:1024/sda re: Trying to GET /v1/a/c/o:
Traceback (most recent call last):
File "/home/zuul/src/opendev.org/openstack/swift/swift/proxy/controllers/obj.py", line 2534, in _iter_bytes_from_response_part
buf += chunk
File "/home/zuul/src/opendev.org/openstack/swift/test/unit/__init__.py", line 783, in __radd__
self.slowdown()
File "/home/zuul/src/opendev.org/openstack/swift/test/unit/__init__.py", line 774, in slowdown
eventlet.sleep(self.slowness)
File "/home/zuul/src/opendev.org/openstack/swift/.tox/py310/lib/python3.10/site-packages/eventlet/greenthread.py", line 36, in sleep
hub.switch()
File "/home/zuul/src/opendev.org/openstack/swift/.tox/py310/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 313, in switch
return self.greenlet.switch()
swift.common.exceptions.ChunkReadTimeout: 0.01 seconds
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/zuul/src/opendev.org/openstack/swift/swift/proxy/controllers/obj.py", line 2669, in _make_node_request
conn = http_connect(
File "/home/zuul/src/opendev.org/openstack/swift/swift/common/bufferedhttp.py", line 280, in http_connect
return http_connect_raw(
File "/home/zuul/src/opendev.org/openstack/swift/test/unit/__init__.py", line 973, in connect
i, status = next(conn_id_and_code_iter)
StopIteration (txn: txc00ea049dc0b478483148-0064d2734b)
proxy-server ERROR: ERROR with Object server 10.0.0.25:1025/sda re: Trying to GET /v1/a/c/o:
Traceback (most recent call last):
File "/home/zuul/src/opendev.org/openstack/swift/swift/proxy/controllers/obj.py", line 2534, in _iter_bytes_from_response_part
buf += chunk
File "/home/zuul/src/opendev.org/openstack/swift/test/unit/__init__.py", line 783, in __radd__
self.slowdown()
File "/home/zuul/src/opendev.org/openstack/swift/test/unit/__init__.py", line 774, in slowdown
eventlet.sleep(self.slowness)
File "/home/zuul/src/opendev.org/openstack/swift/.tox/py310/lib/python3.10/site-packages/eventlet/greenthread.py", line 36, in sleep
hub.switch()
File "/home/zuul/src/opendev.org/openstack/swift/.tox/py310/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 313, in switch
return self.greenlet.switch()
swift.common.exceptions.ChunkReadTimeout: 0.01 seconds
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/zuul/src/opendev.org/openstack/swift/swift/proxy/controllers/obj.py", line 2669, in _make_node_request
conn = http_connect(
File "/home/zuul/src/opendev.org/openstack/swift/swift/common/bufferedhttp.py", line 280, in http_connect
return http_connect_raw(
File "/home/zuul/src/opendev.org/openstack/swift/test/unit/__init__.py", line 973, in connect
i, status = next(conn_id_and_code_iter)
StopIteration (txn: txc00ea049dc0b478483148-0064d2734b)
proxy-server ERROR: ERROR with Object server 10.0.0.26:1026/sda re: Trying to GET /v1/a/c/o:
Traceback (most recent call last):
File "/home/zuul/src/opendev.org/openstack/swift/swift/proxy/controllers/obj.py", line 2534, in _iter_bytes_from_response_part
buf += chunk
File "/home/zuul/src/opendev.org/openstack/swift/test/unit/__init__.py", line 783, in __radd__
self.slowdown()
File "/home/zuul/src/opendev.org/openstack/swift/test/unit/__init__.py", line 774, in slowdown
eventlet.sleep(self.slowness)
File "/home/zuul/src/opendev.org/openstack/swift/.tox/py310/lib/python3.10/site-packages/eventlet/greenthread.py", line 36, in sleep
hub.switch()
File "/home/zuul/src/opendev.org/openstack/swift/.tox/py310/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 313, in switch
return self.greenlet.switch()
swift.common.exceptions.ChunkReadTimeout: 0.01 seconds
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/zuul/src/opendev.org/openstack/swift/swift/proxy/controllers/obj.py", line 2669, in _make_node_request
conn = http_connect(
File "/home/zuul/src/opendev.org/openstack/swift/swift/common/bufferedhttp.py", line 280, in http_connect
return http_connect_raw(
File "/home/zuul/src/opendev.org/openstack/swift/test/unit/__init__.py", line 973, in connect
i, status = next(conn_id_and_code_iter)
StopIteration (txn: txc00ea049dc0b478483148-0064d2734b)
proxy-server ERROR: ERROR with Object server 10.0.0.27:1027/sda re: Trying to GET /v1/a/c/o:
Traceback (most recent call last):
File "/home/zuul/src/opendev.org/openstack/swift/swift/proxy/controllers/obj.py", line 2534, in _iter_bytes_from_response_part
buf += chunk
File "/home/zuul/src/opendev.org/openstack/swift/test/unit/__init__.py", line 783, in __radd__
self.slowdown()
File "/home/zuul/src/opendev.org/openstack/swift/test/unit/__init__.py", line 774, in slowdown
eventlet.sleep(self.slowness)
File "/home/zuul/src/opendev.org/openstack/swift/.tox/py310/lib/python3.10/site-packages/eventlet/greenthread.py", line 36, in sleep
hub.switch()
File "/home/zuul/src/opendev.org/openstack/swift/.tox/py310/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 313, in switch
return self.greenlet.switch()
swift.common.exceptions.ChunkReadTimeout: 0.01 seconds
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/zuul/src/opendev.org/openstack/swift/swift/proxy/controllers/obj.py", line 2669, in _make_node_request
conn = http_connect(
File "/home/zuul/src/opendev.org/openstack/swift/swift/common/bufferedhttp.py", line 280, in http_connect
return http_connect_raw(
File "/home/zuul/src/opendev.org/openstack/swift/test/unit/__init__.py", line 973, in connect
i, status = next(conn_id_and_code_iter)
StopIteration (txn: txc00ea049dc0b478483148-0064d2734b)
proxy-server ERROR: ChunkReadTimeout fetching fragments for '/a/c/o': ChunkReadTimeout (0.01s after 0.05s) (txn: txc00ea049dc0b478483148-0064d2734b)
proxy-server ERROR: ChunkReadTimeout fetching fragments for '/a/c/o': ChunkReadTimeout (0.01s after 0.05s) (txn: txc00ea049dc0b478483148-0064d2734b)
proxy-server ERROR: ChunkReadTimeout fetching fragments for '/a/c/o': ChunkReadTimeout (0.01s after 0.05s) (txn: txc00ea049dc0b478483148-0064d2734b)
proxy-server ERROR: ChunkReadTimeout fetching fragments for '/a/c/o': ChunkReadTimeout (0.01s after 0.05s) (txn: txc00ea049dc0b478483148-0064d2734b)
proxy-server ERROR: ChunkReadTimeout fetching fragments for '/a/c/o': ChunkReadTimeout (0.01s after 0.05s) (txn: txc00ea049dc0b478483148-0064d2734b)
proxy-server ERROR: Trying to read EC fragment during GET (retrying) 10.0.0.11:1011/sdl (txn: txc00ea049dc0b478483148-0064d2734b)
proxy-server ERROR: Trying to read EC fragment during GET (retrying) 10.0.0.9:1009/sdj (txn: txc00ea049dc0b478483148-0064d2734b)
proxy-server WARNING: Un-recoverable fragment rebuild. Only received 5/10 fragments for '/a/c/o' (txn: txc00ea049dc0b478483148-0064d2734b)
```
