After setting up SAIO (Swift All In One) on Ubuntu 22.04 with Python 3.10.12, I've observed that the Swift unit test results are unstable due to the same issue as the one mentioned on https://github.com/eventlet/eventlet/issues/864. Occasionally, it passes 100% (1 out of 20 runs), but it usually fails.
Segmentation fault when run `$HOME/swift/.unittests`, test stopped at
File "/home/parallels/swift/test/unit/common/test_wsgi.py", line 1101 in test_run_server_strategy_plumbing,
the error message points to "/usr/local/lib/python3.10/dist-packages/mock/mock.py", line 699 in __getattr__, which is ` with NonCallableMock._lock:`
```
Current thread 0x0000ffff85261420 (most recent call first):
File "/usr/local/lib/python3.10/dist-packages/mock/mock.py", line 699 in __getattr__
File "/home/parallels/swift/swift/common/wsgi.py", line 888 in run_wsgi
File "/home/parallels/swift/test/unit/common/test_wsgi.py", line 1101 in test_run_server_strategy_plumbing
File "/usr/local/lib/python3.10/dist-packages/mock/mock.py", line 1452 in patched
File "/usr/lib/python3.10/unittest/case.py", line 549 in _callTestMethod
File "/usr/lib/python3.10/unittest/case.py", line 591 in run
File "/usr/lib/python3.10/unittest/case.py", line 650 in __call__
File "/usr/local/lib/python3.10/dist-packages/_pytest/unittest.py", line 333 in runtest
File "/usr/local/lib/python3.10/dist-packages/_pytest/runner.py", line 169 in pytest_runtest_call
File "/usr/local/lib/python3.10/dist-packages/pluggy/_callers.py", line 77 in _multicall
File "/usr/local/lib/python3.10/dist-packages/pluggy/_manager.py", line 115 in _hookexec
File "/usr/local/lib/python3.10/dist-packages/pluggy/_hooks.py", line 493 in __call__
File "/usr/local/lib/python3.10/dist-packages/_pytest/runner.py", line 262 in <lambda>
File "/usr/local/lib/python3.10/dist-packages/_pytest/runner.py", line 341 in from_call
File "/usr/local/lib/python3.10/dist-packages/_pytest/runner.py", line 261 in call_runtest_hook
File "/usr/local/lib/python3.10/dist-packages/_pytest/runner.py", line 222 in call_and_report
File "/usr/local/lib/python3.10/dist-packages/_pytest/runner.py", line 133 in runtestprotocol
File "/usr/local/lib/python3.10/dist-packages/_pytest/runner.py", line 114 in pytest_runtest_protocol
File "/usr/local/lib/python3.10/dist-packages/pluggy/_callers.py", line 77 in _multicall
File "/usr/local/lib/python3.10/dist-packages/pluggy/_manager.py", line 115 in _hookexec
File "/usr/local/lib/python3.10/dist-packages/pluggy/_hooks.py", line 493 in __call__
File "/usr/local/lib/python3.10/dist-packages/_pytest/main.py", line 350 in pytest_runtestloop
File "/usr/local/lib/python3.10/dist-packages/pluggy/_callers.py", line 77 in _multicall
File "/usr/local/lib/python3.10/dist-packages/pluggy/_manager.py", line 115 in _hookexec
File "/usr/local/lib/python3.10/dist-packages/pluggy/_hooks.py", line 493 in __call__
File "/usr/local/lib/python3.10/dist-packages/_pytest/main.py", line 325 in _main
File "/usr/local/lib/python3.10/dist-packages/_pytest/main.py", line 271 in wrap_session
File "/usr/local/lib/python3.10/dist-packages/_pytest/main.py", line 318 in pytest_cmdline_main
File "/usr/local/lib/python3.10/dist-packages/pluggy/_callers.py", line 77 in _multicall
File "/usr/local/lib/python3.10/dist-packages/pluggy/_manager.py", line 115 in _hookexec
File "/usr/local/lib/python3.10/dist-packages/pluggy/_hooks.py", line 493 in __call__
File "/usr/local/lib/python3.10/dist-packages/_pytest/config/__init__.py", line 169 in main
File "/usr/local/lib/python3.10/dist-packages/_pytest/config/__init__.py", line 192 in console_main
File "/usr/local/bin/pytest", line 8 in <module>
Extension modules: greenlet._greenlet, __original_module__thread, __original_module_select, __original_module_time, _cffi_backend, pyeclib_c, lxml._elementpath, lxml.etree, systemd._journal, systemd._reader, systemd.id128 (total: 11)
/home/parallels/swift/.unittests: line 6: 87000 Segmentation fault (core dumped) pytest --cov-report=html:"$TOP_DIR"/cover $@
/tmp
```
When run test_wsgi seperately, no longer receive segmentfault but get three new test failures in addition, for example:
```
test/unit/common/test_wsgi.py:994 (TestWSGI.test_run_server_success)
self = <test.unit.common.test_wsgi.TestWSGI testMethod=test_run_server_success>
def test_run_server_success(self):
calls = defaultdict(int)
def _initrp(conf_file, app_section, *args, **kwargs):
calls['_initrp'] += 1
return (
{'__file__': 'test', 'workers': 0, 'bind_port': 12345},
'logger',
'log_name')
def _loadapp(uri, name=None, **kwargs):
calls['_loadapp'] += 1
logging.logThreads = 1 # reset to default
with mock.patch.object(wsgi, '_initrp', _initrp), \
mock.patch.object(wsgi, 'get_socket'), \
mock.patch.object(wsgi, 'drop_privileges') as _d_privs, \
mock.patch.object(wsgi, 'clean_up_daemon_hygiene') as _c_hyg, \
mock.patch.object(wsgi, 'loadapp', _loadapp), \
mock.patch.object(wsgi, 'capture_stdio'), \
mock.patch.object(wsgi, 'run_server'), \
mock.patch(
'swift.common.wsgi.systemd_notify') as mock_notify, \
mock.patch('swift.common.utils.eventlet') as _utils_evt:
> rc = wsgi.run_wsgi('conf_file', 'app_section')
/home/parallels/swift/test/unit/common/test_wsgi.py:1019:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
/home/parallels/swift/swift/common/wsgi.py:874: in run_wsgi
conf, logger, global_conf, strategy = check_config(
/home/parallels/swift/swift/common/wsgi.py:842: in check_config
utils.monkey_patch()
/home/parallels/swift/swift/common/utils/__init__.py:491: in monkey_patch
eventlet_monkey_patch()
/home/parallels/swift/swift/common/utils/__init__.py:483: in eventlet_monkey_patch
eventlet.patcher.monkey_patch(all=False, socket=True, select=True,
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <MagicMock name='eventlet' id='281473493835136'>, name = 'patcher'
def __getattr__(self, name):
if name in {'_mock_methods', '_mock_unsafe'}:
raise AttributeError(name)
elif self._mock_methods is not None:
if name not in self._mock_methods or name in _all_magics:
raise AttributeError("Mock object has no attribute %r" % name)
elif _is_magic(name):
raise AttributeError(name)
if not self._mock_unsafe and (not self._mock_methods or name not in self._mock_methods):
if name.startswith(('assert', 'assret', 'asert', 'aseert', 'assrt')) or name in _ATTRIB_DENY_LIST:
raise AttributeError(
f"{name!r} is not a valid assertion. Use a spec "
f"for the mock if {name!r} is meant to be an attribute.")
> with NonCallableMock._lock:
E AttributeError: __enter__
/usr/local/lib/python3.10/dist-packages/mock/mock.py:699: AttributeError
```
The failed test are test_run_server_strategy_plumbing, test_run_server_success, test_run_server_test_config, test_run_server_with_latest_eventlet, all of them called the monkey_patch method during test procedure. The error messages also points to "/usr/local/lib/python3.10/dist-packages/mock/mock.py", line 699
It's worth mentioning that if I run these tests one by one, no error will occur. I do believe that this issue is caused by the monkey,patch() method. Once test_run_server_strategy_plumbing has been excecuted, unittest.mock.NonCallableMock._lock becomes empty, which leads to the following three errors.
Now I am doubting if it is the mutex introduced in [CPython PR](https://github.com/python/cpython/pull/98798) this PR which causes this issue as all error messages point to this code snippet in mock.py file:
```
with NonCallableMock._lock:
result = self._mock_children.get(name)
if result is _deleted:
raise AttributeError(name)
elif result is None:
wraps = None
if self._mock_wraps is not None:
# XXXX should we get the attribute without triggering code
# execution?
wraps = getattr(self._mock_wraps, name)
result = self._get_child_mock(
parent=self, name=name, wraps=wraps, _new_name=name,
_new_parent=self
)
self._mock_children[name] = result
elif isinstance(result, _SpecState):
try:
result = create_autospec(
result.spec, result.spec_set, result.instance,
result.parent, result.name
)
except InvalidSpecError:
target_name = self.__dict__['_mock_name'] or self
raise InvalidSpecError(
f'Cannot autospec attr {name!r} from target '
f'{target_name!r} as it has already been mocked out. '
f'[target={self!r}, attr={result.spec!r}]')
self._mock_children[name] = result
```
Note that we've seen this in the gate on https:/
There's also now an upstream bug for cpython; from my testing, it seems to affect 3.7.0+: https:/