| 2021-08-19 07:55:19 |
Akihiro Motoki |
description |
Recently we have been seeing pretty often functional tests issues with errors like:
ft1.13: neutron.tests.functional.plugins.ml2.drivers.ovn.mech_driver.ovsdb.test_maintenance.TestMaintenance.test_check_for_port_security_unknown_addresstesttools.testresult.real._StringException: traceback-1: {{{
Traceback (most recent call last):
File "/home/zuul/src/opendev.org/openstack/neutron/neutron/tests/functional/base.py", line 363, in stop
self.mech_driver.nb_ovn.ovsdb_connection.stop()
File "/home/zuul/src/opendev.org/openstack/neutron/neutron/plugins/ml2/drivers/ovn/mech_driver/mech_driver.py", line 153, in nb_ovn
self._post_fork_event.wait()
File "/usr/lib/python3.8/threading.py", line 558, in wait
signaled = self._cond.wait(timeout)
File "/usr/lib/python3.8/threading.py", line 302, in wait
waiter.acquire()
File "/home/zuul/src/opendev.org/openstack/neutron/.tox/dsvm-functional/lib/python3.8/site-packages/eventlet/semaphore.py", line 120, in acquire
hubs.get_hub().switch()
File "/home/zuul/src/opendev.org/openstack/neutron/.tox/dsvm-functional/lib/python3.8/site-packages/eventlet/hubs/hub.py", line 313, in switch
return self.greenlet.switch()
File "/home/zuul/src/opendev.org/openstack/neutron/.tox/dsvm-functional/lib/python3.8/site-packages/eventlet/hubs/hub.py", line 365, in run
self.wait(sleep_time)
File "/home/zuul/src/opendev.org/openstack/neutron/.tox/dsvm-functional/lib/python3.8/site-packages/eventlet/hubs/poll.py", line 80, in wait
presult = self.do_poll(seconds)
File "/home/zuul/src/opendev.org/openstack/neutron/.tox/dsvm-functional/lib/python3.8/site-packages/eventlet/hubs/epolls.py", line 31, in do_poll
return self.poll.poll(seconds)
File "/home/zuul/src/opendev.org/openstack/neutron/.tox/dsvm-functional/lib/python3.8/site-packages/fixtures/_fixtures/timeout.py", line 52, in signal_handler
raise TimeoutException()
fixtures._fixtures.timeout.TimeoutException
}}}
traceback-2: {{{
Traceback (most recent call last):
File "/home/zuul/src/opendev.org/openstack/neutron/neutron/tests/functional/base.py", line 351, in _collect_processes_logs
self._copy_log_file("%s.log" % northd_log, dst_northd)
File "/home/zuul/src/opendev.org/openstack/neutron/neutron/tests/functional/base.py", line 356, in _copy_log_file
shutil.copyfile(
File "/usr/lib/python3.8/shutil.py", line 264, in copyfile
with open(src, 'rb') as fsrc, open(dst, 'wb') as fdst:
FileNotFoundError: [Errno 2] No such file or directory: '/tmp/tmps9cyr99c/ovn_northd.log'
}}}
Traceback (most recent call last):
File "/home/zuul/src/opendev.org/openstack/neutron/neutron/tests/functional/plugins/ml2/drivers/ovn/mech_driver/ovsdb/test_maintenance.py", line 41, in setUp
super(_TestMaintenanceHelper, self).setUp()
File "/home/zuul/src/opendev.org/openstack/neutron/neutron/tests/functional/base.py", line 217, in setUp
self._start_idls()
File "/home/zuul/src/opendev.org/openstack/neutron/neutron/tests/functional/base.py", line 320, in _start_idls
self.mech_driver.pre_fork_initialize(
File "/home/zuul/src/opendev.org/openstack/neutron/neutron/plugins/ml2/drivers/ovn/mech_driver/mech_driver.py", line 254, in pre_fork_initialize
self._create_neutron_pg_drop()
File "/home/zuul/src/opendev.org/openstack/neutron/neutron/plugins/ml2/drivers/ovn/mech_driver/mech_driver.py", line 272, in _create_neutron_pg_drop
create_default_drop_port_group(pre_ovn_nb_api)
File "/home/zuul/src/opendev.org/openstack/neutron/neutron/plugins/ml2/drivers/ovn/mech_driver/mech_driver.py", line 1241, in create_default_drop_port_group
txn.add(nb_idl.pg_add_ports(pg_name, list(ports_with_pg)))
File "/usr/lib/python3.8/contextlib.py", line 120, in __exit__
next(self.gen)
File "/home/zuul/src/opendev.org/openstack/neutron/neutron/plugins/ml2/drivers/ovn/mech_driver/ovsdb/impl_idl_ovn.py", line 274, in transaction
yield t
File "/usr/lib/python3.8/contextlib.py", line 120, in __exit__
next(self.gen)
File "/home/zuul/src/opendev.org/openstack/neutron/.tox/dsvm-functional/lib/python3.8/site-packages/ovsdbapp/api.py", line 110, in transaction
del self._nested_txns_map[cur_thread_id]
File "/home/zuul/src/opendev.org/openstack/neutron/.tox/dsvm-functional/lib/python3.8/site-packages/ovsdbapp/api.py", line 61, in __exit__
self.result = self.commit()
File "/home/zuul/src/opendev.org/openstack/neutron/.tox/dsvm-functional/lib/python3.8/site-packages/ovsdbapp/backend/ovs_idl/transaction.py", line 65, in commit
raise result.ex
File "/home/zuul/src/opendev.org/openstack/neutron/.tox/dsvm-functional/lib/python3.8/site-packages/ovsdbapp/backend/ovs_idl/connection.py", line 131, in run
txn.results.put(txn.do_commit())
File "/home/zuul/src/opendev.org/openstack/neutron/.tox/dsvm-functional/lib/python3.8/site-packages/ovsdbapp/backend/ovs_idl/transaction.py", line 136, in do_commit
self.post_commit(txn)
File "/home/zuul/src/opendev.org/openstack/neutron/.tox/dsvm-functional/lib/python3.8/site-packages/ovsdbapp/backend/ovs_idl/transaction.py", line 73, in post_commit
command.post_commit(txn)
File "/home/zuul/src/opendev.org/openstack/neutron/.tox/dsvm-functional/lib/python3.8/site-packages/ovsdbapp/backend/ovs_idl/command.py", line 94, in post_commit
row = self.api.tables[self.table_name].rows[real_uuid]
File "/usr/lib/python3.8/collections/__init__.py", line 1010, in __getitem__
raise KeyError(key)
KeyError: UUID('c9570a47-64da-4360-9c94-12ba459344c2')
Examples of such failures:
https://bd41d5ab9bd61c5e04d6-86672578d4e6ceb498f2d932b0da6815.ssl.cf5.rackcdn.com/798961/7/check/neutron-functional-with-uwsgi/772270c/testr_results.html
https://storage.gra.cloud.ovh.net/v1/AUTH_dcaab5e32b234d56b626f72581e3644c/zuul_opendev_logs_856/801310/2/check/neutron-functional-with-uwsgi/856f539/testr_results.html
https://7ee9afbce159ed2d3c7a-ae7c228c39808cabcdb014f8881dacdb.ssl.cf5.rackcdn.com/800278/7/check/neutron-functional-with-uwsgi/6332cdf/testr_results.html
https://fc20912585430bbde389-7e8720184c7a4cb82719c44bcc579362.ssl.cf5.rackcdn.com/802687/1/check/neutron-functional-with-uwsgi/d5d2430/testr_results.html
In all cases the stacktrace is pretty the same but failing tests are different. |
Key failure is KeyError in create_default_drop_port_group().
"No such file or directory" and an error during self.mech_driver.nb_ovn.ovsdb_connection.stop() are part of the cleanup process.
--
Recently we have been seeing pretty often functional tests issues with errors like:
ft1.13: neutron.tests.functional.plugins.ml2.drivers.ovn.mech_driver.ovsdb.test_maintenance.TestMaintenance.test_check_for_port_security_unknown_addresstesttools.testresult.real._StringException: traceback-1: {{{
Traceback (most recent call last):
File "/home/zuul/src/opendev.org/openstack/neutron/neutron/tests/functional/base.py", line 363, in stop
self.mech_driver.nb_ovn.ovsdb_connection.stop()
File "/home/zuul/src/opendev.org/openstack/neutron/neutron/plugins/ml2/drivers/ovn/mech_driver/mech_driver.py", line 153, in nb_ovn
self._post_fork_event.wait()
File "/usr/lib/python3.8/threading.py", line 558, in wait
signaled = self._cond.wait(timeout)
File "/usr/lib/python3.8/threading.py", line 302, in wait
waiter.acquire()
File "/home/zuul/src/opendev.org/openstack/neutron/.tox/dsvm-functional/lib/python3.8/site-packages/eventlet/semaphore.py", line 120, in acquire
hubs.get_hub().switch()
File "/home/zuul/src/opendev.org/openstack/neutron/.tox/dsvm-functional/lib/python3.8/site-packages/eventlet/hubs/hub.py", line 313, in switch
return self.greenlet.switch()
File "/home/zuul/src/opendev.org/openstack/neutron/.tox/dsvm-functional/lib/python3.8/site-packages/eventlet/hubs/hub.py", line 365, in run
self.wait(sleep_time)
File "/home/zuul/src/opendev.org/openstack/neutron/.tox/dsvm-functional/lib/python3.8/site-packages/eventlet/hubs/poll.py", line 80, in wait
presult = self.do_poll(seconds)
File "/home/zuul/src/opendev.org/openstack/neutron/.tox/dsvm-functional/lib/python3.8/site-packages/eventlet/hubs/epolls.py", line 31, in do_poll
return self.poll.poll(seconds)
File "/home/zuul/src/opendev.org/openstack/neutron/.tox/dsvm-functional/lib/python3.8/site-packages/fixtures/_fixtures/timeout.py", line 52, in signal_handler
raise TimeoutException()
fixtures._fixtures.timeout.TimeoutException
}}}
traceback-2: {{{
Traceback (most recent call last):
File "/home/zuul/src/opendev.org/openstack/neutron/neutron/tests/functional/base.py", line 351, in _collect_processes_logs
self._copy_log_file("%s.log" % northd_log, dst_northd)
File "/home/zuul/src/opendev.org/openstack/neutron/neutron/tests/functional/base.py", line 356, in _copy_log_file
shutil.copyfile(
File "/usr/lib/python3.8/shutil.py", line 264, in copyfile
with open(src, 'rb') as fsrc, open(dst, 'wb') as fdst:
FileNotFoundError: [Errno 2] No such file or directory: '/tmp/tmps9cyr99c/ovn_northd.log'
}}}
Traceback (most recent call last):
File "/home/zuul/src/opendev.org/openstack/neutron/neutron/tests/functional/plugins/ml2/drivers/ovn/mech_driver/ovsdb/test_maintenance.py", line 41, in setUp
super(_TestMaintenanceHelper, self).setUp()
File "/home/zuul/src/opendev.org/openstack/neutron/neutron/tests/functional/base.py", line 217, in setUp
self._start_idls()
File "/home/zuul/src/opendev.org/openstack/neutron/neutron/tests/functional/base.py", line 320, in _start_idls
self.mech_driver.pre_fork_initialize(
File "/home/zuul/src/opendev.org/openstack/neutron/neutron/plugins/ml2/drivers/ovn/mech_driver/mech_driver.py", line 254, in pre_fork_initialize
self._create_neutron_pg_drop()
File "/home/zuul/src/opendev.org/openstack/neutron/neutron/plugins/ml2/drivers/ovn/mech_driver/mech_driver.py", line 272, in _create_neutron_pg_drop
create_default_drop_port_group(pre_ovn_nb_api)
File "/home/zuul/src/opendev.org/openstack/neutron/neutron/plugins/ml2/drivers/ovn/mech_driver/mech_driver.py", line 1241, in create_default_drop_port_group
txn.add(nb_idl.pg_add_ports(pg_name, list(ports_with_pg)))
File "/usr/lib/python3.8/contextlib.py", line 120, in __exit__
next(self.gen)
File "/home/zuul/src/opendev.org/openstack/neutron/neutron/plugins/ml2/drivers/ovn/mech_driver/ovsdb/impl_idl_ovn.py", line 274, in transaction
yield t
File "/usr/lib/python3.8/contextlib.py", line 120, in __exit__
next(self.gen)
File "/home/zuul/src/opendev.org/openstack/neutron/.tox/dsvm-functional/lib/python3.8/site-packages/ovsdbapp/api.py", line 110, in transaction
del self._nested_txns_map[cur_thread_id]
File "/home/zuul/src/opendev.org/openstack/neutron/.tox/dsvm-functional/lib/python3.8/site-packages/ovsdbapp/api.py", line 61, in __exit__
self.result = self.commit()
File "/home/zuul/src/opendev.org/openstack/neutron/.tox/dsvm-functional/lib/python3.8/site-packages/ovsdbapp/backend/ovs_idl/transaction.py", line 65, in commit
raise result.ex
File "/home/zuul/src/opendev.org/openstack/neutron/.tox/dsvm-functional/lib/python3.8/site-packages/ovsdbapp/backend/ovs_idl/connection.py", line 131, in run
txn.results.put(txn.do_commit())
File "/home/zuul/src/opendev.org/openstack/neutron/.tox/dsvm-functional/lib/python3.8/site-packages/ovsdbapp/backend/ovs_idl/transaction.py", line 136, in do_commit
self.post_commit(txn)
File "/home/zuul/src/opendev.org/openstack/neutron/.tox/dsvm-functional/lib/python3.8/site-packages/ovsdbapp/backend/ovs_idl/transaction.py", line 73, in post_commit
command.post_commit(txn)
File "/home/zuul/src/opendev.org/openstack/neutron/.tox/dsvm-functional/lib/python3.8/site-packages/ovsdbapp/backend/ovs_idl/command.py", line 94, in post_commit
row = self.api.tables[self.table_name].rows[real_uuid]
File "/usr/lib/python3.8/collections/__init__.py", line 1010, in __getitem__
raise KeyError(key)
KeyError: UUID('c9570a47-64da-4360-9c94-12ba459344c2')
Examples of such failures:
https://bd41d5ab9bd61c5e04d6-86672578d4e6ceb498f2d932b0da6815.ssl.cf5.rackcdn.com/798961/7/check/neutron-functional-with-uwsgi/772270c/testr_results.html
https://storage.gra.cloud.ovh.net/v1/AUTH_dcaab5e32b234d56b626f72581e3644c/zuul_opendev_logs_856/801310/2/check/neutron-functional-with-uwsgi/856f539/testr_results.html
https://7ee9afbce159ed2d3c7a-ae7c228c39808cabcdb014f8881dacdb.ssl.cf5.rackcdn.com/800278/7/check/neutron-functional-with-uwsgi/6332cdf/testr_results.html
https://fc20912585430bbde389-7e8720184c7a4cb82719c44bcc579362.ssl.cf5.rackcdn.com/802687/1/check/neutron-functional-with-uwsgi/d5d2430/testr_results.html
In all cases the stacktrace is pretty the same but failing tests are different. |
|
