Timeouts on netlink related operations in many functional tests causes job's timeout

[Slawek Kaplonski]

Example of failure https://3e402c0e76741e83fc60-d00ff4f1a74cdbc5ea9d8044145b77c0.ssl.cf2.rackcdn.com/888574/3/check/neutron-functional-with-uwsgi/3bea3d7/job-output.txt

It seems that many tests are stuck on the interaction with netlink (add ip, device_exists, etc.)

Example of stacktrace:

2023-07-14 16:22:38.548953 | controller | Captured traceback:
2023-07-14 16:22:38.548978 | controller | ~~~~~~~~~~~~~~~~~~~
2023-07-14 16:22:38.549017 | controller | Traceback (most recent call last):
2023-07-14 16:22:38.549042 | controller |
2023-07-14 16:22:38.549101 | controller | File "/home/zuul/src/opendev.org/openstack/neutron/neutron/tests/base.py", line 178, in func
2023-07-14 16:22:38.549143 | controller | return f(self, *args, **kwargs)
2023-07-14 16:22:38.549164 | controller |
2023-07-14 16:22:38.549202 | controller | File "/home/zuul/src/opendev.org/openstack/neutron/neutron/tests/functional/agent/l2/extensions/test_ovs_agent_qos_extension.py", line 242, in test_port_creation_with_different_bandwidth_limits
2023-07-14 16:22:38.549283 | controller | self.setup_agent_and_ports(port_dicts)
2023-07-14 16:22:38.549305 | controller |
2023-07-14 16:22:38.549330 | controller | File "/home/zuul/src/opendev.org/openstack/neutron/neutron/tests/functional/agent/l2/base.py", line 457, in setup_agent_and_ports
2023-07-14 16:22:38.549362 | controller | self._plug_ports(self.network, self.ports, self.agent,
2023-07-14 16:22:38.549390 | controller |
2023-07-14 16:22:38.549416 | controller | File "/home/zuul/src/opendev.org/openstack/neutron/neutron/tests/functional/agent/l2/base.py", line 268, in _plug_ports
2023-07-14 16:22:38.549505 | controller | self.driver.init_l3(port.get('vif_name'), ip_cidrs,
2023-07-14 16:22:38.549533 | controller |
2023-07-14 16:22:38.549554 | controller | File "/home/zuul/src/opendev.org/openstack/neutron/neutron/agent/linux/interface.py", line 152, in init_l3
2023-07-14 16:22:38.549583 | controller | device.addr.add(ip_cidr)
2023-07-14 16:22:38.549607 | controller |
2023-07-14 16:22:38.549628 | controller | File "/home/zuul/src/opendev.org/openstack/neutron/neutron/agent/linux/ip_lib.py", line 544, in add
2023-07-14 16:22:38.551748 | controller | add_ip_address(cidr, self.name, self._parent.namespace, scope,
2023-07-14 16:22:38.551786 | controller |
2023-07-14 16:22:38.551817 | controller | File "/home/zuul/src/opendev.org/openstack/neutron/neutron/agent/linux/ip_lib.py", line 837, in add_ip_address
2023-07-14 16:22:38.551841 | controller | privileged.add_ip_address(
2023-07-14 16:22:38.551859 | controller |
2023-07-14 16:22:38.551878 | controller | File "/home/zuul/src/opendev.org/openstack/neutron/.tox/dsvm-functional-gate/lib/python3.10/site-packages/oslo_privsep/priv_context.py", line 271, in _wrap
2023-07-14 16:22:38.551899 | controller | return self.channel.remote_call(name, args, kwargs,
2023-07-14 16:22:38.551921 | controller |
2023-07-14 16:22:38.551942 | controller | File "/home/zuul/src/opendev.org/openstack/neutron/.tox/dsvm-functional-gate/lib/python3.10/site-packages/oslo_privsep/daemon.py", line 204, in remote_call
2023-07-14 16:22:38.552058 | controller | result = self.send_recv((comm.Message.CALL.value, name, args, kwargs),
2023-07-14 16:22:38.552087 | controller |
2023-07-14 16:22:38.552106 | controller | File "/home/zuul/src/opendev.org/openstack/neutron/.tox/dsvm-functional-gate/lib/python3.10/site-packages/oslo_privsep/comm.py", line 194, in send_recv
2023-07-14 16:22:38.552155 | controller | reply = future.result()
2023-07-14 16:22:38.552184 | controller |
2023-07-14 16:22:38.552215 | controller | File "/home/zuul/src/opendev.org/openstack/neutron/.tox/dsvm-functional-gate/lib/python3.10/site-packages/oslo_privsep/comm.py", line 121, in result
2023-07-14 16:22:38.552241 | controller | if not self.condvar.wait(timeout=self.timeout):
2023-07-14 16:22:38.552264 | controller |
2023-07-14 16:22:38.552285 | controller | File "/usr/lib/python3.10/threading.py", line 320, in wait
2023-07-14 16:22:38.552302 | controller | waiter.acquire()
2023-07-14 16:22:38.552321 | controller |
2023-07-14 16:22:38.552339 | controller | File "/home/zuul/src/opendev.org/openstack/neutron/.tox/dsvm-functional-gate/lib/python3.10/site-packages/eventlet/semaphore.py", line 115, in acquire
2023-07-14 16:22:38.552358 | controller | hubs.get_hub().switch()
2023-07-14 16:22:38.552381 | controller |
2023-07-14 16:22:38.552456 | controller | File "/home/zuul/src/opendev.org/openstack/neutron/.tox/dsvm-functional-gate/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 313, in switch
2023-07-14 16:22:38.552475 | controller | return self.greenlet.switch()
2023-07-14 16:22:38.552497 | controller |
2023-07-14 16:22:38.552513 | controller | File "/home/zuul/src/opendev.org/openstack/neutron/.tox/dsvm-functional-gate/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 365, in run
2023-07-14 16:22:38.552533 | controller | self.wait(sleep_time)
2023-07-14 16:22:38.552553 | controller |
2023-07-14 16:22:38.552568 | controller | File "/home/zuul/src/opendev.org/openstack/neutron/.tox/dsvm-functional-gate/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 80, in wait
2023-07-14 16:22:38.552595 | controller | presult = self.do_poll(seconds)
2023-07-14 16:22:38.552616 | controller |
2023-07-14 16:22:38.552634 | controller | File "/home/zuul/src/opendev.org/openstack/neutron/.tox/dsvm-functional-gate/lib/python3.10/site-packages/eventlet/hubs/epolls.py", line 31, in do_poll
2023-07-14 16:22:38.552653 | controller | return self.poll.poll(seconds)
2023-07-14 16:22:38.552671 | controller |
2023-07-14 16:22:38.552694 | controller | File "/home/zuul/src/opendev.org/openstack/neutron/.tox/dsvm-functional-gate/lib/python3.10/site-packages/fixtures/_fixtures/timeout.py", line 52, in signal_handler
2023-07-14 16:22:38.552731 | controller | raise TimeoutException()
2023-07-14 16:22:38.552759 | controller |
2023-07-14 16:22:38.552785 | controller | fixtures._fixtures.timeout.TimeoutException

It also looks similarly in https://storage.bhs.cloud.ovh.net/v1/AUTH_dcaab5e32b234d56b626f72581e3644c/zuul_opendev_logs_c52/883246/16/check/neutron-fullstack-with-uwsgi/c524e84/job-output.txt