| 2021-03-23 07:38:52 |
Takashi Kajinami |
description |
We have observed that the following Traceback is logged in metadata-agent.log when metadata-agent encounters MessagingTimeout.
~~~
Traceback (most recent call last):
File "/usr/lib/python3.6/site-packages/oslo_messaging/_drivers/amqpdriver.py", line 397, in get
return self._queues[msg_id].get(block=True, timeout=timeout)
File "/usr/lib/python3.6/site-packages/eventlet/queue.py", line 322, in get
return waiter.wait()
File "/usr/lib/python3.6/site-packages/eventlet/queue.py", line 141, in wait
return get_hub().switch()
File "/usr/lib/python3.6/site-packages/eventlet/hubs/hub.py", line 298, in switch
return self.greenlet.switch()
queue.Empty
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/usr/lib/python3.6/site-packages/neutron_lib/rpc.py", line 157, in call
return self._original_context.call(ctxt, method, **kwargs)
File "/usr/lib/python3.6/site-packages/oslo_messaging/rpc/client.py", line 181, in call
transport_options=self.transport_options)
File "/usr/lib/python3.6/site-packages/oslo_messaging/transport.py", line 129, in _send
transport_options=transport_options)
File "/usr/lib/python3.6/site-packages/oslo_messaging/_drivers/amqpdriver.py", line 646, in send
transport_options=transport_options)
File "/usr/lib/python3.6/site-packages/oslo_messaging/_drivers/amqpdriver.py", line 634, in _send
call_monitor_timeout)
File "/usr/lib/python3.6/site-packages/oslo_messaging/_drivers/amqpdriver.py", line 523, in wait
message = self.waiters.get(msg_id, timeout=timeout)
File "/usr/lib/python3.6/site-packages/oslo_messaging/_drivers/amqpdriver.py", line 401, in get
'to message ID %s' % msg_id)
oslo_messaging.exceptions.MessagingTimeout: Timed out waiting for a reply to message ID <id>
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/usr/lib/python3.6/site-packages/oslo_config/cfg.py", line 2197, in __getattr__
return self._get(name)
File "/usr/lib/python3.6/site-packages/oslo_config/cfg.py", line 2631, in _get
value, loc = self._do_get(name, group, namespace)
File "/usr/lib/python3.6/site-packages/oslo_config/cfg.py", line 2649, in _do_get
info = self._get_opt_info(name, group)
File "/usr/lib/python3.6/site-packages/oslo_config/cfg.py", line 2849, in _get_opt_info
raise NoSuchOptError(opt_name, group)
oslo_config.cfg.NoSuchOptError: no such option rpc_response_max_timeout in group [DEFAULT]
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/usr/lib/python3.6/site-packages/neutron/agent/metadata/agent.py", line 88, in __call__
instance_id, tenant_id = self._get_instance_and_tenant_id(req)
File "/usr/lib/python3.6/site-packages/neutron/agent/metadata/agent.py", line 184, in _get_instance_and_tenant_id
skip_cache=skip_cache)
File "/usr/lib/python3.6/site-packages/neutron/agent/metadata/agent.py", line 169, in _get_ports
skip_cache=skip_cache)
File "/usr/lib/python3.6/site-packages/neutron/common/cache_utils.py", line 122, in __call__
return self.func(target_self, *args, **kwargs)
File "/usr/lib/python3.6/site-packages/neutron/agent/metadata/agent.py", line 146, in _get_ports_for_remote_address
ip_address=remote_address)
File "/usr/lib/python3.6/site-packages/neutron/agent/metadata/agent.py", line 113, in _get_ports_from_server
return self.plugin_rpc.get_ports(self.context, filters)
File "/usr/lib/python3.6/site-packages/neutron/agent/metadata/agent.py", line 71, in get_ports
return cctxt.call(context, 'get_ports', filters=filters)
File "/usr/lib/python3.6/site-packages/neutron_lib/rpc.py", line 173, in call
self._original_context.timeout * 2, self.get_max_timeout())
File "/usr/lib/python3.6/site-packages/neutron_lib/rpc.py", line 133, in get_max_timeout
return cls._max_timeout or _get_rpc_response_max_timeout()
File "/usr/lib/python3.6/site-packages/neutron_lib/rpc.py", line 95, in _get_rpc_response_max_timeout
return TRANSPORT.conf.rpc_response_max_timeout
File "/usr/lib/python3.6/site-packages/oslo_messaging/_drivers/common.py", line 508, in __getattr__
value = getattr(self._conf, name)
File "/usr/lib/python3.6/site-packages/oslo_config/cfg.py", line 2201, in __getattr__
raise NoSuchOptError(name)
oslo_config.cfg.NoSuchOptError: no such option rpc_response_max_timeout in group [DEFAULT]
~~~
A similar issue has been reported in https://bugs.launchpad.net/neutron/+bug/1880934 and fixed already. We need the same fix for metadata-agent.
Note: This issue was found in stable/train deployment |
We have observed that the following Traceback is logged in metadata-agent.log when metadata-agent encounters MessagingTimeout.
~~~
Traceback (most recent call last):
File "/usr/lib/python3.6/site-packages/oslo_messaging/_drivers/amqpdriver.py", line 397, in get
return self._queues[msg_id].get(block=True, timeout=timeout)
File "/usr/lib/python3.6/site-packages/eventlet/queue.py", line 322, in get
return waiter.wait()
File "/usr/lib/python3.6/site-packages/eventlet/queue.py", line 141, in wait
return get_hub().switch()
File "/usr/lib/python3.6/site-packages/eventlet/hubs/hub.py", line 298, in switch
return self.greenlet.switch()
queue.Empty
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/usr/lib/python3.6/site-packages/neutron_lib/rpc.py", line 157, in call
return self._original_context.call(ctxt, method, **kwargs)
File "/usr/lib/python3.6/site-packages/oslo_messaging/rpc/client.py", line 181, in call
transport_options=self.transport_options)
File "/usr/lib/python3.6/site-packages/oslo_messaging/transport.py", line 129, in _send
transport_options=transport_options)
File "/usr/lib/python3.6/site-packages/oslo_messaging/_drivers/amqpdriver.py", line 646, in send
transport_options=transport_options)
File "/usr/lib/python3.6/site-packages/oslo_messaging/_drivers/amqpdriver.py", line 634, in _send
call_monitor_timeout)
File "/usr/lib/python3.6/site-packages/oslo_messaging/_drivers/amqpdriver.py", line 523, in wait
message = self.waiters.get(msg_id, timeout=timeout)
File "/usr/lib/python3.6/site-packages/oslo_messaging/_drivers/amqpdriver.py", line 401, in get
'to message ID %s' % msg_id)
oslo_messaging.exceptions.MessagingTimeout: Timed out waiting for a reply to message ID <id>
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/usr/lib/python3.6/site-packages/oslo_config/cfg.py", line 2197, in __getattr__
return self._get(name)
File "/usr/lib/python3.6/site-packages/oslo_config/cfg.py", line 2631, in _get
value, loc = self._do_get(name, group, namespace)
File "/usr/lib/python3.6/site-packages/oslo_config/cfg.py", line 2649, in _do_get
info = self._get_opt_info(name, group)
File "/usr/lib/python3.6/site-packages/oslo_config/cfg.py", line 2849, in _get_opt_info
raise NoSuchOptError(opt_name, group)
oslo_config.cfg.NoSuchOptError: no such option rpc_response_max_timeout in group [DEFAULT]
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/usr/lib/python3.6/site-packages/neutron/agent/metadata/agent.py", line 88, in __call__
instance_id, tenant_id = self._get_instance_and_tenant_id(req)
File "/usr/lib/python3.6/site-packages/neutron/agent/metadata/agent.py", line 184, in _get_instance_and_tenant_id
skip_cache=skip_cache)
File "/usr/lib/python3.6/site-packages/neutron/agent/metadata/agent.py", line 169, in _get_ports
skip_cache=skip_cache)
File "/usr/lib/python3.6/site-packages/neutron/common/cache_utils.py", line 122, in __call__
return self.func(target_self, *args, **kwargs)
File "/usr/lib/python3.6/site-packages/neutron/agent/metadata/agent.py", line 146, in _get_ports_for_remote_address
ip_address=remote_address)
File "/usr/lib/python3.6/site-packages/neutron/agent/metadata/agent.py", line 113, in _get_ports_from_server
return self.plugin_rpc.get_ports(self.context, filters)
File "/usr/lib/python3.6/site-packages/neutron/agent/metadata/agent.py", line 71, in get_ports
return cctxt.call(context, 'get_ports', filters=filters)
File "/usr/lib/python3.6/site-packages/neutron_lib/rpc.py", line 173, in call
self._original_context.timeout * 2, self.get_max_timeout())
File "/usr/lib/python3.6/site-packages/neutron_lib/rpc.py", line 133, in get_max_timeout
return cls._max_timeout or _get_rpc_response_max_timeout()
File "/usr/lib/python3.6/site-packages/neutron_lib/rpc.py", line 95, in _get_rpc_response_max_timeout
return TRANSPORT.conf.rpc_response_max_timeout
File "/usr/lib/python3.6/site-packages/oslo_messaging/_drivers/common.py", line 508, in __getattr__
value = getattr(self._conf, name)
File "/usr/lib/python3.6/site-packages/oslo_config/cfg.py", line 2201, in __getattr__
raise NoSuchOptError(name)
oslo_config.cfg.NoSuchOptError: no such option rpc_response_max_timeout in group [DEFAULT]
~~~
A similar issue has been reported in https://bugs.launchpad.net/neutron/+bug/1880934 and fixed already. We need the same fix for metadata-agent.
Note: This issue was found in stable/train deployment with ml2+ovs |
|
