Yes,I receive the trace after changing RequestSenderDirect to use zmq.NOBLOCK in sending.
Here are the reasons for using zmq.NOBLOCK: 1、nova-compute send CALL message to nova-conductor , this is normal procedure: [root@host224 home(keystone_admin)]# tail -f /var/log/nova/nova-compute.log 2017-02-08 10:59:51.588 28874 DEBUG oslo_messaging._drivers.zmq_driver.client.zmq_routing_table [-] Processing target ROUTER/conductor for round-robin. _fetch_round_robin_hosts_from_matchmaker /usr/lib/python2.7/site-packages/oslo_messaging/_drivers/zmq_driver/client/zmq_routing_table.py:59 2017-02-08 10:59:51.588 28874 DEBUG oslo_messaging._drivers.zmq_driver.client.zmq_routing_table [-] Host resolved for the current connection is host224:62374 get_round_robin_host /usr/lib/python2.7/site-packages/oslo_messaging/_drivers/zmq_driver/client/zmq_routing_table.py:48 2017-02-08 10:59:51.588 28874 DEBUG oslo_messaging._drivers.zmq_driver.zmq_socket [-] Connecting DEALER socket 3937d22b-4335-4859-8a32-43a834d13675 to tcp://host224:62374 connect_to_address /usr/lib/python2.7/site-packages/oslo_messaging/_drivers/zmq_driver/zmq_socket.py:188 2017-02-08 10:59:51.588 28874 DEBUG oslo_messaging._drivers.zmq_driver.client.zmq_routing_table [-] Processing target ROUTER/conductor for round-robin. _fetch_round_robin_hosts_from_matchmaker /usr/lib/python2.7/site-packages/oslo_messaging/_drivers/zmq_driver/client/zmq_routing_table.py:59 2017-02-08 10:59:51.589 28874 DEBUG oslo_messaging._drivers.zmq_driver.zmq_socket [-] Connecting DEALER socket 3937d22b-4335-4859-8a32-43a834d13675 to tcp://host224:53730 connect_to_address /usr/lib/python2.7/site-packages/oslo_messaging/_drivers/zmq_driver/zmq_socket.py:188 2017-02-08 10:59:51.589 28874 DEBUG oslo_messaging._drivers.zmq_driver.zmq_socket [-] Connecting DEALER socket 3937d22b-4335-4859-8a32-43a834d13675 to tcp://host224:53038 connect_to_address /usr/lib/python2.7/site-packages/oslo_messaging/_drivers/zmq_driver/zmq_socket.py:188 2017-02-08 10:59:51.589 28874 DEBUG oslo_messaging._drivers.zmq_driver.client.zmq_senders [-] Sending CALL message a8ef9d4e-edaa-11e6-9e4b-fa163e6b3caa to target <Target topic=conductor, version=3.0> (v1.0) send /usr/lib/python2.7/site-packages/oslo_messaging/_drivers/zmq_driver/client/zmq_senders.py:155 2017-02-08 10:59:51.589 28874 DEBUG oslo_messaging._drivers.zmq_driver.poller.green_poller [-] Registering socket 3937d22b-4335-4859-8a32-43a834d13675 register /usr/lib/python2.7/site-packages/oslo_messaging/_drivers/zmq_driver/poller/green_poller.py:32 2017-02-08 10:59:51.600 28874 DEBUG oslo_messaging._drivers.zmq_driver.client.zmq_receivers [-] Received REPLY for a8ef9d4e-edaa-11e6-9e4b-fa163e6b3caa _run_loop /usr/lib/python2.7/site-packages/oslo_messaging/_drivers/zmq_driver/client/zmq_receivers.py:125 2017-02-08 10:59:51.605 28874 DEBUG oslo_messaging._drivers.zmq_driver.poller.green_poller [-] Unregistering socket 3937d22b-4335-4859-8a32-43a834d13675 unregister /usr/lib/python2.7/site-packages/oslo_messaging/_drivers/zmq_driver/poller/green_poller.py:40 2017-02-08 10:59:51.605 28874 DEBUG oslo_messaging._drivers.zmq_driver.zmq_socket [-] Socket 3937d22b-4335-4859-8a32-43a834d13675 closed close /usr/lib/python2.7/site-packages/oslo_messaging/_drivers/zmq_driver/zmq_socket.py:179 [root@host224 home(keystone_admin)]# redis-cli smembers ROUTER/conductor 1) "host224:53038" 2) "host224:53730" 3) "host224:58548" 4) "host224:62374"
2、When restart nova-conductor, nova-compute send CALL message to conductor, this is abnormal procedure, socket.send() function has been blocked. [root@host224 home(keystone_admin)]# systemctl restart openstack-nova-conductor.service [root@host224 home(keystone_admin)]# redis-cli smembers ROUTER/conductor/host224 1) "host224:50534" 2) "host224:61061" 3) "host224:65061" 4) "host224:49290" [root@host224 home(keystone_admin)]# tail -f /var/log/nova/nova-compute.log 2017-02-08 11:00:01.588 28874 DEBUG oslo_messaging._drivers.zmq_driver.client.zmq_routing_table [-] Processing target ROUTER/conductor for round-robin. _fetch_round_robin_hosts_from_matchmaker /usr/lib/python2.7/site-packages/oslo_messaging/_drivers/zmq_driver/client/zmq_routing_table.py:59 2017-02-08 11:00:01.588 28874 DEBUG oslo_messaging._drivers.zmq_driver.client.zmq_routing_table [-] Host resolved for the current connection is host224:53730 get_round_robin_host /usr/lib/python2.7/site-packages/oslo_messaging/_drivers/zmq_driver/client/zmq_routing_table.py:48 2017-02-08 11:00:01.588 28874 DEBUG oslo_messaging._drivers.zmq_driver.zmq_socket [-] Connecting DEALER socket f7381ccd-831f-4a20-b677-86f93f3cef91 to tcp://host224:53730 connect_to_address /usr/lib/python2.7/site-packages/oslo_messaging/_drivers/zmq_driver/zmq_socket.py:188 2017-02-08 11:00:01.588 28874 DEBUG oslo_messaging._drivers.zmq_driver.client.zmq_routing_table [-] Processing target ROUTER/conductor for round-robin. _fetch_round_robin_hosts_from_matchmaker /usr/lib/python2.7/site-packages/oslo_messaging/_drivers/zmq_driver/client/zmq_routing_table.py:59 2017-02-08 11:00:01.588 28874 DEBUG oslo_messaging._drivers.zmq_driver.zmq_socket [-] Connecting DEALER socket f7381ccd-831f-4a20-b677-86f93f3cef91 to tcp://host224:53038 connect_to_address /usr/lib/python2.7/site-packages/oslo_messaging/_drivers/zmq_driver/zmq_socket.py:188 2017-02-08 11:00:45.347 28874 DEBUG oslo_service.periodic_task [req-522af5ab-10aa-47d9-8ba5-7f5a4edc35e4 - - - - -] Running periodic task ComputeManager._poll_volume_usage run_periodic_tasks /usr/lib/python2.7/site-packages/oslo_service/periodic_task.py:215 2017-02-08 11:00:45.348 28874 DEBUG oslo_service.periodic_task [req-522af5ab-10aa-47d9-8ba5-7f5a4edc35e4 - - - - -] Running periodic task ComputeManager.update_available_resource run_periodic_tasks /usr/lib/python2.7/site-packages/oslo_service/periodic_task.py:215 2017-02-08 11:00:45.350 28874 DEBUG oslo_messaging._drivers.zmq_driver.client.zmq_routing_table [req-522af5ab-10aa-47d9-8ba5-7f5a4edc35e4 - - - - -] Processing target ROUTER/conductor for round-robin. _fetch_round_robin_hosts_from_matchmaker /usr/lib/python2.7/site-packages/oslo_messaging/_drivers/zmq_driver/client/zmq_routing_table.py:59 2017-02-08 11:00:45.350 28874 DEBUG oslo_messaging._drivers.zmq_driver.client.zmq_routing_table [req-522af5ab-10aa-47d9-8ba5-7f5a4edc35e4 - - - - -] Host resolved for the current connection is host224:53038 get_round_robin_host /usr/lib/python2.7/site-packages/oslo_messaging/_drivers/zmq_driver/client/zmq_routing_table.py:48 2017-02-08 11:00:45.351 28874 DEBUG oslo_messaging._drivers.zmq_driver.zmq_socket [req-522af5ab-10aa-47d9-8ba5-7f5a4edc35e4 - - - - -] Connecting DEALER socket d05b6c5d-69cf-4e4f-887f-ca80db8a2153 to tcp://host224:53038 connect_to_address /usr/lib/python2.7/site-packages/oslo_messaging/_drivers/zmq_driver/zmq_socket.py:188 2017-02-08 11:00:45.351 28874 DEBUG oslo_messaging._drivers.zmq_driver.client.zmq_routing_table [req-522af5ab-10aa-47d9-8ba5-7f5a4edc35e4 - - - - -] Processing target ROUTER/conductor for round-robin. _fetch_round_robin_hosts_from_matchmaker /usr/lib/python2.7/site-packages/oslo_messaging/_drivers/zmq_driver/client/zmq_routing_table.py:59 2017-02-08 11:00:45.351 28874 DEBUG oslo_messaging._drivers.zmq_driver.zmq_socket [req-522af5ab-10aa-47d9-8ba5-7f5a4edc35e4 - - - - -] Connecting DEALER socket d05b6c5d-69cf-4e4f-887f-ca80db8a2153 to tcp://host224:53730 connect_to_address /usr/lib/python2.7/site-packages/oslo_messaging/_drivers/zmq_driver/zmq_socket.py:188 2017-02-08 11:01:45.884 28874 DEBUG oslo_messaging._drivers.zmq_driver.client.zmq_routing_table [-] Updating routing table from the matchmaker. 2 target(s) updated ['ROUTER/scheduler', 'ROUTER/conductor']. _update_routing_table /usr/lib/python2.7/site-packages/oslo_messaging/_drivers/zmq_driver/client/zmq_routing_table.py:194 2017-02-08 11:04:45.887 28874 DEBUG oslo_messaging._drivers.zmq_driver.client.zmq_routing_table [-] Updating routing table from the matchmaker. 2 target(s) updated ['ROUTER/scheduler', 'ROUTER/conductor']. _update_routing_table /usr/lib/python2.7/site-packages/oslo_messaging/_drivers/zmq_driver/client/zmq_routing_table.py:194 2017-02-08 11:07:45.890 28874 DEBUG oslo_messaging._drivers.zmq_driver.client.zmq_routing_table [-] Updating routing table from the matchmaker. 2 target(s) updated ['ROUTER/scheduler', 'ROUTER/conductor']. _update_routing_table /usr/lib/python2.7/site-packages/oslo_messaging/_drivers/zmq_driver/client/zmq_routing_table.py:194 2017-02-08 11:10:45.893 28874 DEBUG oslo_messaging._drivers.zmq_driver.client.zmq_routing_table [-] Updating routing table from the matchmaker. 2 target(s) updated ['ROUTER/scheduler', 'ROUTER/conductor']. _update_routing_table /usr/lib/python2.7/site-packages/oslo_messaging/_drivers/zmq_driver/client/zmq_routing_table.py:194. ....................
Yes,I receive the trace after changing RequestSenderDirect to use zmq.NOBLOCK in sending.
Here are the reasons for using zmq.NOBLOCK: admin)] # tail -f /var/log/ nova/nova- compute. log _drivers. zmq_driver. client. zmq_routing_ table [-] Processing target ROUTER/conductor for round-robin. _fetch_ round_robin_ hosts_from_ matchmaker /usr/lib/ python2. 7/site- packages/ oslo_messaging/ _drivers/ zmq_driver/ client/ zmq_routing_ table.py: 59 _drivers. zmq_driver. client. zmq_routing_ table [-] Host resolved for the current connection is host224:62374 get_round_ robin_host /usr/lib/ python2. 7/site- packages/ oslo_messaging/ _drivers/ zmq_driver/ client/ zmq_routing_ table.py: 48 _drivers. zmq_driver. zmq_socket [-] Connecting DEALER socket 3937d22b- 4335-4859- 8a32-43a834d136 75 to tcp://host224:62374 connect_to_address /usr/lib/ python2. 7/site- packages/ oslo_messaging/ _drivers/ zmq_driver/ zmq_socket. py:188 _drivers. zmq_driver. client. zmq_routing_ table [-] Processing target ROUTER/conductor for round-robin. _fetch_ round_robin_ hosts_from_ matchmaker /usr/lib/ python2. 7/site- packages/ oslo_messaging/ _drivers/ zmq_driver/ client/ zmq_routing_ table.py: 59 _drivers. zmq_driver. zmq_socket [-] Connecting DEALER socket 3937d22b- 4335-4859- 8a32-43a834d136 75 to tcp://host224:53730 connect_to_address /usr/lib/ python2. 7/site- packages/ oslo_messaging/ _drivers/ zmq_driver/ zmq_socket. py:188 _drivers. zmq_driver. zmq_socket [-] Connecting DEALER socket 3937d22b- 4335-4859- 8a32-43a834d136 75 to tcp://host224:53038 connect_to_address /usr/lib/ python2. 7/site- packages/ oslo_messaging/ _drivers/ zmq_driver/ zmq_socket. py:188 _drivers. zmq_driver. client. zmq_senders [-] Sending CALL message a8ef9d4e- edaa-11e6- 9e4b-fa163e6b3c aa to target <Target topic=conductor, version=3.0> (v1.0) send /usr/lib/ python2. 7/site- packages/ oslo_messaging/ _drivers/ zmq_driver/ client/ zmq_senders. py:155 _drivers. zmq_driver. poller. green_poller [-] Registering socket 3937d22b- 4335-4859- 8a32-43a834d136 75 register /usr/lib/ python2. 7/site- packages/ oslo_messaging/ _drivers/ zmq_driver/ poller/ green_poller. py:32 _drivers. zmq_driver. client. zmq_receivers [-] Received REPLY for a8ef9d4e- edaa-11e6- 9e4b-fa163e6b3c aa _run_loop /usr/lib/ python2. 7/site- packages/ oslo_messaging/ _drivers/ zmq_driver/ client/ zmq_receivers. py:125 _drivers. zmq_driver. poller. green_poller [-] Unregistering socket 3937d22b- 4335-4859- 8a32-43a834d136 75 unregister /usr/lib/ python2. 7/site- packages/ oslo_messaging/ _drivers/ zmq_driver/ poller/ green_poller. py:40 _drivers. zmq_driver. zmq_socket [-] Socket 3937d22b- 4335-4859- 8a32-43a834d136 75 closed close /usr/lib/ python2. 7/site- packages/ oslo_messaging/ _drivers/ zmq_driver/ zmq_socket. py:179 admin)] # redis-cli smembers ROUTER/conductor
1、nova-compute send CALL message to nova-conductor , this is normal procedure:
[root@host224 home(keystone_
2017-02-08 10:59:51.588 28874 DEBUG oslo_messaging.
2017-02-08 10:59:51.588 28874 DEBUG oslo_messaging.
2017-02-08 10:59:51.588 28874 DEBUG oslo_messaging.
2017-02-08 10:59:51.588 28874 DEBUG oslo_messaging.
2017-02-08 10:59:51.589 28874 DEBUG oslo_messaging.
2017-02-08 10:59:51.589 28874 DEBUG oslo_messaging.
2017-02-08 10:59:51.589 28874 DEBUG oslo_messaging.
2017-02-08 10:59:51.589 28874 DEBUG oslo_messaging.
2017-02-08 10:59:51.600 28874 DEBUG oslo_messaging.
2017-02-08 10:59:51.605 28874 DEBUG oslo_messaging.
2017-02-08 10:59:51.605 28874 DEBUG oslo_messaging.
[root@host224 home(keystone_
1) "host224:53038"
2) "host224:53730"
3) "host224:58548"
4) "host224:62374"
2、When restart nova-conductor, nova-compute send CALL message to conductor, this is abnormal procedure, socket.send() function has been blocked. admin)] # systemctl restart openstack- nova-conductor. service admin)] # redis-cli smembers ROUTER/ conductor/ host224 admin)] # tail -f /var/log/ nova/nova- compute. log _drivers. zmq_driver. client. zmq_routing_ table [-] Processing target ROUTER/conductor for round-robin. _fetch_ round_robin_ hosts_from_ matchmaker /usr/lib/ python2. 7/site- packages/ oslo_messaging/ _drivers/ zmq_driver/ client/ zmq_routing_ table.py: 59 _drivers. zmq_driver. client. zmq_routing_ table [-] Host resolved for the current connection is host224:53730 get_round_ robin_host /usr/lib/ python2. 7/site- packages/ oslo_messaging/ _drivers/ zmq_driver/ client/ zmq_routing_ table.py: 48 _drivers. zmq_driver. zmq_socket [-] Connecting DEALER socket f7381ccd- 831f-4a20- b677-86f93f3cef 91 to tcp://host224:53730 connect_to_address /usr/lib/ python2. 7/site- packages/ oslo_messaging/ _drivers/ zmq_driver/ zmq_socket. py:188 _drivers. zmq_driver. client. zmq_routing_ table [-] Processing target ROUTER/conductor for round-robin. _fetch_ round_robin_ hosts_from_ matchmaker /usr/lib/ python2. 7/site- packages/ oslo_messaging/ _drivers/ zmq_driver/ client/ zmq_routing_ table.py: 59 _drivers. zmq_driver. zmq_socket [-] Connecting DEALER socket f7381ccd- 831f-4a20- b677-86f93f3cef 91 to tcp://host224:53038 connect_to_address /usr/lib/ python2. 7/site- packages/ oslo_messaging/ _drivers/ zmq_driver/ zmq_socket. py:188 periodic_ task [req-522af5ab- 10aa-47d9- 8ba5-7f5a4edc35 e4 - - - - -] Running periodic task ComputeManager. _poll_volume_ usage run_periodic_tasks /usr/lib/ python2. 7/site- packages/ oslo_service/ periodic_ task.py: 215 periodic_ task [req-522af5ab- 10aa-47d9- 8ba5-7f5a4edc35 e4 - - - - -] Running periodic task ComputeManager. update_ available_ resource run_periodic_tasks /usr/lib/ python2. 7/site- packages/ oslo_service/ periodic_ task.py: 215 _drivers. zmq_driver. client. zmq_routing_ table [req-522af5ab- 10aa-47d9- 8ba5-7f5a4edc35 e4 - - - - -] Processing target ROUTER/conductor for round-robin. _fetch_ round_robin_ hosts_from_ matchmaker /usr/lib/ python2. 7/site- packages/ oslo_messaging/ _drivers/ zmq_driver/ client/ zmq_routing_ table.py: 59 _drivers. zmq_driver. client. zmq_routing_ table [req-522af5ab- 10aa-47d9- 8ba5-7f5a4edc35 e4 - - - - -] Host resolved for the current connection is host224:53038 get_round_ robin_host /usr/lib/ python2. 7/site- packages/ oslo_messaging/ _drivers/ zmq_driver/ client/ zmq_routing_ table.py: 48 _drivers. zmq_driver. zmq_socket [req-522af5ab- 10aa-47d9- 8ba5-7f5a4edc35 e4 - - - - -] Connecting DEALER socket d05b6c5d- 69cf-4e4f- 887f-ca80db8a21 53 to tcp://host224:53038 connect_to_address /usr/lib/ python2. 7/site- packages/ oslo_messaging/ _drivers/ zmq_driver/ zmq_socket. py:188 _drivers. zmq_driver. client. zmq_routing_ table [req-522af5ab- 10aa-47d9- 8ba5-7f5a4edc35 e4 - - - - -] Processing target ROUTER/conductor for round-robin. _fetch_ round_robin_ hosts_from_ matchmaker /usr/lib/ python2. 7/site- packages/ oslo_messaging/ _drivers/ zmq_driver/ client/ zmq_routing_ table.py: 59 _drivers. zmq_driver. zmq_socket [req-522af5ab- 10aa-47d9- 8ba5-7f5a4edc35 e4 - - - - -] Connecting DEALER socket d05b6c5d- 69cf-4e4f- 887f-ca80db8a21 53 to tcp://host224:53730 connect_to_address /usr/lib/ python2. 7/site- packages/ oslo_messaging/ _drivers/ zmq_driver/ zmq_socket. py:188 _drivers. zmq_driver. client. zmq_routing_ table [-] Updating routing table from the matchmaker. 2 target(s) updated ['ROUTER/ scheduler' , 'ROUTER/ conductor' ]. _update_ routing_ table /usr/lib/ python2. 7/site- packages/ oslo_messaging/ _drivers/ zmq_driver/ client/ zmq_routing_ table.py: 194 _drivers. zmq_driver. client. zmq_routing_ table [-] Updating routing table from the matchmaker. 2 target(s) updated ['ROUTER/ scheduler' , 'ROUTER/ conductor' ]. _update_ routing_ table /usr/lib/ python2. 7/site- packages/ oslo_messaging/ _drivers/ zmq_driver/ client/ zmq_routing_ table.py: 194 _drivers. zmq_driver. client. zmq_routing_ table [-] Updating routing table from the matchmaker. 2 target(s) updated ['ROUTER/ scheduler' , 'ROUTER/ conductor' ]. _update_ routing_ table /usr/lib/ python2. 7/site- packages/ oslo_messaging/ _drivers/ zmq_driver/ client/ zmq_routing_ table.py: 194 _drivers. zmq_driver. client. zmq_routing_ table [-] Updating routing table from the matchmaker. 2 target(s) updated ['ROUTER/ scheduler' , 'ROUTER/ conductor' ]. _update_ routing_ table /usr/lib/ python2. 7/site- packages/ oslo_messaging/ _drivers/ zmq_driver/ client/ zmq_routing_ table.py: 194. ....... ......
[root@host224 home(keystone_
[root@host224 home(keystone_
1) "host224:50534"
2) "host224:61061"
3) "host224:65061"
4) "host224:49290"
[root@host224 home(keystone_
2017-02-08 11:00:01.588 28874 DEBUG oslo_messaging.
2017-02-08 11:00:01.588 28874 DEBUG oslo_messaging.
2017-02-08 11:00:01.588 28874 DEBUG oslo_messaging.
2017-02-08 11:00:01.588 28874 DEBUG oslo_messaging.
2017-02-08 11:00:01.588 28874 DEBUG oslo_messaging.
2017-02-08 11:00:45.347 28874 DEBUG oslo_service.
2017-02-08 11:00:45.348 28874 DEBUG oslo_service.
2017-02-08 11:00:45.350 28874 DEBUG oslo_messaging.
2017-02-08 11:00:45.350 28874 DEBUG oslo_messaging.
2017-02-08 11:00:45.351 28874 DEBUG oslo_messaging.
2017-02-08 11:00:45.351 28874 DEBUG oslo_messaging.
2017-02-08 11:00:45.351 28874 DEBUG oslo_messaging.
2017-02-08 11:01:45.884 28874 DEBUG oslo_messaging.
2017-02-08 11:04:45.887 28874 DEBUG oslo_messaging.
2017-02-08 11:07:45.890 28874 DEBUG oslo_messaging.
2017-02-08 11:10:45.893 28874 DEBUG oslo_messaging.
.......