Hi Russel,
Thanks for taking a look. I did the following:
1. Installed devstack on fedora
2. Stopped QPID. nov-network had an exception (below [1] - this is a separate issue and I would guess that one woud want to catch the exception and retry to connect)
3. When QPID restarts nova-network manages to reconnect again (with an exception and then terminates).
The example that I had with the nova-network was with packages and not devstack. Any idea why this may be different behaviour?
I will take a look at devstack with Quantum service and post my findings.
Thanks
Gary
[1] Trace of crach for nova-network when qpid terminates
2012-11-03 18:54:41 INFO nova.openstack.common.rpc.impl_qpid [-] Connected to AMQP server on localhost:5672
2012-11-03 18:54:41 DEBUG nova.service [-] Creating Consumer connection for Service network from (pid=7002) start /opt/stack/nova/nova/service.py:404
Traceback (most recent call last):
File "/usr/lib/python2.7/site-packages/eventlet/hubs/poll.py", line 97, in wait
readers.get(fileno, noop).cb(fileno)
File "/usr/lib/python2.7/site-packages/eventlet/green/select.py", line 48, in on_read
current.switch(([original], [], []))
File "/usr/lib/python2.7/site-packages/eventlet/greenthread.py", line 192, in main
result = function(*args, **kwargs)
File "/opt/stack/nova/nova/service.py", line 123, in run_server
server.start()
File "/opt/stack/nova/nova/service.py", line 412, in start
self.conn.create_consumer(node_topic, rpc_dispatcher, fanout=False)
File "/opt/stack/nova/nova/openstack/common/rpc/amqp.py", line 136, in create_consumer
self.connection.create_consumer(topic, proxy, fanout)
File "/opt/stack/nova/nova/openstack/common/rpc/impl_qpid.py", line 526, in create_consumer
consumer = TopicConsumer(self.conf, self.session, topic, proxy_cb)
File "/opt/stack/nova/nova/openstack/common/rpc/impl_qpid.py", line 187, in __init__
{}, name or topic, {})
File "/opt/stack/nova/nova/openstack/common/rpc/impl_qpid.py", line 130, in __init__
self.reconnect(session)
File "/opt/stack/nova/nova/openstack/common/rpc/impl_qpid.py", line 135, in reconnect
self.receiver = session.receiver(self.address)
File "<string>", line 6, in receiver
File "/usr/lib/python2.7/site-packages/qpid/messaging/endpoints.py", line 619, in receiver
raise e
MalformedAddress: unrecognized characters line:1,18: openstack/network.(none) ; {"node": {"x-declare": {"auto-delete": true, "durable": true}, "type": "topic"}, "create": "always", "link": {"x-declare": {"auto-delete": true, "exclusive": false, "durable": false}, "durable": true, "name": "network.(none)"}}
Removing descriptor: 9
2012-11-03 18:54:41 CRITICAL nova [-] unrecognized characters line:1,18: openstack/network.(none) ; {"node": {"x-declare": {"auto-delete": true, "durable": true}, "type": "topic"}, "create": "always", "link": {"x-declare": {"auto-delete": true, "exclusive": false, "durable": false}, "durable": true, "name": "network.(none)"}}
2012-11-03 18:54:41 TRACE nova Traceback (most recent call last):
2012-11-03 18:54:41 TRACE nova File "/opt/stack/nova/bin/nova-network", line 50, in <module>
2012-11-03 18:54:41 TRACE nova service.wait()
2012-11-03 18:54:41 TRACE nova File "/opt/stack/nova/nova/service.py", line 655, in wait
2012-11-03 18:54:41 TRACE nova _launcher.wait()
2012-11-03 18:54:41 TRACE nova File "/opt/stack/nova/nova/service.py", line 183, in wait
2012-11-03 18:54:41 TRACE nova super(ServiceLauncher, self).wait()
2012-11-03 18:54:41 TRACE nova File "/opt/stack/nova/nova/service.py", line 153, in wait
2012-11-03 18:54:41 TRACE nova service.wait()
2012-11-03 18:54:41 TRACE nova File "/usr/lib/python2.7/site-packages/eventlet/greenthread.py", line 166, in wait
2012-11-03 18:54:41 TRACE nova return self._exit_event.wait()
2012-11-03 18:54:41 TRACE nova File "/usr/lib/python2.7/site-packages/eventlet/event.py", line 116, in wait
2012-11-03 18:54:41 TRACE nova return hubs.get_hub().switch()
2012-11-03 18:54:41 TRACE nova File "/usr/lib/python2.7/site-packages/eventlet/hubs/hub.py", line 177, in switch
2012-11-03 18:54:41 TRACE nova return self.greenlet.switch()
2012-11-03 18:54:41 TRACE nova File "/usr/lib/python2.7/site-packages/eventlet/greenthread.py", line 192, in main
2012-11-03 18:54:41 TRACE nova result = function(*args, **kwargs)
2012-11-03 18:54:41 TRACE nova File "/opt/stack/nova/nova/service.py", line 123, in run_server
2012-11-03 18:54:41 TRACE nova server.start()
2012-11-03 18:54:41 TRACE nova File "/opt/stack/nova/nova/service.py", line 412, in start
2012-11-03 18:54:41 TRACE nova self.conn.create_consumer(node_topic, rpc_dispatcher, fanout=False)
2012-11-03 18:54:41 TRACE nova File "/opt/stack/nova/nova/openstack/common/rpc/amqp.py", line 136, in create_consumer
2012-11-03 18:54:41 TRACE nova self.connection.create_consumer(topic, proxy, fanout)
2012-11-03 18:54:41 TRACE nova File "/opt/stack/nova/nova/openstack/common/rpc/impl_qpid.py", line 526, in create_consumer
2012-11-03 18:54:41 TRACE nova consumer = TopicConsumer(self.conf, self.session, topic, proxy_cb)
2012-11-03 18:54:41 TRACE nova File "/opt/stack/nova/nova/openstack/common/rpc/impl_qpid.py", line 187, in __init__
2012-11-03 18:54:41 TRACE nova {}, name or topic, {})
2012-11-03 18:54:41 TRACE nova File "/opt/stack/nova/nova/openstack/common/rpc/impl_qpid.py", line 130, in __init__
2012-11-03 18:54:41 TRACE nova self.reconnect(session)
2012-11-03 18:54:41 TRACE nova File "/opt/stack/nova/nova/openstack/common/rpc/impl_qpid.py", line 135, in reconnect
2012-11-03 18:54:41 TRACE nova self.receiver = session.receiver(self.address)
2012-11-03 18:54:41 TRACE nova File "<string>", line 6, in receiver
2012-11-03 18:54:41 TRACE nova File "/usr/lib/python2.7/site-packages/qpid/messaging/endpoints.py", line 619, in receiver
2012-11-03 18:54:41 TRACE nova raise e
2012-11-03 18:54:41 TRACE nova MalformedAddress: unrecognized characters line:1,18: openstack/network.(none) ; {"node": {"x-declare": {"auto-delete": true, "durable": true}, "type": "topic"}, "create": "always", "link": {"x-declare": {"auto-delete": true, "exclusive": false, "durable": false}, "durable": true, "name": "network.(none)"}}
2012-11-03 18:54:41 TRACE nova
[openstack@(none) nova]$
Hi Russel,
Thanks for taking a look. I did the following:
1. Installed devstack on fedora
2. Stopped QPID. nov-network had an exception (below [1] - this is a separate issue and I would guess that one woud want to catch the exception and retry to connect)
3. When QPID restarts nova-network manages to reconnect again (with an exception and then terminates).
The example that I had with the nova-network was with packages and not devstack. Any idea why this may be different behaviour?
I will take a look at devstack with Quantum service and post my findings.
Thanks
Gary
[1] Trace of crach for nova-network when qpid terminates
2012-11-03 18:54:41 INFO nova.openstack. common. rpc.impl_ qpid [-] Connected to AMQP server on localhost:5672 nova/nova/ service. py:404 python2. 7/site- packages/ eventlet/ hubs/poll. py", line 97, in wait get(fileno, noop).cb(fileno) python2. 7/site- packages/ eventlet/ green/select. py", line 48, in on_read switch( ([original] , [], [])) python2. 7/site- packages/ eventlet/ greenthread. py", line 192, in main nova/nova/ service. py", line 123, in run_server nova/nova/ service. py", line 412, in start conn.create_ consumer( node_topic, rpc_dispatcher, fanout=False) nova/nova/ openstack/ common/ rpc/amqp. py", line 136, in create_consumer connection. create_ consumer( topic, proxy, fanout) nova/nova/ openstack/ common/ rpc/impl_ qpid.py" , line 526, in create_consumer self.conf, self.session, topic, proxy_cb) nova/nova/ openstack/ common/ rpc/impl_ qpid.py" , line 187, in __init__ nova/nova/ openstack/ common/ rpc/impl_ qpid.py" , line 130, in __init__ reconnect( session) nova/nova/ openstack/ common/ rpc/impl_ qpid.py" , line 135, in reconnect receiver( self.address) python2. 7/site- packages/ qpid/messaging/ endpoints. py", line 619, in receiver network. (none) ; {"node": {"x-declare": {"auto-delete": true, "durable": true}, "type": "topic"}, "create": "always", "link": {"x-declare": {"auto-delete": true, "exclusive": false, "durable": false}, "durable": true, "name": "network.(none)"}} network. (none) ; {"node": {"x-declare": {"auto-delete": true, "durable": true}, "type": "topic"}, "create": "always", "link": {"x-declare": {"auto-delete": true, "exclusive": false, "durable": false}, "durable": true, "name": "network.(none)"}} nova/bin/ nova-network" , line 50, in <module> nova/nova/ service. py", line 655, in wait nova/nova/ service. py", line 183, in wait uncher, self).wait() nova/nova/ service. py", line 153, in wait python2. 7/site- packages/ eventlet/ greenthread. py", line 166, in wait event.wait( ) python2. 7/site- packages/ eventlet/ event.py" , line 116, in wait hub().switch( ) python2. 7/site- packages/ eventlet/ hubs/hub. py", line 177, in switch switch( ) python2. 7/site- packages/ eventlet/ greenthread. py", line 192, in main nova/nova/ service. py", line 123, in run_server nova/nova/ service. py", line 412, in start create_ consumer( node_topic, rpc_dispatcher, fanout=False) nova/nova/ openstack/ common/ rpc/amqp. py", line 136, in create_consumer .create_ consumer( topic, proxy, fanout) nova/nova/ openstack/ common/ rpc/impl_ qpid.py" , line 526, in create_consumer self.conf, self.session, topic, proxy_cb) nova/nova/ openstack/ common/ rpc/impl_ qpid.py" , line 187, in __init__ nova/nova/ openstack/ common/ rpc/impl_ qpid.py" , line 130, in __init__ session) nova/nova/ openstack/ common/ rpc/impl_ qpid.py" , line 135, in reconnect receiver( self.address) python2. 7/site- packages/ qpid/messaging/ endpoints. py", line 619, in receiver network. (none) ; {"node": {"x-declare": {"auto-delete": true, "durable": true}, "type": "topic"}, "create": "always", "link": {"x-declare": {"auto-delete": true, "exclusive": false, "durable": false}, "durable": true, "name": "network.(none)"}}
2012-11-03 18:54:41 DEBUG nova.service [-] Creating Consumer connection for Service network from (pid=7002) start /opt/stack/
Traceback (most recent call last):
File "/usr/lib/
readers.
File "/usr/lib/
current.
File "/usr/lib/
result = function(*args, **kwargs)
File "/opt/stack/
server.start()
File "/opt/stack/
self.
File "/opt/stack/
self.
File "/opt/stack/
consumer = TopicConsumer(
File "/opt/stack/
{}, name or topic, {})
File "/opt/stack/
self.
File "/opt/stack/
self.receiver = session.
File "<string>", line 6, in receiver
File "/usr/lib/
raise e
MalformedAddress: unrecognized characters line:1,18: openstack/
Removing descriptor: 9
2012-11-03 18:54:41 CRITICAL nova [-] unrecognized characters line:1,18: openstack/
2012-11-03 18:54:41 TRACE nova Traceback (most recent call last):
2012-11-03 18:54:41 TRACE nova File "/opt/stack/
2012-11-03 18:54:41 TRACE nova service.wait()
2012-11-03 18:54:41 TRACE nova File "/opt/stack/
2012-11-03 18:54:41 TRACE nova _launcher.wait()
2012-11-03 18:54:41 TRACE nova File "/opt/stack/
2012-11-03 18:54:41 TRACE nova super(ServiceLa
2012-11-03 18:54:41 TRACE nova File "/opt/stack/
2012-11-03 18:54:41 TRACE nova service.wait()
2012-11-03 18:54:41 TRACE nova File "/usr/lib/
2012-11-03 18:54:41 TRACE nova return self._exit_
2012-11-03 18:54:41 TRACE nova File "/usr/lib/
2012-11-03 18:54:41 TRACE nova return hubs.get_
2012-11-03 18:54:41 TRACE nova File "/usr/lib/
2012-11-03 18:54:41 TRACE nova return self.greenlet.
2012-11-03 18:54:41 TRACE nova File "/usr/lib/
2012-11-03 18:54:41 TRACE nova result = function(*args, **kwargs)
2012-11-03 18:54:41 TRACE nova File "/opt/stack/
2012-11-03 18:54:41 TRACE nova server.start()
2012-11-03 18:54:41 TRACE nova File "/opt/stack/
2012-11-03 18:54:41 TRACE nova self.conn.
2012-11-03 18:54:41 TRACE nova File "/opt/stack/
2012-11-03 18:54:41 TRACE nova self.connection
2012-11-03 18:54:41 TRACE nova File "/opt/stack/
2012-11-03 18:54:41 TRACE nova consumer = TopicConsumer(
2012-11-03 18:54:41 TRACE nova File "/opt/stack/
2012-11-03 18:54:41 TRACE nova {}, name or topic, {})
2012-11-03 18:54:41 TRACE nova File "/opt/stack/
2012-11-03 18:54:41 TRACE nova self.reconnect(
2012-11-03 18:54:41 TRACE nova File "/opt/stack/
2012-11-03 18:54:41 TRACE nova self.receiver = session.
2012-11-03 18:54:41 TRACE nova File "<string>", line 6, in receiver
2012-11-03 18:54:41 TRACE nova File "/usr/lib/
2012-11-03 18:54:41 TRACE nova raise e
2012-11-03 18:54:41 TRACE nova MalformedAddress: unrecognized characters line:1,18: openstack/
2012-11-03 18:54:41 TRACE nova
[openstack@(none) nova]$