The problem happens if the nova compute service is restarted. This at times has a situation where is is impossible to restart the nova compute service. We should consider changing this to critical. Please see the trace below when I restarted the compute service:
2013-09-02 00:59:40.979 INFO nova.compute.manager [req-10927006-b0e9-4159-9032-6337bbad97e5 None None] Updating host status
Traceback (most recent call last):
File "/usr/local/lib/python2.7/dist-packages/eventlet/hubs/poll.py", line 97, in wait
readers.get(fileno, noop).cb(fileno)
File "/usr/local/lib/python2.7/dist-packages/eventlet/greenthread.py", line 194, in main
result = function(*args, **kwargs)
File "/opt/stack/nova/nova/openstack/common/service.py", line 65, in run_service
service.start()
File "/opt/stack/nova/nova/service.py", line 154, in start
self.manager.init_host()
File "/opt/stack/nova/nova/compute/manager.py", line 757, in init_host
self._report_driver_status(context)
File "/opt/stack/nova/nova/compute/manager.py", line 4409, in _report_driver_status
capabilities = self.driver.get_host_stats(refresh=True)
File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 363, in get_host_stats
return self.host_state.get_host_stats(refresh=refresh)
File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 434, in host_state
self._cluster)
File "/opt/stack/nova/nova/virt/vmwareapi/host.py", line 155, in __init__
self.update_status()
File "/opt/stack/nova/nova/virt/vmwareapi/host.py", line 201, in update_status
summary.quickStats.overallMemoryUsage
AttributeError: 'Text' object has no attribute 'overallMemoryUsage'
Removing descriptor: 7
2013-09-02 00:59:41.382 ERROR nova.openstack.common.threadgroup [-] 'Text' object has no attribute 'overallMemoryUsage'
2013-09-02 00:59:41.382 TRACE nova.openstack.common.threadgroup Traceback (most recent call last):
2013-09-02 00:59:41.382 TRACE nova.openstack.common.threadgroup File "/opt/stack/nova/nova/openstack/common/threadgroup.py", line 117, in wait
2013-09-02 00:59:41.382 TRACE nova.openstack.common.threadgroup x.wait()
2013-09-02 00:59:41.382 TRACE nova.openstack.common.threadgroup File "/opt/stack/nova/nova/openstack/common/threadgroup.py", line 49, in wait
2013-09-02 00:59:41.382 TRACE nova.openstack.common.threadgroup return self.thread.wait()
2013-09-02 00:59:41.382 TRACE nova.openstack.common.threadgroup File "/usr/local/lib/python2.7/dist-packages/eventlet/greenthread.py", line 168, in wait
2013-09-02 00:59:41.382 TRACE nova.openstack.common.threadgroup return self._exit_event.wait()
2013-09-02 00:59:41.382 TRACE nova.openstack.common.threadgroup File "/usr/local/lib/python2.7/dist-packages/eventlet/event.py", line 116, in wait
2013-09-02 00:59:41.382 TRACE nova.openstack.common.threadgroup return hubs.get_hub().switch()
2013-09-02 00:59:41.382 TRACE nova.openstack.common.threadgroup File "/usr/local/lib/python2.7/dist-packages/eventlet/hubs/hub.py", line 187, in switch
2013-09-02 00:59:41.382 TRACE nova.openstack.common.threadgroup return self.greenlet.switch()
2013-09-02 00:59:41.382 TRACE nova.openstack.common.threadgroup File "/usr/local/lib/python2.7/dist-packages/eventlet/greenthread.py", line 194, in main
2013-09-02 00:59:41.382 TRACE nova.openstack.common.threadgroup result = function(*args, **kwargs)
2013-09-02 00:59:41.382 TRACE nova.openstack.common.threadgroup File "/opt/stack/nova/nova/openstack/common/service.py", line 65, in run_service
2013-09-02 00:59:41.382 TRACE nova.openstack.common.threadgroup service.start()
2013-09-02 00:59:41.382 TRACE nova.openstack.common.threadgroup File "/opt/stack/nova/nova/service.py", line 154, in start
2013-09-02 00:59:41.382 TRACE nova.openstack.common.threadgroup self.manager.init_host()
2013-09-02 00:59:41.382 TRACE nova.openstack.common.threadgroup File "/opt/stack/nova/nova/compute/manager.py", line 757, in init_host
2013-09-02 00:59:41.382 TRACE nova.openstack.common.threadgroup self._report_driver_status(context)
2013-09-02 00:59:41.382 TRACE nova.openstack.common.threadgroup File "/opt/stack/nova/nova/compute/manager.py", line 4409, in _report_driver_status
2013-09-02 00:59:41.382 TRACE nova.openstack.common.threadgroup capabilities = self.driver.get_host_stats(refresh=True)
2013-09-02 00:59:41.382 TRACE nova.openstack.common.threadgroup File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 363, in get_host_stats
2013-09-02 00:59:41.382 TRACE nova.openstack.common.threadgroup return self.host_state.get_host_stats(refresh=refresh)
2013-09-02 00:59:41.382 TRACE nova.openstack.common.threadgroup File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 434, in host_state
2013-09-02 00:59:41.382 TRACE nova.openstack.common.threadgroup self._cluster)
2013-09-02 00:59:41.382 TRACE nova.openstack.common.threadgroup File "/opt/stack/nova/nova/virt/vmwareapi/host.py", line 155, in __init__
2013-09-02 00:59:41.382 TRACE nova.openstack.common.threadgroup self.update_status()
2013-09-02 00:59:41.382 TRACE nova.openstack.common.threadgroup File "/opt/stack/nova/nova/virt/vmwareapi/host.py", line 201, in update_status
2013-09-02 00:59:41.382 TRACE nova.openstack.common.threadgroup summary.quickStats.overallMemoryUsage
2013-09-02 00:59:41.382 TRACE nova.openstack.common.threadgroup AttributeError: 'Text' object has no attribute 'overallMemoryUsage'
2013-09-02 00:59:41.382 TRACE nova.openstack.common.threadgroup
2013-09-02 00:59:41.385 DEBUG amqp [-] Closed channel #1 from (pid=27298) _do_close /usr/local/lib/python2.7/dist-packages/amqp/channel.py:88
nicira@os-devstack:/opt/stack/nova$
The problem happens if the nova compute service is restarted. This at times has a situation where is is impossible to restart the nova compute service. We should consider changing this to critical. Please see the trace below when I restarted the compute service:
2013-09-02 00:59:40.979 INFO nova.compute. manager [req-10927006- b0e9-4159- 9032-6337bbad97 e5 None None] Updating host status lib/python2. 7/dist- packages/ eventlet/ hubs/poll. py", line 97, in wait get(fileno, noop).cb(fileno) lib/python2. 7/dist- packages/ eventlet/ greenthread. py", line 194, in main nova/nova/ openstack/ common/ service. py", line 65, in run_service nova/nova/ service. py", line 154, in start manager. init_host( ) nova/nova/ compute/ manager. py", line 757, in init_host _report_ driver_ status( context) nova/nova/ compute/ manager. py", line 4409, in _report_ driver_ status get_host_ stats(refresh= True) nova/nova/ virt/vmwareapi/ driver. py", line 363, in get_host_stats state.get_ host_stats( refresh= refresh) nova/nova/ virt/vmwareapi/ driver. py", line 434, in host_state nova/nova/ virt/vmwareapi/ host.py" , line 155, in __init__ update_ status( ) nova/nova/ virt/vmwareapi/ host.py" , line 201, in update_status quickStats. overallMemoryUs age sage' common. threadgroup [-] 'Text' object has no attribute 'overallMemoryU sage' common. threadgroup Traceback (most recent call last): common. threadgroup File "/opt/stack/ nova/nova/ openstack/ common/ threadgroup. py", line 117, in wait common. threadgroup x.wait() common. threadgroup File "/opt/stack/ nova/nova/ openstack/ common/ threadgroup. py", line 49, in wait common. threadgroup return self.thread.wait() common. threadgroup File "/usr/local/ lib/python2. 7/dist- packages/ eventlet/ greenthread. py", line 168, in wait common. threadgroup return self._exit_ event.wait( ) common. threadgroup File "/usr/local/ lib/python2. 7/dist- packages/ eventlet/ event.py" , line 116, in wait common. threadgroup return hubs.get_ hub().switch( ) common. threadgroup File "/usr/local/ lib/python2. 7/dist- packages/ eventlet/ hubs/hub. py", line 187, in switch common. threadgroup return self.greenlet. switch( ) common. threadgroup File "/usr/local/ lib/python2. 7/dist- packages/ eventlet/ greenthread. py", line 194, in main common. threadgroup result = function(*args, **kwargs) common. threadgroup File "/opt/stack/ nova/nova/ openstack/ common/ service. py", line 65, in run_service common. threadgroup service.start() common. threadgroup File "/opt/stack/ nova/nova/ service. py", line 154, in start common. threadgroup self.manager. init_host( ) common. threadgroup File "/opt/stack/ nova/nova/ compute/ manager. py", line 757, in init_host common. threadgroup self._report_ driver_ status( context) common. threadgroup File "/opt/stack/ nova/nova/ compute/ manager. py", line 4409, in _report_ driver_ status common. threadgroup capabilities = self.driver. get_host_ stats(refresh= True) common. threadgroup File "/opt/stack/ nova/nova/ virt/vmwareapi/ driver. py", line 363, in get_host_stats common. threadgroup return self.host_ state.get_ host_stats( refresh= refresh) common. threadgroup File "/opt/stack/ nova/nova/ virt/vmwareapi/ driver. py", line 434, in host_state common. threadgroup self._cluster) common. threadgroup File "/opt/stack/ nova/nova/ virt/vmwareapi/ host.py" , line 155, in __init__ common. threadgroup self.update_ status( ) common. threadgroup File "/opt/stack/ nova/nova/ virt/vmwareapi/ host.py" , line 201, in update_status common. threadgroup summary. quickStats. overallMemoryUs age common. threadgroup AttributeError: 'Text' object has no attribute 'overallMemoryU sage' common. threadgroup lib/python2. 7/dist- packages/ amqp/channel. py:88 os-devstack: /opt/stack/ nova$
Traceback (most recent call last):
File "/usr/local/
readers.
File "/usr/local/
result = function(*args, **kwargs)
File "/opt/stack/
service.start()
File "/opt/stack/
self.
File "/opt/stack/
self.
File "/opt/stack/
capabilities = self.driver.
File "/opt/stack/
return self.host_
File "/opt/stack/
self._cluster)
File "/opt/stack/
self.
File "/opt/stack/
summary.
AttributeError: 'Text' object has no attribute 'overallMemoryU
Removing descriptor: 7
2013-09-02 00:59:41.382 ERROR nova.openstack.
2013-09-02 00:59:41.382 TRACE nova.openstack.
2013-09-02 00:59:41.382 TRACE nova.openstack.
2013-09-02 00:59:41.382 TRACE nova.openstack.
2013-09-02 00:59:41.382 TRACE nova.openstack.
2013-09-02 00:59:41.382 TRACE nova.openstack.
2013-09-02 00:59:41.382 TRACE nova.openstack.
2013-09-02 00:59:41.382 TRACE nova.openstack.
2013-09-02 00:59:41.382 TRACE nova.openstack.
2013-09-02 00:59:41.382 TRACE nova.openstack.
2013-09-02 00:59:41.382 TRACE nova.openstack.
2013-09-02 00:59:41.382 TRACE nova.openstack.
2013-09-02 00:59:41.382 TRACE nova.openstack.
2013-09-02 00:59:41.382 TRACE nova.openstack.
2013-09-02 00:59:41.382 TRACE nova.openstack.
2013-09-02 00:59:41.382 TRACE nova.openstack.
2013-09-02 00:59:41.382 TRACE nova.openstack.
2013-09-02 00:59:41.382 TRACE nova.openstack.
2013-09-02 00:59:41.382 TRACE nova.openstack.
2013-09-02 00:59:41.382 TRACE nova.openstack.
2013-09-02 00:59:41.382 TRACE nova.openstack.
2013-09-02 00:59:41.382 TRACE nova.openstack.
2013-09-02 00:59:41.382 TRACE nova.openstack.
2013-09-02 00:59:41.382 TRACE nova.openstack.
2013-09-02 00:59:41.382 TRACE nova.openstack.
2013-09-02 00:59:41.382 TRACE nova.openstack.
2013-09-02 00:59:41.382 TRACE nova.openstack.
2013-09-02 00:59:41.382 TRACE nova.openstack.
2013-09-02 00:59:41.382 TRACE nova.openstack.
2013-09-02 00:59:41.382 TRACE nova.openstack.
2013-09-02 00:59:41.382 TRACE nova.openstack.
2013-09-02 00:59:41.382 TRACE nova.openstack.
2013-09-02 00:59:41.385 DEBUG amqp [-] Closed channel #1 from (pid=27298) _do_close /usr/local/
nicira@