Upgrade from 3.1 build 25 to 3.2 Build 1
All 3 device-managers in this cluster (nodec1, nodec2, nodec3) had DMs failing with below backtrace
------------------------------
10/27/2016 01:40:38 AM [DeviceManager]: SANDESH: CONNECT TO COLLECTOR: True
10/27/2016 01:40:38 AM [DeviceManager]: Cannot write http_port 8096 to /tmp/DeviceManager.12818.http_port
10/27/2016 01:40:38 AM [DeviceManager]: Starting Introspect on HTTP Port 8096
10/27/2016 01:40:38 AM [DeviceManager]: Processing event[EvStart] in state[none]
10/27/2016 01:40:38 AM [DeviceManager]: Sandesh Client: Event[EvStart] => State[none] -> State[Idle]
10/27/2016 01:40:38 AM [DeviceManager]: SANDESH: Logging: LEVEL: [SYS_INFO] -> [SYS_NOTICE]
<type 'exceptions.AttributeError'>
Python 2.7.6: /usr/bin/python
Thu Oct 27 01:40:39 2016
A problem occurred in a Python script. Here is the sequence of
function calls leading up to the error, in the order they occurred.
/usr/bin/contrail-device-manager in <module>()
6
7 if __name__ == '__main__':
8 sys.exit(
9 load_entry_point('device-manager==0.1dev', 'console_scripts', 'contrail-device-manager')()
10 )
load_entry_point = <function load_entry_point>
/usr/lib/python2.7/dist-packages/device_manager/device_manager.py in server_main()
480 def server_main():
481 cgitb.enable(format='text')
482 main()
483 # end server_main
484
global main = <function main>
/usr/lib/python2.7/dist-packages/device_manager/device_manager.py in main(args_str='--conf_file /etc/contrail/contrail-device-manage... --conf_file /etc/contrail/contrail-database.conf')
469 _zookeeper_client.master_election(zk_path_pfx+"/device-manager",
470 os.getpid(), run_device_manager,
471 args)
472 # end main
473
args = Namespace(admin_password='contrail123', admin_te...:2181,10.204.216.60:2181', zk_server_port='2181')
/usr/lib/python2.7/dist-packages/cfgm_common/zkclient.py in master_election(self=<cfgm_common.zkclient.ZookeeperClient object>, path='/device-manager', identifier=1927, func=<function run_device_manager>, *args=(Namespace(admin_password='contrail123', admin_te...:2181,10.204.216.60:2181', zk_server_port='2181'),), **kwargs={})
344 def master_election(self, path, identifier, func, *args, **kwargs):
345 self._election = self._zk_client.Election(path, identifier)
346 self._election.run(func, *args, **kwargs)
347 # end master_election
348
self = <cfgm_common.zkclient.ZookeeperClient object>
self._election = <kazoo.recipe.election.Election object>
self._election.run = <bound method Election.run of <kazoo.recipe.election.Election object>>
func = <function run_device_manager>
args = (Namespace(admin_password='contrail123', admin_te...:2181,10.204.216.60:2181', zk_server_port='2181'),)
kwargs = {}
/usr/lib/python2.7/dist-packages/kazoo/recipe/election.py in run(self=<kazoo.recipe.election.Election object>, func=<function run_device_manager>, *args=(Namespace(admin_password='contrail123', admin_te...:2181,10.204.216.60:2181', zk_server_port='2181'),), **kwargs={})
46 try:
47 with self.lock:
48 func(*args, **kwargs)
49
50 except CancelledError:
func = <function run_device_manager>
args = (Namespace(admin_password='contrail123', admin_te...:2181,10.204.216.60:2181', zk_server_port='2181'),)
kwargs = {}
/usr/lib/python2.7/dist-packages/device_manager/device_manager.py in run_device_manager(args=Namespace(admin_password='contrail123', admin_te...:2181,10.204.216.60:2181', zk_server_port='2181'))
474
475 def run_device_manager(args):
476 device_manager = DeviceManager(args)
477 # end run_device_manager
478
device_manager undefined
global DeviceManager = <class 'device_manager.device_manager.DeviceManager'>
args = Namespace(admin_password='contrail123', admin_te...:2181,10.204.216.60:2181', zk_server_port='2181')
/usr/lib/python2.7/dist-packages/device_manager/device_manager.py in __init__(self=<device_manager.device_manager.DeviceManager object>, args=Namespace(admin_password='contrail123', admin_te...:2181,10.204.216.60:2181', zk_server_port='2181'))
208
209 for obj in pr_obj_list:
210 pr = PhysicalRouterDM.locate(obj['uuid'], obj)
211 li_set = pr.logical_interfaces
212 vmi_set = set()
pr undefined
global PhysicalRouterDM = <class 'device_manager.db.PhysicalRouterDM'>
PhysicalRouterDM.locate = <bound method __metaclass__.locate of <class 'device_manager.db.PhysicalRouterDM'>>
obj = {'display_name': 'br0', 'fq_name': ['default-global-system-config', 'br0'], 'id_perms': {'created': '2016-07-26T04:24:08.053092', 'creator': None, 'description': None, 'enable': True, 'last_modified': '2016-07-26T04:24:13.078715', 'permissions': {'group': 'KeystoneAdmin', 'group_access': 7, 'other_access': 7, 'owner': 'admin', 'owner_access': 7}, 'user_visible': True, 'uuid': {'uuid_lslong': 9275171865080250252L, 'uuid_mslong': 13777450802188469773L}}, 'parent_type': 'global-system-config', 'parent_uuid': 'f8570655-ebcb-4ee2-b85a-6795943ec4d7', 'perms2': {'global_access': 0, 'owner': '24c8d6f768c843a2ac83f5a8ff847073', 'owner_access': 7, 'share': []}, 'physical_router_dataplane_ip': '10.204.216.195', 'physical_router_management_ip': '10.204.216.195', 'physical_router_product_name': u'', 'physical_router_vendor_name': 'openworld', ...}
/usr/lib/python2.7/dist-packages/cfgm_common/vnc_db.py in locate(cls=<class 'device_manager.db.PhysicalRouterDM'>, key=u'bf335682-2edd-460d-80b8-07ab58ce3f8c', *args=({'display_name': 'br0', 'fq_name': ['default-global-system-config', 'br0'], 'id_perms': {'created': '2016-07-26T04:24:08.053092', 'creator': None, 'description': None, 'enable': True, 'last_modified': '2016-07-26T04:24:13.078715', 'permissions': {'group': 'KeystoneAdmin', 'group_access': 7, 'other_access': 7, 'owner': 'admin', 'owner_access': 7}, 'user_visible': True, 'uuid': {'uuid_lslong': 9275171865080250252L, 'uuid_mslong': 13777450802188469773L}}, 'parent_type': 'global-system-config', 'parent_uuid': 'f8570655-ebcb-4ee2-b85a-6795943ec4d7', 'perms2': {'global_access': 0, 'owner': '24c8d6f768c843a2ac83f5a8ff847073', 'owner_access': 7, 'share': []}, 'physical_router_dataplane_ip': '10.204.216.195', 'physical_router_management_ip': '10.204.216.195', 'physical_router_product_name': u'', 'physical_router_vendor_name': 'openworld', ...},))
62 if key not in cls._dict:
63 try:
64 obj = cls(key, *args)
65 cls._dict[key] = obj
66 return obj
obj undefined
cls = <class 'device_manager.db.PhysicalRouterDM'>
key = u'bf335682-2edd-460d-80b8-07ab58ce3f8c'
args = ({'display_name': 'br0', 'fq_name': ['default-global-system-config', 'br0'], 'id_perms': {'created': '2016-07-26T04:24:08.053092', 'creator': None, 'description': None, 'enable': True, 'last_modified': '2016-07-26T04:24:13.078715', 'permissions': {'group': 'KeystoneAdmin', 'group_access': 7, 'other_access': 7, 'owner': 'admin', 'owner_access': 7}, 'user_visible': True, 'uuid': {'uuid_lslong': 9275171865080250252L, 'uuid_mslong': 13777450802188469773L}}, 'parent_type': 'global-system-config', 'parent_uuid': 'f8570655-ebcb-4ee2-b85a-6795943ec4d7', 'perms2': {'global_access': 0, 'owner': '24c8d6f768c843a2ac83f5a8ff847073', 'owner_access': 7, 'share': []}, 'physical_router_dataplane_ip': '10.204.216.195', 'physical_router_management_ip': '10.204.216.195', 'physical_router_product_name': u'', 'physical_router_vendor_name': 'openworld', ...},)
/usr/lib/python2.7/dist-packages/device_manager/db.py in __init__(self=<device_manager.db.PhysicalRouterDM object>, uuid=u'bf335682-2edd-460d-80b8-07ab58ce3f8c', obj_dict={'display_name': 'br0', 'fq_name': ['default-global-system-config', 'br0'], 'id_perms': {'created': '2016-07-26T04:24:08.053092', 'creator': None, 'description': None, 'enable': True, 'last_modified': '2016-07-26T04:24:13.078715', 'permissions': {'group': 'KeystoneAdmin', 'group_access': 7, 'other_access': 7, 'owner': 'admin', 'owner_access': 7}, 'user_visible': True, 'uuid': {'uuid_lslong': 9275171865080250252L, 'uuid_mslong': 13777450802188469773L}}, 'parent_type': 'global-system-config', 'parent_uuid': 'f8570655-ebcb-4ee2-b85a-6795943ec4d7', 'perms2': {'global_access': 0, 'owner': '24c8d6f768c843a2ac83f5a8ff847073', 'owner_access': 7, 'share': []}, 'physical_router_dataplane_ip': '10.204.216.195', 'physical_router_management_ip': '10.204.216.195', 'physical_router_product_name': u'', 'physical_router_vendor_name': 'openworld', ...})
124 self.nc_handler_gl = vnc_greenlets.VncGreenlet("VNC Device Manager",
125 self.nc_handler)
126 self.uve_send()
127 # end __init__
128
self = <device_manager.db.PhysicalRouterDM object>
self.uve_send = <bound method PhysicalRouterDM.uve_send of <device_manager.db.PhysicalRouterDM object>>
/usr/lib/python2.7/dist-packages/device_manager/db.py in uve_send(self=<device_manager.db.PhysicalRouterDM object>, deleted=False)
751
752 pr_msg = UvePhysicalRouterConfigTrace(
753 data=pr_trace, sandesh=PhysicalRouterDM._sandesh)
754 pr_msg.send(sandesh=PhysicalRouterDM._sandesh)
755 # end uve_send
data undefined
pr_trace = UvePhysicalRouterConfig(name='br0', deleted=None...ected_bgp_router=None, last_commit_duration=None)
global sandesh = <module 'device_manager.sandesh.dm_introspect.tt...device_manager/sandesh/dm_introspect/ttypes.pyc'>
global PhysicalRouterDM = <class 'device_manager.db.PhysicalRouterDM'>
PhysicalRouterDM._sandesh = None
/usr/lib/python2.7/dist-packages/cfgm_common/uve/physical_router/ttypes.py in __init__(self=UvePhysicalRouterConfigTrace(_module='', _level=...ast_commit_duration=None), _category='', _role=0), data=UvePhysicalRouterConfig(name='br0', deleted=None...ected_bgp_router=None, last_commit_duration=None), table=None, sandesh=None)
352 sandesh_base.SandeshUVE.__init__(self)
353 self.data = data
354 self._scope = sandesh.scope()
355 self._module = sandesh.module()
356 self._source = sandesh.source_id()
self = UvePhysicalRouterConfigTrace(_module='', _level=...ast_commit_duration=None), _category='', _role=0)
self._scope = ''
sandesh = None
sandesh.scope undefined
<type 'exceptions.AttributeError'>: 'NoneType' object has no attribute 'scope'
__class__ = <type 'exceptions.AttributeError'>
__delattr__ = <method-wrapper '__delattr__' of exceptions.AttributeError object>
__dict__ = {}
__doc__ = 'Attribute not found.'
__format__ = <built-in method __format__ of exceptions.AttributeError object>
__getattribute__ = <method-wrapper '__getattribute__' of exceptions.AttributeError object>
__getitem__ = <method-wrapper '__getitem__' of exceptions.AttributeError object>
__getslice__ = <method-wrapper '__getslice__' of exceptions.AttributeError object>
__hash__ = <method-wrapper '__hash__' of exceptions.AttributeError object>
__init__ = <method-wrapper '__init__' of exceptions.AttributeError object>
__new__ = <built-in method __new__ of type object>
__reduce__ = <built-in method __reduce__ of exceptions.AttributeError object>
__reduce_ex__ = <built-in method __reduce_ex__ of exceptions.AttributeError object>
__repr__ = <method-wrapper '__repr__' of exceptions.AttributeError object>
__setattr__ = <method-wrapper '__setattr__' of exceptions.AttributeError object>
__setstate__ = <built-in method __setstate__ of exceptions.AttributeError object>
__sizeof__ = <built-in method __sizeof__ of exceptions.AttributeError object>
__str__ = <method-wrapper '__str__' of exceptions.AttributeError object>
__subclasshook__ = <built-in method __subclasshook__ of type object>
__unicode__ = <built-in method __unicode__ of exceptions.AttributeError object>
args = ("'NoneType' object has no attribute 'scope'",)
message = "'NoneType' object has no attribute 'scope'"
The above is a description of an error in a Python program. Here is
the original traceback:
Traceback (most recent call last):
File "/usr/bin/contrail-device-manager", line 9, in <module>
load_entry_point('device-manager==0.1dev', 'console_scripts', 'contrail-device-manager')()
File "/usr/lib/python2.7/dist-packages/device_manager/device_manager.py", line 482, in server_main
main()
File "/usr/lib/python2.7/dist-packages/device_manager/device_manager.py", line 471, in main
args)
File "/usr/lib/python2.7/dist-packages/cfgm_common/zkclient.py", line 346, in master_election
self._election.run(func, *args, **kwargs)
File "/usr/lib/python2.7/dist-packages/kazoo/recipe/election.py", line 48, in run
func(*args, **kwargs)
File "/usr/lib/python2.7/dist-packages/device_manager/device_manager.py", line 476, in run_device_manager
device_manager = DeviceManager(args)
File "/usr/lib/python2.7/dist-packages/device_manager/device_manager.py", line 210, in __init__
pr = PhysicalRouterDM.locate(obj['uuid'], obj)
File "/usr/lib/python2.7/dist-packages/cfgm_common/vnc_db.py", line 64, in locate
obj = cls(key, *args)
File "/usr/lib/python2.7/dist-packages/device_manager/db.py", line 126, in __init__
self.uve_send()
File "/usr/lib/python2.7/dist-packages/device_manager/db.py", line 753, in uve_send
data=pr_trace, sandesh=PhysicalRouterDM._sandesh)
File "/usr/lib/python2.7/dist-packages/cfgm_common/uve/physical_router/ttypes.py", line 354, in __init__
self._scope = sandesh.scope()
AttributeError: 'NoneType' object has no attribute 'scope'
------------------------------
Review in progress for https:/
Submitter: Suresh Balineni (<email address hidden>)