service monitor repeatedly crashing after tempest run

[Vedamurthy Joshi]

R1.10 Build 38 Centos 6.5 Icehouse

service monitor is repeatedly crashing on a multi-cfgm setup.

Logs of two config nodes nodei34 and nodei35 will be in http://10.204.216.50/Docs/bugs/#

Tasks: 679 total, 3 running, 676 sleeping, 0 stopped, 0 zombie
Cpu(s): 5.9%us, 0.8%sy, 0.0%ni, 93.2%id, 0.1%wa, 0.0%hi, 0.1%si, 0.0%st
Mem: 198310572k total, 24413604k used, 173896968k free, 186256k buffers
Swap: 4194296k total, 0k used, 4194296k free, 5078268k cached

  PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND
17389 root 20 0 377m 136m 6200 R 75.1 0.1 117:07.64 contrail-svc-mo
19379 root 20 0 2251m 119m 8052 S 59.7 0.1 219:57.99 contrail-collec
17387 root 20 0 579m 318m 6620 S 36.6 0.2 94:04.50 contrail-api
12438 root 20 0 25.7g 7.5g 830m S 25.0 4.0 77:04.24 java
 6344 root 20 0 296m 43m 6288 S 19.3 0.0 10:56.64 contrail-schema
17713 rabbitmq 20 0 4591m 131m 3152 S 15.4 0.1 15:59.57 beam.smp
13112 root 20 0 15424 1672 888 R 3.9 0.0 0:00.02 top
16129 mysql 20 0 7889m 69m 6476 S 3.9 0.0 0:48.57 mysqld
16290 cinder 20 0 304m 50m 5308 S 1.9 0.0 1:43.66 python
16465 nova 20 0 404m 74m 3252 S 1.9 0.0 0:09.58 python
16468 nova 20 0 403m 73m 3252 S 1.9 0.0 0:09.37 python
16471 nova 20 0 404m 74m 3228 S 1.9 0.0 0:09.50 python
16487 nova 20 0 405m 74m 3228 S 1.9 0.0 0:09.68 python
16488 nova 20 0 405m 74m 3252 S 1.9 0.0 0:09.71 python
16494 nova 20 0 406m 76m 3252 S 1.9 0.0 0:09.88 python
16495 nova 20 0 403m 73m 3252 S 1.9 0.0 0:09.50 python
16496 nova 20 0 408m 78m 3252 S 1.9 0.0 0:09.91 python
16497 nova 20 0 403m 73m 3252 S 1.9 0.0 0:09.72 python
16507 nova 20 0 402m 71m 3252 S 1.9 0.0 0:09.17 python
16509 nova 20 0 404m 74m 3252 S 1.9 0.0 0:09.71 python
16510 nova 20 0 402m 72m 3252 S 1.9 0.0 0:09.29 python
16514 nova 20 0 402m 71m 3248 S 1.9 0.0 0:09.55 python
17275 root 20 0 195m 11m 1492 S 1.9 0.0 4:10.34 supervisord
17386 root 20 0 34.9g 818m 11m S 1.9 0.4 5:52.91 java
18797 haproxy 20 0 19468 2088 576 R 1.9 0.0 4:28.15 haproxy
    1 root 20 0 19232 1496 1220 S 0.0 0.0 0:02.44 init
[root@nodei34 contrail]#

The svc-monitor.err log is full of these :

<type 'exceptions.NameError'>
Python 2.6.6: /usr/bin/python
Wed Sep 24 01:39:48 2014

A problem occurred in a Python script. Here is the sequence of
function calls leading up to the error, in the order they occurred.

 /usr/lib/python2.6/site-packages/svc_monitor/svc_monitor.py in launch_arc(monitor=<svc_monitor.svc_monitor.SvcMonitor object>, ssrc_mapc=<cfgm_common.ifmap.client.client instance>)
  566 pollreq = PollRequest(arc_mapc.get_session_id())
  567 result = arc_mapc.call('poll', pollreq)
  568 monitor.process_poll_result(result)
  569 except Exception as e:
  570 if type(e) == socket.error:
monitor = <svc_monitor.svc_monitor.SvcMonitor object>
monitor.process_poll_result = <bound method SvcMonitor.process_poll_result of <svc_monitor.svc_monitor.SvcMonitor object>>
result = '<?xml version="1.0" encoding="UTF-8" standalone=...lResult></ns2:response></ns3:Body></ns3:Envelope>'

 /usr/lib/python2.6/site-packages/svc_monitor/svc_monitor.py in process_poll_result(self=<svc_monitor.svc_monitor.SvcMonitor object>, poll_result_str='<?xml version="1.0" encoding="UTF-8" standalone=...lResult></ns2:response></ns3:Body></ns3:Envelope>')
  522 self.logger.log("%s with %s/%s"
  523 % (funcname, meta_name, idents))
  524 func(idents)
  525 # end for meta
  526 # end for result_type
func = <bound method SvcMonitor._addmsg_service_instanc...e of <svc_monitor.svc_monitor.SvcMonitor object>>
idents = {'service-instance': 'default-domain:RoutersTest-1064616415:si_02e683b1-55b7-4cbf-bb4e-d483c6c9d842', 'service-template': 'default-domain:netns-snat-template'}

 /usr/lib/python2.6/site-packages/svc_monitor/svc_monitor.py in _addmsg_service_instance_service_template(self=<svc_monitor.svc_monitor.SvcMonitor object>, idents={'service-instance': 'default-domain:RoutersTest-1064616415:si_02e683b1-55b7-4cbf-bb4e-d483c6c9d842', 'service-template': 'default-domain:netns-snat-template'})
  458
  459 #launch VMs
  460 self._create_svc_instance(st_obj, si_obj)
  461 # end _addmsg_service_instance_service_template
  462
self = <svc_monitor.svc_monitor.SvcMonitor object>
self._create_svc_instance = <bound method SvcMonitor._create_svc_instance of <svc_monitor.svc_monitor.SvcMonitor object>>
st_obj = <vnc_api.gen.resource_client.ServiceTemplate object>
si_obj = <vnc_api.gen.resource_client.ServiceInstance object>

 /usr/lib/python2.6/site-packages/svc_monitor/svc_monitor.py in _create_svc_instance(self=<svc_monitor.svc_monitor.SvcMonitor object>, st_obj=<vnc_api.gen.resource_client.ServiceTemplate object>, si_obj=<vnc_api.gen.resource_client.ServiceInstance object>)
  291 self.vm_manager.create_service(st_obj, si_obj)
  292 elif virt_type == 'network-namespace':
  293 self.netns_manager.create_service(st_obj, si_obj)
  294
  295 def _delete_svc_instance(self, vm_uuid, proj_name,
self = <svc_monitor.svc_monitor.SvcMonitor object>
self.netns_manager = <svc_monitor.instance_manager.NetworkNamespaceManager object>
self.netns_manager.create_service = <bound method NetworkNamespaceManager.create_ser...instance_manager.NetworkNamespaceManager object>>
st_obj = <vnc_api.gen.resource_client.ServiceTemplate object>
si_obj = <vnc_api.gen.resource_client.ServiceInstance object>

 /usr/lib/python2.6/site-packages/svc_monitor/instance_manager.py in create_service(self=<svc_monitor.instance_manager.NetworkNamespaceManager object>, st_obj=<vnc_api.gen.resource_client.ServiceTemplate object>, si_obj=<vnc_api.gen.resource_client.ServiceInstance object>)
  506 for nic in nics:
  507 vmi_obj = self._create_svc_vm_port(nic, instance_name, st_obj,
  508 si_obj, local_preference)
  509 vmi_obj.set_virtual_machine(vm_obj)
  510 self._vnc_lib.virtual_machine_interface_update(vmi_obj)
si_obj = <vnc_api.gen.resource_client.ServiceInstance object>
local_preference = 200

 /usr/lib/python2.6/site-packages/svc_monitor/instance_manager.py in _create_svc_vm_port(self=<svc_monitor.instance_manager.NetworkNamespaceManager object>, nic={'net-id': u'28fd680f-230c-4b13-a502-96e26f41f064', 'shared-ip': True, 'static-route-enable': False, 'static-routes': None, 'type': u'right'}, instance_name=u'default-domain__RoutersTest-1064616415__si_02e683b1-55b7-4cbf-bb4e-d483c6c9d842_1', st_obj=<vnc_api.gen.resource_client.ServiceTemplate object>, si_obj=<vnc_api.gen.resource_client.ServiceInstance object>, local_preference=200)
  146 if (st_props.service_mode in ['in-network', 'in-network-nat'] and
  147 proj_obj.name != 'default-project'):
  148 sg_obj = self._get_default_security_group(vn_obj)
  149 vmi_obj.set_security_group(sg_obj)
  150 if nic['static-route-enable']:
sg_obj undefined
self = <svc_monitor.instance_manager.NetworkNamespaceManager object>
self._get_default_security_group = <bound method NetworkNamespaceManager._get_defau...instance_manager.NetworkNamespaceManager object>>
vn_obj = <vnc_api.gen.resource_client.VirtualNetwork object>

 /usr/lib/python2.6/site-packages/svc_monitor/instance_manager.py in _get_default_security_group(self=<svc_monitor.instance_manager.NetworkNamespaceManager object>, vn_obj=<vnc_api.gen.resource_client.VirtualNetwork object>)
   62 except Exception as e:
   63 self.logger.log(
   64 "Error: Security group default not found %s" % (proj_obj.name))
   65 return sg_obj
   66
proj_obj undefined
<type 'exceptions.NameError'>: global name 'proj_obj' is not defined
    __class__ = <type 'exceptions.NameError'>
    __delattr__ = <method-wrapper '__delattr__' of exceptions.NameError object>
    __dict__ = {}
    __doc__ = 'Name not found globally.'
    __format__ = <built-in method __format__ of exceptions.NameError object>
    __getattribute__ = <method-wrapper '__getattribute__' of exceptions.NameError object>
    __getitem__ = <method-wrapper '__getitem__' of exceptions.NameError object>
    __getslice__ = <method-wrapper '__getslice__' of exceptions.NameError object>
    __hash__ = <method-wrapper '__hash__' of exceptions.NameError object>
    __init__ = <method-wrapper '__init__' of exceptions.NameError object>
    __new__ = <built-in method __new__ of type object>
    __reduce__ = <built-in method __reduce__ of exceptions.NameError object>
    __reduce_ex__ = <built-in method __reduce_ex__ of exceptions.NameError object>
    __repr__ = <method-wrapper '__repr__' of exceptions.NameError object>
    __setattr__ = <method-wrapper '__setattr__' of exceptions.NameError object>
    __setstate__ = <built-in method __setstate__ of exceptions.NameError object>
    __sizeof__ = <built-in method __sizeof__ of exceptions.NameError object>
    __str__ = <method-wrapper '__str__' of exceptions.NameError object>
    __subclasshook__ = <built-in method __subclasshook__ of type object>
    __unicode__ = <built-in method __unicode__ of exceptions.NameError object>
    __subclasshook__ = <built-in method __subclasshook__ of type object>
    __unicode__ = <built-in method __unicode__ of exceptions.NameError object>
    args = ("global name 'proj_obj' is not defined",)
    message = "global name 'proj_obj' is not defined"

The above is a description of an error in a Python program. Here is
the original traceback:

Traceback (most recent call last):
  File "/usr/lib/python2.6/site-packages/svc_monitor/svc_monitor.py", line 568, in launch_arc
    monitor.process_poll_result(result)
  File "/usr/lib/python2.6/site-packages/svc_monitor/svc_monitor.py", line 524, in process_poll_result
    func(idents)
  File "/usr/lib/python2.6/site-packages/svc_monitor/svc_monitor.py", line 460, in _addmsg_service_instance_service_template
    self._create_svc_instance(st_obj, si_obj)
  File "/usr/lib/python2.6/site-packages/svc_monitor/svc_monitor.py", line 293, in _create_svc_instance
    self.netns_manager.create_service(st_obj, si_obj)
  File "/usr/lib/python2.6/site-packages/svc_monitor/instance_manager.py", line 508, in create_service
    si_obj, local_preference)
  File "/usr/lib/python2.6/site-packages/svc_monitor/instance_manager.py", line 148, in _create_svc_vm_port
    sg_obj = self._get_default_security_group(vn_obj)
  File "/usr/lib/python2.6/site-packages/svc_monitor/instance_manager.py", line 64, in _get_default_security_group
    "Error: Security group default not found %s" % (proj_obj.name))
NameError: global name 'proj_obj' is not defined

[Rudra Rugge (rrugge)]

R1.10:
https://github.com/Juniper/contrail-controller/commit/2cde09024391c863684778e6facdb975af16e700

Master:
https://github.com/Juniper/contrail-controller/commit/507c3b7b541c453a382e6f6a30244eec621cbc13