Authorization for svc-monitor failed after running for a while

[Tony Liu]

After svc-monitor running for a day or so, authentication fails.
Need to restart svc-monitor to work this around. But, after restart, multiple service instances are created
by svc-monitor.

Here is the logging from /var/log/contrail/svc-monitor.err.
------------------------------------------------------------------
<class 'novaclient.exceptions.Unauthorized'>
Python 2.7.5: /opt/contrail/api-venv/bin/python
Mon Jun 30 06:19:07 2014

A problem occurred in a Python script. Here is the sequence of
function calls leading up to the error, in the order they occurred.

 /opt/contrail/api-venv/lib/python2.7/site-packages/svc_monitor/svc_monitor.py in launch_arc(monitor=<__main__.SvcMonitor object>, ssrc_mapc=<cfgm_common.ifmap.client.client instance>)
  998 pollreq = PollRequest(arc_mapc.get_session_id())
  999 result = arc_mapc.call('poll', pollreq)
 1000 monitor.process_poll_result(result)
 1001 except Exception as e:
 1002 if type(e) == socket.error:
monitor = <__main__.SvcMonitor object>
monitor.process_poll_result = <bound method SvcMonitor.process_poll_result of <__main__.SvcMonitor object>>
result = '<?xml version="1.0" encoding="UTF-8" standalone=...lResult></ns2:response></ns3:Body></ns3:Envelope>'

 /opt/contrail/api-venv/lib/python2.7/site-packages/svc_monitor/svc_monitor.py in process_poll_result(self=<__main__.SvcMonitor object>, poll_result_str='<?xml version="1.0" encoding="UTF-8" standalone=...lResult></ns2:response></ns3:Body></ns3:Envelope>')
  844 self._svc_syslog("%s with %s/%s"
  845 % (funcname, meta_name, idents))
  846 func(idents)
  847 # end for meta
  848 # end for result_type
func = <bound method SvcMonitor._addmsg_service_instance_service_template of <__main__.SvcMonitor object>>
idents = {'service-instance': 'default-domain:acme:vsrx-test', 'service-template': 'default-domain:vsrx-nat'}

 /opt/contrail/api-venv/lib/python2.7/site-packages/svc_monitor/svc_monitor.py in _addmsg_service_instance_service_template(self=<__main__.SvcMonitor object>, idents={'service-instance': 'default-domain:acme:vsrx-test', 'service-template': 'default-domain:vsrx-nat'})
  784
  785 #launch VMs
  786 self._create_svc_instance_vm(st_obj, si_obj)
  787 # end _addmsg_service_instance_service_template
  788
self = <__main__.SvcMonitor object>
self._create_svc_instance_vm = <bound method SvcMonitor._create_svc_instance_vm of <__main__.SvcMonitor object>>
st_obj = <vnc_api.gen.resource_client.ServiceTemplate object>
si_obj = <vnc_api.gen.resource_client.ServiceInstance object>

 /opt/contrail/api-venv/lib/python2.7/site-packages/svc_monitor/svc_monitor.py in _create_svc_instance_vm(self=<__main__.SvcMonitor object>, st_obj=<vnc_api.gen.resource_client.ServiceTemplate object>, si_obj=<vnc_api.gen.resource_client.ServiceInstance object>)
  493 else:
  494 vm = self._create_svc_vm(instance_name, image_name,
  495 nics, proj_obj.name, flavor)
  496 if vm is None:
  497 continue
nics = [{'net-id': u'21f6426b-24e1-4459-a844-e6dab422ac97', 'v4-fixed-ip': None}, {'net-id': u'434526ab-e571-4244-86f6-7988e862c634', 'v4-fixed-ip': None}, {'net-id': u'dbebd472-564f-432c-9610-be40ef446561', 'v4-fixed-ip': None}]
proj_obj = <vnc_api.gen.resource_client.Project object>
proj_obj.name = u'acme'
flavor = u'm1.medium'

 /opt/contrail/api-venv/lib/python2.7/site-packages/svc_monitor/svc_monitor.py in _create_svc_vm(self=<__main__.SvcMonitor object>, vm_name=u'vsrx-test_1', image_name=u'vSRX 12.1X46-D20.5', nics=[{'net-id': u'21f6426b-24e1-4459-a844-e6dab422ac97', 'v4-fixed-ip': None}, {'net-id': u'434526ab-e571-4244-86f6-7988e862c634', 'v4-fixed-ip': None}, {'net-id': u'dbebd472-564f-432c-9610-be40ef446561', 'v4-fixed-ip': None}], proj_name=u'acme', flavor_name=u'm1.medium')
  892 n_client = self._novaclient_get(proj_name)
  893 if flavor_name:
  894 flavor = n_client.flavors.find(name=flavor_name)
  895 else:
  896 flavor = n_client.flavors.find(ram=4096)
flavor undefined
n_client = <novaclient.v1_1.client.Client object>
n_client.flavors = <novaclient.v1_1.flavors.FlavorManager object>
n_client.flavors.find = <bound method FlavorManager.find of <novaclient.v1_1.flavors.FlavorManager object>>
name undefined
flavor_name = u'm1.medium'

 /opt/contrail/api-venv/lib/python2.7/site-packages/novaclient/base.py in find(self=<novaclient.v1_1.flavors.FlavorManager object>, **kwargs={'name': u'm1.medium'})
  175 the Python side.
  176 """
  177 matches = self.findall(**kwargs)
  178 num_matches = len(matches)
  179 if num_matches == 0:
matches undefined
self = <novaclient.v1_1.flavors.FlavorManager object>
self.findall = <bound method FlavorManager.findall of <novaclient.v1_1.flavors.FlavorManager object>>
kwargs = {'name': u'm1.medium'}

 /opt/contrail/api-venv/lib/python2.7/site-packages/novaclient/base.py in findall(self=<novaclient.v1_1.flavors.FlavorManager object>, **kwargs={'name': u'm1.medium'})
  195 searches = kwargs.items()
  196
  197 for obj in self.list():
  198 try:
  199 if all(getattr(obj, attr) == value
obj undefined
self = <novaclient.v1_1.flavors.FlavorManager object>
self.list = <bound method FlavorManager.list of <novaclient.v1_1.flavors.FlavorManager object>>

 /opt/contrail/api-venv/lib/python2.7/site-packages/novaclient/v1_1/flavors.py in list(self=<novaclient.v1_1.flavors.FlavorManager object>, detailed=True)
   84 """
   85 if detailed is True:
   86 return self._list("/flavors/detail", "flavors")
   87 else:
   88 return self._list("/flavors", "flavors")
self = <novaclient.v1_1.flavors.FlavorManager object>
self._list = <bound method FlavorManager._list of <novaclient.v1_1.flavors.FlavorManager object>>

 /opt/contrail/api-venv/lib/python2.7/site-packages/novaclient/base.py in _list(self=<novaclient.v1_1.flavors.FlavorManager object>, url='/flavors/detail', response_key='flavors', obj_class=None, body=None)
   60 _resp, body = self.api.client.post(url, body=body)
   61 else:
   62 _resp, body = self.api.client.get(url)
   63
   64 if obj_class is None:
_resp undefined
body = None
self = <novaclient.v1_1.flavors.FlavorManager object>
self.api = <novaclient.v1_1.client.Client object>
self.api.client = <novaclient.client.HTTPClient object>
self.api.client.get = <bound method HTTPClient.get of <novaclient.client.HTTPClient object>>
url = '/flavors/detail'

 /opt/contrail/api-venv/lib/python2.7/site-packages/novaclient/client.py in get(self=<novaclient.client.HTTPClient object>, url='/flavors/detail', **kwargs={})
  228
  229 def get(self, url, **kwargs):
  230 return self._cs_request(url, 'GET', **kwargs)
  231
  232 def post(self, url, **kwargs):
self = <novaclient.client.HTTPClient object>
self._cs_request = <bound method HTTPClient._cs_request of <novaclient.client.HTTPClient object>>
url = '/flavors/detail'
kwargs = {}

 /opt/contrail/api-venv/lib/python2.7/site-packages/novaclient/client.py in _cs_request(self=<novaclient.client.HTTPClient object>, url='/flavors/detail', method='GET', **kwargs={'headers': {'Accept': 'application/json', 'User-Agent': 'python-novaclient', 'X-Auth-Project-Id': u'acme', 'X-Auth-Token': u'MIIKzAYJKoZIhvcNAQcCoIIKvTCCCrkCAQExCTAHBgUrDg...DyaMQO0cma0nkj9K693ygqKBC8TEki0eHd4qOufZ7Gx4Dbc='}})
  225 return resp, body
  226 except exceptions.Unauthorized:
  227 raise e
  228
  229 def get(self, url, **kwargs):
e = Unauthorized()
<class 'novaclient.exceptions.Unauthorized'>: Unauthorized (HTTP 401)
    __class__ = <class 'novaclient.exceptions.Unauthorized'>
    __delattr__ = <method-wrapper '__delattr__' of Unauthorized object>
    __dict__ = {'code': 401, 'details': None, 'message': 'Unauthorized', 'method': 'GET', 'request_id': None, 'url': u'http://10.84.18.4:8774/v1.1/0a835176cb684ed6b8f03955efd96f71/flavors/detail'}
    __doc__ = '\n HTTP 401 - Unauthorized: bad credentials.\n '
    __format__ = <built-in method __format__ of Unauthorized object>
    __getattribute__ = <method-wrapper '__getattribute__' of Unauthorized object>
    __getitem__ = <method-wrapper '__getitem__' of Unauthorized object>
    __getslice__ = <method-wrapper '__getslice__' of Unauthorized object>
    __hash__ = <method-wrapper '__hash__' of Unauthorized object>
    __init__ = <bound method Unauthorized.__init__ of Unauthorized()>
    __module__ = 'novaclient.exceptions'
    __new__ = <built-in method __new__ of type object>
    __reduce__ = <built-in method __reduce__ of Unauthorized object>
    __reduce_ex__ = <built-in method __reduce_ex__ of Unauthorized object>
    __repr__ = <method-wrapper '__repr__' of Unauthorized object>
    __setattr__ = <method-wrapper '__setattr__' of Unauthorized object>
    __setstate__ = <built-in method __setstate__ of Unauthorized object>
    __sizeof__ = <built-in method __sizeof__ of Unauthorized object>
    __str__ = <bound method Unauthorized.__str__ of Unauthorized()>
    __subclasshook__ = <built-in method __subclasshook__ of type object>
    __unicode__ = <built-in method __unicode__ of Unauthorized object>
    __weakref__ = None
    args = ()
    code = 401
    details = None
    http_status = 401
    message = 'Unauthorized'
    method = 'GET'
    request_id = None
    url = u'http://10.84.18.4:8774/v1.1/0a835176cb684ed6b8f03955efd96f71/flavors/detail'

The above is a description of an error in a Python program. Here is
the original traceback:

Traceback (most recent call last):
  File "/opt/contrail/api-venv/lib/python2.7/site-packages/svc_monitor/svc_monitor.py", line 1000, in launch_arc
    monitor.process_poll_result(result)
  File "/opt/contrail/api-venv/lib/python2.7/site-packages/svc_monitor/svc_monitor.py", line 846, in process_poll_result
    func(idents)
  File "/opt/contrail/api-venv/lib/python2.7/site-packages/svc_monitor/svc_monitor.py", line 786, in _addmsg_service_instance_service_template
    self._create_svc_instance_vm(st_obj, si_obj)
  File "/opt/contrail/api-venv/lib/python2.7/site-packages/svc_monitor/svc_monitor.py", line 495, in _create_svc_instance_vm
    nics, proj_obj.name, flavor)
  File "/opt/contrail/api-venv/lib/python2.7/site-packages/svc_monitor/svc_monitor.py", line 894, in _create_svc_vm
    flavor = n_client.flavors.find(name=flavor_name)
  File "/opt/contrail/api-venv/lib/python2.7/site-packages/novaclient/base.py", line 177, in find
    matches = self.findall(**kwargs)
  File "/opt/contrail/api-venv/lib/python2.7/site-packages/novaclient/base.py", line 197, in findall
    for obj in self.list():
  File "/opt/contrail/api-venv/lib/python2.7/site-packages/novaclient/v1_1/flavors.py", line 86, in list
    return self._list("/flavors/detail", "flavors")
  File "/opt/contrail/api-venv/lib/python2.7/site-packages/novaclient/base.py", line 62, in _list
    _resp, body = self.api.client.get(url)
  File "/opt/contrail/api-venv/lib/python2.7/site-packages/novaclient/client.py", line 230, in get
    return self._cs_request(url, 'GET', **kwargs)
  File "/opt/contrail/api-venv/lib/python2.7/site-packages/novaclient/client.py", line 227, in _cs_request
    raise e
Unauthorized: Unauthorized (HTTP 401)

[Rudra Rugge (rrugge)]

Fix committed

https://github.com/Juniper/contrail-controller/commit/9fb5a5dcdabaefa31bf48c6f288035891c44d449

[Rudra Rugge (rrugge)]

Backported to R1.05
https://github.com/Juniper/contrail-controller/commit/5905b2461a0019c97f997da33ab6c87829486bfd