upgrade to 3.1 with healthcheck crashes svc-monitor

[Senthilnathan Murugappan]

upgrade to 3.1 from 3.0.2 with service healthcheck crashes svc-monitor

==> /var/log/contrail/contrail-svc-monitor.log <==
08/04/2016 01:22:26 AM [contrail-svc-monitor]: __default__ [SYS_ERR]: SvcMonitorLog: <type 'exceptions.KeyError'>
Python 2.7.6: /usr/bin/python
Thu Aug 4 01:22:26 2016

A problem occurred in a Python script. Here is the sequence of
function calls leading up to the error, in the order they occurred.

 /usr/lib/python2.7/dist-packages/svc_monitor/svc_monitor.py in sync_sm(self=<svc_monitor.svc_monitor.SvcMonitor object>)
  298 # invoke port tuple handling
  299 try:
  300 self.port_tuple_agent.update_port_tuples()
  301 except Exception:
  302 cgitb_error_log(self)
self = <svc_monitor.svc_monitor.SvcMonitor object>
self.port_tuple_agent = <svc_monitor.port_tuple.PortTupleAgent object>
self.port_tuple_agent.update_port_tuples = <bound method PortTupleAgent.update_port_tuples of <svc_monitor.port_tuple.PortTupleAgent object>>

 /usr/lib/python2.7/dist-packages/svc_monitor/port_tuple.py in update_port_tuples(self=<svc_monitor.port_tuple.PortTupleAgent object>)
  343 for si in ServiceInstanceSM.values():
  344 for pt_id in si.port_tuples:
  345 self.update_port_tuple(pt_id=pt_id)
  346 for iip in InstanceIpSM.values():
  347 self.delete_shared_iip(iip)
self = <svc_monitor.port_tuple.PortTupleAgent object>
self.update_port_tuple = <bound method PortTupleAgent.update_port_tuple of <svc_monitor.port_tuple.PortTupleAgent object>>
pt_id = '0d57778e-acbf-48bd-9db6-bd1c35b32893'

 /usr/lib/python2.7/dist-packages/svc_monitor/port_tuple.py in update_port_tuple(self=<svc_monitor.port_tuple.PortTupleAgent object>, vmi=<svc_monitor.config_db.VirtualMachineInterfaceSM object>, pt_id='0d57778e-acbf-48bd-9db6-bd1c35b32893')
  337 self.set_port_service_chain_ip(si, port, vmi)
  338 self.set_port_allowed_address_pairs(port, vmi, vmi_obj)
  339 self.set_port_service_health_check(si, port, vmi)
  340 self.set_port_static_routes(port, vmi)
  341
self = <svc_monitor.port_tuple.PortTupleAgent object>
self.set_port_service_health_check = <bound method PortTupleAgent.set_port_service_he...f <svc_monitor.port_tuple.PortTupleAgent object>>
si = <svc_monitor.config_db.ServiceInstanceSM object>
port = {'allowed-address-pairs': None, 'interface-route-tables': [], 'service-health-checks': [u'6083755b-3b20-4e91-a9ed-b5d4dc2d9ea6'], 'shared-ip': False, 'static-route-enable': False, 'type': u'left'}
vmi = <svc_monitor.config_db.VirtualMachineInterfaceSM object>

 /usr/lib/python2.7/dist-packages/svc_monitor/port_tuple.py in set_port_service_health_check(self=<svc_monitor.port_tuple.PortTupleAgent object>, si=<svc_monitor.config_db.ServiceInstanceSM object>, port={'allowed-address-pairs': None, 'interface-route-tables': [], 'service-health-checks': [u'6083755b-3b20-4e91-a9ed-b5d4dc2d9ea6'], 'shared-ip': False, 'static-route-enable': False, 'type': u'left'}, vmi=<svc_monitor.config_db.VirtualMachineInterfaceSM object>)
  153
  154 # update health check ip
  155 self.update_health_check_iip(si, port, vmi)
  156
  157 def set_port_static_routes(self, port, vmi):
self = <svc_monitor.port_tuple.PortTupleAgent object>
self.update_health_check_iip = <bound method PortTupleAgent.update_health_check...f <svc_monitor.port_tuple.PortTupleAgent object>>
si = <svc_monitor.config_db.ServiceInstanceSM object>
port = {'allowed-address-pairs': None, 'interface-route-tables': [], 'service-health-checks': [u'6083755b-3b20-4e91-a9ed-b5d4dc2d9ea6'], 'shared-ip': False, 'static-route-enable': False, 'type': u'left'}
vmi = <svc_monitor.config_db.VirtualMachineInterfaceSM object>

 /usr/lib/python2.7/dist-packages/svc_monitor/port_tuple.py in update_health_check_iip(self=<svc_monitor.port_tuple.PortTupleAgent object>, si=<svc_monitor.config_db.ServiceInstanceSM object>, port={'allowed-address-pairs': None, 'interface-route-tables': [], 'service-health-checks': [u'6083755b-3b20-4e91-a9ed-b5d4dc2d9ea6'], 'shared-ip': False, 'static-route-enable': False, 'type': u'left'}, vmi=<svc_monitor.config_db.VirtualMachineInterfaceSM object>)
  117 if if_type['interface_type'] != vmi.if_type:
  118 continue
  119 if health.params['health_check_type'] != 'end-to-end':
  120 continue
  121 allocate_hc_iip = True
health = <svc_monitor.config_db.ServiceHealthCheckSM object>
health.params = {u'delay': 3, u'enabled': True, u'expected_codes': None, u'http_method': None, u'max_retries': 2, u'monitor_type': u'HTTP', u'timeout': 5, u'url_path': u'http://local-ip'}
<type 'exceptions.KeyError'>: 'health_check_type'
    __class__ = <type 'exceptions.KeyError'>
    __delattr__ = <method-wrapper '__delattr__' of exceptions.KeyError object>
    __dict__ = {}
    __doc__ = 'Mapping key not found.'
    __format__ = <built-in method __format__ of exceptions.KeyError object>
    __getattribute__ = <method-wrapper '__getattribute__' of exceptions.KeyError object>
    __getitem__ = <method-wrapper '__getitem__' of exceptions.KeyError object>
    __getslice__ = <method-wrapper '__getslice__' of exceptions.KeyError object>
    __hash__ = <method-wrapper '__hash__' of exceptions.KeyError object>
    __init__ = <method-wrapper '__init__' of exceptions.KeyError object>
    __new__ = <built-in method __new__ of type object>
    __reduce__ = <built-in method __reduce__ of exceptions.KeyError object>
    __reduce_ex__ = <built-in method __reduce_ex__ of exceptions.KeyError object>
    __repr__ = <method-wrapper '__repr__' of exceptions.KeyError object>
    __setattr__ = <method-wrapper '__setattr__' of exceptions.KeyError object>
    __setstate__ = <built-in method __setstate__ of exceptions.KeyError object>
    __sizeof__ = <built-in method __sizeof__ of exceptions.KeyError object>
    __str__ = <method-wrapper '__str__' of exceptions.KeyError object>
    __subclasshook__ = <built-in method __subclasshook__ of type object>
    __unicode__ = <built-in method __unicode__ of exceptions.KeyError object>
    args = ('health_check_type',)
    message = 'health_check_type'

The above is a description of an error in a Python program. Here is
the original traceback:

Traceback (most recent call last):
  File "/usr/lib/python2.7/dist-packages/svc_monitor/svc_monitor.py", line 300, in sync_sm
    self.port_tuple_agent.update_port_tuples()
  File "/usr/lib/python2.7/dist-packages/svc_monitor/port_tuple.py", line 345, in update_port_tuples
    self.update_port_tuple(pt_id=pt_id)
  File "/usr/lib/python2.7/dist-packages/svc_monitor/port_tuple.py", line 339, in update_port_tuple
    self.set_port_service_health_check(si, port, vmi)
  File "/usr/lib/python2.7/dist-packages/svc_monitor/port_tuple.py", line 155, in set_port_service_health_check
    self.update_health_check_iip(si, port, vmi)
  File "/usr/lib/python2.7/dist-packages/svc_monitor/port_tuple.py", line 119, in update_health_check_iip
    if health.params['health_check_type'] != 'end-to-end':
KeyError: 'health_check_type'

[OpenContrail Admin (ci-admin-f)]

Review in progress for https://review.opencontrail.org/22886
Submitter: Senthilnathan Murugappan (<email address hidden>)

[OpenContrail Admin (ci-admin-f)]

Review in progress for https://review.opencontrail.org/22887
Submitter: Senthilnathan Murugappan (<email address hidden>)

[OpenContrail Admin (ci-admin-f)]

Reviewed: https://review.opencontrail.org/22886
Committed: http://github.org/Juniper/contrail-controller/commit/b3247b4681350c316bd7e4ef26069a8a22b92b33
Submitter: Zuul
Branch: R3.1

commit b3247b4681350c316bd7e4ef26069a8a22b92b33
Author: Senthilnathan Murugappan <email address hidden>
Date: Thu Aug 4 12:28:55 2016 -0700

Check for health_check_type key existence before accessing

Change-Id: I0c55073cb6442e15d1bdbb39c2a87450d7dbaba4
Closes-Bug:#1609685

[OpenContrail Admin (ci-admin-f)]

Reviewed: https://review.opencontrail.org/22887
Committed: http://github.org/Juniper/contrail-controller/commit/4fb5d5a4071bb789b1050d1ab7e81612649e281d
Submitter: Zuul
Branch: master

commit 4fb5d5a4071bb789b1050d1ab7e81612649e281d
Author: Senthilnathan Murugappan <email address hidden>
Date: Thu Aug 4 12:28:55 2016 -0700

Check for health_check_type key existence before accessing

Change-Id: I0c55073cb6442e15d1bdbb39c2a87450d7dbaba4
Closes-Bug:#1609685