Schema crashing upon upgrade from 2709 to 2711

[amit surana]

Upgraded the cluster from 3.0 b2709 to 2711. On one of the controller nodes, schema is crashing. BGPaaS configs were present pre-upgrade.

02/08/2016 05:32:32 PM [contrail-schema]: SANDESH: CONNECT TO COLLECTOR: True

02/08/2016 05:32:32 PM [contrail-schema]: SANDESH: Logging: LEVEL: [SYS_INFO] -> [SYS_NOTICE]
<type 'exceptions.IndexError'>
Python 2.7.6: /usr/bin/python
Mon Feb 8 17:32:33 2016

A problem occurred in a Python script. Here is the sequence of
function calls leading up to the error, in the order they occurred.

 /usr/bin/contrail-schema in <module>()
    6
    7 if __name__ == '__main__':
    8 sys.exit(
    9 load_entry_point('schema-transformer==0.1dev', 'console_scripts', 'contrail-schema')()
   10 )
load_entry_point = <function load_entry_point>

 /usr/lib/python2.7/dist-packages/schema_transformer/to_bgp.py in server_main()
  837 def server_main():
  838 cgitb.enable(format='text')
  839 main()
  840 # end server_main
  841
global main = <function main>

 /usr/lib/python2.7/dist-packages/schema_transformer/to_bgp.py in main(args_str='--conf_file /etc/contrail/contrail-schema.conf -... --conf_file /etc/contrail/contrail-database.conf')
  831 _zookeeper_client.master_election(zk_path_pfx + "/schema-transformer",
  832 os.getpid(), run_schema_transformer,
  833 args)
  834 # end main
  835
args = Namespace(admin_password='contrail123', admin_te...7:2181,172.16.180.8:2181', zk_server_port='2181')

 /usr/lib/python2.7/dist-packages/cfgm_common/zkclient.py in master_election(self=<cfgm_common.zkclient.ZookeeperClient object>, path='/schema-transformer', identifier=39898, func=<function run_schema_transformer>, *args=(Namespace(admin_password='contrail123', admin_te...7:2181,172.16.180.8:2181', zk_server_port='2181'),), **kwargs={})
  334 def master_election(self, path, identifier, func, *args, **kwargs):
  335 self._election = self._zk_client.Election(path, identifier)
  336 self._election.run(func, *args, **kwargs)
  337 # end master_election
  338
self = <cfgm_common.zkclient.ZookeeperClient object>
self._election = <kazoo.recipe.election.Election object>
self._election.run = <bound method Election.run of <kazoo.recipe.election.Election object>>
func = <function run_schema_transformer>
args = (Namespace(admin_password='contrail123', admin_te...7:2181,172.16.180.8:2181', zk_server_port='2181'),)
kwargs = {}

 /usr/lib/python2.7/dist-packages/kazoo/recipe/election.py in run(self=<kazoo.recipe.election.Election object>, func=<function run_schema_transformer>, *args=(Namespace(admin_password='contrail123', admin_te...7:2181,172.16.180.8:2181', zk_server_port='2181'),), **kwargs={})
   46 try:
   47 with self.lock:
   48 func(*args, **kwargs)
   49
   50 except CancelledError:
func = <function run_schema_transformer>
args = (Namespace(admin_password='contrail123', admin_te...7:2181,172.16.180.8:2181', zk_server_port='2181'),)
kwargs = {}

 /usr/lib/python2.7/dist-packages/schema_transformer/to_bgp.py in run_schema_transformer(args=Namespace(admin_password='contrail123', admin_te...7:2181,172.16.180.8:2181', zk_server_port='2181'))
  812
  813 global transformer
  814 transformer = SchemaTransformer(args)
  815 gevent.joinall(transformer._vnc_kombu.greenlets())
  816 # end run_schema_transformer
global transformer = None
global SchemaTransformer = <class 'schema_transformer.to_bgp.SchemaTransformer'>
args = Namespace(admin_password='contrail123', admin_te...7:2181,172.16.180.8:2181', zk_server_port='2181')

 /usr/lib/python2.7/dist-packages/schema_transformer/to_bgp.py in __init__(self=<schema_transformer.to_bgp.SchemaTransformer object>, args=Namespace(admin_password='contrail123', admin_te...7:2181,172.16.180.8:2181', zk_server_port='2181'))
  214 DBBaseST._vnc_lib = _vnc_lib
  215 ServiceChain.init()
  216 self.reinit()
  217 # create cpu_info object to send periodic updates
  218 sysinfo_req = False
self = <schema_transformer.to_bgp.SchemaTransformer object>
self.reinit = <bound method SchemaTransformer.reinit of <schema_transformer.to_bgp.SchemaTransformer object>>

 /usr/lib/python2.7/dist-packages/schema_transformer/to_bgp.py in reinit(self=<schema_transformer.to_bgp.SchemaTransformer object>)
  428 VirtualNetworkST.locate(vn.get_fq_name_str(), vn, vn_acl_dict)
  429 for ri_name, ri_obj in ri_dict.items():
  430 RoutingInstanceST.locate(ri_name, ri_obj)
  431
  432 for policy in NetworkPolicyST.list_vnc_obj():
global RoutingInstanceST = <class 'schema_transformer.config_db.RoutingInstanceST'>
RoutingInstanceST.locate = <bound method __metaclass__.locate of <class 'schema_transformer.config_db.RoutingInstanceST'>>
ri_name = u'default-domain:bgp-test:vsrx-bgp-vn:vsrx-bgp-vn'
ri_obj = <vnc_api.gen.resource_client.RoutingInstance object>

 /usr/lib/python2.7/dist-packages/schema_transformer/config_db.py in locate(cls=<class 'schema_transformer.config_db.RoutingInstanceST'>, key=u'default-domain:bgp-test:vsrx-bgp-vn:vsrx-bgp-vn', *args=(<vnc_api.gen.resource_client.RoutingInstance object>,))
   96 @classmethod
   97 def locate(cls, key, *args):
   98 obj = super(DBBaseST, cls).locate(key, *args)
   99 if obj.obj.uuid not in cls._uuid_fq_name_map:
  100 cls._uuid_fq_name_map[obj.obj.uuid] = key
obj undefined
builtinsuper = <type 'super'>
global DBBaseST = <class 'schema_transformer.config_db.DBBaseST'>
cls = <class 'schema_transformer.config_db.RoutingInstanceST'>
).locate = <bound method __metaclass__.locate of <class 'schema_transformer.config_db.RoutingInstanceST'>>
key = u'default-domain:bgp-test:vsrx-bgp-vn:vsrx-bgp-vn'
args = (<vnc_api.gen.resource_client.RoutingInstance object>,)

 /usr/lib/python2.7/dist-packages/cfgm_common/vnc_db.py in locate(cls=<class 'schema_transformer.config_db.RoutingInstanceST'>, key=u'default-domain:bgp-test:vsrx-bgp-vn:vsrx-bgp-vn', *args=(<vnc_api.gen.resource_client.RoutingInstance object>,))
   62 if key not in cls._dict:
   63 try:
   64 obj = cls(key, *args)
   65 cls._dict[key] = obj
   66 return obj
obj undefined
cls = <class 'schema_transformer.config_db.RoutingInstanceST'>
key = u'default-domain:bgp-test:vsrx-bgp-vn:vsrx-bgp-vn'
args = (<vnc_api.gen.resource_client.RoutingInstance object>,)

 /usr/lib/python2.7/dist-packages/schema_transformer/config_db.py in __init__(self=<schema_transformer.config_db.RoutingInstanceST object>, name=u'default-domain:bgp-test:vsrx-bgp-vn:vsrx-bgp-vn', obj=<vnc_api.gen.resource_client.RoutingInstance object>)
 1832 for connection in self.connections:
 1833 remote_ri_fq_name = connection.split(':')
 1834 if remote_ri_fq_name[-1] == remote_ri_fq_name[-2]:
 1835 vn.connections.add(':'.join(remote_ri_fq_name[0:-1] ))
 1836 vmi_refs = self.obj.get_virtual_machine_interface_back_refs() or []
remote_ri_fq_name = ['ERROR']
<type 'exceptions.IndexError'>: list index out of range
    __class__ = <type 'exceptions.IndexError'>
    __delattr__ = <method-wrapper '__delattr__' of exceptions.IndexError object>
    __dict__ = {}
    __doc__ = 'Sequence index out of range.'
    __format__ = <built-in method __format__ of exceptions.IndexError object>
    __getattribute__ = <method-wrapper '__getattribute__' of exceptions.IndexError object>
    __getitem__ = <method-wrapper '__getitem__' of exceptions.IndexError object>
    __getslice__ = <method-wrapper '__getslice__' of exceptions.IndexError object>
    __hash__ = <method-wrapper '__hash__' of exceptions.IndexError object>
    __init__ = <method-wrapper '__init__' of exceptions.IndexError object>
    __new__ = <built-in method __new__ of type object>
    __reduce__ = <built-in method __reduce__ of exceptions.IndexError object>
    __reduce_ex__ = <built-in method __reduce_ex__ of exceptions.IndexError object>
    __repr__ = <method-wrapper '__repr__' of exceptions.IndexError object>
    __setattr__ = <method-wrapper '__setattr__' of exceptions.IndexError object>
    __setstate__ = <built-in method __setstate__ of exceptions.IndexError object>
    __sizeof__ = <built-in method __sizeof__ of exceptions.IndexError object>
    __str__ = <method-wrapper '__str__' of exceptions.IndexError object>
    __subclasshook__ = <built-in method __subclasshook__ of type object>
    __unicode__ = <built-in method __unicode__ of exceptions.IndexError object>
    args = ('list index out of range',)
    message = 'list index out of range'

The above is a description of an error in a Python program. Here is
the original traceback:

Traceback (most recent call last):
  File "/usr/bin/contrail-schema", line 9, in <module>
    load_entry_point('schema-transformer==0.1dev', 'console_scripts', 'contrail-schema')()
  File "/usr/lib/python2.7/dist-packages/schema_transformer/to_bgp.py", line 839, in server_main
    main()
  File "/usr/lib/python2.7/dist-packages/schema_transformer/to_bgp.py", line 833, in main
    args)
  File "/usr/lib/python2.7/dist-packages/cfgm_common/zkclient.py", line 336, in master_election
    self._election.run(func, *args, **kwargs)
  File "/usr/lib/python2.7/dist-packages/kazoo/recipe/election.py", line 48, in run
    func(*args, **kwargs)
  File "/usr/lib/python2.7/dist-packages/schema_transformer/to_bgp.py", line 814, in run_schema_transformer
    transformer = SchemaTransformer(args)
  File "/usr/lib/python2.7/dist-packages/schema_transformer/to_bgp.py", line 216, in __init__
    self.reinit()
  File "/usr/lib/python2.7/dist-packages/schema_transformer/to_bgp.py", line 430, in reinit
    RoutingInstanceST.locate(ri_name, ri_obj)
  File "/usr/lib/python2.7/dist-packages/schema_transformer/config_db.py", line 98, in locate
    obj = super(DBBaseST, cls).locate(key, *args)
  File "/usr/lib/python2.7/dist-packages/cfgm_common/vnc_db.py", line 64, in locate
    obj = cls(key, *args)
  File "/usr/lib/python2.7/dist-packages/schema_transformer/config_db.py", line 1834, in __init__
    if remote_ri_fq_name[-1] == remote_ri_fq_name[-2]:
IndexError: list index out of range

[Suresh Balineni (sbalineni)]

Discussed with Amit, this needs to be reproduced and hence reducing the priority.

[OpenContrail Admin (ci-admin-f)]

Review in progress for https://review.opencontrail.org/17694
Submitter: Sachin Bansal (<email address hidden>)

[OpenContrail Admin (ci-admin-f)]

Reviewed: https://review.opencontrail.org/17694
Committed: http://github.org/Juniper/contrail-controller/commit/40d188c05848e95a592a65d34af3d0a83e07ebec
Submitter: Zuul
Branch: master

commit 40d188c05848e95a592a65d34af3d0a83e07ebec
Author: Sachin Bansal <email address hidden>
Date: Tue Feb 23 15:25:30 2016 -0800

Validate existence of ref uuid before adding ref

Currently, you can add a ref to a non-existent object. This can
cause problems later when the object is read. To resolve this,
deny the ref-update add if the object does not exist.

Change-Id: I4cb5398f65ddfc3589cbc843aa2e5c40de9d299b
Closes-Bug: 1543386