BGPaaS: Schema in infinite loop trying to incorrectly delete bgp-router upon service restart

[amit surana]

contrail version 3.0-2698.

BGPaaS object is attached to a VMI. bgp-router client/server objects are correctly created. Now, if config services are restarted, it is seen that schema is trying to delete the bgpaas-client bgp-router object. This of course fails cause there is a bgpaas back ref present. ST continues to periodically attempt the delete.

01/20/2016 11:22:17 AM [contrail-schema]: SANDESH: Logging: LEVEL: [SYS_INFO] -> [SYS_NOTICE]
<class 'cfgm_common.exceptions.RefsExistError'>
Python 2.7.6: /usr/bin/python
Wed Jan 20 11:22:23 2016

A problem occurred in a Python script. Here is the sequence of
function calls leading up to the error, in the order they occurred.

 /usr/bin/contrail-schema in <module>()
    6
    7 if __name__ == '__main__':
    8 sys.exit(
    9 load_entry_point('schema-transformer==0.1dev', 'console_scripts', 'contrail-schema')()
   10 )
load_entry_point = <function load_entry_point>

 /usr/lib/python2.7/dist-packages/schema_transformer/to_bgp.py in server_main()
  821 def server_main():
  822 cgitb.enable(format='text')
  823 main()
  824 # end server_main
  825
global main = <function main>

 /usr/lib/python2.7/dist-packages/schema_transformer/to_bgp.py in main(args_str='--conf_file /etc/contrail/contrail-schema.conf -... --conf_file /etc/contrail/contrail-database.conf')
  815 _zookeeper_client.master_election(zk_path_pfx + "/schema-transformer",
  816 os.getpid(), run_schema_transformer,
  817 args)
  818 # end main
  819
args = Namespace(admin_password='c0ntrail123', admin_te....2:2181,172.16.80.3:2181', zk_server_port='2181')

 /usr/lib/python2.7/dist-packages/cfgm_common/zkclient.py in master_election(self=<cfgm_common.zkclient.ZookeeperClient object>, path='/schema-transformer', identifier=25533, func=<function run_schema_transformer>, *args=(Namespace(admin_password='c0ntrail123', admin_te....2:2181,172.16.80.3:2181', zk_server_port='2181'),), **kwargs={})
  334 def master_election(self, path, identifier, func, *args, **kwargs):
  335 self._election = self._zk_client.Election(path, identifier)
  336 self._election.run(func, *args, **kwargs)
  337 # end master_election
  338
self = <cfgm_common.zkclient.ZookeeperClient object>
self._election = <kazoo.recipe.election.Election object>
self._election.run = <bound method Election.run of <kazoo.recipe.election.Election object>>
func = <function run_schema_transformer>
args = (Namespace(admin_password='c0ntrail123', admin_te....2:2181,172.16.80.3:2181', zk_server_port='2181'),)
kwargs = {}

 /usr/lib/python2.7/dist-packages/kazoo/recipe/election.py in run(self=<kazoo.recipe.election.Election object>, func=<function run_schema_transformer>, *args=(Namespace(admin_password='c0ntrail123', admin_te....2:2181,172.16.80.3:2181', zk_server_port='2181'),), **kwargs={})
   46 try:
   47 with self.lock:
   48 func(*args, **kwargs)
   49
   50 except CancelledError:
func = <function run_schema_transformer>
args = (Namespace(admin_password='c0ntrail123', admin_te....2:2181,172.16.80.3:2181', zk_server_port='2181'),)
kwargs = {}

 /usr/lib/python2.7/dist-packages/schema_transformer/to_bgp.py in run_schema_transformer(args=Namespace(admin_password='c0ntrail123', admin_te....2:2181,172.16.80.3:2181', zk_server_port='2181'))
  796
  797 global transformer
  798 transformer = SchemaTransformer(args)
  799 gevent.joinall(transformer._vnc_kombu.greenlets())
  800 # end run_schema_transformer
global transformer = None
global SchemaTransformer = <class 'schema_transformer.to_bgp.SchemaTransformer'>
args = Namespace(admin_password='c0ntrail123', admin_te....2:2181,172.16.80.3:2181', zk_server_port='2181')

 /usr/lib/python2.7/dist-packages/schema_transformer/to_bgp.py in __init__(self=<schema_transformer.to_bgp.SchemaTransformer object>, args=Namespace(admin_password='c0ntrail123', admin_te....2:2181,172.16.80.3:2181', zk_server_port='2181'))
  208 DBBaseST._vnc_lib = _vnc_lib
  209 ServiceChain.init()
  210 self.reinit()
  211 # create cpu_info object to send periodic updates
  212 sysinfo_req = False
self = <schema_transformer.to_bgp.SchemaTransformer object>
self.reinit = <bound method SchemaTransformer.reinit of <schema_transformer.to_bgp.SchemaTransformer object>>

 /usr/lib/python2.7/dist-packages/schema_transformer/to_bgp.py in reinit(self=<schema_transformer.to_bgp.SchemaTransformer object>)
  461 for cls in DBBaseST.get_obj_type_map().values():
  462 for obj in cls.values():
  463 obj.evaluate()
  464 self.process_stale_objects()
  465 # end reinit
obj = <schema_transformer.config_db.BgpRouterST object>
obj.evaluate = <bound method BgpRouterST.evaluate of <schema_transformer.config_db.BgpRouterST object>>

 /usr/lib/python2.7/dist-packages/schema_transformer/config_db.py in evaluate(self=<schema_transformer.config_db.BgpRouterST object>)
 2699 except NoIdError:
 2700 pass
 2701 self._vnc_lib.bgp_router_delete(id=self.obj.uuid)
 2702 self.delete(self.name)
 2703 elif ret:
self = <schema_transformer.config_db.BgpRouterST object>
self._vnc_lib = <vnc_api.vnc_api.VncApi object>
self._vnc_lib.bgp_router_delete = <functools.partial object>
builtinid = <built-in function id>
self.obj = <vnc_api.gen.resource_client.BgpRouter object>
self.obj.uuid = u'617100bf-e3e4-4878-ae4d-c20c3a81d2d0'

 /usr/lib/python2.7/dist-packages/vnc_api/vnc_api.py in wrapper(self=<vnc_api.vnc_api.VncApi object>, *args=('bgp-router',), **kwargs={'id': u'617100bf-e3e4-4878-ae4d-c20c3a81d2d0'})
   37 retry_on_error=False)
   38 self._parse_homepage(homepage)
   39 return func(self, *args, **kwargs)
   40 return wrapper
   41
func = <function _object_delete>
self = <vnc_api.vnc_api.VncApi object>
args = ('bgp-router',)
kwargs = {'id': u'617100bf-e3e4-4878-ae4d-c20c3a81d2d0'}

 /usr/lib/python2.7/dist-packages/vnc_api/vnc_api.py in _object_delete(self=<vnc_api.vnc_api.VncApi object>, res_type='bgp-router', fq_name=None, id=u'617100bf-e3e4-4878-ae4d-c20c3a81d2d0', ifmap_id=None)
  495 uri = obj_cls.resource_uri_base[res_type] + '/' + id
  496
  497 content = self._request_server(rest.OP_DELETE, uri)
  498 # end _object_delete
  499
content undefined
self = <vnc_api.vnc_api.VncApi object>
self._request_server = <bound method VncApi._request_server of <vnc_api.vnc_api.VncApi object>>
global rest = <module 'cfgm_common.rest' from '/usr/lib/python2.7/dist-packages/cfgm_common/rest.pyc'>
rest.OP_DELETE = 4
uri = u'/bgp-router/617100bf-e3e4-4878-ae4d-c20c3a81d2d0'

 /usr/lib/python2.7/dist-packages/vnc_api/vnc_api.py in _request_server(self=<vnc_api.vnc_api.VncApi object>, op=4, url=u'/bgp-router/617100bf-e3e4-4878-ae4d-c20c3a81d2d0', data=None, retry_on_error=True, retry_after_authn=False, retry_count=30)
  695 return self._request(op, url, data=data, retry_on_error=retry_on_error,
  696 retry_after_authn=retry_after_authn,
  697 retry_count=retry_count)
  698
  699 def _request(self, op, url, data=None, retry_on_error=True,
retry_count = 30

 /usr/lib/python2.7/dist-packages/vnc_api/vnc_api.py in _request(self=<vnc_api.vnc_api.VncApi object>, op=4, url=u'/bgp-router/617100bf-e3e4-4878-ae4d-c20c3a81d2d0', data=None, retry_on_error=True, retry_after_authn=False, retry_count=30)
  740 raise PermissionDenied(content)
  741 elif status == 409:
  742 raise RefsExistError(content)
  743 elif status == 504:
  744 # Request sent to API server, but no response came within 50s
global RefsExistError = <class 'cfgm_common.exceptions.RefsExistError'>
content = u"Delete when resource still referred: ['http://...a-service/7e8f2540-ebeb-4c66-933c-a90a668c36d1']"
<class 'cfgm_common.exceptions.RefsExistError'>: Delete when resource still referred: ['http://172.16.80.50:8082/bgp-as-a-service/7e8f2540-ebeb-4c66-933c-a90a668c36d1']
    __class__ = <class 'cfgm_common.exceptions.RefsExistError'>
    __delattr__ = <method-wrapper '__delattr__' of RefsExistError object>
    __dict__ = {}
    __doc__ = None
    __format__ = <built-in method __format__ of RefsExistError object>
    __getattribute__ = <method-wrapper '__getattribute__' of RefsExistError object>
    __getitem__ = <method-wrapper '__getitem__' of RefsExistError object>
    __getslice__ = <method-wrapper '__getslice__' of RefsExistError object>
    __hash__ = <method-wrapper '__hash__' of RefsExistError object>
    __init__ = <method-wrapper '__init__' of RefsExistError object>
    __module__ = 'cfgm_common.exceptions'
    __new__ = <built-in method __new__ of type object>
    __reduce__ = <built-in method __reduce__ of RefsExistError object>
    __reduce_ex__ = <built-in method __reduce_ex__ of RefsExistError object>
    __repr__ = <method-wrapper '__repr__' of RefsExistError object>
    __setattr__ = <method-wrapper '__setattr__' of RefsExistError object>
    __setstate__ = <built-in method __setstate__ of RefsExistError object>
    __sizeof__ = <built-in method __sizeof__ of RefsExistError object>
    __str__ = <method-wrapper '__str__' of RefsExistError object>
    __subclasshook__ = <built-in method __subclasshook__ of type object>
    __unicode__ = <built-in method __unicode__ of RefsExistError object>
    __weakref__ = None
    args = (u"Delete when resource still referred: ['http://...a-service/7e8f2540-ebeb-4c66-933c-a90a668c36d1']",)
    message = u"Delete when resource still referred: ['http://...a-service/7e8f2540-ebeb-4c66-933c-a90a668c36d1']"

The above is a description of an error in a Python program. Here is
the original traceback:

Traceback (most recent call last):
  File "/usr/bin/contrail-schema", line 9, in <module>
    load_entry_point('schema-transformer==0.1dev', 'console_scripts', 'contrail-schema')()
  File "/usr/lib/python2.7/dist-packages/schema_transformer/to_bgp.py", line 823, in server_main
    main()
  File "/usr/lib/python2.7/dist-packages/schema_transformer/to_bgp.py", line 817, in main
    args)
  File "/usr/lib/python2.7/dist-packages/cfgm_common/zkclient.py", line 336, in master_election
    self._election.run(func, *args, **kwargs)
  File "/usr/lib/python2.7/dist-packages/kazoo/recipe/election.py", line 48, in run
    func(*args, **kwargs)
  File "/usr/lib/python2.7/dist-packages/schema_transformer/to_bgp.py", line 798, in run_schema_transformer
    transformer = SchemaTransformer(args)
  File "/usr/lib/python2.7/dist-packages/schema_transformer/to_bgp.py", line 210, in __init__
    self.reinit()
  File "/usr/lib/python2.7/dist-packages/schema_transformer/to_bgp.py", line 463, in reinit
    obj.evaluate()
  File "/usr/lib/python2.7/dist-packages/schema_transformer/config_db.py", line 2701, in evaluate
    self._vnc_lib.bgp_router_delete(id=self.obj.uuid)
  File "/usr/lib/python2.7/dist-packages/vnc_api/vnc_api.py", line 39, in wrapper
    return func(self, *args, **kwargs)
  File "/usr/lib/python2.7/dist-packages/vnc_api/vnc_api.py", line 497, in _object_delete
    content = self._request_server(rest.OP_DELETE, uri)
  File "/usr/lib/python2.7/dist-packages/vnc_api/vnc_api.py", line 697, in _request_server
    retry_count=retry_count)
  File "/usr/lib/python2.7/dist-packages/vnc_api/vnc_api.py", line 742, in _request
    raise RefsExistError(content)
RefsExistError: Delete when resource still referred: ['http://172.16.80.50:8082/bgp-as-a-service/7e8f2540-ebeb-4c66-933c-a90a668c36d1']

[OpenContrail Admin (ci-admin-f)]

Review in progress for https://review.opencontrail.org/16760
Submitter: Suresh Balineni (<email address hidden>)

[OpenContrail Admin (ci-admin-f)]

Reviewed: https://review.opencontrail.org/16760
Committed: http://github.org/Juniper/contrail-controller/commit/d239035ec3525365f315192fec8e4f958560f6d7
Submitter: Zuul
Branch: master

commit d239035ec3525365f315192fec8e4f958560f6d7
Author: sbalineni <email address hidden>
Date: Mon Feb 1 11:01:02 2016 -0800

BGPaaS - vmi primary ip address is used as BGPaaS object if not configured.
Read and initalize Bgpaas objects when ST is reinited.

Change-Id: I3f0a4da86d708464de7d0b4ba41ae6e7a5faabb8
Closes-Bug: #1538322
Closes-Bug: #1538318
Closes-Bug: #1536339
Closes-Bug: #1540540