BGPaaS: schema leaking bgpaas port, eventually crashing

[amit surana]

after repeated restarts, schema seems to have leaked source ports used for the BGP connection from vRouter to CN.

02/02/2016 03:16:29 PM [contrail-schema]: SANDESH: CONNECT TO COLLECTOR: True
02/02/2016 03:16:29 PM [contrail-schema]: SANDESH: Logging: LEVEL: [SYS_INFO] -> [SYS_NOTICE]
^F<class 'cfgm_common.exceptions.ResourceExhaustionError'>
Python 2.7.6: /usr/bin/python
Tue Feb 2 15:16:30 2016

A problem occurred in a Python script. Here is the sequence of
function calls leading up to the error, in the order they occurred.

 /usr/bin/contrail-schema in <module>()
    6
    7 if __name__ == '__main__':
    8 sys.exit(
    9 load_entry_point('schema-transformer==0.1dev', 'console_scripts', 'contrail-schema')()
   10 )
load_entry_point = <function load_entry_point>

 /usr/lib/python2.7/dist-packages/schema_transformer/to_bgp.py in server_main()
  830 def server_main():
  831 cgitb.enable(format='text')
  832 main()
  833 # end server_main
  834
global main = <function main>

 /usr/lib/python2.7/dist-packages/schema_transformer/to_bgp.py in main(args_str='--conf_file /etc/contrail/contrail-schema.conf -... --conf_file /etc/contrail/contrail-database.conf')
  824 _zookeeper_client.master_election(zk_path_pfx + "/schema-transformer",
  825 os.getpid(), run_schema_transformer,
  826 args)
  827 # end main
  828
args = Namespace(admin_password='contrail123', admin_te...7:2181,172.16.180.8:2181', zk_server_port='2181')

 /usr/lib/python2.7/dist-packages/cfgm_common/zkclient.py in master_election(self=<cfgm_common.zkclient.ZookeeperClient object>, path='/schema-transformer', identifier=36751, func=<function run_schema_transformer>, *args=(Namespace(admin_password='contrail123', admin_te...7:2181,172.16.180.8:2181', zk_server_port='2181'),), **kwargs={})
  334 def master_election(self, path, identifier, func, *args, **kwargs):
  335 self._election = self._zk_client.Election(path, identifier)
  336 self._election.run(func, *args, **kwargs)
  337 # end master_election
  338
self = <cfgm_common.zkclient.ZookeeperClient object>
self._election = <kazoo.recipe.election.Election object>
self._election.run = <bound method Election.run of <kazoo.recipe.election.Election object>>
func = <function run_schema_transformer>
args = (Namespace(admin_password='contrail123', admin_te...7:2181,172.16.180.8:2181', zk_server_port='2181'),)
kwargs = {}

 /usr/lib/python2.7/dist-packages/kazoo/recipe/election.py in run(self=<kazoo.recipe.election.Election object>, func=<function run_schema_transformer>, *args=(Namespace(admin_password='contrail123', admin_te...7:2181,172.16.180.8:2181', zk_server_port='2181'),), **kwargs={})
   46 try:
   47 with self.lock:
   48 func(*args, **kwargs)
   49
   50 except CancelledError:
func = <function run_schema_transformer>
args = (Namespace(admin_password='contrail123', admin_te...7:2181,172.16.180.8:2181', zk_server_port='2181'),)
kwargs = {}

 /usr/lib/python2.7/dist-packages/schema_transformer/to_bgp.py in run_schema_transformer(args=Namespace(admin_password='contrail123', admin_te...7:2181,172.16.180.8:2181', zk_server_port='2181'))
  805
  806 global transformer
  807 transformer = SchemaTransformer(args)
  808 gevent.joinall(transformer._vnc_kombu.greenlets())
  809 # end run_schema_transformer
global transformer = None
global SchemaTransformer = <class 'schema_transformer.to_bgp.SchemaTransformer'>
args = Namespace(admin_password='contrail123', admin_te...7:2181,172.16.180.8:2181', zk_server_port='2181')

 /usr/lib/python2.7/dist-packages/schema_transformer/to_bgp.py in __init__(self=<schema_transformer.to_bgp.SchemaTransformer object>, args=Namespace(admin_password='contrail123', admin_te...7:2181,172.16.180.8:2181', zk_server_port='2181'))
  214 DBBaseST._vnc_lib = _vnc_lib
  215 ServiceChain.init()
  216 self.reinit()
  217 # create cpu_info object to send periodic updates
  218 sysinfo_req = False
self = <schema_transformer.to_bgp.SchemaTransformer object>
self.reinit = <bound method SchemaTransformer.reinit of <schema_transformer.to_bgp.SchemaTransformer object>>

 /usr/lib/python2.7/dist-packages/schema_transformer/to_bgp.py in reinit(self=<schema_transformer.to_bgp.SchemaTransformer object>)
  470 for cls in DBBaseST.get_obj_type_map().values():
  471 for obj in cls.values():
  472 obj.evaluate()
  473 self.process_stale_objects()
  474 # end reinit
obj = <schema_transformer.config_db.BgpAsAServiceST object>
obj.evaluate = <bound method BgpAsAServiceST.evaluate of <schema_transformer.config_db.BgpAsAServiceST object>>

 /usr/lib/python2.7/dist-packages/schema_transformer/config_db.py in evaluate(self=<schema_transformer.config_db.BgpAsAServiceST object>)
 2889 for name in (self.virtual_machine_interfaces -
 2890 set(self.bgpaas_clients.keys())):
 2891 self.create_bgp_router(name)
 2892 # end evaluate
 2893
self = <schema_transformer.config_db.BgpAsAServiceST object>
self.create_bgp_router = <bound method BgpAsAServiceST.create_bgp_router ...ma_transformer.config_db.BgpAsAServiceST object>>
name = u'default-domain:bgp-test:bgp-vSRX-1'

 /usr/lib/python2.7/dist-packages/schema_transformer/config_db.py in create_bgp_router(self=<schema_transformer.config_db.BgpAsAServiceST object>, name=u'default-domain:bgp-test:bgp-vSRX-1')
 2926 address=ip,
 2927 identifier=ip,
 2928 source_port=self._cassandra.alloc_bgpaas_port(router_fq_name),
 2929 router_type='bgpaas-client')
 2930 bgp_router.set_bgp_router_parameters(params)
source_port undefined
self = <schema_transformer.config_db.BgpAsAServiceST object>
self._cassandra = <schema_transformer.db.SchemaTransformerDB object>
self._cassandra.alloc_bgpaas_port = <bound method SchemaTransformerDB.alloc_bgpaas_p...chema_transformer.db.SchemaTransformerDB object>>
router_fq_name = u'default-domain:bgp-test:vsrx-bgp-vn:vsrx-bgp-vn:bgp-vSRX-1'

 /usr/lib/python2.7/dist-packages/schema_transformer/db.py in alloc_bgpaas_port(self=<schema_transformer.db.SchemaTransformerDB object>, name=u'default-domain:bgp-test:vsrx-bgp-vn:vsrx-bgp-vn:bgp-vSRX-1')
  267
  268 def alloc_bgpaas_port(self, name):
  269 return self._bgpaas_port_allocator.alloc(name)
  270
  271 def free_bgpaas_port(self, port):
self = <schema_transformer.db.SchemaTransformerDB object>
self._bgpaas_port_allocator = <cfgm_common.zkclient.IndexAllocator object>
self._bgpaas_port_allocator.alloc = <bound method IndexAllocator.alloc of <cfgm_common.zkclient.IndexAllocator object>>
name = u'default-domain:bgp-test:vsrx-bgp-vn:vsrx-bgp-vn:bgp-vSRX-1'

 /usr/lib/python2.7/dist-packages/cfgm_common/zkclient.py in alloc(self=<cfgm_common.zkclient.IndexAllocator object>, value=u'default-domain:bgp-test:vsrx-bgp-vn:vsrx-bgp-vn:bgp-vSRX-1')
  152 self._in_use[idx] = 1
  153
  154 idx = self._get_zk_index_from_bit(idx)
  155 try:
  156 # Create a node at path and return its integer value
idx = 257L
self = <cfgm_common.zkclient.IndexAllocator object>
self._get_zk_index_from_bit = <bound method IndexAllocator._get_zk_index_from_bit of <cfgm_common.zkclient.IndexAllocator object>>

 /usr/lib/python2.7/dist-packages/cfgm_common/zkclient.py in _get_zk_index_from_bit(self=<cfgm_common.zkclient.IndexAllocator object>, idx=257L)
   77
   78 raise ResourceExhaustionError(
   79 'Cannot get zk index from bit %s' %(idx))
   80 # end _get_zk_index
   81
idx = 257L
<class 'cfgm_common.exceptions.ResourceExhaustionError'>: Cannot get zk index from bit 257
    __class__ = <class 'cfgm_common.exceptions.ResourceExhaustionError'>
    __delattr__ = <method-wrapper '__delattr__' of ResourceExhaustionError object>
    __dict__ = {}
    __doc__ = None
    __format__ = <built-in method __format__ of ResourceExhaustionError object>
    __getattribute__ = <method-wrapper '__getattribute__' of ResourceExhaustionError object>
    __getitem__ = <method-wrapper '__getitem__' of ResourceExhaustionError object>
    __getslice__ = <method-wrapper '__getslice__' of ResourceExhaustionError object>
    __hash__ = <method-wrapper '__hash__' of ResourceExhaustionError object>
    __init__ = <method-wrapper '__init__' of ResourceExhaustionError object>
    __module__ = 'cfgm_common.exceptions'
    __new__ = <built-in method __new__ of type object>
    __reduce__ = <built-in method __reduce__ of ResourceExhaustionError object>
    __reduce_ex__ = <built-in method __reduce_ex__ of ResourceExhaustionError object>
    __repr__ = <method-wrapper '__repr__' of ResourceExhaustionError object>
    __setattr__ = <method-wrapper '__setattr__' of ResourceExhaustionError object>
    __setstate__ = <built-in method __setstate__ of ResourceExhaustionError object>
    __sizeof__ = <built-in method __sizeof__ of ResourceExhaustionError object>
    __str__ = <method-wrapper '__str__' of ResourceExhaustionError object>
    __subclasshook__ = <built-in method __subclasshook__ of type object>
    __unicode__ = <built-in method __unicode__ of ResourceExhaustionError object>
    __weakref__ = None
    args = ('Cannot get zk index from bit 257',)
    message = 'Cannot get zk index from bit 257'

The above is a description of an error in a Python program. Here is
the original traceback:

Traceback (most recent call last):
  File "/usr/bin/contrail-schema", line 9, in <module>
    load_entry_point('schema-transformer==0.1dev', 'console_scripts', 'contrail-schema')()
  File "/usr/lib/python2.7/dist-packages/schema_transformer/to_bgp.py", line 832, in server_main
    main()
  File "/usr/lib/python2.7/dist-packages/schema_transformer/to_bgp.py", line 826, in main
    args)
  File "/usr/lib/python2.7/dist-packages/cfgm_common/zkclient.py", line 336, in master_election
    self._election.run(func, *args, **kwargs)
  File "/usr/lib/python2.7/dist-packages/kazoo/recipe/election.py", line 48, in run
    func(*args, **kwargs)
  File "/usr/lib/python2.7/dist-packages/schema_transformer/to_bgp.py", line 807, in run_schema_transformer
    transformer = SchemaTransformer(args)
  File "/usr/lib/python2.7/dist-packages/schema_transformer/to_bgp.py", line 216, in __init__
    self.reinit()
  File "/usr/lib/python2.7/dist-packages/schema_transformer/to_bgp.py", line 472, in reinit
    obj.evaluate()
  File "/usr/lib/python2.7/dist-packages/schema_transformer/config_db.py", line 2891, in evaluate
    self.create_bgp_router(name)
  File "/usr/lib/python2.7/dist-packages/schema_transformer/config_db.py", line 2928, in create_bgp_router
    source_port=self._cassandra.alloc_bgpaas_port(router_fq_name),
  File "/usr/lib/python2.7/dist-packages/schema_transformer/db.py", line 269, in alloc_bgpaas_port
    return self._bgpaas_port_allocator.alloc(name)
  File "/usr/lib/python2.7/dist-packages/cfgm_common/zkclient.py", line 154, in alloc
    idx = self._get_zk_index_from_bit(idx)
  File "/usr/lib/python2.7/dist-packages/cfgm_common/zkclient.py", line 79, in _get_zk_index_from_bit
    'Cannot get zk index from bit %s' %(idx))
ResourceExhaustionError: Cannot get zk index from bit 257

^C
root@csol2-node5:/var/tmp/bgp#