KeyError when a service frontend ip is created in kubernetes

[Vedamurthy Joshi]

R4.0 Build 3028

[root@vedu-vm1-1 centos]# kubectl get rc
NAME DESIRED CURRENT READY AGE
nginx-controller 2 2 2 39m
[root@vedu-vm1-1 centos]# kubectl expose rc/nginx-controller
service "nginx-controller" exposed
[root@vedu-vm1-1 centos]# kubectl get service
NAME CLUSTER-IP EXTERNAL-IP PORT(S) AGE
kubernetes 10.96.0.1 <none> 443/TCP 2h
nginx-controller 10.107.177.41 <none> 80/TCP 4s <<<<<<<
[root@vedu-vm1-1 centos]#

contrail-kube-manager console :
------------------------
Traceback (most recent call last):
  File "/usr/lib/python2.7/dist-packages/gevent/greenlet.py", line 327, in run
    result = self._run(*self.args, **self.kwargs)
  File "/usr/lib/python2.7/dist-packages/kube_manager/vnc/vnc_kubernetes.py", line 211, in vnc_process
    self.service_mgr.process(event)
  File "/usr/lib/python2.7/dist-packages/kube_manager/vnc/vnc_service.py", line 484, in process
    service_type, externalIp)
  File "/usr/lib/python2.7/dist-packages/kube_manager/vnc/vnc_service.py", line 395, in vnc_service_add
    selectors, ports)
  File "/usr/lib/python2.7/dist-packages/kube_manager/vnc/vnc_service.py", line 286, in _lb_create
    service_namespace, service_ip, selectors)
  File "/usr/lib/python2.7/dist-packages/kube_manager/vnc/vnc_service.py", line 278, in _vnc_create_lb
    proj_obj, service_ip, selectors)
  File "/usr/lib/python2.7/dist-packages/kube_manager/vnc/loadbalancer.py", line 133, in create
    vn_obj, service_name, cluster_ip)
  File "/usr/lib/python2.7/dist-packages/kube_manager/vnc/loadbalancer.py", line 70, in _create_virtual_interface
    self._vnc_lib.instance_ip_create(iip_obj)
  File "/usr/lib/python2.7/dist-packages/vnc_api/vnc_api.py", line 42, in wrapper
    return func(self, *args, **kwargs)
  File "/usr/lib/python2.7/dist-packages/vnc_api/vnc_api.py", line 392, in _object_create
    data=json_body)
  File "/usr/lib/python2.7/dist-packages/vnc_api/vnc_api.py", line 755, in _request_server
    retry_count=retry_count)
  File "/usr/lib/python2.7/dist-packages/vnc_api/vnc_api.py", line 849, in _request
    raise HttpError(status, content)
HttpError: HTTP Status: 500 Content: <type 'exceptions.KeyError'>
Python 2.7.6: /usr/bin/python
Fri Jan 20 23:06:50 2017

A problem occurred in a Python script. Here is the sequence of
function calls leading up to the error, in the order they occurred.

 /usr/lib/python2.7/dist-packages/vnc_cfg_api_server/vnc_cfg_api_server.py in http_resource_create(self=<vnc_cfg_api_server.vnc_cfg_api_server.VncApiServer object>, obj_type='instance_ip')
  593
  594 try:
  595 ok, result = stateful_create()
  596 except Exception as e:
  597 ok = False
ok = False
result = None
stateful_create = <function stateful_create>

 /usr/lib/python2.7/dist-packages/vnc_cfg_api_server/vnc_cfg_api_server.py in stateful_create()
  534 # type-specific hook
  535 (ok, result) = r_class.pre_dbe_create(tenant_name, obj_dict,
  536 db_conn)
  537 if not ok:
  538 return (ok, result)
db_conn = <vnc_cfg_api_server.vnc_db.VncDbClient object>

 /usr/lib/python2.7/dist-packages/vnc_cfg_api_server/vnc_cfg_types.py in pre_dbe_create(cls=<class 'vnc_cfg_api_server.vnc_cfg_types.InstanceIpServer'>, tenant_name='admin', obj_dict={'display_name': 'nginx-controller', 'fq_name': ['nginx-controller'], 'id_perms': {u'created': None, u'creator': None, u'description': None, u'enable': True, u'last_modified': None, u'permissions': {u'group': u'cloud-admin-group', u'group_access': 7, u'other_access': 7, u'owner': u'cloud-admin', u'owner_access': 7}, u'user_visible': True, u'uuid': {'uuid_lslong': 12019656247259188868L, 'uuid_mslong': 13079667359974376991L}}, 'instance_ip_address': '10.107.177.41', 'instance_ip_local_ip': False, 'instance_ip_secondary': False, 'perms2': {u'global_access': 0, u'owner': u'cloud-admin', u'owner_access': 7, u'share': []}, 'service_health_check_ip': False, 'service_instance_ip': False, 'uuid': 'b584502a-338b-461f-a6ce-65ac8c304e84', ...}, db_conn=<vnc_cfg_api_server.vnc_db.VncDbClient object>)
  418 # for g/w ip, creation allowed but only can ref to router port.
  419 if req_ip and cls.addr_mgmt.is_ip_allocated(req_ip, vn_fq_name,
  420 vn_uuid=vn_id):
  421 if not cls.addr_mgmt.is_gateway_ip(vn_dict, req_ip):
  422 return (False, (400, 'Ip address already in use'))
vn_uuid undefined
vn_id = 'b1026bf6-c7ba-4b1e-90f1-18db097f97d5'

 /usr/lib/python2.7/dist-packages/vnc_cfg_api_server/vnc_addr_mgmt.py in is_ip_allocated(self=<vnc_cfg_api_server.vnc_addr_mgmt.AddrMgmt object>, ip_addr='10.107.177.41', vn_fq_name=['default-domain', 'default', 'cluster-network'], vn_uuid='b1026bf6-c7ba-4b1e-90f1-18db097f97d5', sub=None)
 1604
 1605 try:
 1606 return self._ipam_is_ip_allocated(ip_addr, vn_uuid, sub)
 1607 except cfgm_common.exceptions.VncError:
 1608 try:
self = <vnc_cfg_api_server.vnc_addr_mgmt.AddrMgmt object>
self._ipam_is_ip_allocated = <bound method AddrMgmt._ipam_is_ip_allocated of <vnc_cfg_api_server.vnc_addr_mgmt.AddrMgmt object>>
ip_addr = '10.107.177.41'
vn_uuid = 'b1026bf6-c7ba-4b1e-90f1-18db097f97d5'
sub = None

 /usr/lib/python2.7/dist-packages/vnc_cfg_api_server/vnc_addr_mgmt.py in _ipam_is_ip_allocated(self=<vnc_cfg_api_server.vnc_addr_mgmt.AddrMgmt object>, ip_addr='10.107.177.41', vn_uuid='b1026bf6-c7ba-4b1e-90f1-18db097f97d5', sub=None)
 1571 ipam_uuid = ipam_ref['uuid']
 1572 subnet_objs = self._get_ipam_subnet_objs_from_ipam_uuid(
 1573 ipam_fq_name, ipam_uuid, False)
 1574 if subnet_objs is None:
 1575 continue
ipam_fq_name = ['default-domain', 'default', 'service-ipam']
ipam_uuid = '13dc7d5a-87f0-4082-9ebd-bd1d587284d8'
builtinFalse = False

 /usr/lib/python2.7/dist-packages/vnc_cfg_api_server/vnc_addr_mgmt.py in _get_ipam_subnet_objs_from_ipam_uuid(self=<vnc_cfg_api_server.vnc_addr_mgmt.AddrMgmt object>, ipam_fq_name=['default-domain', 'default', 'service-ipam'], ipam_uuid='13dc7d5a-87f0-4082-9ebd-bd1d587284d8', should_persist=False)
  604 subnet_obj = self._create_subnet_obj_for_ipam_subnet(
  605 ipam_subnet, ipam_fq_name_str, should_persist)
  606 self._subnet_objs[ipam_uuid][subnet_name] = subnet_obj
  607 subnet_objs = self._subnet_objs[ipam_uuid]
  608
self = <vnc_cfg_api_server.vnc_addr_mgmt.AddrMgmt object>
self._subnet_objs = {u'cfeb2aa2-f676-4c25-8896-f08bb46c9602': {}}
ipam_uuid = '13dc7d5a-87f0-4082-9ebd-bd1d587284d8'
subnet_name = '10.96.0.0/12'
subnet_obj = <vnc_cfg_api_server.vnc_addr_mgmt.Subnet object>
<type 'exceptions.KeyError'>: '13dc7d5a-87f0-4082-9ebd-bd1d587284d8'
    __class__ = <type 'exceptions.KeyError'>
    __delattr__ = <method-wrapper '__delattr__' of exceptions.KeyError object>
    __dict__ = {}
    __doc__ = 'Mapping key not found.'
    __format__ = <built-in method __format__ of exceptions.KeyError object>
    __getattribute__ = <method-wrapper '__getattribute__' of exceptions.KeyError object>
    __getitem__ = <method-wrapper '__getitem__' of exceptions.KeyError object>
    __getslice__ = <method-wrapper '__getslice__' of exceptions.KeyError object>
    __hash__ = <method-wrapper '__hash__' of exceptions.KeyError object>
    __init__ = <method-wrapper '__init__' of exceptions.KeyError object>
    __new__ = <built-in method __new__ of type object>
    __reduce__ = <built-in method __reduce__ of exceptions.KeyError object>
    __reduce_ex__ = <built-in method __reduce_ex__ of exceptions.KeyError object>
    __repr__ = <method-wrapper '__repr__' of exceptions.KeyError object>
    __setattr__ = <method-wrapper '__setattr__' of exceptions.KeyError object>
    __setstate__ = <built-in method __setstate__ of exceptions.KeyError object>
    __sizeof__ = <built-in method __sizeof__ of exceptions.KeyError object>
    __str__ = <method-wrapper '__str__' of exceptions.KeyError object>
    __subclasshook__ = <built-in method __subclasshook__ of type object>
    __unicode__ = <built-in method __unicode__ of exceptions.KeyError object>
    args = ('13dc7d5a-87f0-4082-9ebd-bd1d587284d8',)
    message = '13dc7d5a-87f0-4082-9ebd-bd1d587284d8'

The above is a description of an error in a Python program. Here is
the original traceback:

Traceback (most recent call last):
  File "/usr/lib/python2.7/dist-packages/vnc_cfg_api_server/vnc_cfg_api_server.py", line 595, in http_resource_create
    ok, result = stateful_create()
  File "/usr/lib/python2.7/dist-packages/vnc_cfg_api_server/vnc_cfg_api_server.py", line 536, in stateful_create
    db_conn)
  File "/usr/lib/python2.7/dist-packages/vnc_cfg_api_server/vnc_cfg_types.py", line 420, in pre_dbe_create
    vn_uuid=vn_id):
  File "/usr/lib/python2.7/dist-packages/vnc_cfg_api_server/vnc_addr_mgmt.py", line 1606, in is_ip_allocated
    return self._ipam_is_ip_allocated(ip_addr, vn_uuid, sub)
  File "/usr/lib/python2.7/dist-packages/vnc_cfg_api_server/vnc_addr_mgmt.py", line 1573, in _ipam_is_ip_allocated
    ipam_fq_name, ipam_uuid, False)
  File "/usr/lib/python2.7/dist-packages/vnc_cfg_api_server/vnc_addr_mgmt.py", line 606, in _get_ipam_subnet_objs_from_ipam_uuid
    self._subnet_objs[ipam_uuid][subnet_name] = subnet_obj
KeyError: '13dc7d5a-87f0-4082-9ebd-bd1d587284d8' <<<<< key error on subnet_name

[Atul Moghe (moghea)]

Sachin Vaidya and I tried with 4.0.0.0-5, and unable to reproduce. We have fixed many issues in flat-subnet and ip allocation since this was observed. Can you try with latest 4.0 or mainline, If we dont see this problem, we can close it