Unhandled NoIdError during svc-vn-mgmt VN creation , during parallel sanity run

[Vedamurthy Joshi]

2.0-2397~icehouse.el6 on Ubuntu single node setup

Was seen during a parallel run of sanity tests

Contrail-Logs will be in http://10.204.216.50/Docs/bugs/1382385/

<class 'cfgm_common.exceptions.NoIdError'>
Python 2.6.6: /usr/bin/python
Fri Oct 17 11:22:17 2014

A problem occurred in a Python script. Here is the sequence of
function calls leading up to the error, in the order they occurred.

 /usr/lib/python2.6/site-packages/svc_monitor/svc_monitor.py in launch_timer(monitor=<svc_monitor.svc_monitor.SvcMonitor object>)
  540 gevent.sleep(svc_info.get_vm_health_interval())
  541 try:
  542 timer_callback(monitor)
  543 except Exception:
  544 cgitb_error_log(monitor)
global timer_callback = <function timer_callback>
monitor = <svc_monitor.svc_monitor.SvcMonitor object>

 /usr/lib/python2.6/site-packages/svc_monitor/svc_monitor.py in timer_callback(monitor=<svc_monitor.svc_monitor.SvcMonitor object>)
  532 if status == 'ERROR':
  533 monitor.logger.log("Relaunch SI %s" % (si_fq_name_str))
  534 monitor._restart_svc(si_fq_name_str)
  535 elif status == 'DELETE':
  536 monitor._cleanup_si(si_fq_name_str)
monitor = <svc_monitor.svc_monitor.SvcMonitor object>
monitor._restart_svc = <bound method SvcMonitor._restart_svc of <svc_monitor.svc_monitor.SvcMonitor object>>
si_fq_name_str = 'default-domain:TestSvcRegr-70281290:bridge_si-47754329_1'
 /usr/lib/python2.6/site-packages/svc_monitor/svc_monitor.py in _restart_svc(self=<svc_monitor.svc_monitor.SvcMonitor object>, si_fq_str='default-domain:TestSvcRegr-70281290:bridge_si-47754329_1')
  266 fq_name = st_list[0]['to']
  267 st_obj = self._vnc_lib.service_template_read(fq_name=fq_name)
  268 self._create_svc_instance(st_obj, si_obj)
  269 # end _restart_svc
  270
self = <svc_monitor.svc_monitor.SvcMonitor object>
self._create_svc_instance = <bound method SvcMonitor._create_svc_instance of <svc_monitor.svc_monitor.SvcMonitor object>>
st_obj = <vnc_api.gen.resource_client.ServiceTemplate object>
si_obj = <vnc_api.gen.resource_client.ServiceInstance object>

 /usr/lib/python2.6/site-packages/svc_monitor/svc_monitor.py in _create_svc_instance(self=<svc_monitor.svc_monitor.SvcMonitor object>, st_obj=<vnc_api.gen.resource_client.ServiceTemplate object>, si_obj=<vnc_api.gen.resource_client.ServiceInstance object>)
  280 virt_type = self._get_virtualization_type(st_props)
  281 if virt_type == 'virtual-machine':
  282 self.vm_manager.create_service(st_obj, si_obj)
  283 elif virt_type == 'network-namespace':
  284 self.netns_manager.create_service(st_obj, si_obj)
self = <svc_monitor.svc_monitor.SvcMonitor object>
self.vm_manager = <svc_monitor.instance_manager.VirtualMachineManager object>
self.vm_manager.create_service = <bound method VirtualMachineManager.create_servi...r.instance_manager.VirtualMachineManager object>>
st_obj = <vnc_api.gen.resource_client.ServiceTemplate object>
si_obj = <vnc_api.gen.resource_client.ServiceInstance object>

 /usr/lib/python2.6/site-packages/svc_monitor/instance_manager.py in create_service(self=<svc_monitor.instance_manager.VirtualMachineManager object>, st_obj=<vnc_api.gen.resource_client.ServiceTemplate object>, si_obj=<vnc_api.gen.resource_client.ServiceInstance object>)
  439
  440 # populate nic information
  441 nics = self._get_nic_info(si_obj, si_props, st_props)
  442
  443 # get availability zone
nics undefined
self = <svc_monitor.instance_manager.VirtualMachineManager object>
self._get_nic_info = <bound method VirtualMachineManager._get_nic_inf...r.instance_manager.VirtualMachineManager object>>
si_obj = <vnc_api.gen.resource_client.ServiceInstance object>
si_props = <vnc_api.gen.resource_xsd.ServiceInstanceType object>
st_props = <vnc_api.gen.resource_xsd.ServiceTemplateType object>

 /usr/lib/python2.6/site-packages/svc_monitor/instance_manager.py in _get_nic_info(self=<svc_monitor.instance_manager.VirtualMachineManager object>, si_obj=<vnc_api.gen.resource_client.ServiceInstance object>, si_props=<vnc_api.gen.resource_xsd.ServiceInstanceType object>, st_props=<vnc_api.gen.resource_xsd.ServiceTemplateType object>)
  347 else:
  348 vn_id = self._get_vn_id(proj_obj, vn_fq_name_str,
  349 itf_type, si_obj)
  350 if vn_id is None:
  351 continue
itf_type = u'left'
si_obj = <vnc_api.gen.resource_client.ServiceInstance object>

 /usr/lib/python2.6/site-packages/svc_monitor/instance_manager.py in _get_vn_id(self=<svc_monitor.instance_manager.VirtualMachineManager object>, proj_obj=<vnc_api.gen.resource_client.Project object>, vn_fq_name_str=u'', itf_type=u'left', si_obj=<vnc_api.gen.resource_client.ServiceInstance object>)
  289 except NoIdError:
  290 vn_id = self._create_svc_vn(shared_vn_name,
  291 shared_vn_subnet, proj_obj)
  292
  293 si_entry = {}
shared_vn_subnet = '10.250.2.0/24'
proj_obj = <vnc_api.gen.resource_client.Project object>

 /usr/lib/python2.6/site-packages/svc_monitor/instance_manager.py in _create_svc_vn(self=<svc_monitor.instance_manager.VirtualMachineManager object>, vn_name='svc-vn-left', vn_subnet='10.250.2.0/24', proj_obj=<vnc_api.gen.resource_client.Project object>, user_visible=None)
  258 subnet_data = VnSubnetsType([subnet_info])
  259 vn_obj.add_network_ipam(ipam_obj, subnet_data)
  260 self._vnc_lib.virtual_network_create(vn_obj)
  261
  262 return vn_obj.uuid
self = <svc_monitor.instance_manager.VirtualMachineManager object>
self._vnc_lib = <vnc_api.vnc_api.VncApi object>
self._vnc_lib.virtual_network_create = <bound method VncApi.virtual_network_create of <vnc_api.vnc_api.VncApi object>>
vn_obj = <vnc_api.gen.resource_client.VirtualNetwork object>

 /usr/lib/python2.6/site-packages/vnc_api/gen/vnc_api_client_gen.py in virtual_network_create(self=<vnc_api.vnc_api.VncApi object>, obj=<vnc_api.gen.resource_client.VirtualNetwork object>)
 4790 content = self._request_server(rest.OP_POST,
 4791 vnc_api.gen.resource_client.VirtualNetwork.create_uri,
 4792 data = json_body)
 4793
 4794 virtual_network_dict = json.loads(content)['virtual-network']
data undefined
json_body = '{"virtual-network":{"parent_type": "project", "n...SvcRegr-70281290", "svc-vn-left"], "uuid": null}}'

 /usr/lib/python2.6/site-packages/vnc_api/vnc_api.py in _request_server(self=<vnc_api.vnc_api.VncApi object>, op=1, url=u'/virtual-networks', data='{"virtual-network":{"parent_type": "project", "n...SvcRegr-70281290", "svc-vn-left"], "uuid": null}}', retry_on_error=True, retry_after_authn=False, retry_count=30)
  364 elif status == 404:
  365 raise NoIdError('Error: oper %s url %s body %s response %s'
  366 % (op, url, data, content))
  367 elif status == 403:
  368 raise PermissionDenied(content)
op = 1
url = u'/virtual-networks'
data = '{"virtual-network":{"parent_type": "project", "n...SvcRegr-70281290", "svc-vn-left"], "uuid": null}}'
content = u'string indices must be integers, not str'
<class 'cfgm_common.exceptions.NoIdError'>: Unknown id: Error: oper 1 url /virtual-networks body {"virtual-network":{"parent_type": "project", "network_ipam_refs": [{"to": ["default-domain", "default-project", "default-network-ipam"], "attr": {"ipam_subnets": [{"subnet": {"ip_prefix": "10.250.2.0", "ip_prefix_len": 24}, "dns_server_address": null, "enable_dhcp": true, "default_gateway": null, "dns_nameservers": [], "allocation_pools": [], "subnet_uuid": null, "dhcp_option_list": null, "host_routes": null, "addr_from_start": null, "subnet_name": null}], "host_routes": null}, "uuid": "fa6ddd9c-9277-4335-9feb-37d13a6c5d27"}], "fq_name": ["default-domain", "TestSvcRegr-70281290", "svc-vn-left"], "uuid": null}} response string indices must be integers, not str
    __class__ = <class 'cfgm_common.exceptions.NoIdError'>
    __delattr__ = <method-wrapper '__delattr__' of NoIdError object>
    __dict__ = {'_unknown_id': u'Error: oper 1 url /virtual-networks body {"vir...esponse string indices must be integers, not str'}
    __doc__ = None
    __format__ = <built-in method __format__ of NoIdError object>
    __getattribute__ = <method-wrapper '__getattribute__' of NoIdError object>
    __getitem__ = <method-wrapper '__getitem__' of NoIdError object>
    __getslice__ = <method-wrapper '__getslice__' of NoIdError object>
    __hash__ = <method-wrapper '__hash__' of NoIdError object>
    __init__ = <bound method NoIdError.__init__ of NoIdError()>
    __module__ = 'cfgm_common.exceptions'
    __new__ = <built-in method __new__ of type object>
    __reduce__ = <built-in method __reduce__ of NoIdError object>
    __reduce_ex__ = <built-in method __reduce_ex__ of NoIdError object>
    __repr__ = <method-wrapper '__repr__' of NoIdError object>
    __setattr__ = <method-wrapper '__setattr__' of NoIdError object>
    __setstate__ = <built-in method __setstate__ of NoIdError object>
    __sizeof__ = <built-in method __sizeof__ of NoIdError object>
    __str__ = <bound method NoIdError.__str__ of NoIdError()>
    __subclasshook__ = <built-in method __subclasshook__ of type object>
    __unicode__ = <built-in method __unicode__ of NoIdError object>
    __weakref__ = None
    _unknown_id = u'Error: oper 1 url /virtual-networks body {"vir...esponse string indices must be integers, not str'
    args = ()
    message = ''

The above is a description of an error in a Python program. Here is
the original traceback:
Traceback (most recent call last):
  File "/usr/lib/python2.6/site-packages/svc_monitor/svc_monitor.py", line 542, in launch_timer
    timer_callback(monitor)
  File "/usr/lib/python2.6/site-packages/svc_monitor/svc_monitor.py", line 534, in timer_callback
    monitor._restart_svc(si_fq_name_str)
  File "/usr/lib/python2.6/site-packages/svc_monitor/svc_monitor.py", line 268, in _restart_svc
    self._create_svc_instance(st_obj, si_obj)
  File "/usr/lib/python2.6/site-packages/svc_monitor/svc_monitor.py", line 282, in _create_svc_instance
    self.vm_manager.create_service(st_obj, si_obj)
  File "/usr/lib/python2.6/site-packages/svc_monitor/instance_manager.py", line 441, in create_service
    nics = self._get_nic_info(si_obj, si_props, st_props)
  File "/usr/lib/python2.6/site-packages/svc_monitor/instance_manager.py", line 349, in _get_nic_info
    itf_type, si_obj)
  File "/usr/lib/python2.6/site-packages/svc_monitor/instance_manager.py", line 291, in _get_vn_id
    shared_vn_subnet, proj_obj)
  File "/usr/lib/python2.6/site-packages/svc_monitor/instance_manager.py", line 260, in _create_svc_vn
    self._vnc_lib.virtual_network_create(vn_obj)
  File "/usr/lib/python2.6/site-packages/vnc_api/gen/vnc_api_client_gen.py", line 4792, in virtual_network_create
    data = json_body)
  File "/usr/lib/python2.6/site-packages/vnc_api/vnc_api.py", line 366, in _request_server
    % (op, url, data, content))
NoIdError: Unknown id: Error: oper 1 url /virtual-networks body {"virtual-network":{"parent_type": "project", "network_ipam_refs": [{"to": ["default-domain", "default-project", "default-network-ipam"], "attr": {"ipam_subnets": [{"subnet": {"ip_prefix": "10.250.2.0", "ip_prefix_len": 24}, "dns_server_address": null, "enable_dhcp": true, "default_gateway": null, "dns_nameservers": [], "allocation_pools": [], "subnet_uuid": null, "dhcp_option_list": null, "host_routes": null, "addr_from_start": null, "subnet_name": null}], "host_routes": null}, "uuid": "fa6ddd9c-9277-4335-9feb-37d13a6c5d27"}], "fq_name": ["default-domain", "TestSvcRegr-70281290", "svc-vn-left"], "uuid": null}} response string indices must be integers, not str

[Vedamurthy Joshi (vedujoshi)]

See similar error in R1.20 Build 59 as well