Build 1.10 2229
port-delete is failing randomly. This is a 3-node cfgm setup :
env.roledefs = {
'all': [host1, host2, host3, host4, host5],
'cfgm': [host1,host4,host5],
'openstack': [host2],
'control': [host1,host4],
'compute': [host3],
'collector': [host1,host4,host5],
'webui': [host1],
'database': [host1,host4,host5],
'build': [host_build],
}
env.hostnames = {
'all': ['nodec22', 'nodeg30', 'nodeg29', 'nodeh4', 'nodeh5']
}
(Pdb) print body
{'port': {'network_id': u'5352271b-9143-4b15-b1ed-bacfbe5393c5', 'tenant_id': u'ac822370b03c4ed9bc837f391cb0faab'}}
(Pdb) response = self.neutron_h.create_port(body)
(Pdb) print response
{u'port': {u'status': u'ACTIVE', u'name': u'4ce52fc4-a672-43ad-978f-0e7f3b7fad5e', u'admin_state_up': True, u'network_id': u'5352271b-9143-4b15-b1ed-bacfbe5393c5', u'tenant_id': u'ac822370b03c4ed9bc837f391cb0faab', u'device_owner': u'TODO-device-owner', u'mac_address': u'02:4c:e5:2f:c4:a6', u'fixed_ips': [{u'subnet_id': u'aed8632e-3019-47c4-9dab-2435e3d44028', u'ip_address': u'29.172.20.254', u'port_id': u'4ce52fc4-a672-43ad-978f-0e7f3b7fad5e', u'net_id': u'5352271b-9143-4b15-b1ed-bacfbe5393c5'}], u'id': u'4ce52fc4-a672-43ad-978f-0e7f3b7fad5e', u'security_groups': [u'e8d6bb68-5789-4a3a-8a92-1fbba58c9195'], u'device_id': u''}}
(Pdb) self.neutron_h.delete_port(response['port']['id'])
*** NeutronClientException: Request Failed: internal server error while processing your request.
(Pdb)
-------------Log when such error happens :
server.log :
Internal Server Error
2014-07-04 16:32:55.58 ERROR [neutron.api.v2.resource] delete failed
Traceback (most recent call last):
File "/usr/lib/python2.7/dist-packages/neutron/api/v2/resource.py", line 84, in resource
result = method(request=request, **args)
File "/usr/lib/python2.7/dist-packages/neutron/api/v2/base.py", line 432, in delete
obj_deleter(request.context, id, **kwargs)
File "/usr/lib/python2.7/dist-packages/neutron_plugin_contrail/plugins/opencontrail/contrail_plugin_core.py", line 515, in delete_port
self._delete_resource('port', context, port_id)
File "/usr/lib/python2.7/dist-packages/neutron_plugin_contrail/plugins/opencontrail/contrail_plugin_core.py", line 317, in _delete_resource
res_type, 'DELETE')
File "/usr/lib/python2.7/dist-packages/neutron_plugin_contrail/plugins/opencontrail/contrail_plugin_core.py", line 204, in _request_backend
return response.status_code, response.json()
File "/usr/lib/python2.7/dist-packages/requests/models.py", line 651, in json
return json.loads(self.text or self.content, **kwargs)
File "/usr/lib/python2.7/dist-packages/simplejson/__init__.py", line 488, in loads
return _default_decoder.decode(s)
File "/usr/lib/python2.7/dist-packages/simplejson/decoder.py", line 371, in decode
obj, end = self.raw_decode(s)
File "/usr/lib/python2.7/dist-packages/simplejson/decoder.py", line 390, in raw_decode
return self.scan_once(s, idx=_w(s, idx).end())
JSONDecodeError: Expecting value: line 1 column 1 (char 0)
------------
contrail-api log :
AddrMgmt: free IP 29.172.20.254, vn=[u'default-domain', u'admin', u'name-YwHHTV']
Notification Message: {u'imid': u'contrail:instance-ip:4a3c46d9-4962-480e-bbab-8118349a047c',
u'obj_dict': {u'fq_name': [u'4a3c46d9-4962-480e-bbab-8118349a047c'],
u'id_perms': {u'created': u'2014-07-04T11:02:48.888029',
u'description': None,
u'enable': True,
u'last_modified': u'2014-07-04T11:02:48.888029',
u'permissions': {u'group': u'cloud-admin-group',
u'group_access': 7,
u'other_access': 7,
u'owner': u'cloud-admin',
u'owner_access': 7},
u'uuid': {u'uuid_lslong': 13523044247071753340L,
u'uuid_mslong': 5349228356462331918}},
u'instance_ip_address': u'29.172.20.254',
u'uuid': u'4a3c46d9-4962-480e-bbab-8118349a047c',
u'virtual_machine_interface_refs': [{u'attr': None,
u'href': u'http://0.0.0.0:9100/virtual-machine-interface/7e7248da-e172-41f9-96a8-0952f16179db',
u'to': [u'default-domain',
u'admin',
u'7e7248da-e172-41f9-96a8-0952f16179db'],
u'uuid': u'7e7248da-e172-41f9-96a8-0952f16179db'}],
u'virtual_network_refs': [{u'attr': None,
u'href': u'http://0.0.0.0:9100/virtual-network/5352271b-9143-4b15-b1ed-bacfbe5393c5',
u'to': [u'default-domain',
u'admin',
u'name-YwHHTV'],
u'uuid': u'5352271b-9143-4b15-b1ed-bacfbe5393c5'}]},
u'oper': u'DELETE',
u'type': u'instance-ip',
u'uuid': u'4a3c46d9-4962-480e-bbab-8118349a047c'}
<class 'cfgm_common.exceptions.NoIdError'>
Python 2.7.3: /usr/bin/python
Fri Jul 4 16:32:55 2014
A problem occurred in a Python script. Here is the sequence of
function calls leading up to the error, in the order they occurred.
/usr/lib/python2.7/dist-packages/vnc_openstack/neutron_plugin_interface.py in plugin_delete_port(self=<vnc_openstack.neutron_plugin_interface.NeutronPluginInterface object>, context={u'is_admin': True, u'operation': u'DELETE', u'roles': [u'admin', u'KeystoneServiceAdmin', u'KeystoneAdmin'], u'tenant': u'ac822370b03c4ed9bc837f391cb0faab', u'tenant_id': u'ac822370b03c4ed9bc837f391cb0faab', u'type': u'port', u'user_id': u'346eeb198b4346d1bfb1bcbf3affcaa8'}, port={u'fields': None, u'filters': None, u'id': u'7e7248da-e172-41f9-96a8-0952f16179db'})
369 try:
370 cfgdb = self._get_user_cfgdb(context)
371 cfgdb.port_delete(port['id'])
372 LOG.debug("plugin_delete_port(): " + pformat(port['id']))
373 except Exception as e:
cfgdb = <vnc_openstack.neutron_plugin_db.DBInterface object>
cfgdb.port_delete = <bound method DBInterface.port_delete of <vnc_openstack.neutron_plugin_db.DBInterface object>>
port = {u'fields': None, u'filters': None, u'id': u'7e7248da-e172-41f9-96a8-0952f16179db'}
/usr/lib/python2.7/dist-packages/vnc_openstack/neutron_plugin_db.py in port_delete(self=<vnc_openstack.neutron_plugin_db.DBInterface object>, port_id=u'7e7248da-e172-41f9-96a8-0952f16179db')
3213 # update cache on successful deletion
3214 try:
3215 tenant_id = self._get_obj_tenant_id('port', port_id)
3216 self._db_cache['q_tenant_port_count'][tenant_id] = \
3217 self._db_cache['q_tenant_port_count'][tenant_id] - 1
tenant_id undefined
self = <vnc_openstack.neutron_plugin_db.DBInterface object>
self._get_obj_tenant_id = <bound method DBInterface._get_obj_tenant_id of ..._openstack.neutron_plugin_db.DBInterface object>>
port_id = u'7e7248da-e172-41f9-96a8-0952f16179db'
/usr/lib/python2.7/dist-packages/vnc_openstack/neutron_plugin_db.py in _get_obj_tenant_id(self=<vnc_openstack.neutron_plugin_db.DBInterface object>, q_type='port', obj_uuid=u'7e7248da-e172-41f9-96a8-0952f16179db')
389 # Seed the cache and return
390 if q_type == 'port':
391 port_obj = self._virtual_machine_interface_read(obj_uuid)
392 net_id = port_obj.get_virtual_network_refs()[0]['uuid']
393 # recurse up type-hierarchy
port_obj undefined
self = <vnc_openstack.neutron_plugin_db.DBInterface object>
self._virtual_machine_interface_read = <bound method DBInterface._virtual_machine_inter..._openstack.neutron_plugin_db.DBInterface object>>
obj_uuid = u'7e7248da-e172-41f9-96a8-0952f16179db'
/usr/lib/python2.7/dist-packages/vnc_openstack/neutron_plugin_db.py in _virtual_machine_interface_read(self=<vnc_openstack.neutron_plugin_db.DBInterface object>, port_id=u'7e7248da-e172-41f9-96a8-0952f16179db', fq_name=None, fields=None)
615 except KeyError:
616 port_obj = self._vnc_lib.virtual_machine_interface_read(
617 id=port_id, fields=fields)
618 fq_name_str = json.dumps(port_obj.get_fq_name())
619 self._db_cache['vnc_ports'][port_id] = port_obj
builtinid = <built-in function id>
port_id = u'7e7248da-e172-41f9-96a8-0952f16179db'
fields = None
/usr/lib/python2.7/dist-packages/vnc_api/gen/vnc_api_client_gen.py in virtual_machine_interface_read(self=<vnc_api.vnc_api.VncApi object>, fq_name=None, fq_name_str=None, id=u'7e7248da-e172-41f9-96a8-0952f16179db', ifmap_id=None, fields=None)
4831 query_params = {'exclude_back_refs':True,
4832 'exclude_children':True,}
4833 content = self._request_server(rest.OP_GET, uri, query_params)
4834
4835 obj_dict = json.loads(content)['virtual-machine-interface']
content undefined
self = <vnc_api.vnc_api.VncApi object>
self._request_server = <bound method VncApi._request_server of <vnc_api.vnc_api.VncApi object>>
global rest = <module 'cfgm_common.rest' from '/usr/lib/python2.7/dist-packages/cfgm_common/rest.pyc'>
rest.OP_GET = 2
uri = u'/virtual-machine-interface/7e7248da-e172-41f9-96a8-0952f16179db'
query_params = {'exclude_back_refs': True, 'exclude_children': True}
/usr/lib/python2.7/dist-packages/vnc_api/vnc_api.py in _request_server(self=<vnc_api.vnc_api.VncApi object>, op=2, url=u'/virtual-machine-interface/7e7248da-e172-41f9-96a8-0952f16179db', data={'exclude_back_refs': True, 'exclude_children': True}, retry_on_error=True, retry_after_authn=False)
339 elif status == 404:
340 raise NoIdError('Error: oper %s url %s body %s response %s'
341 % (op, url, data, content))
342 elif status == 403:
343 raise PermissionDenied(content)
op = 2
url = u'/virtual-machine-interface/7e7248da-e172-41f9-96a8-0952f16179db'
data = {'exclude_back_refs': True, 'exclude_children': True}
content = u'No virtual-machine-interface object found for id 7e7248da-e172-41f9-96a8-0952f16179db'
<class 'cfgm_common.exceptions.NoIdError'>: Unknown id: Error: oper 2 url /virtual-machine-interface/7e7248da-e172-41f9-96a8-0952f16179db body {'exclude_back_refs': True, 'exclude_children': True} response No virtual-machine-interface object found for id 7e7248da-e172-41f9-96a8-0952f16179db
__class__ = <class 'cfgm_common.exceptions.NoIdError'>
__delattr__ = <method-wrapper '__delattr__' of NoIdError object>
__dict__ = {'_unknown_id': u"Error: oper 2 url /virtual-machine-interface/7...ound for id 7e7248da-e172-41f9-96a8-0952f16179db"}
__doc__ = None
__format__ = <built-in method __format__ of NoIdError object>
__getattribute__ = <method-wrapper '__getattribute__' of NoIdError object>
__getitem__ = <method-wrapper '__getitem__' of NoIdError object>
__getslice__ = <method-wrapper '__getslice__' of NoIdError object>
__hash__ = <method-wrapper '__hash__' of NoIdError object>
__init__ = <bound method NoIdError.__init__ of NoIdError()>
__module__ = 'cfgm_common.exceptions'
__new__ = <built-in method __new__ of type object>
__reduce__ = <built-in method __reduce__ of NoIdError object>
__reduce_ex__ = <built-in method __reduce_ex__ of NoIdError object>
__repr__ = <method-wrapper '__repr__' of NoIdError object>
__setattr__ = <method-wrapper '__setattr__' of NoIdError object>
__setstate__ = <built-in method __setstate__ of NoIdError object>
__sizeof__ = <built-in method __sizeof__ of NoIdError object>
__str__ = <bound method NoIdError.__str__ of NoIdError()>
__subclasshook__ = <built-in method __subclasshook__ of type object>
__unicode__ = <built-in method __unicode__ of NoIdError object>
__weakref__ = None
_unknown_id = u"Error: oper 2 url /virtual-machine-interface/7...ound for id 7e7248da-e172-41f9-96a8-0952f16179db"
args = ()
message = ''
The above is a description of an error in a Python program. Here is
the original traceback:
Traceback (most recent call last):
File "/usr/lib/python2.7/dist-packages/vnc_openstack/neutron_plugin_interface.py", line 371, in plugin_delete_port
cfgdb.port_delete(port['id'])
File "/usr/lib/python2.7/dist-packages/vnc_openstack/neutron_plugin_db.py", line 3215, in port_delete
tenant_id = self._get_obj_tenant_id('port', port_id)
File "/usr/lib/python2.7/dist-packages/vnc_openstack/neutron_plugin_db.py", line 391, in _get_obj_tenant_id
port_obj = self._virtual_machine_interface_read(obj_uuid)
File "/usr/lib/python2.7/dist-packages/vnc_openstack/neutron_plugin_db.py", line 617, in _virtual_machine_interface_read
id=port_id, fields=fields)
File "/usr/lib/python2.7/dist-packages/vnc_api/gen/vnc_api_client_gen.py", line 4833, in virtual_machine_interface_read
content = self._request_server(rest.OP_GET, uri, query_params)
File "/usr/lib/python2.7/dist-packages/vnc_api/vnc_api.py", line 341, in _request_server
% (op, url, data, content))
NoIdError: Unknown id: Error: oper 2 url /virtual-machine-interface/7e7248da-e172-41f9-96a8-0952f16179db body {'exclude_back_refs': True, 'exclude_children': True} response No virtual-machine-interface object found for id 7e7248da-e172-41f9-96a8-0952f16179db
Traceback (most recent call last):
File "/usr/lib/python2.7/dist-packages/bottle.py", line 764, in _handle
return route.call(**args)
File "/usr/lib/python2.7/dist-packages/bottle.py", line 1575, in wrapper
rv = callback(*a, **ka)
File "/usr/lib/python2.7/dist-packages/vnc_openstack/neutron_plugin_interface.py", line 423, in plugin_http_post_port
return self.plugin_delete_port(context, port)
File "/usr/lib/python2.7/dist-packages/vnc_openstack/neutron_plugin_interface.py", line 375, in plugin_delete_port
raise e
NoIdError: Unknown id: Error: oper 2 url /virtual-machine-interface/7e7248da-e172-41f9-96a8-0952f16179db body {'exclude_back_refs': True, 'exclude_children': True} response No virtual-machine-interface object found for id 7e7248da-e172-41f9-96a8-0952f16179db
Could you please provide access to log files and steps to recreate?