R4.0 Build 35 Ubuntu 16.04.2 Newton
On this multi-controller setup, it was seen that tempest test tempest.scenario.test_shelve_instance.TestShelveInstance.test_shelve_volume_backed_instance was failing saying that the default SG deletion is failing. The SG had references to two VMIs
ERROR: tearDownClass (tempest.scenario.test_shelve_instance.TestShelveInstance)
----------------------------------------------------------------------
Traceback (most recent call last):
File "tempest/test.py", line 292, in tearDownClass
six.reraise(etype, value, trace)
File "tempest/test.py", line 275, in tearDownClass
teardown()
File "tempest/test.py", line 563, in clear_credentials
cls._creds_provider.clear_creds()
File "tempest/lib/common/dynamic_creds.py", line 459, in clear_creds
self._cleanup_default_secgroup(creds.tenant_id)
File "tempest/lib/common/dynamic_creds.py", line 406, in _cleanup_default_secgroup
nsg_client.delete_security_group(secgroup['id'])
File "tempest/lib/services/network/security_groups_client.py", line 60, in delete_security_group
return self.delete_resource(uri)
File "tempest/lib/services/network/base.py", line 41, in delete_resource
resp, body = self.delete(req_uri)
File "tempest/lib/common/rest_client.py", line 301, in delete
return self.request('DELETE', url, extra_headers, headers, body)
File "tempest/lib/common/rest_client.py", line 659, in request
self._error_checker(resp, resp_body)
File "tempest/lib/common/rest_client.py", line 780, in _error_checker
raise exceptions.Conflict(resp_body, resp=resp)
tempest.lib.exceptions.Conflict: An object with that identifier already exists
Details: {u'message': u'Security Group 36d05a14-af0e-4007-ad37-6bb567912d86 in use.', u'type': u'SecurityGroupInUse', u'detail': u''}
VMIs :
"virtual_machine_interface_back_refs": [
{
"attr": null,
"href": "http://127.0.0.1:8095/virtual-machine-interface/2a8f4e22-5308-4b76-8219-97626c68776d",
"to": [
"default-domain",
"tempest-TestShelveInstance-1170451400",
"default-domain__tempest-TestShelveInstance-1170451400__snat_c2713679-1b2a-45d2-b3db-eb06646a5b7d_87fa3d04-8303-415c-9a33-ef8dad4deef1__2__left__2"
],
"uuid": "2a8f4e22-5308-4b76-8219-97626c68776d"
},
{
"attr": null,
"href": "http://127.0.0.1:8095/virtual-machine-interface/b2f41f34-1360-48f7-8a55-cac67eb67ef3",
"to": [
"default-domain",
"tempest-TestShelveInstance-1170451400",
"default-domain__tempest-TestShelveInstance-1170451400__snat_c2713679-1b2a-45d2-b3db-eb06646a5b7d_87fa3d04-8303-415c-9a33-ef8dad4deef1__1__left__2"
],
"uuid": "b2f41f34-1360-48f7-8a55-cac67eb67ef3"
}
VMI 2a8f4e22-5308-4b76-8219-97626c68776d had ref to a VM 0e0fee48-f1c8-4f3d-aef9-f0f6d0e639a8:
"virtual_machine_refs": [
{
"attr": null,
"href": "http://127.0.0.1:8095/virtual-machine/0e0fee48-f1c8-4f3d-aef9-f0f6d0e639a8",
"to": [
"default-domain__tempest-TestShelveInstance-1170451400__snat_c2713679-1b2a-45d2-b3db-eb06646a5b7d_87fa3d04-8303-415c-9a33-ef8dad4deef1__2"
],
"uuid": "0e0fee48-f1c8-4f3d-aef9-f0f6d0e639a8"
}
],
VM had ref to a SI:
"service_instance_refs": [
{
"attr": null,
"href": "http://127.0.0.1:8095/service-instance/d232c117-e2dd-4ff2-86a8-6f2050f71742",
"to": [
"default-domain",
"tempest-TestShelveInstance-1170451400",
"snat_c2713679-1b2a-45d2-b3db-eb06646a5b7d_87fa3d04-8303-415c-9a33-ef8dad4deef1"
],
"uuid": "d232c117-e2dd-4ff2-86a8-6f2050f71742"
}
],
svc monitor log had this error :
11/06/2017 03:08:24 PM [contrail-svc-monitor]: __default__ [SYS_ERR]: SvcMonitorLog: <class 'cfgm_common.exceptions.NoIdError'>
Python 2.7.12: /usr/bin/python
Mon Nov 6 15:08:24 2017
A problem occurred in a Python script. Here is the sequence of
function calls leading up to the error, in the order they occurred.
/usr/lib/python2.7/dist-packages/cfgm_common/vnc_amqp.py in _vnc_subscribe_callback(self=<cfgm_common.vnc_amqp.VncAmqpHandle object>, oper_info={u'fq_name': [u'default-domain', u'tempest-TestShelveInstance-730639230', u'tempest-TestShelveInstance-1497752311-router'], u'oper': u'UPDATE', u'request-id': u'req-dcaf2c3f-d84f-4290-83f6-36c2cd5e6a72', u'type': u'logical_router', u'uuid': u'e809fca3-399c-43ac-bf10-a20ad7c0a897'})
59 try:
60 self.oper_info = oper_info
61 self.vnc_subscribe_actions()
62
63 except ConnectionError:
self = <cfgm_common.vnc_amqp.VncAmqpHandle object>
self.vnc_subscribe_actions = <bound method VncAmqpHandle.vnc_subscribe_actions of <cfgm_common.vnc_amqp.VncAmqpHandle object>>
/usr/lib/python2.7/dist-packages/cfgm_common/vnc_amqp.py in vnc_subscribe_actions(self=<cfgm_common.vnc_amqp.VncAmqpHandle object>)
123 (self. obj_type, obj_id, oper))
124 return
125 self.evaluate_dependency()
126
127 def _get_key_from_oper_info(self):
self = <cfgm_common.vnc_amqp.VncAmqpHandle object>
self.evaluate_dependency = <bound method VncAmqpHandle.evaluate_dependency of <cfgm_common.vnc_amqp.VncAmqpHandle object>>
/usr/lib/python2.7/dist-packages/cfgm_common/vnc_amqp.py in evaluate_dependency(self=<cfgm_common.vnc_amqp.VncAmqpHandle object>)
229 res_obj = cls.get(res_id)
230 if res_obj is not None:
231 res_obj.evaluate()
232
233 def close(self):
res_obj = <svc_monitor.config_db.LogicalRouterSM object>
res_obj.evaluate = <bound method LogicalRouterSM.evaluate of <svc_monitor.config_db.LogicalRouterSM object>>
/usr/lib/python2.7/dist-packages/svc_monitor/config_db.py in evaluate(self=<svc_monitor.config_db.LogicalRouterSM object>)
1271
1272 def evaluate(self):
1273 self._manager.snat_agent.update_snat_instance(self)
1274
1275 # end LogicalRouterSM
self = <svc_monitor.config_db.LogicalRouterSM object>
self._manager = <svc_monitor.svc_monitor.SvcMonitor object>
self._manager.snat_agent = <svc_monitor.snat_agent.SNATAgent object>
self._manager.snat_agent.update_snat_instance = <bound method SNATAgent.update_snat_instance of <svc_monitor.snat_agent.SNATAgent object>>
/usr/lib/python2.7/dist-packages/svc_monitor/snat_agent.py in update_snat_instance(self=<svc_monitor.snat_agent.SNATAgent object>, router_obj=<svc_monitor.config_db.LogicalRouterSM object>)
76 if router_obj.virtual_network:
77 if router_obj.service_instance is None:
78 self._add_snat_instance(router_obj)
79 else:
80 if router_obj.service_instance:
self = <svc_monitor.snat_agent.SNATAgent object>
self._add_snat_instance = <bound method SNATAgent._add_snat_instance of <svc_monitor.snat_agent.SNATAgent object>>
router_obj = <svc_monitor.config_db.LogicalRouterSM object>
/usr/lib/python2.7/dist-packages/svc_monitor/snat_agent.py in _add_snat_instance(self=<svc_monitor.snat_agent.SNATAgent object>, router_obj=<svc_monitor.config_db.LogicalRouterSM object>)
208 # Add logical gateway virtual network
209 vnc_rtr_obj.set_service_instance(si_obj)
210 self._vnc_lib.logical_router_update(vnc_rtr_obj)
211 # end add_snat_instance
212
self = <svc_monitor.snat_agent.SNATAgent object>
self._vnc_lib = <vnc_api.vnc_api.VncApi object>
self._vnc_lib.logical_router_update = <functools.partial object>
vnc_rtr_obj = <vnc_api.gen.resource_client.LogicalRouter object>
/usr/lib/python2.7/dist-packages/vnc_api/vnc_api.py in wrapper(self=<vnc_api.vnc_api.VncApi object>, *args=('logical-router', <vnc_api.gen.resource_client.LogicalRouter object>), **kwargs={})
40 retry_on_error=False)
41 self._parse_homepage(homepage)
42 return func(self, *args, **kwargs)
43 return wrapper
44
func = <function _object_update>
self = <vnc_api.vnc_api.VncApi object>
args = ('logical-router', <vnc_api.gen.resource_client.LogicalRouter object>)
kwargs = {}
/usr/lib/python2.7/dist-packages/vnc_api/vnc_api.py in _object_update(self=<vnc_api.vnc_api.VncApi object>, res_type='logical-router', obj=<vnc_api.gen.resource_client.LogicalRouter object>)
574 uri = obj_cls.resource_uri_base[res_type] + '/' + obj.uuid
575 content = self._request_server(
576 rest.OP_PUT, uri, data=json_body)
577
578 # Generate POST on /prop-collection-update if needed/pending
global rest = <module 'cfgm_common.rest' from '/usr/lib/python2.7/dist-packages/cfgm_common/rest.pyc'>
rest.OP_PUT = 3
uri = u'/logical-router/e809fca3-399c-43ac-bf10-a20ad7c0a897'
data undefined
json_body = '{"logical-router":{"service_instance_refs": [{"t... "uuid": "e809fca3-399c-43ac-bf10-a20ad7c0a897"}}'
/usr/lib/python2.7/dist-packages/vnc_api/vnc_api.py in _request_server(self=<vnc_api.vnc_api.VncApi object>, op=3, url=u'/logical-router/e809fca3-399c-43ac-bf10-a20ad7c0a897', data='{"logical-router":{"service_instance_refs": [{"t... "uuid": "e809fca3-399c-43ac-bf10-a20ad7c0a897"}}', retry_on_error=True, retry_after_authn=False, retry_count=30)
871 return self._request(
872 op, url, data=data, retry_on_error=retry_on_error,
873 retry_after_authn=retry_after_authn, retry_count=retry_count)
874 # end _request_server
875
retry_after_authn = False
retry_count = 30
/usr/lib/python2.7/dist-packages/vnc_api/vnc_api.py in _request(self=<vnc_api.vnc_api.VncApi object>, op=3, url=u'/logical-router/e809fca3-399c-43ac-bf10-a20ad7c0a897', data='{"logical-router":{"service_instance_refs": [{"t... "uuid": "e809fca3-399c-43ac-bf10-a20ad7c0a897"}}', retry_on_error=True, retry_after_authn=False, retry_count=30)
949 elif status == 404:
950 raise NoIdError('Error: oper %s url %s body %s response %s'
951 % (op, url, data, content))
952 elif status == 403:
953 raise PermissionDenied(content)
op = 3
url = u'/logical-router/e809fca3-399c-43ac-bf10-a20ad7c0a897'
data = '{"logical-router":{"service_instance_refs": [{"t... "uuid": "e809fca3-399c-43ac-bf10-a20ad7c0a897"}}'
content = u'Unknown id: e809fca3-399c-43ac-bf10-a20ad7c0a897'
<class 'cfgm_common.exceptions.NoIdError'>: Unknown id: Error: oper 3 url /logical-router/e809fca3-399c-43ac-bf10-a20ad7c0a897 body {"logical-router":{"service_instance_refs": [{"to": ["default-domain", "tempest-TestShelveInstance-730639230", "snat_e809fca3-399c-43ac-bf10-a20ad7c0a897_25eb9ae6-b6a3-46ab-803d-d9f784834d45"], "uuid": "4f4aefee-1b41-47a2-b8e6-f7841b344c64"}], "uuid": "e809fca3-399c-43ac-bf10-a20ad7c0a897"}} response Unknown id: e809fca3-399c-43ac-bf10-a20ad7c0a897
__class__ = <class 'cfgm_common.exceptions.NoIdError'>
__delattr__ = <method-wrapper '__delattr__' of NoIdError object>
__dict__ = {'_unknown_id': u'Error: oper 3 url /logical-router/e809fca3-399...Unknown id: e809fca3-399c-43ac-bf10-a20ad7c0a897'}
__doc__ = None
__format__ = <built-in method __format__ of NoIdError object>
__getattribute__ = <method-wrapper '__getattribute__' of NoIdError object>
__getitem__ = <method-wrapper '__getitem__' of NoIdError object>
__getslice__ = <method-wrapper '__getslice__' of NoIdError object>
__hash__ = <method-wrapper '__hash__' of NoIdError object>
__init__ = <bound method NoIdError.__init__ of NoIdError()>
__module__ = 'cfgm_common.exceptions'
__new__ = <built-in method __new__ of type object>
__reduce__ = <built-in method __reduce__ of NoIdError object>
__reduce_ex__ = <built-in method __reduce_ex__ of NoIdError object>
__repr__ = <method-wrapper '__repr__' of NoIdError object>
__setattr__ = <method-wrapper '__setattr__' of NoIdError object>
__setstate__ = <built-in method __setstate__ of NoIdError object>
__sizeof__ = <built-in method __sizeof__ of NoIdError object>
__str__ = <bound method NoIdError.__str__ of NoIdError()>
__subclasshook__ = <built-in method __subclasshook__ of type object>
__unicode__ = <built-in method __unicode__ of NoIdError object>
__weakref__ = None
_unknown_id = u'Error: oper 3 url /logical-router/e809fca3-399...Unknown id: e809fca3-399c-43ac-bf10-a20ad7c0a897'
args = ()
message = ''
The above is a description of an error in a Python program. Here is
the original traceback:
Traceback (most recent call last):
File "/usr/lib/python2.7/dist-packages/cfgm_common/vnc_amqp.py", line 61, in _vnc_subscribe_callback
self.vnc_subscribe_actions()
File "/usr/lib/python2.7/dist-packages/cfgm_common/vnc_amqp.py", line 125, in vnc_subscribe_actions
self.evaluate_dependency()
File "/usr/lib/python2.7/dist-packages/cfgm_common/vnc_amqp.py", line 231, in evaluate_dependency
res_obj.evaluate()
File "/usr/lib/python2.7/dist-packages/svc_monitor/config_db.py", line 1273, in evaluate
self._manager.snat_agent.update_snat_instance(self)
File "/usr/lib/python2.7/dist-packages/svc_monitor/snat_agent.py", line 78, in update_snat_instance
self._add_snat_instance(router_obj)
File "/usr/lib/python2.7/dist-packages/svc_monitor/snat_agent.py", line 210, in _add_snat_instance
self._vnc_lib.logical_router_update(vnc_rtr_obj)
File "/usr/lib/python2.7/dist-packages/vnc_api/vnc_api.py", line 42, in wrapper
return func(self, *args, **kwargs)
File "/usr/lib/python2.7/dist-packages/vnc_api/vnc_api.py", line 576, in _object_update
rest.OP_PUT, uri, data=json_body)
File "/usr/lib/python2.7/dist-packages/vnc_api/vnc_api.py", line 873, in _request_server
retry_after_authn=retry_after_authn, retry_count=retry_count)
File "/usr/lib/python2.7/dist-packages/vnc_api/vnc_api.py", line 951, in _request
% (op, url, data, content))
NoIdError: Unknown id: Error: oper 3 url /logical-router/e809fca3-399c-43ac-bf10-a20ad7c0a897 body {"logical-router":{"service_instance_refs": [{"to": ["default-domain", "tempest-TestShelveInstance-730639230", "snat_e809fca3-399c-43ac-bf10-a20ad7c0a897_25eb9ae6-b6a3-46ab-803d-d9f784834d45"], "uuid": "4f4aefee-1b41-47a2-b8e6-f7841b344c64"}], "uuid": "e809fca3-399c-43ac-bf10-a20ad7c0a897"}} response Unknown id: e809fca3-399c-43ac-bf10-a20ad7c0a897
