With build 29 ubuntu/Kilo Neutron Server Log during vm creation failure 2017-03-14 19:47:09.157 DEBUG [keystoneclient.auth.identity.v2] Making authentication request to http://10.204.217.112:35357/v2.0/tokens 2017-03-14 19:47:09.293 DEBUG [neutron_plugin_contrail.plugins.opencontrail.contrail_plugin] get_port(): filters: {u'device_id': [u'c0c19aad-aa8a-4444-87c8-701a99642af3', u'fbda332a-43d5-43a7-8d48-40dc56a1a701', u'23a5864a-d177-48ed-ac44-37d029652b50', u'0e2e1a08-d1ef-4c36-8dc0-a3b2b455db6b', u'38c6d945-41e8-4bc4-99a7-d31ce3d3306a', u'7dd8eba1-eca6-4b5d-99ea-8eed5583e12f', u'2ab683dc-f009-4b22-92ee-251062b4b548']} data: [{u'status': u'ACTIVE', u'binding:host_id': u'nodeh8', u'name': u'4bffbd51-5218-4662-95e0-4b5789938d33', u'admin_state_up': True, u'network_id': u'90f2b682-5025-47a3-821d-9d9f4975dff6', u'tenant_id': u'c1ac8f5263d3498f9fa13b2bc4f46113', u'binding:vif_details': {u'port_filter': True}, u'binding:vnic_type': u'normal', u'binding:vif_type': u'vrouter', u'device_owner': u'compute:nova', u'mac_address': u'02:4b:ff:bd:51:52', u'port_security_enabled': True, u'device_id': u'38c6d945-41e8-4bc4-99a7-d31ce3d3306a', u'fixed_ips': [{u'subnet_id': u'9c6cb02d-3494-4e5d-8e96-b2ffbec9a5bf', u'ip_address': u'201.153.235.3'}], u'id': u'4bffbd51-5218-4662-95e0-4b5789938d33', u'security_groups': [u'95024f7a-1710-4b21-8bff-d78a3e0c5fa9'], u'contrail:fq_name': [u'default-domain', u'ctest-TestSvcRegr-30759412', u'4bffbd51-5218-4662-95e0-4b5789938d33']}, {u'status': u'ACTIVE', u'binding:host_id': u'nodeh8', u'name': u'16ebcff7-9dcd-4aa4-a90c-269efeacbb80', u'admin_state_up': True, u'network_id': u'36a5b12f-b8e3-4975-b32c-56fc1fe8c97e', u'tenant_id': u'c1ac8f5263d3498f9fa13b2bc4f46113', u'binding:vif_details': {u'port_filter': True}, u'binding:vnic_type': u'normal', u'binding:vif_type': u'vrouter', u'device_owner': u'compute:nova', u'mac_address': u'02:16:eb:cf:f7:9d', u'port_security_enabled': True, u'device_id': u'2ab683dc-f009-4b22-92ee-251062b4b548', u'fixed_ips': [{u'subnet_id': u'404092e2-e62f-4041-ac56-8511e2d7031c', u'ip_address': u'22.228.142.3'}], u'id': u'16ebcff7-9dcd-4aa4-a90c-269efeacbb80', u'security_groups': [u'95024f7a-1710-4b21-8bff-d78a3e0c5fa9'], u'contrail:fq_name': [u'default-domain', u'ctest-TestSvcRegr-30759412', u'16ebcff7-9dcd-4aa4-a90c-269efeacbb80']}, {u'status': u'ACTIVE', u'binding:host_id': u'nodeh8', u'name': u'default-domain__ctest-TestSvcRegr-30759412__ctest-in_net_svc_instance-20658585_1__1__right__3', u'admin_state_up': True, u'network_id': u'90f2b682-5025-47a3-821d-9d9f4975dff6', u'tenant_id': u'c1ac8f5263d3498f9fa13b2bc4f46113', u'binding:vif_details': {u'port_filter': True}, u'binding:vnic_type': u'normal', u'binding:vif_type': u'vrouter', u'device_owner': u'compute:None', u'mac_address': u'02:30:37:35:74:71', u'port_security_enabled': True, u'device_id': u'0e2e1a08-d1ef-4c36-8dc0-a3b2b455db6b', u'fixed_ips': [{u'subnet_id': u'9c6cb02d-3494-4e5d-8e96-b2ffbec9a5bf', u'ip_address': u'201.153.235.4'}], u'id': u'00257e38-816c-4c0a-a12a-a96ce08a2121', u'security_groups': [u'95024f7a-1710-4b21-8bff-d78a3e0c5fa9'], u'contrail:fq_name': [u'default-domain', u'ctest-TestSvcRegr-30759412', u'default-domain__ctest-TestSvcRegr-30759412__ctest-in_net_svc_instance-20658585_1__1__right__3']}, {u'status': u'ACTIVE', u'binding:host_id': u'nodeh8', u'name': u'1e443deb-df95-4a79-bd48-8762c9256f29', u'admin_state_up': True, u'network_id': u'734cf458-6291-48bf-800d-6353073d59db', u'tenant_id': u'1098ce6ea17842f1aed4c535c98f5d41', u'binding:vif_details': {u'port_filter': True}, u'binding:vnic_type': u'normal', u'binding:vif_type': u'vrouter', u'device_owner': u'compute:nova', u'mac_address': u'02:1e:44:3d:eb:df', u'port_security_enabled': True, u'device_id': u'7dd8eba1-eca6-4b5d-99ea-8eed5583e12f', u'fixed_ips': [{u'subnet_id': u'89ac1406-ace0-434d-8fe5-00a3327010f4', u'ip_address': u'2b36:f153:3569:633c::3'}, {u'subnet_id': u'20c54df2-3c27-4a09-848d-d76e7eb0f9bf', u'ip_address': u'178.102.125.3'}], u'id': u'1e443deb-df95-4a79-bd48-8762c9256f29', u'security_groups': [u'42fdce27-2ca3-4af5-b78a-1423b8f88736'], u'contrail:fq_name': [u'default-domain', u'ctest-TestSvcRegrIPv6-61982455', u'1e443deb-df95-4a79-bd48-8762c9256f29']}, {u'status': u'ACTIVE', u'binding:host_id': u'nodeh8', u'name': u'7db75a4a-8781-4592-bc5b-24596260cc48', u'admin_state_up': True, u'network_id': u'26cb7158-e4c3-433c-acaf-72aad0b9a462', u'tenant_id': u'093ad0f500914035b9562e039729a8d2', u'binding:vif_details': {u'port_filter': True}, u'binding:vnic_type': u'normal', u'binding:vif_type': u'vrouter', u'device_owner': u'compute:nova', u'mac_address': u'02:7d:b7:5a:4a:87', u'port_security_enabled': True, u'device_id': u'fbda332a-43d5-43a7-8d48-40dc56a1a701', u'fixed_ips': [], u'id': u'7db75a4a-8781-4592-bc5b-24596260cc48', u'security_groups': [u'40474842-eccc-4aa8-9da5-ca1455db3b9c'], u'contrail:fq_name': [u'default-domain', u'ctest-TestBasicVMVN-88493108', u'7db75a4a-8781-4592-bc5b-24596260cc48']}, {u'status': u'ACTIVE', u'binding:host_id': u'nodeh8', u'name': u'89e90633-4ca7-4533-beed-173caf74381a', u'admin_state_up': True, u'network_id': u'26cb7158-e4c3-433c-acaf-72aad0b9a462', u'tenant_id': u'093ad0f500914035b9562e039729a8d2', u'binding:vif_details': {u'port_filter': True}, u'binding:vnic_type': u'normal', u'binding:vif_type': u'vrouter', u'device_owner': u'compute:nova', u'mac_address': u'02:89:e9:06:33:4c', u'port_security_enabled': True, u'device_id': u'23a5864a-d177-48ed-ac44-37d029652b50', u'fixed_ips': [{u'subnet_id': u'38df70da-5502-4e67-87a4-0b6743df12c6', u'ip_address': u'65.102.163.3'}], u'id': u'89e90633-4ca7-4533-beed-173caf74381a', u'security_groups': [u'40474842-eccc-4aa8-9da5-ca1455db3b9c'], u'contrail:fq_name': [u'default-domain', u'ctest-TestBasicVMVN-88493108', u'89e90633-4ca7-4533-beed-173caf74381a']}, {u'status': u'ACTIVE', u'binding:host_id': u'nodeh8', u'name': u'default-domain__ctest-TestSvcRegr-30759412__ctest-in_net_svc_instance-20658585_1__1__left__2', u'admin_state_up': True, u'network_id': u'36a5b12f-b8e3-4975-b32c-56fc1fe8c97e', u'tenant_id': u'c1ac8f5263d3498f9fa13b2bc4f46113', u'binding:vif_details': {u'port_filter': True}, u'binding:vnic_type': u'normal', u'binding:vif_type': u'vrouter', u'device_owner': u'compute:None', u'mac_address': u'02:b0:8f:0d:5c:92', u'port_security_enabled': True, u'device_id': u'0e2e1a08-d1ef-4c36-8dc0-a3b2b455db6b', u'fixed_ips': [{u'subnet_id': u'404092e2-e62f-4041-ac56-8511e2d7031c', u'ip_address': u'22.228.142.4'}], u'id': u'370a5dea-c063-495a-af71-d2a107e92bbe', u'security_groups': [u'95024f7a-1710-4b21-8bff-d78a3e0c5fa9'], u'contrail:fq_name': [u'default-domain', u'ctest-TestSvcRegr-30759412', u'default-domain__ctest-TestSvcRegr-30759412__ctest-in_net_svc_instance-20658585_1__1__left__2']}, {u'status': u'ACTIVE', u'binding:host_id': u'nodeh8', u'name': u'default-domain__ctest-TestSvcRegr-30759412__ctest-in_net_svc_instance-20658585_1__1__management__1', u'admin_state_up': True, u'network_id': u'9db56f73-d033-4888-82ec-d73e28afc96e', u'tenant_id': u'c1ac8f5263d3498f9fa13b2bc4f46113', u'binding:vif_details': {u'port_filter': True}, u'binding:vnic_type': u'normal', u'binding:vif_type': u'vrouter', u'device_owner': u'compute:None', u'mac_address': u'02:79:47:4e:27:30', u'port_security_enabled': True, u'device_id': u'0e2e1a08-d1ef-4c36-8dc0-a3b2b455db6b', u'fixed_ips': [{u'subnet_id': u'4969d2c4-c9a5-484c-9067-8a92e6cabb2b', u'ip_address': u'30.10.71.3'}], u'id': u'c9e33335-2c9b-453a-889f-a6020466c4f4', u'security_groups': [u'95024f7a-1710-4b21-8bff-d78a3e0c5fa9'], u'contrail:fq_name': [u'default-domain', u'ctest-TestSvcRegr-30759412', u'default-domain__ctest-TestSvcRegr-30759412__ctest-in_net_svc_instance-20658585_1__1__management__1']}] 2017-03-14 19:47:09.296 INFO [neutron.wsgi] 10.204.217.112 - - [14/Mar/2017 19:47:09] "GET /v2.0/ports.json?device_id=c0c19aad-aa8a-4444-87c8-701a99642af3&device_id=fbda332a-43d5-43a7-8d48-40dc56a1a701&device_id=23a5864a-d177-48ed-ac44-37d029652b50&device_id=0e2e1a08-d1ef-4c36-8dc0-a3b2b455db6b&device_id=38c6d945-41e8-4bc4-99a7-d31ce3d3306a&device_id=7dd8eba1-eca6-4b5d-99ea-8eed5583e12f&device_id=2ab683dc-f009-4b22-92ee-251062b4b548 HTTP/1.1" 200 5934 0.2596172017-03-14 19:47:09.325 DEBUG [keystoneclient.session] Request returned failure status: 401 2017-03-14 19:47:09.296 INFO [neutron.wsgi] 10.204.217.112 - - [14/Mar/2017 19:47:09] "GET /v2.0/ports.json?device_id=c0c19aad-aa8a-4444-87c8-701a99642af3&device_id=fbda332a-43d5-43a7-8d48-40dc56a1a701&device_id=23a5864a-d177-48ed-ac44-37d029652b50&device_id=0e2e1a08-d1ef-4c36-8dc0-a3b2b455db6b&device_id=38c6d945-41e8-4bc4-99a7-d31ce3d3306a&device_id=7dd8eba1-eca6-4b5d-99ea-8eed5583e12f&device_id=2ab683dc-f009-4b22-92ee-251062b4b548 HTTP/1.1" 200 5934 0.2596172017-03-14 19:47:09.325 DEBUG [keystoneclient.session] Request returned failure status: 401 2017-03-14 19:47:09.325 ERROR [neutron.notifiers.nova] Failed to notify nova on events: [{'name': 'network-changed', 'server_uuid': u'0e2e1a08-d1ef-4c36-8dc0-a3b2b455db6b'}] Traceback (most recent call last): File "/usr/lib/python2.7/dist-packages/neutron/notifiers/nova.py", line 243, in send_events batched_events) File "/usr/lib/python2.7/dist-packages/novaclient/v2/contrib/server_external_events.py", line 39, in create return_raw=True) File "/usr/lib/python2.7/dist-packages/novaclient/base.py", line 152, in _create _resp, body = self.api.client.post(url, body=body) File "/usr/lib/python2.7/dist-packages/keystoneclient/adapter.py", line 170, in post return self.request(url, 'POST', **kwargs) File "/usr/lib/python2.7/dist-packages/novaclient/client.py", line 89, in request **kwargs) File "/usr/lib/python2.7/dist-packages/keystoneclient/adapter.py", line 200, in request resp = super(LegacyJsonAdapter, self).request(*args, **kwargs) File "/usr/lib/python2.7/dist-packages/keystoneclient/adapter.py", line 89, in request return self.session.request(url, method, **kwargs) File "/usr/lib/python2.7/dist-packages/keystoneclient/utils.py", line 318, in inner return func(*args, **kwargs) File "/usr/lib/python2.7/dist-packages/keystoneclient/session.py", line 307, in request auth_headers = self.get_auth_headers(auth) File "/usr/lib/python2.7/dist-packages/keystoneclient/session.py", line 589, in get_auth_headers return auth.get_headers(self, **kwargs) File "/usr/lib/python2.7/dist-packages/keystoneclient/auth/base.py", line 114, in get_headers token = self.get_token(session) File "/usr/lib/python2.7/dist-packages/keystoneclient/auth/identity/base.py", line 104, in get_token return self.get_access(session).auth_token File "/usr/lib/python2.7/dist-packages/keystoneclient/auth/identity/base.py", line 144, in get_access self.auth_ref = self.get_auth_ref(session) File "/usr/lib/python2.7/dist-packages/keystoneclient/auth/identity/v2.py", line 78, in get_auth_ref authenticated=False, log=False) File "/usr/lib/python2.7/dist-packages/keystoneclient/session.py", line 488, in post return self.request(url, 'POST', **kwargs) File "/usr/lib/python2.7/dist-packages/keystoneclient/utils.py", line 318, in inner return func(*args, **kwargs) File "/usr/lib/python2.7/dist-packages/keystoneclient/session.py", line 389, in request raise exceptions.from_response(resp, method, url) Unauthorized: User a279f824cdf34366b92e759852a59818 is unauthorized for tenant services (Disable debug mode to suppress these details.) (HTTP 401) (Request-ID: req-9cf72a51-5a22-4e5a-b245-ea0d4a20a2ee) @