Some more logs from http://logs.openstack.org/18/581918/5/check/tripleo-ci-centos-7-scenario002-multinode-oooq-container/3cf2734/logs/undercloud/var/log/containers/tempest/tempest.log.txt.gz#_2018-07-13_04_47_43_593 giving 412 status code. 2018-07-13 04:47:43.582 81 INFO tempest.lib.common.rest_client [tx0059ce08a31c4dd4bd072-005b482eef ] Request (ObjectTest:setUpClass): 200 GET http://192.168.24.7:8080/info 0.431s 2018-07-13 04:47:43.583 81 DEBUG tempest.lib.common.rest_client [tx0059ce08a31c4dd4bd072-005b482eef ] Request - Headers: {'Content-Type': 'application/json', 'Accept': 'application/json', 'X-Auth-Token': ''} Body: None Response - Headers: {'status': '200', u'content-length': '1643', 'content-location': 'http://192.168.24.7:8080/info', u'connection': 'close', u'x-trans-id': 'tx0059ce08a31c4dd4bd072-005b482eef', u'date': 'Fri, 13 Jul 2018 04:47:43 GMT', u'content-type': 'application/json; charset=UTF-8', u'x-openstack-request-id': 'tx0059ce08a31c4dd4bd072-005b482eef'} Body: {"formpost": {}, "bulk_delete": {"max_failed_deletes": 1000, "max_deletes_per_request": 10000}, "versioned_writes": {"allowed_flags": ["x-versions-location", "x-history-location"]}, "bulk_upload": {"max_failed_extractions": 1000, "max_containers_per_extraction": 10000}, "container_quotas": {}, "tempurl": {"outgoing_remove_headers": ["x-object-meta-*"], "methods": ["GET", "HEAD", "PUT", "POST", "DELETE"], "incoming_remove_headers": ["x-timestamp"], "allowed_digests": ["sha1", "sha256", "sha512"], "outgoing_allow_headers": ["x-object-meta-public-*"], "incoming_allow_headers": []}, "s3api": {"max_upload_part_num": 1000, "allow_multipart_uploads": true, "max_bucket_listing": 1000, "s3_acl": false, "min_segment_size": 5242880, "max_parts_listing": 1000, "max_multi_delete_objects": 1000}, "ratelimit": {"account_ratelimit": 0.0, "max_sleep_time_seconds": 60.0, "container_ratelimits": [], "container_listing_ratelimits": []}, "slo": {"max_manifest_segments": 1000, "yield_frequency": 10, "min_segment_size": 1, "max_manifest_size": 2097152}, "account_quotas": {}, "staticweb": {}, "swift": {"max_file_size": 5368709122, "account_listing_limit": 10000, "account_autocreate": true, "max_meta_count": 90, "max_meta_value_length": 256, "container_listing_limit": 10000, "extra_header_count": 0, "max_meta_overall_size": 4096, "version": "2.18.1.dev169", "max_meta_name_length": 128, "max_header_size": 8192, "policies": [{"default": true, "name": "Policy-0", "aliases": "Policy-0"}], "max_object_name_length": 1024, "max_account_name_length": 256, "strict_cors_mode": true, "allow_account_management": true, "max_container_name_length": 256}} _log_request_full /usr/lib/python2.7/site-packages/tempest/lib/common/rest_client.py:434 2018-07-13 04:47:43.592 81 INFO tempest.lib.common.rest_client [txb87cc715719a4d748bdbd-005b482eef ] Request (ObjectTest:setUpClass): 412 PUT http://192.168.24.7:8080/tempest-TestContainer-1152572772 0.008s 2018-07-13 04:47:43.592 81 DEBUG tempest.lib.common.rest_client [txb87cc715719a4d748bdbd-005b482eef ] Request - Headers: {'X-Auth-Token': ''} Body: None Response - Headers: {'status': '412', u'content-length': '7', 'content-location': 'http://192.168.24.7:8080/tempest-TestContainer-1152572772', u'connection': 'close', u'x-trans-id': 'txb87cc715719a4d748bdbd-005b482eef', u'date': 'Fri, 13 Jul 2018 04:47:43 GMT', u'content-type': 'text/html; charset=UTF-8', u'x-openstack-request-id': 'txb87cc715719a4d748bdbd-005b482eef'} Body: Bad URL _log_request_full /usr/lib/python2.7/site-packages/tempest/lib/common/rest_client.py:434 2018-07-13 04:47:43.593 81 INFO tempest.test [-] raised in ObjectTest.setUpClass. Invoking tearDownClass. 2018-07-13 04:47:43.936 81 INFO tempest.lib.common.rest_client [req-0017e931-e73b-4779-bd59-a3d554fc50ee ] Request (ObjectTest:tearDownClass): 204 DELETE http://192.168.24.7:35357/v3/users/e3fe9ace45234074845913b8a52705e6 0.343s 2018-07-13 04:47:43.937 81 DEBUG tempest.lib.common.rest_client [req-0017e931-e73b-4779-bd59-a3d554fc50ee ] Request - Headers: {'Content-Type': 'application/json', 'Accept': 'application/json', 'X-Auth-Token': ''} Body: None Response - Headers: {'status': '204', 'content-location': 'http://192.168.24.7:35357/v3/users/e3fe9ace45234074845913b8a52705e6', u'vary': 'X-Auth-Token', u'server': 'Apache', u'connection': 'close', u'date': 'Fri, 13 Jul 2018 04:47:43 GMT', u'content-type': 'text/plain', u'x-openstack-request-id': 'req-0017e931-e73b-4779-bd59-a3d554fc50ee'} Body: _log_request_full /usr/lib/python2.7/site-packages/tempest/lib/common/rest_client.py:434 2018-07-13 04:47:44.578 81 INFO tempest.lib.common.rest_client [req-c8856df2-423a-4488-b691-21d645fe14e0 ] Request (ObjectTest:tearDownClass): 201 POST http://192.168.24.7:5000/v3/auth/tokens 2018-07-13 04:47:44.578 81 DEBUG tempest.lib.common.rest_client [req-c8856df2-423a-4488-b691-21d645fe14e0 ] Request - Headers: {'Content-Type': 'application/json', 'Accept': 'application/json'} Body: Response - Headers: {'status': '201', u'content-length': '11349', 'content-location': 'http://192.168.24.7:5000/v3/auth/tokens', u'x-subject-token': '', u'vary': 'X-Auth-Token', u'server': 'Apache', u'connection': 'close', u'date': 'Fri, 13 Jul 2018 04:47:43 GMT', u'content-type': 'application/json', u'x-openstack-request-id': 'req-c8856df2-423a-4488-b691-21d645fe14e0'} Body: {"token": {"is_domain": false, "methods": ["password"], "roles": [{"id": "9d1e7dd8886a462db73c5f49b06ef834", "name": "heat_stack_owner"}, {"id": "966611393afe470190dbedc09dbbd7f0", "name": "reader"}, {"id": "dc02f9348a064de5b876f00c0d684f71", "name": "admin"}, {"id": "543439cd23e944598d10f1724bff82f9", "name": "member"}], "expires_at": "2018-07-13T05:47:44.000000Z", "project": {"domain": {"id": "default", "name": "Default"}, "id": "b0ac23d633c44766bc016253fcbc880e", "name": "admin"}, "catalog": [{"endpoints": [{"region_id": "regionOne", "url": "http://192.168.24.7:8888", "region": "regionOne", "interface": "internal", "id": "3f4a49a4436e41cdaba7b14281f5c616"}, {"region_id": "regionOne", "url": "http://192.168.24.7:8888", "region": "regionOne", "interface": "admin", "id": "cba243e325ae491b89a371bfbd76d72c"}, {"region_id": "regionOne", "url": "http://192.168.24.7:8888", "region": "regionOne", "interface": "public", "id": "d10f95e1817046cfac3a1c2bf236113d"}], "type": "messaging", "id": "09dea00130924ad0933c044c9c68c218", "name": "zaqar"}, {"endpoints": [{"region_id": "regionOne", "url": "http://192.168.24.7:8776/v2/b0ac23d633c44766bc016253fcbc880e", "region": "regionOne", "interface": "admin", "id": "8f00e77772d24312877e9dd4792e5332"}, {"region_id": "regionOne", "url": "http://192.168.24.7:8776/v2/b0ac23d633c44766bc016253fcbc880e", "region": "regionOne", "interface": "internal", "id": "93f151d0d76046c689bd2fbb1bb2b068"}, {"region_id": "regionOne", "url": "http://192.168.24.7:8776/v2/b0ac23d633c44766bc016253fcbc880e", "region": "regionOne", "interface": "public", "id": "e7efd5f0255947eb9dcb350ca7479263"}], "type": "volumev2", "id": "1f089941887b44b9b98b46f3c2c1c90c", "name": "cinderv2"}, {"endpoints": [{"region_id": "regionOne", "url": "http://192.168.24.7:8004/v1/b0ac23d633c44766bc016253fcbc880e", "region": "regionOne", "interface": "public", "id": "0ace07aa4ee249ebaf9c0da1e226ab49"}, {"region_id": "regionOne", "url": "http://192.168.24.7:8004/v1/b0ac23d633c44766bc016253fcbc880e", "region": "regionOne", "interface": "admin", "id": "5b83753b71004b2fa18a82e475432841"}, {"region_id": "regionOne", "url": "http://192.168.24.7:8004/v1/b0ac23d633c44766bc016253fcbc880e", "region": "regionOne", "interface": "internal", "id": "c61b8560063848e0861eaecaf55059c1"}], "type": "orchestration", "id": "242fcbfc40494e8f9f785318992529e2", "name": "heat"}, {"endpoints": [{"region_id": "regionOne", "url": "http://192.168.24.7:8977", "region": "regionOne", "interface": "public", "id": "73262657f6104f6f958877e3e1b91586"}, {"region_id": "regionOne", "url": "http://192.168.24.7:8977", "region": "regionOne", "interface": "internal", "id": "73fb5b0e781d437bb548b59f44976728"}, {"region_id": "regionOne", "url": "http://192.168.24.7:8977", "region": "regionOne", "interface": "admin", "id": "a4333039e0f2490a9defab995869419d"}], "type": "event", "id": "28e925b12507436c82f33e18fd1c79d4", "name": "panko"}, {"endpoints": [{"region_id": "regionOne", "url": "http://192.168.24.7:35357", "region": "regionOne", "interface": "admin", "id": "7d7822004b0a471793a6e3d9d8ca09e8"}, {"region_id": "regionOne", "url": "http://192.168.24.7:5000", "region": "regionOne", "interface": "internal", "id": "e0bd9e7399474e16893f5b7e09e85259"}, {"region_id": "regionOne", "url": "http://192.168.24.7:5000", "region": "regionOne", "interface": "public", "id": "e9fd8f6e39f04daab2caadceb1bfd472"}], "type": "identity", "id": "2c7012f6faed40dda791aa70a289cc81", "name": "keystone"}, {"endpoints": [{"region_id": "regionOne", "url": "http://192.168.24.7:9311", "region": "regionOne", "interface": "internal", "id": "4dc8424bc5b74ab488a01225c0bbda79"}, {"region_id": "regionOne", "url": "http://192.168.24.7:9311", "region": "regionOne", "interface": "public", "id": "72522106feee44adaba1defae7710631"}, {"region_id": "regionOne", "url": "http://192.168.24.7:9311", "region": "regionOne", "interface": "admin", "id": "d616ce13dad34aff81c040b11dca5ab2"}], "type": "key-manager", "id": "383f7c488e95443981dd2ac0ec43706c", "name": "barbican"}, {"endpoints": [{"region_id": "regionOne", "url": "ws://192.168.2 _log_request_full /usr/lib/python2.7/site-packages/tempest/lib/common/rest_client.py:434 2018-07-13 04:47:45.210 81 INFO tempest.lib.common.rest_client [req-f292d6d1-0f17-42d7-8d64-5f5b5f21e3c5 ] Request (ObjectTest:tearDownClass): 200 GET http://192.168.24.7:9696/v2.0/security-groups?tenant_id=4122e175cae840faabf5ad7e8879f0e5&name=default 0.630s 2018-07-13 04:47:45.211 81 DEBUG tempest.lib.common.rest_client [req-f292d6d1-0f17-42d7-8d64-5f5b5f21e3c5 ] Request - Headers: {'Content-Type': 'application/json', 'Accept': 'application/json', 'X-Auth-Token': ''} Body: None Response - Headers: {'status': '200', u'content-length': '2434', 'content-location': 'http://192.168.24.7:9696/v2.0/security-groups?tenant_id=4122e175cae840faabf5ad7e8879f0e5&name=default', u'date': 'Fri, 13 Jul 2018 04:47:45 GMT', u'content-type': 'application/json', u'connection': 'close', u'x-openstack-request-id': 'req-f292d6d1-0f17-42d7-8d64-5f5b5f21e3c5'} Body: {"security_groups": [{"description": "Default security group", "tags": [], "tenant_id": "4122e175cae840faabf5ad7e8879f0e5", "created_at": "2018-07-13T04:47:45Z", "updated_at": "2018-07-13T04:47:45Z", "security_group_rules": [{"direction": "egress", "protocol": null, "description": null, "tags": [], "port_range_max": null, "updated_at": "2018-07-13T04:47:45Z", "revision_number": 0, "id": "63559b81-f943-4102-a730-bc0ff71e5f38", "remote_group_id": null, "remote_ip_prefix": null, "created_at": "2018-07-13T04:47:45Z", "security_group_id": "d62237da-a680-4431-82e9-a8331c07fe44", "tenant_id": "4122e175cae840faabf5ad7e8879f0e5", "port_range_min": null, "ethertype": "IPv4", "project_id": "4122e175cae840faabf5ad7e8879f0e5"}, {"direction": "ingress", "protocol": null, "description": null, "tags": [], "port_range_max": null, "updated_at": "2018-07-13T04:47:45Z", "revision_number": 0, "id": "705ca520-8cc4-4fee-ba4b-049861782e6e", "remote_group_id": "d62237da-a680-4431-82e9-a8331c07fe44", "remote_ip_prefix": null, "created_at": "2018-07-13T04:47:45Z", "security_group_id": "d62237da-a680-4431-82e9-a8331c07fe44", "tenant_id": "4122e175cae840faabf5ad7e8879f0e5", "port_range_min": null, "ethertype": "IPv6", "project_id": "4122e175cae840faabf5ad7e8879f0e5"}, {"direction": "ingress", "protocol": null, "description": null, "tags": [], "port_range_max": null, "updated_at": "2018-07-13T04:47:45Z", "revision_number": 0, "id": "ad48e3f7-6063-497d-ae6c-a135c4cfb932", "remote_group_id": "d62237da-a680-4431-82e9-a8331c07fe44", "remote_ip_prefix": null, "created_at": "2018-07-13T04:47:45Z", "security_group_id": "d62237da-a680-4431-82e9-a8331c07fe44", "tenant_id": "4122e175cae840faabf5ad7e8879f0e5", "port_range_min": null, "ethertype": "IPv4", "project_id": "4122e175cae840faabf5ad7e8879f0e5"}, {"direction": "egress", "protocol": null, "description": null, "tags": [], "port_range_max": null, "updated_at": "2018-07-13T04:47:45Z", "revision_number": 0, "id": "d4c3b094-f765-4c9f-b9dc-ce258eb23b5b", "remote_group_id": null, "remote_ip_prefix": null, "created_at": "2018-07-13T04:47:45Z", "security_group_id": "d62237da-a680-4431-82e9-a8331c07fe44", "tenant_id": "4122e175cae840faabf5ad7e8879f0e5", "port_range_min": null, "ethertype": "IPv6", "project_id": "4122e175cae840faabf5ad7e8879f0e5"}], "revision_number": 1, "project_id": "4122e175cae840faabf5ad7e8879f0e5", "id": "d62237da-a680-4431-82e9-a8331c07fe44", "name": "default"}]} _log_request_full /usr/lib/python2.7/site-packages/tempest/lib/common/rest_client.py:434 2018-07-13 04:47:45.439 81 INFO tempest.lib.common.rest_client [req-ccdb3aae-b45b-4d5b-8658-7e77b2b5fc76 ] Request (ObjectTest:tearDownClass): 204 DELETE http://192.168.24.7:9696/v2.0/security-groups/d62237da-a680-4431-82e9-a8331c07fe44 0.227s 2018-07-13 04:47:45.439 81 DEBUG tempest.lib.common.rest_client [req-ccdb3aae-b45b-4d5b-8658-7e77b2b5fc76 ] Request - Headers: {'Content-Type': 'application/json', 'Accept': 'application/json', 'X-Auth-Token': ''} Body: None Response - Headers: {'status': '204', u'content-length': '0', 'content-location': 'http://192.168.24.7:9696/v2.0/security-groups/d62237da-a680-4431-82e9-a8331c07fe44', u'date': 'Fri, 13 Jul 2018 04:47:45 GMT', u'connection': 'close', u'x-openstack-request-id': 'req-ccdb3aae-b45b-4d5b-8658-7e77b2b5fc76'} Body: _log_request_full /usr/lib/python2.7/site-packages/tempest/lib/common/rest_client.py:434 2018-07-13 04:47:45.753 81 INFO tempest.lib.common.rest_client [req-cd52d241-2898-49dd-b0e6-7da786118429 ] Request (ObjectTest:tearDownClass): 204 DELETE http://192.168.24.7:35357/v3/projects/4122e175cae840faabf5ad7e8879f0e5 0.313s 2018-07-13 04:47:45.754 81 DEBUG tempest.lib.common.rest_client [req-cd52d241-2898-49dd-b0e6-7da786118429 ] Request - Headers: {'Content-Type': 'application/json', 'Accept': 'application/json', 'X-Auth-Token': ''} Body: None Response - Headers: {'status': '204', 'content-location': 'http://192.168.24.7:35357/v3/projects/4122e175cae840faabf5ad7e8879f0e5', u'vary': 'X-Auth-Token', u'server': 'Apache', u'connection': 'close', u'date': 'Fri, 13 Jul 2018 04:47:45 GMT', u'content-type': 'text/plain', u'x-openstack-request-id': 'req-cd52d241-2898-49dd-b0e6-7da786118429'} Body: _log_request_full /usr/lib/python2.7/site-packages/tempest/lib/common/rest_client.py:434 From Journalctl logs: http://logs.openstack.org/18/581918/5/check/tripleo-ci-centos-7-scenario002-multinode-oooq-container/3cf2734/logs/subnode-2/var/log/journal.txt.gz#_Jul_13_04_04_13 Jul 13 04:04:14 centos-7-rax-ord-0000699054 ansible-file[13666]: Invoked with directory_mode=None force=False remote_src=None path=/var/log/swift owner=None follow=True group=None unsafe_writes=None state=directory content=NOT_LOGGING_PARAMETER serole=None diff_peek=None setype=None selevel=None original_basename=None regexp=None validate=None src=None seuser=None recurse=False delimiter=None mode=None attributes=None backup=None Jul 13 04:04:14 centos-7-rax-ord-0000699054 ansible-file[13685]: Invoked with directory_mode=None force=False remote_src=None path=/var/log/containers/swift owner=None follow=True group=None unsafe_writes=None state=link content=NOT_LOGGING_PARAMETER serole=None diff_peek=None setype=None dest=/var/log/containers/swift selevel=None original_basename=None regexp=None validate=None src=/var/log/swift seuser=None recurse=False delimiter=None mode=None attributes=None backup=None Jul 13 04:04:14 centos-7-rax-ord-0000699054 ansible-file[13705]: Invoked with directory_mode=None force=False remote_src=None path=/srv/node owner=None follow=True group=None unsafe_writes=None state=directory content=NOT_LOGGING_PARAMETER serole=None diff_peek=None setype=None selevel=None original_basename=None regexp=None validate=None src=None seuser=None recurse=False delimiter=None mode=None attributes=None backup=None Jul 13 04:04:14 centos-7-rax-ord-0000699054 ansible-file[13724]: Invoked with directory_mode=None force=False remote_src=None path=/var/log/swift owner=None follow=True group=None unsafe_writes=None state=directory content=NOT_LOGGING_PARAMETER serole=None diff_peek=None setype=None selevel=None original_basename=None regexp=None validate=None src=None seuser=None recurse=False delimiter=None mode=None attributes=None backup=None Jul 13 04:04:15 centos-7-rax-ord-0000699054 ansible-file[13742]: Invoked with directory_mode=None force=False remote_src=None path=/var/log/containers owner=None follow=True group=None unsafe_writes=None state=directory content=NOT_LOGGING_PARAMETER serole=None diff_peek=None setype=None selevel=None original_basename=None regexp=None validate=None src=None seuser=None recurse=False delimiter=None mode=None attributes=None backup=None Jul 13 04:04:15 centos-7-rax-ord-0000699054 ansible-file[13763]: Invoked with directory_mode=None force=False remote_src=None path=/srv/node/d1 owner=None follow=True group=None unsafe_writes=None state=directory content=NOT_LOGGING_PARAMETER serole=None diff_peek=None setype=None selevel=None original_basename=None regexp=None validate=None src=None seuser=None recurse=False delimiter=None mode=None attributes=None backup=None Jul 13 04:04:15 centos-7-rax-ord-0000699054 ansible-stat[13783]: Invoked with checksum_algorithm=sha1 get_checksum=True follow=False checksum_algo=sha1 path=/var/log/swift/readme.txt get_md5=None get_mime=True get_attributes=True Jul 13 04:04:15 centos-7-rax-ord-0000699054 ansible-copy[13793]: Invoked with directory_mode=None force=True remote_src=None owner=None follow=False local_follow=None group=None unsafe_writes=None setype=None content=NOT_LOGGING_PARAMETER serole=None dest=/var/log/swift/readme.txt selevel=None original_basename=tmpon9xoC regexp=None validate=None src=/root/.ansible/tmp/ansible-tmp-1531454655.64-60825008480712/source checksum=42510a6de124722d6efbc2b1bb038bfe97e5b6d3 seuser=None delimiter=None mode=None attributes=None backup=False