swift stat error

[sam]

run "swift --debug stat " error

all user jurisdiction ... very normal

root@controller:~# swift --debug stat
DEBUG:keystoneclient.auth.identity.v3.base:Making authentication request to http://controller:5000/v3/auth/tokens
DEBUG:urllib3.connectionpool:Starting new HTTP connection (1): controller:5000
DEBUG:urllib3.connectionpool:http://controller:5000 "POST /v3/auth/tokens HTTP/1.1" 201 5316
DEBUG:keystoneclient.auth.identity.v3.base:{"token": {"methods": ["password"], "user": {"domain": {"id": "default", "name": "Default"}, "id": "ebe0c2002931431c9f6057f216c9aad1", "name": "swift", "password_expires_at": null}, "audit_ids": ["uxfmM8EtSDuf4TjLXur8LQ"], "expires_at": "2022-11-04T10:03:08.000000Z", "issued_at": "2022-11-04T09:03:08.000000Z", "project": {"domain": {"id": "default", "name": "Default"}, "id": "6c50811c608f44ebb32132f039fd3ac0", "name": "service"}, "is_domain": false, "roles": [{"id": "8eb2665e258844bdb6daa7826ff7fd96", "name": "reader"}, {"id": "527a74431f6349738de18528c6d445e1", "name": "admin"}, {"id": "101015b8b34e41b4962b31440cf61520", "name": "member"}], "catalog": [{"endpoints": [{"id": "6e624d24e7e046e8bda14929bbb345c5", "interface": "internal", "region_id": "RegionOne", "url": "http://controller:5000/v3/", "region": "RegionOne"}, {"id": "bcdb2ad98de94420926cae68b27e4ccc", "interface": "public", "region_id": "RegionOne", "url": "http://controller:5000/v3/", "region": "RegionOne"}, {"id": "cc52757d98b04bdd88026d1087ae9a5d", "interface": "admin", "region_id": "RegionOne", "url": "http://controller:5000/v3/", "region": "RegionOne"}], "id": "03943f266612412ebd2ea111596cfb75", "type": "identity", "name": "keystone"}, {"endpoints": [{"id": "68950739e40340768da67915b2cf6aac", "interface": "admin", "region_id": "RegionOne", "url": "http://controller:9696", "region": "RegionOne"}, {"id": "8a68f4ac2d4845959e9515304dd75eb3", "interface": "internal", "region_id": "RegionOne", "url": "http://controller:9696", "region": "RegionOne"}, {"id": "da32e6e91bff4279bafa3329053e3704", "interface": "public", "region_id": "RegionOne", "url": "http://controller:9696", "region": "RegionOne"}], "id": "33f2243a764d4bdc9a157e6a9d5a9c6a", "type": "network", "name": ""}, {"endpoints": [{"id": "166e07fa638e49b987595323bd3bcf61", "interface": "public", "region_id": "RegionOne", "url": "http://controller:9292", "region": "RegionOne"}, {"id": "cbb01761367e4acf83c658333e2de386", "interface": "admin", "region_id": "RegionOne", "url": "http://controller:9292", "region": "RegionOne"}, {"id": "db8dd3cb0dd04b29bfb96a608fb72d99", "interface": "internal", "region_id": "RegionOne", "url": "http://controller:9292", "region": "RegionOne"}], "id": "4f86e32926a64fb99de6e6919c714926", "type": "image", "name": ""}, {"endpoints": [{"id": "93e135f5a0824caa8b8fc9fa0df74bd4", "interface": "internal", "region_id": "RegionOne", "url": "http://controller:8776/v2/6c50811c608f44ebb32132f039fd3ac0", "region": "RegionOne"}, {"id": "f21e50d72b914333952b7d3d8fd2578d", "interface": "public", "region_id": "RegionOne", "url": "http://controller:8776/v2/6c50811c608f44ebb32132f039fd3ac0", "region": "RegionOne"}, {"id": "f669072bd0a84c5b85ca70714e6b212d", "interface": "admin", "region_id": "RegionOne", "url": "http://controller:8776/v2/6c50811c608f44ebb32132f039fd3ac0", "region": "RegionOne"}], "id": "554089befdd94ee1a87a03de006a33c6", "type": "volumev2", "name": ""}, {"endpoints": [{"id": "7f0d59134e60478ab7fbb0b72772cbab", "interface": "admin", "region_id": "RegionOne", "url": "http://controller:8774/v2.1", "region": "RegionOne"}, {"id": "ac50c3fb0af947508497d8344b8e6d0a", "interface": "internal", "region_id": "RegionOne", "url": "http://controller:8774/v2.1", "region": "RegionOne"}, {"id": "b41204a96b7a43f68a7f8f2613163df3", "interface": "public", "region_id": "RegionOne", "url": "http://controller:8774/v2.1", "region": "RegionOne"}], "id": "5e09e36a7744495389ea6159048eef5e", "type": "compute", "name": ""}, {"endpoints": [{"id": "1e56c094a9ea4e7a80132280df1925f4", "interface": "admin", "region_id": "RegionOne", "url": "http://controller:8778", "region": "RegionOne"}, {"id": "896c40aa1ada41ed82bcb0c39d87d8f5", "interface": "public", "region_id": "RegionOne", "url": "http://controller:8778", "region": "RegionOne"}, {"id": "b4d50334c3174633aa1372f13b7c19ed", "interface": "internal", "region_id": "RegionOne", "url": "http://controller:8778", "region": "RegionOne"}], "id": "90e53b94f1b84b99ae6fa03c793a30b1", "type": "placement", "name": ""}, {"endpoints": [{"id": "50653fa05fde458c9e04be6410d3407f", "interface": "internal", "region_id": "RegionOne", "url": "http://controller:8776/v3/6c50811c608f44ebb32132f039fd3ac0", "region": "RegionOne"}, {"id": "794a0750211b4034bc40d4095fab09df", "interface": "admin", "region_id": "RegionOne", "url": "http://controller:8776/v3/6c50811c608f44ebb32132f039fd3ac0", "region": "RegionOne"}, {"id": "951627fca8974f27a21fcb0a075cecc1", "interface": "public", "region_id": "RegionOne", "url": "http://controller:8776/v3/6c50811c608f44ebb32132f039fd3ac0", "region": "RegionOne"}], "id": "d8cca1398508471f937533ab72ffae96", "type": "volumev3", "name": ""}, {"endpoints": [{"id": "29c38e196a6743aa9ff92c7a26362bb2", "interface": "public", "region_id": "RegionOne", "url": "http://controller:8080/v1/AUTH_6c50811c608f44ebb32132f039fd3ac0", "region": "RegionOne"}, {"id": "adce2914f26f4d1ead7036255a690d42", "interface": "internal", "region_id": "RegionOne", "url": "http://controller:8080/v1/AUTH_6c50811c608f44ebb32132f039fd3ac0", "region": "RegionOne"}, {"id": "bc4246d33f3146be87c49fd47af98172", "interface": "admin", "region_id": "RegionOne", "url": "http://controller:8080/v1", "region": "RegionOne"}], "id": "f256c47fcfaa48f78baaf34066c54960", "type": "object-store", "name": "swift"}]}}
DEBUG:urllib3.connectionpool:Starting new HTTP connection (1): controller:8080
DEBUG:urllib3.connectionpool:http://controller:8080 "HEAD /v1/AUTH_6c50811c608f44ebb32132f039fd3ac0 HTTP/1.1" 401 0
INFO:swiftclient:REQ: curl -i http://controller:8080/v1/AUTH_6c50811c608f44ebb32132f039fd3ac0 -I -H "X-Auth-Token: gAAAAABjZNVMHpSugcariFAL92JRrJXtfLTk8sAtac7CZY9BEUoOxg3hnI2xKyv-DBkYuFQNVEAQiL68-5in7gfh3SrWTn_P5s3QB_GWOVGMOobkSYXiiKtqzL46nUySqLKpJiG0IMd06ufiC3ILCZguXu5yfoAA9U9Z--XhBmyhXmcWLaQ1Ybo"
INFO:swiftclient:RESP STATUS: 401 Unauthorized
INFO:swiftclient:RESP HEADERS: {'Content-Type': 'text/html; charset=UTF-8', 'Www-Authenticate': 'Swift realm="AUTH_6c50811c608f44ebb32132f039fd3ac0"', 'Content-Length': '0', 'X-Trans-Id': 'txe65f29e7acb04920bb9f6-006364d54c', 'X-Openstack-Request-Id': 'txe65f29e7acb04920bb9f6-006364d54c', 'Date': 'Fri, 04 Nov 2022 09:03:08 GMT'}
DEBUG:keystoneclient.auth.identity.v3.base:Making authentication request to http://controller:5000/v3/auth/tokens
DEBUG:urllib3.connectionpool:Starting new HTTP connection (1): controller:5000
DEBUG:urllib3.connectionpool:http://controller:5000 "POST /v3/auth/tokens HTTP/1.1" 201 5316
DEBUG:keystoneclient.auth.identity.v3.base:{"token": {"methods": ["password"], "user": {"domain": {"id": "default", "name": "Default"}, "id": "ebe0c2002931431c9f6057f216c9aad1", "name": "swift", "password_expires_at": null}, "audit_ids": ["_gjaOZg2TquDXq7PTISWNA"], "expires_at": "2022-11-04T10:03:09.000000Z", "issued_at": "2022-11-04T09:03:09.000000Z", "project": {"domain": {"id": "default", "name": "Default"}, "id": "6c50811c608f44ebb32132f039fd3ac0", "name": "service"}, "is_domain": false, "roles": [{"id": "527a74431f6349738de18528c6d445e1", "name": "admin"}, {"id": "8eb2665e258844bdb6daa7826ff7fd96", "name": "reader"}, {"id": "101015b8b34e41b4962b31440cf61520", "name": "member"}], "catalog": [{"endpoints": [{"id": "6e624d24e7e046e8bda14929bbb345c5", "interface": "internal", "region_id": "RegionOne", "url": "http://controller:5000/v3/", "region": "RegionOne"}, {"id": "bcdb2ad98de94420926cae68b27e4ccc", "interface": "public", "region_id": "RegionOne", "url": "http://controller:5000/v3/", "region": "RegionOne"}, {"id": "cc52757d98b04bdd88026d1087ae9a5d", "interface": "admin", "region_id": "RegionOne", "url": "http://controller:5000/v3/", "region": "RegionOne"}], "id": "03943f266612412ebd2ea111596cfb75", "type": "identity", "name": "keystone"}, {"endpoints": [{"id": "68950739e40340768da67915b2cf6aac", "interface": "admin", "region_id": "RegionOne", "url": "http://controller:9696", "region": "RegionOne"}, {"id": "8a68f4ac2d4845959e9515304dd75eb3", "interface": "internal", "region_id": "RegionOne", "url": "http://controller:9696", "region": "RegionOne"}, {"id": "da32e6e91bff4279bafa3329053e3704", "interface": "public", "region_id": "RegionOne", "url": "http://controller:9696", "region": "RegionOne"}], "id": "33f2243a764d4bdc9a157e6a9d5a9c6a", "type": "network", "name": ""}, {"endpoints": [{"id": "166e07fa638e49b987595323bd3bcf61", "interface": "public", "region_id": "RegionOne", "url": "http://controller:9292", "region": "RegionOne"}, {"id": "cbb01761367e4acf83c658333e2de386", "interface": "admin", "region_id": "RegionOne", "url": "http://controller:9292", "region": "RegionOne"}, {"id": "db8dd3cb0dd04b29bfb96a608fb72d99", "interface": "internal", "region_id": "RegionOne", "url": "http://controller:9292", "region": "RegionOne"}], "id": "4f86e32926a64fb99de6e6919c714926", "type": "image", "name": ""}, {"endpoints": [{"id": "93e135f5a0824caa8b8fc9fa0df74bd4", "interface": "internal", "region_id": "RegionOne", "url": "http://controller:8776/v2/6c50811c608f44ebb32132f039fd3ac0", "region": "RegionOne"}, {"id": "f21e50d72b914333952b7d3d8fd2578d", "interface": "public", "region_id": "RegionOne", "url": "http://controller:8776/v2/6c50811c608f44ebb32132f039fd3ac0", "region": "RegionOne"}, {"id": "f669072bd0a84c5b85ca70714e6b212d", "interface": "admin", "region_id": "RegionOne", "url": "http://controller:8776/v2/6c50811c608f44ebb32132f039fd3ac0", "region": "RegionOne"}], "id": "554089befdd94ee1a87a03de006a33c6", "type": "volumev2", "name": ""}, {"endpoints": [{"id": "7f0d59134e60478ab7fbb0b72772cbab", "interface": "admin", "region_id": "RegionOne", "url": "http://controller:8774/v2.1", "region": "RegionOne"}, {"id": "ac50c3fb0af947508497d8344b8e6d0a", "interface": "internal", "region_id": "RegionOne", "url": "http://controller:8774/v2.1", "region": "RegionOne"}, {"id": "b41204a96b7a43f68a7f8f2613163df3", "interface": "public", "region_id": "RegionOne", "url": "http://controller:8774/v2.1", "region": "RegionOne"}], "id": "5e09e36a7744495389ea6159048eef5e", "type": "compute", "name": ""}, {"endpoints": [{"id": "1e56c094a9ea4e7a80132280df1925f4", "interface": "admin", "region_id": "RegionOne", "url": "http://controller:8778", "region": "RegionOne"}, {"id": "896c40aa1ada41ed82bcb0c39d87d8f5", "interface": "public", "region_id": "RegionOne", "url": "http://controller:8778", "region": "RegionOne"}, {"id": "b4d50334c3174633aa1372f13b7c19ed", "interface": "internal", "region_id": "RegionOne", "url": "http://controller:8778", "region": "RegionOne"}], "id": "90e53b94f1b84b99ae6fa03c793a30b1", "type": "placement", "name": ""}, {"endpoints": [{"id": "50653fa05fde458c9e04be6410d3407f", "interface": "internal", "region_id": "RegionOne", "url": "http://controller:8776/v3/6c50811c608f44ebb32132f039fd3ac0", "region": "RegionOne"}, {"id": "794a0750211b4034bc40d4095fab09df", "interface": "admin", "region_id": "RegionOne", "url": "http://controller:8776/v3/6c50811c608f44ebb32132f039fd3ac0", "region": "RegionOne"}, {"id": "951627fca8974f27a21fcb0a075cecc1", "interface": "public", "region_id": "RegionOne", "url": "http://controller:8776/v3/6c50811c608f44ebb32132f039fd3ac0", "region": "RegionOne"}], "id": "d8cca1398508471f937533ab72ffae96", "type": "volumev3", "name": ""}, {"endpoints": [{"id": "29c38e196a6743aa9ff92c7a26362bb2", "interface": "public", "region_id": "RegionOne", "url": "http://controller:8080/v1/AUTH_6c50811c608f44ebb32132f039fd3ac0", "region": "RegionOne"}, {"id": "adce2914f26f4d1ead7036255a690d42", "interface": "internal", "region_id": "RegionOne", "url": "http://controller:8080/v1/AUTH_6c50811c608f44ebb32132f039fd3ac0", "region": "RegionOne"}, {"id": "bc4246d33f3146be87c49fd47af98172", "interface": "admin", "region_id": "RegionOne", "url": "http://controller:8080/v1", "region": "RegionOne"}], "id": "f256c47fcfaa48f78baaf34066c54960", "type": "object-store", "name": "swift"}]}}
DEBUG:urllib3.connectionpool:Starting new HTTP connection (1): controller:8080
DEBUG:urllib3.connectionpool:http://controller:8080 "HEAD /v1/AUTH_6c50811c608f44ebb32132f039fd3ac0 HTTP/1.1" 401 0
INFO:swiftclient:REQ: curl -i http://controller:8080/v1/AUTH_6c50811c608f44ebb32132f039fd3ac0 -I -H "X-Auth-Token: gAAAAABjZNVNWPHppqLAGxDFedVVT4FiMam_wPsK0HL2siFsQhBz2-LewBYcX5FCspKLclWzrGRj0ChEE2Rd-6auT_fpiI5uHFt11zOpDZRN83zxvGTp_FgtN5p7h5dG6I1yvPVi6ICurC8MhfK7SpfKupdK5TZVt7r2hZkdjd7TdHj3-m-Bgdc"
INFO:swiftclient:RESP STATUS: 401 Unauthorized
INFO:swiftclient:RESP HEADERS: {'Content-Type': 'text/html; charset=UTF-8', 'Www-Authenticate': 'Swift realm="AUTH_6c50811c608f44ebb32132f039fd3ac0"', 'Content-Length': '0', 'X-Trans-Id': 'txff99e52141174a038c7d1-006364d54d', 'X-Openstack-Request-Id': 'txff99e52141174a038c7d1-006364d54d', 'Date': 'Fri, 04 Nov 2022 09:03:09 GMT'}
ERROR:swiftclient.service:Account HEAD failed: http://controller:8080/v1/AUTH_6c50811c608f44ebb32132f039fd3ac0 401 Unauthorized (txn: txff99e52141174a038c7d1-006364d54d)
Traceback (most recent call last):
  File "/usr/lib/python3/dist-packages/swiftclient/service.py", line 555, in stat
    items, headers = get_future_result(stats_future)
  File "/usr/lib/python3/dist-packages/swiftclient/service.py", line 251, in get_future_result
    res = f.result(timeout=timeout)
  File "/usr/lib/python3.8/concurrent/futures/_base.py", line 444, in result
    return self.__get_result()
  File "/usr/lib/python3.8/concurrent/futures/_base.py", line 389, in __get_result
    raise self._exception
  File "/usr/lib/python3.8/concurrent/futures/thread.py", line 57, in run
    result = self.fn(*self.args, **self.kwargs)
  File "/usr/lib/python3/dist-packages/swiftclient/multithreading.py", line 201, in conn_fn
    return fn(*conn_args, **kwargs)
  File "/usr/lib/python3/dist-packages/swiftclient/command_helpers.py", line 24, in stat_account
    headers = conn.head_account(headers=req_headers)
  File "/usr/lib/python3/dist-packages/swiftclient/client.py", line 1902, in head_account
    return self._retry(None, head_account, headers=headers)
  File "/usr/lib/python3/dist-packages/swiftclient/client.py", line 1856, in _retry
    rv = func(self.url, self.token, *args,
  File "/usr/lib/python3/dist-packages/swiftclient/client.py", line 924, in head_account
    raise ClientException.from_response(resp, 'Account HEAD failed', body)
swiftclient.exceptions.ClientException: Account HEAD failed: http://controller:8080/v1/AUTH_6c50811c608f44ebb32132f039fd3ac0 401 Unauthorized (txn: txff99e52141174a038c7d1-006364d54d)
Account HEAD failed: http://controller:8080/v1/AUTH_6c50811c608f44ebb32132f039fd3ac0 401 Unauthorized
Failed Transaction ID: txff99e52141174a038c7d1-006364d54d

[sam (sam22926)]

version is ubuntu 20.04 openstack W

[clayg (clay-gerrard)]

the first request to http://controller:5000 "POST /v3/auth/tokens HTTP/1.1" succeeds 201

the second request

INFO:swiftclient:REQ: curl -i http://controller:8080/v1/AUTH_6c50811c608f44ebb32132f039fd3ac0 -I -H "X-Auth-Token: gAAAAABjZNVNWPHppqLAGxDFedVVT4FiMam_wPsK0HL2siFsQhBz2-LewBYcX5FCspKLclWzrGRj0ChEE2Rd-6auT_fpiI5uHFt11zOpDZRN83zxvGTp_FgtN5p7h5dG6I1yvPVi6ICurC8MhfK7SpfKupdK5TZVt7r2hZkdjd7TdHj3-m-Bgdc"

... has the auth token, but it's rejected 401 Unauthorized (txn: txff99e52141174a038c7d1-006364d54d)

If you look through your swift-proxy server logs (maybe for the failed txn id txff99e52141174a038c7d1-006364d54d) you may get more info about why it failed to validate the token with keystone.

This is a pretty typicial configuraiton/workflow; it's probably not a bug. Is this your initial setup, or did this work before until something changed?

[sam (sam22926)]

Sorry, I don't know the log location of swift proxy. I can't see the log about swift in/var/log/. I provide its directory information below

root@controller:/etc/swift# cd /var/log/
apache2/ journal/ placement/
apt/ keystone/ private/
cinder/ landscape/ rabbitmq/
dist-upgrade/ mysql/ unattended-upgrades/
glance/ neutron/ zvmsdk/
installer/ nova/
root@controller:/etc/swift# cd /var/log/

Please tell me the area of swift logs, thank you

Would you like me to provide/etc/swift/proxy server. conf ??

[sam (sam22926)]

Sorry, I don't know the log location of swift proxy. I can't see the log about swift in/var/log/. I provide its directory information below

root@controller:/etc/swift# cd /var/log/
apache2/ journal/ placement/
apt/ keystone/ private/
cinder/ landscape/ rabbitmq/
dist-upgrade/ mysql/ unattended-upgrades/
glance/ neutron/ zvmsdk/
installer/ nova/
root@controller:/etc/swift# cd /var/log/

Please tell me the area of swift logs, thank you

Would you like me to provide/etc/swift/proxy server. conf ??

[sam (sam22926)]

This is my initial setup .

[sam (sam22926)]

i run this two cmd
form tail -f /var/log/* look that:
swift --debug stat
tail -f /var/log/*

Nov 4 13:49:42 controller proxy-server: - - 04/Nov/2022/13/49/42 HEAD /v1/AUTH_6c50811c608f44ebb32132f039fd3ac0%3Fformat%3Djson HTTP/1.0 200 - Swift - - - - tx967ed9b9c29b4828b646d-0063651876 - 0.0039 RL - 1667569782.696980715 1667569782.700921297 -
Nov 4 13:49:42 controller proxy-server: 192.168.100.10 192.168.100.10 04/Nov/2022/13/49/42 HEAD /v1/AUTH_6c50811c608f44ebb32132f039fd3ac0%3Fformat%3Djson HTTP/1.0 401 - python-swiftclient-3.11.0 gAAAAABjZRh2... - - - tx967ed9b9c29b4828b646d-0063651876 - 0.0055 - - 1667569782.696368933 1667569782.701885939 -
Nov 4 13:49:44 controller proxy-server: 192.168.100.10 192.168.100.10 04/Nov/2022/13/49/44 HEAD /v1/AUTH_6c50811c608f44ebb32132f039fd3ac0%3Fformat%3Djson HTTP/1.0 401 - python-swiftclient-3.11.0 gAAAAABjZRh4... - - - tx4ac048f132544deaaac45-0063651878 - 0.0009 - - 1667569784.155652046 1667569784.156536341 -

[clayg (clay-gerrard)]

Depending on the complexity of the tooling you used to configure your setup, troubleshooting an initial setup can take a lot of back and forth hypothesis testing. Launchpad bug comments probably aren't the forum - you could send a message to the mailing list https://lists.openstack.org/cgi-bin/mailman/listinfo/openstack-discuss or come visit in IRC https://docs.openstack.org/swift/latest/contributor/contributing.html#communication

on my dev setup swift logs directly to /var/log/syslog - you just have to grep for "proxy-server" - if you're using systemd you might try journalctl: https://docs.openstack.org/devstack/latest/systemd.html#querying-logs

If you're interested in a reliable dev/testing environment focused on swift I'd recommend looking into:

https://github.com/nvidia/vagrant-swift-all-in-one
https://hub.docker.com/r/openstackswift/saio
https://hub.docker.com/r/dockerswiftaio/docker-swift/