I see glance and neutron REST calls failing in some cases and the log shows this. I have the services running on RHEL 7.2 . The problem disappears temporarily once the services are restarted.
/tmp directory looks like this:
drwx------. 2 nova nova 64 Sep 26 06:49 keystone-signing-IMigWf
drwx------. 2 cinder cinder 64 Sep 26 06:03 keystone-signing-JVBe80
drwx------. 2 ceilometer ceilometer 64 Sep 26 04:44 keystone-signing-Lb30NH
drwx------. 2 cinder cinder 64 Sep 26 06:21 keystone-signing-pOJym0
drwx------. 2 cinder cinder 64 Sep 26 06:43 keystone-signing-QH89cP
drwx------. 2 nova nova 64 Sep 26 06:49 keystone-signing-_qlc31
drwx------. 2 cinder cinder 64 Sep 26 06:46 keystone-signing-qvCp__
drwx------. 2 nova nova 64 Sep 26 06:49 keystone-signing-RS0Shu
drwx------. 2 swift swift 64 Sep 26 06:49 keystone-signing-swift
drwx------. 2 nova nova 64 Sep 26 06:49 keystone-signing-tioqkc
drwx------. 2 cinder cinder 64 Sep 26 06:35 keystone-signing-TJj0U2
drwx------. 2 cinder cinder 64 Sep 26 06:47 keystone-signing-U56FSw
drwx------. 2 cinder cinder 64 Sep 26 06:19 keystone-signing-_uy4Dy
drwx------. 2 cinder cinder 64 Sep 26 06:49 keystone-signing-YqsCSA
Stacktrace from glance
2016-09-20 05:38:47.310 35161 INFO eventlet.wsgi.server [req-b503be53-cbe9-4ae6-a5b3-6d8f264c4702 0688b01e6439ca32d698d20789d52169126fb41fb1a4ddafcebb97d854e836c9 088c8d5388c4437f93b40f195bfaed05 - default default] 127.0.0.1 - - [20/Sep/2016 05:38:47] "GET /v2/schemas/image HTTP/1.1" 200 4352 0.002869
2016-09-20 05:38:47.706 35160 INFO eventlet.wsgi.server [-] Traceback (most recent call last):
File "/usr/lib/python2.7/site-packages/eventlet/wsgi.py", line 470, in handle_one_response
result = self.application(self.environ, start_response)
File "/usr/lib/python2.7/site-packages/webob/dec.py", line 130, in __call__
resp = self.call_func(req, *args, **self.kwargs)
File "/usr/lib/python2.7/site-packages/webob/dec.py", line 195, in call_func
return self.func(req, *args, **kwargs)
File "/usr/lib/python2.7/site-packages/oslo_middleware/base.py", line 126, in __call__
response = req.get_response(self.application)
File "/usr/lib/python2.7/site-packages/webob/request.py", line 1296, in send
application, catch_exc_info=False)
File "/usr/lib/python2.7/site-packages/webob/request.py", line 1260, in call_application
app_iter = application(self.environ, start_response)
File "/usr/lib/python2.7/site-packages/webob/dec.py", line 130, in __call__
resp = self.call_func(req, *args, **self.kwargs)
File "/usr/lib/python2.7/site-packages/webob/dec.py", line 195, in call_func
return self.func(req, *args, **kwargs)
File "/usr/lib/python2.7/site-packages/oslo_middleware/base.py", line 126, in __call__
response = req.get_response(self.application)
File "/usr/lib/python2.7/site-packages/webob/request.py", line 1296, in send
application, catch_exc_info=False)
File "/usr/lib/python2.7/site-packages/webob/request.py", line 1260, in call_application
app_iter = application(self.environ, start_response)
File "/usr/lib/python2.7/site-packages/webob/dec.py", line 130, in __call__
resp = self.call_func(req, *args, **self.kwargs)
File "/usr/lib/python2.7/site-packages/webob/dec.py", line 195, in call_func
return self.func(req, *args, **kwargs)
File "/usr/lib/python2.7/site-packages/oslo_middleware/base.py", line 126, in __call__
response = req.get_response(self.application)
File "/usr/lib/python2.7/site-packages/webob/request.py", line 1296, in send
application, catch_exc_info=False)
File "/usr/lib/python2.7/site-packages/webob/request.py", line 1260, in call_application
app_iter = application(self.environ, start_response)
File "/usr/lib/python2.7/site-packages/webob/dec.py", line 130, in __call__
resp = self.call_func(req, *args, **self.kwargs)
File "/usr/lib/python2.7/site-packages/webob/dec.py", line 195, in call_func
return self.func(req, *args, **kwargs)
File "/usr/lib/python2.7/site-packages/glance/common/wsgi.py", line 804, in __call__
response = req.get_response(self.application)
File "/usr/lib/python2.7/site-packages/webob/request.py", line 1296, in send
application, catch_exc_info=False)
File "/usr/lib/python2.7/site-packages/webob/request.py", line 1260, in call_application
app_iter = application(self.environ, start_response)
File "/usr/lib/python2.7/site-packages/webob/dec.py", line 130, in __call__
resp = self.call_func(req, *args, **self.kwargs)
File "/usr/lib/python2.7/site-packages/webob/dec.py", line 195, in call_func
return self.func(req, *args, **kwargs)
File "/usr/lib/python2.7/site-packages/osprofiler/web.py", line 108, in __call__
return request.get_response(self.application)
File "/usr/lib/python2.7/site-packages/webob/request.py", line 1296, in send
application, catch_exc_info=False)
File "/usr/lib/python2.7/site-packages/webob/request.py", line 1260, in call_application
app_iter = application(self.environ, start_response)
File "/usr/lib/python2.7/site-packages/webob/dec.py", line 130, in __call__
resp = self.call_func(req, *args, **self.kwargs)
File "/usr/lib/python2.7/site-packages/webob/dec.py", line 195, in call_func
return self.func(req, *args, **kwargs)
File "/usr/lib/python2.7/site-packages/keystonemiddleware/auth_token/__init__.py", line 320, in __call__
response = self.process_request(req)
File "/usr/lib/python2.7/site-packages/keystonemiddleware/auth_token/__init__.py", line 552, in process_request
resp = super(AuthProtocol, self).process_request(request)
File "/usr/lib/python2.7/site-packages/keystonemiddleware/auth_token/__init__.py", line 348, in process_request
data, user_auth_ref = self._do_fetch_token(request.user_token)
File "/usr/lib/python2.7/site-packages/keystonemiddleware/auth_token/__init__.py", line 388, in _do_fetch_token
data = self.fetch_token(token)
File "/usr/lib/python2.7/site-packages/keystonemiddleware/auth_token/__init__.py", line 670, in fetch_token
self._revocations.check(token_hashes)
File "/usr/lib/python2.7/site-packages/keystonemiddleware/auth_token/_revocations.py", line 104, in check
if self._any_revoked(token_ids):
File "/usr/lib/python2.7/site-packages/keystonemiddleware/auth_token/_revocations.py", line 99, in _any_revoked
if self._is_revoked(token_id):
File "/usr/lib/python2.7/site-packages/keystonemiddleware/auth_token/_revocations.py", line 90, in _is_revoked
revoked_tokens = self._list.get('revoked', None)
File "/usr/lib/python2.7/site-packages/keystonemiddleware/auth_token/_revocations.py", line 74, in _list
self._list = self._fetch()
File "/usr/lib/python2.7/site-packages/keystonemiddleware/auth_token/_revocations.py", line 61, in _fetch
return self._cms_verify(revocation_list_data)
File "/usr/lib/python2.7/site-packages/keystonemiddleware/auth_token/__init__.py", line 771, in _cms_verify
self._fetch_signing_cert()
File "/usr/lib/python2.7/site-packages/keystonemiddleware/auth_token/__init__.py", line 786, in _fetch_signing_cert
self._identity_server.fetch_signing_cert())
File "/usr/lib/python2.7/site-packages/keystonemiddleware/auth_token/_signing_dir.py", line 54, in write_file
self._verify_signing_dir()
File "/usr/lib/python2.7/site-packages/keystonemiddleware/auth_token/_signing_dir.py", line 90, in _verify_signing_dir
os.makedirs(self._directory_name, stat.S_IRWXU)
File "/usr/lib64/python2.7/os.py", line 157, in makedirs
mkdir(name, mode)
OSError: [Errno 2] No such file or directory: '/tmp/keystone-signing-9zB_p6'
Stacktrace from neutron:
X-Auth-Token: gAAAAABX6NfMz4Lj4sYIDHu0eXr9oxymDrJTDOOrKztp0NElSiZcs9Umr-v8P-s8VP_lz_aVKPobfoj1ROP9X9amp8ACqwa4FNRvFX5IatzwmjAKR42AZZnuD4jxoJoC05iT-UKIY81gqHsOY8v7DbqTLSE2eOFwrFKZIMQBUDlDaeqwpce0LDp-dZrM2JIta9tOz99aOH5CShyu-ihMy3F87CN3cMdK5qHIr7oM1UiXc97zgzbDOTA
2016-09-26 05:29:36.804 28288 ERROR oslo_middleware.catch_errors Traceback (most recent call last):
2016-09-26 05:29:36.804 28288 ERROR oslo_middleware.catch_errors File "/usr/lib/python2.7/site-packages/oslo_middleware/catch_errors.py", line 38, in __call__
2016-09-26 05:29:36.804 28288 ERROR oslo_middleware.catch_errors response = req.get_response(self.application)
2016-09-26 05:29:36.804 28288 ERROR oslo_middleware.catch_errors File "/usr/lib/python2.7/site-packages/webob/request.py", line 1296, in send
2016-09-26 05:29:36.804 28288 ERROR oslo_middleware.catch_errors application, catch_exc_info=False)
2016-09-26 05:29:36.804 28288 ERROR oslo_middleware.catch_errors File "/usr/lib/python2.7/site-packages/webob/request.py", line 1260, in call_application
2016-09-26 05:29:36.804 28288 ERROR oslo_middleware.catch_errors app_iter = application(self.environ, start_response)
2016-09-26 05:29:36.804 28288 ERROR oslo_middleware.catch_errors File "/usr/lib/python2.7/site-packages/webob/dec.py", line 130, in __call__
2016-09-26 05:29:36.804 28288 ERROR oslo_middleware.catch_errors resp = self.call_func(req, *args, **self.kwargs)
2016-09-26 05:29:36.804 28288 ERROR oslo_middleware.catch_errors File "/usr/lib/python2.7/site-packages/webob/dec.py", line 195, in call_func
2016-09-26 05:29:36.804 28288 ERROR oslo_middleware.catch_errors return self.func(req, *args, **kwargs)
2016-09-26 05:29:36.804 28288 ERROR oslo_middleware.catch_errors File "/usr/lib/python2.7/site-packages/keystonemiddleware/auth_token/__init__.py", line 320, in __call__
2016-09-26 05:29:36.804 28288 ERROR oslo_middleware.catch_errors response = self.process_request(req)
2016-09-26 05:29:36.804 28288 ERROR oslo_middleware.catch_errors File "/usr/lib/python2.7/site-packages/keystonemiddleware/auth_token/__init__.py", line 552, in process_request
2016-09-26 05:29:36.804 28288 ERROR oslo_middleware.catch_errors resp = super(AuthProtocol, self).process_request(request)
2016-09-26 05:29:36.804 28288 ERROR oslo_middleware.catch_errors File "/usr/lib/python2.7/site-packages/keystonemiddleware/auth_token/__init__.py", line 348, in process_request
2016-09-26 05:29:36.804 28288 ERROR oslo_middleware.catch_errors data, user_auth_ref = self._do_fetch_token(request.user_token)
2016-09-26 05:29:36.804 28288 ERROR oslo_middleware.catch_errors File "/usr/lib/python2.7/site-packages/keystonemiddleware/auth_token/__init__.py", line 388, in _do_fetch_token
2016-09-26 05:29:36.804 28288 ERROR oslo_middleware.catch_errors data = self.fetch_token(token)
2016-09-26 05:29:36.804 28288 ERROR oslo_middleware.catch_errors File "/usr/lib/python2.7/site-packages/keystonemiddleware/auth_token/__init__.py", line 670, in fetch_token
2016-09-26 05:29:36.804 28288 ERROR oslo_middleware.catch_errors self._revocations.check(token_hashes)
2016-09-26 05:29:36.804 28288 ERROR oslo_middleware.catch_errors File "/usr/lib/python2.7/site-packages/keystonemiddleware/auth_token/_revocations.py", line 104, in check
2016-09-26 05:29:36.804 28288 ERROR oslo_middleware.catch_errors if self._any_revoked(token_ids):
2016-09-26 05:29:36.804 28288 ERROR oslo_middleware.catch_errors File "/usr/lib/python2.7/site-packages/keystonemiddleware/auth_token/_revocations.py", line 99, in _any_revoked
2016-09-26 05:29:36.804 28288 ERROR oslo_middleware.catch_errors if self._is_revoked(token_id):
2016-09-26 05:29:36.804 28288 ERROR oslo_middleware.catch_errors File "/usr/lib/python2.7/site-packages/keystonemiddleware/auth_token/_revocations.py", line 90, in _is_revoked
2016-09-26 05:29:36.804 28288 ERROR oslo_middleware.catch_errors revoked_tokens = self._list.get('revoked', None)
2016-09-26 05:29:36.804 28288 ERROR oslo_middleware.catch_errors File "/usr/lib/python2.7/site-packages/keystonemiddleware/auth_token/_revocations.py", line 74, in _list
2016-09-26 05:29:36.804 28288 ERROR oslo_middleware.catch_errors self._list = self._fetch()
2016-09-26 05:29:36.804 28288 ERROR oslo_middleware.catch_errors File "/usr/lib/python2.7/site-packages/keystonemiddleware/auth_token/_revocations.py", line 61, in _fetch
2016-09-26 05:29:36.804 28288 ERROR oslo_middleware.catch_errors return self._cms_verify(revocation_list_data)
2016-09-26 05:29:36.804 28288 ERROR oslo_middleware.catch_errors File "/usr/lib/python2.7/site-packages/keystonemiddleware/auth_token/__init__.py", line 771, in _cms_verify
2016-09-26 05:29:36.804 28288 ERROR oslo_middleware.catch_errors self._fetch_signing_cert()
2016-09-26 05:29:36.804 28288 ERROR oslo_middleware.catch_errors File "/usr/lib/python2.7/site-packages/keystonemiddleware/auth_token/__init__.py", line 786, in _fetch_signing_cert
2016-09-26 05:29:36.804 28288 ERROR oslo_middleware.catch_errors self._identity_server.fetch_signing_cert())
2016-09-26 05:29:36.804 28288 ERROR oslo_middleware.catch_errors File "/usr/lib/python2.7/site-packages/keystonemiddleware/auth_token/_signing_dir.py", line 54, in write_file
2016-09-26 05:29:36.804 28288 ERROR oslo_middleware.catch_errors self._verify_signing_dir()
2016-09-26 05:29:36.804 28288 ERROR oslo_middleware.catch_errors File "/usr/lib/python2.7/site-packages/keystonemiddleware/auth_token/_signing_dir.py", line 90, in _verify_signing_dir
2016-09-26 05:29:36.804 28288 ERROR oslo_middleware.catch_errors os.makedirs(self._directory_name, stat.S_IRWXU)
2016-09-26 05:29:36.804 28288 ERROR oslo_middleware.catch_errors File "/usr/lib64/python2.7/os.py", line 157, in makedirs
2016-09-26 05:29:36.804 28288 ERROR oslo_middleware.catch_errors mkdir(name, mode)
2016-09-26 05:29:36.804 28288 ERROR oslo_middleware.catch_errors OSError: [Errno 2] No such file or directory: '/tmp/keystone-signing-EVEBkJ'
As per discussion in IRC channel with Adam Young, this could be caused due to SeLinux permission problems (though the audit logs do not have anything on this). I will try with permissive and see if this can be reproduced.