Comment 3 for bug 1616797

It seems that bunch of requests from contrail-analytics-api VNC api client to keystone at around 12:52:40 are missing from keystone logs.

contrail-analytics-api logs
--------------------------------------

Traceback (most recent call last):
  File "/usr/lib/python2.7/dist-packages/bottle.py", line 856, in _handle
    return route.call(**args)
  File "/usr/lib/python2.7/dist-packages/bottle.py", line 1721, in wrapper
    rv = callback(*a, **ka)
  File "/usr/lib/python2.7/dist-packages/opserver/opserver.py", line 411, in _impl
    self._vnc_api_client.is_role_cloud_admin(user_token):
  File "/usr/lib/python2.7/dist-packages/opserver/vnc_cfg_api_client.py", line 61, in is_role_cloud_admin
    result = self._get_user_token_info(user_token)
  File "/usr/lib/python2.7/dist-packages/opserver/vnc_cfg_api_client.py", line 30, in _get_user_token_info
    return self._vnc_api_client.obj_perms(user_token)
  File "/usr/lib/python2.7/dist-packages/vnc_api/vnc_api.py", line 1208, in obj_perms
    del self._headers['X-USER-TOKEN']
KeyError: 'X-USER-TOKEN'
10.204.217.41 - - [2016-08-25 12:52:40] "POST /analytics/uves/database-node HTTP/1.1" 500 156 0.032620
10.204.217.41 - - [2016-08-25 12:52:40] "POST /analytics/uves/analytics-node HTTP/1.1" 200 10437 0.037467
Traceback (most recent call last):
  File "/usr/lib/python2.7/dist-packages/bottle.py", line 856, in _handle
    return route.call(**args)
  File "/usr/lib/python2.7/dist-packages/bottle.py", line 1721, in wrapper
    rv = callback(*a, **ka)
  File "/usr/lib/python2.7/dist-packages/opserver/opserver.py", line 411, in _impl
    self._vnc_api_client.is_role_cloud_admin(user_token):
  File "/usr/lib/python2.7/dist-packages/opserver/vnc_cfg_api_client.py", line 61, in is_role_cloud_admin
    result = self._get_user_token_info(user_token)
  File "/usr/lib/python2.7/dist-packages/opserver/vnc_cfg_api_client.py", line 30, in _get_user_token_info
    return self._vnc_api_client.obj_perms(user_token)
  File "/usr/lib/python2.7/dist-packages/vnc_api/vnc_api.py", line 1208, in obj_perms
    del self._headers['X-USER-TOKEN']
KeyError: 'X-USER-TOKEN'
10.204.217.41 - - [2016-08-25 12:52:40] "POST /analytics/uves/config-node HTTP/1.1" 500 156 0.037808
10.204.217.41 - - [2016-08-25 12:52:40] "POST /analytics/uves/control-node HTTP/1.1" 200 4659 0.026998
10.204.217.41 - - [2016-08-25 12:52:40] "GET /analytics/uves/virtual-networks?cfilt=UveVirtualNetworkAgent HTTP/1.1" 200 243 0.015646
Traceback (most recent call last):
  File "/usr/lib/python2.7/dist-packages/bottle.py", line 856, in _handle
    return route.call(**args)
  File "/usr/lib/python2.7/dist-packages/bottle.py", line 1721, in wrapper
    rv = callback(*a, **ka)
  File "/usr/lib/python2.7/dist-packages/opserver/opserver.py", line 411, in _impl
    self._vnc_api_client.is_role_cloud_admin(user_token):
  File "/usr/lib/python2.7/dist-packages/opserver/vnc_cfg_api_client.py", line 61, in is_role_cloud_admin
    result = self._get_user_token_info(user_token)
  File "/usr/lib/python2.7/dist-packages/opserver/vnc_cfg_api_client.py", line 30, in _get_user_token_info
    return self._vnc_api_client.obj_perms(user_token)
  File "/usr/lib/python2.7/dist-packages/vnc_api/vnc_api.py", line 1208, in obj_perms
    del self._headers['X-USER-TOKEN']
KeyError: 'X-USER-TOKEN'
10.204.217.41 - - [2016-08-25 12:52:40] "GET /analytics/queries HTTP/1.1" 500 156 0.016220

keystone logs
-------------------

2016-08-25 12:52:39.487 4381 INFO eventlet.wsgi.server [req-52eae439-bb84-4291-8648-9502e4bd93c0 a8f8e869d40b4ea2a5df2c2c79388d41 c683a111eeb1423b8fb24a634f525035 - default default] 10.204.217.41 - - [25/Aug/2016 12:52:39] "GET /v3/auth/tokens HTTP/1.1" 200 6486 0.100954
2016-08-25 12:53:10.135 4380 INFO keystone.common.wsgi [req-c6a89140-29cf-44e3-b740-a03702ac7805 - - - - -] POST http://10.204.217.41:35357/v2.0/tokens

It doesn't look like keystone was restarted at that time

root@nodeg1:/var/log/contrail# supervisorctl -s unix:///var/run/supervisord_openstack.sock
barbican-api FATAL Exited too quickly (process log may have details)
barbican-worker RUNNING pid 2541, uptime 11:22:18
cinder-api RUNNING pid 2549, uptime 11:22:18
cinder-scheduler RUNNING pid 2653, uptime 11:22:18
glance-api RUNNING pid 2546, uptime 11:22:18
glance-registry RUNNING pid 2608, uptime 11:22:18
heat-api RUNNING pid 2638, uptime 11:22:18
heat-api-cfn RUNNING pid 2592, uptime 11:22:18
heat-engine RUNNING pid 2607, uptime 11:22:18
keystone RUNNING pid 23684, uptime 10:06:32
nova-api RUNNING pid 2538, uptime 11:22:18
nova-conductor RUNNING pid 2543, uptime 11:22:18
nova-console RUNNING pid 2568, uptime 11:22:18
nova-consoleauth RUNNING pid 2553, uptime 11:22:18
nova-novncproxy RUNNING pid 2537, uptime 11:22:18
nova-scheduler RUNNING pid 2602, uptime 11:22:18
supervisor> quit
root@nodeg1:/var/log/contrail# date
Thu Aug 25 23:06:28 IST 2016
root@nodeg1:/var/log/contrail#