Error creating VMs due to keystone ConnectionRefused exception

[wenqing liang]

r3.2.0.0-5 kilo.

root@cmbu-ceph-perf1:/tmp# nova list
+--------------------------------------+-------+--------+------------+-------------+----------+
| ID | Name | Status | Task State | Power State | Networks |
+--------------------------------------+-------+--------+------------+-------------+----------+
| 368214e1-466c-48f5-b896-c168672c48bb | VM1 | ERROR | spawning | NOSTATE | |
| 096de4b5-8936-4ae3-9dc6-97529b1c837c | test | ERROR | spawning | NOSTATE | |
| 03340d7e-6f1a-4820-b4f6-c8e1264cfcf4 | test1 | ERROR | spawning | NOSTATE | |
+--------------------------------------+-------+--------+------------+-------------+----------+
root@cmbu-ceph-perf1:/tmp#

2016-11-29 15:05:20.832 1897 TRACE nova.compute.manager [instance: 368214e1-466c-48f5-b896-c168672c48bb] File "/usr/lib/python2.7/dist-packages/keystoneclient/auth/identity/base.py", line 144, in get_access
2016-11-29 15:05:20.832 1897 TRACE nova.compute.manager [instance: 368214e1-466c-48f5-b896-c168672c48bb] self.auth_ref = self.get_auth_ref(session)
2016-11-29 15:05:20.832 1897 TRACE nova.compute.manager [instance: 368214e1-466c-48f5-b896-c168672c48bb] File "/usr/lib/python2.7/dist-packages/keystoneclient/auth/identity/v2.py", line 78, in get_auth_ref
2016-11-29 15:05:20.832 1897 TRACE nova.compute.manager [instance: 368214e1-466c-48f5-b896-c168672c48bb] authenticated=False, log=False)
2016-11-29 15:05:20.832 1897 TRACE nova.compute.manager [instance: 368214e1-466c-48f5-b896-c168672c48bb] File "/usr/lib/python2.7/dist-packages/keystoneclient/session.py", line 488, in post
2016-11-29 15:05:20.832 1897 TRACE nova.compute.manager [instance: 368214e1-466c-48f5-b896-c168672c48bb] return self.request(url, 'POST', **kwargs)
2016-11-29 15:05:20.832 1897 TRACE nova.compute.manager [instance: 368214e1-466c-48f5-b896-c168672c48bb] File "/usr/lib/python2.7/dist-packages/keystoneclient/utils.py", line 318, in inner
2016-11-29 15:05:20.832 1897 TRACE nova.compute.manager [instance: 368214e1-466c-48f5-b896-c168672c48bb] return func(*args, **kwargs)
2016-11-29 15:05:20.832 1897 TRACE nova.compute.manager [instance: 368214e1-466c-48f5-b896-c168672c48bb] File "/usr/lib/python2.7/dist-packages/keystoneclient/session.py", line 374, in request
2016-11-29 15:05:20.832 1897 TRACE nova.compute.manager [instance: 368214e1-466c-48f5-b896-c168672c48bb] resp = send(**kwargs)
2016-11-29 15:05:20.832 1897 TRACE nova.compute.manager [instance: 368214e1-466c-48f5-b896-c168672c48bb] File "/usr/lib/python2.7/dist-packages/keystoneclient/session.py", line 417, in _send_request
2016-11-29 15:05:20.832 1897 TRACE nova.compute.manager [instance: 368214e1-466c-48f5-b896-c168672c48bb] raise exceptions.ConnectionRefused(msg)
2016-11-29 15:05:20.832 1897 TRACE nova.compute.manager [instance: 368214e1-466c-48f5-b896-c168672c48bb] ConnectionRefused: Unable to establish connection to http://localhost:5000/v2.0/tokens

root@cmbu-ceph-perf1:/tmp# contrail-version | grep install
contrail-install-packages 3.2.0.0-5~kilo 5
root@cmbu-ceph-perf1:/tmp# contrail-status
== Contrail Control ==
supervisor-control: active
contrail-control active
contrail-control-nodemgr active
contrail-dns active
contrail-named active

== Contrail Analytics ==
supervisor-analytics: active
contrail-alarm-gen:0 active
contrail-analytics-api active
contrail-analytics-nodemgr active
contrail-collector active
contrail-query-engine active
contrail-snmp-collector active
contrail-topology active

== Contrail Config ==
supervisor-config: active
contrail-api:0 active
contrail-config-nodemgr active
contrail-device-manager active
contrail-discovery:0 active
contrail-schema active
contrail-svc-monitor active
ifmap active

== Contrail Web UI ==
supervisor-webui: active
contrail-webui active
contrail-webui-middleware active

== Contrail Database ==
contrail-database: active

== Contrail Supervisor Database ==
supervisor-database: active
contrail-database-nodemgr active
kafka active

== Contrail Support Services ==
supervisor-support-service: active
rabbitmq-server active

== Contrail Storage ==
contrail-storage-stats: active
root@cmbu-ceph-perf1:/tmp# ceph -s
    cluster 7c7613ac-ceeb-49be-bbb1-2bd7e1beae6c
     health HEALTH_WARN
            too many PGs per OSD (796 > max 300)
     monmap e3: 3 mons at {cmbu-ceph-perf1=5.0.0.1:6789/0,cmbu-ceph-perf2=5.0.0.2:6789/0,cmbu-ceph-perf3=5.0.0.3:6789/0}
            election epoch 14, quorum 0,1,2 cmbu-ceph-perf1,cmbu-ceph-perf2,cmbu-ceph-perf3
     osdmap e286: 11 osds: 11 up, 11 in
      pgmap v768: 4410 pgs, 27 pools, 1859 MB data, 2996 objects
            3129 MB used, 8513 GB / 8516 GB avail
                4410 active+clean
root@cmbu-ceph-perf1:/tmp#

[amudhar (amudha)]

This should be fixed by https://review.opencontrail.org/#/c/26562/.
Please test with the latest build and confirm.

Thanks,
Amudha

[wenqing liang (wliang)]

not seen in r3.2.0.0-12 kilo. Will monitor and close if not seen again.