2717 kilo SM provision looked to be looping and not completing with dependency failures.
root@cmbu-auto-esx1-lnx02:/var/log/contrail-server-manager/provision/cmbu-ceph-perf2.englab.juniper.net# server-manager status server
{
"server": [
{
"id": "cmbu-ceph-perf1",
"ip_address": "10.87.140.197",
"mac_address": "00:25:90:AB:9C:88",
"status": "storage_started"
},
{
"id": "cmbu-ceph-perf2",
"ip_address": "10.87.140.198",
"mac_address": "00:25:90:35:8A:1F",
"status": "storage_started"
},
{
"id": "cmbu-ceph-perf3",
"ip_address": "10.87.140.199",
"mac_address": "00:25:90:92:0E:6C",
"status": "storage_started"
},
{
"id": "cmbu-ceph-perf4",
"ip_address": "10.87.140.200",
"mac_address": "00:25:90:92:0D:F2",
"status": "storage_started"
}
]
}
root@cmbu-auto-esx1-lnx02:/var/log/contrail-server-manager/provision/cmbu-ceph-perf2.englab.juniper.net#
root@cmbu-auto-esx1-lnx02:/var/log/contrail-server-manager# server-manager show cluster --detail
{
"cluster": [
{
"base_image_id": "",
"email": "",
"id": "test-cluster",
"package_image_id": "",
"parameters": {
"admin_key": "AQDIgtNTgPLWARAAK6gs/fj8m88LnY9DwxJdYA==",
"analytics_data_ttl": "168",
"database_dir": "/home/cassandra",
"database_minimum_diskGB": "32",
"database_token": "",
"domain": "englab.juniper.net",
"encapsulation_priority": "MPLSoUDP,MPLSoGRE,VXLAN",
"external_bgp": "",
"gateway": "10.87.159.254",
"haproxy": "disable",
"internal_vip": "",
"keystone_password": "contrail123",
"keystone_tenant": "admin",
"keystone_username": "admin",
"live_migration": "enable",
"live_migration_nfs_vm_host": "cmbu-ceph-perf3",
"live_migration_storage_scope": "global",
"multi_tenancy": "False",
"osd_bootstrap_key": "AQCq7NFTeJUoBhAAlTVpxwWQJtBej/JDNhT6+Q==",
"password": "c0ntrail123",
"router_asn": "64512",
"service_token": "contrail123",
"storage_fsid": "1df387e1-9d32-42c4-a751-894108c7beeb",
"storage_mon_secret": "AQBM78tTEMz+GhAA3WiOXQI7UVdIy0YFFuTGdw==",
"storage_virsh_uuid": "86c19945-c06f-49f7-a9c7-c82d15b20d4f",
"subnet_mask": "255.255.224.0",
"use_certificates": "False",
"uuid": "58a724e8-3bb4-435a-8f29-cab50b37ca95"
},
"provision_role_sequence": "{'completed': [('cmbu-ceph-perf1', 'haproxy', '2016_02_24__13_42_33'), ('cmbu-ceph-perf1', 'database', '2016_02_24__13_43_45'), ('cmbu-ceph-perf1', 'openstack', '2016_02_24__13_54_00'), ('cmbu-ceph-perf1', 'config', '2016_02_24__13_57_04'), ('cmbu-ceph-perf1', 'control', '2016_02_24__13_57_41'), ('cmbu-ceph-perf1', 'collector', '2016_02_24__13_58_24'), ('cmbu-ceph-perf1', 'webui', '2016_02_24__13_58_53'), ('cmbu-ceph-perf4', 'compute', '2016_02_24__14_09_17'), ('cmbu-ceph-perf2', 'compute', '2016_02_24__14_09_29'), ('cmbu-ceph-perf3', 'compute', '2016_02_24__14_10_24')], 'steps': [[(u'cmbu-ceph-perf2', 'post_provision'), (u'cmbu-ceph-perf3', 'post_provision'), (u'cmbu-ceph-perf4', 'post_provision'), (u'cmbu-ceph-perf2', 'storage-compute'), (u'cmbu-ceph-perf3', 'storage-compute'), (u'cmbu-ceph-perf4', 'storage-compute'), (u'cmbu-ceph-perf1', 'storage-master'), (u'cmbu-ceph-perf1', 'post_provision')]]}",
"provisioned_id": null
}
]
}
root@cmbu-auto-esx1-lnx02:/var/log/contrail-server-manager#
root@cmbu-auto-esx1-lnx02:/var/log/contrail-server-manager/provision/cmbu-ceph-perf2.englab.juniper.net# tail cmbu-ceph-perf2.englab.juniper.net_24-02-2016_15:10:08.log
24-02-2016_15:10:08:/Stage[storage]/Contrail::Profile::Storage/Contrail::Storage/Contrail::Lib::Storage_common[storage-compute]/Contrail::Lib::Report_status[storage-master_completed]/Exec[contrail-status-storage-master_completed]/returns:executed successfully
24-02-2016_15:10:08:/Stage[storage]/Contrail::Profile::Storage/Contrail::Storage/Contrail::Lib::Storage_common[storage-compute]/Contrail::Lib::Report_status[storage-compute_completed]/Exec[contrail-status-storage-compute_completed]:Dependency Exec[setup-config-storage-compute-live-migration] has failures: true
24-02-2016_15:10:08:/Stage[storage]/Contrail::Profile::Storage/Contrail::Storage/Contrail::Lib::Storage_common[storage-compute]/Contrail::Lib::Report_status[storage-compute_completed]/Exec[contrail-status-storage-compute_completed]:Skipping because of failed dependencies
24-02-2016_15:10:08:/Stage[post]/Contrail::Do_reboot_server/Exec[do-reboot-server]:Dependency Exec[setup-config-storage-compute-live-migration] has failures: true
24-02-2016_15:10:08:/Stage[post]/Contrail::Do_reboot_server/Exec[do-reboot-server]:Skipping because of failed dependencies
24-02-2016_15:10:08:/Stage[post]/Contrail::Do_reboot_server/Notify[executed reboot server]:Dependency Exec[setup-config-storage-compute-live-migration] has failures: true
24-02-2016_15:10:08:/Stage[post]/Contrail::Do_reboot_server/Notify[executed reboot server]:Skipping because of failed dependencies
24-02-2016_15:10:08:/Stage[post]/Contrail::Provision_complete/Contrail::Lib::Report_status[post_provision_completed]/Exec[contrail-status-post_provision_completed]:Dependency Exec[setup-config-storage-compute-live-migration] has failures: true
24-02-2016_15:10:08:/Stage[post]/Contrail::Provision_complete/Contrail::Lib::Report_status[post_provision_completed]/Exec[contrail-status-post_provision_completed]:Skipping because of failed dependencies
24-02-2016_15:10:08:Puppet:Finished catalog run in 31.76 seconds
root@cmbu-auto-esx1-lnx02:/var/log/contrail-server-manager/provision/cmbu-ceph-perf2.englab.juniper.net#
Debugged this issue. openstack services were not running using "service SERVICE-NAME start/stop/restart" and caused storage VM based live-migration to fail.
This is a side-effect of https:/