On R3.0.2.0-43 kilo, provision is stuck at compute (u'cmbu-ceph-perf2', 'compute').
Unlike the config node, some compute nodes mgmt IPs are not reachable. In ipmi, the interface names looked not to have been updated since reimaging, e.g. cmbu-ceph-perf2 still shows mgmt. interface as eth1 though the interface with the same mac is now eth2 and mgmt ip is not configured anywhere after 14.04.4 reimaging. The same is true for the other non-reachable computes.
root@cmbu-auto-esx1-lnx02:/var/log# server-manager status server
{
"server": [
{
"id": "cmbu-ceph-perf1",
"ip_address": "10.87.140.197",
"mac_address": "00:25:90:AB:9C:88",
"status": "provision_completed"
},
{
"id": "cmbu-ceph-perf2",
"ip_address": "10.87.140.198",
"mac_address": "00:25:90:35:8A:1F",
"status": "reimage_completed"
},
{
"id": "cmbu-ceph-perf3",
"ip_address": "10.87.140.199",
"mac_address": "00:25:90:92:0E:6C",
"status": "reimage_completed"
},
{
"id": "cmbu-ceph-perf4",
"ip_address": "10.87.140.200",
"mac_address": "00:25:90:92:0D:F2",
"status": "reimage_completed"
}
]
}
root@cmbu-auto-esx1-lnx02:/var/log# server-manager show cluster --detail
{
"cluster": [
{
"base_image_id": "",
"email": "",
"id": "test-cluster",
"package_image_id": "",
"parameters": {
"admin_key": "AQDIgtNTgPLWARAAK6gs/fj8m88LnY9DwxJdYA==",
"amqp_server_ip": "",
"analytics_config_audit_ttl": "168",
"analytics_data_dir": "",
"analytics_data_ttl": "168",
"analytics_flow_ttl": "2",
"analytics_statistics_ttl": "24",
"analytics_syslog_port": "-1",
"contrail_external_virtual_router_id": "100",
"contrail_internal_virtual_router_id": "100",
"database_dir": "/home/cassandra",
"database_initial_token": "0",
"database_ip_port": "9160",
"database_minimum_diskGB": "32",
"database_token": "",
"domain": "englab.juniper.net",
"enable_ceilometer": "False",
"enable_lbass": "False",
"encapsulation_priority": "MPLSoUDP,MPLSoGRE,VXLAN",
"external_bgp": "",
"external_vip": "",
"external_virtual_router_id": "100",
"gateway": "10.87.159.254",
"haproxy": "disable",
"haproxy_flag": "disable",
"hc_interval": "5",
"heat_encryption_key": "0726FF6263B5",
"internal_vip": "",
"internal_virtual_router_id": "100",
"kernel_upgrade": "yes",
"kernel_version": "",
"keystone_admin_token": "5198BFF7F61D",
"keystone_ip": "",
"keystone_password": "contrail123",
"keystone_region_name": "RegionOne",
"keystone_service_tenant": "service",
"keystone_service_token": "contrail123",
"keystone_tenant": "admin",
"keystone_username": "admin",
"live_migration": "enable",
"live_migration_host": "",
"live_migration_ip": "",
"live_migration_nfs_vm_host": "cmbu-ceph-perf3",
"live_migration_storage_scope": "global",
"manage_neutron": "True",
"multi_tenancy": "False",
"mysql_root_password": "40C112673C44",
"mysql_service_password": "2A927473A283",
"nfs_glance_path": "",
"nfs_server": "",
"openstack_manage_amqp": "False",
"osd_bootstrap_key": "AQCq7NFTeJUoBhAAlTVpxwWQJtBej/JDNhT6+Q==",
"password": "c0ntrail123",
"router_asn": "64512",
"service_token": "contrail123",
"snmp_fast_scan_frequency": "60",
"snmp_scan_frequency": "600",
"ssd_data_dir": "",
"storage_chassis_config": "",
"storage_fsid": "c686cce3-dd27-4487-af97-e95bc7624992",
"storage_mon_secret": "AQBM78tTEMz+GhAA3WiOXQI7UVdIy0YFFuTGdw==",
"storage_osd_disks": "",
"storage_virsh_uuid": "bb3d6a9f-c52d-4b75-91be-da6e7ecf9811",
"subnet_mask": "255.255.224.0",
"topology_scan_frequency": "60",
"use_certificates": "False",
"uuid": "9c0e69a3-f75c-495b-b411-59bfd519164a",
"vgw_gateway_routes": "",
"vgw_interface": "",
"vgw_public_subnet": "",
"vgw_public_vn_name": "",
"vmware_ip": "",
"vmware_password": "",
"vmware_username": "",
"vmware_vswitch": ""
},
"provision_role_sequence": "{'completed': [('cmbu-ceph-perf1', 'haproxy', '2016_05_26__17_28_25'), ('cmbu-ceph-perf1', 'database', '2016_05_26__17_29_42'), ('cmbu-ceph-perf1', 'openstack', '2016_05_26__17_40_53'), ('cmbu-ceph-perf1', 'config', '2016_05_26__17_43_51'), ('cmbu-ceph-perf1', 'control', '2016_05_26__17_44_35'), ('cmbu-ceph-perf1', 'collector', '2016_05_26__17_45_30'), ('cmbu-ceph-perf1', 'webui', '2016_05_26__17_46_03'), ('cmbu-ceph-perf1', 'post_provision', '2016_05_26__17_46_22')], 'steps': [[(u'cmbu-ceph-perf2', 'compute'), (u'cmbu-ceph-perf2', 'post_provision'), (u'cmbu-ceph-perf3', 'compute'), (u'cmbu-ceph-perf3', 'post_provision'), (u'cmbu-ceph-perf4', 'compute'), (u'cmbu-ceph-perf4', 'post_provision')]]}",
"provisioned_id": null
}
]
}
root@cmbu-auto-esx1-lnx02:/var/log# server-manager show server --detail
{
"server": [
{
"base_image_id": "",
"cluster_id": "test-cluster",
"contrail": {
"control_data_interface": "p2p2"
},
"discovered": "false",
"domain": "englab.juniper.net",
"email": null,
"gateway": "10.87.159.254",
"host_name": "",
"id": "cmbu-ceph-perf1",
"intf_bond": null,
"intf_control": null,
"intf_data": null,
"ip_address": "10.87.140.197",
"ipmi_address": "10.87.140.201",
"ipmi_password": "ADMIN",
"ipmi_type": "",
"ipmi_username": "",
"last_update": "2016-05-26 23:59:53",
"mac_address": "00:25:90:AB:9C:88",
"network": {
"interfaces": [
{
"default_gateway": "10.87.159.254",
"dhcp": true,
"ip_address": "10.87.140.197/19",
"mac_address": "00:25:90:ab:9c:88",
"name": "p4p1"
},
{
"default_gateway": "5.0.0.1",
"gateway": "",
"ip_address": "5.0.0.1/24",
"mac_address": "90:e2:ba:2b:53:5d",
"name": "p2p2"
}
],
"management_interface": "p4p1",
"provisioning": "kickstart"
},
"package_image_id": "",
"parameters": {
"interface_name": "p4p1",
"kernel_upgrade": "yes",
"kernel_version": "3.13.0-40",
"partition": "/dev/sda",
"storage_repo_id": "contrail-storage-packages"
},
"password": "n1keenA",
"provisioned_id": "contrail_test_pkg_43kilo",
"reimaged_id": "ubuntu-server1",
"roles": [
"config",
"openstack",
"control",
"collector",
"webui",
"database"
],
"ssh_public_key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC2Xxzyurb7cFj5frPe7gsZUtuVUq6NOtTGKzGHnVM/fbY12ver8kQnAGxjdHRIBkA6JfclC/7rTLZjagwO7E1MpbML939gXY8+dNq2UAHeloftGOqPgQB3+8Ffh2elrMN0dXBo6XfsQ8bixFmEVHg/qZ7tQe2PXCack4lZ2M2C1jENFBX9JM7LyKsDZ2Sy6uddhD3JYxhLlwGAxUSbuHH1GTJTBcfjTgoSgyTTErDOAxjefgunEZSgqA6rUfmQStXDa5dqdbRJLfeLkklig0MyVJyyaw/vm1nSzmV7NTYFQp7jzsgEV8wGmBFsBXzt6zImCLqMu6HcZCA9u1XR0ent",
"static_ip": "N",
"status": "provision_completed",
"subnet_mask": "255.255.224.0",
"tag": {},
"top_of_rack": {}
},
{
"base_image_id": "",
"cluster_id": "test-cluster",
"contrail": {
"control_data_interface": "p6p2"
},
"discovered": "false",
"domain": "englab.juniper.net",
"email": null,
"gateway": "10.87.159.254",
"host_name": "",
"id": "cmbu-ceph-perf2",
"intf_bond": null,
"intf_control": null,
"intf_data": null,
"ip_address": "10.87.140.198",
"ipmi_address": "10.87.140.202",
"ipmi_password": "ADMIN",
"ipmi_type": "",
"ipmi_username": "",
"last_update": "2016-05-26 23:59:53",
"mac_address": "00:25:90:35:8A:1F",
"network": {
"interfaces": [
{
"default_gateway": "10.87.159.254",
"dhcp": true,
"ip_address": "10.87.140.198/19",
"mac_address": "00:25:90:35:8a:1f",
"name": "eth1"
},
{
"default_gateway": "5.0.0.1",
"gateway": "",
"ip_address": "5.0.0.2/24",
"mac_address": "90:e2:ba:1c:be:69",
"name": "p6p2"
}
],
"management_interface": "eth1",
"provisioning": "kickstart"
},
"package_image_id": "",
"parameters": {
"disks": [
"/dev/sdc:/dev/sdb",
"/dev/sdd:/dev/sdb",
"/dev/sde:/dev/sdb",
"/dev/sdf:/dev/sdb"
],
"interface_name": "eth1",
"kernel_upgrade": "yes",
"kernel_version": "3.13.0-40",
"partition": "/dev/sda",
"storage_repo_id": "contrail-storage-packages"
},
"password": "n1keenA",
"provisioned_id": "contrail_test_pkg_43kilo",
"reimaged_id": "ubuntu-server1",
"roles": [
"compute"
],
"ssh_public_key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCYQq/Ggfq7RC+Y+XObRtJ3Rx0nO0Th+D0720Zrlu2lRg1ViOqn0DLAAL/hffI6wOJKZt+WT8Vvra5xkagxUF0G39zmGC3FZkxQrpvXQJLsLG5YikEZwzZlxmBvc+yyUpdITFRhM8OReKf7Ombt2aRMn8edIER6yw6/uzBouK9Cqr4s3TPeGEKPyc7xZNtKKAwQVnGzYOkzVwQuLuyoGAvFgyjWaCNYziOF5ngssBzUOr/wWvjUHKUsHNbtMuf7KfgKrLkh8kkmlgDlq3DBxijxQoZl1xBiRyPEQyw1xvDjr6aCVjzAbly5uRCZVSnCT2RoNGNOyp31AbvvDzO7PpGL",
"static_ip": "N",
"status": "reimage_completed",
"subnet_mask": "255.255.224.0",
"tag": {},
"top_of_rack": {}
},
{
"base_image_id": "",
"cluster_id": "test-cluster",
"contrail": {
"control_data_interface": "p6p2"
},
"discovered": "false",
"domain": "englab.juniper.net",
"email": null,
"gateway": "10.87.159.254",
"host_name": "",
"id": "cmbu-ceph-perf3",
"intf_bond": null,
"intf_control": null,
"intf_data": null,
"ip_address": "10.87.140.199",
"ipmi_address": "10.87.140.203",
"ipmi_password": "ADMIN",
"ipmi_type": "",
"ipmi_username": "",
"last_update": "2016-05-26 23:59:54",
"mac_address": "00:25:90:92:0E:6C",
"network": {
"interfaces": [
{
"default_gateway": "10.87.159.254",
"dhcp": true,
"ip_address": "10.87.140.199/19",
"mac_address": "00:25:90:92:0e:6c",
"name": "eth0"
},
{
"default_gateway": "5.0.0.1",
"gateway": "",
"ip_address": "5.0.0.3/24",
"mac_address": "90:e2:ba:1c:c0:ad",
"name": "p6p2"
}
],
"management_interface": "eth0",
"provisioning": "kickstart"
},
"package_image_id": "",
"parameters": {
"disks": [
"/dev/sdc:/dev/sdb",
"/dev/sdd:/dev/sdb",
"/dev/sde:/dev/sdb",
"/dev/sdf:/dev/sdb"
],
"interface_name": "eth0",
"kernel_upgrade": "yes",
"kernel_version": "3.13.0-40",
"partition": "/dev/sda",
"storage_repo_id": "contrail-storage-packages"
},
"password": "n1keenA",
"provisioned_id": "contrail_test_pkg_43kilo",
"reimaged_id": "ubuntu-server1",
"roles": [
"compute"
],
"ssh_public_key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDN9EvZH77yeJjC+GLx/NW1PhGeB5/7QoQNFA6vc3qN0NL7mN2VixY/jWrBwPiNShlYORa1+DMl03z3C0Dn9vKrEj4SpMVGJxNLm4be+LnX7yRTmILyPFvzMgxw7cZZnHQazKgUz4yAinD19p0swfmG1RIuHWzsFT4QXXZ4ZqUVckUC0f0LNWV3o4NvWUFxjy4hP7z1T248apUMKXbzPOb3NfRHQ5ReGfMFw/QMOUlp6rD93SzHvITsPy2KTO/EEu8RXa0fx9nZ2Ka1L3c0FE428XBfv2kxaj/a1RK3A8YKIi0XCuGosDaVR9dKl//cy5pLshc3zVtJeyfWENRZwUPV",
"static_ip": "N",
"status": "reimage_completed",
"subnet_mask": "255.255.224.0",
"tag": {},
"top_of_rack": {}
},
{
"base_image_id": "",
"cluster_id": "test-cluster",
"contrail": {
"control_data_interface": "p6p2"
},
"discovered": "false",
"domain": "englab.juniper.net",
"email": null,
"gateway": "10.87.159.254",
"host_name": "",
"id": "cmbu-ceph-perf4",
"intf_bond": null,
"intf_control": null,
"intf_data": null,
"ip_address": "10.87.140.200",
"ipmi_address": "10.87.140.204",
"ipmi_password": "ADMIN",
"ipmi_type": "",
"ipmi_username": "",
"last_update": "2016-05-26 23:59:55",
"mac_address": "00:25:90:92:0D:F2",
"network": {
"interfaces": [
{
"default_gateway": "10.87.159.254",
"dhcp": true,
"ip_address": "10.87.140.200/19",
"mac_address": "00:25:90:92:0d:f2",
"name": "eth0"
},
{
"default_gateway": "5.0.0.1",
"gateway": "",
"ip_address": "5.0.0.4/24",
"mac_address": "90:e2:ba:1c:c4:05",
"name": "p6p2"
}
],
"management_interface": "eth0",
"provisioning": "kickstart"
},
"package_image_id": "",
"parameters": {
"disks": [
"/dev/sdc:/dev/sdb",
"/dev/sdd:/dev/sdb",
"/dev/sde:/dev/sdb",
"/dev/sdf:/dev/sdb"
],
"interface_name": "eth0",
"kernel_upgrade": "yes",
"kernel_version": "3.13.0-40",
"partition": "/dev/sda",
"storage_repo_id": "contrail-storage-packages"
},
"password": "n1keenA",
"provisioned_id": "contrail_test_pkg_43kilo",
"reimaged_id": "ubuntu-server1",
"roles": [
"compute"
],
"ssh_public_key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC4xQJysUDToTnxE8sgggEENVw93cKv8Jnz6UBeK8lsNTweGudI/kcXRF3abQi2HOvmtJf2iRcVLPj7lvMyuZen++AURE1XZEVCqvNklCCMo2Qcb0TuosLYYAWQ3j0DhPcw4unHaZhRzAhQo40K6JFGdyjDA35LkowmhbgTmHHjUazA6hJaswH/PPL5Qam4k7SWPNm9NYxkMGtGj/zqwZTqf6mfWJBMcLSwJur9AwA79TddW93nsfXjc1qh+7bfLgdGQhMDA9cXXBdvr0z1dSL4IA9slekvCsDI3pKqEU/HYu8Sftzk1zlh1gIGHTQc5nLhCMCVZQESUS7mIpLeeHB3",
"static_ip": "N",
"status": "reimage_completed",
"subnet_mask": "255.255.224.0",
"tag": {},
"top_of_rack": {}
}
]
}
root@cmbu-auto-esx1-lnx02:/var/log#
1. The first boot fix in SM for 14.04.04 is available only in build 43. Since build 35 is used you are seeing the issue.
2. After upgrading to build 43, since this fix has changes in the kickstart file you need to delte and add the existing ubuntu images for the new kickstart files to change effect. We will release note it