On a Cluster which was migrated using ISSU, on Restarting one of the control nodes teh below alarm is raised on all control nodes.
Severity
Major
Time
2017-03-02 11:36:20
Type
default-global-system-config:system-defined-address-mismatch-control
Source
cl2-controller-1 (Control Node)
Status
Unacknowledged
Alarm
Control Node IP Address mismatch
Introspect shows below , which has uve's for old controllers also. Restart of ifmap server on all nodes fixes it.
{
uuid: ""581728d3-f68a-44c2-9d36-3480321383bb"",
fq_name: "["default-domain", "default-project", "ip-fabric", "__default__", "cl2-controller-3"]",
parent_uuid: ""eaf33c94-cce0-46d7-aa44-b3e8f6361907"",
parent_type: ""routing-instance"",
bgp_router_parameters: "{"gateway_address": null, "vendor": "contrail", "admin_down": false, "ipv6_gateway_address": null, "port": 179, "local_autonomous_system": null, "auth_data": null, "address": "141.102.0.17", "source_port": null, "router_type": "control-node", "identifier": "141.102.0.17", "hold_time": 0, "autonomous_system": 64512, "address_families": {"family": ["route-target", "inet-vpn", "e-vpn", "erm-vpn", "inet6-vpn"]}}",
perms2: "{"owner": "7ffaa60bf08e49fcaefe719268bfa704", "owner_access": 7, "global_access": 0, "share": []}",
id_perms: "{"enable": true, "uuid": {"uuid_mslong": 6347587090641470658, "uuid_lslong": 11328299637897659323}, "created": "2017-03-01T20:06:24.957866", "description": null, "creator": null, "user_visible": true, "last_modified": "2017-03-01T20:06:25.314826", "permissions": {"owner": "admin", "owner_access": 7, "other_access": 7, "group": "cloud-admin", "group_access": 7}}",
display_name: ""cl2-controller-3"",
bgp_router_refs: "[{"to": ["default-domain", "default-project", "ip-fabric", "__default__", "cl1-controller-1"], "attr": {"session": [{"attributes": [{"passive": false, "admin_down": false, "hold_time": 0, "bgp_router": null, "auth_data": null, "family_attributes": [], "as_override": false, "loop_count": 0, "address_families": {"family": ["route-target", "inet-vpn", "e-vpn", "erm-vpn", "inet6-vpn"]}}], "uuid": null}]}, "uuid": "51d64f49-f6be-45a6-9b4a-fc62569b4d21"}, {"to": ["default-domain", "default-project", "ip-fabric", "__default__", "cl2-controller-2"], "attr": {"session": [{"attributes": [{"passive": false, "admin_down": false, "hold_time": 0, "bgp_router": null, "auth_data": null, "family_attributes": [], "as_override": false, "loop_count": 0, "address_families": {"family": ["route-target", "inet-vpn", "e-vpn", "erm-vpn", "inet6-vpn"]}}], "uuid": null}]}, "uuid": "f032de97-85ce-407f-ba76-876aea3c5c79"}, {"to": ["default-domain", "default-project", "ip-fabric", "__default__", "cl2-controller-1"], "attr": {"session": [{"attributes": [{"passive": false, "admin_down": false, "hold_time": 0, "bgp_router": null, "auth_data": null, "family_attributes": [], "as_override": false, "loop_count": 0, "address_families": {"family": ["route-target", "inet-vpn", "e-vpn", "erm-vpn", "inet6-vpn"]}}], "uuid": null}]}, "uuid": "92048fae-34af-4c5f-a8f8-d73b044fc5dd"}, {"to": ["default-domain", "default-project", "ip-fabric", "__default__", "cl1-controller-2"], "attr": {"session": [{"attributes": [{"passive": false, "admin_down": false, "hold_time": 0, "bgp_router": null, "auth_data": null, "family_attributes": [], "as_override": false, "loop_count": 0, "address_families": {"family": ["route-target", "inet-vpn", "e-vpn", "erm-vpn", "inet6-vpn"]}}], "uuid": null}]}, "uuid": "ac64db6f-8fef-4292-904b-a9d0fca94061"}, {"to": ["default-domain", "default-project", "ip-fabric", "__default__", "montreal"], "attr": {"session": [{"attributes": [{"passive": false, "admin_down": false, "hold_time": 0, "bgp_router": null, "auth_data": null, "family_attributes": [], "as_override": false, "loop_count": 0, "address_families": {"family": ["route-target", "inet-vpn", "e-vpn", "erm-vpn", "inet6-vpn"]}}], "uuid": null}]}, "uuid": "56ac784d-f09a-4ca1-bbd6-22627ba01214"}, {"to": ["default-domain", "default-project", "ip-fabric", "__default__", "cl1-controller-3"], "attr": {"session": [{"attributes": [{"passive": false, "admin_down": false, "hold_time": 0, "bgp_router": null, "auth_data": null, "family_attributes": [], "as_override": false, "loop_count": 0, "address_families": {"family": ["route-target", "inet-vpn", "e-vpn", "erm-vpn", "inet6-vpn"]}}], "uuid": null}]}, "uuid": "623cba03-6d38-4944-b6ef-98cf1c1ba948"}]"
},
"cl2-controller-3:
AFTER RESTART IF IFMAP
=========================
ContrailConfig: {
deleted: false,
elements: {
uuid: ""92048fae-34af-4c5f-a8f8-d73b044fc5dd"",
fq_name: "["default-domain", "default-project", "ip-fabric", "__default__", "cl2-controller-1"]",
parent_uuid: ""eaf33c94-cce0-46d7-aa44-b3e8f6361907"",
parent_type: ""routing-instance"",
bgp_router_parameters: "{"gateway_address": null, "vendor": "contrail", "admin_down": false, "ipv6_gateway_address": null, "port": 179, "local_autonomous_system": null, "auth_data": null, "address": "141.102.0.15", "source_port": null, "router_type": "control-node", "identifier": "141.102.0.15", "hold_time": 0, "autonomous_system": 64512, "address_families": {"family": ["route-target", "inet-vpn", "e-vpn", "erm-vpn", "inet6-vpn"]}}",
perms2: "{"owner": "7ffaa60bf08e49fcaefe719268bfa704", "owner_access": 7, "global_access": 0, "share": []}",
id_perms: "{"enable": true, "uuid": {"uuid_mslong": 10521692607815306335, "uuid_lslong": 12175718241071515101}, "created": "2017-03-01T20:06:20.852587", "description": null, "creator": null, "user_visible": true, "last_modified": "2017-03-01T20:06:21.204938", "permissions": {"owner": "admin", "owner_access": 7, "other_access": 7, "group": "cloud-admin", "group_access": 7}}",
display_name: ""cl2-controller-1"",
bgp_router_refs: "[{"to": ["default-domain", "default-project", "ip-fabric", "__default__", "cl2-controller-3"], "attr": {"session": [{"attributes": [{"passive": false, "admin_down": false, "hold_time": 0, "bgp_router": null, "auth_data": null, "family_attributes": [], "as_override": false, "loop_count": 0, "address_families": {"family": ["route-target", "inet-vpn", "e-vpn", "erm-vpn", "inet6-vpn"]}}], "uuid": null}]}, "uuid": "581728d3-f68a-44c2-9d36-3480321383bb"}, {"to": ["default-domain", "default-project", "ip-fabric", "__default__", "cl2-controller-2"], "attr": {"session": [{"attributes": [{"passive": false, "admin_down": false, "hold_time": 0, "bgp_router": null, "auth_data": null, "family_attributes": [], "as_override": false, "loop_count": 0, "address_families": {"family": ["route-target", "inet-vpn", "e-vpn", "erm-vpn", "inet6-vpn"]}}], "uuid": null}]}, "uuid": "f032de97-85ce-407f-ba76-876aea3c5c79"}, {"to": ["default-domain", "default-project", "ip-fabric", "__default__", "montreal"], "attr": {"session": [{"attributes": [{"passive": false, "admin_down": false, "hold_time": 0, "bgp_router": null, "auth_data": null, "family_attributes": [], "as_override": false, "loop_count": 0, "address_families": {"family": []}}], "uuid": null}]}, "uuid": "56ac784d-f09a-4ca1-bbd6-22627ba01214"}]"
}
}
}
V1 Version: 3.1.2.0-62~kilo
V2 Version: 3.2.1.0-26~kilo
Hi,
Root cause has been found. Sandesh uve lib needs to be modified to handle
the new way in which UVEs are being sent from api-server. Earlier, uve sent
from api-server had incremental changes. Now, uve always has the entire
configuration json object string.
Analytics team will provide the necessary fix.
Regards,
Aannth