Observing ip address mismatch alarm for both config and control nodes on a multi cfgm cluster even if the alarm rules match.
{
"BgpRouterState": {
"num_down_static_routes": 0,
"bgp_router_ip_list": [
null
],
"num_up_xmpp_peer": 2,
"output_queue_depth": 0,
"num_down_service_chains": 0,
"uptime": 1493894996545630,
"num_deleting_xmpp_peer": 0,
"local_asn": 64512,
"db_conn_info": {
"cluster": "10.10.0.10, 10.10.0.5, 10.10.0.11",
"connection_status_change_at": 1493894936608392,
"connection_status": true
},
"num_xmpp_peer": 2,
"num_deleting_bgp_peer": 0,
"num_static_routes": 0,
"router_id": "10.10.0.10",
"admin_down": false,
"num_up_bgpaas_peer": 0,
"num_deleted_routing_instance": 0,
"num_service_chains": 0,
"global_asn": 64512,
"num_routing_instance": 22,
"build_info": "{\"build-info\":[{\"build-time\":\"2017-05-03 14:19:46.672871\",\"build-hostname\":\"ubuntu\",\"build-user\":\"contrail-builder\",\"build-version\":\"4.0.0.0\",\"build-id\":\"4.0.0.0-3068\",\"build-number\":\"3068\"}]}",
"ifmap_server_info": {
"num_peer_clients": 1
},
"num_up_bgp_peer": 2,
"amqp_conn_info": {
"url": "amqp://guest:guest@10.10.0.4:5672",
"connection_status_change_at": 1493894936621218,
"connection_status": true
},
"num_bgpaas_peer": 0,
"num_bgp_peer": 2,
"num_deleting_bgpaas_peer": 0
},
"ContrailConfig": {
"deleted": [
[
false,
"server2:Config:contrail-api:0"
],
[
false,
"server5:Config:contrail-api:0"
],
[
false,
"server6:Config:contrail-api:0"
]
],
"elements": [
[
{
"uuid": "\"375975f0-1ded-47cf-8906-753c067a2c08\"",
"fq_name": "[\"default-domain\", \"default-project\", \"ip-fabric\", \"__default__\", \"server5\"]",
"parent_uuid": "\"2327986d-b0aa-4904-b99b-217cd72425e3\"",
"parent_type": "\"routing-instance\"",
"bgp_router_parameters": "{\"gateway_address\": null, \"vendor\": \"contrail\", \"admin_down\": false, \"ipv6_gateway_address\": null, \"port\": 179, \"local_autonomous_system\": null, \"auth_data\": null, \"address\": \"10.10.0.10\", \"source_port\": null, \"router_type\": \"control-node\", \"identifier\": \"10.10.0.10\", \"hold_time\": 0, \"autonomous_system\": 64512, \"address_families\": {\"family\": [\"route-target\", \"inet-vpn\", \"e-vpn\", \"erm-vpn\", \"inet6-vpn\"]}}",
"perms2": "{\"owner\": \"cloud-admin\", \"owner_access\": 7, \"global_access\": 0, \"share\": []}",
"id_perms": "{\"enable\": true, \"uuid\": {\"uuid_mslong\": 3988348619167975375, \"uuid_lslong\": 9873708133723548680}, \"creator\": null, \"created\": \"2017-05-03T22:54:58.134730\", \"user_visible\": true, \"last_modified\": \"2017-05-04T10:37:21.665399\", \"permissions\": {\"owner\": \"cloud-admin\", \"owner_access\": 7, \"other_access\": 7, \"group\": \"admin\", \"group_access\": 7}, \"description\": null}",
"display_name": "\"server5\"",
"bgp_router_refs": "[{\"to\": [\"default-domain\", \"default-project\", \"ip-fabric\", \"__default__\", \"server6\"], \"attr\": {\"session\": [{\"attributes\": [{\"passive\": false, \"admin_down\": false, \"private_as_action\": null, \"hold_time\": 0, \"bgp_router\": null, \"auth_data\": null, \"family_attributes\": [], \"as_override\": false, \"loop_count\": 0, \"address_families\": {\"family\": []}}], \"uuid\": null}]}, \"uuid\": \"9e48bb31-24f4-48a1-8b89-21243ed34dd0\"}, {\"to\": [\"default-domain\", \"default-project\", \"ip-fabric\", \"__default__\", \"server2\"], \"attr\": {\"session\": [{\"attributes\": [{\"passive\": false, \"admin_down\": false, \"private_as_action\": null, \"hold_time\": 0, \"bgp_router\": null, \"auth_data\": null, \"family_attributes\": [], \"as_override\": false, \"loop_count\": 0, \"address_families\": {\"family\": [\"route-target\", \"inet-vpn\", \"e-vpn\", \"erm-vpn\", \"inet6-vpn\"]}}], \"uuid\": null}]}, \"uuid\": \"b60ce400-7519-4376-bc53-5dbf50ce3f06\"}]"
},
"server2:Config:contrail-api:0"
],
[
{
"uuid": "\"375975f0-1ded-47cf-8906-753c067a2c08\"",
"fq_name": "[\"default-domain\", \"default-project\", \"ip-fabric\", \"__default__\", \"server5\"]",
"parent_uuid": "\"2327986d-b0aa-4904-b99b-217cd72425e3\"",
"parent_type": "\"routing-instance\"",
"bgp_router_parameters": "{\"gateway_address\": null, \"vendor\": \"contrail\", \"admin_down\": false, \"ipv6_gateway_address\": null, \"port\": 179, \"local_autonomous_system\": null, \"auth_data\": null, \"address\": \"10.10.0.10\", \"source_port\": null, \"router_type\": \"control-node\", \"identifier\": \"10.10.0.10\", \"hold_time\": 0, \"autonomous_system\": 64512, \"address_families\": {\"family\": [\"route-target\", \"inet-vpn\", \"e-vpn\", \"erm-vpn\", \"inet6-vpn\"]}}",
"perms2": "{\"owner\": \"cloud-admin\", \"owner_access\": 7, \"global_access\": 0, \"share\": []}",
"id_perms": "{\"enable\": true, \"uuid\": {\"uuid_mslong\": 3988348619167975375, \"uuid_lslong\": 9873708133723548680}, \"creator\": null, \"created\": \"2017-05-03T22:54:58.134730\", \"user_visible\": true, \"last_modified\": \"2017-05-04T10:37:21.665399\", \"permissions\": {\"owner\": \"cloud-admin\", \"owner_access\": 7, \"other_access\": 7, \"group\": \"admin\", \"group_access\": 7}, \"description\": null}",
"display_name": "\"server5\"",
"bgp_router_refs": "[{\"to\": [\"default-domain\", \"default-project\", \"ip-fabric\", \"__default__\", \"server6\"], \"attr\": {\"session\": [{\"attributes\": [{\"passive\": false, \"admin_down\": false, \"private_as_action\": null, \"hold_time\": 0, \"bgp_router\": null, \"auth_data\": null, \"family_attributes\": [], \"as_override\": false, \"loop_count\": 0, \"address_families\": {\"family\": []}}], \"uuid\": null}]}, \"uuid\": \"9e48bb31-24f4-48a1-8b89-21243ed34dd0\"}, {\"to\": [\"default-domain\", \"default-project\", \"ip-fabric\", \"__default__\", \"server2\"], \"attr\": {\"session\": [{\"attributes\": [{\"passive\": false, \"admin_down\": false, \"private_as_action\": null, \"hold_time\": 0, \"bgp_router\": null, \"auth_data\": null, \"family_attributes\": [], \"as_override\": false, \"loop_count\": 0, \"address_families\": {\"family\": [\"route-target\", \"inet-vpn\", \"e-vpn\", \"erm-vpn\", \"inet6-vpn\"]}}], \"uuid\": null}]}, \"uuid\": \"b60ce400-7519-4376-bc53-5dbf50ce3f06\"}]"
},
"server5:Config:contrail-api:0"
],
[
{
"uuid": "\"375975f0-1ded-47cf-8906-753c067a2c08\"",
"fq_name": "[\"default-domain\", \"default-project\", \"ip-fabric\", \"__default__\", \"server5\"]",
"parent_uuid": "\"2327986d-b0aa-4904-b99b-217cd72425e3\"",
"parent_type": "\"routing-instance\"",
"bgp_router_parameters": "{\"gateway_address\": null, \"vendor\": \"contrail\", \"admin_down\": false, \"ipv6_gateway_address\": null, \"port\": 179, \"local_autonomous_system\": null, \"auth_data\": null, \"address\": \"10.10.0.10\", \"source_port\": null, \"router_type\": \"control-node\", \"identifier\": \"10.10.0.10\", \"hold_time\": 0, \"autonomous_system\": 64512, \"address_families\": {\"family\": [\"route-target\", \"inet-vpn\", \"e-vpn\", \"erm-vpn\", \"inet6-vpn\"]}}",
"perms2": "{\"owner\": \"cloud-admin\", \"owner_access\": 7, \"global_access\": 0, \"share\": []}",
"id_perms": "{\"enable\": true, \"uuid\": {\"uuid_mslong\": 3988348619167975375, \"uuid_lslong\": 9873708133723548680}, \"creator\": null, \"created\": \"2017-05-03T22:54:58.134730\", \"user_visible\": true, \"last_modified\": \"2017-05-04T10:37:21.665399\", \"permissions\": {\"owner\": \"cloud-admin\", \"owner_access\": 7, \"other_access\": 7, \"group\": \"admin\", \"group_access\": 7}, \"description\": null}",
"display_name": "\"server5\"",
"bgp_router_refs": "[{\"to\": [\"default-domain\", \"default-project\", \"ip-fabric\", \"__default__\", \"server6\"], \"attr\": {\"session\": [{\"attributes\": [{\"passive\": false, \"admin_down\": false, \"private_as_action\": null, \"hold_time\": 0, \"bgp_router\": null, \"auth_data\": null, \"family_attributes\": [], \"as_override\": false, \"loop_count\": 0, \"address_families\": {\"family\": []}}], \"uuid\": null}]}, \"uuid\": \"9e48bb31-24f4-48a1-8b89-21243ed34dd0\"}, {\"to\": [\"default-domain\", \"default-project\", \"ip-fabric\", \"__default__\", \"server2\"], \"attr\": {\"session\": [{\"attributes\": [{\"passive\": false, \"admin_down\": false, \"private_as_action\": null, \"hold_time\": 0, \"bgp_router\": null, \"auth_data\": null, \"family_attributes\": [], \"as_override\": false, \"loop_count\": 0, \"address_families\": {\"family\": [\"route-target\", \"inet-vpn\", \"e-vpn\", \"erm-vpn\", \"inet6-vpn\"]}}], \"uuid\": null}]}, \"uuid\": \"b60ce400-7519-4376-bc53-5dbf50ce3f06\"}]"
},
"server6:Config:contrail-api:0"
]
]
},
"UVEAlarms": {
"alarms": [
{
"severity": 1,
"alarm_rules": {},
"timestamp": 1493894997301782,
"ack": false,
"token": "eyJ0aW1lc3RhbXAiOiAxNDkzODk0OTk3MzAxNzgyLCAiaHR0cF9wb3J0IjogNTk5NSwgImhvc3RfaXAiOiAiMTAuMTAuMC4xMSJ9",
"type": "default-global-system-config:system-defined-address-mismatch-control",
"description": "Control Node IP Address mismatch."
}
]
}
}
contrail-collector adds timestamp to every attribute in the UVE for aggregation and that seems to be causing this issue.
127.0.0.1:6381[1]> hgetall VALUES: ObjectBgpRouter :server2: server5: Config: contrail- api:0:ContrailC onfig \"1493894966005 530\">false< /deleted> " \"1493894966005 462\">< map key=\"string\" value=\"string\" size=\" 9\"><element> bgp_router_ parameters< /element> <element> {\"gateway_ address\ ": null, \"vendor\": \"contrail\", \"admin_down\": false, \"ipv6_ gateway_ address\ ": null, \"port\": 179, \"local_ autonomous_ system\ ": null, \"auth_data\": null, \"address\": \"10.10.0.5\", \"source_port\": null, \"router_type\": \"control-node\", \"identifier\": \"10.10.0.5\", \"hold_time\": 0, \"autonomous_ system\ ": 64512, \"address_ families\ ": {\"family\": [\"route-target\", \"inet-vpn\", \"e-vpn\", \"erm-vpn\", \"inet6- vpn\"]} }</element> <element> bgp_router_ refs</element> <element> [{\"to\ ": [\"default- domain\ ", \"default- project\ ", \"ip-fabric\", \"__default__\", \"server6\"], \"attr\": {\"session\": [{\"attributes\": [{\"passive\": false, \"admin_down\": false, \"private_ as_action\ ": null, \"hold_time\": 0, \"bgp_router\": null, \"auth_data\": null, \"family_ attributes\ ": [], \"as_override\": false, \"loop_count\": 0, \"address_ families\ ": {\"family\": []}}], \"uuid\": null}]}, \"uuid\": \"9e48bb31- 24f4-48a1- 8b89-21243ed34d d0\"}, {\"to\": [\"default- domain\ ", \"default- project\ ", \"ip-fabric\", \"__default__\", \"server5\"], \"attr\": {\"session\": [{\"attributes\": [{\"passive\": false, \"admin_down\": false, \"private_ as_action\ ": null, \"hold_time\": 0, \"bgp_router\": null, \"auth_data\": null, \"family_ attributes\ ": [], \"as_override\": false, \"loop_count\": 0, \"address_ families\ ": {\"family\": [\"route-target\", \"inet-vpn\", \"e-vpn\", \"erm-vpn\", \"inet6-vpn\"]}}], \"uuid\": null}]}, \"uuid\": \"375975f0- 1ded-47cf- 8906-753c067a2c 08\"}]< /element> <element> display_ name</element> <element> \"server2\ "</element> <element> fq_name< /element> <element> [\"default- domain\ ", \"default- project\ ", \"ip-fabric\", \"__default__\", \"server2\ "]</element> <element> id_perms< /element> <element> {\"enable\ ": true, \"uuid\": {\"uuid_mslong\": 131181105052391 35094, \"uuid_lslong\": 135702931784732 66950}, \"created\": \"2017- 05-03T22: 55:28.687268\ ", \"description\": null, \"creator\": null, \"user_visible\": true, \"last_modified\": \"2017- 05-04T10: 37:20.539238\ ", \"permissions\": {\"owner\": \"cloud-admin\", \"owner_access\": 7, \"other_access\": 7, \"group\": \"admin\", \"group_access\": 7}}</element> <element> parent_ type</element> <element> \"routing- instance\ "</element> <element> parent_ uuid</element> <element> \"2327986d- b0aa-4904- b99b-217cd72425 e3\"</element> <element> perms2< /element> <element> {\"owner\ ": \"cloud-admin\", \"owner_access\": 7, \"global_access\": 0, \"share\": []}</element> <element> uuid</element> <element> \"b60ce400- 7519-4376- bc53-5dbf50ce3f 06\"</element> </map>< /elements> "
1) "deleted"
2) "<deleted type=\"bool\" timestamp=
3) "elements"
4) "<elements type=\"map\" timestamp=
127.0.0.1:6381[1]> hgetall VALUES: ObjectBgpRouter :server2: server2: Config: contrail- api:0:ContrailC onfig \"1493894973759 524\">< map key=\"s...
1) "elements"
2) "<elements type=\"map\" timestamp=