The objects in error state are,
LI = default-global-system-config:QFX1:xe-0/0/2:xe-0/0/2.2001
VMI = default-domain:tky:7c73d281-705f-41d4-a7a3-2a7b4e2d705b
There is inconsistency between the ifmap output and the gcore file. IFMap output has adjacency between LI and VMI but not in the gcore.
Trace buffer 'IFMapAgentTrace' has following messages,
2015-07-03 09:06:31.124416 IFMapAgentTrace: seq = 1199 event = "Deleting Link between default-global-system-config:QFX1:xe-0/0/2:xe-0/0/2.2001default-domain:tky:7c73d281-705f-41d4-a7a3-2a7b4e2d705b" file = "controller/src/ifmap/ifmap_agent_table.cc" line = 605
2015-07-03 09:06:31.124419 IFMapAgentTrace: seq = 1199 event = "Deleting Link between default-global-system-config:QFX1:xe-0/0/2:xe-0/0/2.2001default-global-system-config:QFX1:xe-0/0/2" file = "controller/src/ifmap/ifmap_agent_table.cc" line = 605
The link between VMI and LI is deleted at 09:06:31 by stale timer. We dont have enough information to find why stale timer deleted the configuration.
XmppTraceBuffer contains logs from 08:50:00 to >10:00:00 and no new links between VMI and LI are received after this time. This is confirmed from DBGraph also. Dont have enough information to find why links got deleted by stale timer audit.
The vertex_id_ for interface is 0x7f69b000dbb0. This vertex does not have any adjacency to logical-interfaces.
IFMapNodeState for VMI is 0x7f69b000f730
IFMapNode for VMI is 0x7f69b000fba0
vertex_id_ for VMI is 0x7f69b000dbb0
The adjacencies are,
(gdb) p *(IFMapLink *) 0x7f69b000e4d0
$1 = (IFMapLink) {
<DBGraphEdge> = {
<DBEntry> = {
<DBEntryBase> = { _vptr.DBEntryBase = 0xe99bd0 <vtable for IFMapLink+16>,
chg_list_ = <boost::intrusive_hook> next = 0x0 prev = 0x0,
tpart_ = 0x7f6a2c0152b0,
state_ = std::map with 0 elements,
flags = 0 '\000',
onremoveq_ = (tbb::atomic) false, last_change_at_ = 1435307483974210
},
members of DBEntry:
node_ = <boost::intrusive_hook> parent = 0x7f6a2c015338 left = 0x7f69e801a4e0 right = 0x7f69c40a5b00
},
members of DBGraphEdge:
edge_id_ = {
<boost::detail::edge_base<boost::undirected_tag, void*>> = {
m_source = 0x7f69b000dbb0,
m_target = 0x7f6a00015180
},
members of boost::detail::edge_desc_impl<boost::undirected_tag, void*>:
m_eproperty = 0x7f69b000dd30
}
},
members of IFMapLink:
link_name_ = "virtual-machine-interface-security-group,virtual-machine-interface:default-domain:tky:7c73d281-705f-41d4-a7a3-2a7b4e2d705b,security-group:default-domain:tky:default",
metadata_ = "virtual-machine-interface-security-group",
left_id_ = {
first = "virtual-machine-interface",
second = "default-domain:tky:7c73d281-705f-41d4-a7a3-2a7b4e2d705b"
},
right_id_ = {
first = "security-group",
second = "default-domain:tky:default"
},
left_node_ = 0x7f69b000fba0,
right_node_ = 0x7f6a0001bb00,
origin_info_ = std::vector of length 1, capacity 1 = {{
origin = {
origin = IFMapOrigin::UNKNOWN
},
sequence_number = 1209
}}
}
(gdb) p *(IFMapLink *) 0x7f69b0010080
$2 = (IFMapLink) {
<DBGraphEdge> = {
<DBEntry> = {
<DBEntryBase> = { _vptr.DBEntryBase = 0xe99bd0 <vtable for IFMapLink+16>,
chg_list_ = <boost::intrusive_hook> next = 0x0 prev = 0x0,
tpart_ = 0x7f6a2c0152b0,
state_ = std::map with 0 elements,
flags = 0 '\000',
onremoveq_ = (tbb::atomic) false, last_change_at_ = 1435307483974240
},
members of DBEntry:
node_ = <boost::intrusive_hook> parent = 0x7f6a1c020f10 left = 0x0 right = 0x0
},
members of DBGraphEdge:
edge_id_ = {
<boost::detail::edge_base<boost::undirected_tag, void*>> = {
m_source = 0x7f69b000dbb0,
m_target = 0x7f6a1c020080
},
members of boost::detail::edge_desc_impl<boost::undirected_tag, void*>:
m_eproperty = 0x7f69b000e470
}
},
members of IFMapLink:
link_name_ = "virtual-machine-interface-virtual-network,virtual-machine-interface:default-domain:tky:7c73d281-705f-41d4-a7a3-2a7b4e2d705b,virtual-network:default-domain:tky:tky_192.168.10.0/24",
metadata_ = "virtual-machine-interface-virtual-network",
left_id_ = {
first = "virtual-machine-interface",
second = "default-domain:tky:7c73d281-705f-41d4-a7a3-2a7b4e2d705b"
},
right_id_ = {
first = "virtual-network",
second = "default-domain:tky:tky_192.168.10.0/24"
},
left_node_ = 0x7f69b000fba0,
right_node_ = 0x7f6a1c022730,
origin_info_ = std::vector of length 1, capacity 1 = {{
origin = {
origin = IFMapOrigin::UNKNOWN
},
sequence_number = 1209
}}
}
(gdb) p *(IFMapLink *) 0x7f69b0010180
$3 = (IFMapLink) {
<DBGraphEdge> = {
<DBEntry> = {
<DBEntryBase> = { _vptr.DBEntryBase = 0xe99bd0 <vtable for IFMapLink+16>,
chg_list_ = <boost::intrusive_hook> next = 0x0 prev = 0x0,
tpart_ = 0x7f6a2c0152b0,
state_ = std::map with 0 elements,
flags = 0 '\000',
onremoveq_ = (tbb::atomic) false, last_change_at_ = 1435307483974277
},
members of DBEntry:
node_ = <boost::intrusive_hook> parent = 0x7f6a00002a90 left = 0x0 right = 0x7f6a2c303c40
},
members of DBGraphEdge:
edge_id_ = {
<boost::detail::edge_base<boost::undirected_tag, void*>> = {
m_source = 0x7f69b000e870,
m_target = 0x7f69b000dbb0
},
members of boost::detail::edge_desc_impl<boost::undirected_tag, void*>:
m_eproperty = 0x7f69b000e100
}
},
members of IFMapLink:
link_name_ = "instance-ip-virtual-machine-interface,instance-ip:456d98a6-7f02-4812-998a-1f60d29b68c3,virtual-machine-interface:default-domain:tky:7c73d281-705f-41d4-a7a3-2a7b4e2d705b",
metadata_ = "instance-ip-virtual-machine-interface",
left_id_ = {
first = "instance-ip",
second = "456d98a6-7f02-4812-998a-1f60d29b68c3"
},
right_id_ = {
first = "virtual-machine-interface",
second = "default-domain:tky:7c73d281-705f-41d4-a7a3-2a7b4e2d705b"
},
left_node_ = 0x7f69b000dc00,
right_node_ = 0x7f69b000fba0,
origin_info_ = std::vector of length 1, capacity 1 = {{
origin = {
origin = IFMapOrigin::UNKNOWN
},
sequence_number = 1209
}}
}
(gdb) p *(IFMapLink *) 0x7f69b0010280
$4 = (IFMapLink) {
<DBGraphEdge> = {
<DBEntry> = {
<DBEntryBase> = { _vptr.DBEntryBase = 0xe99bd0 <vtable for IFMapLink+16>,
chg_list_ = <boost::intrusive_hook> next = 0x0 prev = 0x0,
tpart_ = 0x7f6a2c0152b0,
state_ = std::map with 0 elements,
flags = 0 '\000',
onremoveq_ = (tbb::atomic) false, last_change_at_ = 1435307483974319
},
members of DBEntry:
node_ = <boost::intrusive_hook> parent = 0x7f6a0001c2c0 left = 0x0 right = 0x7f6a0c018380
},
members of DBGraphEdge:
edge_id_ = {
<boost::detail::edge_base<boost::undirected_tag, void*>> = {
m_source = 0x7f69b000dbb0,
m_target = 0x7f69b000ed70
},
members of boost::detail::edge_desc_impl<boost::undirected_tag, void*>:
m_eproperty = 0x7f69b000ef70
}
},
members of IFMapLink:
link_name_ = "virtual-machine-interface-routing-instance,virtual-machine-interface-routing-instance:attr(default-domain:tky:7c73d281-705f-41d4-a7a3-2a7b4e2d705b,default-domain:tky:tky_192.168.10.0/24:tky_192.168.10"...,
metadata_ = "virtual-machine-interface-routing-instance",
left_id_ = {
first = "virtual-machine-interface",
second = "default-domain:tky:7c73d281-705f-41d4-a7a3-2a7b4e2d705b"
},
right_id_ = {
first = "virtual-machine-interface-routing-instance",
second = "attr(default-domain:tky:7c73d281-705f-41d4-a7a3-2a7b4e2d705b,default-domain:tky:tky_192.168.10.0/24:tky_192.168.10.0/24)"
},
left_node_ = 0x7f69b000fba0,
right_node_ = 0x7f69b000fea0,
origin_info_ = std::vector of length 1, capacity 1 = {{
origin = {
origin = IFMapOrigin::UNKNOWN
},
sequence_number = 1209
}}
}
Update for interface xe-0/0/2.2001
The objects in error state are, global- system- config: QFX1:xe- 0/0/2:xe- 0/0/2.2001 domain: tky:7c73d281- 705f-41d4- a7a3-2a7b4e2d70 5b
LI = default-
VMI = default-
There is inconsistency between the ifmap output and the gcore file. IFMap output has adjacency between LI and VMI but not in the gcore.
Trace buffer 'IFMapAgentTrace' has following messages,
2015-07-03 09:06:31.124416 IFMapAgentTrace: seq = 1199 event = "Deleting Link between default- global- system- config: QFX1:xe- 0/0/2:xe- 0/0/2.2001defau lt-domain: tky:7c73d281- 705f-41d4- a7a3-2a7b4e2d70 5b" file = "controller/ src/ifmap/ ifmap_agent_ table.cc" line = 605 global- system- config: QFX1:xe- 0/0/2:xe- 0/0/2.2001defau lt-global- system- config: QFX1:xe- 0/0/2" file = "controller/ src/ifmap/ ifmap_agent_ table.cc" line = 605
2015-07-03 09:06:31.124419 IFMapAgentTrace: seq = 1199 event = "Deleting Link between default-
The link between VMI and LI is deleted at 09:06:31 by stale timer. We dont have enough information to find why stale timer deleted the configuration.
XmppTraceBuffer contains logs from 08:50:00 to >10:00:00 and no new links between VMI and LI are received after this time. This is confirmed from DBGraph also. Dont have enough information to find why links got deleted by stale timer audit.
The vertex_id_ for interface is 0x7f69b000dbb0. This vertex does not have any adjacency to logical-interfaces.
IFMapNodeState for VMI is 0x7f69b000f730
IFMapNode for VMI is 0x7f69b000fba0
vertex_id_ for VMI is 0x7f69b000dbb0
The adjacencies are,
_vptr. DBEntryBase = 0xe99bd0 <vtable for IFMapLink+16>, :intrusive_ hook> next = 0x0 prev = 0x0,
last_change_ at_ = 1435307483974210 :intrusive_ hook> parent = 0x7f6a2c015338 left = 0x7f69e801a4e0 right = 0x7f69c40a5b00 boost:: detail: :edge_base< boost:: undirected_ tag, void*>> = { detail: :edge_desc_ impl<boost: :undirected_ tag, void*>: machine- interface- security- group,virtual- machine- interface: default- domain: tky:7c73d281- 705f-41d4- a7a3-2a7b4e2d70 5b,security- group:default- domain: tky:default" , machine- interface- security- group", machine- interface" , domain: tky:7c73d281- 705f-41d4- a7a3-2a7b4e2d70 5b" domain: tky:default" :UNKNOWN number = 1209
_vptr. DBEntryBase = 0xe99bd0 <vtable for IFMapLink+16>, :intrusive_ hook> next = 0x0 prev = 0x0,
last_change_ at_ = 1435307483974240 :intrusive_ hook> parent = 0x7f6a1c020f10 left = 0x0 right = 0x0 boost:: detail: :edge_base< boost:: undirected_ tag, void*>> = { detail: :edge_desc_ impl<boost: :undirected_ tag, void*>: machine- interface- virtual- network, virtual- machine- interface: default- domain: tky:7c73d281- 705f-41d4- a7a3-2a7b4e2d70 5b,virtual- network: default- domain: tky:tky_ 192.168. 10.0/24" , machine- interface- virtual- network" , machine- interface" , domain: tky:7c73d281- 705f-41d4- a7a3-2a7b4e2d70 5b" domain: tky:tky_ 192.168. 10.0/24" :UNKNOWN number = 1209
_vptr. DBEntryBase = 0xe99bd0 <vtable for IFMapLink+16>, :intrusive_ hook> next = 0x0 prev = 0x0,
last_change_ at_ = 1435307483974277 :intrusive_ hook> parent = 0x7f6a00002a90 left = 0x0 right = 0x7f6a2c303c40 boost:: detail: :edge_base< boost:: undirected_ tag, void*>> = { detail: :edge_desc_ impl<boost: :undirected_ tag, void*>: ip-virtual- machine- interface, instance- ip:456d98a6- 7f02-4812- 998a-1f60d29b68 c3,virtual- machine- interface: default- domain: tky:7c73d281- 705f-41d4- a7a3-2a7b4e2d70 5b", ip-virtual- machine- interface" , 7f02-4812- 998a-1f60d29b68 c3" machine- interface" , domain: tky:7c73d281- 705f-41d4- a7a3-2a7b4e2d70 5b" :UNKNOWN number = 1209
_vptr. DBEntryBase = 0xe99bd0 <vtable for IFMapLink+16>, :intrusive_ hook> next = 0x0 prev = 0x0,
last_change_ at_ = 1435307483974319 :intrusive_ hook> parent = 0x7f6a0001c2c0 left = 0x0 right = 0x7f6a0c018380 boost:: detail: :edge_base< boost:: undirected_ tag, void*>> = { detail: :edge_desc_ impl<boost: :undirected_ tag, void*>: machine- interface- routing- instance, virtual- machine- interface- routing- instance: attr(default- domain: tky:7c73d281- 705f-41d4- a7a3-2a7b4e2d70 5b,default- domain: tky:tky_ 192.168. 10.0/24: tky_192. 168.10" ..., machine- interface- routing- instance" , machine- interface" , domain: tky:7c73d281- 705f-41d4- a7a3-2a7b4e2d70 5b" machine- interface- routing- instance" , domain: tky:7c73d281- 705f-41d4- a7a3-2a7b4e2d70 5b,default- domain: tky:tky_ 192.168. 10.0/24: tky_192. 168.10. 0/24)" :UNKNOWN number = 1209
(gdb) p *(IFMapLink *) 0x7f69b000e4d0
$1 = (IFMapLink) {
<DBGraphEdge> = {
<DBEntry> = {
<DBEntryBase> = {
chg_list_ = <boost:
tpart_ = 0x7f6a2c0152b0,
state_ = std::map with 0 elements,
flags = 0 '\000',
onremoveq_ = (tbb::atomic) false,
},
members of DBEntry:
node_ = <boost:
},
members of DBGraphEdge:
edge_id_ = {
<
m_source = 0x7f69b000dbb0,
m_target = 0x7f6a00015180
},
members of boost::
m_eproperty = 0x7f69b000dd30
}
},
members of IFMapLink:
link_name_ = "virtual-
metadata_ = "virtual-
left_id_ = {
first = "virtual-
second = "default-
},
right_id_ = {
first = "security-group",
second = "default-
},
left_node_ = 0x7f69b000fba0,
right_node_ = 0x7f6a0001bb00,
origin_info_ = std::vector of length 1, capacity 1 = {{
origin = {
origin = IFMapOrigin:
},
sequence_
}}
}
(gdb) p *(IFMapLink *) 0x7f69b0010080
$2 = (IFMapLink) {
<DBGraphEdge> = {
<DBEntry> = {
<DBEntryBase> = {
chg_list_ = <boost:
tpart_ = 0x7f6a2c0152b0,
state_ = std::map with 0 elements,
flags = 0 '\000',
onremoveq_ = (tbb::atomic) false,
},
members of DBEntry:
node_ = <boost:
},
members of DBGraphEdge:
edge_id_ = {
<
m_source = 0x7f69b000dbb0,
m_target = 0x7f6a1c020080
},
members of boost::
m_eproperty = 0x7f69b000e470
}
},
members of IFMapLink:
link_name_ = "virtual-
metadata_ = "virtual-
left_id_ = {
first = "virtual-
second = "default-
},
right_id_ = {
first = "virtual-network",
second = "default-
},
left_node_ = 0x7f69b000fba0,
right_node_ = 0x7f6a1c022730,
origin_info_ = std::vector of length 1, capacity 1 = {{
origin = {
origin = IFMapOrigin:
},
sequence_
}}
}
(gdb) p *(IFMapLink *) 0x7f69b0010180
$3 = (IFMapLink) {
<DBGraphEdge> = {
<DBEntry> = {
<DBEntryBase> = {
chg_list_ = <boost:
tpart_ = 0x7f6a2c0152b0,
state_ = std::map with 0 elements,
flags = 0 '\000',
onremoveq_ = (tbb::atomic) false,
},
members of DBEntry:
node_ = <boost:
},
members of DBGraphEdge:
edge_id_ = {
<
m_source = 0x7f69b000e870,
m_target = 0x7f69b000dbb0
},
members of boost::
m_eproperty = 0x7f69b000e100
}
},
members of IFMapLink:
link_name_ = "instance-
metadata_ = "instance-
left_id_ = {
first = "instance-ip",
second = "456d98a6-
},
right_id_ = {
first = "virtual-
second = "default-
},
left_node_ = 0x7f69b000dc00,
right_node_ = 0x7f69b000fba0,
origin_info_ = std::vector of length 1, capacity 1 = {{
origin = {
origin = IFMapOrigin:
},
sequence_
}}
}
(gdb) p *(IFMapLink *) 0x7f69b0010280
$4 = (IFMapLink) {
<DBGraphEdge> = {
<DBEntry> = {
<DBEntryBase> = {
chg_list_ = <boost:
tpart_ = 0x7f6a2c0152b0,
state_ = std::map with 0 elements,
flags = 0 '\000',
onremoveq_ = (tbb::atomic) false,
},
members of DBEntry:
node_ = <boost:
},
members of DBGraphEdge:
edge_id_ = {
<
m_source = 0x7f69b000dbb0,
m_target = 0x7f69b000ed70
},
members of boost::
m_eproperty = 0x7f69b000ef70
}
},
members of IFMapLink:
link_name_ = "virtual-
metadata_ = "virtual-
left_id_ = {
first = "virtual-
second = "default-
},
right_id_ = {
first = "virtual-
second = "attr(default-
},
left_node_ = 0x7f69b000fba0,
right_node_ = 0x7f69b000fea0,
origin_info_ = std::vector of length 1, capacity 1 = {{
origin = {
origin = IFMapOrigin:
},
sequence_
}}
}