Config:
Server VM with the following fat flow config :
FatFlow rules:
1:* AggrDst 3880:e2d6:f8c0:739b::/64 125
Server VM on nodeh3:
150.254.212.3
3b61:81f7:da60:2e45::3
Client VMs on nodeh7:
166.173.157.11
3880:e2d6:f8c0:739b::b
3880:e2d6:f8c0:739b::3
166.173.157.3
Steps :
Start ICMPv6 traffic from client VMs to Server VM
Observation : Traffic drop and a flow in hold state on nodeh7
Logs:
Listing flows matching ([3b61:81f7:da60:2e45::3]:*, Protocol ICMPv6)
Index Source:Port/Destination:Port Proto(V)
-----------------------------------------------------------------------------------
87512 3880:e2d6:f8c0:739b:::0 58 (2)
3b61:81f7:da60:2e45::3:0
(Gen: 1, K(nh):22, Action:H, Flags:, QOS:0, S(nh):0, Stats:7/728, SPort 0,
TTL 0, Sinfo 7.7.7.7)
vif0/3 OS: tap8b5e3be0-71
Type:Virtual HWaddr:00:00:5e:00:01:00 IPaddr:150.254.212.3
IP6addr:3b61:81f7:da60:2e45::3
Vrf:2 Mcast Vrf:2 Flags:PL3L2DEr QOS:-1 Ref:6
RX packets:549 bytes:40094 errors:0
TX packets:525 bytes:55028 errors:0
Drops:92
FatFlow rules:
1:* AggrDst 3880:e2d6:f8c0:739b::/64 125
FatFlows IPv4 exclude prefix list:
150.254.212.1
150.254.212.2
169.254.0.0
FatFlows IPv6 exclude prefix list:
3b61:81f7:da60:2e45::1
3b61:81f7:da60:2e45::2
fe80:
Introspect:
2018-12-11 11:16:02.928 FlowTrace: operation = DEL info= [ gen_id = 1 flow_index = 282776 nh_id = 5 source_ip = 7.7.7.3 source_port = 8097 destination_ip = 169.254.0.3 destination_port = 51866 protocol = 6 vrf = 0 allow = 1 nat = 1 mirror_l= [ [ ] ] mirror_vrf = 65535 implicit_deny = 0 short_flow = 0 source_vn_list= [ [ (*_iter127) = default-domain:default-project:ip-fabric, ] ] dest_vn_list= [ [ (*_iter128) = default-domain:ctest-FatFlow-33315198:ctest-vn-78116912, ] ] source_vn_match = default-domain:default-project:ip-fabric dest_vn_match = default-domain:ctest-FatFlow-33315198:ctest-vn-78116912 source_sg_id_l= [ [ ] ] dest_sg_id_l= [ [ (*_iter130) = 8000040, ] ] vrf_assign = l3_flow = 1 smac = 00:00:5e:00:01:00 dmac = 02:8b:5e:3b:e0:71 drop_reason = UNKNOWN table_id = 0 short_flow_reason = UNKNOWN rpf_nh = 10 src_ip_nh = 10 ] file = controller/src/vnsw/agent/pkt/flow_mgmt.cc line = 632
2018-12-11 11:16:02.928 FlowTrace: operation = DEL info= [ gen_id = 1 flow_index = 179324 nh_id = 22 source_ip = 150.254.212.3 source_port = 51866 destination_ip = 169.254.169.254 destination_port = 80 protocol = 6 vrf = 2 allow = 1 nat = 1 mirror_l= [ [ ] ] mirror_vrf = 65535 implicit_deny = 0 short_flow = 1 source_vn_list= [ [ (*_iter127) = default-domain:ctest-FatFlow-33315198:ctest-vn-78116912, ] ] dest_vn_list= [ [ (*_iter128) = default-domain:default-project:ip-fabric, ] ] source_vn_match = default-domain:ctest-FatFlow-33315198:ctest-vn-78116912 dest_vn_match = default-domain:default-project:ip-fabric source_sg_id_l= [ [ (*_iter129) = 8000040, ] ] dest_sg_id_l= [ [ ] ] vrf_assign = l3_flow = 1 smac = 02:8b:5e:3b:e0:71 dmac = 00:00:5e:00:01:00 drop_reason = UNKNOWN table_id = 0 short_flow_reason = Short flow No Reverse flow rpf_nh = 22 src_ip_nh = 22 ] file = controller/src/vnsw/agent/pkt/flow_mgmt.cc line = 632
2018-12-11 11:16:02.928 FlowTrace: operation = DEL info= [ gen_id = 1 flow_index = 3936 nh_id = 5 source_ip = 7.7.7.3 source_port = 8097 destination_ip = 169.254.0.3 destination_port = 51867 protocol = 6 vrf = 0 allow = 1 nat = 1 mirror_l= [ [ ] ] mirror_vrf = 65535 implicit_deny = 0 short_flow = 0 source_vn_list= [ [ (*_iter127) = default-domain:default-project:ip-fabric, ] ] dest_vn_list= [ [ (*_iter128) = default-domain:ctest-FatFlow-33315198:ctest-vn-78116912, ] ] source_vn_match = default-domain:default-project:ip-fabric dest_vn_match = default-domain:ctest-FatFlow-33315198:ctest-vn-78116912 source_sg_id_l= [ [ ] ] dest_sg_id_l= [ [ (*_iter130) = 8000040, ] ] vrf_assign = l3_flow = 1 smac = 00:00:5e:00:01:00 dmac = 02:8b:5e:3b:e0:71 drop_reason = UNKNOWN table_id = 0 short_flow_reason = UNKNOWN rpf_nh = 10 src_ip_nh = 10 ] file = controller/src/vnsw/agent/pkt/flow_mgmt.cc line = 632
2018-12-11 11:16:02.929 FlowTrace: operation = DEL info= [ gen_id = 1 flow_index = 16768 nh_id = 22 source_ip = 150.254.212.3 source_port = 51867 destination_ip = 169.254.169.254 destination_port = 80 protocol = 6 vrf = 2 allow = 1 nat = 1 mirror_l= [ [ ] ] mirror_vrf = 65535 implicit_deny = 0 short_flow = 1 source_vn_list= [ [ (*_iter127) = default-domain:ctest-FatFlow-33315198:ctest-vn-78116912, ] ] dest_vn_list= [ [ (*_iter128) = default-domain:default-project:ip-fabric, ] ] source_vn_match = default-domain:ctest-FatFlow-33315198:ctest-vn-78116912 dest_vn_match = default-domain:default-project:ip-fabric source_sg_id_l= [ [ (*_iter129) = 8000040, ] ] dest_sg_id_l= [ [ ] ] vrf_assign = l3_flow = 1 smac = 02:8b:5e:3b:e0:71 dmac = 00:00:5e:00:01:00 drop_reason = UNKNOWN table_id = 0 short_flow_reason = Short flow No Reverse flow rpf_nh = 22 src_ip_nh = 22 ] file = controller/src/vnsw/agent/pkt/flow_mgmt.cc line = 632
2018-12-11 11:16:02.929 FlowTrace: operation = DEL info= [ gen_id = 1 flow_index = 35308 nh_id = 5 source_ip = 7.7.7.3 source_port = 8097 destination_ip = 169.254.0.3 destination_port = 51868 protocol = 6 vrf = 0 allow = 1 nat = 1 mirror_l= [ [ ] ] mirror_vrf = 65535 implicit_deny = 0 short_flow = 0 source_vn_list= [ [ (*_iter127) = default-domain:default-project:ip-fabric, ] ] dest_vn_list= [ [ (*_iter128) = default-domain:ctest-FatFlow-33315198:ctest-vn-78116912, ] ] source_vn_match = default-domain:default-project:ip-fabric dest_vn_match = default-domain:ctest-FatFlow-33315198:ctest-vn-78116912 source_sg_id_l= [ [ ] ] dest_sg_id_l= [ [ (*_iter130) = 8000040, ] ] vrf_assign = l3_flow = 1 smac = 00:00:5e:00:01:00 dmac = 02:8b:5e:3b:e0:71 drop_reason = UNKNOWN table_id = 0 short_flow_reason = UNKNOWN rpf_nh = 10 src_ip_nh = 10 ] file = controller/src/vnsw/agent/pkt/flow_mgmt.cc line = 632
2018-12-11 11:16:02.929 FlowTrace: operation = DEL info= [ gen_id = 2 flow_index = 517384 nh_id = 22 source_ip = 150.254.212.3 source_port = 51868 destination_ip = 169.254.169.254 destination_port = 80 protocol = 6 vrf = 2 allow = 1 nat = 1 mirror_l= [ [ ] ] mirror_vrf = 65535 implicit_deny = 0 short_flow = 1 source_vn_list= [ [ (*_iter127) = default-domain:ctest-FatFlow-33315198:ctest-vn-78116912, ] ] dest_vn_list= [ [ (*_iter128) = default-domain:default-project:ip-fabric, ] ] source_vn_match = default-domain:ctest-FatFlow-33315198:ctest-vn-78116912 dest_vn_match = default-domain:default-project:ip-fabric source_sg_id_l= [ [ (*_iter129) = 8000040, ] ] dest_sg_id_l= [ [ ] ] vrf_assign = l3_flow = 1 smac = 02:8b:5e:3b:e0:71 dmac = 00:00:5e:00:01:00 drop_reason = UNKNOWN table_id = 0 short_flow_reason = Short flow No Reverse flow rpf_nh = 22 src_ip_nh = 22 ] file = controller/src/vnsw/agent/pkt/flow_mgmt.cc line = 632
2018-12-11 11:16:02.935 FlowRouteUpdate: vrf = default-domain:ctest-FatFlow-33315198:ctest-vn-78116912:ctest-vn-78116912 ip = 150.254.212.3 plen = 32 vn_list= [ [ (*_iter212) = default-domain:ctest-FatFlow-33315198:ctest-vn-78116912, ] ] deleted = 0 marked_delete = 0 num_sg = 1 sg_list= [ [ (*_iter213) = 8000040, ] ] file = controller/src/vnsw/agent/pkt/flow_mgmt_dbclient.cc line = 497
2018-12-11 11:16:02.935 FlowRouteUpdate: vrf = default-domain:ctest-FatFlow-33315198:ctest-vn-78116912:ctest-vn-78116912 ip = 3b61:81f7:da60:2e45::3 plen = 128 vn_list= [ [ (*_iter212) = default-domain:ctest-FatFlow-33315198:ctest-vn-78116912, ] ] deleted = 0 marked_delete = 0 num_sg = 1 sg_list= [ [ (*_iter213) = 8000040, ] ] file = controller/src/vnsw/agent/pkt/flow_mgmt_dbclient.cc line = 497
2018-12-11 11:16:02.936 FlowRouteUpdate: vrf = default-domain:ctest-FatFlow-33315198:ctest-vn-78116912:ctest-vn-78116912 ip = ff:ff:ff:ff:ff:ff plen = 0 vn_list= [ [ (*_iter212) = default-domain:ctest-FatFlow-33315198:ctest-vn-78116912, ] ] deleted = 0 marked_delete = 0 num_sg = 0 sg_list= [ [ ] ] file = controller/src/vnsw/agent/pkt/flow_mgmt_dbclient.cc line = 510
2018-12-11 11:16:02.937 FlowRouteUpdate: vrf = default-domain:ctest-FatFlow-33315198:ctest-vn-78116912:ctest-vn-78116912 ip = 3b61:81f7:da60:2e45::3 plen = 128 vn_list= [ [ (*_iter212) = default-domain:ctest-FatFlow-33315198:ctest-vn-78116912, ] ] deleted = 0 marked_delete = 0 num_sg = 1 sg_list= [ [ (*_iter213) = 8000040, ] ] file = controller/src/vnsw/agent/pkt/flow_mgmt_dbclient.cc line = 497
2018-12-11 11:16:02.937 FlowTrace: operation = ADD info= [ gen_id = 1 flow_index = 512856 nh_id = 5 source_ip = 7.7.7.3 source_port = 8097 destination_ip = 169.254.0.3 destination_port = 51870 protocol = 6 vrf = 0 allow = 1 nat = 1 nat_source_ip = 169.254.169.254 nat_destination_ip = 150.254.212.3 nat_source_port = 80 nat_protocol = 6 nat_vrf = 2 nat_mirror_vrf = 65535 mirror_l= [ [ ] ] mirror_vrf = 65535 reverse_index = 279252 implicit_deny = 0 short_flow = 0 source_vn_list= [ [ (*_iter127) = default-domain:default-project:ip-fabric, ] ] dest_vn_list= [ [ (*_iter128) = default-domain:ctest-FatFlow-33315198:ctest-vn-78116912, ] ] source_vn_match = default-domain:default-project:ip-fabric dest_vn_match = default-domain:ctest-FatFlow-33315198:ctest-vn-78116912 source_sg_id_l= [ [ ] ] dest_sg_id_l= [ [ (*_iter130) = 8000040, ] ] vrf_assign = l3_flow = 1 smac = 00:00:5e:00:01:00 dmac = 02:8b:5e:3b:e0:71 drop_reason = UNKNOWN table_id = 0 short_flow_reason = UNKNOWN rpf_nh = 10 src_ip_nh = 10 ] file = controller/src/vnsw/agent/pkt/flow_mgmt.cc line = 632
2018-12-11 11:16:02.937 FlowTrace: operation = ADD info= [ gen_id = 1 flow_index = 279252 nh_id = 22 source_ip = 150.254.212.3 source_port = 51870 destination_ip = 169.254.169.254 destination_port = 80 protocol = 6 vrf = 2 allow = 1 nat = 1 nat_source_ip = 169.254.0.3 nat_destination_ip = 7.7.7.3 nat_destination_port = 8097 nat_protocol = 6 nat_vrf = 0 nat_mirror_vrf = 65535 mirror_l= [ [ ] ] mirror_vrf = 65535 reverse_index = 512856 implicit_deny = 0 short_flow = 0 source_vn_list= [ [ (*_iter127) = default-domain:ctest-FatFlow-33315198:ctest-vn-78116912, ] ] dest_vn_list= [ [ (*_iter128) = default-domain:default-project:ip-fabric, ] ] source_vn_match = default-domain:ctest-FatFlow-33315198:ctest-vn-78116912 dest_vn_match = default-domain:default-project:ip-fabric source_sg_id_l= [ [ (*_iter129) = 8000040, ] ] dest_sg_id_l= [ [ ] ] vrf_assign = l3_flow = 1 smac = 02:8b:5e:3b:e0:71 dmac = 00:00:5e:00:01:00 drop_reason = UNKNOWN table_id = 0 short_flow_reason = UNKNOWN rpf_nh = 22 src_ip_nh = 22 ] file = controller/src/vnsw/agent/pkt/flow_mgmt.cc line = 632
2018-12-11 11:16:02.938 FlowRouteUpdate: vrf = default-domain:ctest-FatFlow-33315198:ctest-vn-78116912:ctest-vn-78116912 ip = 02:8b:5e:3b:e0:71 plen = 0 vn_list= [ [ (*_iter212) = default-domain:ctest-FatFlow-33315198:ctest-vn-78116912, ] ] deleted = 0 marked_delete = 0 num_sg = 1 sg_list= [ [ (*_iter213) = 8000040, ] ] file = controller/src/vnsw/agent/pkt/flow_mgmt_dbclient.cc line = 510
2018-12-11 11:16:02.941 FlowRouteUpdate: vrf = default-domain:ctest-FatFlow-33315198:ctest-vn-78116912:ctest-vn-78116912 ip = 3b61:81f7:da60:2e45::3 plen = 128 vn_list= [ [ (*_iter212) = default-domain:ctest-FatFlow-33315198:ctest-vn-78116912, ] ] deleted = 0 marked_delete = 0 num_sg = 1 sg_list= [ [ (*_iter213) = 8000040, ] ] file = controller/src/vnsw/agent/pkt/flow_mgmt_dbclient.cc line = 497
2018-12-11 11:16:02.952 FlowTrace: operation = ADD info= [ gen_id = 1 flow_index = 220664 nh_id = 5 source_ip = 7.7.7.3 source_port = 8097 destination_ip = 169.254.0.3 destination_port = 51871 protocol = 6 vrf = 0 allow = 1 nat = 1 nat_source_ip = 169.254.169.254 nat_destination_ip = 150.254.212.3 nat_source_port = 80 nat_protocol = 6 nat_vrf = 2 nat_mirror_vrf = 65535 mirror_l= [ [ ] ] mirror_vrf = 65535 reverse_index = 371464 implicit_deny = 0 short_flow = 0 source_vn_list= [ [ (*_iter127) = default-domain:default-project:ip-fabric, ] ] dest_vn_list= [ [ (*_iter128) = default-domain:ctest-FatFlow-33315198:ctest-vn-78116912, ] ] source_vn_match = default-domain:default-project:ip-fabric dest_vn_match = default-domain:ctest-FatFlow-33315198:ctest-vn-78116912 source_sg_id_l= [ [ ] ] dest_sg_id_l= [ [ (*_iter130) = 8000040, ] ] vrf_assign = l3_flow = 1 smac = 00:00:5e:00:01:00 dmac = 02:8b:5e:3b:e0:71 drop_reason = UNKNOWN table_id = 0 short_flow_reason = UNKNOWN rpf_nh = 10 src_ip_nh = 10 ] file = controller/src/vnsw/agent/pkt/flow_mgmt.cc line = 632
2018-12-11 11:16:02.953 FlowTrace: operation = ADD info= [ gen_id = 1 flow_index = 371464 nh_id = 22 source_ip = 150.254.212.3 source_port = 51871 destination_ip = 169.254.169.254 destination_port = 80 protocol = 6 vrf = 2 allow = 1 nat = 1 nat_source_ip = 169.254.0.3 nat_destination_ip = 7.7.7.3 nat_destination_port = 8097 nat_protocol = 6 nat_vrf = 0 nat_mirror_vrf = 65535 mirror_l= [ [ ] ] mirror_vrf = 65535 reverse_index = 220664 implicit_deny = 0 short_flow = 0 source_vn_list= [ [ (*_iter127) = default-domain:ctest-FatFlow-33315198:ctest-vn-78116912, ] ] dest_vn_list= [ [ (*_iter128) = default-domain:default-project:ip-fabric, ] ] source_vn_match = default-domain:ctest-FatFlow-33315198:ctest-vn-78116912 dest_vn_match = default-domain:default-project:ip-fabric source_sg_id_l= [ [ (*_iter129) = 8000040, ] ] dest_sg_id_l= [ [ ] ] vrf_assign = l3_flow = 1 smac = 02:8b:5e:3b:e0:71 dmac = 00:00:5e:00:01:00 drop_reason = UNKNOWN table_id = 0 short_flow_reason = UNKNOWN rpf_nh = 22 src_ip_nh = 22 ] file = controller/src/vnsw/agent/pkt/flow_mgmt.cc line = 632
2018-12-11 11:16:02.964 FlowTrace: operation = ADD info= [ gen_id = 1 flow_index = 187004 nh_id = 5 source_ip = 7.7.7.3 source_port = 8097 destination_ip = 169.254.0.3 destination_port = 51872 protocol = 6 vrf = 0 allow = 1 nat = 1 nat_source_ip = 169.254.169.254 nat_destination_ip = 150.254.212.3 nat_source_port = 80 nat_protocol = 6 nat_vrf = 2 nat_mirror_vrf = 65535 mirror_l= [ [ ] ] mirror_vrf = 65535 reverse_index = 364208 implicit_deny = 0 short_flow = 0 source_vn_list= [ [ (*_iter127) = default-domain:default-project:ip-fabric, ] ] dest_vn_list= [ [ (*_iter128) = default-domain:ctest-FatFlow-33315198:ctest-vn-78116912, ] ] source_vn_match = default-domain:default-project:ip-fabric dest_vn_match = default-domain:ctest-FatFlow-33315198:ctest-vn-78116912 source_sg_id_l= [ [ ] ] dest_sg_id_l= [ [ (*_iter130) = 8000040, ] ] vrf_assign = l3_flow = 1 smac = 00:00:5e:00:01:00 dmac = 02:8b:5e:3b:e0:71 drop_reason = UNKNOWN table_id = 0 short_flow_reason = UNKNOWN rpf_nh = 10 src_ip_nh = 10 ] file = controller/src/vnsw/agent/pkt/flow_mgmt.cc line = 632
2018-12-11 11:16:02.964 FlowTrace: operation = ADD info= [ gen_id = 1 flow_index = 364208 nh_id = 22 source_ip = 150.254.212.3 source_port = 51872 destination_ip = 169.254.169.254 destination_port = 80 protocol = 6 vrf = 2 allow = 1 nat = 1 nat_source_ip = 169.254.0.3 nat_destination_ip = 7.7.7.3 nat_destination_port = 8097 nat_protocol = 6 nat_vrf = 0 nat_mirror_vrf = 65535 mirror_l= [ [ ] ] mirror_vrf = 65535 reverse_index = 187004 implicit_deny = 0 short_flow = 0 source_vn_list= [ [ (*_iter127) = default-domain:ctest-FatFlow-33315198:ctest-vn-78116912, ] ] dest_vn_list= [ [ (*_iter128) = default-domain:default-project:ip-fabric, ] ] source_vn_match = default-domain:ctest-FatFlow-33315198:ctest-vn-78116912 dest_vn_match = default-domain:default-project:ip-fabric source_sg_id_l= [ [ (*_iter129) = 8000040, ] ] dest_sg_id_l= [ [ ] ] vrf_assign = l3_flow = 1 smac = 02:8b:5e:3b:e0:71 dmac = 00:00:5e:00:01:00 drop_reason = UNKNOWN table_id = 0 short_flow_reason = UNKNOWN rpf_nh = 22 src_ip_nh = 22 ] file = controller/src/vnsw/agent/pkt/flow_mgmt.cc line = 632
2018-12-11 11:16:02.974 FlowTrace: operation = ADD info= [ gen_id = 0 flow_index = 4294967295 nh_id = 5 source_ip = 7.7.7.3 source_port = 8097 destination_ip = 169.254.0.3 destination_port = 51873 protocol = 6 vrf = 0 allow = 1 nat = 1 nat_source_ip = 169.254.169.254 nat_destination_ip = 150.254.212.3 nat_source_port = 80 nat_protocol = 6 nat_vrf = 2 nat_mirror_vrf = 65535 mirror_l= [ [ ] ] mirror_vrf = 65535 reverse_index = 269864 implicit_deny = 0 short_flow = 0 source_vn_list= [ [ (*_iter127) = default-domain:default-project:ip-fabric, ] ] dest_vn_list= [ [ (*_iter128) = default-domain:ctest-FatFlow-33315198:ctest-vn-78116912, ] ] source_vn_match = default-domain:default-project:ip-fabric dest_vn_match = default-domain:ctest-FatFlow-33315198:ctest-vn-78116912 source_sg_id_l= [ [ ] ] dest_sg_id_l= [ [ (*_iter130) = 8000040, ] ] vrf_assign = l3_flow = 1 smac = 00:00:5e:00:01:00 dmac = 02:8b:5e:3b:e0:71 drop_reason = UNKNOWN table_id = 0 short_flow_reason = UNKNOWN rpf_nh = 10 src_ip_nh = 10 ] file = controller/src/vnsw/agent/pkt/flow_mgmt.cc line = 632
