1. I dumped all VM and all host each other, and I found all icmp request and reply disappeared in host . request don't even reach the VM. 2. Only openvswitch_vswitchd helped to solve the problem. when we restarted neutron-openvswith_agent, but didn;t solve it. 3. I checked the openflow rules, but, I couldn't when issue happened. If it occur, I will check it. now there are no problem. troubled server : kdash-portal01 floating IP : 172.29.75.11 internal IP : 20.21.21.7 I checked the normal status. If problem occured, we will gather some info again. (virtenv) [root@2020c5lut005 ~]# openstack server show 1efccd39-68bd-4ec6-9f27-5a3604956cb8 +-------------------------------------+----------------------------------------------------------+ | Field | Value | +-------------------------------------+----------------------------------------------------------+ | OS-DCF:diskConfig | AUTO | | OS-EXT-AZ:availability_zone | dash_zone | | OS-EXT-SRV-ATTR:host | 2020c5lkt070 | | OS-EXT-SRV-ATTR:hypervisor_hostname | 2020c5lkt070 | | OS-EXT-SRV-ATTR:instance_name | instance-0000047f | | OS-EXT-STS:power_state | Running | | OS-EXT-STS:task_state | None | | OS-EXT-STS:vm_state | active | | OS-SRV-USG:launched_at | 2020-11-25T07:45:28.000000 | | OS-SRV-USG:terminated_at | None | | accessIPv4 | | | accessIPv6 | | | addresses | dash-network=20.21.21.75, 172.29.75.11 | | config_drive | True | | created | 2020-11-02T07:40:12Z | | flavor | c04r16os50 (33bf3602-ae10-4d2b-aeff-ba0500fb0ec3) | | hostId | 4aad83584416e1459b112b0ce665895c9eef3e8a541b2a244e924c60 | | id | 1efccd39-68bd-4ec6-9f27-5a3604956cb8 | | image | | | key_name | None | | name | kdash-portal01 | | progress | 0 | | project_id | e347a41cea154277867246edaba897c8 | | properties | | | security_groups | name='default' | | | name='dash-sg' | | status | ACTIVE | | updated | 2020-12-01T06:41:08Z | | user_id | fc210d00bf404509843ae1036747f2b1 | | volumes_attached | id='36673e51-dea6-4209-9cd8-64c4fcd95186' | +-------------------------------------+----------------------------------------------------------+ (virtenv) [root@2020c5lut005 ~]# openstack port list -f yaml | grep 20.21.21.75 -A 3 - ip_address: 20.21.21.75 subnet_id: aa7cdfdf-e74b-40e3-857c-a610800a79d7 ID: 70bb4570-2ef2-4fe2-b111-c9b2ca87bcd5 MAC Address: fa:16:3e:bd:aa:30 [root@2020c5lkt070 ~]# brctl show | grep 70b qbr70bb4570-2e 8000.8e0048a96171 no qvb70bb4570-2e tap70bb4570-2e [root@2020c5lkt070 ~]# iptables -S | grep 70bb -N neutron-openvswi-i70bb4570-2 -N neutron-openvswi-o70bb4570-2 -N neutron-openvswi-s70bb4570-2 -A neutron-openvswi-FORWARD -m physdev --physdev-out tap70bb4570-2e --physdev-is-bridged -m comment --comment "Direct traffic from the VM interface to the security group chain." -j neutron-openvswi-sg-chain -A neutron-openvswi-FORWARD -m physdev --physdev-in tap70bb4570-2e --physdev-is-bridged -m comment --comment "Direct traffic from the VM interface to the security group chain." -j neutron-openvswi-sg-chain -A neutron-openvswi-INPUT -m physdev --physdev-in tap70bb4570-2e --physdev-is-bridged -m comment --comment "Direct incoming traffic from VM to the security group chain." -j neutron-openvswi-o70bb4570-2 -A neutron-openvswi-i70bb4570-2 -m state --state RELATED,ESTABLISHED -m comment --comment "Direct packets associated with a known session to the RETURN chain." -j RETURN -A neutron-openvswi-i70bb4570-2 -d 20.21.21.75/32 -p udp -m udp --sport 67 --dport 68 -j RETURN -A neutron-openvswi-i70bb4570-2 -d 255.255.255.255/32 -p udp -m udp --sport 67 --dport 68 -j RETURN -A neutron-openvswi-i70bb4570-2 -p tcp -m tcp -m multiport --dports 1:65535 -j RETURN -A neutron-openvswi-i70bb4570-2 -p icmp -j RETURN -A neutron-openvswi-i70bb4570-2 -p udp -m udp -m multiport --dports 1:65535 -j RETURN -A neutron-openvswi-i70bb4570-2 -m set --match-set NIPv4200e55b8-e8de-45c1-b37e- src -j RETURN -A neutron-openvswi-i70bb4570-2 -m state --state INVALID -m comment --comment "Drop packets that appear related to an existing connection (e.g. TCP ACK/FIN) but do not have an entry in conntrack." -j DROP -A neutron-openvswi-i70bb4570-2 -m comment --comment "Send unmatched traffic to the fallback chain." -j neutron-openvswi-sg-fallback -A neutron-openvswi-o70bb4570-2 -s 0.0.0.0/32 -d 255.255.255.255/32 -p udp -m udp --sport 68 --dport 67 -m comment --comment "Allow DHCP client traffic." -j RETURN -A neutron-openvswi-o70bb4570-2 -j neutron-openvswi-s70bb4570-2 -A neutron-openvswi-o70bb4570-2 -p udp -m udp --sport 68 --dport 67 -m comment --comment "Allow DHCP client traffic." -j RETURN -A neutron-openvswi-o70bb4570-2 -p udp -m udp --sport 67 --dport 68 -m comment --comment "Prevent DHCP Spoofing by VM." -j DROP -A neutron-openvswi-o70bb4570-2 -m state --state RELATED,ESTABLISHED -m comment --comment "Direct packets associated with a known session to the RETURN chain." -j RETURN -A neutron-openvswi-o70bb4570-2 -j RETURN -A neutron-openvswi-o70bb4570-2 -m state --state INVALID -m comment --comment "Drop packets that appear related to an existing connection (e.g. TCP ACK/FIN) but do not have an entry in conntrack." -j DROP -A neutron-openvswi-o70bb4570-2 -m comment --comment "Send unmatched traffic to the fallback chain." -j neutron-openvswi-sg-fallback -A neutron-openvswi-s70bb4570-2 -s 20.21.21.75/32 -m mac --mac-source FA:16:3E:BD:AA:30 -m comment --comment "Allow traffic from defined IP/MAC pairs." -j RETURN -A neutron-openvswi-s70bb4570-2 -m comment --comment "Drop traffic without an IP/MAC allow rule." -j DROP -A neutron-openvswi-sg-chain -m physdev --physdev-out tap70bb4570-2e --physdev-is-bridged -m comment --comment "Jump to the VM specific chain." -j neutron-openvswi-i70bb4570-2 -A neutron-openvswi-sg-chain -m physdev --physdev-in tap70bb4570-2e --physdev-is-bridged -m comment --comment "Jump to the VM specific chain." -j neutron-openvswi-o70bb4570-2 checked the qvo [root@2020c5lkt070 ~]# docker exec -uroot -it neutron_openvswitch_agent ovs-vsctl show | grep 70b Port "qvo70bb4570-2e" Interface "qvo70bb4570-2e" and checked the qr [root@2020c5lkt070 ~]# docker exec -uroot -it neutron_openvswitch_agent ovs-vsctl show | grep qr Port "qr-df9d9aae-0c" tag: 8 Interface "qr-df9d9aae-0c" type: internal we installed by kolla [root@2020c5lkt070 ~]# docker exec -it openvswitch_vswitchd ls -a /var/run/openvswitch/ovs-vswitchd.*.ctl /var/run/openvswitch/ovs-vswitchd.17.ctl /var/run/openvswitch/ovs-vswitchd.18.ctl [root@2020c5lkt070 ~]# docker exec -it openvswitch_vswitchd ovs-appctl -t /var/run/openvswitch/ovs-vswitchd.17.ctl fdb/show br-int port VLAN MAC Age 87 8 fa:16:3e:bd:aa:30 0 [root@2020c5lkt070 ~]# docker exec -it openvswitch_vswitchd ovs-ofctl show br-int 87(qvo70bb4570-2e): addr:1a:c4:f9:ed:a7:db config: 0 state: 0 current: 10GB-FD COPPER speed: 10000 Mbps now, 0 Mbps max [root@2020c5lkt070 ~]# ip netns fip-319c89cc-3cff-4a83-9c49-1fadf6e9e05c (id: 3) qrouter-877b3a4b-c414-4081-86ee-43f616e643b8 (id: 1) [root@2020c5lkt070 ~]# ip netns exec qrouter-877b3a4b-c414-4081-86ee-43f616e643b8 ip a 1: lo: mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000 link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 inet 127.0.0.1/8 scope host lo valid_lft forever preferred_lft forever inet6 ::1/128 scope host valid_lft forever preferred_lft forever 2: rfp-877b3a4b-c@if2: mtu 9000 qdisc noqueue state UP group default qlen 1000 link/ether f6:98:1c:89:35:19 brd ff:ff:ff:ff:ff:ff link-netnsid 0 inet 169.254.93.94/31 scope global rfp-877b3a4b-c valid_lft forever preferred_lft forever inet6 fe80::f498:1cff:fe89:3519/64 scope link valid_lft forever preferred_lft forever 238: qr-df9d9aae-0c: mtu 8950 qdisc noqueue state UNKNOWN group default qlen 1000 link/ether fa:16:3e:f5:27:75 brd ff:ff:ff:ff:ff:ff inet 20.21.21.1/24 brd 20.21.21.255 scope global qr-df9d9aae-0c valid_lft forever preferred_lft forever inet6 fe80::f816:3eff:fef5:2775/64 scope link valid_lft forever preferred_lft forever [root@2020c5lkt070 ~]# ip netns exec qrouter-877b3a4b-c414-4081-86ee-43f616e643b8 route -n Kernel IP routing table Destination Gateway Genmask Flags Metric Ref Use Iface 20.21.21.0 0.0.0.0 255.255.255.0 U 0 0 0 qr-df9d9aae-0c 169.254.93.94 0.0.0.0 255.255.255.254 U 0 0 0 rfp-877b3a4b-c [root@2020c5lkt070 ~]# ip netns exec qrouter-877b3a4b-c414-4081-86ee-43f616e643b8 ip rule ls 0: from all lookup local 32766: from all lookup main 32767: from all lookup default 46137: from 20.21.21.75 lookup 16 336925953: from 20.21.21.1/24 lookup 336925953 [root@2020c5lkt070 ~]# ip netns exec fip-319c89cc-3cff-4a83-9c49-1fadf6e9e05c ip a 1: lo: mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000 link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 inet 127.0.0.1/8 scope host lo valid_lft forever preferred_lft forever inet6 ::1/128 scope host valid_lft forever preferred_lft forever 2: fpr-877b3a4b-c@if2: mtu 9000 qdisc noqueue state UP group default qlen 1000 link/ether 2a:2c:ca:85:8e:4e brd ff:ff:ff:ff:ff:ff link-netnsid 0 inet 169.254.93.95/31 scope global fpr-877b3a4b-c valid_lft forever preferred_lft forever inet6 fe80::282c:caff:fe85:8e4e/64 scope link valid_lft forever preferred_lft forever 241: fg-cd25bd41-5d: mtu 9000 qdisc noqueue state UNKNOWN group default qlen 1000 link/ether fa:16:3e:78:f3:17 brd ff:ff:ff:ff:ff:ff inet 172.29.80.47/24 brd 172.29.80.255 scope global fg-cd25bd41-5d valid_lft forever preferred_lft forever inet6 fe80::f816:3eff:fe78:f317/64 scope link valid_lft forever preferred_lft forever [root@2020c5lkt070 ~]# ip netns exec fip-319c89cc-3cff-4a83-9c49-1fadf6e9e05c ip rule 0: from all lookup local 32766: from all lookup main 32767: from all lookup default 2852019551: from all iif fpr-877b3a4b-c lookup 2852019551 [root@2020c5lkt070 ~]# ip netns exec fip-319c89cc-3cff-4a83-9c49-1fadf6e9e05c ip route ls table 2852019551 default via 172.29.80.1 dev fg-cd25bd41-5d proto static [root@2020c5lut006 ~]# ip netns | awk '{print "ip netns exec "$1 " arp -a"}' | sh | grep 172.29.80.1 ? (172.29.80.1) at 00:00:5e:00:01:6e [ether] on qg-06bfdc74-36 gateway (172.29.80.1) at 00:00:5e:00:01:6e [ether] on qg-22dcd86a-c8 gateway (172.29.80.1) at 00:00:5e:00:01:6e [ether] on qg-a1058ca2-47 gateway (172.29.80.1) at 00:00:5e:00:01:6e [ether] on qg-a56e88c6-90 gateway (172.29.80.1) at 00:00:5e:00:01:6e [ether] on qg-b8578f0f-b3 gateway (172.29.80.1) at 00:00:5e:00:01:6e [ether] on qg-c7fe5cfc-a1 gateway (172.29.80.1) at 00:00:5e:00:01:6e [ether] on qg-2bc84656-56 gateway (172.29.80.1) at 00:00:5e:00:01:6e [ether] on qg-c5352319-58 [root@2020c5lkt070 ~]# docker exec -it openvswitch_vswitchd ovs-appctl -t /var/run/openvswitch/ovs-vswitchd.17.ctl fdb/show br-int | grep 00:00:5e:00:01:6e 2 2 00:00:5e:00:01:6e 0 [root@2020c5lkt070 ~]# docker exec -it openvswitch_vswitchd ovs-ofctl show br-int OFPT_FEATURES_REPLY (xid=0x2): dpid:00000acb3a0b6443 n_tables:254, n_buffers:0 capabilities: FLOW_STATS TABLE_STATS PORT_STATS QUEUE_STATS ARP_MATCH_IP actions: output enqueue set_vlan_vid set_vlan_pcp strip_vlan mod_dl_src mod_dl_dst mod_nw_src mod_nw_dst mod_nw_tos mod_tp_src mod_tp_dst 1(int-br-cephfs): addr:ea:5d:bc:46:0b:36 config: 0 state: 0 speed: 0 Mbps now, 0 Mbps max 2(int-br-ex): addr:06:33:16:a8:df:03 config: 0 state: 0 speed: 0 Mbps now, 0 Mbps max [root@2020c5lkt070 ~]# docker exec -it openvswitch_vswitchd ovs-vsctl show ec1990ff-f429-474c-9b6c-fc6a5cef406a Manager "ptcp:6640:127.0.0.1" is_connected: true Bridge br-int Controller "tcp:127.0.0.1:6633" is_connected: true fail_mode: secure datapath_type: system Port "qvo1e677f09-5a" tag: 8 Interface "qvo1e677f09-5a" Port "qvo6e72f871-7f" tag: 8 Interface "qvo6e72f871-7f" Port "qvo6091f829-14" tag: 8 Interface "qvo6091f829-14" Port "qvof21eb7ea-89" tag: 8 Interface "qvof21eb7ea-89" Port "qvobd7f6057-f5" tag: 8 Interface "qvobd7f6057-f5" Port int-br-api Interface int-br-api type: patch options: {peer=phy-br-api} Port "qr-df9d9aae-0c" tag: 8 Interface "qr-df9d9aae-0c" type: internal Port br-int Interface br-int type: internal Port "qvo70bb4570-2e" tag: 8 Interface "qvo70bb4570-2e" Port int-br-ex Interface int-br-ex type: patch options: {peer=phy-br-ex} Port "qvo812ad92d-07" tag: 8 Interface "qvo812ad92d-07" Port int-br-cephfs Interface int-br-cephfs type: patch options: {peer=phy-br-cephfs} Port "qvo3538c0cf-2d" tag: 8 Interface "qvo3538c0cf-2d" Port "qvo18cc8245-09" tag: 8 Interface "qvo18cc8245-09" Bridge br-ex Controller "tcp:127.0.0.1:6633" is_connected: true fail_mode: secure datapath_type: system Port phy-br-ex Interface phy-br-ex type: patch options: {peer=int-br-ex} Port br-ex Interface br-ex type: internal Port "bond_serv.110" Interface "bond_serv.110" Bridge br-tun Controller "tcp:127.0.0.1:6633" is_connected: true fail_mode: secure datapath_type: system Port "vxlan-14150250" Interface "vxlan-14150250" type: vxlan options: {df_default="true", egress_pkt_mark="0", in_key=flow, local_ip="20.21.2.87", out_key=flow, remote_ip="20.21.2.80"} Port "vxlan-1415025c" Interface "vxlan-1415025c" type: vxlan options: {df_default="true", egress_pkt_mark="0", in_key=flow, local_ip="20.21.2.87", out_key=flow, remote_ip="20.21.2.92"} Port br-tun Interface br-tun type: internal Port "vxlan-14150243" Interface "vxlan-14150243" type: vxlan options: {df_default="true", egress_pkt_mark="0", in_key=flow, local_ip="20.21.2.87", out_key=flow, remote_ip="20.21.2.67"} Port "vxlan-14150266" Interface "vxlan-14150266" type: vxlan options: {df_default="true", egress_pkt_mark="0", in_key=flow, local_ip="20.21.2.87", out_key=flow, remote_ip="20.21.2.102"} Port patch-int Interface patch-int type: patch options: {peer=patch-tun} Port "vxlan-14150235" Interface "vxlan-14150235" type: vxlan options: {df_default="true", egress_pkt_mark="0", in_key=flow, local_ip="20.21.2.87", out_key=flow, remote_ip="20.21.2.53"} Port "vxlan-14150265" Interface "vxlan-14150265" type: vxlan options: {df_default="true", egress_pkt_mark="0", in_key=flow, local_ip="20.21.2.87", out_key=flow, remote_ip="20.21.2.101"} Port "vxlan-1415024f" Interface "vxlan-1415024f" type: vxlan options: {df_default="true", egress_pkt_mark="0", in_key=flow, local_ip="20.21.2.87", out_key=flow, remote_ip="20.21.2.79"} Port "vxlan-1415023a" Interface "vxlan-1415023a" type: vxlan options: {df_default="true", egress_pkt_mark="0", in_key=flow, local_ip="20.21.2.87", out_key=flow, remote_ip="20.21.2.58"} Port "vxlan-14150267" Interface "vxlan-14150267" type: vxlan options: {df_default="true", egress_pkt_mark="0", in_key=flow, local_ip="20.21.2.87", out_key=flow, remote_ip="20.21.2.103"} Port "vxlan-1415025d" Interface "vxlan-1415025d" type: vxlan options: {df_default="true", egress_pkt_mark="0", in_key=flow, local_ip="20.21.2.87", out_key=flow, remote_ip="20.21.2.93"} Bridge br-api Controller "tcp:127.0.0.1:6633" is_connected: true fail_mode: secure datapath_type: system Port "bond_api.101" Interface "bond_api.101" Port phy-br-api Interface phy-br-api type: patch options: {peer=int-br-api} Port br-api Interface br-api type: internal Bridge br-cephfs Controller "tcp:127.0.0.1:6633" is_connected: true fail_mode: secure datapath_type: system Port veth_ovs Interface veth_ovs Port phy-br-cephfs Interface phy-br-cephfs type: patch options: {peer=int-br-cephfs} Port br-cephfs Interface br-cephfs type: internal