In some scenarios, the VM needs access to its own Floating IP when tenant network type is Geneve. In my production environment, VMs cannot connect their own FIPs, and they also cannot connect other VMs on the same compute node through FIPs. Using ovn-trace to track lfows ,it shows that no lfow matched after SNAT. On different compute nodes network works fine through FIP.
I add an OVS flow on compute node like this:
ovs-ofctl add-flow br-int "table=8,reg14=0x1,metadata=0x12,dl_src=fa:16:3e:85:0e:cb actions=resubmit(,9)"
then the FIP(10.0.41.228) works fine.
Verions:
openstack: victoria
ovn-northd: 20.03.2
ovn-sbctl: 20.03.2
ovn-controller: 20.03.2
Open vSwitch Library: 2.13.3
OpenFlow versions: 0x4:0x4
neutron-server: 17.1.3.dev4
Network information:
+--------+-------------------+--------------------+---------------------------------------+---------------------------------------+--------------------------------------+------------+--------------------------------------+------------+
|type | IP | MAC | neutron port ID | neutron network ID | OVN Port_Binding UUID |tunnel_key | OVN Datapath_Binding UUID |tunnel_key |
+--------+-------------------+--------------------+---------------------------------------+---------------------------------------+--------------------------------------+------------+--------------------------------------+------------+
|VM NIC | 172.20.25.162/24 | fa:16:3e:30:4b:85 | 103e7777-a376-419d-9629-31bc6c7fedb1 | fb1dc9cb-df50-433a-9388-89dc31e47d99 | 18c93367-0944-4572-a601-67ae9cf13198 | 0x1e | 3eb22556-9dea-46bd-972d-9097252927c9 | 0x12 |
+--------+-------------------+--------------------+---------------------------------------+---------------------------------------+--------------------------------------+------------+--------------------------------------+------------+
|VM GW | 172.20.25.1/24 | fa:16:3e:9c:8b:f3 | 039ba2bc-997c-448a-adb7-54bd3af697c1 | fb1dc9cb-df50-433a-9388-89dc31e47d99 | fded7818-5a70-46a2-84cd-b88b9b82ea56 | 0x11 | 3eb22556-9dea-46bd-972d-9097252927c9 | 0x12 |
+--------+-------------------+--------------------+---------------------------------------+---------------------------------------+--------------------------------------+------------+--------------------------------------+------------+
|VM FIP | 10.0.41.228/24 | fa:16:3e:85:0e:cb | 7266cfce-f20d-4281-872d-b89f79394467 | d32cbd3e-15ba-487e-82fc-23646d7b3d91 | 18c93367-0944-4572-a601-67ae9cf13198 | 0x1e | c01f5fca-0b89-42a3-8d15-f9da96509496 | 0x3 |
+--------+-------------------+--------------------+---------------------------------------+---------------------------------------+--------------------------------------+------------+--------------------------------------+------------+
|FIP GW | 10.0.40.1/24 | b4:09:31:47:30:ef | N/A | d32cbd3e-15ba-487e-82fc-23646d7b3d91 | N/A | N/A | c01f5fca-0b89-42a3-8d15-f9da96509496 | 0x3 |
+--------+-------------------+--------------------+---------------------------------------+---------------------------------------+--------------------------------------+------------+--------------------------------------+------------+
|router | 10.0.40.233/24 | fa:16:3e:4a:e3:b5 | e9e8c0cb-2e81-4d20-9c28-44962ad1cbb9 | d32cbd3e-15ba-487e-82fc-23646d7b3d91 | 883bca93-6db8-44df-9ff6-3f66b1798d95 | 0x1 | c01f5fca-0b89-42a3-8d15-f9da96509496 | 0x3 |
|ext_port| | | | | | | | |
+--------+-------------------+--------------------+---------------------------------------+---------------------------------------+--------------------------------------+------------+--------------------------------------+------------+
ovn-trace output:
(ovn-controller)# ovn-trace --db=tcp:172.16.xx.xx:6642,tcp:172.16.xx.yy:6642,tcp:172.16.xx.zz:6642 --no-friendly-names --ovs neutron-fb1dc9cb-df50-433a-9388-89dc31e47d99 'inport == "18c93367-0944-4572-a601-67ae9cf13198" && eth.src == fa:16:3e:30:4b:85 && eth.dst == fa:16:3e:9c:8b:f3 && ip4.src == 172.20.25.162 && ip4.dst == 10.0.41.228 && ip.ttl == 64 && icmp4.type == 8'
# icmp,reg14=0x1e,vlan_tci=0x0000,dl_src=fa:16:3e:30:4b:85,dl_dst=fa:16:3e:9c:8b:f3,nw_src=172.20.25.162,nw_dst=10.0.41.228,nw_tos=0,nw_ecn=0,nw_ttl=64,icmp_type=8,icmp_code=0
ingress(dp="af17a9c0-b600-407f-880f-535dfab2877d", inport="103e7777-a376-419d-9629-31bc6c7fedb1")
-------------------------------------------------------------------------------------------------
0. ls_in_port_sec_l2 (ovn-northd.c:4514): inport == "103e7777-a376-419d-9629-31bc6c7fedb1" && eth.src == {fa:16:3e:30:4b:85}, priority 50, uuid 36c79b1c
cookie=0x36c79b1c, duration=74244.455s, table=8, n_packets=288859, n_bytes=936110898, priority=50,reg14=0x1e,metadata=0xc,dl_src=fa:16:3e:30:4b:85 actions=resubmit(,9)
next;
1. ls_in_port_sec_ip (ovn-northd.c:4186): inport == "103e7777-a376-419d-9629-31bc6c7fedb1" && eth.src == fa:16:3e:30:4b:85 && ip4.src == {172.20.25.162}, priority 90, uuid c58a60a0
cookie=0xc58a60a0, duration=74244.323s, table=9, n_packets=287956, n_bytes=936072972, priority=90,ip,reg14=0x1e,metadata=0xc,dl_src=fa:16:3e:30:4b:85,nw_src=172.20.25.162 actions=resubmit(,10)
next;
3. ls_in_pre_acl (ovn-northd.c:4703): ip, priority 100, uuid 38b64b35
cookie=0x38b64b35, duration=74244.434s, table=11, n_packets=456863, n_bytes=954013905, priority=100,ip,metadata=0xc actions=load:0x1->NXM_NX_XXREG0[96],resubmit(,12)
cookie=0x38b64b35, duration=74244.380s, table=11, n_packets=0, n_bytes=0, priority=100,ipv6,metadata=0xc actions=load:0x1->NXM_NX_XXREG0[96],resubmit(,12)
reg0[0] = 1;
next;
5. ls_in_pre_stateful (ovn-northd.c:4890): reg0[0] == 1, priority 100, uuid dcd05000
cookie=0xdcd05000, duration=74244.366s, table=13, n_packets=0, n_bytes=0, priority=100,ipv6,reg0=0x1/0x1,metadata=0xc actions=ct(table=14,zone=NXM_NX_REG13[0..15])
cookie=0xdcd05000, duration=74244.334s, table=13, n_packets=456863, n_bytes=954013905, priority=100,ip,reg0=0x1/0x1,metadata=0xc actions=ct(table=14,zone=NXM_NX_REG13[0..15])
ct_next;
ct_next(ct_state=est|trk /* default (use --ct to customize) */)
---------------------------------------------------------------
6. ls_in_acl (ovn-northd.c:5083): (!ct.trk || (!ct.new && ct.est && !ct.rpl && ct_label.blocked == 0)) && (inport == @pg_79068b7d_a57d_4f24_b0ed_5d689d6a6859 && ip4), priority 2002, uuid e2c75b95
cookie=0xe2c75b95, duration=74244.497s, table=14, n_packets=0, n_bytes=0, priority=2002,ct_state=-trk,ip,reg14=0x22,metadata=0xc actions=resubmit(,15)
cookie=0xe2c75b95, duration=74244.270s, table=14, n_packets=166679, n_bytes=16622180, priority=2002,ct_state=-new+est-rpl+trk,ct_label=0/0x1,ip,reg14=0x1e,metadata=0xc actions=resubmit(,15)
cookie=0xe2c75b95, duration=74244.283s, table=14, n_packets=4456, n_bytes=361337, priority=2002,ct_state=-new+est-rpl+trk,ct_label=0/0x1,ip,reg14=0x22,metadata=0xc actions=resubmit(,15)
cookie=0xe2c75b95, duration=74244.304s, table=14, n_packets=0, n_bytes=0, priority=2002,ct_state=-trk,ip,reg14=0x13,metadata=0xc actions=resubmit(,15)
cookie=0xe2c75b95, duration=74244.371s, table=14, n_packets=0, n_bytes=0, priority=2002,ct_state=-trk,ip,reg14=0x1e,metadata=0xc actions=resubmit(,15)
cookie=0xe2c75b95, duration=74244.380s, table=14, n_packets=3672, n_bytes=555588, priority=2002,ct_state=-new+est-rpl+trk,ct_label=0/0x1,ip,reg14=0x28,metadata=0xc actions=resubmit(,15)
cookie=0xe2c75b95, duration=74244.420s, table=14, n_packets=3350, n_bytes=307439, priority=2002,ct_state=-new+est-rpl+trk,ct_label=0/0x1,ip,reg14=0x13,metadata=0xc actions=resubmit(,15)
cookie=0xe2c75b95, duration=74244.438s, table=14, n_packets=114262, n_bytes=11563996, priority=2002,ct_state=-new+est-rpl+trk,ct_label=0/0x1,ip,reg14=0x23,metadata=0xc actions=resubmit(,15)
cookie=0xe2c75b95, duration=74244.451s, table=14, n_packets=0, n_bytes=0, priority=2002,ct_state=-trk,ip,reg14=0x28,metadata=0xc actions=resubmit(,15)
cookie=0xe2c75b95, duration=74244.488s, table=14, n_packets=0, n_bytes=0, priority=2002,ct_state=-trk,ip,reg14=0x23,metadata=0xc actions=resubmit(,15)
next;
19. ls_in_l2_lkup (ovn-northd.c:6841): eth.dst == fa:16:3e:9c:8b:f3, priority 50, uuid 325f29ce
cookie=0x325f29ce, duration=74244.407s, table=27, n_packets=456618, n_bytes=953953284, priority=50,metadata=0xc,dl_dst=fa:16:3e:9c:8b:f3 actions=set_field:0xb->reg15,resubmit(,32)
outport = "039ba2bc-997c-448a-adb7-54bd3af697c1";
output;
egress(dp="af17a9c0-b600-407f-880f-535dfab2877d", inport="103e7777-a376-419d-9629-31bc6c7fedb1", outport="039ba2bc-997c-448a-adb7-54bd3af697c1")
------------------------------------------------------------------------------------------------------------------------------------------------
1. ls_out_pre_acl (ovn-northd.c:4658): ip && outport == "039ba2bc-997c-448a-adb7-54bd3af697c1", priority 110, uuid 4dba6a66
cookie=0x4dba6a66, duration=74244.407s, table=41, n_packets=456824, n_bytes=954006123, priority=110,ip,reg15=0xb,metadata=0xc actions=resubmit(,42)
cookie=0x4dba6a66, duration=74244.346s, table=41, n_packets=0, n_bytes=0, priority=110,ipv6,reg15=0xb,metadata=0xc actions=resubmit(,42)
next;
9. ls_out_port_sec_l2 (ovn-northd.c:4580): outport == "039ba2bc-997c-448a-adb7-54bd3af697c1", priority 50, uuid ea1250eb
cookie=0xea1250eb, duration=74244.492s, table=49, n_packets=456618, n_bytes=953953284, priority=50,reg15=0xb,metadata=0xc actions=resubmit(,64)
output;
/* output to "039ba2bc-997c-448a-adb7-54bd3af697c1", type "patch" */
ingress(dp="32d08999-d00a-49d2-bf8d-9169894b2a45", inport="lrp-039ba2bc-997c-448a-adb7-54bd3af697c1")
-----------------------------------------------------------------------------------------------------
0. lr_in_admission (ovn-northd.c:7836): eth.dst == fa:16:3e:9c:8b:f3 && inport == "lrp-039ba2bc-997c-448a-adb7-54bd3af697c1", priority 50, uuid 9a133c69
cookie=0x9a133c69, duration=74244.305s, table=8, n_packets=456618, n_bytes=953953284, priority=50,reg14=0x3,metadata=0x12,dl_dst=fa:16:3e:9c:8b:f3 actions=resubmit(,9)
next;
1. lr_in_lookup_neighbor (ovn-northd.c:7885): 1, priority 0, uuid 2d7be36f
cookie=0x2d7be36f, duration=74244.478s, table=9, n_packets=666756, n_bytes=1140691558, priority=0,metadata=0x12 actions=load:0x1->OXM_OF_PKT_REG4[3],resubmit(,10)
reg9[3] = 1;
next;
2. lr_in_learn_neighbor (ovn-northd.c:7890): reg9[3] == 1 || reg9[2] == 1, priority 100, uuid 664e0889
cookie=0x664e0889, duration=74244.321s, table=10, n_packets=0, n_bytes=0, priority=100,reg9=0x4/0x4,metadata=0x12 actions=resubmit(,11)
cookie=0x664e0889, duration=74244.318s, table=10, n_packets=666756, n_bytes=1140691558, priority=100,reg9=0x8/0x8,metadata=0x12 actions=resubmit(,11)
next;
9. lr_in_ip_routing (ovn-northd.c:7463): ip4.dst == 0.0.0.0/0, priority 1, uuid 60838bf6
cookie=0x60838bf6, duration=74244.364s, table=17, n_packets=456618, n_bytes=953953284, priority=1,ip,metadata=0x12 actions=dec_ttl(),load:0->OXM_OF_PKT_REG4[32..47],load:0xa002801->NXM_NX_XXREG0[96..127],load:0xa0028e9->NXM_NX_XXREG0[64..95],set_field:fa:16:3e:4a:e3:b5->eth_src,set_field:0x1->reg15,load:0x1->NXM_NX_REG10[0],resubmit(,18)
ip.ttl--;
reg8[0..15] = 0;
reg0 = 10.0.40.1;
reg1 = 10.0.40.233;
eth.src = fa:16:3e:4a:e3:b5;
outport = "lrp-e9e8c0cb-2e81-4d20-9c28-44962ad1cbb9";
flags.loopback = 1;
next;
10. lr_in_ip_routing_ecmp (ovn-northd.c:9374): reg8[0..15] == 0, priority 150, uuid a0d19c8d
cookie=0xa0d19c8d, duration=74244.353s, table=18, n_packets=649421, n_bytes=1138767765, priority=150,reg8=0/0xffff,metadata=0x12 actions=resubmit(,19)
next;
12. lr_in_arp_resolve (ovn-northd.c:9854): ip4, priority 0, uuid 7d603a2d
cookie=0x7d603a2d, duration=74244.302s, table=20, n_packets=456618, n_bytes=953953284, priority=0,ip,metadata=0x12 actions=push:NXM_NX_REG0[],push:NXM_NX_XXREG0[96..127],pop:NXM_NX_REG0[],set_field:00:00:00:00:00:00->eth_dst,resubmit(,66),pop:NXM_NX_REG0[],resubmit(,21)
get_arp(outport, reg0);
/* MAC binding to b4:09:31:47:30:ef. */
next;
15. lr_in_gw_redirect (ovn-northd.c:9085): ip4.src == 172.20.25.162 && outport == "lrp-e9e8c0cb-2e81-4d20-9c28-44962ad1cbb9", priority 100, uuid 2c2a44fc
cookie=0x2c2a44fc, duration=10206.559s, table=23, n_packets=42796, n_bytes=156952532, priority=100,ip,reg15=0x1,metadata=0x12,nw_src=172.20.25.162 actions=resubmit(,24)
next;
16. lr_in_arp_request (ovn-northd.c:10055): 1, priority 0, uuid f0c7e08b
cookie=0xf0c7e08b, duration=74244.285s, table=24, n_packets=649421, n_bytes=1138767765, priority=0,metadata=0x12 actions=resubmit(,32)
output;
egress(dp="32d08999-d00a-49d2-bf8d-9169894b2a45", inport="lrp-039ba2bc-997c-448a-adb7-54bd3af697c1", outport="lrp-e9e8c0cb-2e81-4d20-9c28-44962ad1cbb9")
--------------------------------------------------------------------------------------------------------------------------------------------------------
0. lr_out_undnat (ovn-northd.c:8983): ip && ip4.src == 172.20.25.162 && outport == "lrp-e9e8c0cb-2e81-4d20-9c28-44962ad1cbb9", priority 100, uuid ca864081
cookie=0xca864081, duration=10206.560s, table=40, n_packets=42796, n_bytes=156952532, priority=100,ip,reg15=0x1,metadata=0x12,nw_src=172.20.25.162 actions=set_field:fa:16:3e:85:0e:cb->eth_src,ct(table=41,zone=NXM_NX_REG11[0..15],nat)
eth.src = fa:16:3e:85:0e:cb;
ct_dnat;
ct_dnat /* assuming no un-dnat entry, so no change */
-----------------------------------------------------
1. lr_out_snat (ovn-northd.c:9015): ip && ip4.src == 172.20.25.0/24 && outport == "lrp-e9e8c0cb-2e81-4d20-9c28-44962ad1cbb9" && is_chassis_resident("cr-lrp-e9e8c0cb-2e81-4d20-9c28-44962ad1cbb9"), priority 153, uuid e041811b
*** no OpenFlow flows
ct_snat(10.0.40.233);
ct_snat(ip4.src=10.0.40.233)
----------------------------
2. lr_out_egr_loop (ovn-northd.c:9120): ip4.dst == 10.0.41.228 && outport == "lrp-e9e8c0cb-2e81-4d20-9c28-44962ad1cbb9" && is_chassis_resident("103e7777-a376-419d-9629-31bc6c7fedb1"), priority 100, uuid 5a133ec9
cookie=0x5a133ec9, duration=10206.560s, table=42, n_packets=47, n_bytes=4606, priority=100,ip,reg15=0x1,metadata=0x12,nw_dst=10.0.41.228 actions=clone(ct_clear,move:NXM_NX_REG15[]->NXM_NX_REG14[],set_field:0->reg15,set_field:0->reg10,load:0x1->NXM_NX_REG10[0],load:0->NXM_NX_XXREG0[96..127],load:0->NXM_NX_XXREG0[64..95],load:0->NXM_NX_XXREG0[32..63],load:0->NXM_NX_XXREG0[0..31],load:0->NXM_NX_XXREG1[96..127],load:0->NXM_NX_XXREG1[64..95],load:0->NXM_NX_XXREG1[32..63],load:0->NXM_NX_XXREG1[0..31],load:0->OXM_OF_PKT_REG4[32..63],load:0->OXM_OF_PKT_REG4[0..31],load:0x1->OXM_OF_PKT_REG4[0],resubmit(,8))
clone { ct_clear; inport = outport; outport = ""; flags = 0; flags.loopback = 1; reg0 = 0; reg1 = 0; reg2 = 0; reg3 = 0; reg4 = 0; reg5 = 0; reg6 = 0; reg7 = 0; reg8 = 0; reg9 = 0; reg9[0] = 1; next(pipeline=ingress, table=0); };
clone
-----
ct_clear;
inport = outport;
outport = "";
flags = 0;
flags.loopback = 1;
reg0 = 0;
reg1 = 0;
reg2 = 0;
reg3 = 0;
reg4 = 0;
reg5 = 0;
reg6 = 0;
reg7 = 0;
reg8 = 0;
reg9 = 0;
reg9[0] = 1;
next(pipeline=ingress, table=0);
ingress(dp="32d08999-d00a-49d2-bf8d-9169894b2a45", inport="lrp-e9e8c0cb-2e81-4d20-9c28-44962ad1cbb9")
-----------------------------------------------------------------------------------------------------
0. lr_in_admission: no match (implicit drop)
I also have this problem