Comment 1 for bug 1842988

Revision history for this message
Andreas Karis (akaris) wrote :

I ran into the same issue downstream Red Hat OSP 13 in 2 different, brand new environments. I'm going to add further details from my troubleshooting.

First of all, a workaround or way to "fix" this is to reboot the instance. The external_mac field then will be populated. However, if one detaches and reattaches the VIP, one can easily reproduce the issue.

From my lab, here are steps to reproduce and verify this:

Create server without FIP:
~~~
(overcloud) [stack@undercloud-0 ~]$ nova list
+--------------------------------------+------------+--------+------------+-------------+--------------------------------------------------------------------------+
| ID | Name | Status | Task State | Power State | Networks |
+--------------------------------------+------------+--------+------------+-------------+--------------------------------------------------------------------------+
| ac73af43-c22e-4e5b-be90-f2bc79b30ca4 | rhel-test1 | ACTIVE | - | Running | private1=2000:192:168:0:f816:3eff:fe3f:9ee4, 192.168.0.101, 172.31.0.201 |
| f8dccb8d-b39a-4231-b2f1-46dae48744b4 | rhel-test2 | ACTIVE | - | Running | private1=2000:192:168:0:f816:3eff:fe00:bcd6, 192.168.0.103, 172.31.0.217 |
| 1cd9d144-9951-4bff-8b97-3a6e02b79467 | rhel-test3 | ACTIVE | - | Running | private2=192.168.1.101, 2000:192:168:1:f816:3eff:feb7:5d7d |
+--------------------------------------+------------+--------+------------+-------------+--------------------------------------------------------------------------+
~~~

Check NAT rules:
~~~
[root@overcloud-controller-0 ~]# export SB=$(sudo ovs-vsctl get open . external_ids:ovn-remote | sed -e 's/\"//g')
[root@overcloud-controller-0 ~]# export NB=$(sudo ovs-vsctl get open . external_ids:ovn-remote | sed -e 's/\"//g' | sed -e 's/6642/6641/g')
[root@overcloud-controller-0 ~]# alias ovn-sbctl='sudo docker exec ovn_controller ovn-sbctl --db=$SB'
[root@overcloud-controller-0 ~]# alias ovn-nbctl='sudo docker exec ovn_controller ovn-nbctl --db=$NB'
[root@overcloud-controller-0 ~]# alias ovn-trace='sudo docker exec ovn_controller ovn-trace --db=$SB'
[root@overcloud-controller-0 ~]# ovn-nbctl find NAT type=dnat_and_snat
_uuid : 61b357c0-1c06-4c22-b0e5-50ef64813cc8
external_ids : {"neutron:fip_external_mac"="fa:16:3e:ee:4b:3b", "neutron:fip_id"="b0c03864-8c79-43b3-9240-370cfbcde904", "neutron:fip_port_id"="b7e9fb53-db4e-4b3e-a6e4-2acbbc4ba9ba", "neutron:revision_number"="2", "neutron:router_name"="neutron-e75c5fb4-29ee-450d-840f-a911c9896256"}
external_ip : "172.31.0.201"
external_mac : "fa:16:3e:ee:4b:3b"
logical_ip : "192.168.0.101"
logical_port : "b7e9fb53-db4e-4b3e-a6e4-2acbbc4ba9ba"
type : dnat_and_snat

_uuid : b0782824-00e9-4b66-bc4b-9d54cf8ec737
external_ids : {"neutron:fip_external_mac"="fa:16:3e:8b:04:4f", "neutron:fip_id"="4fc2b87a-a188-4468-ad40-8b03ed26b56d", "neutron:fip_port_id"="581de757-7b2e-4995-aa84-3c4bc58edd4d", "neutron:revision_number"="2", "neutron:router_name"="neutron-e75c5fb4-29ee-450d-840f-a911c9896256"}
external_ip : "172.31.0.217"
external_mac : "fa:16:3e:8b:04:4f"
logical_ip : "192.168.0.103"
logical_port : "581de757-7b2e-4995-aa84-3c4bc58edd4d"
type : dnat_and_snat
[root@overcloud-controller-0 ~]#
~~~

Add FIP:
~~~
(overcloud) [stack@undercloud-0 ~]$ openstack server add floating ip rhel-test3 172.31.0.210
(overcloud) [stack@undercloud-0 ~]$
~~~

Check NAT rules:
~~~
[root@overcloud-controller-0 ~]# ovn-nbctl find NAT type=dnat_and_snat
_uuid : 61b357c0-1c06-4c22-b0e5-50ef64813cc8
external_ids : {"neutron:fip_external_mac"="fa:16:3e:ee:4b:3b", "neutron:fip_id"="b0c03864-8c79-43b3-9240-370cfbcde904", "neutron:fip_port_id"="b7e9fb53-db4e-4b3e-a6e4-2acbbc4ba9ba", "neutron:revision_number"="2", "neutron:router_name"="neutron-e75c5fb4-29ee-450d-840f-a911c9896256"}
external_ip : "172.31.0.201"
external_mac : "fa:16:3e:ee:4b:3b"
logical_ip : "192.168.0.101"
logical_port : "b7e9fb53-db4e-4b3e-a6e4-2acbbc4ba9ba"
type : dnat_and_snat

_uuid : b0782824-00e9-4b66-bc4b-9d54cf8ec737
external_ids : {"neutron:fip_external_mac"="fa:16:3e:8b:04:4f", "neutron:fip_id"="4fc2b87a-a188-4468-ad40-8b03ed26b56d", "neutron:fip_port_id"="581de757-7b2e-4995-aa84-3c4bc58edd4d", "neutron:revision_number"="2", "neutron:router_name"="neutron-e75c5fb4-29ee-450d-840f-a911c9896256"}
external_ip : "172.31.0.217"
external_mac : "fa:16:3e:8b:04:4f"
logical_ip : "192.168.0.103"
logical_port : "581de757-7b2e-4995-aa84-3c4bc58edd4d"
type : dnat_and_snat

_uuid : f406e2aa-697d-4a44-a078-ac50f5699202
external_ids : {"neutron:fip_external_mac"="fa:16:3e:9e:96:da", "neutron:fip_id"="cdeb3dc7-fdc8-493b-9536-edb2163c3d1c", "neutron:fip_port_id"="79f8f865-30c7-48ab-9017-76ce6c007c1b", "neutron:revision_number"="18", "neutron:router_name"="neutron-e75c5fb4-29ee-450d-840f-a911c9896256"}
external_ip : "172.31.0.210"
external_mac : []
logical_ip : "192.168.1.101"
logical_port : "79f8f865-30c7-48ab-9017-76ce6c007c1b"
type : dnat_and_snat
[root@overcloud-controller-0 ~]#
~~~

Reboot server:
~~~
(overcloud) [stack@undercloud-0 ~]$ nova reboot rhel-test3
Request to reboot server <Server: rhel-test3> has been accepted.
(overcloud) [stack@undercloud-0 ~]$
~~~

Check NAT rules:
~~~
[root@overcloud-controller-0 ~]# ovn-nbctl find NAT type=dnat_and_snat
_uuid : 61b357c0-1c06-4c22-b0e5-50ef64813cc8
external_ids : {"neutron:fip_external_mac"="fa:16:3e:ee:4b:3b", "neutron:fip_id"="b0c03864-8c79-43b3-9240-370cfbcde904", "neutron:fip_port_id"="b7e9fb53-db4e-4b3e-a6e4-2acbbc4ba9ba", "neutron:revision_number"="2", "neutron:router_name"="neutron-e75c5fb4-29ee-450d-840f-a911c9896256"}
external_ip : "172.31.0.201"
external_mac : "fa:16:3e:ee:4b:3b"
logical_ip : "192.168.0.101"
logical_port : "b7e9fb53-db4e-4b3e-a6e4-2acbbc4ba9ba"
type : dnat_and_snat

_uuid : b0782824-00e9-4b66-bc4b-9d54cf8ec737
external_ids : {"neutron:fip_external_mac"="fa:16:3e:8b:04:4f", "neutron:fip_id"="4fc2b87a-a188-4468-ad40-8b03ed26b56d", "neutron:fip_port_id"="581de757-7b2e-4995-aa84-3c4bc58edd4d", "neutron:revision_number"="2", "neutron:router_name"="neutron-e75c5fb4-29ee-450d-840f-a911c9896256"}
external_ip : "172.31.0.217"
external_mac : "fa:16:3e:8b:04:4f"
logical_ip : "192.168.0.103"
logical_port : "581de757-7b2e-4995-aa84-3c4bc58edd4d"
type : dnat_and_snat

_uuid : f406e2aa-697d-4a44-a078-ac50f5699202
external_ids : {"neutron:fip_external_mac"="fa:16:3e:9e:96:da", "neutron:fip_id"="cdeb3dc7-fdc8-493b-9536-edb2163c3d1c", "neutron:fip_port_id"="79f8f865-30c7-48ab-9017-76ce6c007c1b", "neutron:revision_number"="18", "neutron:router_name"="neutron-e75c5fb4-29ee-450d-840f-a911c9896256"}
external_ip : "172.31.0.210"
external_mac : "fa:16:3e:9e:96:da"
logical_ip : "192.168.1.101"
logical_port : "79f8f865-30c7-48ab-9017-76ce6c007c1b"
type : dnat_and_snat
~~~