Juniper Openstack

Comment 0 for bug 1795834

Revision history for this message

Bernhard Koessler (bkoessler) wrote on 2018-10-03:

in a CC13/Contrail 5.0.1 setup with X710 ortville NICs, for non-DPDK workloads, the DPDK vrouter fragments packets with default 1500 MTU and above despite vrouter/vhost MTU being set to 9000.

# vifdump on vif 0/0
17:38:55 | (vrouter-agent-dpdk)[root@overcloud63m-compdpdk-25 /]$ vifdump vif0/0 -nvv host 100.64.128.3
17:38:55 | vif0/0 PCI: 0000:00:00.0 (Speed 20000, Duplex 0)
# packet from SDN-GW to VM
17:39:45 | 15:39:45.686691 IP (tos 0x0, ttl 64, id 47127, offset 0, flags [none], proto UDP (17), length 1560)
17:39:45 | 100.64.128.3.56632 > 100.64.144.61.6635: [no cksum] UDP, length 1532
# packet from VM to SDN-GW
17:39:45 | 15:39:45.687314 IP (tos 0x0, ttl 64, id 27886, offset 0, flags [none], proto UDP (17), length 1492, bad cksum 0 (->2f6a)!)
17:39:45 | 100.64.144.61.49969 > 100.64.128.3.6635: [no cksum] UDP, length 1464
17:39:45 | 15:39:45.687326 IP (tos 0x0, ttl 64, id 27886, offset 0, flags [none], proto UDP (17), length 120, bad cksum 0 (->34c6)!)
17:39:45 | 100.64.144.61.49969 > 100.64.128.3.6635: [no cksum] UDP, length 92

# config
[root@overcloud63m-compdpdk-25 ~]# ifconfig vhost0
vhost0: flags=4163<UP,BROADCAST,RUNNING,MULTICAST> mtu 9000
        inet 100.64.144.61 netmask 255.255.248.0 broadcast 100.64.151.255
        inet6 fe80::6a05:caff:fe41:ffe0 prefixlen 64 scopeid 0x20<link>
        ether 68:05:ca:41:ff:e0 txqueuelen 1000 (Ethernet)
        RX packets 26076733 bytes 1839255148 (1.7 GiB)
        RX errors 0 dropped 0 overruns 0 frame 0
        TX packets 1476132 bytes 97125795 (92.6 MiB)
        TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0

[root@overcloud63m-compdpdk-25 ~]# ip l | grep -i mtu
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN mode DEFAULT group default qlen 1000
2: eno1: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 9000 qdisc mq state UP mode DEFAULT group default qlen 1000
3: eno2: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 9000 qdisc mq state UP mode DEFAULT group default qlen 1000
4: eno3: <NO-CARRIER,BROADCAST,MULTICAST,UP> mtu 1500 qdisc mq state DOWN mode DEFAULT group default qlen 1000
5: eno4: <NO-CARRIER,BROADCAST,MULTICAST,UP> mtu 1500 qdisc mq state DOWN mode DEFAULT group default qlen 1000
8: ens5f0: <BROADCAST,MULTICAST,SLAVE,UP,LOWER_UP> mtu 9000 qdisc mq master bond1 state UP mode DEFAULT group default qlen 1000
9: enp0s20u1u5: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UNKNOWN mode DEFAULT group default qlen 1000
10: ens5f1: <BROADCAST,MULTICAST,SLAVE,UP,LOWER_UP> mtu 9000 qdisc mq master bond1 state UP mode DEFAULT group default qlen 1000
11: docker0: <NO-CARRIER,BROADCAST,MULTICAST,UP> mtu 1500 qdisc noqueue state DOWN mode DEFAULT group default
14: bond1: <BROADCAST,MULTICAST,MASTER,UP,LOWER_UP> mtu 9000 qdisc noqueue state UP mode DEFAULT group default qlen 1000
15: vlan303@bond1: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 9000 qdisc noqueue state UP mode DEFAULT group default qlen 1000
16: vlan304@bond1: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 9000 qdisc noqueue state UP mode DEFAULT group default qlen 1000
17: bond0: <BROADCAST,MULTICAST> mtu 1500 qdisc noop state DOWN mode DEFAULT group default qlen 1000
18: vhost0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 9000 qdisc pfifo_fast state UNKNOWN mode DEFAULT group default qlen 1000

[root@overcloud63m-compdpdk-25 ~]# cat /etc/sysconfig/network-scripts/ifcfg-vhost0
# This file is autogenerated by os-net-config
DEVICE=vhost0
ONBOOT=yes
HOTPLUG=no
NM_CONTROLLED=no
PEERDNS=no
MTU=9000
BOOTPROTO=static
IPADDR=100.64.144.61
NETMASK=255.255.248.0
DEVICETYPE=vhost
TYPE=dpdk
BIND_INT=0000:06:00.0,0000:06:00.1
BOND_MODE=4
BOND_POLICY=layer3+4
DRIVER=vfio-pci
CPU_LIST=0xe00000e
VLAN_ID=301

# vif list
17:41:07 | (vrouter-agent-dpdk)[root@overcloud63m-compdpdk-25 /]$ vif --list
17:41:07 | Vrouter Interface Table
17:41:07 |
17:41:07 | Flags: P=Policy, X=Cross Connect, S=Service Chain, Mr=Receive Mirror
17:41:07 | Mt=Transmit Mirror, Tc=Transmit Checksum Offload, L3=Layer 3, L2=Layer 2
17:41:07 | D=DHCP, Vp=Vhost Physical, Pr=Promiscuous, Vnt=Native Vlan Tagged
17:41:07 | Mnp=No MAC Proxy, Dpdk=DPDK PMD Interface, Rfl=Receive Filtering Offload, Mon=Interface is Monitored
17:41:07 | Uuf=Unknown Unicast Flood, Vof=VLAN insert/strip offload, Df=Drop New Flows, L=MAC Learning Enabled
17:41:07 | Proxy=MAC Requests Proxied Always, Er=Etree Root, Mn=Mirror without Vlan Tag, Ig=Igmp Trap Enabled
17:41:07 |
17:41:07 | vif0/0 PCI: 0000:00:00.0 (Speed 20000, Duplex 0)
17:41:07 | Type:Physical HWaddr:68:05:ca:41:ff:e0 IPaddr:0.0.0.0
17:41:07 | Vrf:0 Mcast Vrf:65535 Flags:TcL3L2VpEr QOS:-1 Ref:183
17:41:07 | RX device packets:148760677 bytes:100772950203 errors:2
17:41:07 | RX port packets:22096245 errors:0
17:41:07 | RX queue errors to lcore 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
17:41:07 | RX packets:21872441 bytes:5047649117 errors:0
17:41:07 | TX packets:13461281 bytes:4024415957 errors:0
17:41:07 | Drops:1426580
17:41:07 | TX port packets:13376148 errors:86652
17:41:07 | TX device packets:128200703 bytes:22037988996 errors:0
17:41:07 |
17:41:07 | vif0/1 PMD: vhost0
17:41:07 | Type:Host HWaddr:68:05:ca:41:ff:e0 IPaddr:100.64.144.61
17:41:07 | Vrf:0 Mcast Vrf:65535 Flags:L3DEr QOS:-1 Ref:17
17:41:07 | RX device packets:325655 bytes:19709666 errors:0
17:41:07 | RX queue packets:325655 errors:0
17:41:07 | RX queue errors to lcore 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
17:41:07 | RX packets:325655 bytes:19709666 errors:0
17:41:07 | TX packets:7513215 bytes:522931356 errors:0
17:41:07 | Drops:0
17:41:07 | TX queue packets:6625285 errors:887930
17:41:07 | TX device packets:6625263 bytes:435257461 errors:0
17:41:07 |
17:41:07 | vif0/2 Socket: unix
17:41:07 | Type:Agent HWaddr:00:00:5e:00:01:00 IPaddr:0.0.0.0
17:41:07 | Vrf:65535 Mcast Vrf:65535 Flags:L3Er QOS:-1 Ref:3
17:41:07 | RX port packets:1731003 errors:0
17:41:07 | RX queue errors to lcore 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
17:41:07 | RX packets:1731003 bytes:297731006 errors:0
17:41:07 | TX packets:4281847 bytes:661636460 errors:0
17:41:07 | Drops:0
17:41:07 |
17:41:07 | vif0/3 PMD: tapf8e31b10-43
17:41:07 | Type:Virtual HWaddr:00:00:5e:00:01:00 IPaddr:87.140.203.126
17:41:07 | Vrf:2 Mcast Vrf:2 Flags:PL3L2DEr QOS:-1 Ref:20
17:41:07 | RX port packets:63302 errors:0 syscalls:1
17:41:07 | RX queue packets:51539 errors:0
17:41:07 | RX queue errors to lcore 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
17:41:07 | RX packets:63302 bytes:8483331 errors:0
17:41:07 | TX packets:38552 bytes:8532265 errors:0
17:41:07 | Drops:88746
17:41:07 | TX port packets:37676 errors:22 syscalls:37415

in a CC13/Contrail 5.0.1 setup with X710 ortville NICs, for non-DPDK workloads, the DPDK vrouter fragments packets with default 1500 MTU and above despite vrouter/vhost MTU being set to 9000.

# vifdump on vif 0/0
17:38:55 | (vrouter-agent-dpdk)[root@overcloud63m-compdpdk-25 /]$ vifdump vif0/0 -nvv host 100.64.128.3
17:38:55 | vif0/0      PCI: 0000:00:00.0 (Speed 20000, Duplex 0)
# packet from SDN-GW to VM
17:39:45 | 15:39:45.686691 IP (tos 0x0, ttl 64, id 47127, offset 0, flags [none], proto UDP (17), length 1560)
17:39:45 |     100.64.128.3.56632 > 100.64.144.61.6635: [no cksum] UDP, length 1532
# packet from VM to SDN-GW
17:39:45 | 15:39:45.687314 IP (tos 0x0, ttl 64, id 27886, offset 0, flags [none], proto UDP (17), length 1492, bad cksum 0 (->2f6a)!)
17:39:45 |     100.64.144.61.49969 > 100.64.128.3.6635: [no cksum] UDP, length 1464
17:39:45 | 15:39:45.687326 IP (tos 0x0, ttl 64, id 27886, offset 0, flags [none], proto UDP (17), length 120, bad cksum 0 (->34c6)!)
17:39:45 |     100.64.144.61.49969 > 100.64.128.3.6635: [no cksum] UDP, length 92

# config
[root@overcloud63m-compdpdk-25 ~]# ifconfig vhost0
vhost0: flags=4163<UP,BROADCAST,RUNNING,MULTICAST>  mtu 9000
        inet 100.64.144.61  netmask 255.255.248.0  broadcast 100.64.151.255
        inet6 fe80::6a05:caff:fe41:ffe0  prefixlen 64  scopeid 0x20<link>
        ether 68:05:ca:41:ff:e0  txqueuelen 1000  (Ethernet)
        RX packets 26076733  bytes 1839255148 (1.7 GiB)
        RX errors 0  dropped 0  overruns 0  frame 0
        TX packets 1476132  bytes 97125795 (92.6 MiB)
        TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0

[root@overcloud63m-compdpdk-25 ~]# ip l | grep -i mtu
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN mode DEFAULT group default qlen 1000
2: eno1: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 9000 qdisc mq state UP mode DEFAULT group default qlen 1000
3: eno2: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 9000 qdisc mq state UP mode DEFAULT group default qlen 1000
4: eno3: <NO-CARRIER,BROADCAST,MULTICAST,UP> mtu 1500 qdisc mq state DOWN mode DEFAULT group default qlen 1000
5: eno4: <NO-CARRIER,BROADCAST,MULTICAST,UP> mtu 1500 qdisc mq state DOWN mode DEFAULT group default qlen 1000
8: ens5f0: <BROADCAST,MULTICAST,SLAVE,UP,LOWER_UP> mtu 9000 qdisc mq master bond1 state UP mode DEFAULT group default qlen 1000
9: enp0s20u1u5: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UNKNOWN mode DEFAULT group default qlen 1000
10: ens5f1: <BROADCAST,MULTICAST,SLAVE,UP,LOWER_UP> mtu 9000 qdisc mq master bond1 state UP mode DEFAULT group default qlen 1000
11: docker0: <NO-CARRIER,BROADCAST,MULTICAST,UP> mtu 1500 qdisc noqueue state DOWN mode DEFAULT group default 
14: bond1: <BROADCAST,MULTICAST,MASTER,UP,LOWER_UP> mtu 9000 qdisc noqueue state UP mode DEFAULT group default qlen 1000
15: vlan303@bond1: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 9000 qdisc noqueue state UP mode DEFAULT group default qlen 1000
16: vlan304@bond1: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 9000 qdisc noqueue state UP mode DEFAULT group default qlen 1000
17: bond0: <BROADCAST,MULTICAST> mtu 1500 qdisc noop state DOWN mode DEFAULT group default qlen 1000
18: vhost0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 9000 qdisc pfifo_fast state UNKNOWN mode DEFAULT group default qlen 1000

[root@overcloud63m-compdpdk-25 ~]# cat /etc/sysconfig/network-scripts/ifcfg-vhost0 
# This file is autogenerated by os-net-config
DEVICE=vhost0
ONBOOT=yes
HOTPLUG=no
NM_CONTROLLED=no
PEERDNS=no
MTU=9000
BOOTPROTO=static
IPADDR=100.64.144.61
NETMASK=255.255.248.0
DEVICETYPE=vhost
TYPE=dpdk
BIND_INT=0000:06:00.0,0000:06:00.1
BOND_MODE=4
BOND_POLICY=layer3+4
DRIVER=vfio-pci
CPU_LIST=0xe00000e
VLAN_ID=301

# contrail-status
17:43:14 | [root@overcloud63m-compdpdk-25 ~]# contrail-status
17:43:15 | Pod      Service     Original Name                State    Status     
17:43:15 | vrouter  agent       contrail-vrouter-agent       running  Up 2 days  
17:43:15 | vrouter  agent-dpdk  contrail-vrouter-agent-dpdk  running  Up 8 days  
17:43:15 | vrouter  nodemgr     contrail-nodemgr             running  Up 2 days  
17:43:15 | 
17:43:15 | vrouter DPDK module is PRESENT
17:43:15 | == Contrail vrouter ==
17:43:15 | nodemgr: active
17:43:15 | agent: active
17:43:15 |
 
# vif list
17:41:07 | (vrouter-agent-dpdk)[root@overcloud63m-compdpdk-25 /]$ vif --list
17:41:07 | Vrouter Interface Table
17:41:07 | 
17:41:07 | Flags: P=Policy, X=Cross Connect, S=Service Chain, Mr=Receive Mirror
17:41:07 |        Mt=Transmit Mirror, Tc=Transmit Checksum Offload, L3=Layer 3, L2=Layer 2
17:41:07 |        D=DHCP, Vp=Vhost Physical, Pr=Promiscuous, Vnt=Native Vlan Tagged
17:41:07 |        Mnp=No MAC Proxy, Dpdk=DPDK PMD Interface, Rfl=Receive Filtering Offload, Mon=Interface is Monitored
17:41:07 |        Uuf=Unknown Unicast Flood, Vof=VLAN insert/strip offload, Df=Drop New Flows, L=MAC Learning Enabled
17:41:07 |        Proxy=MAC Requests Proxied Always, Er=Etree Root, Mn=Mirror without Vlan Tag, Ig=Igmp Trap Enabled
17:41:07 | 
17:41:07 | vif0/0      PCI: 0000:00:00.0 (Speed 20000, Duplex 0)
17:41:07 |             Type:Physical HWaddr:68:05:ca:41:ff:e0 IPaddr:0.0.0.0
17:41:07 |             Vrf:0 Mcast Vrf:65535 Flags:TcL3L2VpEr QOS:-1 Ref:183
17:41:07 |             RX device packets:148760677  bytes:100772950203 errors:2
17:41:07 |             RX port   packets:22096245 errors:0
17:41:07 |             RX queue errors to lcore 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
17:41:07 |             RX packets:21872441  bytes:5047649117 errors:0
17:41:07 |             TX packets:13461281  bytes:4024415957 errors:0
17:41:07 |             Drops:1426580
17:41:07 |             TX port   packets:13376148 errors:86652
17:41:07 |             TX device packets:128200703  bytes:22037988996 errors:0
17:41:07 | 
17:41:07 | vif0/1      PMD: vhost0
17:41:07 |             Type:Host HWaddr:68:05:ca:41:ff:e0 IPaddr:100.64.144.61
17:41:07 |             Vrf:0 Mcast Vrf:65535 Flags:L3DEr QOS:-1 Ref:17
17:41:07 |             RX device packets:325655  bytes:19709666 errors:0
17:41:07 |             RX queue  packets:325655 errors:0
17:41:07 |             RX queue errors to lcore 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
17:41:07 |             RX packets:325655  bytes:19709666 errors:0
17:41:07 |             TX packets:7513215  bytes:522931356 errors:0
17:41:07 |             Drops:0
17:41:07 |             TX queue  packets:6625285 errors:887930
17:41:07 |             TX device packets:6625263  bytes:435257461 errors:0
17:41:07 | 
17:41:07 | vif0/2      Socket: unix
17:41:07 |             Type:Agent HWaddr:00:00:5e:00:01:00 IPaddr:0.0.0.0
17:41:07 |             Vrf:65535 Mcast Vrf:65535 Flags:L3Er QOS:-1 Ref:3
17:41:07 |             RX port   packets:1731003 errors:0
17:41:07 |             RX queue errors to lcore 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
17:41:07 |             RX packets:1731003  bytes:297731006 errors:0
17:41:07 |             TX packets:4281847  bytes:661636460 errors:0
17:41:07 |             Drops:0
17:41:07 | 
17:41:07 | vif0/3      PMD: tapf8e31b10-43
17:41:07 |             Type:Virtual HWaddr:00:00:5e:00:01:00 IPaddr:87.140.203.126
17:41:07 |             Vrf:2 Mcast Vrf:2 Flags:PL3L2DEr QOS:-1 Ref:20
17:41:07 |             RX port   packets:63302 errors:0 syscalls:1
17:41:07 |             RX queue  packets:51539 errors:0
17:41:07 |             RX queue errors to lcore 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
17:41:07 |             RX packets:63302  bytes:8483331 errors:0
17:41:07 |             TX packets:38552  bytes:8532265 errors:0
17:41:07 |             Drops:88746
17:41:07 |             TX port   packets:37676 errors:22 syscalls:37415