Installing Mellanox OFED in Xenial with Kernel 4.4:
# lsb_release -a No LSB modules are available. Distributor ID: Ubuntu Description: Ubuntu 16.04.1 LTS Release: 16.04 Codename: xenial
# uname -a Linux heatmor 4.4.0-36-generic #55-Ubuntu SMP Thu Aug 11 18:01:55 UTC 2016 x86_64 x86_64 x86_64 GNU/Linux
And, after configuring SR-IOV for ConnectX-4 (after configured with mlxconfig):
# lspci | grep -i mellanox 08:00.0 Infiniband controller: Mellanox Technologies MT27700 Family [ConnectX-4] 08:00.1 Infiniband controller: Mellanox Technologies MT27700 Family [ConnectX-4] 08:00.2 Infiniband controller: Mellanox Technologies MT27700 Family [ConnectX-4 Virtual Function] 08:00.3 Infiniband controller: Mellanox Technologies MT27700 Family [ConnectX-4 Virtual Function] 08:00.4 Infiniband controller: Mellanox Technologies MT27700 Family [ConnectX-4 Virtual Function] 08:00.5 Infiniband controller: Mellanox Technologies MT27700 Family [ConnectX-4 Virtual Function]
echo 4 > /sys/class/infiniband/mlx5_0/device/sriov_numvfs
echo Follow > /sys/class/infiniband/mlx5_0/device/sriov/0/policy
echo e4:1d:2d:03:00:af:4f:06 > /sys/class/infiniband/mlx5_0/device/sriov/0/node echo e4:1d:2d:03:00:af:5f:06 > /sys/class/infiniband/mlx5_0/device/sriov/0/port
echo e4:1d:2d:03:00:af:4f:07 > /sys/class/infiniband/mlx5_0/device/sriov/1/node echo e4:1d:2d:03:00:af:5f:07 > /sys/class/infiniband/mlx5_0/device/sriov/1/port
echo e4:1d:2d:03:00:af:4f:08 > /sys/class/infiniband/mlx5_0/device/sriov/2/node echo e4:1d:2d:03:00:af:5f:08 > /sys/class/infiniband/mlx5_0/device/sriov/2/port
echo e4:1d:2d:03:00:af:4f:09 > /sys/class/infiniband/mlx5_0/device/sriov/3/node echo e4:1d:2d:03:00:af:5f:09 > /sys/class/infiniband/mlx5_0/device/sriov/3/port
echo 0000:08:00.2 > /sys/bus/pci/drivers/mlx5_core/unbind echo 0000:08:00.2 > /sys/bus/pci/drivers/mlx5_core/bind
echo 0000:08:00.3 > /sys/bus/pci/drivers/mlx5_core/unbind echo 0000:08:00.3 > /sys/bus/pci/drivers/mlx5_core/bind
echo 0000:08:00.4 > /sys/bus/pci/drivers/mlx5_core/unbind echo 0000:08:00.4 > /sys/bus/pci/drivers/mlx5_core/bind
echo 0000:08:00.5 > /sys/bus/pci/drivers/mlx5_core/unbind echo 0000:08:00.5 > /sys/bus/pci/drivers/mlx5_core/bind
And attaching this XML to a guest:
<interface type='hostdev' managed='yes'> <source> <address type='pci' domain='0' bus='8' slot='0' function='2'/> </source> </interface>
root@heatmor:~# virsh attach-device ibdhcprelay ./new-device.xml --config
I can't start the guest in question:
# virsh start ibdhcprelay error: Failed to start domain ibdhcprelay error: internal error: missing IFLA_VF_INFO in netlink response
Looks like there is an incompatibility between Mellanox OFED DKMS packages and Ubuntu kernel (specifically about netlink support).
Installing Mellanox OFED in Xenial with Kernel 4.4:
# lsb_release -a
No LSB modules are available.
Distributor ID: Ubuntu
Description: Ubuntu 16.04.1 LTS
Release: 16.04
Codename: xenial
# uname -a
Linux heatmor 4.4.0-36-generic #55-Ubuntu SMP Thu Aug 11 18:01:55 UTC 2016 x86_64 x86_64 x86_64 GNU/Linux
And, after configuring SR-IOV for ConnectX-4 (after configured with mlxconfig):
# lspci | grep -i mellanox
08:00.0 Infiniband controller: Mellanox Technologies MT27700 Family [ConnectX-4]
08:00.1 Infiniband controller: Mellanox Technologies MT27700 Family [ConnectX-4]
08:00.2 Infiniband controller: Mellanox Technologies MT27700 Family [ConnectX-4 Virtual Function]
08:00.3 Infiniband controller: Mellanox Technologies MT27700 Family [ConnectX-4 Virtual Function]
08:00.4 Infiniband controller: Mellanox Technologies MT27700 Family [ConnectX-4 Virtual Function]
08:00.5 Infiniband controller: Mellanox Technologies MT27700 Family [ConnectX-4 Virtual Function]
echo 4 > /sys/class/ infiniband/ mlx5_0/ device/ sriov_numvfs
echo Follow > /sys/class/ infiniband/ mlx5_0/ device/ sriov/0/ policy
echo e4:1d:2d: 03:00:af: 4f:06 > /sys/class/ infiniband/ mlx5_0/ device/ sriov/0/ node 03:00:af: 5f:06 > /sys/class/ infiniband/ mlx5_0/ device/ sriov/0/ port
echo e4:1d:2d:
echo e4:1d:2d: 03:00:af: 4f:07 > /sys/class/ infiniband/ mlx5_0/ device/ sriov/1/ node 03:00:af: 5f:07 > /sys/class/ infiniband/ mlx5_0/ device/ sriov/1/ port
echo e4:1d:2d:
echo e4:1d:2d: 03:00:af: 4f:08 > /sys/class/ infiniband/ mlx5_0/ device/ sriov/2/ node 03:00:af: 5f:08 > /sys/class/ infiniband/ mlx5_0/ device/ sriov/2/ port
echo e4:1d:2d:
echo e4:1d:2d: 03:00:af: 4f:09 > /sys/class/ infiniband/ mlx5_0/ device/ sriov/3/ node 03:00:af: 5f:09 > /sys/class/ infiniband/ mlx5_0/ device/ sriov/3/ port
echo e4:1d:2d:
echo 0000:08:00.2 > /sys/bus/ pci/drivers/ mlx5_core/ unbind pci/drivers/ mlx5_core/ bind
echo 0000:08:00.2 > /sys/bus/
echo 0000:08:00.3 > /sys/bus/ pci/drivers/ mlx5_core/ unbind pci/drivers/ mlx5_core/ bind
echo 0000:08:00.3 > /sys/bus/
echo 0000:08:00.4 > /sys/bus/ pci/drivers/ mlx5_core/ unbind pci/drivers/ mlx5_core/ bind
echo 0000:08:00.4 > /sys/bus/
echo 0000:08:00.5 > /sys/bus/ pci/drivers/ mlx5_core/ unbind pci/drivers/ mlx5_core/ bind
echo 0000:08:00.5 > /sys/bus/
And attaching this XML to a guest:
<interface type='hostdev' managed='yes'>
<source>
<address type='pci' domain='0' bus='8' slot='0' function='2'/>
</source>
</interface>
root@heatmor:~# virsh attach-device ibdhcprelay ./new-device.xml --config
I can't start the guest in question:
# virsh start ibdhcprelay
error: Failed to start domain ibdhcprelay
error: internal error: missing IFLA_VF_INFO in netlink response
Looks like there is an incompatibility between Mellanox OFED DKMS packages and Ubuntu kernel (specifically about netlink support).