Comment 62 for bug 1541902

------- Comment From <email address hidden> 2016-09-13 16:35 EDT-------
Some additional information from our testing:

System was a Firestone / S822LC, 2 sockets, 8 cores each (POWER8), 64GB memory, running Ubuntu 16.10 with SMT disabled (for KVM), using qemu-system-common/qemu-system-pcc/qemu-kvm packages from Jon's PPA.

We tested 2 kernels, a local build of 4.7, and 4.4.0-30-generic from the install ISO.

We tested physical function passthrough of 2 adapters, a Mellanox ConnectX-3 and ConnectX-4

We don't have any GPUs on this system. I'm still trying to get some results with GPUs, but we're having some issues with the machine we have available (unrelated to the DDW code). From IBM's perspective however the Mellanox test case is the more important driver of the DDW work/feature request so our hope is verification against Mellanox adapters is enough confirm the DDW functionality is working as intended, and that any additional requirements for GPU passthrough be handled as separate issues.

Test steps are below (retried for each adapter/kernel):

# Bind device to vfio-pci module

root@ltc-fire1:~/mdroth# lspci | grep Mellanox
0001:01:00.0 Ethernet controller: Mellanox Technologies MT27700 Family [ConnectX-4]
0001:01:00.1 Ethernet controller: Mellanox Technologies MT27700 Family [ConnectX-4]
0004:01:00.0 Ethernet controller: Mellanox Technologies MT27500 Family [ConnectX-3]

root@ltc-fire1:~/mdroth# echo 0004:01:00.0 >/sys/bus/pci/devices/0004:01:00.0/driver/unbind
root@ltc-fire1:~/mdroth# echo 0x15b3 0x1003 >/sys/bus/pci/drivers/vfio-pci/
bind module/ new_id remove_id uevent unbind
root@ltc-fire1:~/mdroth# echo 0004:01:00.0 >/sys/bus/pci/drivers/vfio-pci/bind

# Start guest

root@ltc-fire1:~/mdroth# qemu-system-ppc64 --smp sockets=1,cores=2,threads=8 --enable-kvm -M pseries -m 3G,slots=32,maxmem=8G -device virtio-blk-pci,id=disk1,drive=drive1 -drive file=ubuntu16.04.base.img.3,aio=threads,cache=none,if=none,id=drive1 -device spapr-vscsi,id=scsi0,reg=0x2000 -monitor unix:/tmp/mdroth-vm3-hmp.sock,server,nowait -qmp unix:/tmp/mdroth-vm3-qmp.sock,server,nowait -device virtio-net-pci,mac=52:54:00:12:34:01,netdev=net1,id=nic1 -netdev user,hostfwd=tcp:0.0.0.0:2223-:22,id=net1 -device vfio-pci,host=0004:01:00.0,id=vfio0 -L /root/mdroth-qemu/pc-bios -boot c -vga none -nographic

# Verify that DDW window ("window#1") gets created after guest kernel boots and passthrough device driver enables 64-bit DMA

root@ltc-fire1:~/mdroth# dmesg | tail -9
[ 5801.875253] pci 0004:01 : [PE# 000] Disabling 64-bit DMA bypass
[ 5801.875262] pci 0004:01 : [PE# 000] Removing DMA window #0
[ 5801.875840] pci 0004:01 : [PE# 000] Setting up window#0 0..7fffffff pg=1000
[ 5802.053473] pci 0004:01 : [PE# 000] Removing DMA window #0
[ 5802.101984] vfio-pci 0004:01:00.0: enabling device (0400 -> 0402)
[ 5806.123998] vfio_ecap_init: 0004:01:00.0 hiding ecap 0x19@0x18c
[ 5806.137354] KVM guest htab at c000000f2b000000 (order 26), LPID 1
[ 5810.159668] pci 0004:01 : [PE# 000] Setting up window#0 0..3fffffff pg=1000
[ 5862.518987] pci 0004:01 : [PE# 000] Setting up window#1 800000000000000..8000001ffffffff pg=10000

# Shutdown guest, verify that windows are destroyed and default 32-bit window is recreated

root@ltc-fire1:~/mdroth# dmesg | tail -4
[ 6187.635524] pci 0004:01 : [PE# 000] Removing DMA window #0
[ 6187.635533] pci 0004:01 : [PE# 000] Removing DMA window #1
[ 6187.641791] pci 0004:01 : [PE# 000] Setting up window#0 0..7fffffff pg=1000
[ 6187.641795] pci 0004:01 : [PE# 000] Enabling 64-bit DMA bypass