Hi Christian,
I have re-tested with Ubuntu 21.04 (Hirsute Hippo).
It took me a while to set up test environment.
Summary:
Ubuntu Version:
$ cat /etc/*-release DISTRIB_ID=Ubuntu DISTRIB_RELEASE=21.04 DISTRIB_CODENAME=hirsute DISTRIB_DESCRIPTION="Ubuntu 21.04" NAME="Ubuntu" VERSION="21.04 (Hirsute Hippo)" ID=ubuntu ID_LIKE=debian PRETTY_NAME="Ubuntu 21.04" VERSION_ID="21.04" HOME_URL="https://www.ubuntu.com/" SUPPORT_URL="https://help.ubuntu.com/" BUG_REPORT_URL="https://bugs.launchpad.net/ubuntu/" PRIVACY_POLICY_URL="https://www.ubuntu.com/legal/terms-and-policies/privacy-policy" VERSION_CODENAME=hirsute UBUNTU_CODENAME=hirsute
Linux Version:
$ uname -a Linux green 5.11.0-17-generic #18-Ubuntu SMP Thu May 6 20:10:11 UTC 2021 x86_64 x86_64 x86_64 GNU/Linux
QEMU / Libvirt Version:
$ sudo virsh version Compiled against library: libvirt 7.0.0 Using library: libvirt 7.0.0 Using API: QEMU 7.0.0 Running hypervisor: QEMU 5.2.0
Nesting Scenario:
Layer 0 - Ubuntu 21.04 Layer 1 - FreeBSD 12.2 Bhyve Host Layer 2 - FreeBSD 12.2 Guest
Result:
Virtual Machine Freezes (without work around of turning off APIC interrupt delivery as per existing diagnosis:
Workaround is to disable APIC virtual interrupt delivery:
1. Add entry into Layer 1 - FreeBSD Guest / Host: /boot/loader.conf: hw.vmm.vmx.use_apic_vid=0
Here is the libvirt log taken from Layer 0 - Ubuntu host:
<<Layer 0 - Ubuntu 21.04 QEMU / KVM Host>> 2021-05-16 09:57:28.970+0000: starting up libvirt version: 7.0.0, package: 2ubuntu2 (Christian Ehrhardt <email address hidden> Wed, 07 Apr 2021 13:33:46 +0200), qemu version: 5.2.0Debian 1:5.2+dfsg-9ubuntu3, kernel: 5.11.0-17-generic, hostname: green.in.graphica.com.au LC_ALL=C \ PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/snap/bin \ HOME=/var/lib/libvirt/qemu/domain-10-hive-dev-freebsd-12. \ XDG_DATA_HOME=/var/lib/libvirt/qemu/domain-10-hive-dev-freebsd-12./.local/share \ XDG_CACHE_HOME=/var/lib/libvirt/qemu/domain-10-hive-dev-freebsd-12./.cache \ XDG_CONFIG_HOME=/var/lib/libvirt/qemu/domain-10-hive-dev-freebsd-12./.config \ QEMU_AUDIO_DRV=spice \ /usr/bin/qemu-system-x86_64 \ -name guest=hive-dev-freebsd-12.2,debug-threads=on \ -S \ -object secret,id=masterKey0,format=raw,file=/var/lib/libvirt/qemu/domain-10-hive-dev-freebsd-12./master-key.aes \ -blockdev '{"driver":"file","filename":"/usr/share/OVMF/OVMF_CODE_4M.fd","node-name":"libvirt-pflash0-storage","auto-read-only":true,"discard":"unmap"}' \ -blockdev '{"node-name":"libvirt-pflash0-format","read-only":true,"driver":"raw","file":"libvirt-pflash0-storage"}' \ -blockdev '{"driver":"file","filename":"/home/jbh/Documents/virtual-machines/hive.dev.freebsd/OVMF_VARS.fd","node-name":"libvirt-pflash1-storage","auto-read-only":true,"discard":"unmap"}' \ -blockdev '{"node-name":"libvirt-pflash1-format","read-only":false,"driver":"raw","file":"libvirt-pflash1-storage"}' \ -machine pc-q35-5.2,accel=kvm,usb=off,vmport=off,dump-guest-core=off,pflash0=libvirt-pflash0-format,pflash1=libvirt-pflash1-format,memory-backend=pc.ram \ -cpu Broadwell-IBRS,vme=on,ss=on,vmx=on,pdcm=on,f16c=on,rdrand=on,hypervisor=on,arat=on,tsc-adjust=on,umip=on,md-clear=on,stibp=on,arch-capabilities=on,ssbd=on,xsaveopt=on,pdpe1gb=on,abm=on,ibpb=on,ibrs=on,amd-stibp=on,amd-ssbd=on,skip-l1dfl-vmentry=on,pschange-mc-no=on \ -m 4096 \ -object memory-backend-ram,id=pc.ram,size=4294967296 \ -overcommit mem-lock=off \ -smp 4,sockets=4,cores=1,threads=1 \ -uuid 459ff0b9-e0d1-44d4-9862-83315419eeee \ -no-user-config \ -nodefaults \ -chardev socket,id=charmonitor,fd=32,server,nowait \ -mon chardev=charmonitor,id=monitor,mode=control \ -rtc base=utc,driftfix=slew \ -global kvm-pit.lost_tick_policy=delay \ -no-hpet \ -no-shutdown \ -global ICH9-LPC.disable_s3=1 \ -global ICH9-LPC.disable_s4=1 \ -boot strict=on \ -device pcie-root-port,port=0x10,chassis=1,id=pci.1,bus=pcie.0,multifunction=on,addr=0x2 \ -device pcie-root-port,port=0x11,chassis=2,id=pci.2,bus=pcie.0,addr=0x2.0x1 \ -device pcie-root-port,port=0x12,chassis=3,id=pci.3,bus=pcie.0,addr=0x2.0x2 \ -device pcie-root-port,port=0x13,chassis=4,id=pci.4,bus=pcie.0,addr=0x2.0x3 \ -device pcie-root-port,port=0x14,chassis=5,id=pci.5,bus=pcie.0,addr=0x2.0x4 \ -device pcie-root-port,port=0x15,chassis=6,id=pci.6,bus=pcie.0,addr=0x2.0x5 \ -device pcie-pci-bridge,id=pci.7,bus=pci.1,addr=0x0 \ -device ich9-usb-ehci1,id=usb,bus=pcie.0,addr=0x1d.0x7 \ -device ich9-usb-uhci1,masterbus=usb.0,firstport=0,bus=pcie.0,multifunction=on,addr=0x1d \ -device ich9-usb-uhci2,masterbus=usb.0,firstport=2,bus=pcie.0,addr=0x1d.0x1 \ -device ich9-usb-uhci3,masterbus=usb.0,firstport=4,bus=pcie.0,addr=0x1d.0x2 \ -device virtio-serial-pci,id=virtio-serial0,bus=pci.2,addr=0x0 \ -device ide-cd,bus=ide.0,id=sata0-0-0,bootindex=1 \ -blockdev '{"driver":"file","filename":"/home/jbh/Documents/virtual-machines/hive.dev.freebsd/hive-hd1-01.qcow2","node-name":"libvirt-1-storage","auto-read-only":true,"discard":"unmap"}' \ -blockdev '{"node-name":"libvirt-1-format","read-only":false,"driver":"qcow2","file":"libvirt-1-storage","backing":null}' \ -device ide-hd,bus=ide.1,drive=libvirt-1-format,id=sata0-0-1,bootindex=2 \ -netdev tap,fd=35,id=hostnet0 \ -device vmxnet3,netdev=hostnet0,id=net0,mac=52:54:00:c8:8b:95,bus=pci.7,addr=0x1 \ -netdev tap,fd=36,id=hostnet1 \ -device vmxnet3,netdev=hostnet1,id=net1,mac=52:54:00:6c:c9:c1,bus=pci.7,addr=0x2 \ -chardev pty,id=charserial0 \ -device isa-serial,chardev=charserial0,id=serial0 \ -chardev spicevmc,id=charchannel0,name=vdagent \ -device virtserialport,bus=virtio-serial0.0,nr=1,chardev=charchannel0,id=channel0,name=com.redhat.spice.0 \ -device usb-kbd,id=input2,bus=usb.0,port=3 \ -spice port=5902,addr=127.0.0.1,disable-ticketing,image-compression=off,seamless-migration=on \ -device qxl-vga,id=video0,ram_size=67108864,vram_size=67108864,vram64_size_mb=0,vgamem_mb=16,max_outputs=1,bus=pcie.0,addr=0x1 \ -chardev spicevmc,id=charredir0,name=usbredir \ -device usb-redir,chardev=charredir0,id=redir0,bus=usb.0,port=1 \ -chardev spicevmc,id=charredir1,name=usbredir \ -device usb-redir,chardev=charredir1,id=redir1,bus=usb.0,port=2 \ -device virtio-balloon-pci,id=balloon0,bus=pci.4,addr=0x0 \ -sandbox on,obsolete=deny,elevateprivileges=deny,spawn=deny,resourcecontrol=deny \ -msg timestamp=on char device redirected to /dev/pts/2 (label charserial0) KVM internal error. Suberror: 1 emulation failure RAX=0000000000000000 RBX=0000000000000000 RCX=0000000000000000 RDX=0000000000000f00 RSI=0000000000000000 RDI=0000000000000000 RBP=0000000000000000 RSP=fffffe002d9f9700 R8 =0000000000000000 R9 =0000000000000000 R10=0000000000000000 R11=0000000000000000 R12=0000000000000000 R13=0000000000000000 R14=0000000000000000 R15=0000000000000000 RIP=ffffffff828fc5d9 RFL=00000046 [---Z-P-] CPL=0 II=0 A20=1 SMM=0 HLT=0 ES =003b 0000000000000000 ffffffff 00c0f300 DPL=3 DS [-WA] CS =0020 0000000000000000 ffffffff 00a09b00 DPL=0 CS64 [-RA] SS =0028 0000000000000000 ffffffff 00c09300 DPL=0 DS [-WA] DS =003b 0000000000000000 ffffffff 00c0f300 DPL=3 DS [-WA] FS =0013 0000000800b368d0 ffffffff 00c0f300 DPL=3 DS [-WA] GS =001b ffffffff82611000 ffffffff 00c0f300 DPL=3 DS [-WA] LDT=0000 0000000000000000 ffffffff 00c00000 TR =0048 ffffffff81f15e08 00002068 00008b00 DPL=0 TSS64-busy GDT= ffffffff81f1c608 00000067 IDT= ffffffff81f14da0 00000fff CR0=8005003b CR2=0000000000000000 CR3=0000000043b2b6ee CR4=003726e0 DR0=0000000000000000 DR1=0000000000000000 DR2=0000000000000000 DR3=0000000000000000 DR6=00000000ffff0ff0 DR7=0000000000000400 EFER=0000000000000d01 Code=50 4c 8b 67 58 4c 8b 6f 60 4c 8b 77 68 4c 8b 7f 70 48 8b 3f <0f> 01 c2 48 89 e7 b8 02 00 00 00 eb 07 b8 03 00 00 00 eb 00 41 bb 02 00 00 00 74 06 41 bb 2021-05-16T11:49:21.487500Z qemu-system-x86_64: terminating on signal 15 from pid 1885 (/usr/sbin/libvirtd) 2021-05-16 11:49:21.889+0000: shutting down, reason=destroyed <<END>>
So result is unchanged with update from: 20.04 -> 21.04 and FreeBSD 12.1 -> FreeBSD 12.2
NOTE: I also did testing of all 21.04 virtualization nesting: Layer 0 - Ubuntu 21.04 Layer 1 - Ubuntu 21.04 Layer 2 - Ubuntu 21.04 All Working OK !!
Also Christian, I see your name in log:
2021-05-16 09:57:28.970+0000: starting up libvirt version: 7.0.0, package: 2ubuntu2 (Christian Ehrhardt <email address hidden> Wed, 07 Apr 2021 13:33:46 +0200), qemu version: 5.2
Thanks for looking into this, please let me know if there is additional testing you would like to see.
Cheers,
John.
Hi Christian,
I have re-tested with Ubuntu 21.04 (Hirsute Hippo).
It took me a while to set up test environment.
Summary:
Ubuntu Version:
$ cat /etc/*-release RELEASE= 21.04 CODENAME= hirsute DESCRIPTION= "Ubuntu 21.04" /www.ubuntu. com/" /help.ubuntu. com/" /bugs.launchpad .net/ubuntu/" POLICY_ URL="https:/ /www.ubuntu. com/legal/ terms-and- policies/ privacy- policy" CODENAME= hirsute CODENAME= hirsute
DISTRIB_ID=Ubuntu
DISTRIB_
DISTRIB_
DISTRIB_
NAME="Ubuntu"
VERSION="21.04 (Hirsute Hippo)"
ID=ubuntu
ID_LIKE=debian
PRETTY_NAME="Ubuntu 21.04"
VERSION_ID="21.04"
HOME_URL="https:/
SUPPORT_URL="https:/
BUG_REPORT_URL="https:/
PRIVACY_
VERSION_
UBUNTU_
Linux Version:
$ uname -a
Linux green 5.11.0-17-generic #18-Ubuntu SMP Thu May 6 20:10:11 UTC 2021 x86_64 x86_64 x86_64 GNU/Linux
QEMU / Libvirt Version:
$ sudo virsh version
Compiled against library: libvirt 7.0.0
Using library: libvirt 7.0.0
Using API: QEMU 7.0.0
Running hypervisor: QEMU 5.2.0
Nesting Scenario:
Layer 0 - Ubuntu 21.04
Layer 1 - FreeBSD 12.2 Bhyve Host
Layer 2 - FreeBSD 12.2 Guest
Result:
Virtual Machine Freezes (without work around of turning off APIC interrupt delivery as per existing diagnosis:
Workaround is to disable APIC virtual interrupt delivery:
1. Add entry into Layer 1 - FreeBSD Guest / Host: /boot/loader.conf: vmx.use_ apic_vid= 0
hw.vmm.
Here is the libvirt log taken from Layer 0 - Ubuntu host:
<<Layer 0 - Ubuntu 21.04 QEMU / KVM Host>> 9ubuntu3, kernel: 5.11.0-17-generic, hostname: green.in. graphica. com.au local/sbin: /usr/local/ bin:/usr/ sbin:/usr/ bin:/sbin: /bin:/snap/ bin \ lib/libvirt/ qemu/domain- 10-hive- dev-freebsd- 12. \ HOME=/var/ lib/libvirt/ qemu/domain- 10-hive- dev-freebsd- 12./.local/ share \ HOME=/var/ lib/libvirt/ qemu/domain- 10-hive- dev-freebsd- 12./.cache \ HOME=/var/ lib/libvirt/ qemu/domain- 10-hive- dev-freebsd- 12./.config \ DRV=spice \ qemu-system- x86_64 \ dev-freebsd- 12.2,debug- threads= on \ id=masterKey0, format= raw,file= /var/lib/ libvirt/ qemu/domain- 10-hive- dev-freebsd- 12./master- key.aes \ :"file" ,"filename" :"/usr/ share/OVMF/ OVMF_CODE_ 4M.fd", "node-name" :"libvirt- pflash0- storage" ,"auto- read-only" :true," discard" :"unmap" }' \ name":" libvirt- pflash0- format" ,"read- only":true, "driver" :"raw", "file": "libvirt- pflash0- storage" }' \ :"file" ,"filename" :"/home/ jbh/Documents/ virtual- machines/ hive.dev. freebsd/ OVMF_VARS. fd","node- name":" libvirt- pflash1- storage" ,"auto- read-only" :true," discard" :"unmap" }' \ name":" libvirt- pflash1- format" ,"read- only":false, "driver" :"raw", "file": "libvirt- pflash1- storage" }' \ 5.2,accel= kvm,usb= off,vmport= off,dump- guest-core= off,pflash0= libvirt- pflash0- format, pflash1= libvirt- pflash1- format, memory- backend= pc.ram \ IBRS,vme= on,ss=on, vmx=on, pdcm=on, f16c=on, rdrand= on,hypervisor= on,arat= on,tsc- adjust= on,umip= on,md-clear= on,stibp= on,arch- capabilities= on,ssbd= on,xsaveopt= on,pdpe1gb= on,abm= on,ibpb= on,ibrs= on,amd- stibp=on, amd-ssbd= on,skip- l1dfl-vmentry= on,pschange- mc-no=on \ backend- ram,id= pc.ram, size=4294967296 \ 4,cores= 1,threads= 1 \ e0d1-44d4- 9862-83315419ee ee \ id=charmonitor, fd=32,server, nowait \ charmonitor, id=monitor, mode=control \ driftfix= slew \ lost_tick_ policy= delay \ disable_ s3=1 \ disable_ s4=1 \ port,port= 0x10,chassis= 1,id=pci. 1,bus=pcie. 0,multifunction =on,addr= 0x2 \ port,port= 0x11,chassis= 2,id=pci. 2,bus=pcie. 0,addr= 0x2.0x1 \ port,port= 0x12,chassis= 3,id=pci. 3,bus=pcie. 0,addr= 0x2.0x2 \ port,port= 0x13,chassis= 4,id=pci. 4,bus=pcie. 0,addr= 0x2.0x3 \ port,port= 0x14,chassis= 5,id=pci. 5,bus=pcie. 0,addr= 0x2.0x4 \ port,port= 0x15,chassis= 6,id=pci. 6,bus=pcie. 0,addr= 0x2.0x5 \ bridge, id=pci. 7,bus=pci. 1,addr= 0x0 \ ehci1,id= usb,bus= pcie.0, addr=0x1d. 0x7 \ uhci1,masterbus =usb.0, firstport= 0,bus=pcie. 0,multifunction =on,addr= 0x1d \ uhci2,masterbus =usb.0, firstport= 2,bus=pcie. 0,addr= 0x1d.0x1 \ uhci3,masterbus =usb.0, firstport= 4,bus=pcie. 0,addr= 0x1d.0x2 \ serial- pci,id= virtio- serial0, bus=pci. 2,addr= 0x0 \ bus=ide. 0,id=sata0- 0-0,bootindex= 1 \ :"file" ,"filename" :"/home/ jbh/Documents/ virtual- machines/ hive.dev. freebsd/ hive-hd1- 01.qcow2" ,"node- name":" libvirt- 1-storage" ,"auto- read-only" :true," discard" :"unmap" }' \ name":" libvirt- 1-format" ,"read- only":false, "driver" :"qcow2" ,"file" :"libvirt- 1-storage" ,"backing" :null}' \ bus=ide. 1,drive= libvirt- 1-format, id=sata0- 0-1,bootindex= 2 \ 35,id=hostnet0 \ netdev= hostnet0, id=net0, mac=52: 54:00:c8: 8b:95,bus= pci.7,addr= 0x1 \ 36,id=hostnet1 \ netdev= hostnet1, id=net1, mac=52: 54:00:6c: c9:c1,bus= pci.7,addr= 0x2 \ chardev= charserial0, id=serial0 \ id=charchannel0 ,name=vdagent \ bus=virtio- serial0. 0,nr=1, chardev= charchannel0, id=channel0, name=com. redhat. spice.0 \ id=input2, bus=usb. 0,port= 3 \ addr=127. 0.0.1,disable- ticketing, image-compressi on=off, seamless- migration= on \ id=video0, ram_size= 67108864, vram_size= 67108864, vram64_ size_mb= 0,vgamem_ mb=16,max_ outputs= 1,bus=pcie. 0,addr= 0x1 \ id=charredir0, name=usbredir \ chardev= charredir0, id=redir0, bus=usb. 0,port= 1 \ id=charredir1, name=usbredir \ chardev= charredir1, id=redir1, bus=usb. 0,port= 2 \ balloon- pci,id= balloon0, bus=pci. 4,addr= 0x0 \ deny,elevatepri vileges= deny,spawn= deny,resourceco ntrol=deny \ 00000 RBX=00000000000 00000 RCX=00000000000 00000 RDX=00000000000 00f00 00000 RDI=00000000000 00000 RBP=00000000000 00000 RSP=fffffe002d9 f9700 00000 R11=00000000000 00000 00000 R13=00000000000 00000 R14=00000000000 00000 R15=00000000000 00000 fc5d9 RFL=00000046 [---Z-P-] CPL=0 II=0 A20=1 SMM=0 HLT=0 00000 CR3=0000000043b 2b6ee CR4=003726e0 00000 DR1=00000000000 00000 DR2=00000000000 00000 DR3=00000000000 00000 f0ff0 DR7=00000000000 00400 000d01 16T11:49: 21.487500Z qemu-system-x86_64: terminating on signal 15 from pid 1885 (/usr/sbin/ libvirtd)
2021-05-16 09:57:28.970+0000: starting up libvirt version: 7.0.0, package: 2ubuntu2 (Christian Ehrhardt <email address hidden> Wed, 07 Apr 2021 13:33:46 +0200), qemu version: 5.2.0Debian 1:5.2+dfsg-
LC_ALL=C \
PATH=/usr/
HOME=/var/
XDG_DATA_
XDG_CACHE_
XDG_CONFIG_
QEMU_AUDIO_
/usr/bin/
-name guest=hive-
-S \
-object secret,
-blockdev '{"driver"
-blockdev '{"node-
-blockdev '{"driver"
-blockdev '{"node-
-machine pc-q35-
-cpu Broadwell-
-m 4096 \
-object memory-
-overcommit mem-lock=off \
-smp 4,sockets=
-uuid 459ff0b9-
-no-user-config \
-nodefaults \
-chardev socket,
-mon chardev=
-rtc base=utc,
-global kvm-pit.
-no-hpet \
-no-shutdown \
-global ICH9-LPC.
-global ICH9-LPC.
-boot strict=on \
-device pcie-root-
-device pcie-root-
-device pcie-root-
-device pcie-root-
-device pcie-root-
-device pcie-root-
-device pcie-pci-
-device ich9-usb-
-device ich9-usb-
-device ich9-usb-
-device ich9-usb-
-device virtio-
-device ide-cd,
-blockdev '{"driver"
-blockdev '{"node-
-device ide-hd,
-netdev tap,fd=
-device vmxnet3,
-netdev tap,fd=
-device vmxnet3,
-chardev pty,id=charserial0 \
-device isa-serial,
-chardev spicevmc,
-device virtserialport,
-device usb-kbd,
-spice port=5902,
-device qxl-vga,
-chardev spicevmc,
-device usb-redir,
-chardev spicevmc,
-device usb-redir,
-device virtio-
-sandbox on,obsolete=
-msg timestamp=on
char device redirected to /dev/pts/2 (label charserial0)
KVM internal error. Suberror: 1
emulation failure
RAX=00000000000
RSI=00000000000
R8 =0000000000000000 R9 =0000000000000000 R10=00000000000
R12=00000000000
RIP=ffffffff828
ES =003b 0000000000000000 ffffffff 00c0f300 DPL=3 DS [-WA]
CS =0020 0000000000000000 ffffffff 00a09b00 DPL=0 CS64 [-RA]
SS =0028 0000000000000000 ffffffff 00c09300 DPL=0 DS [-WA]
DS =003b 0000000000000000 ffffffff 00c0f300 DPL=3 DS [-WA]
FS =0013 0000000800b368d0 ffffffff 00c0f300 DPL=3 DS [-WA]
GS =001b ffffffff82611000 ffffffff 00c0f300 DPL=3 DS [-WA]
LDT=0000 0000000000000000 ffffffff 00c00000
TR =0048 ffffffff81f15e08 00002068 00008b00 DPL=0 TSS64-busy
GDT= ffffffff81f1c608 00000067
IDT= ffffffff81f14da0 00000fff
CR0=8005003b CR2=00000000000
DR0=00000000000
DR6=00000000fff
EFER=0000000000
Code=50 4c 8b 67 58 4c 8b 6f 60 4c 8b 77 68 4c 8b 7f 70 48 8b 3f <0f> 01 c2 48 89 e7 b8 02 00 00 00 eb 07 b8 03 00 00 00 eb 00 41 bb 02 00 00 00 74 06 41 bb
2021-05-
2021-05-16 11:49:21.889+0000: shutting down, reason=destroyed
<<END>>
So result is unchanged with update from: 20.04 -> 21.04 and FreeBSD 12.1 -> FreeBSD 12.2
NOTE: I also did testing of all 21.04 virtualization nesting:
Layer 0 - Ubuntu 21.04
Layer 1 - Ubuntu 21.04
Layer 2 - Ubuntu 21.04
All Working OK !!
Also Christian, I see your name in log:
2021-05-16 09:57:28.970+0000: starting up libvirt version: 7.0.0, package: 2ubuntu2 (Christian Ehrhardt <email address hidden> Wed, 07 Apr 2021 13:33:46 +0200), qemu version: 5.2
Thanks for looking into this, please let me know if there is additional testing you would like to see.
Cheers,
John.