kvm: freeze a guest with a NIC PCI passthrough

[AlexanderYT]

After attach NIC to guest domain, this domain freeze and CPU load 100% on host system.

# virsh start w

# top
top - 19:08:14 up 28 min, 1 user, load average: 1.00, 0.97, 0.75
Tasks: 277 total, 1 running, 276 sleeping, 0 stopped, 0 zombie
Cpu(s): 1.2%us, 1.2%sy, 0.0%ni, 97.5%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st
Mem: 99035416k total, 2537044k used, 96498372k free, 84844k buffers
Swap: 0k total, 0k used, 0k free, 484432k cached

  PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND
21168 libvirt- 20 0 837m 544m 6916 S 103 0.6 9:57.66 kvm

# cat /etc/libvirt/qemuw.xml
<!--
WARNING: THIS IS AN AUTO-GENERATED FILE. CHANGES TO IT ARE LIKELY TO BE
OVERWRITTEN AND LOST. Changes to this xml configuration should be made using:
  virsh edit w
or other application using the libvirt API.
-->

<domain type='kvm'>
  <name>w</name>
  <uuid>640ff4d4-665a-5a73-9e85-aecd4e75f18f</uuid>
  <memory>524288</memory>
  <currentMemory>524288</currentMemory>
  <vcpu>1</vcpu>
  <os>
    <type arch='x86_64' machine='pc-1.0'>hvm</type>
    <boot dev='cdrom'/>
  </os>
  <features>
    <acpi/>
    <apic/>
    <pae/>
  </features>
  <cpu match='exact'>
    <model>Westmere</model>
    <vendor>Intel</vendor>
    <feature policy='require' name='tm2'/>
    <feature policy='require' name='est'/>
    <feature policy='require' name='vmx'/>
    <feature policy='require' name='ds'/>
    <feature policy='require' name='ss'/>
    <feature policy='require' name='vme'/>
    <feature policy='require' name='rdtscp'/>
    <feature policy='require' name='ht'/>
    <feature policy='require' name='dca'/>
    <feature policy='require' name='pbe'/>
    <feature policy='require' name='tm'/>
    <feature policy='require' name='monitor'/>
    <feature policy='require' name='pdpe1gb'/>
    <feature policy='require' name='ds_cpl'/>
    <feature policy='require' name='xtpr'/>
    <feature policy='require' name='acpi'/>
    <feature policy='require' name='x2apic'/>
  </cpu>
  <clock offset='utc'/>
  <on_poweroff>destroy</on_poweroff>
  <on_reboot>restart</on_reboot>
  <on_crash>restart</on_crash>
  <devices>
    <emulator>/usr/bin/kvm</emulator>
    <disk type='file' device='disk'>
      <driver name='qemu' type='raw' cache='writeback'/>
      <source file='/var/lib/libvirt/images/w.img'/>
      <target dev='vda' bus='virtio'/>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x04' function='0x0'/>
    </disk>
    <disk type='file' device='cdrom'>
      <driver name='qemu' type='raw'/>
      <source file='/home/sanches/sysrescue33.iso'/>
      <target dev='hdc' bus='ide'/>
      <readonly/>
      <address type='drive' controller='0' bus='1' unit='0'/>
    </disk>
    <controller type='ide' index='0'>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x01' function='0x1'/>
    </controller>
    <interface type='network'>
      <mac address='52:54:00:08:ad:b9'/>
      <source network='default'/>
      <model type='virtio'/>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x03' function='0x0'/>
    </interface>
    <serial type='pty'>
      <target port='0'/>
    </serial>
    <console type='pty'>
      <target type='serial' port='0'/>
    </console>
    <input type='mouse' bus='ps2'/>
    <graphics type='vnc' port='5932' autoport='no' listen='10.1.110.133' passwd='123'>
      <listen type='address' address='10.1.110.133'/>
    </graphics>
    <video>
      <model type='cirrus' vram='9216' heads='1'/>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x0'/>
    </video>
    <hostdev mode='subsystem' type='pci' managed='yes'>
      <source>
        <address domain='0x0000' bus='0x03' slot='0x00' function='0x1'/>
      </source>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x06' function='0x0'/>
    </hostdev>
    <memballoon model='virtio'>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x05' function='0x0'/>
    </memballoon>
  </devices>
</domain>

# strace
read(21, 0x7fffd9558200, 16) = -1 EAGAIN (Resource temporarily unavailable)
read(7, "\16\0\0\0\0\0\0\0\376\377\377\377\0\0\0\0\0\0\0\0\0\0\0\0\1\0\0\0\0\0\0\0"..., 128) = 128
rt_sigaction(SIGALRM, NULL, {0x7f21895d1250, ~[KILL STOP RTMIN RT_1], SA_RESTORER, 0x7f2185fa7cb0}, 8) = 0
write(9, "\1\0\0\0\0\0\0\0", 8) = 8
read(7, 0x7fffd9558180, 128) = -1 EAGAIN (Resource temporarily unavailable)
timer_gettime(0x1, {it_interval={0, 0}, it_value={0, 0}}) = 0
timer_settime(0x1, 0, {it_interval={0, 0}, it_value={0, 8640000}}, NULL) = 0
futex(0x7f218a223460, FUTEX_WAKE_PRIVATE, 1) = 1
select(24, [7 8 11 15 21 22 23], [], [], {1, 0}) = 1 (in [8], left {0, 999998})
read(21, 0x7fffd9558200, 16) = -1 EAGAIN (Resource temporarily unavailable)
read(8, "\1\0\0\0\0\0\0\0", 512) = 8
futex(0x7f218a223460, FUTEX_WAKE_PRIVATE, 1) = 1
select(24, [7 8 11 15 21 22 23], [], [], {1, 0}) = 1 (in [7], left {0, 991512})
read(21, 0x7fffd9558200, 16) = -1 EAGAIN (Resource temporarily unavailable)
read(7, "\16\0\0\0\0\0\0\0\376\377\377\377\0\0\0\0\0\0\0\0\0\0\0\0\1\0\0\0\0\0\0\0"..., 128) = 128
rt_sigaction(SIGALRM, NULL, {0x7f21895d1250, ~[KILL STOP RTMIN RT_1], SA_RESTORER, 0x7f2185fa7cb0}, 8) = 0
write(9, "\1\0\0\0\0\0\0\0", 8) = 8
read(7, 0x7fffd9558180, 128) = -1 EAGAIN (Resource temporarily unavailable)
timer_gettime(0x1, {it_interval={0, 0}, it_value={0, 0}}) = 0
timer_settime(0x1, 0, {it_interval={0, 0}, it_value={0, 250000}}, NULL) = 0
timer_gettime(0x1, {it_interval={0, 0}, it_value={0, 228319}}) = 0
futex(0x7f218a223460, FUTEX_WAKE_PRIVATE, 1) = 1
select(24, [7 8 11 15 21 22 23], [], [], {1, 0}) = 1 (in [8], left {0, 999998})
read(21, 0x7fffd9558200, 16) = -1 EAGAIN (Resource temporarily unavailable)
read(8, "\1\0\0\0\0\0\0\0", 512) = 8
futex(0x7f218a223460, FUTEX_WAKE_PRIVATE, 1) = 1
select(24, [7 8 11 15 21 22 23], [], [], {1, 0}) = 1 (in [7], left {0, 999936})
read(21, 0x7fffd9558200, 16) = -1 EAGAIN (Resource temporarily unavailable)
read(7, "\16\0\0\0\0\0\0\0\376\377\377\377\0\0\0\0\0\0\0\0\0\0\0\0\1\0\0\0\0\0\0\0"..., 128) = 128
rt_sigaction(SIGALRM, NULL, {0x7f21895d1250, ~[KILL STOP RTMIN RT_1], SA_RESTORER, 0x7f2185fa7cb0}, 8) = 0
write(9, "\1\0\0\0\0\0\0\0", 8) = 8
read(7, 0x7fffd9558180, 128) = -1 EAGAIN (Resource temporarily unavailable)
timer_gettime(0x1, {it_interval={0, 0}, it_value={0, 0}}) = 0
timer_settime(0x1, 0, {it_interval={0, 0}, it_value={0, 763493546}}, NULL) = 0
futex(0x7f218a223460, FUTEX_WAKE_PRIVATE, 1) = 1
select(24, [7 8 11 15 21 22 23], [], [], {1, 0}) = 1 (in [8], left {0, 999998})
read(21, 0x7fffd9558200, 16) = -1 EAGAIN (Resource temporarily unavailable)
read(8, "\1\0\0\0\0\0\0\0", 512) = 8
futex(0x7f218a223460, FUTEX_WAKE_PRIVATE, 1) = 1
select(24, [7 8 11 15 21 22 23], [], [], {1, 0}) = 1 (in [7], left {0, 236654})
read(21, 0x7fffd9558200, 16) = -1 EAGAIN (Resource temporarily unavailable)
read(7, "\16\0\0\0\0\0\0\0\376\377\377\377\0\0\0\0\0\0\0\0\0\0\0\0\1\0\0\0\0\0\0\0"..., 128) = 128
rt_sigaction(SIGALRM, NULL, {0x7f21895d1250, ~[KILL STOP RTMIN RT_1], SA_RESTORER, 0x7f2185fa7cb0}, 8) = 0
write(9, "\1\0\0\0\0\0\0\0", 8) = 8
read(7, 0x7fffd9558180, 128) = -1 EAGAIN (Resource temporarily unavailable)
timer_gettime(0x1, {it_interval={0, 0}, it_value={0, 0}}) = 0
timer_settime(0x1, 0, {it_interval={0, 0}, it_value={0, 250000}}, NULL) = 0
timer_gettime(0x1, {it_interval={0, 0}, it_value={0, 225636}}) = 0
futex(0x7f218a223460, FUTEX_WAKE_PRIVATE, 1) = 1
select(24, [4 7 8 11 15 21 22 23], [], [], {1, 0}) = 2 (in [4 8], left {0, 999996})
read(21, 0x7fffd9558200, 16) = -1 EAGAIN (Resource temporarily unavailable)
read(4, 0x7fffd9557210, 1) = -1 EIO (Input/output error)
timer_gettime(0x1, {it_interval={0, 0}, it_value={0, 87734}}) = 0
read(8, "\1\0\0\0\0\0\0\0", 512) = 8
futex(0x7f218a223460, FUTEX_WAKE_PRIVATE, 1) = 1
select(24, [7 8 11 15 21 22 23], [], [], {1, 0}) = 1 (in [7], left {0, 999986})
read(21, 0x7fffd9558200, 16) = -1 EAGAIN (Resource temporarily unavailable)
read(7, "\16\0\0\0\0\0\0\0\376\377\377\377\0\0\0\0\0\0\0\0\0\0\0\0\1\0\0\0\0\0\0\0"..., 128) = 128
rt_sigaction(SIGALRM, NULL, {0x7f21895d1250, ~[KILL STOP RTMIN RT_1], SA_RESTORER, 0x7f2185fa7cb0}, 8) = 0
write(9, "\1\0\0\0\0\0\0\0", 8) = 8
read(7, 0x7fffd9558180, 128) = -1 EAGAIN (Resource temporarily unavailable)
timer_gettime(0x1, {it_interval={0, 0}, it_value={0, 0}}) = 0
timer_settime(0x1, 0, {it_interval={0, 0}, it_value={0, 224423000}}, NULL) = 0
futex(0x7f218a223460, FUTEX_WAKE_PRIVATE, 1) = 1
select(24, [7 8 11 15 21 22 23], [], [], {1, 0}) = 1 (in [8], left {0, 999998})
read(21, 0x7fffd9558200, 16) = -1 EAGAIN (Resource temporarily unavailable)
read(8, "\1\0\0\0\0\0\0\0", 512) = 8
futex(0x7f218a223460, FUTEX_WAKE_PRIVATE, 1) = 1
select(24, [7 8 11 15 21 22 23], [], [], {1, 0} <unfinished ...>

ProblemType: Bug
DistroRelease: Ubuntu 12.04
Package: qemu-kvm 1.0+noroms-0ubuntu14.8
ProcVersionSignature: Ubuntu 3.2.0-43.68-generic 3.2.42
Uname: Linux 3.2.0-43-generic x86_64
ApportVersion: 2.0.1-0ubuntu17.2
Architecture: amd64
Date: Sun May 19 19:02:24 2013
ExecutablePath: /usr/bin/qemu-system-x86_64
MachineType: HP ProLiant DL380p Gen8
MarkForUpload: True
ProcEnviron: PATH=(custom, no user)
ProcKernelCmdLine: BOOT_IMAGE=/boot/vmlinuz-3.2.0-43-generic root=/dev/mapper/CtrlSlot0--raid5-root ro console=tty0 console=ttyS1,115200 nomodeset intel_iommu=on
SourcePackage: qemu-kvm
UpgradeStatus: No upgrade log present (probably fresh install)
dmi.bios.date: 02/25/2012
dmi.bios.vendor: HP
dmi.bios.version: P70
dmi.chassis.type: 23
dmi.chassis.vendor: HP
dmi.modalias: dmi:bvnHP:bvrP70:bd02/25/2012:svnHP:pnProLiantDL380pGen8:pvr:cvnHP:ct23:cvr:
dmi.product.name: ProLiant DL380p Gen8
dmi.sys.vendor: HP

[Serge Hallyn (serge-hallyn)]

Thanks for reporting this bug.

In order to help me reproduce it, could you please tell us in detail how you are attaching the NIC, and what OS/release is running inside the guest?

[AlexanderYT (arcam)]

Hello.
This is the step by step history of commands. The domain "w" is not rinnung.
# virsh list --all
 Id Name State
----------------------------------
  - w shut off

There are all NICs on the host system
# virsh nodedev-list --tree | grep -A 18 '+- pci_0000_00_02_0'
  +- pci_0000_00_02_0
  | |
  | +- pci_0000_03_00_0
  | | |
  | | +- net_eth0_2c_76_8a_4f_45_00
  | |
  | +- pci_0000_03_00_1
  | | |
  | | +- net_eth1_2c_76_8a_4f_45_01
  | |
  | +- pci_0000_03_00_2
  | | |
  | | +- net_eth2_2c_76_8a_4f_45_02
  | |
  | +- pci_0000_03_00_3
  | |
  | +- net_eth3_2c_76_8a_4f_45_03

Dump xml for eht1
# virsh nodedev-dumpxml pci_0000_03_00_1
<device>
  <name>pci_0000_03_00_1</name>
  <parent>pci_0000_00_02_0</parent>
  <driver>
    <name>tg3</name>
  </driver>
  <capability type='pci'>
    <domain>0</domain>
    <bus>3</bus>
    <slot>0</slot>
    <function>1</function>
    <product id='0x1657'>NetXtreme BCM5719 Gigabit Ethernet PCIe</product>
    <vendor id='0x14e4'>Broadcom Corporation</vendor>
    <capability type='virt_functions'>
    </capability>
  </capability>
</device>

Add the host device
# virsh edit w
...
    <hostdev mode='subsystem' type='pci' managed='yes'>
      <source>
        <address domain='0x0000' bus='0x03' slot='0x00' function='0x1'/>
      </source>
    </hostdev>
...

Dettach the NIC from host
# virsh nodedev-dettach pci_0000_03_00_1

# virsh start w

The image file of guest OS is not contain a OS, this is only a RAW partition. I'm trying to boot the guest with live cd ISO.
But problem is while running seabios or iPXE (IMHO). Please, see the screenshot.

[Serge Hallyn (serge-hallyn)]

Unfortunately I don't have any hardware to reproduce this with.

Could you please:

1. Attach /var/log/libvirt/qemu/w.log

2. Try running this vm with kvm by hand as per http://www.linux-kvm.org/page/How_to_assign_devices_with_VT-d_in_KVM ?
     (i.e. the kvm command in step 6, something like "kvm ... -device pci-assign,host=xx:yy.z)

[Serge Hallyn (serge-hallyn)]

Would you be able to test this with upstream qemu?

git clone git://git.qemu.org/qemu.git
cd qemu
./configure --target-list=x86_64-softmmu
make -j4
cd x86_64-softmmu
./qemu-system-x86_64 -enable-kvm -m 512 -device pci-assign,host=03:00.1 -boot d

If that works, then we just need to look for the commit which fixed it. If not, then either there is an existing bug upstream, or some bios setting is likely not right on your system.

[AlexanderYT (arcam)]

I tried upstream qemu. But the issue not solved yet.
On Centos it work fine. qemu in Centos use gPXE instead iPXE. Please see screenshot.

[Serge Hallyn (serge-hallyn)]

Thanks! So it sounds like the problem is actually in ipxe.

Saucy has a new ipxe (from 9 days ago). However simply installing it alongside precise's qemu-kvm probably won't work.

(I do see a few commits in the git tree which could perhaps fix this) Is it possible for you to test this with saucy, perhaps installing on a spare partition or in a chroot?

[AlexanderYT (arcam)]

Yes, I have a possibility.

[AlexanderYT (arcam)]

I tried qemu in saucy. No issue found. IPXE works fine.

[Serge Hallyn (serge-hallyn)]

Thanks - so we can mark this fix released once the nomination for precise is accepted. We'll have to look through the git tree for the fix, but great to know the fix is in there.

[Marius (marius-m)]

I have the same problem on a ProLiant ML350p Gen8 when trying to install (using virt-install with PCI passthrough --host-device=....etc) using the integrated Broadcom NetXtreme BCM5719.

I also have an Intel dual port I350 on the system and with it everything is works as expected (virt-install with PCI passthrough).

[Serge Hallyn (serge-hallyn)]

Interesting to note the original bug reporter also had BCM5719

[Serge Hallyn (serge-hallyn)]

An ipxe package is building in ppa:serge-hallyn/virt which has a newer broadcom driver. See https://launchpad.net/~serge-hallyn/+archive/virt for details.

When it builds, please grab kvm-ipxe_1.0.0+git-3.55f6c88-0ubuntu2.12.04.1~ppa1_all.deb (and, if you have ipxe itself installed, also ipxe_1.0.0+git-3.55f6c88-0ubuntu2.12.04.1~ppa1_all.deb) and install them with 'dpkg -i', and let us know if that fixes the hang. If you need more detailed instructions, please let me know and I will send them as soon as the ppa has built the packages.

Thank you!

[AlexanderYT (arcam)]

Thank you Serge!
The issue not solved.

# aptitude show ipxe | grep -i version
Version: 1.0.0+git-3.55f6c88-0ubuntu2.12.04.1~ppa1

# aptitude show kvm-ipxe | grep -i version
Version: 1.0.0+git-3.55f6c88-0ubuntu2.12.04.1~ppa1

Strace log:
---cut----
select(16, [5 6 15], [], [], {0, 987950}) = 1 (in [5], left {0, 970520})
read(15, 0x7fffb8dcc480, 16) = -1 EAGAIN (Resource temporarily unavailable)
read(5, "\16\0\0\0\0\0\0\0\376\377\377\377\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0"..., 128) = 128
rt_sigaction(SIGALRM, NULL, {0x7fc3dedf4250, ~[KILL STOP RTMIN RT_1], SA_RESTORER, 0x7fc3db7c9cb0}, 8) = 0
write(7, "\1\0\0\0\0\0\0\0", 8) = 8
read(5, 0x7fffb8dcc400, 128) = -1 EAGAIN (Resource temporarily unavailable)
timer_gettime(0, {it_interval={0, 0}, it_value={0, 0}}) = 0
timer_settime(0, 0, {it_interval={0, 0}, it_value={0, 250000}}, NULL) = 0
recvfrom(10, 0x7fc3e00d3ec4, 4096, 0, 0, 0) = -1 EAGAIN (Resource temporarily unavailable)
select(11, [10], NULL, NULL, {0, 0}) = 0 (Timeout)
timer_gettime(0, {it_interval={0, 0}, it_value={0, 951}}) = 0
futex(0x7fc3dfa46460, FUTEX_WAKE_PRIVATE, 1) = 1
select(16, [5 6 15], [], [], {1, 0}) = 2 (in [5 6], left {0, 999998})
read(15, 0x7fffb8dcc480, 16) = -1 EAGAIN (Resource temporarily unavailable)
read(6, "\1\0\0\0\0\0\0\0", 512) = 8
read(5, "\16\0\0\0\0\0\0\0\376\377\377\377\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0"..., 128) = 128
rt_sigaction(SIGALRM, NULL, {0x7fc3dedf4250, ~[KILL STOP RTMIN RT_1], SA_RESTORER, 0x7fc3db7c9cb0}, 8) = 0
write(7, "\1\0\0\0\0\0\0\0", 8) = 8
read(5, 0x7fffb8dcc400, 128) = -1 EAGAIN (Resource temporarily unavailable)
timer_gettime(0, {it_interval={0, 0}, it_value={0, 0}}) = 0
timer_settime(0, 0, {it_interval={0, 0}, it_value={0, 28933959}}, NULL) = 0
futex(0x7fc3dfa46460, FUTEX_WAKE_PRIVATE, 1) = 1
select(16, [5 6 15], [], [], {1, 0}) = 1 (in [6], left {0, 999998})
read(15, 0x7fffb8dcc480, 16) = -1 EAGAIN (Resource temporarily unavailable)
read(6, "\1\0\0\0\0\0\0\0", 512) = 8
futex(0x7fc3dfa46460, FUTEX_WAKE_PRIVATE, 1) = 1
select(16, [5 6 15], [], [], {1, 0}) = 1 (in [5], left {0, 971287})
read(15, 0x7fffb8dcc480, 16) = -1 EAGAIN (Resource temporarily unavailable)
read(5, "\16\0\0\0\0\0\0\0\376\377\377\377\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0"..., 128) = 128
rt_sigaction(SIGALRM, NULL, {0x7fc3dedf4250, ~[KILL STOP RTMIN RT_1], SA_RESTORER, 0x7fc3db7c9cb0}, 8) = 0
write(7, "\1\0\0\0\0\0\0\0", 8) = 8
read(5, 0x7fffb8dcc400, 128) = -1 EAGAIN (Resource temporarily unavailable)
timer_gettime(0, {it_interval={0, 0}, it_value={0, 0}}) = 0
timer_settime(0, 0, {it_interval={0, 0}, it_value={0, 250000}}, NULL) = 0
recvfrom(10, 0x7fc3e00d3ec4, 4096, 0, 0, 0) = -1 EAGAIN (Resource temporarily unavailable)
select(11, [10], NULL, NULL, {0, 0}) = 0 (Timeout)
timer_gettime(0, {it_interval={0, 0}, it_value={0, 173410}}) = 0
futex(0x7fc3dfa46460, FUTEX_WAKE_PRIVATE, 1) = 1
select(16, [5 6 15], [], [], {1, 0}) = 1 (in [6], left {0, 999998})
read(15, 0x7fffb8dcc480, 16) = -1 EAGAIN (Resource temporarily unavailable)
read(6, "\1\0\0\0\0\0\0\0", 512) ...

[Serge Hallyn (serge-hallyn)]

Thanks for testing. That's a shame. I had hoped the new driver for your card would fix it. I'll try some more testing to see if this is a more general problem that I can somehow reproduce.

[Serge Hallyn (serge-hallyn)]

I've tried some more setups like having a virtio nic on virbr0 and a second e1000 nic on br0, where only br0 was on a network with a pxeboot server - but wasn't able to reproduce. So it appears to be purely PCI passthrough.

I'm building a package (in the same ppa) which is the saucy ipxe, built for precise. It built and installed fine for me locally. I'll post another comment when it is ready for testing.

[Serge Hallyn (serge-hallyn)]

Hi,

the backported saucy ipxe packages have finished building in my ppa. Please download
them using

 wget https://launchpad.net/~serge-hallyn/+archive/virt/+files/kvm-ipxe_1.0.0%2Bgit-20130710.936134e-0ubuntu1.12.04.1%7Eppa1_all.deb
 wget https://launchpad.net/~serge-hallyn/+archive/virt/+files/ipxe-qemu_1.0.0%2Bgit-20130710.936134e-0ubuntu1.12.04.1%7Eppa1_all.deb
 wget https://launchpad.net/~serge-hallyn/+archive/virt/+files/ipxe_1.0.0%2Bgit-20130710.936134e-0ubuntu1.12.04.1%7Eppa1_all.deb

then install them using 'dpkg -i'.

(Note to self - if this solves the problem, then we'll need to finagle the
package to move ipxe-qemu back into kvm-ipxe)

[AlexanderYT (arcam)]

Hi,
not solved yet.

  PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND
10710 root 20 0 805m 543m 6724 S 100 0.6 0:10.81 qemu-system-x86

# apt-file search qemu-system-x86
qemu-kvm: /usr/bin/qemu-system-x86_64
qemu-kvm: /usr/share/man/man1/qemu-system-x86_64.1.gz
qemu-kvm-spice: /usr/bin/qemu-system-x86_64-spice

Not sure, but may be the problem in qemu-kvm package?

[Serge Hallyn (serge-hallyn)]

I don't believe it can be the qemu-kvm package, since upstream qemu git HEAD didn't fix it, while it wasn't broken on centos.

Can you show 'dpkg -l | grep ipxe' on the system from comment #19?

If the new ipxe didn't solve it, then my next guess would be seabios...

[AlexanderYT (arcam)]

dpkg -l | grep ipxe
ii ipxe 1.0.0+git-20130710.936134e-0ubuntu1.12.04.1~ppa1 PXE boot firmware
ii ipxe-qemu 1.0.0+git-20130710.936134e-0ubuntu1.12.04.1~ppa1 Virtual package to support use of kvm-ipxe with qemu
ii kvm-ipxe 1.0.0+git-20130710.936134e-0ubuntu1.12.04.1~ppa1 transitional dummy package

[Serge Hallyn (serge-hallyn)]

Does it solve your problem if you grab and install

https://launchpad.net/ubuntu/+source/seabios/1.7.3-1/+build/4779258/+files/seabios_1.7.3-1_all.deb

on the precise system?

You said upstream qemu git head reproduced this - have you been able to
test the Ubuntu raring and saucy (13.04 and 13.10) releases?

[AlexanderYT (arcam)]

Not solved yet. Please see the screenshot.

# dpkg -l | grep -e ipxe -e seab
ii ipxe 1.0.0+git-20130710.936134e-0ubuntu1.12.04.1~ppa1 PXE boot firmware
ii ipxe-qemu 1.0.0+git-20130710.936134e-0ubuntu1.12.04.1~ppa1 Virtual package to support use of kvm-ipxe with qemu
ii kvm-ipxe 1.0.0+git-20130710.936134e-0ubuntu1.12.04.1~ppa1 transitional dummy package
ii seabios 1.7.3-1 Legacy BIOS implementation

I tested 13.04 and 13.10 full installation (#10 was the test in chroot in 12.04).
But I got new problem with
# virsh nodedev-detach pci_0000_02_00_1
After this command I found some messages in dmesg about kernel errors (please see attach 1304 and 1310)

[AlexanderYT (arcam)]

About #22 #25: https://bugzilla.kernel.org/show_bug.cgi?id=60850

[Nigel Jones (dev-nigelj)]

After getting the bug mail for this bug and reading it with interest, I happened to run into this issue today myself and did some investigation at a seabios level.

My original hit of the issue was with RHEL, but I reproduced it with the same card (Realtek Semiconductor Co., Ltd. RTL8111/8168/8411 PCI Express Gigabit Ethernet Controller (PCI ID 10ec:8168)) on Ubuntu saucy.

The logs in QEMU logs list however the follow:

2013-09-18 11:09:37.307+0000: starting up
LC_ALL=C PATH=/usr/local/sbin:/usr/local/bin:/usr/bin:/usr/sbin:/sbin:/bin QEMU_AUDIO_DRV=none /usr/bin/kvm-spice -name test -S -machine pc-i440fx-1.5,accel=kvm,usb=off -m 1024 -realtime mlock=off -smp 1,sockets=1,cores=1,threads=1 -uuid e2481510-dcb8-8bc3-8cc5-dfd51f5d73d7 -no-user-config -nodefaults -chardev socket,id=charmonitor,path=/var/lib/libvirt/qemu/test.monitor,server,nowait -mon chardev=charmonitor,id=monitor,mode=control -rtc base=utc -no-shutdown -device piix3-usb-uhci,id=usb,bus=pci.0,addr=0x1.0x2 -drive file=/var/lib/libvirt/images/test.img,if=none,id=drive-ide0-0-0,format=raw -device ide-hd,bus=ide.0,unit=0,drive=drive-ide0-0-0,id=ide0-0-0,bootindex=1 -drive file=/home/njones/pfSense-LiveCD-2.1-RELEASE-amd64.iso,if=none,id=drive-ide0-1-0,readonly=on,format=raw -device ide-cd,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0 -chardev pty,id=charserial0 -device isa-serial,chardev=charserial0,id=serial0 -vnc 127.0.0.1:0 -vga cirrus -device intel-hda,id=sound0,bus=pci.0,addr=0x3 -device hda-duplex,id=sound0-codec0,bus=sound0.0,cad=0 -device pci-assign,configfd=24,host=06:00.0,id=hostdev0,bus=pci.0,addr=0x5 -device virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x4
W: kvm binary is deprecated, please use qemu-system-x86_64 instead
char device redirected to /dev/pts/0 (label charserial0)
qemu-system-x86_64: -device pci-assign,configfd=24,host=06:00.0,id=hostdev0,bus=pci.0,addr=0x5: pci-assign: Cannot read from host /sys/bus/pci/devices/0000:06:00.0/rom
Device option ROM contents are probably invalid (check dmesg).
Skip option ROM probe with rombar=0, or load from file with romfile=
[after virsh destroy]
qemu: terminating on signal 15 from pid 1143
2013-09-18 11:13:54.183+0000: shutting down

At the same time, dmesg reports:

[ 412.463287] pci-stub 0000:06:00.0: claimed by stub
[ 413.177444] type=1400 audit(1379502837.636:78): apparmor="STATUS" operation="profile_load" parent=2874 profile="unconfined" name="libvirt-e2481510-dcb8-8bc3-8cc5-dfd51f5d73d7" pid=2875 comm="apparmor_parser"
[ 413.178897] type=1400 audit(1379502837.636:79): apparmor="DENIED" operation="capable" parent=1 profile="/usr/sbin/libvirtd" pid=1148 comm="libvirtd" pid=1148 comm="libvirtd" capability=29 capname="audit_write"
[ 414.330150] assign device 0:6:0.0
[ 414.330585] pci-stub 0000:06:00.0: Invalid ROM contents

virsh edit'ing the VM and adding "<rom bar='off'/>" to the <hostdev> entry to read:

    <hostdev mode='subsystem' type='pci' managed='yes'>
      <source>
        <address domain='0x0000' bus='0x06' slot='0x00' function='0x0'/>
      </source>
      <rom bar='off'/>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x05' function='0x0'/>
    </hostdev>

caused the message to go a...

[Serge Hallyn (serge-hallyn)]

Thanks, Nigel.

Alexander,

can you confirm whether adding "<rom bar='off'/> to your hostdev entry works around the bug for you as well?

[AlexanderYT (arcam)]

Yes. It's works for me. But I can't find the message about ROM invalid:
grep -i -e invalid -e " rom "
[ 0.000000] ACPI Warning: Invalid length for Pm1aControlBlock: 32, using default 16 (20110623/tbfadt-610)
[ 0.000000] ACPI Warning: Invalid length for Pm2ControlBlock: 32, using default 8 (20110623/tbfadt-610)
[ 6.233547] IOMMU 0 0xfbdfe000: using Queued invalidation
[ 6.238509] IOMMU 1 0xf4ffe000: using Queued invalidation

Thanks all!

[Launchpad Janitor (janitor)]

Status changed to 'Confirmed' because the bug affects multiple users.

[Pierre Schweitzer (pierre-jean-schweitzer)]

We're also affected by this bug here. You can actually see my questioning on libvirt-users about this: https://www.redhat.com/archives/libvirt-users/2014-July/msg00134.html
Using <rom bar='off'/> "fixes" it.

[Pierre Schweitzer (pierre-jean-schweitzer)]

This workaround unfortunately appear to totally break the network card in the VM making it unusable. TG3 driver (which is supposed to handle it) cannot properly communicate with it.

The kernel outputs the following lines:
Jul 30 13:32:03 carrier-commander kernel: [ 0.981658] tg3 0000:00:05.0: PCI INT B -> Link[LNKB] -> GSI 10 (level, high) -> IRQ 10
Jul 30 13:32:03 carrier-commander kernel: [ 0.981722] tg3 0000:00:05.0: setting latency timer to 64
Jul 30 13:32:03 carrier-commander kernel: [ 0.988207] tg3 0000:00:05.0: phy probe failed, err -19
Jul 30 13:32:03 carrier-commander kernel: [ 0.993954] tg3 0000:00:05.0: Problem fetching invariants of chip, aborting
Jul 30 13:32:03 carrier-commander kernel: [ 1.047883] tg3 0000:00:05.0: Refused to change power state, currently in D0
Jul 30 13:32:03 carrier-commander kernel: [ 1.047916] tg3 0000:00:05.0: PCI INT B disabled

The weird thing being this "-19" error, ie, -ENODEV.

[Pierre Schweitzer (pierre-jean-schweitzer)]

This also affects Trusty.

[Steve Langasek (vorlon)]

The Precise Pangolin has reached end of life, so this bug will not be fixed for that release