We recently encountered an issue when trying to stop nova instances.
| Apr 16 09:35:10 ginzel kernel: [6925958.071665] unregister_netdevice: waiting for tape3e33cb9-d6 to become free. Usage count = 2
When I try to stop manually the qemu process, it errors
"""
ballot@ginzel:~$ sudo virsh destroy instance-000718af
setlocale: No such file or directory
error: Failed to destroy domain instance-000718af
error: Failed to terminate process 20953 with SIGKILL: Device or resource busy
"""
The qemu process is in D state because of this
"""
libvirt+ 20953 12.9 0.5 9001400 2925908 ? D Apr15 84:25 /usr/bin/qemu-system-x86_64 -name instance-000718af -S -machine pc-i440fx-trusty,accel=kvm,usb=off -cpu EPYC-IBPB-2.0,+perfctr_nb,+perfctr_core,+t
opoext,+tce,+wdt,+skinit,+extapic,+cmp_legacy,+osxsave,+ht -m 4096 -realtime mlock=off -smp 2,sockets=2,cores=1,threads=1 -uuid 020ecf87-4c96-47af-8d3f-d86d51b445f4 -smbios type=1,manufacturer=OpenStack Foundati
on,product=OpenStack Nova,version=2014.1.5,serial=993710e9-0a44-428e-88ec-9210c6dfed55,uuid=020ecf87-4c96-47af-8d3f-d86d51b445f4 -no-user-config -nodefaults -chardev socket,id=charmonitor,path=/var/lib/libvirt/q
emu/instance-000718af.monitor,server,nowait -mon chardev=charmonitor,id=monitor,mode=control -rtc base=utc,driftfix=slew -global kvm-pit.lost_tick_policy=discard -no-hpet -no-shutdown -boot strict=on -device pii
x3-usb-uhci,id=usb,bus=pci.0,addr=0x1.0x2 -drive file=/srv/nova/instances/020ecf87-4c96-47af-8d3f-d86d51b445f4/disk,if=none,id=drive-virtio-disk0,format=qcow2,cache=none -device virtio-blk-pci,scsi=off,bus=pci.0
,addr=0x4,drive=drive-virtio-disk0,id=virtio-disk0,bootindex=1 -netdev tap,fd=24,id=hostnet0,vhost=on,vhostfd=29 -device virtio-net-pci,netdev=hostnet0,id=net0,mac=fa:16:3e:eb:99:5e,bus=pci.0,addr=0x3 -chardev f
ile,id=charserial0,path=/srv/nova/instances/020ecf87-4c96-47af-8d3f-d86d51b445f4/console.log -device isa-serial,chardev=charserial0,id=serial0 -chardev pty,id=charserial1 -device isa-serial,chardev=charserial1,i
d=serial1 -device usb-tablet,id=input0 -vnc 0.0.0.0:4 -k en-us -device cirrus-vga,id=video0,bus=pci.0,addr=0x2 -device virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x5
ballot@ginzel:~$ sudo cat /proc/20953/sta
stack stat statm status
"""
and we can see the stack of the process
"""
ballot@ginzel:~$ sudo cat /proc/20953/stack
[<0>] msleep+0x2d/0x40
[<0>] netdev_run_todo+0x11c/0x320
[<0>] rtnl_unlock+0xe/0x10
[<0>] tun_chr_close+0x28/0x30
[<0>] __fput+0xea/0x220
[<0>] ____fput+0xe/0x10
[<0>] task_work_run+0x9d/0xc0
[<0>] exit_to_usermode_loop+0xc0/0xd0
[<0>] do_syscall_64+0x115/0x130
[<0>] entry_SYSCALL_64_after_hwframe+0x3d/0xa2
[<0>] 0xffffffffffffffff
"""
Hello,
We recently encountered an issue when trying to stop nova instances.
| Apr 16 09:35:10 ginzel kernel: [6925958.071665] unregister_ netdevice: waiting for tape3e33cb9-d6 to become free. Usage count = 2
When I try to stop manually the qemu process, it errors
"""
ballot@ginzel:~$ sudo virsh destroy instance-000718af
setlocale: No such file or directory
error: Failed to destroy domain instance-000718af
error: Failed to terminate process 20953 with SIGKILL: Device or resource busy
"""
The qemu process is in D state because of this qemu-system- x86_64 -name instance-000718af -S -machine pc-i440fx- trusty, accel=kvm, usb=off -cpu EPYC-IBPB- 2.0,+perfctr_ nb,+perfctr_ core,+t +tce,+wdt, +skinit, +extapic, +cmp_legacy, +osxsave, +ht -m 4096 -realtime mlock=off -smp 2,sockets= 2,cores= 1,threads= 1 -uuid 020ecf87- 4c96-47af- 8d3f-d86d51b445 f4 -smbios type=1, manufacturer= OpenStack Foundati OpenStack Nova,version= 2014.1. 5,serial= 993710e9- 0a44-428e- 88ec-9210c6dfed 55,uuid= 020ecf87- 4c96-47af- 8d3f-d86d51b445 f4 -no-user-config -nodefaults -chardev socket, id=charmonitor, path=/var/ lib/libvirt/ q 000718af. monitor, server, nowait -mon chardev= charmonitor, id=monitor, mode=control -rtc base=utc, driftfix= slew -global kvm-pit. lost_tick_ policy= discard -no-hpet -no-shutdown -boot strict=on -device pii uhci,id= usb,bus= pci.0,addr= 0x1.0x2 -drive file=/srv/ nova/instances/ 020ecf87- 4c96-47af- 8d3f-d86d51b445 f4/disk, if=none, id=drive- virtio- disk0,format= qcow2,cache= none -device virtio- blk-pci, scsi=off, bus=pci. 0 drive=drive- virtio- disk0,id= virtio- disk0,bootindex =1 -netdev tap,fd= 24,id=hostnet0, vhost=on, vhostfd= 29 -device virtio- net-pci, netdev= hostnet0, id=net0, mac=fa: 16:3e:eb: 99:5e,bus= pci.0,addr= 0x3 -chardev f charserial0, path=/srv/ nova/instances/ 020ecf87- 4c96-47af- 8d3f-d86d51b445 f4/console. log -device isa-serial, chardev= charserial0, id=serial0 -chardev pty,id=charserial1 -device isa-serial, chardev= charserial1, i id=input0 -vnc 0.0.0.0:4 -k en-us -device cirrus- vga,id= video0, bus=pci. 0,addr= 0x2 -device virtio- balloon- pci,id= balloon0, bus=pci. 0,addr= 0x5
"""
libvirt+ 20953 12.9 0.5 9001400 2925908 ? D Apr15 84:25 /usr/bin/
opoext,
on,product=
emu/instance-
x3-usb-
,addr=0x4,
ile,id=
d=serial1 -device usb-tablet,
ballot@ginzel:~$ sudo cat /proc/20953/sta
stack stat statm status
"""
and we can see the stack of the process run_todo+ 0x11c/0x320 0xe/0x10 close+0x28/ 0x30 run+0x9d/ 0xc0 usermode_ loop+0xc0/ 0xd0 64+0x115/ 0x130 64_after_ hwframe+ 0x3d/0xa2
"""
ballot@ginzel:~$ sudo cat /proc/20953/stack
[<0>] msleep+0x2d/0x40
[<0>] netdev_
[<0>] rtnl_unlock+
[<0>] tun_chr_
[<0>] __fput+0xea/0x220
[<0>] ____fput+0xe/0x10
[<0>] task_work_
[<0>] exit_to_
[<0>] do_syscall_
[<0>] entry_SYSCALL_
[<0>] 0xffffffffffffffff
"""
Will attach the relevant files