Comment 2 for bug 1816170

Revision history for this message
Dmitrii Shcherbakov (dmitriis) wrote :

/var/log from a host with a bunch of failed kvm "containers":
https://private-fileshare.canonical.com/~dima/juju-dumps/kvm-host-2019-03-14.tar.gz

Relevant VM logs: /var/log/libvirt/qemu/*

Some VMs have started successfully, some have not:

pgrep -af qemu | grep -ioP 'qemu.*?guest=.*?\s'
qemu-system-x86_64 -enable-kvm -name guest=juju-5ad6c1-6-kvm-0,debug-threads=on
qemu-system-x86_64 -enable-kvm -name guest=juju-5ad6c1-6-kvm-2,debug-threads=on
qemu-system-x86_64 -enable-kvm -name guest=juju-5ad6c1-6-kvm-3,debug-threads=on

root@control-7:~# pgrep -af qemu | grep -ioP 'drive file=.*?\s'
drive file=/var/lib/juju/kvm/guests/juju-5ad6c1-6-kvm-0.qcow,format=qcow2,if=none,id=drive-virtio-disk0
drive file=/var/lib/juju/kvm/guests/juju-5ad6c1-6-kvm-0-ds.iso,format=raw,if=none,id=drive-virtio-disk1
drive file=/var/lib/juju/kvm/guests/juju-5ad6c1-6-kvm-2.qcow,format=qcow2,if=none,id=drive-virtio-disk0
drive file=/var/lib/juju/kvm/guests/juju-5ad6c1-6-kvm-2-ds.iso,format=raw,if=none,id=drive-virtio-disk1
drive file=/var/lib/juju/kvm/guests/juju-5ad6c1-6-kvm-3.qcow,format=qcow2,if=none,id=drive-virtio-disk0
drive file=/var/lib/juju/kvm/guests/juju-5ad6c1-6-kvm-3-ds.iso,format=raw,if=none,id=drive-virtio-disk1

root@control-7:~# ls /var/lib/juju/kvm/guests/
bionic-amd64-backing-file.qcow juju-5ad6c1-6-kvm-1-ds.iso juju-5ad6c1-6-kvm-2.qcow juju-5ad6c1-6-kvm-4-ds.iso juju-5ad6c1-6-kvm-5.qcow
juju-5ad6c1-6-kvm-0-ds.iso juju-5ad6c1-6-kvm-1.qcow juju-5ad6c1-6-kvm-3-ds.iso juju-5ad6c1-6-kvm-4.qcow
juju-5ad6c1-6-kvm-0.qcow juju-5ad6c1-6-kvm-2-ds.iso juju-5ad6c1-6-kvm-3.qcow juju-5ad6c1-6-kvm-5-ds.iso

root@control-7:~# grep -RiP 'Failed' /var/log/libvirt/qemu/
/var/log/libvirt/qemu/juju-5ad6c1-6-kvm-4.log:2019-03-14T19:44:09.095078Z qemu-system-x86_64: -device virtio-blk-pci,scsi=off,bus=pci.0,addr=0x3,drive=drive-virtio-disk0,id=virtio-disk0,bootindex=1: Failed to get shared "write" lock
/var/log/libvirt/qemu/juju-5ad6c1-6-kvm-4.log:2019-03-14 19:44:09.184+0000: shutting down, reason=failed
/var/log/libvirt/qemu/juju-5ad6c1-6-kvm-5.log:2019-03-14T19:44:11.875801Z qemu-system-x86_64: -device virtio-blk-pci,scsi=off,bus=pci.0,addr=0x3,drive=drive-virtio-disk0,id=virtio-disk0,bootindex=1: Failed to get shared "write" lock
/var/log/libvirt/qemu/juju-5ad6c1-6-kvm-5.log:2019-03-14 19:44:12.160+0000: shutting down, reason=failed
/var/log/libvirt/qemu/juju-5ad6c1-6-kvm-1.log:2019-03-14T19:44:09.219746Z qemu-system-x86_64: -device virtio-blk-pci,scsi=off,bus=pci.0,addr=0x4,drive=drive-virtio-disk0,id=virtio-disk0,bootindex=1: Failed to get shared "write" lock
/var/log/libvirt/qemu/juju-5ad6c1-6-kvm-1.log:2019-03-14 19:44:09.497+0000: shutting down, reason=failed

Each VM uses the following files (an ISO and a qcow image):

-drive file=/var/lib/juju/kvm/guests/juju-5ad6c1-6-kvm-0.qcow
-drive file=/var/lib/juju/kvm/guests/juju-5ad6c1-6-kvm-0-ds.iso

ls -il /var/lib/juju/kvm/guests/
total 1519192
125175532 -rw-r--r-- 1 root root 2361393152 Mar 14 19:46 bionic-amd64-backing-file.qcow
125175524 -rw-r--r-- 1 libvirt-qemu kvm 391168 Mar 14 19:44 juju-5ad6c1-6-kvm-0-ds.iso
125175525 -rw-r--r-- 1 libvirt-qemu kvm 393216 Mar 14 19:44 juju-5ad6c1-6-kvm-0.qcow
125175520 -rw-r--r-- 1 root root 391168 Mar 14 19:44 juju-5ad6c1-6-kvm-1-ds.iso
125175521 -rw-r--r-- 1 root root 196736 Mar 14 19:44 juju-5ad6c1-6-kvm-1.qcow
125175527 -rw-r--r-- 1 libvirt-qemu kvm 391168 Mar 14 19:44 juju-5ad6c1-6-kvm-2-ds.iso
125175528 -rw-r--r-- 1 libvirt-qemu kvm 469237760 Mar 14 22:09 juju-5ad6c1-6-kvm-2.qcow
125175530 -rw-r--r-- 1 libvirt-qemu kvm 391168 Mar 14 19:46 juju-5ad6c1-6-kvm-3-ds.iso
125175531 -rw-r--r-- 1 libvirt-qemu kvm 393216 Mar 14 19:46 juju-5ad6c1-6-kvm-3.qcow
125175518 -rw-r--r-- 1 root root 391168 Mar 14 19:44 juju-5ad6c1-6-kvm-4-ds.iso
125175519 -rw-r--r-- 1 root root 196736 Mar 14 19:44 juju-5ad6c1-6-kvm-4.qcow
125175522 -rw-r--r-- 1 root root 391168 Mar 14 19:44 juju-5ad6c1-6-kvm-5-ds.iso
125175523 -rw-r--r-- 1 root root 196736 Mar 14 19:44 juju-5ad6c1-6-kvm-5.qcow

All images have bionic-amd64-backing-file.qcow as a backing file:

qemu-img info /var/lib/juju/kvm/guests/juju-5ad6c1-6-kvm-5.qcow
image: /var/lib/juju/kvm/guests/juju-5ad6c1-6-kvm-5.qcow
file format: qcow2
virtual size: 8.0G (8589934592 bytes)
disk size: 196K
cluster_size: 65536
backing file: /var/lib/juju/kvm/guests/bionic-amd64-backing-file.qcow
Format specific information:
    compat: 1.1
    lazy refcounts: false
    refcount bits: 16
    corrupt: false

Domains can be manually started:

root@control-7:~# virsh list --all
 Id Name State
----------------------------------------------------
 4 juju-5ad6c1-6-kvm-0 running
 5 juju-5ad6c1-6-kvm-2 running
 6 juju-5ad6c1-6-kvm-3 running
 - juju-5ad6c1-6-kvm-1 shut off
 - juju-5ad6c1-6-kvm-4 shut off
 - juju-5ad6c1-6-kvm-5 shut off

root@control-7:~# virsh start juju-5ad6c1-6-kvm-1
Domain juju-5ad6c1-6-kvm-1 started

The machines that were started initially are stuck in initramfs:

virsh console juju-5ad6c1-6-kvm-0
Connected to domain juju-5ad6c1-6-kvm-0
Escape character is ^]

(initramfs)