ls -il /var/lib/juju/kvm/guests/
total 1519192
125175532 -rw-r--r-- 1 root root 2361393152 Mar 14 19:46 bionic-amd64-backing-file.qcow
125175524 -rw-r--r-- 1 libvirt-qemu kvm 391168 Mar 14 19:44 juju-5ad6c1-6-kvm-0-ds.iso
125175525 -rw-r--r-- 1 libvirt-qemu kvm 393216 Mar 14 19:44 juju-5ad6c1-6-kvm-0.qcow
125175520 -rw-r--r-- 1 root root 391168 Mar 14 19:44 juju-5ad6c1-6-kvm-1-ds.iso
125175521 -rw-r--r-- 1 root root 196736 Mar 14 19:44 juju-5ad6c1-6-kvm-1.qcow
125175527 -rw-r--r-- 1 libvirt-qemu kvm 391168 Mar 14 19:44 juju-5ad6c1-6-kvm-2-ds.iso
125175528 -rw-r--r-- 1 libvirt-qemu kvm 469237760 Mar 14 22:09 juju-5ad6c1-6-kvm-2.qcow
125175530 -rw-r--r-- 1 libvirt-qemu kvm 391168 Mar 14 19:46 juju-5ad6c1-6-kvm-3-ds.iso
125175531 -rw-r--r-- 1 libvirt-qemu kvm 393216 Mar 14 19:46 juju-5ad6c1-6-kvm-3.qcow
125175518 -rw-r--r-- 1 root root 391168 Mar 14 19:44 juju-5ad6c1-6-kvm-4-ds.iso
125175519 -rw-r--r-- 1 root root 196736 Mar 14 19:44 juju-5ad6c1-6-kvm-4.qcow
125175522 -rw-r--r-- 1 root root 391168 Mar 14 19:44 juju-5ad6c1-6-kvm-5-ds.iso
125175523 -rw-r--r-- 1 root root 196736 Mar 14 19:44 juju-5ad6c1-6-kvm-5.qcow
All images have bionic-amd64-backing-file.qcow as a backing file:
qemu-img info /var/lib/juju/kvm/guests/juju-5ad6c1-6-kvm-5.qcow
image: /var/lib/juju/kvm/guests/juju-5ad6c1-6-kvm-5.qcow
file format: qcow2
virtual size: 8.0G (8589934592 bytes)
disk size: 196K
cluster_size: 65536
backing file: /var/lib/juju/kvm/guests/bionic-amd64-backing-file.qcow
Format specific information:
compat: 1.1
lazy refcounts: false
refcount bits: 16
corrupt: false
Domains can be manually started:
root@control-7:~# virsh list --all
Id Name State
----------------------------------------------------
4 juju-5ad6c1-6-kvm-0 running
5 juju-5ad6c1-6-kvm-2 running
6 juju-5ad6c1-6-kvm-3 running
- juju-5ad6c1-6-kvm-1 shut off
- juju-5ad6c1-6-kvm-4 shut off
- juju-5ad6c1-6-kvm-5 shut off
root@control-7:~# virsh start juju-5ad6c1-6-kvm-1
Domain juju-5ad6c1-6-kvm-1 started
The machines that were started initially are stuck in initramfs:
virsh console juju-5ad6c1-6-kvm-0
Connected to domain juju-5ad6c1-6-kvm-0
Escape character is ^]
/var/log from a host with a bunch of failed kvm "containers": /private- fileshare. canonical. com/~dima/ juju-dumps/ kvm-host- 2019-03- 14.tar. gz
https:/
Relevant VM logs: /var/log/ libvirt/ qemu/*
Some VMs have started successfully, some have not:
pgrep -af qemu | grep -ioP 'qemu.* ?guest= .*?\s' 5ad6c1- 6-kvm-0, debug-threads= on 5ad6c1- 6-kvm-2, debug-threads= on 5ad6c1- 6-kvm-3, debug-threads= on
qemu-system-x86_64 -enable-kvm -name guest=juju-
qemu-system-x86_64 -enable-kvm -name guest=juju-
qemu-system-x86_64 -enable-kvm -name guest=juju-
root@control-7:~# pgrep -af qemu | grep -ioP 'drive file=.*?\s' lib/juju/ kvm/guests/ juju-5ad6c1- 6-kvm-0. qcow,format= qcow2,if= none,id= drive-virtio- disk0 lib/juju/ kvm/guests/ juju-5ad6c1- 6-kvm-0- ds.iso, format= raw,if= none,id= drive-virtio- disk1 lib/juju/ kvm/guests/ juju-5ad6c1- 6-kvm-2. qcow,format= qcow2,if= none,id= drive-virtio- disk0 lib/juju/ kvm/guests/ juju-5ad6c1- 6-kvm-2- ds.iso, format= raw,if= none,id= drive-virtio- disk1 lib/juju/ kvm/guests/ juju-5ad6c1- 6-kvm-3. qcow,format= qcow2,if= none,id= drive-virtio- disk0 lib/juju/ kvm/guests/ juju-5ad6c1- 6-kvm-3- ds.iso, format= raw,if= none,id= drive-virtio- disk1
drive file=/var/
drive file=/var/
drive file=/var/
drive file=/var/
drive file=/var/
drive file=/var/
root@control-7:~# ls /var/lib/ juju/kvm/ guests/ amd64-backing- file.qcow juju-5ad6c1- 6-kvm-1- ds.iso juju-5ad6c1- 6-kvm-2. qcow juju-5ad6c1- 6-kvm-4- ds.iso juju-5ad6c1- 6-kvm-5. qcow 6-kvm-0- ds.iso juju-5ad6c1- 6-kvm-1. qcow juju-5ad6c1- 6-kvm-3- ds.iso juju-5ad6c1- 6-kvm-4. qcow 6-kvm-0. qcow juju-5ad6c1- 6-kvm-2- ds.iso juju-5ad6c1- 6-kvm-3. qcow juju-5ad6c1- 6-kvm-5- ds.iso
bionic-
juju-5ad6c1-
juju-5ad6c1-
root@control-7:~# grep -RiP 'Failed' /var/log/ libvirt/ qemu/ libvirt/ qemu/juju- 5ad6c1- 6-kvm-4. log:2019- 03-14T19: 44:09.095078Z qemu-system-x86_64: -device virtio- blk-pci, scsi=off, bus=pci. 0,addr= 0x3,drive= drive-virtio- disk0,id= virtio- disk0,bootindex =1: Failed to get shared "write" lock libvirt/ qemu/juju- 5ad6c1- 6-kvm-4. log:2019- 03-14 19:44:09.184+0000: shutting down, reason=failed libvirt/ qemu/juju- 5ad6c1- 6-kvm-5. log:2019- 03-14T19: 44:11.875801Z qemu-system-x86_64: -device virtio- blk-pci, scsi=off, bus=pci. 0,addr= 0x3,drive= drive-virtio- disk0,id= virtio- disk0,bootindex =1: Failed to get shared "write" lock libvirt/ qemu/juju- 5ad6c1- 6-kvm-5. log:2019- 03-14 19:44:12.160+0000: shutting down, reason=failed libvirt/ qemu/juju- 5ad6c1- 6-kvm-1. log:2019- 03-14T19: 44:09.219746Z qemu-system-x86_64: -device virtio- blk-pci, scsi=off, bus=pci. 0,addr= 0x4,drive= drive-virtio- disk0,id= virtio- disk0,bootindex =1: Failed to get shared "write" lock libvirt/ qemu/juju- 5ad6c1- 6-kvm-1. log:2019- 03-14 19:44:09.497+0000: shutting down, reason=failed
/var/log/
/var/log/
/var/log/
/var/log/
/var/log/
/var/log/
Each VM uses the following files (an ISO and a qcow image):
-drive file=/var/ lib/juju/ kvm/guests/ juju-5ad6c1- 6-kvm-0. qcow lib/juju/ kvm/guests/ juju-5ad6c1- 6-kvm-0- ds.iso
-drive file=/var/
ls -il /var/lib/ juju/kvm/ guests/ amd64-backing- file.qcow 6-kvm-0- ds.iso 6-kvm-0. qcow 6-kvm-1- ds.iso 6-kvm-1. qcow 6-kvm-2- ds.iso 6-kvm-2. qcow 6-kvm-3- ds.iso 6-kvm-3. qcow 6-kvm-4- ds.iso 6-kvm-4. qcow 6-kvm-5- ds.iso 6-kvm-5. qcow
total 1519192
125175532 -rw-r--r-- 1 root root 2361393152 Mar 14 19:46 bionic-
125175524 -rw-r--r-- 1 libvirt-qemu kvm 391168 Mar 14 19:44 juju-5ad6c1-
125175525 -rw-r--r-- 1 libvirt-qemu kvm 393216 Mar 14 19:44 juju-5ad6c1-
125175520 -rw-r--r-- 1 root root 391168 Mar 14 19:44 juju-5ad6c1-
125175521 -rw-r--r-- 1 root root 196736 Mar 14 19:44 juju-5ad6c1-
125175527 -rw-r--r-- 1 libvirt-qemu kvm 391168 Mar 14 19:44 juju-5ad6c1-
125175528 -rw-r--r-- 1 libvirt-qemu kvm 469237760 Mar 14 22:09 juju-5ad6c1-
125175530 -rw-r--r-- 1 libvirt-qemu kvm 391168 Mar 14 19:46 juju-5ad6c1-
125175531 -rw-r--r-- 1 libvirt-qemu kvm 393216 Mar 14 19:46 juju-5ad6c1-
125175518 -rw-r--r-- 1 root root 391168 Mar 14 19:44 juju-5ad6c1-
125175519 -rw-r--r-- 1 root root 196736 Mar 14 19:44 juju-5ad6c1-
125175522 -rw-r--r-- 1 root root 391168 Mar 14 19:44 juju-5ad6c1-
125175523 -rw-r--r-- 1 root root 196736 Mar 14 19:44 juju-5ad6c1-
All images have bionic- amd64-backing- file.qcow as a backing file:
qemu-img info /var/lib/ juju/kvm/ guests/ juju-5ad6c1- 6-kvm-5. qcow juju/kvm/ guests/ juju-5ad6c1- 6-kvm-5. qcow juju/kvm/ guests/ bionic- amd64-backing- file.qcow
image: /var/lib/
file format: qcow2
virtual size: 8.0G (8589934592 bytes)
disk size: 196K
cluster_size: 65536
backing file: /var/lib/
Format specific information:
compat: 1.1
lazy refcounts: false
refcount bits: 16
corrupt: false
Domains can be manually started:
root@control-7:~# virsh list --all ------- ------- ------- ------- ------- ------- ---
Id Name State
-------
4 juju-5ad6c1-6-kvm-0 running
5 juju-5ad6c1-6-kvm-2 running
6 juju-5ad6c1-6-kvm-3 running
- juju-5ad6c1-6-kvm-1 shut off
- juju-5ad6c1-6-kvm-4 shut off
- juju-5ad6c1-6-kvm-5 shut off
root@control-7:~# virsh start juju-5ad6c1-6-kvm-1
Domain juju-5ad6c1-6-kvm-1 started
The machines that were started initially are stuck in initramfs:
virsh console juju-5ad6c1-6-kvm-0
Connected to domain juju-5ad6c1-6-kvm-0
Escape character is ^]
(initramfs)