Comment 11 for bug 1773084

I'm here to resurrect a year old bug! Or to at least figure out what is causing this.
We (Solutions-qa) ran into this while trying to enable lvl2 CIS Hardened images.

I can reliably (at least 2/2 fresh deploys so far) reproduce this (just takes 3hrs to get everything up).

Let me know if there is anything else I can provide to help.

---

Here's a link to one of the runs on our dashboard:
https://solutions.qa.canonical.com/testruns/testRun/e614dd36-3c26-4abf-b976-9b0c9c6bd8f3

Artifacts/logs/bundles here:
https://oil-jenkins.canonical.com/artifacts/e614dd36-3c26-4abf-b976-9b0c9c6bd8f3/index.html

Juju model crashdump here:
https://oil-jenkins.canonical.com/artifacts/e614dd36-3c26-4abf-b976-9b0c9c6bd8f3/generated/generated/openstack/juju-crashdump-openstack-2021-03-16-19.01.01.tar.gz

---

And here's the output you previously requested:
ubuntu@juju-533a8f-1-lxd-10:~$ sudo fuser /var/lib/dpkg/lock
/var/lib/dpkg/lock: 44677
ubuntu@juju-533a8f-1-lxd-10:~$ ps faxuw
USER PID %CPU %MEM VSZ RSS TTY STAT START TIME COMMAND
root 1 0.7 0.0 77992 9392 ? Ss 18:40 0:15 /sbin/init
root 58 0.1 0.0 103300 24700 ? Ss 18:40 0:02 /lib/systemd/systemd-journald
root 62 0.6 0.0 33356 3508 ? Ss 18:40 0:12 /lib/systemd/systemd-udevd
systemd+ 156 0.0 0.0 70672 4984 ? Ss 18:40 0:00 /lib/systemd/systemd-resolved
systemd+ 185 0.0 0.0 71896 5292 ? Ss 18:40 0:00 /lib/systemd/systemd-networkd
daemon 218 0.0 0.0 28340 2472 ? Ss 18:40 0:00 /usr/sbin/atd -f
root 219 0.0 0.0 31756 3164 ? Ss 18:40 0:00 /usr/sbin/cron -f
root 220 0.0 0.0 70676 6316 ? Ss 18:40 0:00 /lib/systemd/systemd-logind
root 223 0.0 0.0 170872 17432 ? Ssl 18:40 0:00 /usr/bin/python3 /usr/bin/networkd-dispatcher --run-startup-triggers
root 228 0.0 0.0 288004 6836 ? Ssl 18:40 0:00 /usr/lib/accountsservice/accounts-daemon
message+ 229 0.1 0.0 50212 4680 ? Ss 18:40 0:02 /usr/bin/dbus-daemon --system --address=systemd: --nofork --nopidfile --systemd-activation --syslog-onl
root 238 0.0 0.0 72308 6468 ? Ss 18:40 0:00 /usr/sbin/sshd -D
root 34692 0.0 0.0 118456 7400 ? Ss 19:23 0:00 \_ sshd: ubuntu [priv]
ubuntu 34771 0.0 0.0 118456 4016 ? S 19:23 0:00 | \_ sshd: ubuntu@pts/0
root 34772 0.0 0.0 78672 4540 pts/0 Ss+ 19:23 0:00 | \_ sudo /bin/bash -c F=$(mktemp); echo IyEvYmluL2Jhc2gKKApjbGVhbnVwX29uX2V4aXQoKSAKeyAKCWVjaG8
root 34773 0.0 0.0 13324 3368 pts/0 S+ 19:23 0:00 | \_ /bin/bash -c F=$(mktemp); echo IyEvYmluL2Jhc2gKKApjbGVhbnVwX29uX2V4aXQoKSAKeyAKCWVjaG8g
root 34778 0.0 0.0 13324 2100 pts/0 S+ 19:23 0:00 | \_ /bin/bash -c F=$(mktemp); echo IyEvYmluL2Jhc2gKKApjbGVhbnVwX29uX2V4aXQoKSAKeyAKCWVj
root 34780 0.0 0.0 13324 2100 pts/0 S+ 19:23 0:00 | \_ /bin/bash -c F=$(mktemp); echo IyEvYmluL2Jhc2gKKApjbGVhbnVwX29uX2V4aXQoKSAKeyAK
root 34784 0.0 0.0 19924 3432 pts/0 S+ 19:23 0:00 | \_ tmux attach-session -t rabbitmq-server/0
root 41273 0.0 0.0 118456 7560 ? Ss 19:29 0:00 \_ sshd: ubuntu [priv]
ubuntu 41366 0.0 0.0 118456 3764 ? S 19:29 0:00 \_ sshd: ubuntu@pts/3
ubuntu 41371 0.0 0.0 23104 5040 pts/3 Ss 19:29 0:00 \_ -bash
ubuntu 44907 0.0 0.0 37968 3328 pts/3 R+ 19:33 0:00 \_ ps faxuw
root 248 0.0 0.0 288888 6616 ? Ssl 18:40 0:00 /usr/lib/policykit-1/polkitd --no-debug
root 258 0.0 0.0 16420 2328 pts/2 Ss+ 18:40 0:00 /sbin/agetty -o -p -- \u --noclear --keep-baud console 115200,38400,9600 linux
syslog 3394 0.0 0.0 197636 4164 ? Ssl 18:41 0:00 /usr/sbin/rsyslogd -n
_chrony 20916 0.0 0.0 108464 3104 ? S 18:43 0:00 /usr/sbin/chronyd -x
root 21385 0.0 0.0 67384 4016 ? Ss 18:43 0:00 /usr/lib/postfix/sbin/master -w
postfix 21386 0.0 0.0 73816 5228 ? S 18:43 0:00 \_ pickup -l -t unix -u -c
postfix 21387 0.0 0.0 73864 5340 ? S 18:43 0:00 \_ qmgr -l -t unix -u
root 24868 0.0 0.0 187680 20356 ? Ssl 18:44 0:00 /usr/bin/python3 /usr/share/unattended-upgrades/unattended-upgrade-shutdown --wait-for-signal
root 25322 0.0 0.0 21776 3496 ? Ss 18:44 0:00 bash /etc/systemd/system/jujud-machine-1-lxd-10-exec-start.sh
root 25327 0.0 0.0 828956 80156 ? Sl 18:44 0:01 \_ /var/lib/juju/tools/machine-1-lxd-10/jujud machine --data-dir /var/lib/juju --machine-id 1/lxd/10 -
root 25466 1.8 0.0 2466916 30664 ? Ssl 18:44 0:31 /usr/lib/lxd/lxd --group lxd --logfile=/var/log/lxd/lxd.log
root 25483 0.0 0.0 21776 3636 ? Ss 18:44 0:00 bash /etc/systemd/system/jujud-unit-rabbitmq-server-0-exec-start.sh
root 25487 0.0 0.0 828444 80696 ? Sl 18:44 0:01 \_ /var/lib/juju/tools/unit-rabbitmq-server-0/jujud unit --data-dir /var/lib/juju --unit-name rabbitmq
root 35265 0.0 0.0 21776 3604 ? S 19:23 0:00 \_ /bin/bash -s
root 44894 0.0 0.0 7932 748 ? S 19:33 0:00 | \_ sleep 1
root 35266 0.0 0.0 12192 892 ? S 19:23 0:00 \_ flock /tmp/juju-unit-rabbitmq-server-0-debug-hooks-exit -c true
ubuntu 34694 0.0 0.0 76580 7392 ? Ss 19:23 0:00 /lib/systemd/systemd --user
ubuntu 34695 0.0 0.0 122332 3632 ? S 19:23 0:00 \_ (sd-pam)
root 34788 0.0 0.0 28800 4104 ? Ss 19:23 0:03 tmux new-session -d -s rabbitmq-server/0
root 34792 0.0 0.0 22972 4660 pts/2 Ss+ 19:23 0:00 \_ /bin/bash
root 35270 0.0 0.0 22056 3976 pts/1 Ss 19:23 0:00 \_ /bin/bash --noprofile --init-file /tmp/juju-debug-hooks-088448928/init.sh
root 44587 0.0 0.0 115900 36588 pts/1 S+ 19:32 0:00 \_ /usr/bin/python3 /var/lib/juju/agents/unit-rabbitmq-server-0/charm/hooks/install.real
root 44644 0.0 0.0 120096 66148 pts/1 S+ 19:32 0:00 \_ apt-get --assume-yes --option=Dpkg::Options::=--force-confold install
root 44677 0.0 0.0 17644 4432 pts/4 Ss+ 19:32 0:00 \_ /usr/bin/dpkg --force-confold --status-fd 45 --configure --pending
root 44678 0.0 0.0 4636 1728 pts/4 S+ 19:32 0:00 \_ /bin/sh /var/lib/dpkg/info/rabbitmq-server.postinst configure
root 44735 0.0 0.0 4636 1760 pts/4 S+ 19:32 0:00 \_ /bin/sh /usr/sbin/invoke-rc.d rabbitmq-server start
root 44772 0.0 0.0 62024 3144 pts/4 S+ 19:32 0:00 \_ systemctl start rabbitmq-server.service
root 44773 0.0 0.0 61872 3144 pts/4 S+ 19:32 0:00 \_ /bin/systemd-tty-ask-password-agent --watch