Manually copying the keyring into the container fixes the problem in the sense that ceph commands now work in the container:

(nova-compute)[root@chrnc-area51-os-compute-03 /]# ceph --id nova health
HEALTH_OK

But I get a new error when I try to build a VM now:

2021-05-28 17:00:36.141 20 ERROR nova.scheduler.utils [req-1763e349-bf76-4112-a914-fe790d57d924 f1cc3cd2fe734a93ab1ed6ad8143decc b7c712174e1c41c9bd5bde4721a7458d - default default] [instance: 475f2fd9-947b-4300-abdc-aab7bdf80fb3] Error from last host: chrnc-area51-os-compute-02.chtrse.com (node chrnc-area51-os-compute-02.chtrse.com): ['Traceback (most recent call last):\n', '  File "/var/lib/kolla/venv/lib/python3.6/site-packages/nova/compute/manager.py", line 2385, in _build_and_run_instance\n    accel_info=accel_info)\n', '  File "/var/lib/kolla/venv/lib/python3.6/site-packages/nova/virt/libvirt/driver.py", line 3690, in spawn\n    cleanup_instance_disks=created_disks)\n', '  File "/var/lib/kolla/venv/lib/python3.6/site-packages/nova/virt/libvirt/driver.py", line 6627, in _create_domain_and_network\n    cleanup_instance_disks=cleanup_instance_disks)\n', '  File "/var/lib/kolla/venv/lib/python3.6/site-packages/oslo_utils/excutils.py", line 220, in __exit__\n    self.force_reraise()\n', '  File "/var/lib/kolla/venv/lib/python3.6/site-packages/oslo_utils/excutils.py", line 196, in force_reraise\n    six.reraise(self.type_, self.value, self.tb)\n', '  File "/usr/lib/python3.6/site-packages/six.py", line 703, in reraise\n    raise value\n', '  File "/var/lib/kolla/venv/lib/python3.6/site-packages/nova/virt/libvirt/driver.py", line 6596, in _create_domain_and_network\n    post_xml_callback=post_xml_callback)\n', '  File "/var/lib/kolla/venv/lib/python3.6/site-packages/nova/virt/libvirt/driver.py", line 6532, in _create_domain\n    guest.launch(pause=pause)\n', '  File "/var/lib/kolla/venv/lib/python3.6/site-packages/nova/virt/libvirt/guest.py", line 142, in launch\n    self._encoded_xml, errors=\'ignore\')\n', '  File "/var/lib/kolla/venv/lib/python3.6/site-packages/oslo_utils/excutils.py", line 220, in __exit__\n    self.force_reraise()\n', '  File "/var/lib/kolla/venv/lib/python3.6/site-packages/oslo_utils/excutils.py", line 196, in force_reraise\n    six.reraise(self.type_, self.value, self.tb)\n', '  File "/usr/lib/python3.6/site-packages/six.py", line 703, in reraise\n    raise value\n', '  File "/var/lib/kolla/venv/lib/python3.6/site-packages/nova/virt/libvirt/guest.py", line 137, in launch\n    return self._domain.createWithFlags(flags)\n', '  File "/var/lib/kolla/venv/lib/python3.6/site-packages/eventlet/tpool.py", line 190, in doit\n    result = proxy_call(self._autowrap, f, *args, **kwargs)\n', '  File "/var/lib/kolla/venv/lib/python3.6/site-packages/eventlet/tpool.py", line 148, in proxy_call\n    rv = execute(f, *args, **kwargs)\n', '  File "/var/lib/kolla/venv/lib/python3.6/site-packages/eventlet/tpool.py", line 129, in execute\n    six.reraise(c, e, tb)\n', '  File "/usr/lib/python3.6/site-packages/six.py", line 703, in reraise\n    raise value\n', '  File "/var/lib/kolla/venv/lib/python3.6/site-packages/eventlet/tpool.py", line 83, in tworker\n    rv = meth(*args, **kwargs)\n', '  File "/usr/lib64/python3.6/site-packages/libvirt.py", line 1270, in createWithFlags\n    if ret == -1: raise libvirtError (\'virDomainCreateWithFlags() failed\', dom=self)\n', 'libvirt.libvirtError: internal error: qemu unexpectedly closed the monitor: 2021-05-28T17:00:34.444521Z qemu-kvm: -blockdev {"driver":"rbd","pool":"vms","image":"475f2fd9-947b-4300-abdc-aab7bdf80fb3_disk","server":[{"host":"192.168.2.31","port":"6789"},{"host":"192.168.2.32","port":"6789"},{"host":"192.168.2.33","port":"6789"}],"user":"nova","auth-client-required":["cephx","none"],"key-secret":"libvirt-1-storage-auth-secret0","node-name":"libvirt-1-storage","cache":{"direct":false,"no-flush":false},"auto-read-only":true,"discard":"unmap"}: error connecting: Operation not permitted\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', '  File "/var/lib/kolla/venv/lib/python3.6/site-packages/nova/compute/manager.py", line 2207, in _do_build_and_run_instance\n    filter_properties, request_spec, accel_uuids)\n', '  File "/var/lib/kolla/venv/lib/python3.6/site-packages/nova/compute/manager.py", line 2485, in _build_and_run_instance\n    instance_uuid=instance.uuid, reason=six.text_type(e))\n', 'nova.exception.RescheduledException: Build of instance 475f2fd9-947b-4300-abdc-aab7bdf80fb3 was re-scheduled: internal error: qemu unexpectedly closed the monitor: 2021-05-28T17:00:34.444521Z qemu-kvm: -blockdev {"driver":"rbd","pool":"vms","image":"475f2fd9-947b-4300-abdc-aab7bdf80fb3_disk","server":[{"host":"192.168.2.31","port":"6789"},{"host":"192.168.2.32","port":"6789"},{"host":"192.168.2.33","port":"6789"}],"user":"nova","auth-client-required":["cephx","none"],"key-secret":"libvirt-1-storage-auth-secret0","node-name":"libvirt-1-storage","cache":{"direct":false,"no-flush":false},"auto-read-only":true,"discard":"unmap"}: error connecting: Operation not permitted\n']

Now nova is failing to auth with libvirt. It looks like nova keys are not being copied correctly during the upgrade.