Comment 6 for bug 1831315

So, I have had this reproduced for a while, just had'nt time to dig deeper.

This happens when running 'nova_cell_v2_discover_hosts':

    "Error running ['podman', 'run', '--name', 'nova_cell_v2_discover_hosts', '--label', 'config_id=tripleo_step5', '--label', 'container_name=nova_cell_v2_discover_hosts', '--label', 'managed_by=paunch', '--lab
el', 'config_data={\"start_order\": 0, \"command\": \"/usr/bin/bootstrap_host_exec nova_ironic su nova -s /bin/bash -c \\'/container-config-scripts/pyshim.sh /container-config-scripts/nova_cell_v2_discover_hosts
.py\\'\", \"user\": \"root\", \"volumes\": [\"/etc/hosts:/etc/hosts:ro\", \"/etc/localtime:/etc/localtime:ro\", \"/etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro\", \"/etc/pki/ca-trust/source/anchors:
/etc/pki/ca-trust/source/anchors:ro\", \"/etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro\", \"/etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro\", \"/etc/pki/t
ls/cert.pem:/etc/pki/tls/cert.pem:ro\", \"/dev/log:/dev/log\", \"/etc/ssh/ssh_known_hosts:/etc/ssh/ssh_known_hosts:ro\", \"/etc/puppet:/etc/puppet:ro\", \"/var/lib/config-data/nova/etc/my.cnf.d/:/etc/my.cnf.d/:r
o\", \"/var/lib/config-data/nova/etc/nova/:/etc/nova/:ro\", \"/var/log/containers/nova:/var/log/nova\", \"/var/lib/container-config-scripts/:/container-config-scripts/\"], \"image\": \"docker.io/tripleomaster/ce
ntos-binary-nova-compute-ironic:current-tripleo\", \"detach\": false, \"net\": \"host\"}', '--conmon-pidfile=/var/run/nova_cell_v2_discover_hosts.pid', '--log-driver', 'json-file', '--log-opt', 'path=/var/log/co
ntainers/stdouts/nova_cell_v2_discover_hosts.log', '--net=host', '--user=root', '--volume=/etc/hosts:/etc/hosts:ro', '--volume=/etc/localtime:/etc/localtime:ro', '--volume=/etc/pki/ca-trust/extracted:/etc/pki/ca
-trust/extracted:ro', '--volume=/etc/pki/ca-trust/source/anchors:/etc/pki/ca-trust/source/anchors:ro', '--volume=/etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro', '--volume=/etc/pki/tls/cert
s/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro', '--volume=/etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro', '--volume=/dev/log:/dev/log', '--volume=/etc/ssh/ssh_known_hosts:/etc/ssh/ssh_known_ho
sts:ro', '--volume=/etc/puppet:/etc/puppet:ro', '--volume=/var/lib/config-data/nova/etc/my.cnf.d/:/etc/my.cnf.d/:ro', '--volume=/var/lib/config-data/nova/etc/nova/:/etc/nova/:ro', '--volume=/var/log/containers/n
ova:/var/log/nova', '--volume=/var/lib/container-config-scripts/:/container-config-scripts/', 'docker.io/tripleomaster/centos-binary-nova-compute-ironic:current-tripleo', '/usr/bin/bootstrap_host_exec', 'nova_ir
onic', 'su', 'nova', '-s', '/bin/bash', '-c', \"'/container-config-scripts/pyshim.sh\", \"/container-config-scripts/nova_cell_v2_discover_hosts.py'\"]. [1]",

-- The scipts simply call's nova-manage afict.

import logging
import os
import random
import subprocess
import sys
import time

random.seed()

debug = os.getenv('__OS_DEBUG', 'false')

if debug.lower() == 'true':
    loglevel = logging.DEBUG
else:
    loglevel = logging.INFO

logging.basicConfig(stream=sys.stdout, level=loglevel)
LOG = logging.getLogger('nova_cell_v2_discover_hosts')

iterations = 10
timeout_max = 30
nova_cfg = '/etc/nova/nova.conf'

if __name__ == '__main__':
    if not os.path.isfile(nova_cfg):
        LOG.error('Nova configuration file %s does not exist', nova_cfg)
        sys.exit(1)

    for i in range(iterations):
        try:
            subprocess.check_call([
                '/usr/bin/nova-manage',
                'cell_v2',
                'discover_hosts',
                '--by-service',
                '--verbose'
            ])
            sys.exit(0)
        except subprocess.CalledProcessError as e:
            LOG.error('Cell v2 discovery failed with exit code %d, retrying',
                      e.returncode)
        except Exception as e:
            LOG.exception('Error during host discovery')
        time.sleep(random.randint(1, timeout_max))
sys.exit(1)