So, I have had this reproduced for a while, just had'nt time to dig deeper.
This happens when running 'nova_cell_v2_discover_hosts':
"Error running ['podman', 'run', '--name', 'nova_cell_v2_discover_hosts', '--label', 'config_id=tripleo_step5', '--label', 'container_name=nova_cell_v2_discover_hosts', '--label', 'managed_by=paunch', '--lab el', 'config_data={\"start_order\": 0, \"command\": \"/usr/bin/bootstrap_host_exec nova_ironic su nova -s /bin/bash -c \\'/container-config-scripts/pyshim.sh /container-config-scripts/nova_cell_v2_discover_hosts .py\\'\", \"user\": \"root\", \"volumes\": [\"/etc/hosts:/etc/hosts:ro\", \"/etc/localtime:/etc/localtime:ro\", \"/etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro\", \"/etc/pki/ca-trust/source/anchors: /etc/pki/ca-trust/source/anchors:ro\", \"/etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro\", \"/etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro\", \"/etc/pki/t ls/cert.pem:/etc/pki/tls/cert.pem:ro\", \"/dev/log:/dev/log\", \"/etc/ssh/ssh_known_hosts:/etc/ssh/ssh_known_hosts:ro\", \"/etc/puppet:/etc/puppet:ro\", \"/var/lib/config-data/nova/etc/my.cnf.d/:/etc/my.cnf.d/:r o\", \"/var/lib/config-data/nova/etc/nova/:/etc/nova/:ro\", \"/var/log/containers/nova:/var/log/nova\", \"/var/lib/container-config-scripts/:/container-config-scripts/\"], \"image\": \"docker.io/tripleomaster/ce ntos-binary-nova-compute-ironic:current-tripleo\", \"detach\": false, \"net\": \"host\"}', '--conmon-pidfile=/var/run/nova_cell_v2_discover_hosts.pid', '--log-driver', 'json-file', '--log-opt', 'path=/var/log/co ntainers/stdouts/nova_cell_v2_discover_hosts.log', '--net=host', '--user=root', '--volume=/etc/hosts:/etc/hosts:ro', '--volume=/etc/localtime:/etc/localtime:ro', '--volume=/etc/pki/ca-trust/extracted:/etc/pki/ca -trust/extracted:ro', '--volume=/etc/pki/ca-trust/source/anchors:/etc/pki/ca-trust/source/anchors:ro', '--volume=/etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro', '--volume=/etc/pki/tls/cert s/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro', '--volume=/etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro', '--volume=/dev/log:/dev/log', '--volume=/etc/ssh/ssh_known_hosts:/etc/ssh/ssh_known_ho sts:ro', '--volume=/etc/puppet:/etc/puppet:ro', '--volume=/var/lib/config-data/nova/etc/my.cnf.d/:/etc/my.cnf.d/:ro', '--volume=/var/lib/config-data/nova/etc/nova/:/etc/nova/:ro', '--volume=/var/log/containers/n ova:/var/log/nova', '--volume=/var/lib/container-config-scripts/:/container-config-scripts/', 'docker.io/tripleomaster/centos-binary-nova-compute-ironic:current-tripleo', '/usr/bin/bootstrap_host_exec', 'nova_ir onic', 'su', 'nova', '-s', '/bin/bash', '-c', \"'/container-config-scripts/pyshim.sh\", \"/container-config-scripts/nova_cell_v2_discover_hosts.py'\"]. [1]",
-- The scipts simply call's nova-manage afict.
import logging import os import random import subprocess import sys import time
random.seed()
debug = os.getenv('__OS_DEBUG', 'false')
if debug.lower() == 'true': loglevel = logging.DEBUG else: loglevel = logging.INFO
logging.basicConfig(stream=sys.stdout, level=loglevel) LOG = logging.getLogger('nova_cell_v2_discover_hosts')
iterations = 10 timeout_max = 30 nova_cfg = '/etc/nova/nova.conf'
if __name__ == '__main__': if not os.path.isfile(nova_cfg): LOG.error('Nova configuration file %s does not exist', nova_cfg) sys.exit(1)
for i in range(iterations): try: subprocess.check_call([ '/usr/bin/nova-manage', 'cell_v2', 'discover_hosts', '--by-service', '--verbose' ]) sys.exit(0) except subprocess.CalledProcessError as e: LOG.error('Cell v2 discovery failed with exit code %d, retrying', e.returncode) except Exception as e: LOG.exception('Error during host discovery') time.sleep(random.randint(1, timeout_max)) sys.exit(1)
So, I have had this reproduced for a while, just had'nt time to dig deeper.
This happens when running 'nova_cell_ v2_discover_ hosts':
"Error running ['podman', 'run', '--name', 'nova_cell_ v2_discover_ hosts', '--label', 'config_ id=tripleo_ step5', '--label', 'container_ name=nova_ cell_v2_ discover_ hosts', '--label', 'managed_ by=paunch' , '--lab data={\ "start_ order\" : 0, \"command\": \"/usr/ bin/bootstrap_ host_exec nova_ironic su nova -s /bin/bash -c \\'/container- config- scripts/ pyshim. sh /container- config- scripts/ nova_cell_ v2_discover_ hosts hosts:/ etc/hosts: ro\", \"/etc/ localtime: /etc/localtime: ro\", \"/etc/ pki/ca- trust/extracted :/etc/pki/ ca-trust/ extracted: ro\", \"/etc/ pki/ca- trust/source/ anchors: ca-trust/ source/ anchors: ro\", \"/etc/ pki/tls/ certs/ca- bundle. crt:/etc/ pki/tls/ certs/ca- bundle. crt:ro\ ", \"/etc/ pki/tls/ certs/ca- bundle. trust.crt: /etc/pki/ tls/certs/ ca-bundle. trust.crt: ro\", \"/etc/pki/t pem:/etc/ pki/tls/ cert.pem: ro\", \"/dev/ log:/dev/ log\", \"/etc/ ssh/ssh_ known_hosts: /etc/ssh/ ssh_known_ hosts:ro\ ", \"/etc/ puppet: /etc/puppet: ro\", \"/var/ lib/config- data/nova/ etc/my. cnf.d/: /etc/my. cnf.d/: r lib/config- data/nova/ etc/nova/ :/etc/nova/ :ro\", \"/var/ log/containers/ nova:/var/ log/nova\ ", \"/var/ lib/container- config- scripts/ :/container- config- scripts/ \"], \"image\": \"docker. io/tripleomaste r/ce nova-compute- ironic: current- tripleo\ ", \"detach\": false, \"net\": \"host\"}', '--conmon- pidfile= /var/run/ nova_cell_ v2_discover_ hosts.pid' , '--log-driver', 'json-file', '--log-opt', 'path=/var/log/co stdouts/ nova_cell_ v2_discover_ hosts.log' , '--net=host', '--user=root', '--volume= /etc/hosts: /etc/hosts: ro', '--volume= /etc/localtime: /etc/localtime: ro', '--volume= /etc/pki/ ca-trust/ extracted: /etc/pki/ ca extracted: ro', '--volume= /etc/pki/ ca-trust/ source/ anchors: /etc/pki/ ca-trust/ source/ anchors: ro', '--volume= /etc/pki/ tls/certs/ ca-bundle. crt:/etc/ pki/tls/ certs/ca- bundle. crt:ro' , '--volume= /etc/pki/ tls/cert trust.crt: /etc/pki/ tls/certs/ ca-bundle. trust.crt: ro', '--volume= /etc/pki/ tls/cert. pem:/etc/ pki/tls/ cert.pem: ro', '--volume= /dev/log: /dev/log' , '--volume= /etc/ssh/ ssh_known_ hosts:/ etc/ssh/ ssh_known_ ho /etc/puppet: /etc/puppet: ro', '--volume= /var/lib/ config- data/nova/ etc/my. cnf.d/: /etc/my. cnf.d/: ro', '--volume= /var/lib/ config- data/nova/ etc/nova/ :/etc/nova/ :ro', '--volume= /var/log/ containers/ n /var/lib/ container- config- scripts/ :/container- config- scripts/ ', 'docker. io/tripleomaste r/centos- binary- nova-compute- ironic: current- tripleo' , '/usr/bin/ bootstrap_ host_exec' , 'nova_ir config- scripts/ pyshim. sh\", \"/container- config- scripts/ nova_cell_ v2_discover_ hosts.py' \"]. [1]",
el', 'config_
.py\\'\", \"user\": \"root\", \"volumes\": [\"/etc/
/etc/pki/
ls/cert.
o\", \"/var/
ntos-binary-
ntainers/
-trust/
s/ca-bundle.
sts:ro', '--volume=
ova:/var/log/nova', '--volume=
onic', 'su', 'nova', '-s', '/bin/bash', '-c', \"'/container-
-- The scipts simply call's nova-manage afict.
import logging
import os
import random
import subprocess
import sys
import time
random.seed()
debug = os.getenv( '__OS_DEBUG' , 'false')
if debug.lower() == 'true':
loglevel = logging.DEBUG
else:
loglevel = logging.INFO
logging. basicConfig( stream= sys.stdout, level=loglevel) getLogger( 'nova_cell_ v2_discover_ hosts')
LOG = logging.
iterations = 10 nova.conf'
timeout_max = 30
nova_cfg = '/etc/nova/
if __name__ == '__main__': isfile( nova_cfg) :
LOG.error( 'Nova configuration file %s does not exist', nova_cfg)
if not os.path.
sys.exit(1)
for i in range(iterations):
subprocess .check_ call([
'/usr/ bin/nova- manage' ,
'cell_ v2',
'discover_ hosts',
'--by- service' ,
'--verbose'
sys. exit(0) CalledProcessEr ror as e:
LOG. error(' Cell v2 discovery failed with exit code %d, retrying',
e.returncode)
LOG. exception( 'Error during host discovery')
time.sleep( random. randint( 1, timeout_max))
try:
])
except subprocess.
except Exception as e:
sys.exit(1)