Deployed 3 controller / 1 compute / 3 ceph overcloud with:
EnablePaunch: False
ConfigDebug: True
Deployment during config-download here:
TASK [tripleo_container_manage : Check podman create status] *********************************
************
changed: [overcloud-controller-0] => (item={'ansible_loop_var': u'container_data', u'ansible_j
ob_id': u'445898066548.184603', u'started': 1, 'changed': True, 'failed': False, u'finished':
0, u'results_file': u'/root/.ansible_async/445898066548.184603', u'container_data': {u'swift_r
sync_fix': {u'start_order': 0, u'image': u'192.168.24.1:8787/tripleomaster/centos-binary-swift
-object:current-tripleo', u'environment': {u'TRIPLEO_CONFIG_HASH': u'821aedcc1a6664fc2b2d21d9f
337cabe'}, u'command': [u'/bin/bash', u'-c', u'sed -i "/pid file/d" /var/lib/kolla/config_file
s/src/etc/rsyncd.conf'], u'user': u'root', u'volumes': [u'/var/lib/config-data/puppet-generate
d/swift:/var/lib/kolla/config_files/src:rw,z'], u'net': u'host', u'detach': False}}}) => {"act
ions": ["started swift_rsync_fix"], "ansible_job_id": "445898066548.184603", "ansible_loop_var
": "create_async_result_item", "attempts": 1, "changed": true, "container": {}, "create_async_
result_item": {"ansible_job_id": "445898066548.184603", "ansible_loop_var": "container_data",
"changed": true, "container_data": {"swift_rsync_fix": {"command": ["/bin/bash", "-c", "sed -i
\"/pid file/d\" /var/lib/kolla/config_files/src/etc/rsyncd.conf"], "detach": false, "environm
ent": {"TRIPLEO_CONFIG_HASH": "821aedcc1a6664fc2b2d21d9f337cabe"}, "image": "192.168.24.1:8787
/tripleomaster/centos-binary-swift-object:current-tripleo", "net": "host", "start_order": 0, "
user": "root", "volumes": ["/var/lib/config-data/puppet-generated/swift:/var/lib/kolla/config_
files/src:rw,z"]}}, "failed": false, "finished": 0, "results_file": "/root/.ansible_async/4458
98066548.184603", "started": 1}, "finished": 1, "podman_actions": ["podman run --name swift_rs
ync_fix --tty=False --detach=False --network host --conmon-pidfile /var/run/swift_rsync_fix.pi
d --label config_data={'start_order': 0, 'image': '192.168.24.1:8787/tripleomaster/centos-bina
ry-swift-object:current-tripleo', 'environment': {'TRIPLEO_CONFIG_HASH': '821aedcc1a6664fc2b2d
21d9f337cabe'}, 'command': ['/bin/bash', '-c', 'sed -i \"/pid file/d\" /var/lib/kolla/config_f
iles/src/etc/rsyncd.conf'], 'user': 'root', 'volumes': ['/var/lib/config-data/puppet-generated
/swift:/var/lib/kolla/config_files/src:rw,z'], 'net': 'host', 'detach': False} --label managed
_by=tripleo_ansible --label config_id=tripleo_step3 --label container_name=swift_rsync_fix --r
m=False --volume /var/lib/config-data/puppet-generated/swift:/var/lib/kolla/config_files/src:r
w,z --log-opt path=/var/log/containers/stdouts/swift_rsync_fix.log --interactive=False --stop-
timeout 0 --env TRIPLEO_CONFIG_HASH=821aedcc1a6664fc2b2d21d9f337cabe --privileged=False --log-
driver k8s-file --user root 192.168.24.1:8787/tripleomaster/centos-binary-swift-object:current
-tripleo /bin/bash -c sed -i \"/pid file/d\" /var/lib/kolla/config_files/src/etc/rsyncd.conf"]
, "podman_version": "1.5.1", "stderr": "", "stderr_lines": [], "stdout": "", "stdout_lines": [
]}
failed: [overcloud-controller-0] (item={'ansible_loop_var': u'container_data', u'ansible_job_i
d': u'97491261257.184873', u'started': 1, 'changed': True, 'failed': False, u'finished': 0, u'
results_file': u'/root/.ansible_async/97491261257.184873', u'container_data': {u'heat_engine_d
b_sync': {u'start_order': 0, u'image': u'192.168.24.1:8787/tripleomaster/centos-binary-heat-en
gine:current-tripleo', u'environment': {u'TRIPLEO_CONFIG_HASH': u'ac70006d5a3d2f50a563194ea8ab
739b-ac70006d5a3d2f50a563194ea8ab739b', u'TRIPLEO_DEPLOY_IDENTIFIER': u'1582730903'}, u'comman
d': u"/usr/bin/bootstrap_host_exec heat_engine su heat -s /bin/bash -c 'heat-manage db_sync'",
u'user': u'root', u'volumes': [u'/etc/hosts:/etc/hosts:ro', u'/etc/localtime:/etc/localtime:r
o', u'/etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro', u'/etc/pki/ca-trust/source/
anchors:/etc/pki/ca-trust/source/anchors:ro', u'/etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/
certs/ca-bundle.crt:ro', u'/etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle
.trust.crt:ro', u'/etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro', u'/dev/log:/dev/log', u'/et
c/ssh/ssh_known_hosts:/etc/ssh/ssh_known_hosts:ro', u'/etc/puppet:/etc/puppet:ro', u'/var/log/
containers/heat:/var/log/heat:z', u'/var/lib/config-data/heat/etc/my.cnf.d/tripleo.cnf:/etc/my
.cnf.d/tripleo.cnf:ro', u'/var/lib/config-data/heat/etc/heat/:/etc/heat/:ro'], u'net': u'host'
, u'detach': False, u'privileged': False}}}) => {"ansible_job_id": "97491261257.184873", "ansi
ble_loop_var": "create_async_result_item", "attempts": 1, "changed": false, "create_async_resu
lt_item": {"ansible_job_id": "97491261257.184873", "ansible_loop_var": "container_data", "chan
ged": true, "container_data": {"heat_engine_db_sync": {"command": "/usr/bin/bootstrap_host_exe
c heat_engine su heat -s /bin/bash -c 'heat-manage db_sync'", "detach": false, "environment":
{"TRIPLEO_CONFIG_HASH": "ac70006d5a3d2f50a563194ea8ab739b-ac70006d5a3d2f50a563194ea8ab739b", "
TRIPLEO_DEPLOY_IDENTIFIER": "1582730903"}, "image": "192.168.24.1:8787/tripleomaster/centos-bi
nary-heat-engine:current-tripleo", "net": "host", "privileged": false, "start_order": 0, "user
": "root", "volumes": ["/etc/hosts:/etc/hosts:ro", "/etc/localtime:/etc/localtime:ro", "/etc/p
ki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro", "/etc/pki/ca-trust/source/anchors:/etc/
pki/ca-trust/source/anchors:ro", "/etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundl
e.crt:ro", "/etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro",
"/etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro", "/dev/log:/dev/log", "/etc/ssh/ssh_known_ho
sts:/etc/ssh/ssh_known_hosts:ro", "/etc/puppet:/etc/puppet:ro", "/var/log/containers/heat:/var
/log/heat:z", "/var/lib/config-data/heat/etc/my.cnf.d/tripleo.cnf:/etc/my.cnf.d/tripleo.cnf:ro
", "/var/lib/config-data/heat/etc/heat/:/etc/heat/:ro"]}}, "failed": false, "finished": 0, "re
sults_file": "/root/.ansible_async/97491261257.184873", "started": 1}, "finished": 1, "msg": "
Can't run container heat_engine_db_sync", "stderr": "ERROR: (pymysql.err.OperationalError) (10
45, u\"Access denied for user 'heat'@'overcloud-controller-2.localdomain' (using password: YES
)\") (Background on this error at: http://sqlalche.me/e/e3q8)\n", "stderr_lines": ["ERROR: (py
mysql.err.OperationalError) (1045, u\"Access denied for user 'heat'@'overcloud-controller-2.lo
caldomain' (using password: YES)\") (Background on this error at: http://sqlalche.me/e/e3q8)"]
, "stdout": "", "stdout_lines": []}
FAILED - RETRYING: Check podman create status (30 retries left).
So did the DB go away?
Network seemed good:
[root@overcloud-controller-0 ~]# telnet overcloud-controller-2.localdomain 3306
Trying 172.16.13.198...
Connected to overcloud-controller-2.localdomain.
Escape character is '^]'.
Y
5.5.5-10.3.10-MariaDB#H$xTy&K���*0Q'lnb!wZ<Zmysql_native_password^]
telnet>
^CConnection closed by foreign host.
[root@overcloud-controller-0 ~]#
Behaviour is like the DB password is no good:
[root@overcloud-controller-0 ~]# podman exec -ti fd50cbb76c37 mysql -u heat -h overcloud-controller-2.localdomain
ERROR 1045 (28000): Access denied for user 'heat'@'overcloud-controller-0.localdomain' (using password: YES)
Error: non zero exit code: 1: OCI runtime error
[root@overcloud-controller-0 ~]#
[root@overcloud-controller-0 ~]# grep pymysql /var/lib/config-data/heat/etc/heat/heat.conf
connection=mysql+pymysql://heat:ziyv7tG93ulmpDdKDd8rFTXO5@172.16.13.39/heat?read_default_group=tripleo&read_default_file=/etc/my.cnf.d/tripleo.cnf
[root@overcloud-controller-0 ~]#
after I copied/pasted the password it still didn't work.
[root@overcloud-controller-0 ~]# grep pymysql /var/lib/config-data/heat/etc/heat/heat.conf
connection=mysql+pymysql://heat:ziyv7tG93ulmpDdKDd8rFTXO5@172.16.13.39/heat?read_default_group=tripleo&read_default_file=/etc/my.cnf.d/tripleo.cnf
[root@overcloud-controller-0 ~]# podman exec -ti fd50cbb76c37 mysql -u heat -h overcloud-controller-2.localdomain -p
Enter password:
ERROR 1045 (28000): Access denied for user 'heat'@'overcloud-controller-0.localdomain' (using password: YES)
Error: non zero exit code: 1: OCI runtime error
[root@overcloud-controller-0 ~]#
formatting ugly, this looks a little better http://