Did a short investigation since https://review.opendev.org/c/starlingx/config/+/773451 landed. There is a small error observerd in the logs introduced by that that commit, but it is not the cause for the issue observed here. This will be the fix for that error: diff --git a/sysinv/sysinv/sysinv/sysinv/conductor/manager.py b/sysinv/sysinv/sysinv/sysinv/conductor/manager.py index b5189f65..6fb2616e 100644 --- a/sysinv/sysinv/sysinv/sysinv/conductor/manager.py +++ b/sysinv/sysinv/sysinv/sysinv/conductor/manager.py @@ -11908,8 +11908,8 @@ class ConductorManager(service.PeriodicService): LOG.exception("Failed to regenerate the overrides for app %s. %s" % (app.name, e)) else: - LOG.info("{} app active:{} status:{} does not warrant re-apply", - app.name, app.active, app.status) + LOG.info("{} app active:{} status:{} does not warrant re-apply" + "".format(app.name, app.active, app.status)) def app_lifecycle_actions(self, context, rpc_app, hook_info): """Perform any lifecycle actions for the operation and timing supplied. -- 2.30.0 Back to the issue: Seems armada/kubernetes related. sysinv 2021-03-01 11:36:32.372 2356122 INFO sysinv.conductor.kube_app [-] lifecycle hook for application stx-openstack (1.0-78-centos-stable-versioned) started {'lifecycle_type': u'manifest', 'relative_timing': u'pre', 'mode': u'auto', 'operation': u'apply', 'extra': {'was_applied': True}}. sysinv 2021-03-01 11:36:32.372 2356122 INFO k8sapp_openstack.lifecycle.lifecycle_openstack [-] Wait if there are openstack charts in pending install... sysinv 2021-03-01 11:36:32.781 2356122 ERROR sysinv.conductor.kube_app [-] Helm operation failure: Failed to obtain pending charts list: Helm operation failure: Error: write tcp 172.16.192.176:45960->10.10.59.10:5432: write: broken pipe command terminated with exit code 1 : HelmTillerFailure: Helm operation failure: Failed to obtain pending charts list: Helm operation failure: Error: write tcp 172.16.192.176:45960->10.10.59.10:5432: write: broken pipe command terminated with exit code 1 2021-03-01 11:36:32.781 2356122 ERROR sysinv.conductor.kube_app Traceback (most recent call last): var/log/containers$ grep -R "10.10.59.10" | grep armada-api armada-api-b86d46465-xdbjt_armada_tiller-a00cf66fa21b19f28771a99a2aa85643c1fbfd2ed9d19d0f10c2a8ac7925cc1b.log:2021-03-01T10:44:38.71962272Z stderr F [storage/driver] 2021/03/01 10:44:38 list: failed to list: write tcp 172.16.192.176:60758->10.10.59.10:5432: write: broken pipe armada-api-b86d46465-xdbjt_armada_tiller-a00cf66fa21b19f28771a99a2aa85643c1fbfd2ed9d19d0f10c2a8ac7925cc1b.log:2021-03-01T11:36:32.776510152Z stderr F [storage/driver] 2021/03/01 11:36:32 list: failed to list: write tcp 172.16.192.176:45960->10.10.59.10:5432: write: broken pipe armada-api-b86d46465-xdbjt_armada_tiller-a00cf66fa21b19f28771a99a2aa85643c1fbfd2ed9d19d0f10c2a8ac7925cc1b.log:2021-03-01T11:38:56.600564874Z stderr F [storage/driver] 2021/03/01 11:38:56 list: failed to list: write tcp 172.16.192.176:35854->10.10.59.10:5432: write: broken pipe armada-api-b86d46465-xdbjt_armada_tiller-3e0d5dae071494b4c73c9316b798a80c842443b8a99d52319d7cfaaaea458322.log:2021-03-01T01:11:07.873173025Z stderr F [storage/driver] 2021/03/01 01:11:07 list: failed to list: write tcp 172.16.192.115:53838->10.10.59.10:5432: write: broken pipe armada-api-b86d46465-xdbjt_armada_tiller-3e0d5dae071494b4c73c9316b798a80c842443b8a99d52319d7cfaaaea458322.log:2021-03-01T09:38:51.986229937Z stderr F [storage/driver] 2021/03/01 09:38:51 list: failed to list: write tcp 172.16.192.115:47882->10.10.59.10:5432: write: broken pipe