Trying again with 1.30 release on maas bare metal; results in endless configuration loop for all calico instances. See debug output below: unit-calico-3: 00:47:39 INFO unit.calico/3.juju-log cni:17: Configured Calico IP pool. unit-calico-3: 00:47:39 ERROR unit.calico/3.juju-log cni:17: env={'JUJU_UNIT_NAME': 'calico/3', 'JUJU_VERSION': '3.5.1', 'JUJU_CHARM_HTTP_PROXY': '', 'APT_LISTCHANGES_FRONTEND': 'none', 'JUJU_CONTEXT_ID': 'calico/3-cni-relation-changed-1657795354183363859', 'JUJU_AGENT_SOCKET_NETWORK': 'unix', 'JUJU_API_ADDRESSES': '192.168.2.56:17070', 'JUJU_CHARM_HTTPS_PROXY': '', 'JUJU_AGENT_SOCKET_ADDRESS': '@/var/lib/juju/agents/unit-calico-3/agent.socket', 'JUJU_MODEL_NAME': 'k8s', 'JUJU_DISPATCH_PATH': 'hooks/cni-relation-changed', 'JUJU_AVAILABILITY_ZONE': 'default', 'JUJU_REMOTE_UNIT': 'kubernetes-control-plane/1', 'JUJU_CHARM_DIR': '/var/lib/juju/agents/unit-calico-3/charm', 'TERM': 'tmux-256color', 'JUJU_RELATION': 'cni', 'PATH': '/var/lib/juju/tools/unit-calico-3:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/snap/bin', 'JUJU_RELATION_ID': 'cni:17', 'JUJU_METER_STATUS': 'AMBER', 'JUJU_HOOK_NAME': 'cni-relation-changed', 'LANG': 'C.UTF-8', 'CLOUD_API_VERSION': '', 'DEBIAN_FRONTEND': 'noninteractive', 'JUJU_SLA': 'unsupported', 'JUJU_MODEL_UUID': '746a6374-c778-49f8-88fe-f85e86ebceab', 'JUJU_MACHINE_ID': '13', 'JUJU_CHARM_FTP_PROXY': '', 'JUJU_METER_INFO': 'not set', 'PWD': '/var/lib/juju/agents/unit-calico-3/charm', 'JUJU_PRINCIPAL_UNIT': 'kubernetes-control-plane/1', 'JUJU_CHARM_NO_PROXY': '127.0.0.1,localhost,::1', 'PYTHONPATH': 'lib:venv', 'CHARM_DIR': '/var/lib/juju/agents/unit-calico-3/charm', 'JUJU_REMOTE_APP': 'kubernetes-control-plane', 'OPERATOR_DISPATCH': '1', 'ETCD_ENDPOINTS': 'https://192.168.2.2:2379,https://192.168.2.40:2379,https://192.168.2.71:2379', 'ETCD_KEY_FILE': '/opt/calicoctl/etcd-key', 'ETCD_CERT_FILE': '/opt/calicoctl/etcd-cert', 'ETCD_CA_CERT_FILE': '/opt/calicoctl/etcd-ca'} unit-calico-3: 00:47:39 ERROR unit.calico/3.juju-log cni:17: out=null unit-calico-3: 00:47:39 ERROR unit.calico/3.juju-log cni:17: err=time="2024-07-17T05:47:39Z" level=info msg="Log level set to debug" time="2024-07-17T05:47:39Z" level=info msg="Executing config command" time="2024-07-17T05:47:39Z" level=info msg="Config file: /etc/calico/calicoctl.cfg cannot be read - reading config from environment" time="2024-07-17T05:47:39Z" level=debug msg="Datastore type isn't set, trying to detect it" time="2024-07-17T05:47:39Z" level=debug msg="EtcdEndpoints specified, detected etcdv3." time="2024-07-17T05:47:39Z" level=info msg="Loaded client config: apiconfig.CalicoAPIConfigSpec{DatastoreType:\"etcdv3\", EtcdConfig:apiconfig.EtcdConfig{EtcdEndpoints:\"https://192.168.2.2:2379,https://192.168.2.40:2379,https://192.168.2.71:2379\", EtcdDiscoverySrv:\"\", EtcdUsername:\"\", EtcdPassword:\"\", EtcdKeyFile:\"/opt/calicoctl/etcd-key\", EtcdCertFile:\"/opt/calicoctl/etcd-cert\", EtcdCACertFile:\"/opt/calicoctl/etcd-ca\", EtcdKey:\"\", EtcdCert:\"\", EtcdCACert:\"\", EtcdFIPSModeEnabled:false}, KubeConfig:apiconfig.KubeConfig{Kubeconfig:\"\", K8sAPIEndpoint:\"\", K8sKeyFile:\"\", K8sCertFile:\"\", K8sCAFile:\"\", K8sAPIToken:\"\", K8sInsecureSkipTLSVerify:false, K8sDisableNodePoll:false, K8sUsePodCIDR:false, KubeconfigInline:\"\", K8sClientQPS:0, K8sCurrentContext:\"\"}}" time="2024-07-17T05:47:39Z" level=debug msg="Using datastore type 'etcdv3'" time="2024-07-17T05:47:39Z" level=debug msg="creating a TLS config" BuiltWithBoringCrypto=false fipsMode=false time="2024-07-17T05:47:39Z" level=debug msg="Processing Get request" model-etcdKey="ClusterInformation(default)" rev= time="2024-07-17T05:47:39Z" level=debug msg="Calling Get on etcdv3 client" etcdv3-etcdKey=/calico/resources/v3/projectcalico.org/clusterinformations/default model-etcdKey="ClusterInformation(default)" rev= time="2024-07-17T05:47:39Z" level=debug msg="No results returned from etcdv3 client" etcdv3-etcdKey=/calico/resources/v3/projectcalico.org/clusterinformations/default model-etcdKey="ClusterInformation(default)" rev= time="2024-07-17T05:47:39Z" level=info msg="Skip version mismatch checking due to ClusterInformation not being present" time="2024-07-17T05:47:39Z" level=debug msg="Resource: projectcalico.org/v3, Kind=Node" time="2024-07-17T05:47:39Z" level=debug msg="Data: - apiVersion: projectcalico.org/v3\n kind: Node\n metadata:\n creationTimestamp: null\n name: sweet-shiner\n spec: {}\n status: {}\n" time="2024-07-17T05:47:39Z" level=info msg="Config file: /etc/calico/calicoctl.cfg cannot be read - reading config from environment" time="2024-07-17T05:47:39Z" level=debug msg="Datastore type isn't set, trying to detect it" time="2024-07-17T05:47:39Z" level=debug msg="EtcdEndpoints specified, detected etcdv3." time="2024-07-17T05:47:39Z" level=info msg="Loaded client config: apiconfig.CalicoAPIConfigSpec{DatastoreType:\"etcdv3\", EtcdConfig:apiconfig.EtcdConfig{EtcdEndpoints:\"https://192.168.2.2:2379,https://192.168.2.40:2379,https://192.168.2.71:2379\", EtcdDiscoverySrv:\"\", EtcdUsername:\"\", EtcdPassword:\"\", EtcdKeyFile:\"/opt/calicoctl/etcd-key\", EtcdCertFile:\"/opt/calicoctl/etcd-cert\", EtcdCACertFile:\"/opt/calicoctl/etcd-ca\", EtcdKey:\"\", EtcdCert:\"\", EtcdCACert:\"\", EtcdFIPSModeEnabled:false}, KubeConfig:apiconfig.KubeConfig{Kubeconfig:\"\", K8sAPIEndpoint:\"\", K8sKeyFile:\"\", K8sCertFile:\"\", K8sCAFile:\"\", K8sAPIToken:\"\", K8sInsecureSkipTLSVerify:false, K8sDisableNodePoll:false, K8sUsePodCIDR:false, KubeconfigInline:\"\", K8sClientQPS:0, K8sCurrentContext:\"\"}}" time="2024-07-17T05:47:39Z" level=debug msg="Using datastore type 'etcdv3'" time="2024-07-17T05:47:39Z" level=debug msg="creating a TLS config" BuiltWithBoringCrypto=false fipsMode=false time="2024-07-17T05:47:39Z" level=info msg="Client: {{{CalicoAPIConfig projectcalico.org/v3} { 0 {{0 0 }} map[] map[] [] [] []} {etcdv3 {https://192.168.2.2:2379,https://192.168.2.40:2379,https://192.168.2.71:2379 /opt/calicoctl/etcd-key /opt/calicoctl/etcd-cert /opt/calicoctl/etcd-ca false} { false false false 0 }}} 0xc0004f4050 0xc000062240}" time="2024-07-17T05:47:39Z" level=debug msg="Processing Get request" model-etcdKey="Node(sweet-shiner)" rev= time="2024-07-17T05:47:39Z" level=debug msg="Calling Get on etcdv3 client" etcdv3-etcdKey=/calico/resources/v3/projectcalico.org/nodes/sweet-shiner model-etcdKey="Node(sweet-shiner)" rev= time="2024-07-17T05:47:39Z" level=debug msg="No results returned from etcdv3 client" etcdv3-etcdKey=/calico/resources/v3/projectcalico.org/nodes/sweet-shiner model-etcdKey="Node(sweet-shiner)" rev= time="2024-07-17T05:47:39Z" level=info msg="results: {FileInvalid:false NumResources:1 NumHandled:0 Err: SingleKind:Node Resources:[] ResErrs:[resource does not exist: Node(sweet-shiner) with error: ] Client:{config:{TypeMeta:{Kind:CalicoAPIConfig APIVersion:projectcalico.org/v3} ObjectMeta:{Name: GenerateName: Namespace: SelfLink: UID: ResourceVersion: Generation:0 CreationTimestamp:{Time:{wall:0 ext:0 loc:}} DeletionTimestamp: DeletionGracePeriodSeconds: Labels:map[] Annotations:map[] OwnerReferences:[] Finalizers:[] ManagedFields:[]} Spec:{DatastoreType:etcdv3 EtcdConfig:{EtcdEndpoints:https://192.168.2.2:2379,https://192.168.2.40:2379,https://192.168.2.71:2379 EtcdDiscoverySrv: EtcdUsername: EtcdPassword: EtcdKeyFile:/opt/calicoctl/etcd-key EtcdCertFile:/opt/calicoctl/etcd-cert EtcdCACertFile:/opt/calicoctl/etcd-ca EtcdKey: EtcdCert: EtcdCACert: EtcdFIPSModeEnabled:false} KubeConfig:{Kubeconfig: K8sAPIEndpoint: K8sKeyFile: K8sCertFile: K8sCAFile: K8sAPIToken: K8sInsecureSkipTLSVerify:false K8sDisableNodePoll:false K8sUsePodCIDR:false KubeconfigInline: K8sClientQPS:0 K8sCurrentContext:}}} backend:0xc0004f4050 resources:0xc000062240}}" resource does not exist: Node(sweet-shiner) with error: unit-calico-3: 00:47:39 ERROR unit.calico/3.juju-log cni:17: Failed to configure node. Traceback (most recent call last): File "/var/lib/juju/agents/unit-calico-3/charm/./src/charm.py", line 298, in _configure_node node = self._calicoctl_get("node", node_name) File "/var/lib/juju/agents/unit-calico-3/charm/./src/charm.py", line 641, in _calicoctl_get output = self.calicoctl(*args) File "/var/lib/juju/agents/unit-calico-3/charm/./src/charm.py", line 632, in calicoctl return subprocess.check_output(cmd, env=env, stderr=subprocess.PIPE, timeout=timeout) File "/usr/lib/python3.10/subprocess.py", line 421, in check_output return run(*popenargs, stdout=PIPE, timeout=timeout, check=True, File "/usr/lib/python3.10/subprocess.py", line 526, in run raise CalledProcessError(retcode, process.args, subprocess.CalledProcessError: Command '['/opt/calicoctl/calicoctl', '--log-level=debug', 'get', '-o', 'yaml', '--export', 'node', 'sweet-shiner']' returned non-zero exit status 1. unit-calico-3: 00:47:39 ERROR unit.calico/3.juju-log cni:17: Failed to configure Calico, will retry. Traceback (most recent call last): File "/var/lib/juju/agents/unit-calico-3/charm/./src/charm.py", line 174, in _install_or_upgrade self._configure_calico() File "/var/lib/juju/agents/unit-calico-3/charm/./src/charm.py", line 125, in _configure_calico self._configure_node() File "/var/lib/juju/agents/unit-calico-3/charm/./src/charm.py", line 305, in _configure_node raise e File "/var/lib/juju/agents/unit-calico-3/charm/./src/charm.py", line 298, in _configure_node node = self._calicoctl_get("node", node_name) File "/var/lib/juju/agents/unit-calico-3/charm/./src/charm.py", line 641, in _calicoctl_get output = self.calicoctl(*args) File "/var/lib/juju/agents/unit-calico-3/charm/./src/charm.py", line 632, in calicoctl return subprocess.check_output(cmd, env=env, stderr=subprocess.PIPE, timeout=timeout) File "/usr/lib/python3.10/subprocess.py", line 421, in check_output return run(*popenargs, stdout=PIPE, timeout=timeout, check=True, File "/usr/lib/python3.10/subprocess.py", line 526, in run raise CalledProcessError(retcode, process.args, subprocess.CalledProcessError: Command '['/opt/calicoctl/calicoctl', '--log-level=debug', 'get', '-o', 'yaml', '--export', 'node', 'sweet-shiner']' returned non-zero exit status 1. Result of running command from one of the calico instances: ubuntu@rich-poodle:~$ /opt/calicoctl/calicoctl --log-level=debug get -o yaml --export node sweet-shiner INFO[0000] Log level set to debug INFO[0000] Executing config command INFO[0000] Config file: /etc/calico/calicoctl.cfg cannot be read - reading config from environment DEBU[0000] Datastore type isn't set, trying to detect it DEBU[0000] No EtcdEndpoints specified, defaulting to kubernetes. DEBU[0000] Using default kubeconfig path. kubeconfig=/home/ubuntu/.kube/config INFO[0000] Loaded client config: apiconfig.CalicoAPIConfigSpec{DatastoreType:"kubernetes", EtcdConfig:apiconfig.EtcdConfig{EtcdEndpoints:"", EtcdDiscoverySrv:"", EtcdUsername:"", EtcdPassword:"", EtcdKeyFile:"", EtcdCertFile:"", EtcdCACertFile:"", EtcdKey:"", EtcdCert:"", EtcdCACert:"", EtcdFIPSModeEnabled:false}, KubeConfig:apiconfig.KubeConfig{Kubeconfig:"/home/ubuntu/.kube/config", K8sAPIEndpoint:"", K8sKeyFile:"", K8sCertFile:"", K8sCAFile:"", K8sAPIToken:"", K8sInsecureSkipTLSVerify:false, K8sDisableNodePoll:false, K8sUsePodCIDR:false, KubeconfigInline:"", K8sClientQPS:0, K8sCurrentContext:""}} DEBU[0000] Using datastore type 'kubernetes' DEBU[0000] Calico is configured to use calico-ipam DEBU[0000] Performing 'Get' for ClusterInformation(default) DEBU[0000] Get custom Kubernetes resource Key="ClusterInformation(default)" Resource=ClusterInformations Revision= DEBU[0000] Get custom Kubernetes resource by name Key="ClusterInformation(default)" Name=default Namespace= Resource=ClusterInformations Revision= DEBU[0000] Error getting resource Key="ClusterInformation(default)" Name=default Namespace= Resource=ClusterInformations Revision= error="the server could not find the requested resource (get ClusterInformations.crd.projectcalico.org default)" INFO[0000] Skip version mismatch checking due to ClusterInformation not being present DEBU[0000] Resource: projectcalico.org/v3, Kind=Node DEBU[0000] Data: - apiVersion: projectcalico.org/v3 kind: Node metadata: creationTimestamp: null name: sweet-shiner spec: {} status: {} INFO[0000] Config file: /etc/calico/calicoctl.cfg cannot be read - reading config from environment DEBU[0000] Datastore type isn't set, trying to detect it DEBU[0000] No EtcdEndpoints specified, defaulting to kubernetes. DEBU[0000] Using default kubeconfig path. kubeconfig=/home/ubuntu/.kube/config INFO[0000] Loaded client config: apiconfig.CalicoAPIConfigSpec{DatastoreType:"kubernetes", EtcdConfig:apiconfig.EtcdConfig{EtcdEndpoints:"", EtcdDiscoverySrv:"", EtcdUsername:"", EtcdPassword:"", EtcdKeyFile:"", EtcdCertFile:"", EtcdCACertFile:"", EtcdKey:"", EtcdCert:"", EtcdCACert:"", EtcdFIPSModeEnabled:false}, KubeConfig:apiconfig.KubeConfig{Kubeconfig:"/home/ubuntu/.kube/config", K8sAPIEndpoint:"", K8sKeyFile:"", K8sCertFile:"", K8sCAFile:"", K8sAPIToken:"", K8sInsecureSkipTLSVerify:false, K8sDisableNodePoll:false, K8sUsePodCIDR:false, KubeconfigInline:"", K8sClientQPS:0, K8sCurrentContext:""}} DEBU[0000] Using datastore type 'kubernetes' DEBU[0000] Calico is configured to use calico-ipam INFO[0000] Client: {{{CalicoAPIConfig projectcalico.org/v3} { 0 {{0 0 }} map[] map[] [] [] []} {kubernetes { false} {/home/ubuntu/.kube/config false false false 0 }}} 0xc00038d640 0xc0005182d0} DEBU[0000] Performing 'Get' for Node(sweet-shiner) DEBU[0000] Received Get request on Node type INFO[0000] results: {FileInvalid:false NumResources:1 NumHandled:1 Err: SingleKind:Node Resources:[0xc0005bc1c0] ResErrs:[] Client:{config:{TypeMeta:{Kind:CalicoAPIConfig APIVersion:projectcalico.org/v3} ObjectMeta:{Name: GenerateName: Namespace: SelfLink: UID: ResourceVersion: Generation:0 CreationTimestamp:{Time:{wall:0 ext:0 loc:}} DeletionTimestamp: DeletionGracePeriodSeconds: Labels:map[] Annotations:map[] OwnerReferences:[] Finalizers:[] ManagedFields:[]} Spec:{DatastoreType:kubernetes EtcdConfig:{EtcdEndpoints: EtcdDiscoverySrv: EtcdUsername: EtcdPassword: EtcdKeyFile: EtcdCertFile: EtcdCACertFile: EtcdKey: EtcdCert: EtcdCACert: EtcdFIPSModeEnabled:false} KubeConfig:{Kubeconfig:/home/ubuntu/.kube/config K8sAPIEndpoint: K8sKeyFile: K8sCertFile: K8sCAFile: K8sAPIToken: K8sInsecureSkipTLSVerify:false K8sDisableNodePoll:false K8sUsePodCIDR:false KubeconfigInline: K8sClientQPS:0 K8sCurrentContext:}}} backend:0xc00038d640 resources:0xc0005182d0}} apiVersion: projectcalico.org/v3 kind: Node metadata: annotations: projectcalico.org/kube-labels: '{"beta.kubernetes.io/arch":"amd64","beta.kubernetes.io/os":"linux","juju-application":"kubernetes-control-plane","juju-charm":"kubernetes-control-plane","kubernetes.io/arch":"amd64","kubernetes.io/hostname":"sweet-shiner","kubernetes.io/os":"linux","node-role.kubernetes.io/control-plane":""}' creationTimestamp: null labels: beta.kubernetes.io/arch: amd64 beta.kubernetes.io/os: linux juju-application: kubernetes-control-plane juju-charm: kubernetes-control-plane kubernetes.io/arch: amd64 kubernetes.io/hostname: sweet-shiner kubernetes.io/os: linux node-role.kubernetes.io/control-plane: "" name: sweet-shiner spec: addresses: - address: 192.168.2.72 type: InternalIP orchRefs: - nodeName: sweet-shiner orchestrator: k8s status: {}