cmanso@proxima:~$ kubectl describe pod horizon-mysql-0   -n openstack
Name:             horizon-mysql-0
Namespace:        openstack
Priority:         0
Service Account:  horizon-mysql
Node:             compute-2/10.1.1.82
Start Time:       Tue, 27 Feb 2024 09:25:00 +0000
Labels:           app.juju.is/created-by=horizon-mysql
                  app.kubernetes.io/name=horizon-mysql
                  apps.kubernetes.io/pod-index=0
                  cluster-name=cluster-7573647c1e26466beeeac32b95ab8f91
                  controller-revision-hash=horizon-mysql-7cfd594c48
                  role=primary
                  statefulset.kubernetes.io/pod-name=horizon-mysql-0
Annotations:      cni.projectcalico.org/containerID: 281686cb304c7df9ff2d5210f681b00adc5e994d30886e6c6006902d42579ac7
                  cni.projectcalico.org/podIP: 10.1.225.122/32
                  cni.projectcalico.org/podIPs: 10.1.225.122/32
                  controller.juju.is/id: d1679821-8219-4d3f-8515-5d599743ee74
                  juju.is/version: 3.2.4
                  model.juju.is/id: dc471c1a-9e9d-4b34-8e34-bd5ea63c8f0e
                  unit.juju.is/id: horizon-mysql/0
Status:           Running
IP:               10.1.225.122
IPs:
  IP:           10.1.225.122
Controlled By:  StatefulSet/horizon-mysql
Init Containers:
  charm-init:
    Container ID:  containerd://36bfb74c9acff0d10b280a5a6355c6fc2a7bd1d7ab297fa7a507b845d4f0d6ac
    Image:         jujusolutions/jujud-operator:3.2.4
    Image ID:      docker.io/jujusolutions/jujud-operator@sha256:e1a1d97aadf141c5bb30bdc2d3b3820c03333008c411d7eac7b33079ae0cc4ff
    Port:          <none>
    Host Port:     <none>
    Command:
      /opt/containeragent
    Args:
      init
      --containeragent-pebble-dir
      /containeragent/pebble
      --charm-modified-version
      0
      --data-dir
      /var/lib/juju
      --bin-dir
      /charm/bin
    State:          Terminated
      Reason:       Completed
      Exit Code:    0
      Started:      Tue, 27 Feb 2024 10:57:36 +0000
      Finished:     Tue, 27 Feb 2024 10:57:37 +0000
    Ready:          True
    Restart Count:  1
    Environment Variables from:
      horizon-mysql-application-config  Secret  Optional: false
    Environment:
      JUJU_CONTAINER_NAMES:  mysql
      JUJU_K8S_POD_NAME:     horizon-mysql-0 (v1:metadata.name)
      JUJU_K8S_POD_UUID:      (v1:metadata.uid)
    Mounts:
      /charm/bin from charm-data (rw,path="charm/bin")
      /charm/containers from charm-data (rw,path="charm/containers")
      /containeragent/pebble from charm-data (rw,path="containeragent/pebble")
      /var/lib/juju from charm-data (rw,path="var/lib/juju")
      /var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-5xwvq (ro)
Containers:
  charm:
    Container ID:  containerd://d3242e3edc9f63762a1f503e0cd3e6d4dea6860b910a00abb4429a0d44421a0e
    Image:         jujusolutions/charm-base:ubuntu-22.04
    Image ID:      docker.io/jujusolutions/charm-base@sha256:8e36ae0684ee404fe1a4b78a1c26f3464aa6bae1033690feb01ba5e7a89d8a4b
    Port:          <none>
    Host Port:     <none>
    Command:
      /charm/bin/pebble
    Args:
      run
      --http
      :38812
      --verbose
    State:          Running
      Started:      Tue, 27 Feb 2024 10:57:40 +0000
    Last State:     Terminated
      Reason:       Unknown
      Exit Code:    255
      Started:      Tue, 27 Feb 2024 09:25:03 +0000
      Finished:     Tue, 27 Feb 2024 10:55:01 +0000
    Ready:          True
    Restart Count:  1
    Limits:
      memory:  2Gi
    Requests:
      memory:   2Gi
    Liveness:   http-get http://:38812/v1/health%3Flevel=alive delay=30s timeout=1s period=5s #success=1 #failure=1
    Readiness:  http-get http://:38812/v1/health%3Flevel=ready delay=30s timeout=1s period=5s #success=1 #failure=1
    Startup:    http-get http://:38812/v1/health%3Flevel=alive delay=30s timeout=1s period=5s #success=1 #failure=1
    Environment:
      JUJU_CONTAINER_NAMES:  mysql
      HTTP_PROBE_PORT:       3856
    Mounts:
      /charm/bin from charm-data (ro,path="charm/bin")
      /charm/containers from charm-data (rw,path="charm/containers")
      /var/lib/juju from charm-data (rw,path="var/lib/juju")
      /var/lib/juju/storage/database/0 from horizon-mysql-database-2c58ffc2 (rw)
      /var/lib/pebble/default from charm-data (rw,path="containeragent/pebble")
      /var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-5xwvq (ro)
  mysql:
    Container ID:  containerd://dd976c06dbd1f11a6050fc544ebf67c38c22c2cce6bcf3849794c7bea0fba31f
    Image:         registry.jujucharms.com/charm/62ptdfbrjpw3n9tcnswjpart30jauc6wc5wbi/mysql-image@sha256:b0d2e028aa86173918c319a0728a1124b6f025985bc6e9774f32d30bbfc96722
    Image ID:      registry.jujucharms.com/charm/62ptdfbrjpw3n9tcnswjpart30jauc6wc5wbi/mysql-image@sha256:b0d2e028aa86173918c319a0728a1124b6f025985bc6e9774f32d30bbfc96722
    Port:          <none>
    Host Port:     <none>
    Command:
      /charm/bin/pebble
    Args:
      run
      --create-dirs
      --hold
      --http
      :38813
      --verbose
    State:          Running
      Started:      Tue, 05 Mar 2024 12:01:05 +0000
    Last State:     Terminated
      Reason:       Error
      Exit Code:    137
      Started:      Tue, 05 Mar 2024 10:52:05 +0000
      Finished:     Tue, 05 Mar 2024 12:01:04 +0000
    Ready:          True
    Restart Count:  130
    Limits:
      memory:  2Gi
    Requests:
      memory:   2Gi
    Liveness:   http-get http://:38813/v1/health%3Flevel=alive delay=30s timeout=1s period=5s #success=1 #failure=1
    Readiness:  http-get http://:38813/v1/health%3Flevel=ready delay=30s timeout=1s period=5s #success=1 #failure=1
    Environment:
      JUJU_CONTAINER_NAME:  mysql
      PEBBLE_SOCKET:        /charm/container/pebble.socket
    Mounts:
      /charm/bin/pebble from charm-data (ro,path="charm/bin/pebble")
      /charm/container from charm-data (rw,path="charm/containers/mysql")
      /var/lib/mysql from horizon-mysql-database-2c58ffc2 (rw)
      /var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-5xwvq (ro)
Conditions:
  Type              Status
  Initialized       True 
  Ready             True 
  ContainersReady   True 
  PodScheduled      True 
Volumes:
  horizon-mysql-database-2c58ffc2:
    Type:       PersistentVolumeClaim (a reference to a PersistentVolumeClaim in the same namespace)
    ClaimName:  horizon-mysql-database-2c58ffc2-horizon-mysql-0
    ReadOnly:   false
  charm-data:
    Type:       EmptyDir (a temporary directory that shares a pod's lifetime)
    Medium:     
    SizeLimit:  <unset>
  kube-api-access-5xwvq:
    Type:                    Projected (a volume that contains injected data from multiple sources)
    TokenExpirationSeconds:  3607
    ConfigMapName:           kube-root-ca.crt
    ConfigMapOptional:       <nil>
    DownwardAPI:             true
QoS Class:                   Burstable
Node-Selectors:              kubernetes.io/arch=amd64
Tolerations:                 node.kubernetes.io/not-ready:NoExecute op=Exists for 300s
                             node.kubernetes.io/unreachable:NoExecute op=Exists for 300s
Events:
  Type    Reason   Age                     From     Message
  ----    ------   ----                    ----     -------
  Normal  Pulled   2m24s (x130 over 7d1h)  kubelet  Container image "registry.jujucharms.com/charm/62ptdfbrjpw3n9tcnswjpart30jauc6wc5wbi/mysql-image@sha256:b0d2e028aa86173918c319a0728a1124b6f025985bc6e9774f32d30bbfc96722" already present on machine
  Normal  Created  2m24s (x130 over 7d1h)  kubelet  Created container mysql
  Normal  Started  2m24s (x130 over 7d1h)  kubelet  Started container mysql


























































 cmanso@proxima:~$ kubectl logs horizon-mysql-0   -n openstack






2024-02-27T10:57:40.049Z [pebble] HTTP API server listening on ":38812".
2024-02-27T10:57:40.049Z [pebble] Started daemon.
2024-02-27T10:57:40.101Z [pebble] POST /v1/services 50.428702ms 202
2024-02-27T10:57:40.101Z [pebble] Started default services with change 3.
2024-02-27T10:57:40.119Z [pebble] Service "container-agent" starting: /charm/bin/containeragent unit --data-dir /var/lib/juju --append-env "PATH=$PATH:/charm/bin" --show-log --charm-modified-version 0
2024-02-27T10:57:40.196Z [container-agent] 2024-02-27 10:57:40 INFO juju.cmd supercommand.go:56 running containerAgent [3.2.4 3052b293cbab49b1a2f994724d2c90e80954cfe3 gc go1.21.4]
2024-02-27T10:57:40.196Z [container-agent] starting containeragent unit command
2024-02-27T10:57:40.197Z [container-agent] containeragent unit "unit-horizon-mysql-0" start (3.2.4 [gc])
2024-02-27T10:57:40.197Z [container-agent] 2024-02-27 10:57:40 INFO juju.cmd.containeragent.unit runner.go:578 start "unit"
2024-02-27T10:57:40.197Z [container-agent] 2024-02-27 10:57:40 INFO juju.worker.upgradesteps worker.go:60 upgrade steps for 3.2.4 have already been run.
2024-02-27T10:57:40.201Z [container-agent] 2024-02-27 10:57:40 INFO juju.worker.probehttpserver server.go:157 starting http server on 127.0.0.1:65301
2024-02-27T10:57:40.248Z [container-agent] 2024-02-27 10:57:40 INFO juju.api apiclient.go:706 connection established to "wss://10.1.1.82:17070/model/dc471c1a-9e9d-4b34-8e34-bd5ea63c8f0e/api"
2024-02-27T10:57:40.260Z [container-agent] 2024-02-27 10:57:40 INFO juju.worker.apicaller connect.go:163 [dc471c] "unit-horizon-mysql-0" successfully connected to "10.1.1.82:17070"
2024-02-27T10:57:40.294Z [container-agent] 2024-02-27 10:57:40 INFO juju.api apiclient.go:706 connection established to "wss://10.1.6.82:17070/model/dc471c1a-9e9d-4b34-8e34-bd5ea63c8f0e/api"
2024-02-27T10:57:40.298Z [container-agent] 2024-02-27 10:57:40 INFO juju.worker.apicaller connect.go:163 [dc471c] "unit-horizon-mysql-0" successfully connected to "10.1.6.82:17070"
2024-02-27T10:57:40.320Z [container-agent] 2024-02-27 10:57:40 INFO juju.worker.migrationminion worker.go:142 migration phase is now: NONE
2024-02-27T10:57:40.320Z [container-agent] 2024-02-27 10:57:40 INFO juju.worker.logger logger.go:120 logger worker started
2024-02-27T10:57:40.324Z [container-agent] 2024-02-27 10:57:40 WARNING juju.worker.proxyupdater proxyupdater.go:241 unable to set snap core settings [proxy.http= proxy.https= proxy.store=]: exec: "snap": executable file not found in $PATH, output: ""
2024-02-27T10:57:40.435Z [container-agent] 2024-02-27 10:57:40 INFO juju.agent.tools symlinks.go:20 ensure jujuc symlinks in /var/lib/juju/tools/unit-horizon-mysql-0
2024-02-27T10:57:40.438Z [container-agent] 2024-02-27 10:57:40 INFO juju.worker.leadership tracker.go:194 horizon-mysql/0 promoted to leadership of horizon-mysql
2024-02-27T10:57:40.448Z [container-agent] 2024-02-27 10:57:40 INFO juju.worker.caasupgrader upgrader.go:113 abort check blocked until version event received
2024-02-27T10:57:40.448Z [container-agent] 2024-02-27 10:57:40 INFO juju.worker.caasupgrader upgrader.go:119 unblocking abort check
2024-02-27T10:57:40.832Z [container-agent] 2024-02-27 10:57:40 INFO juju.worker.uniter uniter.go:363 unit "horizon-mysql/0" started
2024-02-27T10:57:40.844Z [container-agent] 2024-02-27 10:57:40 INFO juju.worker.uniter uniter.go:389 hooks are retried true
2024-02-27T10:57:41.000Z [container-agent] 2024-02-27 10:57:41 INFO juju.worker.uniter resolver.go:76 reboot detected; triggering implicit start hook to notify charm
2024-02-27T10:57:46.512Z [container-agent] 2024-02-27 10:57:46 INFO juju-log Running legacy hooks/start.
2024-02-27T10:57:48.151Z [container-agent] 2024-02-27 10:57:48 INFO juju-log Starting the log rotate manager
2024-02-27T10:57:48.164Z [container-agent] 2024-02-27 10:57:48 INFO juju-log Started log rotate manager process with PID 99
2024-02-27T10:57:48.957Z [container-agent] 2024-02-27 10:57:48 INFO juju.worker.uniter.operation runhook.go:186 ran "start" hook (via hook dispatching script: dispatch)
2024-02-27T10:57:51.159Z [container-agent] 2024-02-27 10:57:51 INFO juju-log Setting up the logrotate configurations
2024-02-27T10:57:51.255Z [container-agent] 2024-02-27 10:57:51 INFO juju-log Adding pebble layer
2024-02-27T10:57:58.462Z [container-agent] 2024-02-27 10:57:58 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-02-27T10:57:58.511Z [container-agent] 2024-02-27 10:57:58 INFO juju-log Attempting reboot from complete outage.
2024-02-27T10:58:02.312Z [container-agent] 2024-02-27 10:58:02 INFO juju.worker.uniter.operation runhook.go:186 ran "mysql-pebble-ready" hook (via hook dispatching script: dispatch)
2024-02-27T11:02:51.908Z [container-agent] 2024-02-27 11:02:51 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-27T11:02:54.402Z [container-agent] 2024-02-27 11:02:54 WARNING juju-log No relation: certificates
2024-02-27T11:02:54.808Z [container-agent] 2024-02-27 11:02:54 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-27T11:07:53.721Z [container-agent] 2024-02-27 11:07:53 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-27T11:07:56.455Z [container-agent] 2024-02-27 11:07:56 WARNING juju-log No relation: certificates
2024-02-27T11:07:56.826Z [container-agent] 2024-02-27 11:07:56 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-27T11:12:18.263Z [container-agent] 2024-02-27 11:12:18 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-27T11:12:21.445Z [container-agent] 2024-02-27 11:12:21 WARNING juju-log No relation: certificates
2024-02-27T11:12:21.803Z [container-agent] 2024-02-27 11:12:21 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-27T11:17:09.514Z [container-agent] 2024-02-27 11:17:09 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-27T11:17:13.358Z [container-agent] 2024-02-27 11:17:13 WARNING juju-log No relation: certificates
2024-02-27T11:17:13.716Z [container-agent] 2024-02-27 11:17:13 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-27T11:21:18.516Z [container-agent] 2024-02-27 11:21:18 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-27T11:21:22.744Z [container-agent] 2024-02-27 11:21:22 WARNING juju-log No relation: certificates
2024-02-27T11:21:23.090Z [container-agent] 2024-02-27 11:21:23 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-27T11:25:28.559Z [container-agent] 2024-02-27 11:25:28 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-27T11:25:33.252Z [container-agent] 2024-02-27 11:25:33 WARNING juju-log No relation: certificates
2024-02-27T11:25:33.637Z [container-agent] 2024-02-27 11:25:33 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-27T11:31:18.233Z [container-agent] 2024-02-27 11:31:18 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-27T11:31:23.349Z [container-agent] 2024-02-27 11:31:23 WARNING juju-log No relation: certificates
2024-02-27T11:31:23.685Z [container-agent] 2024-02-27 11:31:23 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-27T11:35:53.144Z [container-agent] 2024-02-27 11:35:53 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-27T11:35:58.756Z [container-agent] 2024-02-27 11:35:58 WARNING juju-log No relation: certificates
2024-02-27T11:35:59.127Z [container-agent] 2024-02-27 11:35:59 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-27T11:41:41.611Z [container-agent] 2024-02-27 11:41:41 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-27T11:41:47.760Z [container-agent] 2024-02-27 11:41:47 WARNING juju-log No relation: certificates
2024-02-27T11:41:48.108Z [container-agent] 2024-02-27 11:41:48 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-27T11:46:34.567Z [container-agent] 2024-02-27 11:46:34 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-27T11:46:41.010Z [container-agent] 2024-02-27 11:46:41 WARNING juju-log No relation: certificates
2024-02-27T11:46:41.354Z [container-agent] 2024-02-27 11:46:41 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-27T11:52:26.160Z [container-agent] 2024-02-27 11:52:26 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-27T11:52:33.025Z [container-agent] 2024-02-27 11:52:33 WARNING juju-log No relation: certificates
2024-02-27T11:52:33.391Z [container-agent] 2024-02-27 11:52:33 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-27T11:55:12.455Z [container-agent] 2024-02-27 11:55:12 ERROR juju-log Failed to flush SLOW LOGS logs.
2024-02-27T11:55:12.455Z [container-agent] Traceback (most recent call last):
2024-02-27T11:55:12.455Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 666, in _run_mysqlsh_script
2024-02-27T11:55:12.455Z [container-agent]     stdout, _ = process.wait_output()
2024-02-27T11:55:12.455Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1441, in wait_output
2024-02-27T11:55:12.455Z [container-agent]     raise ExecError[AnyStr](self._command, exit_code, out_value, err_value)
2024-02-27T11:55:12.455Z [container-agent] ops.pebble.ExecError: non-zero exit code 1 executing ['/usr/bin/mysqlsh', '--no-wizard', '--python', '--verbose=1', '-f', '/tmp/script.py', ';', 'rm', '/tmp/script.py'], stdout='', stderr='Cannot set LC_ALL to locale en_US.UTF-8: No such file or directory\nverbose: 2024-02-27T11:55:11Z: Loading startup files...\nverbose: 2024-02-27T11:55:11Z: Loading plugins...\nverbose: 2024-02-27T11:55:11Z: Connecting to MySQL at: serverconfig@horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local\nTraceback (most recent call last):\n  File "<string>", line 1, in <module>\nmysqlsh.DBError: MySQL Error (2003): Shell.connect: Can\'t connect to MySQL server on \'horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local:3306\' (111)\n'
2024-02-27T11:55:12.455Z [container-agent] 
2024-02-27T11:55:12.455Z [container-agent] During handling of the above exception, another exception occurred:
2024-02-27T11:55:12.455Z [container-agent] 
2024-02-27T11:55:12.455Z [container-agent] Traceback (most recent call last):
2024-02-27T11:55:12.455Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-02-27T11:55:12.455Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-02-27T11:55:12.455Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 669, in _run_mysqlsh_script
2024-02-27T11:55:12.455Z [container-agent]     raise MySQLClientError(e.stderr)
2024-02-27T11:55:12.455Z [container-agent] charms.mysql.v0.mysql.MySQLClientError: Cannot set LC_ALL to locale en_US.UTF-8: No such file or directory
2024-02-27T11:55:12.455Z [container-agent] verbose: 2024-02-27T11:55:11Z: Loading startup files...
2024-02-27T11:55:12.455Z [container-agent] verbose: 2024-02-27T11:55:11Z: Loading plugins...
2024-02-27T11:55:12.455Z [container-agent] verbose: 2024-02-27T11:55:11Z: Connecting to MySQL at: serverconfig@horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local
2024-02-27T11:55:12.455Z [container-agent] Traceback (most recent call last):
2024-02-27T11:55:12.455Z [container-agent]   File "<string>", line 1, in <module>
2024-02-27T11:55:12.455Z [container-agent] mysqlsh.DBError: MySQL Error (2003): Shell.connect: Can't connect to MySQL server on 'horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local:3306' (111)
2024-02-27T11:55:12.455Z [container-agent] 
2024-02-27T11:57:46.719Z [container-agent] 2024-02-27 11:57:46 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-02-27T11:57:46.746Z [container-agent] 2024-02-27 11:57:46 INFO juju-log Attempting reboot from complete outage.
2024-02-27T11:57:53.960Z [container-agent] 2024-02-27 11:57:53 WARNING juju-log No relation: certificates
2024-02-27T11:57:54.349Z [container-agent] 2024-02-27 11:57:54 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-27T12:00:09.798Z [container-agent] 2024-02-27 12:00:09 ERROR juju-log Uncaught exception while in charm code:
2024-02-27T12:00:09.798Z [container-agent] Traceback (most recent call last):
2024-02-27T12:00:09.798Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/./src/charm.py", line 786, in <module>
2024-02-27T12:00:09.798Z [container-agent]     main(MySQLOperatorCharm)
2024-02-27T12:00:09.798Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 436, in main
2024-02-27T12:00:09.798Z [container-agent]     _emit_charm_event(charm, dispatcher.event_name)
2024-02-27T12:00:09.798Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 144, in _emit_charm_event
2024-02-27T12:00:09.798Z [container-agent]     event_to_emit.emit(*args, **kwargs)
2024-02-27T12:00:09.798Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 351, in emit
2024-02-27T12:00:09.798Z [container-agent]     framework._emit(event)
2024-02-27T12:00:09.798Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 853, in _emit
2024-02-27T12:00:09.798Z [container-agent]     self._reemit(event_path)
2024-02-27T12:00:09.798Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 942, in _reemit
2024-02-27T12:00:09.798Z [container-agent]     custom_handler(event)
2024-02-27T12:00:09.798Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/rotate_mysql_logs.py", line 57, in _rotate_mysql_logs
2024-02-27T12:00:09.798Z [container-agent]     self.charm._mysql.flush_mysql_logs(MySQLTextLogs.SLOW)
2024-02-27T12:00:09.798Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-02-27T12:00:09.798Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-02-27T12:00:09.798Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 665, in _run_mysqlsh_script
2024-02-27T12:00:09.798Z [container-agent]     process = self.container.exec(cmd, timeout=timeout)
2024-02-27T12:00:09.798Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/model.py", line 2688, in exec
2024-02-27T12:00:09.798Z [container-agent]     return self._pebble.exec(
2024-02-27T12:00:09.798Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 2501, in exec
2024-02-27T12:00:09.798Z [container-agent]     resp = self._request('POST', '/v1/exec', body=body)
2024-02-27T12:00:09.798Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1655, in _request
2024-02-27T12:00:09.798Z [container-agent]     response = self._request_raw(method, path, query, headers, data)
2024-02-27T12:00:09.798Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1690, in _request_raw
2024-02-27T12:00:09.798Z [container-agent]     response = self.opener.open(request, timeout=self.timeout)
2024-02-27T12:00:09.798Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 519, in open
2024-02-27T12:00:09.798Z [container-agent]     response = self._open(req, data)
2024-02-27T12:00:09.798Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 536, in _open
2024-02-27T12:00:09.798Z [container-agent]     result = self._call_chain(self.handle_open, protocol, protocol +
2024-02-27T12:00:09.798Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 496, in _call_chain
2024-02-27T12:00:09.798Z [container-agent]     result = func(*args)
2024-02-27T12:00:09.798Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 311, in http_open
2024-02-27T12:00:09.798Z [container-agent]     return self.do_open(_UnixSocketConnection, req,  # type:ignore
2024-02-27T12:00:09.798Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 1352, in do_open
2024-02-27T12:00:09.798Z [container-agent]     r = h.getresponse()
2024-02-27T12:00:09.798Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1375, in getresponse
2024-02-27T12:00:09.798Z [container-agent]     response.begin()
2024-02-27T12:00:09.798Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 318, in begin
2024-02-27T12:00:09.798Z [container-agent]     version, status, reason = self._read_status()
2024-02-27T12:00:09.798Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 287, in _read_status
2024-02-27T12:00:09.798Z [container-agent]     raise RemoteDisconnected("Remote end closed connection without"
2024-02-27T12:00:09.798Z [container-agent] http.client.RemoteDisconnected: Remote end closed connection without response
2024-02-27T12:00:09.946Z [container-agent] 2024-02-27 12:00:09 INFO juju.util.exec exec.go:209 run result: exit status 1
2024-02-27T12:00:15.322Z [container-agent] 2024-02-27 12:00:15 INFO juju-log Setting up the logrotate configurations
2024-02-27T12:00:15.391Z [container-agent] 2024-02-27 12:00:15 INFO juju-log Adding pebble layer
2024-02-27T12:00:22.141Z [container-agent] 2024-02-27 12:00:22 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-02-27T12:00:22.162Z [container-agent] 2024-02-27 12:00:22 INFO juju-log Attempting reboot from complete outage.
2024-02-27T12:00:25.119Z [container-agent] 2024-02-27 12:00:25 INFO juju.worker.uniter.operation runhook.go:186 ran "mysql-pebble-ready" hook (via hook dispatching script: dispatch)
2024-02-27T12:03:02.485Z [container-agent] 2024-02-27 12:03:02 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-27T12:03:04.496Z [container-agent] 2024-02-27 12:03:04 WARNING juju-log No relation: certificates
2024-02-27T12:03:04.837Z [container-agent] 2024-02-27 12:03:04 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-27T12:08:31.988Z [container-agent] 2024-02-27 12:08:31 INFO juju-log Starting the log rotate manager
2024-02-27T12:08:32.001Z [container-agent] 2024-02-27 12:08:32 INFO juju-log Started log rotate manager process with PID 6718
2024-02-27T12:08:32.292Z [container-agent] 2024-02-27 12:08:32 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-27T12:08:34.120Z [container-agent] 2024-02-27 12:08:34 WARNING juju-log No relation: certificates
2024-02-27T12:08:34.501Z [container-agent] 2024-02-27 12:08:34 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-27T12:14:25.385Z [container-agent] 2024-02-27 12:14:25 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-27T12:14:27.706Z [container-agent] 2024-02-27 12:14:27 WARNING juju-log No relation: certificates
2024-02-27T12:14:28.077Z [container-agent] 2024-02-27 12:14:28 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-27T12:20:06.528Z [container-agent] 2024-02-27 12:20:06 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-27T12:20:09.603Z [container-agent] 2024-02-27 12:20:09 WARNING juju-log No relation: certificates
2024-02-27T12:20:09.987Z [container-agent] 2024-02-27 12:20:09 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-27T12:24:15.717Z [container-agent] 2024-02-27 12:24:15 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-27T12:24:19.321Z [container-agent] 2024-02-27 12:24:19 WARNING juju-log No relation: certificates
2024-02-27T12:24:19.665Z [container-agent] 2024-02-27 12:24:19 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-27T12:28:31.466Z [container-agent] 2024-02-27 12:28:31 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-27T12:28:35.524Z [container-agent] 2024-02-27 12:28:35 WARNING juju-log No relation: certificates
2024-02-27T12:28:35.860Z [container-agent] 2024-02-27 12:28:35 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-27T12:33:54.641Z [container-agent] 2024-02-27 12:33:54 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-27T12:33:59.250Z [container-agent] 2024-02-27 12:33:59 WARNING juju-log No relation: certificates
2024-02-27T12:33:59.626Z [container-agent] 2024-02-27 12:33:59 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-27T12:38:33.516Z [container-agent] 2024-02-27 12:38:33 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-27T12:38:38.461Z [container-agent] 2024-02-27 12:38:38 WARNING juju-log No relation: certificates
2024-02-27T12:38:38.841Z [container-agent] 2024-02-27 12:38:38 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-27T12:44:32.404Z [container-agent] 2024-02-27 12:44:32 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-27T12:44:37.759Z [container-agent] 2024-02-27 12:44:37 WARNING juju-log No relation: certificates
2024-02-27T12:44:38.102Z [container-agent] 2024-02-27 12:44:38 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-27T12:48:57.181Z [container-agent] 2024-02-27 12:48:57 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-27T12:49:02.812Z [container-agent] 2024-02-27 12:49:02 WARNING juju-log No relation: certificates
2024-02-27T12:49:03.175Z [container-agent] 2024-02-27 12:49:03 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-27T12:53:11.395Z [container-agent] 2024-02-27 12:53:11 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-27T12:53:17.457Z [container-agent] 2024-02-27 12:53:17 WARNING juju-log No relation: certificates
2024-02-27T12:53:17.805Z [container-agent] 2024-02-27 12:53:17 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-27T12:57:49.506Z [container-agent] 2024-02-27 12:57:49 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-27T12:57:55.911Z [container-agent] 2024-02-27 12:57:55 WARNING juju-log No relation: certificates
2024-02-27T12:57:56.252Z [container-agent] 2024-02-27 12:57:56 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-27T13:02:17.201Z [container-agent] 2024-02-27 13:02:17 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-27T13:02:24.364Z [container-agent] 2024-02-27 13:02:24 WARNING juju-log No relation: certificates
2024-02-27T13:02:24.730Z [container-agent] 2024-02-27 13:02:24 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-27T13:07:17.232Z [container-agent] 2024-02-27 13:07:17 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-27T13:07:24.785Z [container-agent] 2024-02-27 13:07:24 WARNING juju-log No relation: certificates
2024-02-27T13:07:25.127Z [container-agent] 2024-02-27 13:07:25 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-27T13:12:47.681Z [container-agent] 2024-02-27 13:12:47 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-02-27T13:12:47.715Z [container-agent] 2024-02-27 13:12:47 INFO juju-log Attempting reboot from complete outage.
2024-02-27T13:12:55.252Z [container-agent] 2024-02-27 13:12:55 WARNING juju-log No relation: certificates
2024-02-27T13:12:55.614Z [container-agent] 2024-02-27 13:12:55 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-27T13:17:04.862Z [container-agent] 2024-02-27 13:17:04 ERROR juju-log Uncaught exception while in charm code:
2024-02-27T13:17:04.862Z [container-agent] Traceback (most recent call last):
2024-02-27T13:17:04.862Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/./src/charm.py", line 786, in <module>
2024-02-27T13:17:04.862Z [container-agent]     main(MySQLOperatorCharm)
2024-02-27T13:17:04.862Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 436, in main
2024-02-27T13:17:04.862Z [container-agent]     _emit_charm_event(charm, dispatcher.event_name)
2024-02-27T13:17:04.862Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 144, in _emit_charm_event
2024-02-27T13:17:04.862Z [container-agent]     event_to_emit.emit(*args, **kwargs)
2024-02-27T13:17:04.862Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 351, in emit
2024-02-27T13:17:04.862Z [container-agent]     framework._emit(event)
2024-02-27T13:17:04.862Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 853, in _emit
2024-02-27T13:17:04.862Z [container-agent]     self._reemit(event_path)
2024-02-27T13:17:04.862Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 942, in _reemit
2024-02-27T13:17:04.862Z [container-agent]     custom_handler(event)
2024-02-27T13:17:04.862Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/rotate_mysql_logs.py", line 55, in _rotate_mysql_logs
2024-02-27T13:17:04.862Z [container-agent]     self.charm._mysql.flush_mysql_logs(MySQLTextLogs.ERROR)
2024-02-27T13:17:04.862Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-02-27T13:17:04.862Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-02-27T13:17:04.862Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 666, in _run_mysqlsh_script
2024-02-27T13:17:04.862Z [container-agent]     stdout, _ = process.wait_output()
2024-02-27T13:17:04.862Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1436, in wait_output
2024-02-27T13:17:04.862Z [container-agent]     exit_code: int = self._wait()
2024-02-27T13:17:04.862Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1375, in _wait
2024-02-27T13:17:04.862Z [container-agent]     change = self._client.wait_change(self._change_id, timeout=timeout)
2024-02-27T13:17:04.862Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1893, in wait_change
2024-02-27T13:17:04.862Z [container-agent]     return self._wait_change_using_wait(change_id, timeout)
2024-02-27T13:17:04.862Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1914, in _wait_change_using_wait
2024-02-27T13:17:04.862Z [container-agent]     return self._wait_change(change_id, this_timeout)
2024-02-27T13:17:04.862Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1928, in _wait_change
2024-02-27T13:17:04.862Z [container-agent]     resp = self._request('GET', f'/v1/changes/{change_id}/wait', query)
2024-02-27T13:17:04.862Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1655, in _request
2024-02-27T13:17:04.862Z [container-agent]     response = self._request_raw(method, path, query, headers, data)
2024-02-27T13:17:04.862Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1690, in _request_raw
2024-02-27T13:17:04.862Z [container-agent]     response = self.opener.open(request, timeout=self.timeout)
2024-02-27T13:17:04.862Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 519, in open
2024-02-27T13:17:04.862Z [container-agent]     response = self._open(req, data)
2024-02-27T13:17:04.862Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 536, in _open
2024-02-27T13:17:04.862Z [container-agent]     result = self._call_chain(self.handle_open, protocol, protocol +
2024-02-27T13:17:04.862Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 496, in _call_chain
2024-02-27T13:17:04.862Z [container-agent]     result = func(*args)
2024-02-27T13:17:04.862Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 311, in http_open
2024-02-27T13:17:04.862Z [container-agent]     return self.do_open(_UnixSocketConnection, req,  # type:ignore
2024-02-27T13:17:04.862Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 1352, in do_open
2024-02-27T13:17:04.862Z [container-agent]     r = h.getresponse()
2024-02-27T13:17:04.862Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1375, in getresponse
2024-02-27T13:17:04.862Z [container-agent]     response.begin()
2024-02-27T13:17:04.862Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 318, in begin
2024-02-27T13:17:04.862Z [container-agent]     version, status, reason = self._read_status()
2024-02-27T13:17:04.862Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 287, in _read_status
2024-02-27T13:17:04.862Z [container-agent]     raise RemoteDisconnected("Remote end closed connection without"
2024-02-27T13:17:04.862Z [container-agent] http.client.RemoteDisconnected: Remote end closed connection without response
2024-02-27T13:17:05.045Z [container-agent] 2024-02-27 13:17:05 INFO juju.util.exec exec.go:209 run result: exit status 1
2024-02-27T13:17:08.135Z [container-agent] 2024-02-27 13:17:08 INFO juju-log Setting up the logrotate configurations
2024-02-27T13:17:08.208Z [container-agent] 2024-02-27 13:17:08 INFO juju-log Adding pebble layer
2024-02-27T13:17:14.990Z [container-agent] 2024-02-27 13:17:14 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-02-27T13:17:15.016Z [container-agent] 2024-02-27 13:17:15 INFO juju-log Attempting reboot from complete outage.
2024-02-27T13:17:17.987Z [container-agent] 2024-02-27 13:17:17 INFO juju.worker.uniter.operation runhook.go:186 ran "mysql-pebble-ready" hook (via hook dispatching script: dispatch)
2024-02-27T13:17:46.613Z [container-agent] 2024-02-27 13:17:46 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-27T13:17:48.561Z [container-agent] 2024-02-27 13:17:48 WARNING juju-log No relation: certificates
2024-02-27T13:17:48.939Z [container-agent] 2024-02-27 13:17:48 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-27T13:21:56.572Z [container-agent] 2024-02-27 13:21:56 INFO juju-log Starting the log rotate manager
2024-02-27T13:21:56.585Z [container-agent] 2024-02-27 13:21:56 INFO juju-log Started log rotate manager process with PID 13868
2024-02-27T13:21:56.861Z [container-agent] 2024-02-27 13:21:56 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-27T13:21:58.838Z [container-agent] 2024-02-27 13:21:58 WARNING juju-log No relation: certificates
2024-02-27T13:21:59.225Z [container-agent] 2024-02-27 13:21:59 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-27T13:27:40.794Z [container-agent] 2024-02-27 13:27:40 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-27T13:27:43.279Z [container-agent] 2024-02-27 13:27:43 WARNING juju-log No relation: certificates
2024-02-27T13:27:43.636Z [container-agent] 2024-02-27 13:27:43 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-27T13:32:15.181Z [container-agent] 2024-02-27 13:32:15 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-27T13:32:18.403Z [container-agent] 2024-02-27 13:32:18 WARNING juju-log No relation: certificates
2024-02-27T13:32:18.772Z [container-agent] 2024-02-27 13:32:18 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-27T13:37:15.845Z [container-agent] 2024-02-27 13:37:15 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-27T13:37:19.197Z [container-agent] 2024-02-27 13:37:19 WARNING juju-log No relation: certificates
2024-02-27T13:37:19.532Z [container-agent] 2024-02-27 13:37:19 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-27T13:43:15.806Z [container-agent] 2024-02-27 13:43:15 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-27T13:43:20.132Z [container-agent] 2024-02-27 13:43:20 WARNING juju-log No relation: certificates
2024-02-27T13:43:20.471Z [container-agent] 2024-02-27 13:43:20 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-27T13:48:51.915Z [container-agent] 2024-02-27 13:48:51 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-27T13:48:55.941Z [container-agent] 2024-02-27 13:48:55 WARNING juju-log No relation: certificates
2024-02-27T13:48:56.282Z [container-agent] 2024-02-27 13:48:56 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-27T13:53:25.113Z [container-agent] 2024-02-27 13:53:25 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-27T13:53:29.790Z [container-agent] 2024-02-27 13:53:29 WARNING juju-log No relation: certificates
2024-02-27T13:53:30.140Z [container-agent] 2024-02-27 13:53:30 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-27T13:58:09.795Z [container-agent] 2024-02-27 13:58:09 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-27T13:58:15.272Z [container-agent] 2024-02-27 13:58:15 WARNING juju-log No relation: certificates
2024-02-27T13:58:15.632Z [container-agent] 2024-02-27 13:58:15 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-27T14:02:55.257Z [container-agent] 2024-02-27 14:02:55 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-27T14:03:01.132Z [container-agent] 2024-02-27 14:03:01 WARNING juju-log No relation: certificates
2024-02-27T14:03:01.509Z [container-agent] 2024-02-27 14:03:01 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-27T14:07:18.082Z [container-agent] 2024-02-27 14:07:18 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-27T14:07:24.314Z [container-agent] 2024-02-27 14:07:24 WARNING juju-log No relation: certificates
2024-02-27T14:07:24.655Z [container-agent] 2024-02-27 14:07:24 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-27T14:11:26.074Z [container-agent] 2024-02-27 14:11:26 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-27T14:11:32.957Z [container-agent] 2024-02-27 14:11:32 WARNING juju-log No relation: certificates
2024-02-27T14:11:33.320Z [container-agent] 2024-02-27 14:11:33 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-27T14:17:03.194Z [container-agent] 2024-02-27 14:17:03 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-27T14:17:09.694Z [container-agent] 2024-02-27 14:17:09 WARNING juju-log No relation: certificates
2024-02-27T14:17:10.026Z [container-agent] 2024-02-27 14:17:10 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-27T14:21:37.382Z [container-agent] 2024-02-27 14:21:37 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-27T14:21:44.614Z [container-agent] 2024-02-27 14:21:44 WARNING juju-log No relation: certificates
2024-02-27T14:21:44.969Z [container-agent] 2024-02-27 14:21:44 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-27T14:26:15.248Z [container-agent] 2024-02-27 14:26:15 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-27T14:26:23.235Z [container-agent] 2024-02-27 14:26:23 WARNING juju-log No relation: certificates
2024-02-27T14:26:23.592Z [container-agent] 2024-02-27 14:26:23 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-27T14:30:20.415Z [container-agent] 2024-02-27 14:30:20 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-27T14:30:28.623Z [container-agent] 2024-02-27 14:30:28 WARNING juju-log No relation: certificates
2024-02-27T14:30:28.972Z [container-agent] 2024-02-27 14:30:28 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-27T14:34:21.170Z [container-agent] 2024-02-27 14:34:21 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-02-27T14:34:21.197Z [container-agent] 2024-02-27 14:34:21 INFO juju-log Attempting reboot from complete outage.
2024-02-27T14:34:29.263Z [container-agent] 2024-02-27 14:34:29 WARNING juju-log No relation: certificates
2024-02-27T14:34:29.660Z [container-agent] 2024-02-27 14:34:29 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-27T14:39:04.741Z [container-agent] 2024-02-27 14:39:04 ERROR juju-log Uncaught exception while in charm code:
2024-02-27T14:39:04.741Z [container-agent] Traceback (most recent call last):
2024-02-27T14:39:04.741Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 2507, in exec
2024-02-27T14:39:04.741Z [container-agent]     control_ws = self._connect_websocket(task_id, 'control')
2024-02-27T14:39:04.741Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 2589, in _connect_websocket
2024-02-27T14:39:04.741Z [container-agent]     ws.connect(url, socket=sock)
2024-02-27T14:39:04.741Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_core.py", line 253, in connect
2024-02-27T14:39:04.741Z [container-agent]     self.handshake_response = handshake(self.sock, url, *addrs, **options)
2024-02-27T14:39:04.741Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_handshake.py", line 57, in handshake
2024-02-27T14:39:04.741Z [container-agent]     status, resp = _get_resp_headers(sock)
2024-02-27T14:39:04.741Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_handshake.py", line 143, in _get_resp_headers
2024-02-27T14:39:04.741Z [container-agent]     status, resp_headers, status_message = read_headers(sock)
2024-02-27T14:39:04.741Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_http.py", line 312, in read_headers
2024-02-27T14:39:04.741Z [container-agent]     line = recv_line(sock)
2024-02-27T14:39:04.741Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_socket.py", line 131, in recv_line
2024-02-27T14:39:04.741Z [container-agent]     c = recv(sock, 1)
2024-02-27T14:39:04.741Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_socket.py", line 122, in recv
2024-02-27T14:39:04.741Z [container-agent]     raise WebSocketConnectionClosedException(
2024-02-27T14:39:04.741Z [container-agent] websocket._exceptions.WebSocketConnectionClosedException: Connection to remote host was lost.
2024-02-27T14:39:04.741Z [container-agent] 
2024-02-27T14:39:04.741Z [container-agent] During handling of the above exception, another exception occurred:
2024-02-27T14:39:04.741Z [container-agent] 
2024-02-27T14:39:04.741Z [container-agent] Traceback (most recent call last):
2024-02-27T14:39:04.741Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 1348, in do_open
2024-02-27T14:39:04.741Z [container-agent]     h.request(req.get_method(), req.selector, req.data, headers,
2024-02-27T14:39:04.741Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1283, in request
2024-02-27T14:39:04.741Z [container-agent]     self._send_request(method, url, body, headers, encode_chunked)
2024-02-27T14:39:04.741Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1329, in _send_request
2024-02-27T14:39:04.741Z [container-agent]     self.endheaders(body, encode_chunked=encode_chunked)
2024-02-27T14:39:04.741Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1278, in endheaders
2024-02-27T14:39:04.741Z [container-agent]     self._send_output(message_body, encode_chunked=encode_chunked)
2024-02-27T14:39:04.741Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1038, in _send_output
2024-02-27T14:39:04.741Z [container-agent]     self.send(msg)
2024-02-27T14:39:04.741Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 976, in send
2024-02-27T14:39:04.741Z [container-agent]     self.connect()
2024-02-27T14:39:04.741Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 297, in connect
2024-02-27T14:39:04.741Z [container-agent]     self.sock.connect(self.socket_path)
2024-02-27T14:39:04.741Z [container-agent] ConnectionRefusedError: [Errno 111] Connection refused
2024-02-27T14:39:04.741Z [container-agent] 
2024-02-27T14:39:04.741Z [container-agent] During handling of the above exception, another exception occurred:
2024-02-27T14:39:04.741Z [container-agent] 
2024-02-27T14:39:04.741Z [container-agent] Traceback (most recent call last):
2024-02-27T14:39:04.741Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1690, in _request_raw
2024-02-27T14:39:04.741Z [container-agent]     response = self.opener.open(request, timeout=self.timeout)
2024-02-27T14:39:04.741Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 519, in open
2024-02-27T14:39:04.741Z [container-agent]     response = self._open(req, data)
2024-02-27T14:39:04.741Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 536, in _open
2024-02-27T14:39:04.741Z [container-agent]     result = self._call_chain(self.handle_open, protocol, protocol +
2024-02-27T14:39:04.741Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 496, in _call_chain
2024-02-27T14:39:04.741Z [container-agent]     result = func(*args)
2024-02-27T14:39:04.741Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 311, in http_open
2024-02-27T14:39:04.741Z [container-agent]     return self.do_open(_UnixSocketConnection, req,  # type:ignore
2024-02-27T14:39:04.741Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 1351, in do_open
2024-02-27T14:39:04.741Z [container-agent]     raise URLError(err)
2024-02-27T14:39:04.742Z [container-agent] urllib.error.URLError: <urlopen error [Errno 111] Connection refused>
2024-02-27T14:39:04.742Z [container-agent] 
2024-02-27T14:39:04.742Z [container-agent] The above exception was the direct cause of the following exception:
2024-02-27T14:39:04.742Z [container-agent] 
2024-02-27T14:39:04.742Z [container-agent] Traceback (most recent call last):
2024-02-27T14:39:04.742Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/./src/charm.py", line 786, in <module>
2024-02-27T14:39:04.742Z [container-agent]     main(MySQLOperatorCharm)
2024-02-27T14:39:04.742Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 436, in main
2024-02-27T14:39:04.742Z [container-agent]     _emit_charm_event(charm, dispatcher.event_name)
2024-02-27T14:39:04.742Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 144, in _emit_charm_event
2024-02-27T14:39:04.742Z [container-agent]     event_to_emit.emit(*args, **kwargs)
2024-02-27T14:39:04.742Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 351, in emit
2024-02-27T14:39:04.742Z [container-agent]     framework._emit(event)
2024-02-27T14:39:04.742Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 853, in _emit
2024-02-27T14:39:04.742Z [container-agent]     self._reemit(event_path)
2024-02-27T14:39:04.742Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 942, in _reemit
2024-02-27T14:39:04.742Z [container-agent]     custom_handler(event)
2024-02-27T14:39:04.742Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/rotate_mysql_logs.py", line 55, in _rotate_mysql_logs
2024-02-27T14:39:04.742Z [container-agent]     self.charm._mysql.flush_mysql_logs(MySQLTextLogs.ERROR)
2024-02-27T14:39:04.742Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-02-27T14:39:04.742Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-02-27T14:39:04.742Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 665, in _run_mysqlsh_script
2024-02-27T14:39:04.742Z [container-agent]     process = self.container.exec(cmd, timeout=timeout)
2024-02-27T14:39:04.742Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/model.py", line 2688, in exec
2024-02-27T14:39:04.742Z [container-agent]     return self._pebble.exec(
2024-02-27T14:39:04.742Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 2514, in exec
2024-02-27T14:39:04.742Z [container-agent]     change = self.wait_change(ChangeID(change_id))
2024-02-27T14:39:04.742Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1893, in wait_change
2024-02-27T14:39:04.742Z [container-agent]     return self._wait_change_using_wait(change_id, timeout)
2024-02-27T14:39:04.742Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1914, in _wait_change_using_wait
2024-02-27T14:39:04.742Z [container-agent]     return self._wait_change(change_id, this_timeout)
2024-02-27T14:39:04.742Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1928, in _wait_change
2024-02-27T14:39:04.742Z [container-agent]     resp = self._request('GET', f'/v1/changes/{change_id}/wait', query)
2024-02-27T14:39:04.742Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1655, in _request
2024-02-27T14:39:04.742Z [container-agent]     response = self._request_raw(method, path, query, headers, data)
2024-02-27T14:39:04.742Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1707, in _request_raw
2024-02-27T14:39:04.742Z [container-agent]     raise ConnectionError(e.reason) from e
2024-02-27T14:39:04.742Z [container-agent] ops.pebble.ConnectionError: [Errno 111] Connection refused
2024-02-27T14:39:04.868Z [container-agent] 2024-02-27 14:39:04 INFO juju.util.exec exec.go:209 run result: exit status 1
2024-02-27T14:39:06.770Z [container-agent] 2024-02-27 14:39:06 INFO juju-log Setting up the logrotate configurations
2024-02-27T14:39:06.847Z [container-agent] 2024-02-27 14:39:06 INFO juju-log Adding pebble layer
2024-02-27T14:39:13.587Z [container-agent] 2024-02-27 14:39:13 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-02-27T14:39:13.609Z [container-agent] 2024-02-27 14:39:13 INFO juju-log Attempting reboot from complete outage.
2024-02-27T14:39:16.445Z [container-agent] 2024-02-27 14:39:16 INFO juju.worker.uniter.operation runhook.go:186 ran "mysql-pebble-ready" hook (via hook dispatching script: dispatch)
2024-02-27T14:40:01.512Z [container-agent] 2024-02-27 14:40:01 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-27T14:40:03.556Z [container-agent] 2024-02-27 14:40:03 WARNING juju-log No relation: certificates
2024-02-27T14:40:03.948Z [container-agent] 2024-02-27 14:40:03 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-27T14:45:05.030Z [container-agent] 2024-02-27 14:45:05 INFO juju-log Starting the log rotate manager
2024-02-27T14:45:05.046Z [container-agent] 2024-02-27 14:45:05 INFO juju-log Started log rotate manager process with PID 21905
2024-02-27T14:45:05.365Z [container-agent] 2024-02-27 14:45:05 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-27T14:45:07.806Z [container-agent] 2024-02-27 14:45:07 WARNING juju-log No relation: certificates
2024-02-27T14:45:08.185Z [container-agent] 2024-02-27 14:45:08 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-27T14:49:48.761Z [container-agent] 2024-02-27 14:49:48 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-27T14:49:51.235Z [container-agent] 2024-02-27 14:49:51 WARNING juju-log No relation: certificates
2024-02-27T14:49:51.593Z [container-agent] 2024-02-27 14:49:51 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-27T14:54:06.636Z [container-agent] 2024-02-27 14:54:06 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-27T14:54:09.316Z [container-agent] 2024-02-27 14:54:09 WARNING juju-log No relation: certificates
2024-02-27T14:54:09.667Z [container-agent] 2024-02-27 14:54:09 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-27T14:59:00.292Z [container-agent] 2024-02-27 14:59:00 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-27T14:59:03.830Z [container-agent] 2024-02-27 14:59:03 WARNING juju-log No relation: certificates
2024-02-27T14:59:04.219Z [container-agent] 2024-02-27 14:59:04 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-27T15:03:37.791Z [container-agent] 2024-02-27 15:03:37 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-27T15:03:41.531Z [container-agent] 2024-02-27 15:03:41 WARNING juju-log No relation: certificates
2024-02-27T15:03:41.874Z [container-agent] 2024-02-27 15:03:41 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-27T15:09:29.901Z [container-agent] 2024-02-27 15:09:29 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-27T15:09:34.304Z [container-agent] 2024-02-27 15:09:34 WARNING juju-log No relation: certificates
2024-02-27T15:09:34.648Z [container-agent] 2024-02-27 15:09:34 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-27T15:14:20.626Z [container-agent] 2024-02-27 15:14:20 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-27T15:14:25.383Z [container-agent] 2024-02-27 15:14:25 WARNING juju-log No relation: certificates
2024-02-27T15:14:25.746Z [container-agent] 2024-02-27 15:14:25 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-27T15:20:13.363Z [container-agent] 2024-02-27 15:20:13 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-27T15:20:18.661Z [container-agent] 2024-02-27 15:20:18 WARNING juju-log No relation: certificates
2024-02-27T15:20:19.004Z [container-agent] 2024-02-27 15:20:19 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-27T15:24:54.550Z [container-agent] 2024-02-27 15:24:54 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-27T15:24:59.743Z [container-agent] 2024-02-27 15:24:59 WARNING juju-log No relation: certificates
2024-02-27T15:25:00.088Z [container-agent] 2024-02-27 15:25:00 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-27T15:30:53.398Z [container-agent] 2024-02-27 15:30:53 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-27T15:30:59.720Z [container-agent] 2024-02-27 15:30:59 WARNING juju-log No relation: certificates
2024-02-27T15:31:00.075Z [container-agent] 2024-02-27 15:31:00 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-27T15:36:44.160Z [container-agent] 2024-02-27 15:36:44 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-27T15:36:51.108Z [container-agent] 2024-02-27 15:36:51 WARNING juju-log No relation: certificates
2024-02-27T15:36:51.447Z [container-agent] 2024-02-27 15:36:51 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-27T15:42:18.724Z [container-agent] 2024-02-27 15:42:18 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-27T15:42:26.078Z [container-agent] 2024-02-27 15:42:26 WARNING juju-log No relation: certificates
2024-02-27T15:42:26.415Z [container-agent] 2024-02-27 15:42:26 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-27T15:47:54.283Z [container-agent] 2024-02-27 15:47:54 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-27T15:48:01.947Z [container-agent] 2024-02-27 15:48:01 WARNING juju-log No relation: certificates
2024-02-27T15:48:02.401Z [container-agent] 2024-02-27 15:48:02 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-27T15:52:43.178Z [container-agent] 2024-02-27 15:52:43 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-27T15:52:51.284Z [container-agent] 2024-02-27 15:52:51 WARNING juju-log No relation: certificates
2024-02-27T15:52:51.622Z [container-agent] 2024-02-27 15:52:51 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-27T15:58:13.940Z [container-agent] 2024-02-27 15:58:13 ERROR juju-log Failed to flush SLOW LOGS logs.
2024-02-27T15:58:13.940Z [container-agent] Traceback (most recent call last):
2024-02-27T15:58:13.940Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 666, in _run_mysqlsh_script
2024-02-27T15:58:13.940Z [container-agent]     stdout, _ = process.wait_output()
2024-02-27T15:58:13.940Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1441, in wait_output
2024-02-27T15:58:13.940Z [container-agent]     raise ExecError[AnyStr](self._command, exit_code, out_value, err_value)
2024-02-27T15:58:13.940Z [container-agent] ops.pebble.ExecError: non-zero exit code 1 executing ['/usr/bin/mysqlsh', '--no-wizard', '--python', '--verbose=1', '-f', '/tmp/script.py', ';', 'rm', '/tmp/script.py'], stdout='', stderr='Cannot set LC_ALL to locale en_US.UTF-8: No such file or directory\nverbose: 2024-02-27T15:58:12Z: Loading startup files...\nverbose: 2024-02-27T15:58:12Z: Loading plugins...\nverbose: 2024-02-27T15:58:12Z: Connecting to MySQL at: serverconfig@horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local\nTraceback (most recent call last):\n  File "<string>", line 1, in <module>\nmysqlsh.DBError: MySQL Error (2003): Shell.connect: Can\'t connect to MySQL server on \'horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local:3306\' (111)\n'
2024-02-27T15:58:13.941Z [container-agent] 
2024-02-27T15:58:13.941Z [container-agent] During handling of the above exception, another exception occurred:
2024-02-27T15:58:13.941Z [container-agent] 
2024-02-27T15:58:13.941Z [container-agent] Traceback (most recent call last):
2024-02-27T15:58:13.941Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-02-27T15:58:13.941Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-02-27T15:58:13.941Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 669, in _run_mysqlsh_script
2024-02-27T15:58:13.941Z [container-agent]     raise MySQLClientError(e.stderr)
2024-02-27T15:58:13.941Z [container-agent] charms.mysql.v0.mysql.MySQLClientError: Cannot set LC_ALL to locale en_US.UTF-8: No such file or directory
2024-02-27T15:58:13.941Z [container-agent] verbose: 2024-02-27T15:58:12Z: Loading startup files...
2024-02-27T15:58:13.941Z [container-agent] verbose: 2024-02-27T15:58:12Z: Loading plugins...
2024-02-27T15:58:13.941Z [container-agent] verbose: 2024-02-27T15:58:12Z: Connecting to MySQL at: serverconfig@horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local
2024-02-27T15:58:13.941Z [container-agent] Traceback (most recent call last):
2024-02-27T15:58:13.941Z [container-agent]   File "<string>", line 1, in <module>
2024-02-27T15:58:13.941Z [container-agent] mysqlsh.DBError: MySQL Error (2003): Shell.connect: Can't connect to MySQL server on 'horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local:3306' (111)
2024-02-27T15:58:13.941Z [container-agent] 
2024-02-27T15:58:18.000Z [container-agent] 2024-02-27 15:58:18 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-02-27T15:58:18.028Z [container-agent] 2024-02-27 15:58:18 INFO juju-log Attempting reboot from complete outage.
2024-02-27T15:58:25.991Z [container-agent] 2024-02-27 15:58:25 WARNING juju-log No relation: certificates
2024-02-27T15:58:26.353Z [container-agent] 2024-02-27 15:58:26 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-27T16:03:09.755Z [container-agent] 2024-02-27 16:03:09 ERROR juju-log Uncaught exception while in charm code:
2024-02-27T16:03:09.755Z [container-agent] Traceback (most recent call last):
2024-02-27T16:03:09.755Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 2507, in exec
2024-02-27T16:03:09.755Z [container-agent]     control_ws = self._connect_websocket(task_id, 'control')
2024-02-27T16:03:09.755Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 2589, in _connect_websocket
2024-02-27T16:03:09.755Z [container-agent]     ws.connect(url, socket=sock)
2024-02-27T16:03:09.755Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_core.py", line 253, in connect
2024-02-27T16:03:09.755Z [container-agent]     self.handshake_response = handshake(self.sock, url, *addrs, **options)
2024-02-27T16:03:09.755Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_handshake.py", line 57, in handshake
2024-02-27T16:03:09.755Z [container-agent]     status, resp = _get_resp_headers(sock)
2024-02-27T16:03:09.755Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_handshake.py", line 143, in _get_resp_headers
2024-02-27T16:03:09.755Z [container-agent]     status, resp_headers, status_message = read_headers(sock)
2024-02-27T16:03:09.755Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_http.py", line 312, in read_headers
2024-02-27T16:03:09.755Z [container-agent]     line = recv_line(sock)
2024-02-27T16:03:09.755Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_socket.py", line 131, in recv_line
2024-02-27T16:03:09.755Z [container-agent]     c = recv(sock, 1)
2024-02-27T16:03:09.755Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_socket.py", line 122, in recv
2024-02-27T16:03:09.755Z [container-agent]     raise WebSocketConnectionClosedException(
2024-02-27T16:03:09.755Z [container-agent] websocket._exceptions.WebSocketConnectionClosedException: Connection to remote host was lost.
2024-02-27T16:03:09.755Z [container-agent] 
2024-02-27T16:03:09.755Z [container-agent] During handling of the above exception, another exception occurred:
2024-02-27T16:03:09.755Z [container-agent] 
2024-02-27T16:03:09.755Z [container-agent] Traceback (most recent call last):
2024-02-27T16:03:09.755Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 1348, in do_open
2024-02-27T16:03:09.755Z [container-agent]     h.request(req.get_method(), req.selector, req.data, headers,
2024-02-27T16:03:09.755Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1283, in request
2024-02-27T16:03:09.755Z [container-agent]     self._send_request(method, url, body, headers, encode_chunked)
2024-02-27T16:03:09.755Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1329, in _send_request
2024-02-27T16:03:09.755Z [container-agent]     self.endheaders(body, encode_chunked=encode_chunked)
2024-02-27T16:03:09.755Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1278, in endheaders
2024-02-27T16:03:09.755Z [container-agent]     self._send_output(message_body, encode_chunked=encode_chunked)
2024-02-27T16:03:09.755Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1038, in _send_output
2024-02-27T16:03:09.755Z [container-agent]     self.send(msg)
2024-02-27T16:03:09.755Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 976, in send
2024-02-27T16:03:09.755Z [container-agent]     self.connect()
2024-02-27T16:03:09.755Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 297, in connect
2024-02-27T16:03:09.755Z [container-agent]     self.sock.connect(self.socket_path)
2024-02-27T16:03:09.755Z [container-agent] ConnectionRefusedError: [Errno 111] Connection refused
2024-02-27T16:03:09.755Z [container-agent] 
2024-02-27T16:03:09.755Z [container-agent] During handling of the above exception, another exception occurred:
2024-02-27T16:03:09.755Z [container-agent] 
2024-02-27T16:03:09.755Z [container-agent] Traceback (most recent call last):
2024-02-27T16:03:09.755Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1690, in _request_raw
2024-02-27T16:03:09.755Z [container-agent]     response = self.opener.open(request, timeout=self.timeout)
2024-02-27T16:03:09.755Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 519, in open
2024-02-27T16:03:09.755Z [container-agent]     response = self._open(req, data)
2024-02-27T16:03:09.755Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 536, in _open
2024-02-27T16:03:09.755Z [container-agent]     result = self._call_chain(self.handle_open, protocol, protocol +
2024-02-27T16:03:09.755Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 496, in _call_chain
2024-02-27T16:03:09.755Z [container-agent]     result = func(*args)
2024-02-27T16:03:09.755Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 311, in http_open
2024-02-27T16:03:09.755Z [container-agent]     return self.do_open(_UnixSocketConnection, req,  # type:ignore
2024-02-27T16:03:09.755Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 1351, in do_open
2024-02-27T16:03:09.755Z [container-agent]     raise URLError(err)
2024-02-27T16:03:09.755Z [container-agent] urllib.error.URLError: <urlopen error [Errno 111] Connection refused>
2024-02-27T16:03:09.755Z [container-agent] 
2024-02-27T16:03:09.755Z [container-agent] The above exception was the direct cause of the following exception:
2024-02-27T16:03:09.755Z [container-agent] 
2024-02-27T16:03:09.755Z [container-agent] Traceback (most recent call last):
2024-02-27T16:03:09.755Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/./src/charm.py", line 786, in <module>
2024-02-27T16:03:09.755Z [container-agent]     main(MySQLOperatorCharm)
2024-02-27T16:03:09.755Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 436, in main
2024-02-27T16:03:09.755Z [container-agent]     _emit_charm_event(charm, dispatcher.event_name)
2024-02-27T16:03:09.755Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 144, in _emit_charm_event
2024-02-27T16:03:09.755Z [container-agent]     event_to_emit.emit(*args, **kwargs)
2024-02-27T16:03:09.755Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 351, in emit
2024-02-27T16:03:09.755Z [container-agent]     framework._emit(event)
2024-02-27T16:03:09.755Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 853, in _emit
2024-02-27T16:03:09.755Z [container-agent]     self._reemit(event_path)
2024-02-27T16:03:09.755Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 942, in _reemit
2024-02-27T16:03:09.755Z [container-agent]     custom_handler(event)
2024-02-27T16:03:09.755Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/rotate_mysql_logs.py", line 57, in _rotate_mysql_logs
2024-02-27T16:03:09.755Z [container-agent]     self.charm._mysql.flush_mysql_logs(MySQLTextLogs.SLOW)
2024-02-27T16:03:09.755Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-02-27T16:03:09.755Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-02-27T16:03:09.755Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 665, in _run_mysqlsh_script
2024-02-27T16:03:09.755Z [container-agent]     process = self.container.exec(cmd, timeout=timeout)
2024-02-27T16:03:09.755Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/model.py", line 2688, in exec
2024-02-27T16:03:09.755Z [container-agent]     return self._pebble.exec(
2024-02-27T16:03:09.755Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 2514, in exec
2024-02-27T16:03:09.755Z [container-agent]     change = self.wait_change(ChangeID(change_id))
2024-02-27T16:03:09.755Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1893, in wait_change
2024-02-27T16:03:09.755Z [container-agent]     return self._wait_change_using_wait(change_id, timeout)
2024-02-27T16:03:09.755Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1914, in _wait_change_using_wait
2024-02-27T16:03:09.755Z [container-agent]     return self._wait_change(change_id, this_timeout)
2024-02-27T16:03:09.755Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1928, in _wait_change
2024-02-27T16:03:09.755Z [container-agent]     resp = self._request('GET', f'/v1/changes/{change_id}/wait', query)
2024-02-27T16:03:09.755Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1655, in _request
2024-02-27T16:03:09.755Z [container-agent]     response = self._request_raw(method, path, query, headers, data)
2024-02-27T16:03:09.755Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1707, in _request_raw
2024-02-27T16:03:09.755Z [container-agent]     raise ConnectionError(e.reason) from e
2024-02-27T16:03:09.755Z [container-agent] ops.pebble.ConnectionError: [Errno 111] Connection refused
2024-02-27T16:03:09.874Z [container-agent] 2024-02-27 16:03:09 INFO juju.util.exec exec.go:209 run result: exit status 1
2024-02-27T16:03:14.696Z [container-agent] 2024-02-27 16:03:14 INFO juju-log Setting up the logrotate configurations
2024-02-27T16:03:14.757Z [container-agent] 2024-02-27 16:03:14 INFO juju-log Adding pebble layer
2024-02-27T16:03:21.460Z [container-agent] 2024-02-27 16:03:21 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-02-27T16:03:21.482Z [container-agent] 2024-02-27 16:03:21 INFO juju-log Attempting reboot from complete outage.
2024-02-27T16:03:24.291Z [container-agent] 2024-02-27 16:03:24 INFO juju.worker.uniter.operation runhook.go:186 ran "mysql-pebble-ready" hook (via hook dispatching script: dispatch)
2024-02-27T16:03:32.478Z [container-agent] 2024-02-27 16:03:32 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-27T16:03:34.515Z [container-agent] 2024-02-27 16:03:34 WARNING juju-log No relation: certificates
2024-02-27T16:03:34.887Z [container-agent] 2024-02-27 16:03:34 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-27T16:08:20.858Z [container-agent] 2024-02-27 16:08:20 INFO juju-log Starting the log rotate manager
2024-02-27T16:08:20.872Z [container-agent] 2024-02-27 16:08:20 INFO juju-log Started log rotate manager process with PID 29762
2024-02-27T16:08:21.143Z [container-agent] 2024-02-27 16:08:21 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-27T16:08:23.059Z [container-agent] 2024-02-27 16:08:23 WARNING juju-log No relation: certificates
2024-02-27T16:08:23.414Z [container-agent] 2024-02-27 16:08:23 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-27T16:12:43.653Z [container-agent] 2024-02-27 16:12:43 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-27T16:12:45.957Z [container-agent] 2024-02-27 16:12:45 WARNING juju-log No relation: certificates
2024-02-27T16:12:46.354Z [container-agent] 2024-02-27 16:12:46 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-27T16:18:36.657Z [container-agent] 2024-02-27 16:18:36 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-27T16:18:39.566Z [container-agent] 2024-02-27 16:18:39 WARNING juju-log No relation: certificates
2024-02-27T16:18:39.943Z [container-agent] 2024-02-27 16:18:39 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-27T16:23:24.130Z [container-agent] 2024-02-27 16:23:24 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-27T16:23:27.343Z [container-agent] 2024-02-27 16:23:27 WARNING juju-log No relation: certificates
2024-02-27T16:23:27.698Z [container-agent] 2024-02-27 16:23:27 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-27T16:27:44.661Z [container-agent] 2024-02-27 16:27:44 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-27T16:27:48.632Z [container-agent] 2024-02-27 16:27:48 WARNING juju-log No relation: certificates
2024-02-27T16:27:48.993Z [container-agent] 2024-02-27 16:27:48 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-27T16:33:38.160Z [container-agent] 2024-02-27 16:33:38 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-27T16:33:42.347Z [container-agent] 2024-02-27 16:33:42 WARNING juju-log No relation: certificates
2024-02-27T16:33:42.681Z [container-agent] 2024-02-27 16:33:42 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-27T16:39:09.523Z [container-agent] 2024-02-27 16:39:09 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-27T16:39:14.425Z [container-agent] 2024-02-27 16:39:14 WARNING juju-log No relation: certificates
2024-02-27T16:39:14.776Z [container-agent] 2024-02-27 16:39:14 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-27T16:44:11.143Z [container-agent] 2024-02-27 16:44:11 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-27T16:44:16.689Z [container-agent] 2024-02-27 16:44:16 WARNING juju-log No relation: certificates
2024-02-27T16:44:17.043Z [container-agent] 2024-02-27 16:44:17 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-27T16:48:50.012Z [container-agent] 2024-02-27 16:48:50 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-27T16:48:55.761Z [container-agent] 2024-02-27 16:48:55 WARNING juju-log No relation: certificates
2024-02-27T16:48:56.097Z [container-agent] 2024-02-27 16:48:56 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-27T16:53:49.363Z [container-agent] 2024-02-27 16:53:49 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-27T16:53:55.272Z [container-agent] 2024-02-27 16:53:55 WARNING juju-log No relation: certificates
2024-02-27T16:53:55.608Z [container-agent] 2024-02-27 16:53:55 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-27T16:59:01.262Z [container-agent] 2024-02-27 16:59:01 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-27T16:59:07.682Z [container-agent] 2024-02-27 16:59:07 WARNING juju-log No relation: certificates
2024-02-27T16:59:08.027Z [container-agent] 2024-02-27 16:59:08 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-27T17:02:12.841Z [container-agent] 2024-02-27 17:02:12 ERROR juju-log Failed to flush SLOW LOGS logs.
2024-02-27T17:02:12.841Z [container-agent] Traceback (most recent call last):
2024-02-27T17:02:12.841Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 666, in _run_mysqlsh_script
2024-02-27T17:02:12.841Z [container-agent]     stdout, _ = process.wait_output()
2024-02-27T17:02:12.841Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1441, in wait_output
2024-02-27T17:02:12.841Z [container-agent]     raise ExecError[AnyStr](self._command, exit_code, out_value, err_value)
2024-02-27T17:02:12.841Z [container-agent] ops.pebble.ExecError: non-zero exit code 1 executing ['/usr/bin/mysqlsh', '--no-wizard', '--python', '--verbose=1', '-f', '/tmp/script.py', ';', 'rm', '/tmp/script.py'], stdout='', stderr='Cannot set LC_ALL to locale en_US.UTF-8: No such file or directory\nverbose: 2024-02-27T17:02:11Z: Loading startup files...\nverbose: 2024-02-27T17:02:11Z: Loading plugins...\nverbose: 2024-02-27T17:02:11Z: Connecting to MySQL at: serverconfig@horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local\nTraceback (most recent call last):\n  File "<string>", line 1, in <module>\nmysqlsh.DBError: MySQL Error (2003): Shell.connect: Can\'t connect to MySQL server on \'horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local:3306\' (111)\n'
2024-02-27T17:02:12.841Z [container-agent] 
2024-02-27T17:02:12.841Z [container-agent] During handling of the above exception, another exception occurred:
2024-02-27T17:02:12.841Z [container-agent] 
2024-02-27T17:02:12.841Z [container-agent] Traceback (most recent call last):
2024-02-27T17:02:12.841Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-02-27T17:02:12.841Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-02-27T17:02:12.841Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 669, in _run_mysqlsh_script
2024-02-27T17:02:12.841Z [container-agent]     raise MySQLClientError(e.stderr)
2024-02-27T17:02:12.841Z [container-agent] charms.mysql.v0.mysql.MySQLClientError: Cannot set LC_ALL to locale en_US.UTF-8: No such file or directory
2024-02-27T17:02:12.841Z [container-agent] verbose: 2024-02-27T17:02:11Z: Loading startup files...
2024-02-27T17:02:12.841Z [container-agent] verbose: 2024-02-27T17:02:11Z: Loading plugins...
2024-02-27T17:02:12.841Z [container-agent] verbose: 2024-02-27T17:02:11Z: Connecting to MySQL at: serverconfig@horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local
2024-02-27T17:02:12.841Z [container-agent] Traceback (most recent call last):
2024-02-27T17:02:12.841Z [container-agent]   File "<string>", line 1, in <module>
2024-02-27T17:02:12.841Z [container-agent] mysqlsh.DBError: MySQL Error (2003): Shell.connect: Can't connect to MySQL server on 'horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local:3306' (111)
2024-02-27T17:02:12.841Z [container-agent] 
2024-02-27T17:04:42.711Z [container-agent] 2024-02-27 17:04:42 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-02-27T17:04:42.761Z [container-agent] 2024-02-27 17:04:42 INFO juju-log Attempting reboot from complete outage.
2024-02-27T17:04:49.693Z [container-agent] 2024-02-27 17:04:49 WARNING juju-log No relation: certificates
2024-02-27T17:04:50.081Z [container-agent] 2024-02-27 17:04:50 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-27T17:07:09.840Z [container-agent] 2024-02-27 17:07:09 ERROR juju-log Uncaught exception while in charm code:
2024-02-27T17:07:09.840Z [container-agent] Traceback (most recent call last):
2024-02-27T17:07:09.840Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/./src/charm.py", line 786, in <module>
2024-02-27T17:07:09.840Z [container-agent]     main(MySQLOperatorCharm)
2024-02-27T17:07:09.840Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 436, in main
2024-02-27T17:07:09.840Z [container-agent]     _emit_charm_event(charm, dispatcher.event_name)
2024-02-27T17:07:09.840Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 144, in _emit_charm_event
2024-02-27T17:07:09.840Z [container-agent]     event_to_emit.emit(*args, **kwargs)
2024-02-27T17:07:09.840Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 351, in emit
2024-02-27T17:07:09.840Z [container-agent]     framework._emit(event)
2024-02-27T17:07:09.840Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 853, in _emit
2024-02-27T17:07:09.840Z [container-agent]     self._reemit(event_path)
2024-02-27T17:07:09.840Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 942, in _reemit
2024-02-27T17:07:09.840Z [container-agent]     custom_handler(event)
2024-02-27T17:07:09.840Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/rotate_mysql_logs.py", line 57, in _rotate_mysql_logs
2024-02-27T17:07:09.840Z [container-agent]     self.charm._mysql.flush_mysql_logs(MySQLTextLogs.SLOW)
2024-02-27T17:07:09.840Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-02-27T17:07:09.840Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-02-27T17:07:09.840Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 666, in _run_mysqlsh_script
2024-02-27T17:07:09.840Z [container-agent]     stdout, _ = process.wait_output()
2024-02-27T17:07:09.840Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1436, in wait_output
2024-02-27T17:07:09.840Z [container-agent]     exit_code: int = self._wait()
2024-02-27T17:07:09.840Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1375, in _wait
2024-02-27T17:07:09.840Z [container-agent]     change = self._client.wait_change(self._change_id, timeout=timeout)
2024-02-27T17:07:09.840Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1893, in wait_change
2024-02-27T17:07:09.840Z [container-agent]     return self._wait_change_using_wait(change_id, timeout)
2024-02-27T17:07:09.840Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1914, in _wait_change_using_wait
2024-02-27T17:07:09.840Z [container-agent]     return self._wait_change(change_id, this_timeout)
2024-02-27T17:07:09.840Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1928, in _wait_change
2024-02-27T17:07:09.840Z [container-agent]     resp = self._request('GET', f'/v1/changes/{change_id}/wait', query)
2024-02-27T17:07:09.840Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1655, in _request
2024-02-27T17:07:09.840Z [container-agent]     response = self._request_raw(method, path, query, headers, data)
2024-02-27T17:07:09.840Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1690, in _request_raw
2024-02-27T17:07:09.840Z [container-agent]     response = self.opener.open(request, timeout=self.timeout)
2024-02-27T17:07:09.840Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 519, in open
2024-02-27T17:07:09.840Z [container-agent]     response = self._open(req, data)
2024-02-27T17:07:09.840Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 536, in _open
2024-02-27T17:07:09.840Z [container-agent]     result = self._call_chain(self.handle_open, protocol, protocol +
2024-02-27T17:07:09.840Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 496, in _call_chain
2024-02-27T17:07:09.840Z [container-agent]     result = func(*args)
2024-02-27T17:07:09.840Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 311, in http_open
2024-02-27T17:07:09.840Z [container-agent]     return self.do_open(_UnixSocketConnection, req,  # type:ignore
2024-02-27T17:07:09.840Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 1352, in do_open
2024-02-27T17:07:09.840Z [container-agent]     r = h.getresponse()
2024-02-27T17:07:09.840Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1375, in getresponse
2024-02-27T17:07:09.840Z [container-agent]     response.begin()
2024-02-27T17:07:09.840Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 318, in begin
2024-02-27T17:07:09.840Z [container-agent]     version, status, reason = self._read_status()
2024-02-27T17:07:09.840Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 287, in _read_status
2024-02-27T17:07:09.840Z [container-agent]     raise RemoteDisconnected("Remote end closed connection without"
2024-02-27T17:07:09.840Z [container-agent] http.client.RemoteDisconnected: Remote end closed connection without response
2024-02-27T17:07:10.024Z [container-agent] 2024-02-27 17:07:10 INFO juju.util.exec exec.go:209 run result: exit status 1
2024-02-27T17:07:12.369Z [container-agent] 2024-02-27 17:07:12 INFO juju-log Setting up the logrotate configurations
2024-02-27T17:07:12.431Z [container-agent] 2024-02-27 17:07:12 INFO juju-log Adding pebble layer
2024-02-27T17:07:19.203Z [container-agent] 2024-02-27 17:07:19 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-02-27T17:07:19.224Z [container-agent] 2024-02-27 17:07:19 INFO juju-log Attempting reboot from complete outage.
2024-02-27T17:07:22.036Z [container-agent] 2024-02-27 17:07:22 INFO juju.worker.uniter.operation runhook.go:186 ran "mysql-pebble-ready" hook (via hook dispatching script: dispatch)
2024-02-27T17:09:59.372Z [container-agent] 2024-02-27 17:09:59 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-27T17:10:01.295Z [container-agent] 2024-02-27 17:10:01 WARNING juju-log No relation: certificates
2024-02-27T17:10:01.679Z [container-agent] 2024-02-27 17:10:01 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-27T17:15:20.040Z [container-agent] 2024-02-27 17:15:20 INFO juju-log Starting the log rotate manager
2024-02-27T17:15:20.053Z [container-agent] 2024-02-27 17:15:20 INFO juju-log Started log rotate manager process with PID 35858
2024-02-27T17:15:20.322Z [container-agent] 2024-02-27 17:15:20 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-27T17:15:22.170Z [container-agent] 2024-02-27 17:15:22 WARNING juju-log No relation: certificates
2024-02-27T17:15:22.523Z [container-agent] 2024-02-27 17:15:22 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-27T17:19:44.013Z [container-agent] 2024-02-27 17:19:44 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-27T17:19:46.693Z [container-agent] 2024-02-27 17:19:46 WARNING juju-log No relation: certificates
2024-02-27T17:19:47.040Z [container-agent] 2024-02-27 17:19:47 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-27T17:24:25.180Z [container-agent] 2024-02-27 17:24:25 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-27T17:24:28.138Z [container-agent] 2024-02-27 17:24:28 WARNING juju-log No relation: certificates
2024-02-27T17:24:28.477Z [container-agent] 2024-02-27 17:24:28 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-27T17:28:37.426Z [container-agent] 2024-02-27 17:28:37 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-27T17:28:40.496Z [container-agent] 2024-02-27 17:28:40 WARNING juju-log No relation: certificates
2024-02-27T17:28:40.821Z [container-agent] 2024-02-27 17:28:40 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-27T17:32:50.069Z [container-agent] 2024-02-27 17:32:50 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-27T17:32:53.757Z [container-agent] 2024-02-27 17:32:53 WARNING juju-log No relation: certificates
2024-02-27T17:32:54.103Z [container-agent] 2024-02-27 17:32:54 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-27T17:37:30.443Z [container-agent] 2024-02-27 17:37:30 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-27T17:37:34.774Z [container-agent] 2024-02-27 17:37:34 WARNING juju-log No relation: certificates
2024-02-27T17:37:35.138Z [container-agent] 2024-02-27 17:37:35 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-27T17:42:21.452Z [container-agent] 2024-02-27 17:42:21 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-27T17:42:25.650Z [container-agent] 2024-02-27 17:42:25 WARNING juju-log No relation: certificates
2024-02-27T17:42:26.000Z [container-agent] 2024-02-27 17:42:25 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-27T17:47:47.391Z [container-agent] 2024-02-27 17:47:47 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-27T17:47:52.512Z [container-agent] 2024-02-27 17:47:52 WARNING juju-log No relation: certificates
2024-02-27T17:47:52.902Z [container-agent] 2024-02-27 17:47:52 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-27T17:52:58.250Z [container-agent] 2024-02-27 17:52:58 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-27T17:53:03.474Z [container-agent] 2024-02-27 17:53:03 WARNING juju-log No relation: certificates
2024-02-27T17:53:03.873Z [container-agent] 2024-02-27 17:53:03 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-27T17:57:10.587Z [container-agent] 2024-02-27 17:57:10 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-27T17:57:16.894Z [container-agent] 2024-02-27 17:57:16 WARNING juju-log No relation: certificates
2024-02-27T17:57:17.247Z [container-agent] 2024-02-27 17:57:17 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-27T18:01:59.798Z [container-agent] 2024-02-27 18:01:59 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-27T18:02:05.940Z [container-agent] 2024-02-27 18:02:05 WARNING juju-log No relation: certificates
2024-02-27T18:02:06.302Z [container-agent] 2024-02-27 18:02:06 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-27T18:07:41.781Z [container-agent] 2024-02-27 18:07:41 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-27T18:07:48.640Z [container-agent] 2024-02-27 18:07:48 WARNING juju-log No relation: certificates
2024-02-27T18:07:49.046Z [container-agent] 2024-02-27 18:07:49 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-27T18:12:48.386Z [container-agent] 2024-02-27 18:12:48 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-27T18:12:55.934Z [container-agent] 2024-02-27 18:12:55 WARNING juju-log No relation: certificates
2024-02-27T18:12:56.279Z [container-agent] 2024-02-27 18:12:56 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-27T18:17:19.478Z [container-agent] 2024-02-27 18:17:19 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-27T18:17:27.252Z [container-agent] 2024-02-27 18:17:27 WARNING juju-log No relation: certificates
2024-02-27T18:17:27.593Z [container-agent] 2024-02-27 18:17:27 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-27T18:22:51.232Z [container-agent] 2024-02-27 18:22:51 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-27T18:22:59.554Z [container-agent] 2024-02-27 18:22:59 WARNING juju-log No relation: certificates
2024-02-27T18:22:59.906Z [container-agent] 2024-02-27 18:22:59 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-27T18:26:08.171Z [container-agent] 2024-02-27 18:26:08 ERROR juju-log Failed to flush ERROR LOGS logs.
2024-02-27T18:26:08.171Z [container-agent] Traceback (most recent call last):
2024-02-27T18:26:08.171Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 666, in _run_mysqlsh_script
2024-02-27T18:26:08.171Z [container-agent]     stdout, _ = process.wait_output()
2024-02-27T18:26:08.171Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1441, in wait_output
2024-02-27T18:26:08.171Z [container-agent]     raise ExecError[AnyStr](self._command, exit_code, out_value, err_value)
2024-02-27T18:26:08.171Z [container-agent] ops.pebble.ExecError: non-zero exit code 1 executing ['/usr/bin/mysqlsh', '--no-wizard', '--python', '--verbose=1', '-f', '/tmp/script.py', ';', 'rm', '/tmp/script.py'], stdout='', stderr='Cannot set LC_ALL to locale en_US.UTF-8: No such file or directory\nverbose: 2024-02-27T18:26:06Z: Loading startup files...\nverbose: 2024-02-27T18:26:06Z: Loading plugins...\nverbose: 2024-02-27T18:26:06Z: Connecting to MySQL at: serverconfig@horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local\nTraceback (most recent call last):\n  File "<string>", line 1, in <module>\nmysqlsh.DBError: MySQL Error (2003): Shell.connect: Can\'t connect to MySQL server on \'horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local:3306\' (111)\n'
2024-02-27T18:26:08.171Z [container-agent] 
2024-02-27T18:26:08.171Z [container-agent] During handling of the above exception, another exception occurred:
2024-02-27T18:26:08.171Z [container-agent] 
2024-02-27T18:26:08.171Z [container-agent] Traceback (most recent call last):
2024-02-27T18:26:08.171Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-02-27T18:26:08.171Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-02-27T18:26:08.171Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 669, in _run_mysqlsh_script
2024-02-27T18:26:08.171Z [container-agent]     raise MySQLClientError(e.stderr)
2024-02-27T18:26:08.171Z [container-agent] charms.mysql.v0.mysql.MySQLClientError: Cannot set LC_ALL to locale en_US.UTF-8: No such file or directory
2024-02-27T18:26:08.171Z [container-agent] verbose: 2024-02-27T18:26:06Z: Loading startup files...
2024-02-27T18:26:08.171Z [container-agent] verbose: 2024-02-27T18:26:06Z: Loading plugins...
2024-02-27T18:26:08.171Z [container-agent] verbose: 2024-02-27T18:26:06Z: Connecting to MySQL at: serverconfig@horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local
2024-02-27T18:26:08.171Z [container-agent] Traceback (most recent call last):
2024-02-27T18:26:08.171Z [container-agent]   File "<string>", line 1, in <module>
2024-02-27T18:26:08.171Z [container-agent] mysqlsh.DBError: MySQL Error (2003): Shell.connect: Can't connect to MySQL server on 'horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local:3306' (111)
2024-02-27T18:26:08.171Z [container-agent] 
2024-02-27T18:27:56.022Z [container-agent] 2024-02-27 18:27:56 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-02-27T18:27:56.048Z [container-agent] 2024-02-27 18:27:56 INFO juju-log Attempting reboot from complete outage.
2024-02-27T18:28:03.635Z [container-agent] 2024-02-27 18:28:03 WARNING juju-log No relation: certificates
2024-02-27T18:28:04.009Z [container-agent] 2024-02-27 18:28:04 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-27T18:31:04.790Z [container-agent] 2024-02-27 18:31:04 ERROR juju-log Uncaught exception while in charm code:
2024-02-27T18:31:04.790Z [container-agent] Traceback (most recent call last):
2024-02-27T18:31:04.790Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/./src/charm.py", line 786, in <module>
2024-02-27T18:31:04.790Z [container-agent]     main(MySQLOperatorCharm)
2024-02-27T18:31:04.790Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 436, in main
2024-02-27T18:31:04.790Z [container-agent]     _emit_charm_event(charm, dispatcher.event_name)
2024-02-27T18:31:04.790Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 144, in _emit_charm_event
2024-02-27T18:31:04.790Z [container-agent]     event_to_emit.emit(*args, **kwargs)
2024-02-27T18:31:04.790Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 351, in emit
2024-02-27T18:31:04.790Z [container-agent]     framework._emit(event)
2024-02-27T18:31:04.790Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 853, in _emit
2024-02-27T18:31:04.790Z [container-agent]     self._reemit(event_path)
2024-02-27T18:31:04.790Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 942, in _reemit
2024-02-27T18:31:04.790Z [container-agent]     custom_handler(event)
2024-02-27T18:31:04.790Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/rotate_mysql_logs.py", line 55, in _rotate_mysql_logs
2024-02-27T18:31:04.790Z [container-agent]     self.charm._mysql.flush_mysql_logs(MySQLTextLogs.ERROR)
2024-02-27T18:31:04.790Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-02-27T18:31:04.790Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-02-27T18:31:04.790Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 666, in _run_mysqlsh_script
2024-02-27T18:31:04.790Z [container-agent]     stdout, _ = process.wait_output()
2024-02-27T18:31:04.790Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1436, in wait_output
2024-02-27T18:31:04.790Z [container-agent]     exit_code: int = self._wait()
2024-02-27T18:31:04.790Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1375, in _wait
2024-02-27T18:31:04.790Z [container-agent]     change = self._client.wait_change(self._change_id, timeout=timeout)
2024-02-27T18:31:04.790Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1893, in wait_change
2024-02-27T18:31:04.790Z [container-agent]     return self._wait_change_using_wait(change_id, timeout)
2024-02-27T18:31:04.790Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1914, in _wait_change_using_wait
2024-02-27T18:31:04.790Z [container-agent]     return self._wait_change(change_id, this_timeout)
2024-02-27T18:31:04.790Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1928, in _wait_change
2024-02-27T18:31:04.790Z [container-agent]     resp = self._request('GET', f'/v1/changes/{change_id}/wait', query)
2024-02-27T18:31:04.790Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1655, in _request
2024-02-27T18:31:04.790Z [container-agent]     response = self._request_raw(method, path, query, headers, data)
2024-02-27T18:31:04.790Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1690, in _request_raw
2024-02-27T18:31:04.790Z [container-agent]     response = self.opener.open(request, timeout=self.timeout)
2024-02-27T18:31:04.790Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 519, in open
2024-02-27T18:31:04.790Z [container-agent]     response = self._open(req, data)
2024-02-27T18:31:04.790Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 536, in _open
2024-02-27T18:31:04.790Z [container-agent]     result = self._call_chain(self.handle_open, protocol, protocol +
2024-02-27T18:31:04.790Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 496, in _call_chain
2024-02-27T18:31:04.790Z [container-agent]     result = func(*args)
2024-02-27T18:31:04.790Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 311, in http_open
2024-02-27T18:31:04.790Z [container-agent]     return self.do_open(_UnixSocketConnection, req,  # type:ignore
2024-02-27T18:31:04.790Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 1352, in do_open
2024-02-27T18:31:04.790Z [container-agent]     r = h.getresponse()
2024-02-27T18:31:04.790Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1375, in getresponse
2024-02-27T18:31:04.790Z [container-agent]     response.begin()
2024-02-27T18:31:04.790Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 318, in begin
2024-02-27T18:31:04.790Z [container-agent]     version, status, reason = self._read_status()
2024-02-27T18:31:04.790Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 287, in _read_status
2024-02-27T18:31:04.790Z [container-agent]     raise RemoteDisconnected("Remote end closed connection without"
2024-02-27T18:31:04.790Z [container-agent] http.client.RemoteDisconnected: Remote end closed connection without response
2024-02-27T18:31:04.920Z [container-agent] 2024-02-27 18:31:04 INFO juju.util.exec exec.go:209 run result: exit status 1
2024-02-27T18:31:10.468Z [container-agent] 2024-02-27 18:31:10 INFO juju-log Setting up the logrotate configurations
2024-02-27T18:31:10.549Z [container-agent] 2024-02-27 18:31:10 INFO juju-log Adding pebble layer
2024-02-27T18:31:17.371Z [container-agent] 2024-02-27 18:31:17 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-02-27T18:31:17.393Z [container-agent] 2024-02-27 18:31:17 INFO juju-log Attempting reboot from complete outage.
2024-02-27T18:31:20.225Z [container-agent] 2024-02-27 18:31:20 INFO juju.worker.uniter.operation runhook.go:186 ran "mysql-pebble-ready" hook (via hook dispatching script: dispatch)
2024-02-27T18:33:26.016Z [container-agent] 2024-02-27 18:33:26 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-27T18:33:27.845Z [container-agent] 2024-02-27 18:33:27 WARNING juju-log No relation: certificates
2024-02-27T18:33:28.198Z [container-agent] 2024-02-27 18:33:28 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-27T18:38:54.643Z [container-agent] 2024-02-27 18:38:54 INFO juju-log Starting the log rotate manager
2024-02-27T18:38:54.656Z [container-agent] 2024-02-27 18:38:54 INFO juju-log Started log rotate manager process with PID 43687
2024-02-27T18:38:54.932Z [container-agent] 2024-02-27 18:38:54 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-27T18:38:56.765Z [container-agent] 2024-02-27 18:38:56 WARNING juju-log No relation: certificates
2024-02-27T18:38:57.125Z [container-agent] 2024-02-27 18:38:57 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-27T18:44:38.640Z [container-agent] 2024-02-27 18:44:38 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-27T18:44:40.967Z [container-agent] 2024-02-27 18:44:40 WARNING juju-log No relation: certificates
2024-02-27T18:44:41.314Z [container-agent] 2024-02-27 18:44:41 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-27T18:49:55.702Z [container-agent] 2024-02-27 18:49:55 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-27T18:49:58.714Z [container-agent] 2024-02-27 18:49:58 WARNING juju-log No relation: certificates
2024-02-27T18:49:59.115Z [container-agent] 2024-02-27 18:49:59 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-27T18:55:14.576Z [container-agent] 2024-02-27 18:55:14 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-27T18:55:17.849Z [container-agent] 2024-02-27 18:55:17 WARNING juju-log No relation: certificates
2024-02-27T18:55:18.250Z [container-agent] 2024-02-27 18:55:18 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-27T18:59:23.407Z [container-agent] 2024-02-27 18:59:23 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-27T18:59:27.265Z [container-agent] 2024-02-27 18:59:27 WARNING juju-log No relation: certificates
2024-02-27T18:59:27.657Z [container-agent] 2024-02-27 18:59:27 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-27T19:05:12.407Z [container-agent] 2024-02-27 19:05:12 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-27T19:05:17.083Z [container-agent] 2024-02-27 19:05:17 WARNING juju-log No relation: certificates
2024-02-27T19:05:17.459Z [container-agent] 2024-02-27 19:05:17 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-27T19:11:09.886Z [container-agent] 2024-02-27 19:11:09 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-27T19:11:14.934Z [container-agent] 2024-02-27 19:11:14 WARNING juju-log No relation: certificates
2024-02-27T19:11:15.301Z [container-agent] 2024-02-27 19:11:15 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-27T19:15:32.300Z [container-agent] 2024-02-27 19:15:32 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-27T19:15:37.726Z [container-agent] 2024-02-27 19:15:37 WARNING juju-log No relation: certificates
2024-02-27T19:15:38.070Z [container-agent] 2024-02-27 19:15:38 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-27T19:20:02.416Z [container-agent] 2024-02-27 19:20:02 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-27T19:20:08.360Z [container-agent] 2024-02-27 19:20:08 WARNING juju-log No relation: certificates
2024-02-27T19:20:08.720Z [container-agent] 2024-02-27 19:20:08 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-27T19:24:54.971Z [container-agent] 2024-02-27 19:24:54 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-27T19:25:01.069Z [container-agent] 2024-02-27 19:25:01 WARNING juju-log No relation: certificates
2024-02-27T19:25:01.406Z [container-agent] 2024-02-27 19:25:01 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-27T19:30:47.937Z [container-agent] 2024-02-27 19:30:47 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-27T19:30:54.797Z [container-agent] 2024-02-27 19:30:54 WARNING juju-log No relation: certificates
2024-02-27T19:30:55.140Z [container-agent] 2024-02-27 19:30:55 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-27T19:35:59.587Z [container-agent] 2024-02-27 19:35:59 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-27T19:36:06.268Z [container-agent] 2024-02-27 19:36:06 WARNING juju-log No relation: certificates
2024-02-27T19:36:06.612Z [container-agent] 2024-02-27 19:36:06 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-27T19:41:30.106Z [container-agent] 2024-02-27 19:41:30 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-27T19:41:37.798Z [container-agent] 2024-02-27 19:41:37 WARNING juju-log No relation: certificates
2024-02-27T19:41:38.150Z [container-agent] 2024-02-27 19:41:38 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-27T19:46:32.011Z [container-agent] 2024-02-27 19:46:32 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-02-27T19:46:32.045Z [container-agent] 2024-02-27 19:46:32 INFO juju-log Attempting reboot from complete outage.
2024-02-27T19:46:39.525Z [container-agent] 2024-02-27 19:46:39 WARNING juju-log No relation: certificates
2024-02-27T19:46:39.869Z [container-agent] 2024-02-27 19:46:39 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-27T19:47:04.753Z [container-agent] 2024-02-27 19:47:04 ERROR juju-log Uncaught exception while in charm code:
2024-02-27T19:47:04.753Z [container-agent] Traceback (most recent call last):
2024-02-27T19:47:04.753Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/./src/charm.py", line 786, in <module>
2024-02-27T19:47:04.753Z [container-agent]     main(MySQLOperatorCharm)
2024-02-27T19:47:04.753Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 436, in main
2024-02-27T19:47:04.753Z [container-agent]     _emit_charm_event(charm, dispatcher.event_name)
2024-02-27T19:47:04.753Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 144, in _emit_charm_event
2024-02-27T19:47:04.753Z [container-agent]     event_to_emit.emit(*args, **kwargs)
2024-02-27T19:47:04.753Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 351, in emit
2024-02-27T19:47:04.753Z [container-agent]     framework._emit(event)
2024-02-27T19:47:04.753Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 853, in _emit
2024-02-27T19:47:04.753Z [container-agent]     self._reemit(event_path)
2024-02-27T19:47:04.753Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 942, in _reemit
2024-02-27T19:47:04.753Z [container-agent]     custom_handler(event)
2024-02-27T19:47:04.753Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/rotate_mysql_logs.py", line 55, in _rotate_mysql_logs
2024-02-27T19:47:04.753Z [container-agent]     self.charm._mysql.flush_mysql_logs(MySQLTextLogs.ERROR)
2024-02-27T19:47:04.753Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-02-27T19:47:04.753Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-02-27T19:47:04.753Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 665, in _run_mysqlsh_script
2024-02-27T19:47:04.753Z [container-agent]     process = self.container.exec(cmd, timeout=timeout)
2024-02-27T19:47:04.753Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/model.py", line 2688, in exec
2024-02-27T19:47:04.753Z [container-agent]     return self._pebble.exec(
2024-02-27T19:47:04.753Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 2501, in exec
2024-02-27T19:47:04.753Z [container-agent]     resp = self._request('POST', '/v1/exec', body=body)
2024-02-27T19:47:04.753Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1655, in _request
2024-02-27T19:47:04.753Z [container-agent]     response = self._request_raw(method, path, query, headers, data)
2024-02-27T19:47:04.753Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1690, in _request_raw
2024-02-27T19:47:04.753Z [container-agent]     response = self.opener.open(request, timeout=self.timeout)
2024-02-27T19:47:04.753Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 519, in open
2024-02-27T19:47:04.753Z [container-agent]     response = self._open(req, data)
2024-02-27T19:47:04.753Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 536, in _open
2024-02-27T19:47:04.753Z [container-agent]     result = self._call_chain(self.handle_open, protocol, protocol +
2024-02-27T19:47:04.753Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 496, in _call_chain
2024-02-27T19:47:04.753Z [container-agent]     result = func(*args)
2024-02-27T19:47:04.753Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 311, in http_open
2024-02-27T19:47:04.753Z [container-agent]     return self.do_open(_UnixSocketConnection, req,  # type:ignore
2024-02-27T19:47:04.753Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 1352, in do_open
2024-02-27T19:47:04.753Z [container-agent]     r = h.getresponse()
2024-02-27T19:47:04.753Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1375, in getresponse
2024-02-27T19:47:04.753Z [container-agent]     response.begin()
2024-02-27T19:47:04.753Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 318, in begin
2024-02-27T19:47:04.753Z [container-agent]     version, status, reason = self._read_status()
2024-02-27T19:47:04.753Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 287, in _read_status
2024-02-27T19:47:04.753Z [container-agent]     raise RemoteDisconnected("Remote end closed connection without"
2024-02-27T19:47:04.753Z [container-agent] http.client.RemoteDisconnected: Remote end closed connection without response
2024-02-27T19:47:04.880Z [container-agent] 2024-02-27 19:47:04 INFO juju.util.exec exec.go:209 run result: exit status 1
2024-02-27T19:47:08.281Z [container-agent] 2024-02-27 19:47:08 INFO juju-log Setting up the logrotate configurations
2024-02-27T19:47:08.342Z [container-agent] 2024-02-27 19:47:08 INFO juju-log Adding pebble layer
2024-02-27T19:47:15.114Z [container-agent] 2024-02-27 19:47:15 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-02-27T19:47:15.138Z [container-agent] 2024-02-27 19:47:15 INFO juju-log Attempting reboot from complete outage.
2024-02-27T19:47:18.218Z [container-agent] 2024-02-27 19:47:18 INFO juju.worker.uniter.operation runhook.go:186 ran "mysql-pebble-ready" hook (via hook dispatching script: dispatch)
2024-02-27T19:51:49.286Z [container-agent] 2024-02-27 19:51:49 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-27T19:51:51.117Z [container-agent] 2024-02-27 19:51:51 WARNING juju-log No relation: certificates
2024-02-27T19:51:51.467Z [container-agent] 2024-02-27 19:51:51 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-27T19:57:09.408Z [container-agent] 2024-02-27 19:57:09 INFO juju-log Starting the log rotate manager
2024-02-27T19:57:09.422Z [container-agent] 2024-02-27 19:57:09 INFO juju-log Started log rotate manager process with PID 50652
2024-02-27T19:57:09.748Z [container-agent] 2024-02-27 19:57:09 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-27T19:57:11.559Z [container-agent] 2024-02-27 19:57:11 WARNING juju-log No relation: certificates
2024-02-27T19:57:11.894Z [container-agent] 2024-02-27 19:57:11 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-27T20:01:28.312Z [container-agent] 2024-02-27 20:01:28 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-27T20:01:30.756Z [container-agent] 2024-02-27 20:01:30 WARNING juju-log No relation: certificates
2024-02-27T20:01:31.113Z [container-agent] 2024-02-27 20:01:31 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-27T20:05:37.125Z [container-agent] 2024-02-27 20:05:37 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-27T20:05:39.928Z [container-agent] 2024-02-27 20:05:39 WARNING juju-log No relation: certificates
2024-02-27T20:05:40.271Z [container-agent] 2024-02-27 20:05:40 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-27T20:10:26.302Z [container-agent] 2024-02-27 20:10:26 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-27T20:10:29.573Z [container-agent] 2024-02-27 20:10:29 WARNING juju-log No relation: certificates
2024-02-27T20:10:29.932Z [container-agent] 2024-02-27 20:10:29 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-27T20:14:50.882Z [container-agent] 2024-02-27 20:14:50 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-27T20:14:54.308Z [container-agent] 2024-02-27 20:14:54 WARNING juju-log No relation: certificates
2024-02-27T20:14:54.649Z [container-agent] 2024-02-27 20:14:54 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-27T20:20:48.922Z [container-agent] 2024-02-27 20:20:48 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-27T20:20:52.954Z [container-agent] 2024-02-27 20:20:52 WARNING juju-log No relation: certificates
2024-02-27T20:20:53.323Z [container-agent] 2024-02-27 20:20:53 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-27T20:25:16.679Z [container-agent] 2024-02-27 20:25:16 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-27T20:25:21.349Z [container-agent] 2024-02-27 20:25:21 WARNING juju-log No relation: certificates
2024-02-27T20:25:21.701Z [container-agent] 2024-02-27 20:25:21 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-27T20:30:32.503Z [container-agent] 2024-02-27 20:30:32 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-27T20:30:37.492Z [container-agent] 2024-02-27 20:30:37 WARNING juju-log No relation: certificates
2024-02-27T20:30:37.854Z [container-agent] 2024-02-27 20:30:37 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-27T20:34:39.188Z [container-agent] 2024-02-27 20:34:39 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-27T20:34:44.698Z [container-agent] 2024-02-27 20:34:44 WARNING juju-log No relation: certificates
2024-02-27T20:34:45.052Z [container-agent] 2024-02-27 20:34:45 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-27T20:38:54.842Z [container-agent] 2024-02-27 20:38:54 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-27T20:39:01.009Z [container-agent] 2024-02-27 20:39:01 WARNING juju-log No relation: certificates
2024-02-27T20:39:01.375Z [container-agent] 2024-02-27 20:39:01 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-27T20:44:00.243Z [container-agent] 2024-02-27 20:44:00 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-27T20:44:06.202Z [container-agent] 2024-02-27 20:44:06 WARNING juju-log No relation: certificates
2024-02-27T20:44:06.539Z [container-agent] 2024-02-27 20:44:06 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-27T20:49:12.328Z [container-agent] 2024-02-27 20:49:12 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-27T20:49:19.069Z [container-agent] 2024-02-27 20:49:19 WARNING juju-log No relation: certificates
2024-02-27T20:49:19.429Z [container-agent] 2024-02-27 20:49:19 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-27T20:53:16.656Z [container-agent] 2024-02-27 20:53:16 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-27T20:53:24.409Z [container-agent] 2024-02-27 20:53:24 WARNING juju-log No relation: certificates
2024-02-27T20:53:24.743Z [container-agent] 2024-02-27 20:53:24 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-27T20:57:29.242Z [container-agent] 2024-02-27 20:57:29 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-27T20:57:37.114Z [container-agent] 2024-02-27 20:57:37 WARNING juju-log No relation: certificates
2024-02-27T20:57:37.472Z [container-agent] 2024-02-27 20:57:37 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-27T21:01:33.002Z [container-agent] 2024-02-27 21:01:33 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-27T21:01:40.853Z [container-agent] 2024-02-27 21:01:40 WARNING juju-log No relation: certificates
2024-02-27T21:01:41.210Z [container-agent] 2024-02-27 21:01:41 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-27T21:06:16.215Z [container-agent] 2024-02-27 21:06:16 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-27T21:06:24.405Z [container-agent] 2024-02-27 21:06:24 WARNING juju-log No relation: certificates
2024-02-27T21:06:24.755Z [container-agent] 2024-02-27 21:06:24 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-27T21:10:48.227Z [container-agent] 2024-02-27 21:10:48 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-02-27T21:10:48.290Z [container-agent] 2024-02-27 21:10:48 INFO juju-log Attempting reboot from complete outage.
2024-02-27T21:10:56.217Z [container-agent] 2024-02-27 21:10:56 WARNING juju-log No relation: certificates
2024-02-27T21:10:56.562Z [container-agent] 2024-02-27 21:10:56 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-27T21:14:09.757Z [container-agent] 2024-02-27 21:14:09 ERROR juju-log Uncaught exception while in charm code:
2024-02-27T21:14:09.757Z [container-agent] Traceback (most recent call last):
2024-02-27T21:14:09.757Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/./src/charm.py", line 786, in <module>
2024-02-27T21:14:09.757Z [container-agent]     main(MySQLOperatorCharm)
2024-02-27T21:14:09.757Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 436, in main
2024-02-27T21:14:09.757Z [container-agent]     _emit_charm_event(charm, dispatcher.event_name)
2024-02-27T21:14:09.757Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 144, in _emit_charm_event
2024-02-27T21:14:09.757Z [container-agent]     event_to_emit.emit(*args, **kwargs)
2024-02-27T21:14:09.757Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 351, in emit
2024-02-27T21:14:09.757Z [container-agent]     framework._emit(event)
2024-02-27T21:14:09.757Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 853, in _emit
2024-02-27T21:14:09.757Z [container-agent]     self._reemit(event_path)
2024-02-27T21:14:09.757Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 942, in _reemit
2024-02-27T21:14:09.757Z [container-agent]     custom_handler(event)
2024-02-27T21:14:09.757Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/rotate_mysql_logs.py", line 57, in _rotate_mysql_logs
2024-02-27T21:14:09.757Z [container-agent]     self.charm._mysql.flush_mysql_logs(MySQLTextLogs.SLOW)
2024-02-27T21:14:09.757Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-02-27T21:14:09.757Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-02-27T21:14:09.757Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 665, in _run_mysqlsh_script
2024-02-27T21:14:09.757Z [container-agent]     process = self.container.exec(cmd, timeout=timeout)
2024-02-27T21:14:09.757Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/model.py", line 2688, in exec
2024-02-27T21:14:09.757Z [container-agent]     return self._pebble.exec(
2024-02-27T21:14:09.757Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 2501, in exec
2024-02-27T21:14:09.757Z [container-agent]     resp = self._request('POST', '/v1/exec', body=body)
2024-02-27T21:14:09.757Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1655, in _request
2024-02-27T21:14:09.757Z [container-agent]     response = self._request_raw(method, path, query, headers, data)
2024-02-27T21:14:09.757Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1690, in _request_raw
2024-02-27T21:14:09.757Z [container-agent]     response = self.opener.open(request, timeout=self.timeout)
2024-02-27T21:14:09.757Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 519, in open
2024-02-27T21:14:09.757Z [container-agent]     response = self._open(req, data)
2024-02-27T21:14:09.757Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 536, in _open
2024-02-27T21:14:09.757Z [container-agent]     result = self._call_chain(self.handle_open, protocol, protocol +
2024-02-27T21:14:09.757Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 496, in _call_chain
2024-02-27T21:14:09.757Z [container-agent]     result = func(*args)
2024-02-27T21:14:09.757Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 311, in http_open
2024-02-27T21:14:09.757Z [container-agent]     return self.do_open(_UnixSocketConnection, req,  # type:ignore
2024-02-27T21:14:09.757Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 1352, in do_open
2024-02-27T21:14:09.757Z [container-agent]     r = h.getresponse()
2024-02-27T21:14:09.757Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1375, in getresponse
2024-02-27T21:14:09.757Z [container-agent]     response.begin()
2024-02-27T21:14:09.757Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 318, in begin
2024-02-27T21:14:09.757Z [container-agent]     version, status, reason = self._read_status()
2024-02-27T21:14:09.757Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 287, in _read_status
2024-02-27T21:14:09.757Z [container-agent]     raise RemoteDisconnected("Remote end closed connection without"
2024-02-27T21:14:09.757Z [container-agent] http.client.RemoteDisconnected: Remote end closed connection without response
2024-02-27T21:14:09.918Z [container-agent] 2024-02-27 21:14:09 INFO juju.util.exec exec.go:209 run result: exit status 1
2024-02-27T21:14:11.773Z [container-agent] 2024-02-27 21:14:11 INFO juju-log Setting up the logrotate configurations
2024-02-27T21:14:11.841Z [container-agent] 2024-02-27 21:14:11 INFO juju-log Adding pebble layer
2024-02-27T21:14:18.666Z [container-agent] 2024-02-27 21:14:18 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-02-27T21:14:18.695Z [container-agent] 2024-02-27 21:14:18 INFO juju-log Attempting reboot from complete outage.
2024-02-27T21:14:21.675Z [container-agent] 2024-02-27 21:14:21 INFO juju.worker.uniter.operation runhook.go:186 ran "mysql-pebble-ready" hook (via hook dispatching script: dispatch)
2024-02-27T21:16:28.299Z [container-agent] 2024-02-27 21:16:28 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-27T21:16:30.159Z [container-agent] 2024-02-27 21:16:30 WARNING juju-log No relation: certificates
2024-02-27T21:16:30.554Z [container-agent] 2024-02-27 21:16:30 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-27T21:21:45.175Z [container-agent] 2024-02-27 21:21:45 INFO juju-log Starting the log rotate manager
2024-02-27T21:21:45.188Z [container-agent] 2024-02-27 21:21:45 INFO juju-log Started log rotate manager process with PID 58646
2024-02-27T21:21:45.677Z [container-agent] 2024-02-27 21:21:45 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-27T21:21:47.768Z [container-agent] 2024-02-27 21:21:47 WARNING juju-log No relation: certificates
2024-02-27T21:21:48.122Z [container-agent] 2024-02-27 21:21:48 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-27T21:26:00.720Z [container-agent] 2024-02-27 21:26:00 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-27T21:26:03.214Z [container-agent] 2024-02-27 21:26:03 WARNING juju-log No relation: certificates
2024-02-27T21:26:03.625Z [container-agent] 2024-02-27 21:26:03 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-27T21:30:06.581Z [container-agent] 2024-02-27 21:30:06 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-27T21:30:09.418Z [container-agent] 2024-02-27 21:30:09 WARNING juju-log No relation: certificates
2024-02-27T21:30:09.743Z [container-agent] 2024-02-27 21:30:09 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-27T21:35:12.559Z [container-agent] 2024-02-27 21:35:12 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-27T21:35:16.066Z [container-agent] 2024-02-27 21:35:16 WARNING juju-log No relation: certificates
2024-02-27T21:35:16.407Z [container-agent] 2024-02-27 21:35:16 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-27T21:39:25.398Z [container-agent] 2024-02-27 21:39:25 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-27T21:39:29.009Z [container-agent] 2024-02-27 21:39:29 WARNING juju-log No relation: certificates
2024-02-27T21:39:29.358Z [container-agent] 2024-02-27 21:39:29 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-27T21:43:34.765Z [container-agent] 2024-02-27 21:43:34 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-27T21:43:38.680Z [container-agent] 2024-02-27 21:43:38 WARNING juju-log No relation: certificates
2024-02-27T21:43:39.008Z [container-agent] 2024-02-27 21:43:39 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-27T21:47:36.882Z [container-agent] 2024-02-27 21:47:36 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-27T21:47:41.287Z [container-agent] 2024-02-27 21:47:41 WARNING juju-log No relation: certificates
2024-02-27T21:47:41.634Z [container-agent] 2024-02-27 21:47:41 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-27T21:51:37.261Z [container-agent] 2024-02-27 21:51:37 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-27T21:51:41.924Z [container-agent] 2024-02-27 21:51:41 WARNING juju-log No relation: certificates
2024-02-27T21:51:42.253Z [container-agent] 2024-02-27 21:51:42 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-27T21:56:38.303Z [container-agent] 2024-02-27 21:56:38 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-27T21:56:43.146Z [container-agent] 2024-02-27 21:56:43 WARNING juju-log No relation: certificates
2024-02-27T21:56:43.507Z [container-agent] 2024-02-27 21:56:43 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-27T22:01:59.273Z [container-agent] 2024-02-27 22:01:59 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-27T22:02:04.937Z [container-agent] 2024-02-27 22:02:04 WARNING juju-log No relation: certificates
2024-02-27T22:02:05.273Z [container-agent] 2024-02-27 22:02:05 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-27T22:06:28.972Z [container-agent] 2024-02-27 22:06:28 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-27T22:06:35.449Z [container-agent] 2024-02-27 22:06:35 WARNING juju-log No relation: certificates
2024-02-27T22:06:35.789Z [container-agent] 2024-02-27 22:06:35 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-27T22:10:50.811Z [container-agent] 2024-02-27 22:10:50 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-27T22:10:57.615Z [container-agent] 2024-02-27 22:10:57 WARNING juju-log No relation: certificates
2024-02-27T22:10:58.001Z [container-agent] 2024-02-27 22:10:58 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-27T22:16:49.900Z [container-agent] 2024-02-27 22:16:49 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-27T22:16:57.040Z [container-agent] 2024-02-27 22:16:57 WARNING juju-log No relation: certificates
2024-02-27T22:16:57.377Z [container-agent] 2024-02-27 22:16:57 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-27T22:21:35.836Z [container-agent] 2024-02-27 22:21:35 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-27T22:21:43.654Z [container-agent] 2024-02-27 22:21:43 WARNING juju-log No relation: certificates
2024-02-27T22:21:44.039Z [container-agent] 2024-02-27 22:21:44 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-27T22:26:57.790Z [container-agent] 2024-02-27 22:26:57 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-27T22:27:06.188Z [container-agent] 2024-02-27 22:27:06 WARNING juju-log No relation: certificates
2024-02-27T22:27:06.534Z [container-agent] 2024-02-27 22:27:06 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-27T22:30:10.634Z [container-agent] 2024-02-27 22:30:10 ERROR juju-log Failed to flush ERROR LOGS logs.
2024-02-27T22:30:10.634Z [container-agent] Traceback (most recent call last):
2024-02-27T22:30:10.634Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 666, in _run_mysqlsh_script
2024-02-27T22:30:10.634Z [container-agent]     stdout, _ = process.wait_output()
2024-02-27T22:30:10.634Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1441, in wait_output
2024-02-27T22:30:10.634Z [container-agent]     raise ExecError[AnyStr](self._command, exit_code, out_value, err_value)
2024-02-27T22:30:10.634Z [container-agent] ops.pebble.ExecError: non-zero exit code 1 executing ['/usr/bin/mysqlsh', '--no-wizard', '--python', '--verbose=1', '-f', '/tmp/script.py', ';', 'rm', '/tmp/script.py'], stdout='', stderr='Cannot set LC_ALL to locale en_US.UTF-8: No such file or directory\nverbose: 2024-02-27T22:30:06Z: Loading startup files...\nverbose: 2024-02-27T22:30:06Z: Loading plugins...\nverbose: 2024-02-27T22:30:06Z: Connecting to MySQL at: serverconfig@horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local\nTraceback (most recent call last):\n  File "<string>", line 1, in <module>\nmysqlsh.DBError: MySQL Error (2013): Shell.connect: Lost connection to MySQL server at \'reading initial communication packet\', system error: 104\n'
2024-02-27T22:30:10.634Z [container-agent] 
2024-02-27T22:30:10.634Z [container-agent] During handling of the above exception, another exception occurred:
2024-02-27T22:30:10.634Z [container-agent] 
2024-02-27T22:30:10.634Z [container-agent] Traceback (most recent call last):
2024-02-27T22:30:10.634Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-02-27T22:30:10.634Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-02-27T22:30:10.634Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 669, in _run_mysqlsh_script
2024-02-27T22:30:10.634Z [container-agent]     raise MySQLClientError(e.stderr)
2024-02-27T22:30:10.634Z [container-agent] charms.mysql.v0.mysql.MySQLClientError: Cannot set LC_ALL to locale en_US.UTF-8: No such file or directory
2024-02-27T22:30:10.634Z [container-agent] verbose: 2024-02-27T22:30:06Z: Loading startup files...
2024-02-27T22:30:10.634Z [container-agent] verbose: 2024-02-27T22:30:06Z: Loading plugins...
2024-02-27T22:30:10.634Z [container-agent] verbose: 2024-02-27T22:30:06Z: Connecting to MySQL at: serverconfig@horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local
2024-02-27T22:30:10.634Z [container-agent] Traceback (most recent call last):
2024-02-27T22:30:10.634Z [container-agent]   File "<string>", line 1, in <module>
2024-02-27T22:30:10.634Z [container-agent] mysqlsh.DBError: MySQL Error (2013): Shell.connect: Lost connection to MySQL server at 'reading initial communication packet', system error: 104
2024-02-27T22:30:10.634Z [container-agent] 
2024-02-27T22:30:13.751Z [container-agent] 2024-02-27 22:30:13 ERROR juju-log Failed to flush GENERAL LOGS logs.
2024-02-27T22:30:13.751Z [container-agent] Traceback (most recent call last):
2024-02-27T22:30:13.751Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 666, in _run_mysqlsh_script
2024-02-27T22:30:13.751Z [container-agent]     stdout, _ = process.wait_output()
2024-02-27T22:30:13.751Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1441, in wait_output
2024-02-27T22:30:13.751Z [container-agent]     raise ExecError[AnyStr](self._command, exit_code, out_value, err_value)
2024-02-27T22:30:13.751Z [container-agent] ops.pebble.ExecError: non-zero exit code 1 executing ['/usr/bin/mysqlsh', '--no-wizard', '--python', '--verbose=1', '-f', '/tmp/script.py', ';', 'rm', '/tmp/script.py'], stdout='', stderr='Cannot set LC_ALL to locale en_US.UTF-8: No such file or directory\nverbose: 2024-02-27T22:30:11Z: Loading startup files...\nverbose: 2024-02-27T22:30:11Z: Loading plugins...\nverbose: 2024-02-27T22:30:12Z: Connecting to MySQL at: serverconfig@horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local\nTraceback (most recent call last):\n  File "<string>", line 1, in <module>\nmysqlsh.DBError: MySQL Error (2003): Shell.connect: Can\'t connect to MySQL server on \'horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local:3306\' (111)\n'
2024-02-27T22:30:13.751Z [container-agent] 
2024-02-27T22:30:13.751Z [container-agent] During handling of the above exception, another exception occurred:
2024-02-27T22:30:13.751Z [container-agent] 
2024-02-27T22:30:13.751Z [container-agent] Traceback (most recent call last):
2024-02-27T22:30:13.751Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-02-27T22:30:13.751Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-02-27T22:30:13.751Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 669, in _run_mysqlsh_script
2024-02-27T22:30:13.751Z [container-agent]     raise MySQLClientError(e.stderr)
2024-02-27T22:30:13.751Z [container-agent] charms.mysql.v0.mysql.MySQLClientError: Cannot set LC_ALL to locale en_US.UTF-8: No such file or directory
2024-02-27T22:30:13.751Z [container-agent] verbose: 2024-02-27T22:30:11Z: Loading startup files...
2024-02-27T22:30:13.751Z [container-agent] verbose: 2024-02-27T22:30:11Z: Loading plugins...
2024-02-27T22:30:13.751Z [container-agent] verbose: 2024-02-27T22:30:12Z: Connecting to MySQL at: serverconfig@horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local
2024-02-27T22:30:13.751Z [container-agent] Traceback (most recent call last):
2024-02-27T22:30:13.751Z [container-agent]   File "<string>", line 1, in <module>
2024-02-27T22:30:13.751Z [container-agent] mysqlsh.DBError: MySQL Error (2003): Shell.connect: Can't connect to MySQL server on 'horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local:3306' (111)
2024-02-27T22:30:13.751Z [container-agent] 
2024-02-27T22:30:13.768Z [container-agent] 2024-02-27 22:30:13 ERROR juju-log Uncaught exception while in charm code:
2024-02-27T22:30:13.769Z [container-agent] Traceback (most recent call last):
2024-02-27T22:30:13.769Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/./src/charm.py", line 786, in <module>
2024-02-27T22:30:13.769Z [container-agent]     main(MySQLOperatorCharm)
2024-02-27T22:30:13.769Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 436, in main
2024-02-27T22:30:13.769Z [container-agent]     _emit_charm_event(charm, dispatcher.event_name)
2024-02-27T22:30:13.769Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 144, in _emit_charm_event
2024-02-27T22:30:13.769Z [container-agent]     event_to_emit.emit(*args, **kwargs)
2024-02-27T22:30:13.769Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 351, in emit
2024-02-27T22:30:13.769Z [container-agent]     framework._emit(event)
2024-02-27T22:30:13.769Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 853, in _emit
2024-02-27T22:30:13.769Z [container-agent]     self._reemit(event_path)
2024-02-27T22:30:13.769Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 942, in _reemit
2024-02-27T22:30:13.769Z [container-agent]     custom_handler(event)
2024-02-27T22:30:13.769Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/rotate_mysql_logs.py", line 57, in _rotate_mysql_logs
2024-02-27T22:30:13.769Z [container-agent]     self.charm._mysql.flush_mysql_logs(MySQLTextLogs.SLOW)
2024-02-27T22:30:13.769Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-02-27T22:30:13.769Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-02-27T22:30:13.769Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 649, in _run_mysqlsh_script
2024-02-27T22:30:13.769Z [container-agent]     self.container.push(path=MYSQLSH_SCRIPT_FILE, source=script)
2024-02-27T22:30:13.769Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/model.py", line 2272, in push
2024-02-27T22:30:13.769Z [container-agent]     self._pebble.push(str(path), source, encoding=encoding,
2024-02-27T22:30:13.769Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 2125, in push
2024-02-27T22:30:13.769Z [container-agent]     response = self._request_raw('POST', '/v1/files', None, headers, data)
2024-02-27T22:30:13.769Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1704, in _request_raw
2024-02-27T22:30:13.769Z [container-agent]     raise ConnectionError(
2024-02-27T22:30:13.769Z [container-agent] ops.pebble.ConnectionError: Could not connect to Pebble: socket not found at '/charm/containers/mysql/pebble.socket' (container restarted?)
2024-02-27T22:30:13.930Z [container-agent] 2024-02-27 22:30:13 INFO juju.util.exec exec.go:209 run result: exit status 1
2024-02-27T22:30:15.404Z [container-agent] 2024-02-27 22:30:15 INFO juju-log Starting the log rotate manager
2024-02-27T22:30:15.417Z [container-agent] 2024-02-27 22:30:15 INFO juju-log Started log rotate manager process with PID 65455
2024-02-27T22:30:15.797Z [container-agent] 2024-02-27 22:30:15 INFO juju-log Setting up the logrotate configurations
2024-02-27T22:30:15.857Z [container-agent] 2024-02-27 22:30:15 INFO juju-log Adding pebble layer
2024-02-27T22:30:22.594Z [container-agent] 2024-02-27 22:30:22 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-02-27T22:30:22.622Z [container-agent] 2024-02-27 22:30:22 INFO juju-log Attempting reboot from complete outage.
2024-02-27T22:30:25.457Z [container-agent] 2024-02-27 22:30:25 INFO juju.worker.uniter.operation runhook.go:186 ran "mysql-pebble-ready" hook (via hook dispatching script: dispatch)
2024-02-27T22:32:05.762Z [container-agent] 2024-02-27 22:32:05 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-27T22:32:07.881Z [container-agent] 2024-02-27 22:32:07 WARNING juju-log No relation: certificates
2024-02-27T22:32:08.225Z [container-agent] 2024-02-27 22:32:08 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-27T22:37:19.664Z [container-agent] 2024-02-27 22:37:19 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-27T22:37:22.167Z [container-agent] 2024-02-27 22:37:22 WARNING juju-log No relation: certificates
2024-02-27T22:37:22.508Z [container-agent] 2024-02-27 22:37:22 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-27T22:42:31.621Z [container-agent] 2024-02-27 22:42:31 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-27T22:42:34.872Z [container-agent] 2024-02-27 22:42:34 WARNING juju-log No relation: certificates
2024-02-27T22:42:35.221Z [container-agent] 2024-02-27 22:42:35 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-27T22:48:07.942Z [container-agent] 2024-02-27 22:48:07 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-27T22:48:11.102Z [container-agent] 2024-02-27 22:48:11 WARNING juju-log No relation: certificates
2024-02-27T22:48:11.484Z [container-agent] 2024-02-27 22:48:11 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-27T22:52:16.648Z [container-agent] 2024-02-27 22:52:16 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-27T22:52:20.617Z [container-agent] 2024-02-27 22:52:20 WARNING juju-log No relation: certificates
2024-02-27T22:52:20.997Z [container-agent] 2024-02-27 22:52:20 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-27T22:57:30.653Z [container-agent] 2024-02-27 22:57:30 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-27T22:57:35.170Z [container-agent] 2024-02-27 22:57:35 WARNING juju-log No relation: certificates
2024-02-27T22:57:35.507Z [container-agent] 2024-02-27 22:57:35 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-27T23:01:48.424Z [container-agent] 2024-02-27 23:01:48 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-27T23:01:53.023Z [container-agent] 2024-02-27 23:01:53 WARNING juju-log No relation: certificates
2024-02-27T23:01:53.356Z [container-agent] 2024-02-27 23:01:53 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-27T23:07:10.366Z [container-agent] 2024-02-27 23:07:10 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-27T23:07:15.723Z [container-agent] 2024-02-27 23:07:15 WARNING juju-log No relation: certificates
2024-02-27T23:07:16.095Z [container-agent] 2024-02-27 23:07:16 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-27T23:11:56.362Z [container-agent] 2024-02-27 23:11:56 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-27T23:12:01.744Z [container-agent] 2024-02-27 23:12:01 WARNING juju-log No relation: certificates
2024-02-27T23:12:02.136Z [container-agent] 2024-02-27 23:12:02 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-27T23:16:26.476Z [container-agent] 2024-02-27 23:16:26 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-27T23:16:31.882Z [container-agent] 2024-02-27 23:16:31 WARNING juju-log No relation: certificates
2024-02-27T23:16:32.219Z [container-agent] 2024-02-27 23:16:32 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-27T23:22:20.375Z [container-agent] 2024-02-27 23:22:20 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-27T23:22:26.322Z [container-agent] 2024-02-27 23:22:26 WARNING juju-log No relation: certificates
2024-02-27T23:22:26.667Z [container-agent] 2024-02-27 23:22:26 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-27T23:27:12.390Z [container-agent] 2024-02-27 23:27:12 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-27T23:27:19.501Z [container-agent] 2024-02-27 23:27:19 WARNING juju-log No relation: certificates
2024-02-27T23:27:19.860Z [container-agent] 2024-02-27 23:27:19 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-27T23:32:46.598Z [container-agent] 2024-02-27 23:32:46 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-27T23:32:54.398Z [container-agent] 2024-02-27 23:32:54 WARNING juju-log No relation: certificates
2024-02-27T23:32:54.735Z [container-agent] 2024-02-27 23:32:54 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-27T23:38:14.105Z [container-agent] 2024-02-27 23:38:14 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-27T23:38:22.112Z [container-agent] 2024-02-27 23:38:22 WARNING juju-log No relation: certificates
2024-02-27T23:38:22.486Z [container-agent] 2024-02-27 23:38:22 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-27T23:43:19.159Z [container-agent] 2024-02-27 23:43:19 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-27T23:43:27.526Z [container-agent] 2024-02-27 23:43:27 WARNING juju-log No relation: certificates
2024-02-27T23:43:27.864Z [container-agent] 2024-02-27 23:43:27 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-27T23:47:45.667Z [container-agent] 2024-02-27 23:47:45 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-27T23:47:54.427Z [container-agent] 2024-02-27 23:47:54 WARNING juju-log No relation: certificates
2024-02-27T23:47:54.769Z [container-agent] 2024-02-27 23:47:54 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-27T23:48:13.642Z [container-agent] 2024-02-27 23:48:13 ERROR juju-log Failed to flush SLOW LOGS logs.
2024-02-27T23:48:13.642Z [container-agent] Traceback (most recent call last):
2024-02-27T23:48:13.642Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 666, in _run_mysqlsh_script
2024-02-27T23:48:13.642Z [container-agent]     stdout, _ = process.wait_output()
2024-02-27T23:48:13.642Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1441, in wait_output
2024-02-27T23:48:13.642Z [container-agent]     raise ExecError[AnyStr](self._command, exit_code, out_value, err_value)
2024-02-27T23:48:13.642Z [container-agent] ops.pebble.ExecError: non-zero exit code 1 executing ['/usr/bin/mysqlsh', '--no-wizard', '--python', '--verbose=1', '-f', '/tmp/script.py', ';', 'rm', '/tmp/script.py'], stdout='', stderr='Cannot set LC_ALL to locale en_US.UTF-8: No such file or directory\nverbose: 2024-02-27T23:48:12Z: Loading startup files...\nverbose: 2024-02-27T23:48:12Z: Loading plugins...\nverbose: 2024-02-27T23:48:12Z: Connecting to MySQL at: serverconfig@horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local\nTraceback (most recent call last):\n  File "<string>", line 1, in <module>\nmysqlsh.DBError: MySQL Error (2003): Shell.connect: Can\'t connect to MySQL server on \'horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local:3306\' (111)\n'
2024-02-27T23:48:13.642Z [container-agent] 
2024-02-27T23:48:13.642Z [container-agent] During handling of the above exception, another exception occurred:
2024-02-27T23:48:13.642Z [container-agent] 
2024-02-27T23:48:13.642Z [container-agent] Traceback (most recent call last):
2024-02-27T23:48:13.642Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-02-27T23:48:13.642Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-02-27T23:48:13.642Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 669, in _run_mysqlsh_script
2024-02-27T23:48:13.642Z [container-agent]     raise MySQLClientError(e.stderr)
2024-02-27T23:48:13.642Z [container-agent] charms.mysql.v0.mysql.MySQLClientError: Cannot set LC_ALL to locale en_US.UTF-8: No such file or directory
2024-02-27T23:48:13.642Z [container-agent] verbose: 2024-02-27T23:48:12Z: Loading startup files...
2024-02-27T23:48:13.642Z [container-agent] verbose: 2024-02-27T23:48:12Z: Loading plugins...
2024-02-27T23:48:13.642Z [container-agent] verbose: 2024-02-27T23:48:12Z: Connecting to MySQL at: serverconfig@horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local
2024-02-27T23:48:13.642Z [container-agent] Traceback (most recent call last):
2024-02-27T23:48:13.642Z [container-agent]   File "<string>", line 1, in <module>
2024-02-27T23:48:13.642Z [container-agent] mysqlsh.DBError: MySQL Error (2003): Shell.connect: Can't connect to MySQL server on 'horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local:3306' (111)
2024-02-27T23:48:13.642Z [container-agent] 
2024-02-27T23:52:52.654Z [container-agent] 2024-02-27 23:52:52 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-02-27T23:52:52.682Z [container-agent] 2024-02-27 23:52:52 INFO juju-log Attempting reboot from complete outage.
2024-02-27T23:53:01.052Z [container-agent] 2024-02-27 23:53:01 WARNING juju-log No relation: certificates
2024-02-27T23:53:01.380Z [container-agent] 2024-02-27 23:53:01 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-27T23:53:09.795Z [container-agent] 2024-02-27 23:53:09 ERROR juju-log Uncaught exception while in charm code:
2024-02-27T23:53:09.795Z [container-agent] Traceback (most recent call last):
2024-02-27T23:53:09.795Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/./src/charm.py", line 786, in <module>
2024-02-27T23:53:09.795Z [container-agent]     main(MySQLOperatorCharm)
2024-02-27T23:53:09.795Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 436, in main
2024-02-27T23:53:09.795Z [container-agent]     _emit_charm_event(charm, dispatcher.event_name)
2024-02-27T23:53:09.795Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 144, in _emit_charm_event
2024-02-27T23:53:09.795Z [container-agent]     event_to_emit.emit(*args, **kwargs)
2024-02-27T23:53:09.795Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 351, in emit
2024-02-27T23:53:09.795Z [container-agent]     framework._emit(event)
2024-02-27T23:53:09.795Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 853, in _emit
2024-02-27T23:53:09.795Z [container-agent]     self._reemit(event_path)
2024-02-27T23:53:09.795Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 942, in _reemit
2024-02-27T23:53:09.795Z [container-agent]     custom_handler(event)
2024-02-27T23:53:09.795Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/rotate_mysql_logs.py", line 56, in _rotate_mysql_logs
2024-02-27T23:53:09.795Z [container-agent]     self.charm._mysql.flush_mysql_logs(MySQLTextLogs.GENERAL)
2024-02-27T23:53:09.795Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-02-27T23:53:09.795Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-02-27T23:53:09.795Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 666, in _run_mysqlsh_script
2024-02-27T23:53:09.795Z [container-agent]     stdout, _ = process.wait_output()
2024-02-27T23:53:09.795Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1436, in wait_output
2024-02-27T23:53:09.795Z [container-agent]     exit_code: int = self._wait()
2024-02-27T23:53:09.795Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1375, in _wait
2024-02-27T23:53:09.795Z [container-agent]     change = self._client.wait_change(self._change_id, timeout=timeout)
2024-02-27T23:53:09.795Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1893, in wait_change
2024-02-27T23:53:09.795Z [container-agent]     return self._wait_change_using_wait(change_id, timeout)
2024-02-27T23:53:09.795Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1914, in _wait_change_using_wait
2024-02-27T23:53:09.795Z [container-agent]     return self._wait_change(change_id, this_timeout)
2024-02-27T23:53:09.795Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1928, in _wait_change
2024-02-27T23:53:09.795Z [container-agent]     resp = self._request('GET', f'/v1/changes/{change_id}/wait', query)
2024-02-27T23:53:09.795Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1655, in _request
2024-02-27T23:53:09.795Z [container-agent]     response = self._request_raw(method, path, query, headers, data)
2024-02-27T23:53:09.795Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1690, in _request_raw
2024-02-27T23:53:09.795Z [container-agent]     response = self.opener.open(request, timeout=self.timeout)
2024-02-27T23:53:09.795Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 519, in open
2024-02-27T23:53:09.795Z [container-agent]     response = self._open(req, data)
2024-02-27T23:53:09.795Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 536, in _open
2024-02-27T23:53:09.795Z [container-agent]     result = self._call_chain(self.handle_open, protocol, protocol +
2024-02-27T23:53:09.795Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 496, in _call_chain
2024-02-27T23:53:09.795Z [container-agent]     result = func(*args)
2024-02-27T23:53:09.795Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 311, in http_open
2024-02-27T23:53:09.795Z [container-agent]     return self.do_open(_UnixSocketConnection, req,  # type:ignore
2024-02-27T23:53:09.795Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 1352, in do_open
2024-02-27T23:53:09.795Z [container-agent]     r = h.getresponse()
2024-02-27T23:53:09.795Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1375, in getresponse
2024-02-27T23:53:09.795Z [container-agent]     response.begin()
2024-02-27T23:53:09.795Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 318, in begin
2024-02-27T23:53:09.795Z [container-agent]     version, status, reason = self._read_status()
2024-02-27T23:53:09.795Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 287, in _read_status
2024-02-27T23:53:09.795Z [container-agent]     raise RemoteDisconnected("Remote end closed connection without"
2024-02-27T23:53:09.795Z [container-agent] http.client.RemoteDisconnected: Remote end closed connection without response
2024-02-27T23:53:09.920Z [container-agent] 2024-02-27 23:53:09 INFO juju.util.exec exec.go:209 run result: exit status 1
2024-02-27T23:53:13.689Z [container-agent] 2024-02-27 23:53:13 INFO juju-log Setting up the logrotate configurations
2024-02-27T23:53:13.803Z [container-agent] 2024-02-27 23:53:13 INFO juju-log Adding pebble layer
2024-02-27T23:53:20.536Z [container-agent] 2024-02-27 23:53:20 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-02-27T23:53:20.581Z [container-agent] 2024-02-27 23:53:20 INFO juju-log Attempting reboot from complete outage.
2024-02-27T23:53:23.354Z [container-agent] 2024-02-27 23:53:23 INFO juju.worker.uniter.operation runhook.go:186 ran "mysql-pebble-ready" hook (via hook dispatching script: dispatch)
2024-02-27T23:58:08.619Z [container-agent] 2024-02-27 23:58:08 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-27T23:58:10.495Z [container-agent] 2024-02-27 23:58:10 WARNING juju-log No relation: certificates
2024-02-27T23:58:10.851Z [container-agent] 2024-02-27 23:58:10 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T00:03:51.299Z [container-agent] 2024-02-28 00:03:51 INFO juju-log Starting the log rotate manager
2024-02-28T00:03:51.312Z [container-agent] 2024-02-28 00:03:51 INFO juju-log Started log rotate manager process with PID 73994
2024-02-28T00:03:51.590Z [container-agent] 2024-02-28 00:03:51 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T00:03:53.588Z [container-agent] 2024-02-28 00:03:53 WARNING juju-log No relation: certificates
2024-02-28T00:03:53.927Z [container-agent] 2024-02-28 00:03:53 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T00:09:05.857Z [container-agent] 2024-02-28 00:09:05 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T00:09:08.375Z [container-agent] 2024-02-28 00:09:08 WARNING juju-log No relation: certificates
2024-02-28T00:09:08.714Z [container-agent] 2024-02-28 00:09:08 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T00:14:56.703Z [container-agent] 2024-02-28 00:14:56 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T00:14:59.852Z [container-agent] 2024-02-28 00:14:59 WARNING juju-log No relation: certificates
2024-02-28T00:15:00.200Z [container-agent] 2024-02-28 00:15:00 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T00:20:33.783Z [container-agent] 2024-02-28 00:20:33 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T00:20:37.202Z [container-agent] 2024-02-28 00:20:37 WARNING juju-log No relation: certificates
2024-02-28T00:20:37.572Z [container-agent] 2024-02-28 00:20:37 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T00:25:31.178Z [container-agent] 2024-02-28 00:25:31 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T00:25:35.327Z [container-agent] 2024-02-28 00:25:35 WARNING juju-log No relation: certificates
2024-02-28T00:25:35.670Z [container-agent] 2024-02-28 00:25:35 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T00:30:52.207Z [container-agent] 2024-02-28 00:30:52 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T00:30:56.625Z [container-agent] 2024-02-28 00:30:56 WARNING juju-log No relation: certificates
2024-02-28T00:30:56.971Z [container-agent] 2024-02-28 00:30:56 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T00:35:28.865Z [container-agent] 2024-02-28 00:35:28 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T00:35:33.966Z [container-agent] 2024-02-28 00:35:33 WARNING juju-log No relation: certificates
2024-02-28T00:35:34.304Z [container-agent] 2024-02-28 00:35:34 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T00:40:20.931Z [container-agent] 2024-02-28 00:40:20 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T00:40:25.878Z [container-agent] 2024-02-28 00:40:25 WARNING juju-log No relation: certificates
2024-02-28T00:40:26.216Z [container-agent] 2024-02-28 00:40:26 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T00:46:11.173Z [container-agent] 2024-02-28 00:46:11 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T00:46:17.299Z [container-agent] 2024-02-28 00:46:17 WARNING juju-log No relation: certificates
2024-02-28T00:46:17.650Z [container-agent] 2024-02-28 00:46:17 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T00:51:11.919Z [container-agent] 2024-02-28 00:51:11 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T00:51:18.371Z [container-agent] 2024-02-28 00:51:18 WARNING juju-log No relation: certificates
2024-02-28T00:51:18.713Z [container-agent] 2024-02-28 00:51:18 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T00:56:30.398Z [container-agent] 2024-02-28 00:56:30 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T00:56:37.281Z [container-agent] 2024-02-28 00:56:37 WARNING juju-log No relation: certificates
2024-02-28T00:56:37.650Z [container-agent] 2024-02-28 00:56:37 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T01:00:55.658Z [container-agent] 2024-02-28 01:00:55 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T01:01:02.483Z [container-agent] 2024-02-28 01:01:02 WARNING juju-log No relation: certificates
2024-02-28T01:01:02.866Z [container-agent] 2024-02-28 01:01:02 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T01:06:14.130Z [container-agent] 2024-02-28 01:06:14 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-02-28T01:06:14.168Z [container-agent] 2024-02-28 01:06:14 INFO juju-log Attempting reboot from complete outage.
2024-02-28T01:06:21.812Z [container-agent] 2024-02-28 01:06:21 WARNING juju-log No relation: certificates
2024-02-28T01:06:22.197Z [container-agent] 2024-02-28 01:06:22 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T01:08:04.922Z [container-agent] 2024-02-28 01:08:04 ERROR juju-log Uncaught exception while in charm code:
2024-02-28T01:08:04.922Z [container-agent] Traceback (most recent call last):
2024-02-28T01:08:04.922Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/./src/charm.py", line 786, in <module>
2024-02-28T01:08:04.922Z [container-agent]     main(MySQLOperatorCharm)
2024-02-28T01:08:04.922Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 436, in main
2024-02-28T01:08:04.922Z [container-agent]     _emit_charm_event(charm, dispatcher.event_name)
2024-02-28T01:08:04.922Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 144, in _emit_charm_event
2024-02-28T01:08:04.922Z [container-agent]     event_to_emit.emit(*args, **kwargs)
2024-02-28T01:08:04.922Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 351, in emit
2024-02-28T01:08:04.922Z [container-agent]     framework._emit(event)
2024-02-28T01:08:04.922Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 853, in _emit
2024-02-28T01:08:04.922Z [container-agent]     self._reemit(event_path)
2024-02-28T01:08:04.922Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 942, in _reemit
2024-02-28T01:08:04.922Z [container-agent]     custom_handler(event)
2024-02-28T01:08:04.922Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/rotate_mysql_logs.py", line 55, in _rotate_mysql_logs
2024-02-28T01:08:04.922Z [container-agent]     self.charm._mysql.flush_mysql_logs(MySQLTextLogs.ERROR)
2024-02-28T01:08:04.922Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-02-28T01:08:04.922Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-02-28T01:08:04.922Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 665, in _run_mysqlsh_script
2024-02-28T01:08:04.922Z [container-agent]     process = self.container.exec(cmd, timeout=timeout)
2024-02-28T01:08:04.922Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/model.py", line 2688, in exec
2024-02-28T01:08:04.922Z [container-agent]     return self._pebble.exec(
2024-02-28T01:08:04.922Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 2501, in exec
2024-02-28T01:08:04.922Z [container-agent]     resp = self._request('POST', '/v1/exec', body=body)
2024-02-28T01:08:04.922Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1655, in _request
2024-02-28T01:08:04.922Z [container-agent]     response = self._request_raw(method, path, query, headers, data)
2024-02-28T01:08:04.922Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1690, in _request_raw
2024-02-28T01:08:04.922Z [container-agent]     response = self.opener.open(request, timeout=self.timeout)
2024-02-28T01:08:04.922Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 519, in open
2024-02-28T01:08:04.922Z [container-agent]     response = self._open(req, data)
2024-02-28T01:08:04.922Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 536, in _open
2024-02-28T01:08:04.922Z [container-agent]     result = self._call_chain(self.handle_open, protocol, protocol +
2024-02-28T01:08:04.922Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 496, in _call_chain
2024-02-28T01:08:04.922Z [container-agent]     result = func(*args)
2024-02-28T01:08:04.922Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 311, in http_open
2024-02-28T01:08:04.922Z [container-agent]     return self.do_open(_UnixSocketConnection, req,  # type:ignore
2024-02-28T01:08:04.922Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 1352, in do_open
2024-02-28T01:08:04.922Z [container-agent]     r = h.getresponse()
2024-02-28T01:08:04.922Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1375, in getresponse
2024-02-28T01:08:04.922Z [container-agent]     response.begin()
2024-02-28T01:08:04.922Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 318, in begin
2024-02-28T01:08:04.922Z [container-agent]     version, status, reason = self._read_status()
2024-02-28T01:08:04.922Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 287, in _read_status
2024-02-28T01:08:04.922Z [container-agent]     raise RemoteDisconnected("Remote end closed connection without"
2024-02-28T01:08:04.922Z [container-agent] http.client.RemoteDisconnected: Remote end closed connection without response
2024-02-28T01:08:05.060Z [container-agent] 2024-02-28 01:08:05 INFO juju.util.exec exec.go:209 run result: exit status 1
2024-02-28T01:08:06.942Z [container-agent] 2024-02-28 01:08:06 INFO juju-log Setting up the logrotate configurations
2024-02-28T01:08:07.008Z [container-agent] 2024-02-28 01:08:07 INFO juju-log Adding pebble layer
2024-02-28T01:08:13.759Z [container-agent] 2024-02-28 01:08:13 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-02-28T01:08:13.779Z [container-agent] 2024-02-28 01:08:13 INFO juju-log Attempting reboot from complete outage.
2024-02-28T01:08:16.839Z [container-agent] 2024-02-28 01:08:16 INFO juju.worker.uniter.operation runhook.go:186 ran "mysql-pebble-ready" hook (via hook dispatching script: dispatch)
2024-02-28T01:11:04.853Z [container-agent] 2024-02-28 01:11:04 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T01:11:06.823Z [container-agent] 2024-02-28 01:11:06 WARNING juju-log No relation: certificates
2024-02-28T01:11:07.228Z [container-agent] 2024-02-28 01:11:07 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T01:15:52.057Z [container-agent] 2024-02-28 01:15:52 INFO juju-log Starting the log rotate manager
2024-02-28T01:15:52.071Z [container-agent] 2024-02-28 01:15:52 INFO juju-log Started log rotate manager process with PID 80627
2024-02-28T01:15:52.340Z [container-agent] 2024-02-28 01:15:52 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T01:15:54.279Z [container-agent] 2024-02-28 01:15:54 WARNING juju-log No relation: certificates
2024-02-28T01:15:54.638Z [container-agent] 2024-02-28 01:15:54 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T01:20:08.295Z [container-agent] 2024-02-28 01:20:08 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T01:20:10.746Z [container-agent] 2024-02-28 01:20:10 WARNING juju-log No relation: certificates
2024-02-28T01:20:11.086Z [container-agent] 2024-02-28 01:20:11 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T01:25:01.714Z [container-agent] 2024-02-28 01:25:01 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T01:25:04.569Z [container-agent] 2024-02-28 01:25:04 WARNING juju-log No relation: certificates
2024-02-28T01:25:04.898Z [container-agent] 2024-02-28 01:25:04 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T01:30:30.195Z [container-agent] 2024-02-28 01:30:30 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T01:30:33.819Z [container-agent] 2024-02-28 01:30:33 WARNING juju-log No relation: certificates
2024-02-28T01:30:34.180Z [container-agent] 2024-02-28 01:30:34 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T01:35:17.111Z [container-agent] 2024-02-28 01:35:17 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T01:35:20.946Z [container-agent] 2024-02-28 01:35:20 WARNING juju-log No relation: certificates
2024-02-28T01:35:21.280Z [container-agent] 2024-02-28 01:35:21 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T01:40:45.428Z [container-agent] 2024-02-28 01:40:45 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T01:40:49.918Z [container-agent] 2024-02-28 01:40:49 WARNING juju-log No relation: certificates
2024-02-28T01:40:50.267Z [container-agent] 2024-02-28 01:40:50 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T01:46:43.138Z [container-agent] 2024-02-28 01:46:43 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T01:46:48.176Z [container-agent] 2024-02-28 01:46:48 WARNING juju-log No relation: certificates
2024-02-28T01:46:48.520Z [container-agent] 2024-02-28 01:46:48 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T01:52:13.422Z [container-agent] 2024-02-28 01:52:13 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T01:52:19.170Z [container-agent] 2024-02-28 01:52:19 WARNING juju-log No relation: certificates
2024-02-28T01:52:19.520Z [container-agent] 2024-02-28 01:52:19 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T01:56:38.526Z [container-agent] 2024-02-28 01:56:38 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T01:56:44.293Z [container-agent] 2024-02-28 01:56:44 WARNING juju-log No relation: certificates
2024-02-28T01:56:44.648Z [container-agent] 2024-02-28 01:56:44 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T02:01:11.644Z [container-agent] 2024-02-28 02:01:11 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T02:01:18.059Z [container-agent] 2024-02-28 02:01:18 WARNING juju-log No relation: certificates
2024-02-28T02:01:18.407Z [container-agent] 2024-02-28 02:01:18 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T02:05:56.429Z [container-agent] 2024-02-28 02:05:56 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T02:06:02.797Z [container-agent] 2024-02-28 02:06:02 WARNING juju-log No relation: certificates
2024-02-28T02:06:03.201Z [container-agent] 2024-02-28 02:06:03 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T02:11:21.164Z [container-agent] 2024-02-28 02:11:21 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T02:11:28.430Z [container-agent] 2024-02-28 02:11:28 WARNING juju-log No relation: certificates
2024-02-28T02:11:28.771Z [container-agent] 2024-02-28 02:11:28 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T02:16:42.510Z [container-agent] 2024-02-28 02:16:42 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T02:16:50.436Z [container-agent] 2024-02-28 02:16:50 WARNING juju-log No relation: certificates
2024-02-28T02:16:50.823Z [container-agent] 2024-02-28 02:16:50 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T02:20:49.111Z [container-agent] 2024-02-28 02:20:49 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-02-28T02:20:49.141Z [container-agent] 2024-02-28 02:20:49 INFO juju-log Attempting reboot from complete outage.
2024-02-28T02:20:56.600Z [container-agent] 2024-02-28 02:20:56 WARNING juju-log No relation: certificates
2024-02-28T02:20:56.952Z [container-agent] 2024-02-28 02:20:56 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T02:24:04.754Z [container-agent] 2024-02-28 02:24:04 ERROR juju-log Uncaught exception while in charm code:
2024-02-28T02:24:04.754Z [container-agent] Traceback (most recent call last):
2024-02-28T02:24:04.754Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 2507, in exec
2024-02-28T02:24:04.754Z [container-agent]     control_ws = self._connect_websocket(task_id, 'control')
2024-02-28T02:24:04.754Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 2589, in _connect_websocket
2024-02-28T02:24:04.754Z [container-agent]     ws.connect(url, socket=sock)
2024-02-28T02:24:04.754Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_core.py", line 253, in connect
2024-02-28T02:24:04.754Z [container-agent]     self.handshake_response = handshake(self.sock, url, *addrs, **options)
2024-02-28T02:24:04.754Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_handshake.py", line 57, in handshake
2024-02-28T02:24:04.754Z [container-agent]     status, resp = _get_resp_headers(sock)
2024-02-28T02:24:04.754Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_handshake.py", line 143, in _get_resp_headers
2024-02-28T02:24:04.754Z [container-agent]     status, resp_headers, status_message = read_headers(sock)
2024-02-28T02:24:04.754Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_http.py", line 312, in read_headers
2024-02-28T02:24:04.754Z [container-agent]     line = recv_line(sock)
2024-02-28T02:24:04.754Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_socket.py", line 131, in recv_line
2024-02-28T02:24:04.754Z [container-agent]     c = recv(sock, 1)
2024-02-28T02:24:04.754Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_socket.py", line 122, in recv
2024-02-28T02:24:04.754Z [container-agent]     raise WebSocketConnectionClosedException(
2024-02-28T02:24:04.754Z [container-agent] websocket._exceptions.WebSocketConnectionClosedException: Connection to remote host was lost.
2024-02-28T02:24:04.754Z [container-agent] 
2024-02-28T02:24:04.754Z [container-agent] During handling of the above exception, another exception occurred:
2024-02-28T02:24:04.754Z [container-agent] 
2024-02-28T02:24:04.754Z [container-agent] Traceback (most recent call last):
2024-02-28T02:24:04.754Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 1348, in do_open
2024-02-28T02:24:04.754Z [container-agent]     h.request(req.get_method(), req.selector, req.data, headers,
2024-02-28T02:24:04.754Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1283, in request
2024-02-28T02:24:04.754Z [container-agent]     self._send_request(method, url, body, headers, encode_chunked)
2024-02-28T02:24:04.754Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1329, in _send_request
2024-02-28T02:24:04.754Z [container-agent]     self.endheaders(body, encode_chunked=encode_chunked)
2024-02-28T02:24:04.754Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1278, in endheaders
2024-02-28T02:24:04.754Z [container-agent]     self._send_output(message_body, encode_chunked=encode_chunked)
2024-02-28T02:24:04.754Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1038, in _send_output
2024-02-28T02:24:04.754Z [container-agent]     self.send(msg)
2024-02-28T02:24:04.754Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 976, in send
2024-02-28T02:24:04.754Z [container-agent]     self.connect()
2024-02-28T02:24:04.754Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 297, in connect
2024-02-28T02:24:04.754Z [container-agent]     self.sock.connect(self.socket_path)
2024-02-28T02:24:04.754Z [container-agent] ConnectionRefusedError: [Errno 111] Connection refused
2024-02-28T02:24:04.754Z [container-agent] 
2024-02-28T02:24:04.754Z [container-agent] During handling of the above exception, another exception occurred:
2024-02-28T02:24:04.754Z [container-agent] 
2024-02-28T02:24:04.754Z [container-agent] Traceback (most recent call last):
2024-02-28T02:24:04.754Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1690, in _request_raw
2024-02-28T02:24:04.754Z [container-agent]     response = self.opener.open(request, timeout=self.timeout)
2024-02-28T02:24:04.754Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 519, in open
2024-02-28T02:24:04.754Z [container-agent]     response = self._open(req, data)
2024-02-28T02:24:04.754Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 536, in _open
2024-02-28T02:24:04.754Z [container-agent]     result = self._call_chain(self.handle_open, protocol, protocol +
2024-02-28T02:24:04.754Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 496, in _call_chain
2024-02-28T02:24:04.754Z [container-agent]     result = func(*args)
2024-02-28T02:24:04.754Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 311, in http_open
2024-02-28T02:24:04.754Z [container-agent]     return self.do_open(_UnixSocketConnection, req,  # type:ignore
2024-02-28T02:24:04.754Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 1351, in do_open
2024-02-28T02:24:04.754Z [container-agent]     raise URLError(err)
2024-02-28T02:24:04.754Z [container-agent] urllib.error.URLError: <urlopen error [Errno 111] Connection refused>
2024-02-28T02:24:04.754Z [container-agent] 
2024-02-28T02:24:04.754Z [container-agent] The above exception was the direct cause of the following exception:
2024-02-28T02:24:04.754Z [container-agent] 
2024-02-28T02:24:04.754Z [container-agent] Traceback (most recent call last):
2024-02-28T02:24:04.754Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/./src/charm.py", line 786, in <module>
2024-02-28T02:24:04.754Z [container-agent]     main(MySQLOperatorCharm)
2024-02-28T02:24:04.754Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 436, in main
2024-02-28T02:24:04.754Z [container-agent]     _emit_charm_event(charm, dispatcher.event_name)
2024-02-28T02:24:04.754Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 144, in _emit_charm_event
2024-02-28T02:24:04.754Z [container-agent]     event_to_emit.emit(*args, **kwargs)
2024-02-28T02:24:04.754Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 351, in emit
2024-02-28T02:24:04.754Z [container-agent]     framework._emit(event)
2024-02-28T02:24:04.754Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 853, in _emit
2024-02-28T02:24:04.754Z [container-agent]     self._reemit(event_path)
2024-02-28T02:24:04.754Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 942, in _reemit
2024-02-28T02:24:04.754Z [container-agent]     custom_handler(event)
2024-02-28T02:24:04.754Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/rotate_mysql_logs.py", line 55, in _rotate_mysql_logs
2024-02-28T02:24:04.754Z [container-agent]     self.charm._mysql.flush_mysql_logs(MySQLTextLogs.ERROR)
2024-02-28T02:24:04.754Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-02-28T02:24:04.754Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-02-28T02:24:04.754Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 665, in _run_mysqlsh_script
2024-02-28T02:24:04.754Z [container-agent]     process = self.container.exec(cmd, timeout=timeout)
2024-02-28T02:24:04.754Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/model.py", line 2688, in exec
2024-02-28T02:24:04.754Z [container-agent]     return self._pebble.exec(
2024-02-28T02:24:04.754Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 2514, in exec
2024-02-28T02:24:04.754Z [container-agent]     change = self.wait_change(ChangeID(change_id))
2024-02-28T02:24:04.754Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1893, in wait_change
2024-02-28T02:24:04.754Z [container-agent]     return self._wait_change_using_wait(change_id, timeout)
2024-02-28T02:24:04.754Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1914, in _wait_change_using_wait
2024-02-28T02:24:04.754Z [container-agent]     return self._wait_change(change_id, this_timeout)
2024-02-28T02:24:04.754Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1928, in _wait_change
2024-02-28T02:24:04.754Z [container-agent]     resp = self._request('GET', f'/v1/changes/{change_id}/wait', query)
2024-02-28T02:24:04.754Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1655, in _request
2024-02-28T02:24:04.754Z [container-agent]     response = self._request_raw(method, path, query, headers, data)
2024-02-28T02:24:04.754Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1707, in _request_raw
2024-02-28T02:24:04.754Z [container-agent]     raise ConnectionError(e.reason) from e
2024-02-28T02:24:04.754Z [container-agent] ops.pebble.ConnectionError: [Errno 111] Connection refused
2024-02-28T02:24:04.936Z [container-agent] 2024-02-28 02:24:04 INFO juju.util.exec exec.go:209 run result: exit status 1
2024-02-28T02:24:10.277Z [container-agent] 2024-02-28 02:24:10 INFO juju-log Setting up the logrotate configurations
2024-02-28T02:24:10.346Z [container-agent] 2024-02-28 02:24:10 INFO juju-log Adding pebble layer
2024-02-28T02:24:17.099Z [container-agent] 2024-02-28 02:24:17 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-02-28T02:24:17.123Z [container-agent] 2024-02-28 02:24:17 INFO juju-log Attempting reboot from complete outage.
2024-02-28T02:24:20.141Z [container-agent] 2024-02-28 02:24:20 INFO juju.worker.uniter.operation runhook.go:186 ran "mysql-pebble-ready" hook (via hook dispatching script: dispatch)
2024-02-28T02:25:11.616Z [container-agent] 2024-02-28 02:25:11 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T02:25:13.676Z [container-agent] 2024-02-28 02:25:13 WARNING juju-log No relation: certificates
2024-02-28T02:25:14.027Z [container-agent] 2024-02-28 02:25:14 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T02:29:18.173Z [container-agent] 2024-02-28 02:29:18 INFO juju-log Starting the log rotate manager
2024-02-28T02:29:18.186Z [container-agent] 2024-02-28 02:29:18 INFO juju-log Started log rotate manager process with PID 87695
2024-02-28T02:29:18.454Z [container-agent] 2024-02-28 02:29:18 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T02:29:20.584Z [container-agent] 2024-02-28 02:29:20 WARNING juju-log No relation: certificates
2024-02-28T02:29:20.923Z [container-agent] 2024-02-28 02:29:20 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T02:33:20.199Z [container-agent] 2024-02-28 02:33:20 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T02:33:22.505Z [container-agent] 2024-02-28 02:33:22 WARNING juju-log No relation: certificates
2024-02-28T02:33:22.852Z [container-agent] 2024-02-28 02:33:22 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T02:38:44.474Z [container-agent] 2024-02-28 02:38:44 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T02:38:47.468Z [container-agent] 2024-02-28 02:38:47 WARNING juju-log No relation: certificates
2024-02-28T02:38:47.814Z [container-agent] 2024-02-28 02:38:47 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T02:43:33.242Z [container-agent] 2024-02-28 02:43:33 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T02:43:36.608Z [container-agent] 2024-02-28 02:43:36 WARNING juju-log No relation: certificates
2024-02-28T02:43:36.954Z [container-agent] 2024-02-28 02:43:36 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T02:48:43.399Z [container-agent] 2024-02-28 02:48:43 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T02:48:47.459Z [container-agent] 2024-02-28 02:48:47 WARNING juju-log No relation: certificates
2024-02-28T02:48:47.843Z [container-agent] 2024-02-28 02:48:47 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T02:54:07.992Z [container-agent] 2024-02-28 02:54:07 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T02:54:12.271Z [container-agent] 2024-02-28 02:54:12 WARNING juju-log No relation: certificates
2024-02-28T02:54:12.609Z [container-agent] 2024-02-28 02:54:12 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T02:59:00.761Z [container-agent] 2024-02-28 02:59:00 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T02:59:05.363Z [container-agent] 2024-02-28 02:59:05 WARNING juju-log No relation: certificates
2024-02-28T02:59:05.747Z [container-agent] 2024-02-28 02:59:05 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T03:03:12.263Z [container-agent] 2024-02-28 03:03:12 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T03:03:17.478Z [container-agent] 2024-02-28 03:03:17 WARNING juju-log No relation: certificates
2024-02-28T03:03:17.828Z [container-agent] 2024-02-28 03:03:17 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T03:08:34.956Z [container-agent] 2024-02-28 03:08:34 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T03:08:40.640Z [container-agent] 2024-02-28 03:08:40 WARNING juju-log No relation: certificates
2024-02-28T03:08:41.011Z [container-agent] 2024-02-28 03:08:41 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T03:13:00.224Z [container-agent] 2024-02-28 03:13:00 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T03:13:05.869Z [container-agent] 2024-02-28 03:13:05 WARNING juju-log No relation: certificates
2024-02-28T03:13:06.245Z [container-agent] 2024-02-28 03:13:06 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T03:17:54.246Z [container-agent] 2024-02-28 03:17:54 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T03:18:00.468Z [container-agent] 2024-02-28 03:18:00 WARNING juju-log No relation: certificates
2024-02-28T03:18:00.811Z [container-agent] 2024-02-28 03:18:00 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T03:23:03.156Z [container-agent] 2024-02-28 03:23:03 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T03:23:09.519Z [container-agent] 2024-02-28 03:23:09 WARNING juju-log No relation: certificates
2024-02-28T03:23:09.861Z [container-agent] 2024-02-28 03:23:09 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T03:28:02.877Z [container-agent] 2024-02-28 03:28:02 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T03:28:09.550Z [container-agent] 2024-02-28 03:28:09 WARNING juju-log No relation: certificates
2024-02-28T03:28:09.899Z [container-agent] 2024-02-28 03:28:09 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T03:32:34.748Z [container-agent] 2024-02-28 03:32:34 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T03:32:42.477Z [container-agent] 2024-02-28 03:32:42 WARNING juju-log No relation: certificates
2024-02-28T03:32:42.821Z [container-agent] 2024-02-28 03:32:42 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T03:37:12.760Z [container-agent] 2024-02-28 03:37:12 ERROR juju-log Failed to flush SLOW LOGS logs.
2024-02-28T03:37:12.760Z [container-agent] Traceback (most recent call last):
2024-02-28T03:37:12.760Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 666, in _run_mysqlsh_script
2024-02-28T03:37:12.760Z [container-agent]     stdout, _ = process.wait_output()
2024-02-28T03:37:12.760Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1441, in wait_output
2024-02-28T03:37:12.760Z [container-agent]     raise ExecError[AnyStr](self._command, exit_code, out_value, err_value)
2024-02-28T03:37:12.760Z [container-agent] ops.pebble.ExecError: non-zero exit code 1 executing ['/usr/bin/mysqlsh', '--no-wizard', '--python', '--verbose=1', '-f', '/tmp/script.py', ';', 'rm', '/tmp/script.py'], stdout='', stderr='Cannot set LC_ALL to locale en_US.UTF-8: No such file or directory\nverbose: 2024-02-28T03:37:11Z: Loading startup files...\nverbose: 2024-02-28T03:37:11Z: Loading plugins...\nverbose: 2024-02-28T03:37:11Z: Connecting to MySQL at: serverconfig@horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local\nTraceback (most recent call last):\n  File "<string>", line 1, in <module>\nmysqlsh.DBError: MySQL Error (2003): Shell.connect: Can\'t connect to MySQL server on \'horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local:3306\' (111)\n'
2024-02-28T03:37:12.760Z [container-agent] 
2024-02-28T03:37:12.760Z [container-agent] During handling of the above exception, another exception occurred:
2024-02-28T03:37:12.760Z [container-agent] 
2024-02-28T03:37:12.760Z [container-agent] Traceback (most recent call last):
2024-02-28T03:37:12.760Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-02-28T03:37:12.760Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-02-28T03:37:12.760Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 669, in _run_mysqlsh_script
2024-02-28T03:37:12.760Z [container-agent]     raise MySQLClientError(e.stderr)
2024-02-28T03:37:12.760Z [container-agent] charms.mysql.v0.mysql.MySQLClientError: Cannot set LC_ALL to locale en_US.UTF-8: No such file or directory
2024-02-28T03:37:12.760Z [container-agent] verbose: 2024-02-28T03:37:11Z: Loading startup files...
2024-02-28T03:37:12.760Z [container-agent] verbose: 2024-02-28T03:37:11Z: Loading plugins...
2024-02-28T03:37:12.760Z [container-agent] verbose: 2024-02-28T03:37:11Z: Connecting to MySQL at: serverconfig@horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local
2024-02-28T03:37:12.760Z [container-agent] Traceback (most recent call last):
2024-02-28T03:37:12.760Z [container-agent]   File "<string>", line 1, in <module>
2024-02-28T03:37:12.760Z [container-agent] mysqlsh.DBError: MySQL Error (2003): Shell.connect: Can't connect to MySQL server on 'horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local:3306' (111)
2024-02-28T03:37:12.760Z [container-agent] 
2024-02-28T03:38:15.323Z [container-agent] 2024-02-28 03:38:15 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-02-28T03:38:15.378Z [container-agent] 2024-02-28 03:38:15 INFO juju-log Attempting reboot from complete outage.
2024-02-28T03:38:23.136Z [container-agent] 2024-02-28 03:38:23 WARNING juju-log No relation: certificates
2024-02-28T03:38:23.477Z [container-agent] 2024-02-28 03:38:23 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T03:42:09.796Z [container-agent] 2024-02-28 03:42:09 ERROR juju-log Uncaught exception while in charm code:
2024-02-28T03:42:09.796Z [container-agent] Traceback (most recent call last):
2024-02-28T03:42:09.796Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/./src/charm.py", line 786, in <module>
2024-02-28T03:42:09.796Z [container-agent]     main(MySQLOperatorCharm)
2024-02-28T03:42:09.796Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 436, in main
2024-02-28T03:42:09.796Z [container-agent]     _emit_charm_event(charm, dispatcher.event_name)
2024-02-28T03:42:09.796Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 144, in _emit_charm_event
2024-02-28T03:42:09.796Z [container-agent]     event_to_emit.emit(*args, **kwargs)
2024-02-28T03:42:09.796Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 351, in emit
2024-02-28T03:42:09.796Z [container-agent]     framework._emit(event)
2024-02-28T03:42:09.796Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 853, in _emit
2024-02-28T03:42:09.796Z [container-agent]     self._reemit(event_path)
2024-02-28T03:42:09.796Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 942, in _reemit
2024-02-28T03:42:09.796Z [container-agent]     custom_handler(event)
2024-02-28T03:42:09.796Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/rotate_mysql_logs.py", line 57, in _rotate_mysql_logs
2024-02-28T03:42:09.796Z [container-agent]     self.charm._mysql.flush_mysql_logs(MySQLTextLogs.SLOW)
2024-02-28T03:42:09.796Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-02-28T03:42:09.796Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-02-28T03:42:09.796Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 665, in _run_mysqlsh_script
2024-02-28T03:42:09.796Z [container-agent]     process = self.container.exec(cmd, timeout=timeout)
2024-02-28T03:42:09.796Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/model.py", line 2688, in exec
2024-02-28T03:42:09.796Z [container-agent]     return self._pebble.exec(
2024-02-28T03:42:09.796Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 2501, in exec
2024-02-28T03:42:09.796Z [container-agent]     resp = self._request('POST', '/v1/exec', body=body)
2024-02-28T03:42:09.796Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1655, in _request
2024-02-28T03:42:09.796Z [container-agent]     response = self._request_raw(method, path, query, headers, data)
2024-02-28T03:42:09.796Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1690, in _request_raw
2024-02-28T03:42:09.796Z [container-agent]     response = self.opener.open(request, timeout=self.timeout)
2024-02-28T03:42:09.796Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 519, in open
2024-02-28T03:42:09.796Z [container-agent]     response = self._open(req, data)
2024-02-28T03:42:09.796Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 536, in _open
2024-02-28T03:42:09.796Z [container-agent]     result = self._call_chain(self.handle_open, protocol, protocol +
2024-02-28T03:42:09.796Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 496, in _call_chain
2024-02-28T03:42:09.796Z [container-agent]     result = func(*args)
2024-02-28T03:42:09.796Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 311, in http_open
2024-02-28T03:42:09.796Z [container-agent]     return self.do_open(_UnixSocketConnection, req,  # type:ignore
2024-02-28T03:42:09.796Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 1352, in do_open
2024-02-28T03:42:09.796Z [container-agent]     r = h.getresponse()
2024-02-28T03:42:09.796Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1375, in getresponse
2024-02-28T03:42:09.796Z [container-agent]     response.begin()
2024-02-28T03:42:09.796Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 318, in begin
2024-02-28T03:42:09.796Z [container-agent]     version, status, reason = self._read_status()
2024-02-28T03:42:09.796Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 287, in _read_status
2024-02-28T03:42:09.796Z [container-agent]     raise RemoteDisconnected("Remote end closed connection without"
2024-02-28T03:42:09.796Z [container-agent] http.client.RemoteDisconnected: Remote end closed connection without response
2024-02-28T03:42:09.899Z [container-agent] 2024-02-28 03:42:09 INFO juju.util.exec exec.go:209 run result: exit status 1
2024-02-28T03:42:13.161Z [container-agent] 2024-02-28 03:42:13 INFO juju-log Setting up the logrotate configurations
2024-02-28T03:42:13.225Z [container-agent] 2024-02-28 03:42:13 INFO juju-log Adding pebble layer
2024-02-28T03:42:19.975Z [container-agent] 2024-02-28 03:42:19 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-02-28T03:42:19.999Z [container-agent] 2024-02-28 03:42:19 INFO juju-log Attempting reboot from complete outage.
2024-02-28T03:42:22.916Z [container-agent] 2024-02-28 03:42:22 INFO juju.worker.uniter.operation runhook.go:186 ran "mysql-pebble-ready" hook (via hook dispatching script: dispatch)
2024-02-28T03:43:21.350Z [container-agent] 2024-02-28 03:43:21 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T03:43:23.130Z [container-agent] 2024-02-28 03:43:23 WARNING juju-log No relation: certificates
2024-02-28T03:43:23.486Z [container-agent] 2024-02-28 03:43:23 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T03:48:47.511Z [container-agent] 2024-02-28 03:48:47 INFO juju-log Starting the log rotate manager
2024-02-28T03:48:47.526Z [container-agent] 2024-02-28 03:48:47 INFO juju-log Started log rotate manager process with PID 95182
2024-02-28T03:48:47.824Z [container-agent] 2024-02-28 03:48:47 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T03:48:49.872Z [container-agent] 2024-02-28 03:48:49 WARNING juju-log No relation: certificates
2024-02-28T03:48:50.258Z [container-agent] 2024-02-28 03:48:50 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T03:54:07.607Z [container-agent] 2024-02-28 03:54:07 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T03:54:10.257Z [container-agent] 2024-02-28 03:54:10 WARNING juju-log No relation: certificates
2024-02-28T03:54:10.707Z [container-agent] 2024-02-28 03:54:10 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T03:58:30.692Z [container-agent] 2024-02-28 03:58:30 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T03:58:33.751Z [container-agent] 2024-02-28 03:58:33 WARNING juju-log No relation: certificates
2024-02-28T03:58:34.163Z [container-agent] 2024-02-28 03:58:34 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T04:02:31.653Z [container-agent] 2024-02-28 04:02:31 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T04:02:35.115Z [container-agent] 2024-02-28 04:02:35 WARNING juju-log No relation: certificates
2024-02-28T04:02:35.456Z [container-agent] 2024-02-28 04:02:35 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T04:08:09.985Z [container-agent] 2024-02-28 04:08:09 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T04:08:13.632Z [container-agent] 2024-02-28 04:08:13 WARNING juju-log No relation: certificates
2024-02-28T04:08:13.993Z [container-agent] 2024-02-28 04:08:13 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T04:13:34.709Z [container-agent] 2024-02-28 04:13:34 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T04:13:39.210Z [container-agent] 2024-02-28 04:13:39 WARNING juju-log No relation: certificates
2024-02-28T04:13:39.564Z [container-agent] 2024-02-28 04:13:39 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T04:18:55.942Z [container-agent] 2024-02-28 04:18:55 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T04:19:00.668Z [container-agent] 2024-02-28 04:19:00 WARNING juju-log No relation: certificates
2024-02-28T04:19:01.031Z [container-agent] 2024-02-28 04:19:01 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T04:24:15.827Z [container-agent] 2024-02-28 04:24:15 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T04:24:21.374Z [container-agent] 2024-02-28 04:24:21 WARNING juju-log No relation: certificates
2024-02-28T04:24:21.738Z [container-agent] 2024-02-28 04:24:21 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T04:28:51.754Z [container-agent] 2024-02-28 04:28:51 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T04:28:57.943Z [container-agent] 2024-02-28 04:28:57 WARNING juju-log No relation: certificates
2024-02-28T04:28:58.292Z [container-agent] 2024-02-28 04:28:58 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T04:34:25.502Z [container-agent] 2024-02-28 04:34:25 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T04:34:31.410Z [container-agent] 2024-02-28 04:34:31 WARNING juju-log No relation: certificates
2024-02-28T04:34:31.753Z [container-agent] 2024-02-28 04:34:31 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T04:40:11.838Z [container-agent] 2024-02-28 04:40:11 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T04:40:18.587Z [container-agent] 2024-02-28 04:40:18 WARNING juju-log No relation: certificates
2024-02-28T04:40:18.953Z [container-agent] 2024-02-28 04:40:18 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T04:45:54.366Z [container-agent] 2024-02-28 04:45:54 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T04:46:01.222Z [container-agent] 2024-02-28 04:46:01 WARNING juju-log No relation: certificates
2024-02-28T04:46:01.601Z [container-agent] 2024-02-28 04:46:01 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T04:50:02.194Z [container-agent] 2024-02-28 04:50:02 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T04:50:09.138Z [container-agent] 2024-02-28 04:50:09 WARNING juju-log No relation: certificates
2024-02-28T04:50:09.551Z [container-agent] 2024-02-28 04:50:09 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T04:54:55.289Z [container-agent] 2024-02-28 04:54:55 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T04:55:02.874Z [container-agent] 2024-02-28 04:55:02 WARNING juju-log No relation: certificates
2024-02-28T04:55:03.251Z [container-agent] 2024-02-28 04:55:03 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T04:59:54.650Z [container-agent] 2024-02-28 04:59:54 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T05:00:02.477Z [container-agent] 2024-02-28 05:00:02 WARNING juju-log No relation: certificates
2024-02-28T05:00:02.816Z [container-agent] 2024-02-28 05:00:02 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T05:01:07.690Z [container-agent] 2024-02-28 05:01:07 ERROR juju-log Failed to flush ERROR LOGS logs.
2024-02-28T05:01:07.690Z [container-agent] Traceback (most recent call last):
2024-02-28T05:01:07.690Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 666, in _run_mysqlsh_script
2024-02-28T05:01:07.690Z [container-agent]     stdout, _ = process.wait_output()
2024-02-28T05:01:07.690Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1441, in wait_output
2024-02-28T05:01:07.690Z [container-agent]     raise ExecError[AnyStr](self._command, exit_code, out_value, err_value)
2024-02-28T05:01:07.690Z [container-agent] ops.pebble.ExecError: non-zero exit code 1 executing ['/usr/bin/mysqlsh', '--no-wizard', '--python', '--verbose=1', '-f', '/tmp/script.py', ';', 'rm', '/tmp/script.py'], stdout='', stderr='Cannot set LC_ALL to locale en_US.UTF-8: No such file or directory\nverbose: 2024-02-28T05:01:06Z: Loading startup files...\nverbose: 2024-02-28T05:01:06Z: Loading plugins...\nverbose: 2024-02-28T05:01:06Z: Connecting to MySQL at: serverconfig@horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local\nTraceback (most recent call last):\n  File "<string>", line 1, in <module>\nmysqlsh.DBError: MySQL Error (2003): Shell.connect: Can\'t connect to MySQL server on \'horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local:3306\' (111)\n'
2024-02-28T05:01:07.690Z [container-agent] 
2024-02-28T05:01:07.690Z [container-agent] During handling of the above exception, another exception occurred:
2024-02-28T05:01:07.690Z [container-agent] 
2024-02-28T05:01:07.690Z [container-agent] Traceback (most recent call last):
2024-02-28T05:01:07.690Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-02-28T05:01:07.690Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-02-28T05:01:07.690Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 669, in _run_mysqlsh_script
2024-02-28T05:01:07.690Z [container-agent]     raise MySQLClientError(e.stderr)
2024-02-28T05:01:07.690Z [container-agent] charms.mysql.v0.mysql.MySQLClientError: Cannot set LC_ALL to locale en_US.UTF-8: No such file or directory
2024-02-28T05:01:07.690Z [container-agent] verbose: 2024-02-28T05:01:06Z: Loading startup files...
2024-02-28T05:01:07.690Z [container-agent] verbose: 2024-02-28T05:01:06Z: Loading plugins...
2024-02-28T05:01:07.690Z [container-agent] verbose: 2024-02-28T05:01:06Z: Connecting to MySQL at: serverconfig@horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local
2024-02-28T05:01:07.690Z [container-agent] Traceback (most recent call last):
2024-02-28T05:01:07.690Z [container-agent]   File "<string>", line 1, in <module>
2024-02-28T05:01:07.690Z [container-agent] mysqlsh.DBError: MySQL Error (2003): Shell.connect: Can't connect to MySQL server on 'horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local:3306' (111)
2024-02-28T05:01:07.690Z [container-agent] 
2024-02-28T05:05:16.275Z [container-agent] 2024-02-28 05:05:16 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-02-28T05:05:16.298Z [container-agent] 2024-02-28 05:05:16 INFO juju-log Attempting reboot from complete outage.
2024-02-28T05:05:23.908Z [container-agent] 2024-02-28 05:05:23 WARNING juju-log No relation: certificates
2024-02-28T05:05:24.297Z [container-agent] 2024-02-28 05:05:24 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T05:06:04.797Z [container-agent] 2024-02-28 05:06:04 ERROR juju-log Uncaught exception while in charm code:
2024-02-28T05:06:04.797Z [container-agent] Traceback (most recent call last):
2024-02-28T05:06:04.797Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/./src/charm.py", line 786, in <module>
2024-02-28T05:06:04.797Z [container-agent]     main(MySQLOperatorCharm)
2024-02-28T05:06:04.797Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 436, in main
2024-02-28T05:06:04.797Z [container-agent]     _emit_charm_event(charm, dispatcher.event_name)
2024-02-28T05:06:04.797Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 144, in _emit_charm_event
2024-02-28T05:06:04.797Z [container-agent]     event_to_emit.emit(*args, **kwargs)
2024-02-28T05:06:04.797Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 351, in emit
2024-02-28T05:06:04.797Z [container-agent]     framework._emit(event)
2024-02-28T05:06:04.797Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 853, in _emit
2024-02-28T05:06:04.797Z [container-agent]     self._reemit(event_path)
2024-02-28T05:06:04.797Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 942, in _reemit
2024-02-28T05:06:04.797Z [container-agent]     custom_handler(event)
2024-02-28T05:06:04.797Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/rotate_mysql_logs.py", line 55, in _rotate_mysql_logs
2024-02-28T05:06:04.797Z [container-agent]     self.charm._mysql.flush_mysql_logs(MySQLTextLogs.ERROR)
2024-02-28T05:06:04.797Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-02-28T05:06:04.797Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-02-28T05:06:04.797Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 665, in _run_mysqlsh_script
2024-02-28T05:06:04.797Z [container-agent]     process = self.container.exec(cmd, timeout=timeout)
2024-02-28T05:06:04.797Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/model.py", line 2688, in exec
2024-02-28T05:06:04.797Z [container-agent]     return self._pebble.exec(
2024-02-28T05:06:04.797Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 2501, in exec
2024-02-28T05:06:04.797Z [container-agent]     resp = self._request('POST', '/v1/exec', body=body)
2024-02-28T05:06:04.797Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1655, in _request
2024-02-28T05:06:04.797Z [container-agent]     response = self._request_raw(method, path, query, headers, data)
2024-02-28T05:06:04.797Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1690, in _request_raw
2024-02-28T05:06:04.797Z [container-agent]     response = self.opener.open(request, timeout=self.timeout)
2024-02-28T05:06:04.797Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 519, in open
2024-02-28T05:06:04.797Z [container-agent]     response = self._open(req, data)
2024-02-28T05:06:04.798Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 536, in _open
2024-02-28T05:06:04.798Z [container-agent]     result = self._call_chain(self.handle_open, protocol, protocol +
2024-02-28T05:06:04.798Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 496, in _call_chain
2024-02-28T05:06:04.798Z [container-agent]     result = func(*args)
2024-02-28T05:06:04.798Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 311, in http_open
2024-02-28T05:06:04.798Z [container-agent]     return self.do_open(_UnixSocketConnection, req,  # type:ignore
2024-02-28T05:06:04.798Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 1352, in do_open
2024-02-28T05:06:04.798Z [container-agent]     r = h.getresponse()
2024-02-28T05:06:04.798Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1375, in getresponse
2024-02-28T05:06:04.798Z [container-agent]     response.begin()
2024-02-28T05:06:04.798Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 318, in begin
2024-02-28T05:06:04.798Z [container-agent]     version, status, reason = self._read_status()
2024-02-28T05:06:04.798Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 287, in _read_status
2024-02-28T05:06:04.798Z [container-agent]     raise RemoteDisconnected("Remote end closed connection without"
2024-02-28T05:06:04.798Z [container-agent] http.client.RemoteDisconnected: Remote end closed connection without response
2024-02-28T05:06:04.972Z [container-agent] 2024-02-28 05:06:04 INFO juju.util.exec exec.go:209 run result: exit status 1
2024-02-28T05:06:07.045Z [container-agent] 2024-02-28 05:06:07 INFO juju-log Setting up the logrotate configurations
2024-02-28T05:06:07.102Z [container-agent] 2024-02-28 05:06:07 INFO juju-log Adding pebble layer
2024-02-28T05:06:13.845Z [container-agent] 2024-02-28 05:06:13 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-02-28T05:06:13.869Z [container-agent] 2024-02-28 05:06:13 INFO juju-log Attempting reboot from complete outage.
2024-02-28T05:06:16.953Z [container-agent] 2024-02-28 05:06:16 INFO juju.worker.uniter.operation runhook.go:186 ran "mysql-pebble-ready" hook (via hook dispatching script: dispatch)
2024-02-28T05:10:16.763Z [container-agent] 2024-02-28 05:10:16 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T05:10:18.872Z [container-agent] 2024-02-28 05:10:18 WARNING juju-log No relation: certificates
2024-02-28T05:10:19.256Z [container-agent] 2024-02-28 05:10:19 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T05:16:05.219Z [container-agent] 2024-02-28 05:16:05 INFO juju-log Starting the log rotate manager
2024-02-28T05:16:05.233Z [container-agent] 2024-02-28 05:16:05 INFO juju-log Started log rotate manager process with PID 103135
2024-02-28T05:16:05.586Z [container-agent] 2024-02-28 05:16:05 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T05:16:07.593Z [container-agent] 2024-02-28 05:16:07 WARNING juju-log No relation: certificates
2024-02-28T05:16:07.944Z [container-agent] 2024-02-28 05:16:07 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T05:20:35.307Z [container-agent] 2024-02-28 05:20:35 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T05:20:37.668Z [container-agent] 2024-02-28 05:20:37 WARNING juju-log No relation: certificates
2024-02-28T05:20:38.024Z [container-agent] 2024-02-28 05:20:38 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T05:25:54.069Z [container-agent] 2024-02-28 05:25:54 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T05:25:56.950Z [container-agent] 2024-02-28 05:25:56 WARNING juju-log No relation: certificates
2024-02-28T05:25:57.286Z [container-agent] 2024-02-28 05:25:57 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T05:30:41.764Z [container-agent] 2024-02-28 05:30:41 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T05:30:45.259Z [container-agent] 2024-02-28 05:30:45 WARNING juju-log No relation: certificates
2024-02-28T05:30:45.622Z [container-agent] 2024-02-28 05:30:45 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T05:35:29.745Z [container-agent] 2024-02-28 05:35:29 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T05:35:33.759Z [container-agent] 2024-02-28 05:35:33 WARNING juju-log No relation: certificates
2024-02-28T05:35:34.103Z [container-agent] 2024-02-28 05:35:34 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T05:39:47.291Z [container-agent] 2024-02-28 05:39:47 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T05:39:51.681Z [container-agent] 2024-02-28 05:39:51 WARNING juju-log No relation: certificates
2024-02-28T05:39:52.023Z [container-agent] 2024-02-28 05:39:52 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T05:44:33.137Z [container-agent] 2024-02-28 05:44:33 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T05:44:37.967Z [container-agent] 2024-02-28 05:44:37 WARNING juju-log No relation: certificates
2024-02-28T05:44:38.311Z [container-agent] 2024-02-28 05:44:38 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T05:49:11.822Z [container-agent] 2024-02-28 05:49:11 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T05:49:16.866Z [container-agent] 2024-02-28 05:49:16 WARNING juju-log No relation: certificates
2024-02-28T05:49:17.208Z [container-agent] 2024-02-28 05:49:17 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T05:54:00.940Z [container-agent] 2024-02-28 05:54:00 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T05:54:06.574Z [container-agent] 2024-02-28 05:54:06 WARNING juju-log No relation: certificates
2024-02-28T05:54:06.944Z [container-agent] 2024-02-28 05:54:06 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T05:59:33.382Z [container-agent] 2024-02-28 05:59:33 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T05:59:39.430Z [container-agent] 2024-02-28 05:59:39 WARNING juju-log No relation: certificates
2024-02-28T05:59:39.780Z [container-agent] 2024-02-28 05:59:39 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T06:05:26.907Z [container-agent] 2024-02-28 06:05:26 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T06:05:33.674Z [container-agent] 2024-02-28 06:05:33 WARNING juju-log No relation: certificates
2024-02-28T06:05:34.043Z [container-agent] 2024-02-28 06:05:34 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T06:10:12.510Z [container-agent] 2024-02-28 06:10:12 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T06:10:19.885Z [container-agent] 2024-02-28 06:10:19 WARNING juju-log No relation: certificates
2024-02-28T06:10:20.250Z [container-agent] 2024-02-28 06:10:20 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T06:14:24.574Z [container-agent] 2024-02-28 06:14:24 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T06:14:32.030Z [container-agent] 2024-02-28 06:14:32 WARNING juju-log No relation: certificates
2024-02-28T06:14:32.369Z [container-agent] 2024-02-28 06:14:32 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T06:17:07.833Z [container-agent] 2024-02-28 06:17:07 ERROR juju-log Failed to flush ERROR LOGS logs.
2024-02-28T06:17:07.833Z [container-agent] Traceback (most recent call last):
2024-02-28T06:17:07.833Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 666, in _run_mysqlsh_script
2024-02-28T06:17:07.833Z [container-agent]     stdout, _ = process.wait_output()
2024-02-28T06:17:07.833Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1441, in wait_output
2024-02-28T06:17:07.833Z [container-agent]     raise ExecError[AnyStr](self._command, exit_code, out_value, err_value)
2024-02-28T06:17:07.833Z [container-agent] ops.pebble.ExecError: non-zero exit code 1 executing ['/usr/bin/mysqlsh', '--no-wizard', '--python', '--verbose=1', '-f', '/tmp/script.py', ';', 'rm', '/tmp/script.py'], stdout='', stderr='Cannot set LC_ALL to locale en_US.UTF-8: No such file or directory\nverbose: 2024-02-28T06:17:06Z: Loading startup files...\nverbose: 2024-02-28T06:17:06Z: Loading plugins...\nverbose: 2024-02-28T06:17:06Z: Connecting to MySQL at: serverconfig@horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local\nTraceback (most recent call last):\n  File "<string>", line 1, in <module>\nmysqlsh.DBError: MySQL Error (2003): Shell.connect: Can\'t connect to MySQL server on \'horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local:3306\' (111)\n'
2024-02-28T06:17:07.833Z [container-agent] 
2024-02-28T06:17:07.833Z [container-agent] During handling of the above exception, another exception occurred:
2024-02-28T06:17:07.833Z [container-agent] 
2024-02-28T06:17:07.833Z [container-agent] Traceback (most recent call last):
2024-02-28T06:17:07.833Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-02-28T06:17:07.833Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-02-28T06:17:07.833Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 669, in _run_mysqlsh_script
2024-02-28T06:17:07.833Z [container-agent]     raise MySQLClientError(e.stderr)
2024-02-28T06:17:07.833Z [container-agent] charms.mysql.v0.mysql.MySQLClientError: Cannot set LC_ALL to locale en_US.UTF-8: No such file or directory
2024-02-28T06:17:07.833Z [container-agent] verbose: 2024-02-28T06:17:06Z: Loading startup files...
2024-02-28T06:17:07.833Z [container-agent] verbose: 2024-02-28T06:17:06Z: Loading plugins...
2024-02-28T06:17:07.833Z [container-agent] verbose: 2024-02-28T06:17:06Z: Connecting to MySQL at: serverconfig@horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local
2024-02-28T06:17:07.833Z [container-agent] Traceback (most recent call last):
2024-02-28T06:17:07.833Z [container-agent]   File "<string>", line 1, in <module>
2024-02-28T06:17:07.834Z [container-agent] mysqlsh.DBError: MySQL Error (2003): Shell.connect: Can't connect to MySQL server on 'horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local:3306' (111)
2024-02-28T06:17:07.834Z [container-agent] 
2024-02-28T06:19:56.831Z [container-agent] 2024-02-28 06:19:56 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-02-28T06:19:56.863Z [container-agent] 2024-02-28 06:19:56 INFO juju-log Attempting reboot from complete outage.
2024-02-28T06:20:03.907Z [container-agent] 2024-02-28 06:20:03 WARNING juju-log No relation: certificates
2024-02-28T06:20:04.247Z [container-agent] 2024-02-28 06:20:04 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T06:22:04.810Z [container-agent] 2024-02-28 06:22:04 ERROR juju-log Uncaught exception while in charm code:
2024-02-28T06:22:04.810Z [container-agent] Traceback (most recent call last):
2024-02-28T06:22:04.810Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/./src/charm.py", line 786, in <module>
2024-02-28T06:22:04.810Z [container-agent]     main(MySQLOperatorCharm)
2024-02-28T06:22:04.810Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 436, in main
2024-02-28T06:22:04.810Z [container-agent]     _emit_charm_event(charm, dispatcher.event_name)
2024-02-28T06:22:04.810Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 144, in _emit_charm_event
2024-02-28T06:22:04.810Z [container-agent]     event_to_emit.emit(*args, **kwargs)
2024-02-28T06:22:04.810Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 351, in emit
2024-02-28T06:22:04.810Z [container-agent]     framework._emit(event)
2024-02-28T06:22:04.810Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 853, in _emit
2024-02-28T06:22:04.810Z [container-agent]     self._reemit(event_path)
2024-02-28T06:22:04.810Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 942, in _reemit
2024-02-28T06:22:04.810Z [container-agent]     custom_handler(event)
2024-02-28T06:22:04.810Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/rotate_mysql_logs.py", line 55, in _rotate_mysql_logs
2024-02-28T06:22:04.810Z [container-agent]     self.charm._mysql.flush_mysql_logs(MySQLTextLogs.ERROR)
2024-02-28T06:22:04.810Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-02-28T06:22:04.810Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-02-28T06:22:04.810Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 666, in _run_mysqlsh_script
2024-02-28T06:22:04.810Z [container-agent]     stdout, _ = process.wait_output()
2024-02-28T06:22:04.810Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1436, in wait_output
2024-02-28T06:22:04.810Z [container-agent]     exit_code: int = self._wait()
2024-02-28T06:22:04.810Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1375, in _wait
2024-02-28T06:22:04.810Z [container-agent]     change = self._client.wait_change(self._change_id, timeout=timeout)
2024-02-28T06:22:04.810Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1893, in wait_change
2024-02-28T06:22:04.810Z [container-agent]     return self._wait_change_using_wait(change_id, timeout)
2024-02-28T06:22:04.810Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1914, in _wait_change_using_wait
2024-02-28T06:22:04.810Z [container-agent]     return self._wait_change(change_id, this_timeout)
2024-02-28T06:22:04.810Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1928, in _wait_change
2024-02-28T06:22:04.810Z [container-agent]     resp = self._request('GET', f'/v1/changes/{change_id}/wait', query)
2024-02-28T06:22:04.810Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1655, in _request
2024-02-28T06:22:04.810Z [container-agent]     response = self._request_raw(method, path, query, headers, data)
2024-02-28T06:22:04.810Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1690, in _request_raw
2024-02-28T06:22:04.810Z [container-agent]     response = self.opener.open(request, timeout=self.timeout)
2024-02-28T06:22:04.810Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 519, in open
2024-02-28T06:22:04.810Z [container-agent]     response = self._open(req, data)
2024-02-28T06:22:04.810Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 536, in _open
2024-02-28T06:22:04.810Z [container-agent]     result = self._call_chain(self.handle_open, protocol, protocol +
2024-02-28T06:22:04.810Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 496, in _call_chain
2024-02-28T06:22:04.810Z [container-agent]     result = func(*args)
2024-02-28T06:22:04.810Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 311, in http_open
2024-02-28T06:22:04.810Z [container-agent]     return self.do_open(_UnixSocketConnection, req,  # type:ignore
2024-02-28T06:22:04.810Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 1352, in do_open
2024-02-28T06:22:04.810Z [container-agent]     r = h.getresponse()
2024-02-28T06:22:04.810Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1375, in getresponse
2024-02-28T06:22:04.810Z [container-agent]     response.begin()
2024-02-28T06:22:04.810Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 318, in begin
2024-02-28T06:22:04.810Z [container-agent]     version, status, reason = self._read_status()
2024-02-28T06:22:04.810Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 287, in _read_status
2024-02-28T06:22:04.810Z [container-agent]     raise RemoteDisconnected("Remote end closed connection without"
2024-02-28T06:22:04.810Z [container-agent] http.client.RemoteDisconnected: Remote end closed connection without response
2024-02-28T06:22:04.971Z [container-agent] 2024-02-28 06:22:04 INFO juju.util.exec exec.go:209 run result: exit status 1
2024-02-28T06:22:10.167Z [container-agent] 2024-02-28 06:22:10 INFO juju-log Setting up the logrotate configurations
2024-02-28T06:22:10.234Z [container-agent] 2024-02-28 06:22:10 INFO juju-log Adding pebble layer
2024-02-28T06:22:17.007Z [container-agent] 2024-02-28 06:22:17 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-02-28T06:22:17.035Z [container-agent] 2024-02-28 06:22:17 INFO juju-log Attempting reboot from complete outage.
2024-02-28T06:22:20.127Z [container-agent] 2024-02-28 06:22:20 INFO juju.worker.uniter.operation runhook.go:186 ran "mysql-pebble-ready" hook (via hook dispatching script: dispatch)
2024-02-28T06:23:57.295Z [container-agent] 2024-02-28 06:23:57 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T06:23:59.171Z [container-agent] 2024-02-28 06:23:59 WARNING juju-log No relation: certificates
2024-02-28T06:23:59.532Z [container-agent] 2024-02-28 06:23:59 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T06:28:05.474Z [container-agent] 2024-02-28 06:28:05 INFO juju-log Starting the log rotate manager
2024-02-28T06:28:05.491Z [container-agent] 2024-02-28 06:28:05 INFO juju-log Started log rotate manager process with PID 110045
2024-02-28T06:28:05.869Z [container-agent] 2024-02-28 06:28:05 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T06:28:07.872Z [container-agent] 2024-02-28 06:28:07 WARNING juju-log No relation: certificates
2024-02-28T06:28:08.207Z [container-agent] 2024-02-28 06:28:08 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T06:33:42.040Z [container-agent] 2024-02-28 06:33:42 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T06:33:44.708Z [container-agent] 2024-02-28 06:33:44 WARNING juju-log No relation: certificates
2024-02-28T06:33:45.074Z [container-agent] 2024-02-28 06:33:45 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T06:38:21.072Z [container-agent] 2024-02-28 06:38:21 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T06:38:23.839Z [container-agent] 2024-02-28 06:38:23 WARNING juju-log No relation: certificates
2024-02-28T06:38:24.183Z [container-agent] 2024-02-28 06:38:24 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T06:44:16.592Z [container-agent] 2024-02-28 06:44:16 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T06:44:20.323Z [container-agent] 2024-02-28 06:44:20 WARNING juju-log No relation: certificates
2024-02-28T06:44:20.665Z [container-agent] 2024-02-28 06:44:20 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T06:50:08.151Z [container-agent] 2024-02-28 06:50:08 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T06:50:12.114Z [container-agent] 2024-02-28 06:50:12 WARNING juju-log No relation: certificates
2024-02-28T06:50:12.521Z [container-agent] 2024-02-28 06:50:12 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T06:55:08.493Z [container-agent] 2024-02-28 06:55:08 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T06:55:12.641Z [container-agent] 2024-02-28 06:55:12 WARNING juju-log No relation: certificates
2024-02-28T06:55:12.976Z [container-agent] 2024-02-28 06:55:12 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T07:00:09.176Z [container-agent] 2024-02-28 07:00:09 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T07:00:13.889Z [container-agent] 2024-02-28 07:00:13 WARNING juju-log No relation: certificates
2024-02-28T07:00:14.227Z [container-agent] 2024-02-28 07:00:14 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T07:04:31.006Z [container-agent] 2024-02-28 07:04:31 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T07:04:36.387Z [container-agent] 2024-02-28 07:04:36 WARNING juju-log No relation: certificates
2024-02-28T07:04:36.824Z [container-agent] 2024-02-28 07:04:36 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T07:09:50.575Z [container-agent] 2024-02-28 07:09:50 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T07:09:56.058Z [container-agent] 2024-02-28 07:09:56 WARNING juju-log No relation: certificates
2024-02-28T07:09:56.384Z [container-agent] 2024-02-28 07:09:56 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T07:14:16.438Z [container-agent] 2024-02-28 07:14:16 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T07:14:22.658Z [container-agent] 2024-02-28 07:14:22 WARNING juju-log No relation: certificates
2024-02-28T07:14:23.045Z [container-agent] 2024-02-28 07:14:23 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T07:19:12.593Z [container-agent] 2024-02-28 07:19:12 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T07:19:19.540Z [container-agent] 2024-02-28 07:19:19 WARNING juju-log No relation: certificates
2024-02-28T07:19:19.927Z [container-agent] 2024-02-28 07:19:19 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T07:23:31.666Z [container-agent] 2024-02-28 07:23:31 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T07:23:38.828Z [container-agent] 2024-02-28 07:23:38 WARNING juju-log No relation: certificates
2024-02-28T07:23:39.216Z [container-agent] 2024-02-28 07:23:39 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T07:28:52.574Z [container-agent] 2024-02-28 07:28:52 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-02-28T07:28:52.596Z [container-agent] 2024-02-28 07:28:52 INFO juju-log Attempting reboot from complete outage.
2024-02-28T07:28:59.589Z [container-agent] 2024-02-28 07:28:59 WARNING juju-log No relation: certificates
2024-02-28T07:28:59.986Z [container-agent] 2024-02-28 07:28:59 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T07:32:04.793Z [container-agent] 2024-02-28 07:32:04 ERROR juju-log Uncaught exception while in charm code:
2024-02-28T07:32:04.793Z [container-agent] Traceback (most recent call last):
2024-02-28T07:32:04.793Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 2507, in exec
2024-02-28T07:32:04.793Z [container-agent]     control_ws = self._connect_websocket(task_id, 'control')
2024-02-28T07:32:04.793Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 2589, in _connect_websocket
2024-02-28T07:32:04.793Z [container-agent]     ws.connect(url, socket=sock)
2024-02-28T07:32:04.793Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_core.py", line 253, in connect
2024-02-28T07:32:04.793Z [container-agent]     self.handshake_response = handshake(self.sock, url, *addrs, **options)
2024-02-28T07:32:04.793Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_handshake.py", line 57, in handshake
2024-02-28T07:32:04.793Z [container-agent]     status, resp = _get_resp_headers(sock)
2024-02-28T07:32:04.793Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_handshake.py", line 143, in _get_resp_headers
2024-02-28T07:32:04.793Z [container-agent]     status, resp_headers, status_message = read_headers(sock)
2024-02-28T07:32:04.793Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_http.py", line 312, in read_headers
2024-02-28T07:32:04.793Z [container-agent]     line = recv_line(sock)
2024-02-28T07:32:04.793Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_socket.py", line 131, in recv_line
2024-02-28T07:32:04.793Z [container-agent]     c = recv(sock, 1)
2024-02-28T07:32:04.793Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_socket.py", line 122, in recv
2024-02-28T07:32:04.793Z [container-agent]     raise WebSocketConnectionClosedException(
2024-02-28T07:32:04.793Z [container-agent] websocket._exceptions.WebSocketConnectionClosedException: Connection to remote host was lost.
2024-02-28T07:32:04.793Z [container-agent] 
2024-02-28T07:32:04.793Z [container-agent] During handling of the above exception, another exception occurred:
2024-02-28T07:32:04.793Z [container-agent] 
2024-02-28T07:32:04.793Z [container-agent] Traceback (most recent call last):
2024-02-28T07:32:04.793Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 1348, in do_open
2024-02-28T07:32:04.793Z [container-agent]     h.request(req.get_method(), req.selector, req.data, headers,
2024-02-28T07:32:04.793Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1283, in request
2024-02-28T07:32:04.793Z [container-agent]     self._send_request(method, url, body, headers, encode_chunked)
2024-02-28T07:32:04.793Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1329, in _send_request
2024-02-28T07:32:04.793Z [container-agent]     self.endheaders(body, encode_chunked=encode_chunked)
2024-02-28T07:32:04.793Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1278, in endheaders
2024-02-28T07:32:04.793Z [container-agent]     self._send_output(message_body, encode_chunked=encode_chunked)
2024-02-28T07:32:04.793Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1038, in _send_output
2024-02-28T07:32:04.793Z [container-agent]     self.send(msg)
2024-02-28T07:32:04.793Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 976, in send
2024-02-28T07:32:04.793Z [container-agent]     self.connect()
2024-02-28T07:32:04.793Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 297, in connect
2024-02-28T07:32:04.793Z [container-agent]     self.sock.connect(self.socket_path)
2024-02-28T07:32:04.793Z [container-agent] ConnectionRefusedError: [Errno 111] Connection refused
2024-02-28T07:32:04.793Z [container-agent] 
2024-02-28T07:32:04.793Z [container-agent] During handling of the above exception, another exception occurred:
2024-02-28T07:32:04.793Z [container-agent] 
2024-02-28T07:32:04.793Z [container-agent] Traceback (most recent call last):
2024-02-28T07:32:04.793Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1690, in _request_raw
2024-02-28T07:32:04.793Z [container-agent]     response = self.opener.open(request, timeout=self.timeout)
2024-02-28T07:32:04.793Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 519, in open
2024-02-28T07:32:04.793Z [container-agent]     response = self._open(req, data)
2024-02-28T07:32:04.793Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 536, in _open
2024-02-28T07:32:04.793Z [container-agent]     result = self._call_chain(self.handle_open, protocol, protocol +
2024-02-28T07:32:04.793Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 496, in _call_chain
2024-02-28T07:32:04.793Z [container-agent]     result = func(*args)
2024-02-28T07:32:04.793Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 311, in http_open
2024-02-28T07:32:04.793Z [container-agent]     return self.do_open(_UnixSocketConnection, req,  # type:ignore
2024-02-28T07:32:04.793Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 1351, in do_open
2024-02-28T07:32:04.793Z [container-agent]     raise URLError(err)
2024-02-28T07:32:04.793Z [container-agent] urllib.error.URLError: <urlopen error [Errno 111] Connection refused>
2024-02-28T07:32:04.793Z [container-agent] 
2024-02-28T07:32:04.793Z [container-agent] The above exception was the direct cause of the following exception:
2024-02-28T07:32:04.793Z [container-agent] 
2024-02-28T07:32:04.793Z [container-agent] Traceback (most recent call last):
2024-02-28T07:32:04.793Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/./src/charm.py", line 786, in <module>
2024-02-28T07:32:04.793Z [container-agent]     main(MySQLOperatorCharm)
2024-02-28T07:32:04.793Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 436, in main
2024-02-28T07:32:04.793Z [container-agent]     _emit_charm_event(charm, dispatcher.event_name)
2024-02-28T07:32:04.793Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 144, in _emit_charm_event
2024-02-28T07:32:04.793Z [container-agent]     event_to_emit.emit(*args, **kwargs)
2024-02-28T07:32:04.793Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 351, in emit
2024-02-28T07:32:04.793Z [container-agent]     framework._emit(event)
2024-02-28T07:32:04.793Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 853, in _emit
2024-02-28T07:32:04.793Z [container-agent]     self._reemit(event_path)
2024-02-28T07:32:04.793Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 942, in _reemit
2024-02-28T07:32:04.793Z [container-agent]     custom_handler(event)
2024-02-28T07:32:04.793Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/rotate_mysql_logs.py", line 55, in _rotate_mysql_logs
2024-02-28T07:32:04.793Z [container-agent]     self.charm._mysql.flush_mysql_logs(MySQLTextLogs.ERROR)
2024-02-28T07:32:04.793Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-02-28T07:32:04.793Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-02-28T07:32:04.793Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 665, in _run_mysqlsh_script
2024-02-28T07:32:04.793Z [container-agent]     process = self.container.exec(cmd, timeout=timeout)
2024-02-28T07:32:04.793Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/model.py", line 2688, in exec
2024-02-28T07:32:04.793Z [container-agent]     return self._pebble.exec(
2024-02-28T07:32:04.793Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 2514, in exec
2024-02-28T07:32:04.793Z [container-agent]     change = self.wait_change(ChangeID(change_id))
2024-02-28T07:32:04.793Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1893, in wait_change
2024-02-28T07:32:04.793Z [container-agent]     return self._wait_change_using_wait(change_id, timeout)
2024-02-28T07:32:04.793Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1914, in _wait_change_using_wait
2024-02-28T07:32:04.793Z [container-agent]     return self._wait_change(change_id, this_timeout)
2024-02-28T07:32:04.793Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1928, in _wait_change
2024-02-28T07:32:04.793Z [container-agent]     resp = self._request('GET', f'/v1/changes/{change_id}/wait', query)
2024-02-28T07:32:04.793Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1655, in _request
2024-02-28T07:32:04.793Z [container-agent]     response = self._request_raw(method, path, query, headers, data)
2024-02-28T07:32:04.793Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1707, in _request_raw
2024-02-28T07:32:04.793Z [container-agent]     raise ConnectionError(e.reason) from e
2024-02-28T07:32:04.793Z [container-agent] ops.pebble.ConnectionError: [Errno 111] Connection refused
2024-02-28T07:32:04.958Z [container-agent] 2024-02-28 07:32:04 INFO juju.util.exec exec.go:209 run result: exit status 1
2024-02-28T07:32:07.880Z [container-agent] 2024-02-28 07:32:07 INFO juju-log Setting up the logrotate configurations
2024-02-28T07:32:07.952Z [container-agent] 2024-02-28 07:32:07 INFO juju-log Adding pebble layer
2024-02-28T07:32:14.702Z [container-agent] 2024-02-28 07:32:14 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-02-28T07:32:14.723Z [container-agent] 2024-02-28 07:32:14 INFO juju-log Attempting reboot from complete outage.
2024-02-28T07:32:17.770Z [container-agent] 2024-02-28 07:32:17 INFO juju.worker.uniter.operation runhook.go:186 ran "mysql-pebble-ready" hook (via hook dispatching script: dispatch)
2024-02-28T07:34:25.893Z [container-agent] 2024-02-28 07:34:25 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T07:34:27.671Z [container-agent] 2024-02-28 07:34:27 WARNING juju-log No relation: certificates
2024-02-28T07:34:28.055Z [container-agent] 2024-02-28 07:34:28 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T07:40:12.381Z [container-agent] 2024-02-28 07:40:12 INFO juju-log Starting the log rotate manager
2024-02-28T07:40:12.396Z [container-agent] 2024-02-28 07:40:12 INFO juju-log Started log rotate manager process with PID 116634
2024-02-28T07:40:12.685Z [container-agent] 2024-02-28 07:40:12 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T07:40:14.723Z [container-agent] 2024-02-28 07:40:14 WARNING juju-log No relation: certificates
2024-02-28T07:40:15.072Z [container-agent] 2024-02-28 07:40:15 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T07:44:51.788Z [container-agent] 2024-02-28 07:44:51 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T07:44:54.059Z [container-agent] 2024-02-28 07:44:54 WARNING juju-log No relation: certificates
2024-02-28T07:44:54.417Z [container-agent] 2024-02-28 07:44:54 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T07:49:06.273Z [container-agent] 2024-02-28 07:49:06 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T07:49:09.025Z [container-agent] 2024-02-28 07:49:09 WARNING juju-log No relation: certificates
2024-02-28T07:49:09.375Z [container-agent] 2024-02-28 07:49:09 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T07:54:32.128Z [container-agent] 2024-02-28 07:54:32 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T07:54:35.565Z [container-agent] 2024-02-28 07:54:35 WARNING juju-log No relation: certificates
2024-02-28T07:54:35.905Z [container-agent] 2024-02-28 07:54:35 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T08:00:07.530Z [container-agent] 2024-02-28 08:00:07 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T08:00:11.071Z [container-agent] 2024-02-28 08:00:11 WARNING juju-log No relation: certificates
2024-02-28T08:00:11.419Z [container-agent] 2024-02-28 08:00:11 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T08:05:08.813Z [container-agent] 2024-02-28 08:05:08 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T08:05:12.793Z [container-agent] 2024-02-28 08:05:12 WARNING juju-log No relation: certificates
2024-02-28T08:05:13.135Z [container-agent] 2024-02-28 08:05:13 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T08:10:31.148Z [container-agent] 2024-02-28 08:10:31 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T08:10:35.820Z [container-agent] 2024-02-28 08:10:35 WARNING juju-log No relation: certificates
2024-02-28T08:10:36.180Z [container-agent] 2024-02-28 08:10:36 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T08:14:42.939Z [container-agent] 2024-02-28 08:14:42 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T08:14:48.436Z [container-agent] 2024-02-28 08:14:48 WARNING juju-log No relation: certificates
2024-02-28T08:14:48.796Z [container-agent] 2024-02-28 08:14:48 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T08:20:14.063Z [container-agent] 2024-02-28 08:20:14 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T08:20:19.916Z [container-agent] 2024-02-28 08:20:19 WARNING juju-log No relation: certificates
2024-02-28T08:20:20.291Z [container-agent] 2024-02-28 08:20:20 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T08:25:45.621Z [container-agent] 2024-02-28 08:25:45 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T08:25:51.686Z [container-agent] 2024-02-28 08:25:51 WARNING juju-log No relation: certificates
2024-02-28T08:25:52.065Z [container-agent] 2024-02-28 08:25:52 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T08:29:59.640Z [container-agent] 2024-02-28 08:29:59 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T08:30:06.152Z [container-agent] 2024-02-28 08:30:06 WARNING juju-log No relation: certificates
2024-02-28T08:30:06.549Z [container-agent] 2024-02-28 08:30:06 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T08:34:46.194Z [container-agent] 2024-02-28 08:34:46 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T08:34:53.109Z [container-agent] 2024-02-28 08:34:53 WARNING juju-log No relation: certificates
2024-02-28T08:34:53.488Z [container-agent] 2024-02-28 08:34:53 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T08:38:47.423Z [container-agent] 2024-02-28 08:38:47 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T08:38:54.822Z [container-agent] 2024-02-28 08:38:54 WARNING juju-log No relation: certificates
2024-02-28T08:38:55.164Z [container-agent] 2024-02-28 08:38:55 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T08:43:21.808Z [container-agent] 2024-02-28 08:43:21 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T08:43:28.966Z [container-agent] 2024-02-28 08:43:28 WARNING juju-log No relation: certificates
2024-02-28T08:43:29.338Z [container-agent] 2024-02-28 08:43:29 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T08:47:07.805Z [container-agent] 2024-02-28 08:47:07 ERROR juju-log Failed to flush ERROR LOGS logs.
2024-02-28T08:47:07.805Z [container-agent] Traceback (most recent call last):
2024-02-28T08:47:07.805Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 666, in _run_mysqlsh_script
2024-02-28T08:47:07.805Z [container-agent]     stdout, _ = process.wait_output()
2024-02-28T08:47:07.805Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1441, in wait_output
2024-02-28T08:47:07.805Z [container-agent]     raise ExecError[AnyStr](self._command, exit_code, out_value, err_value)
2024-02-28T08:47:07.805Z [container-agent] ops.pebble.ExecError: non-zero exit code 1 executing ['/usr/bin/mysqlsh', '--no-wizard', '--python', '--verbose=1', '-f', '/tmp/script.py', ';', 'rm', '/tmp/script.py'], stdout='', stderr='Cannot set LC_ALL to locale en_US.UTF-8: No such file or directory\nverbose: 2024-02-28T08:47:06Z: Loading startup files...\nverbose: 2024-02-28T08:47:06Z: Loading plugins...\nverbose: 2024-02-28T08:47:06Z: Connecting to MySQL at: serverconfig@horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local\nTraceback (most recent call last):\n  File "<string>", line 1, in <module>\nmysqlsh.DBError: MySQL Error (2003): Shell.connect: Can\'t connect to MySQL server on \'horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local:3306\' (111)\n'
2024-02-28T08:47:07.805Z [container-agent] 
2024-02-28T08:47:07.805Z [container-agent] During handling of the above exception, another exception occurred:
2024-02-28T08:47:07.805Z [container-agent] 
2024-02-28T08:47:07.805Z [container-agent] Traceback (most recent call last):
2024-02-28T08:47:07.805Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-02-28T08:47:07.805Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-02-28T08:47:07.805Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 669, in _run_mysqlsh_script
2024-02-28T08:47:07.805Z [container-agent]     raise MySQLClientError(e.stderr)
2024-02-28T08:47:07.805Z [container-agent] charms.mysql.v0.mysql.MySQLClientError: Cannot set LC_ALL to locale en_US.UTF-8: No such file or directory
2024-02-28T08:47:07.805Z [container-agent] verbose: 2024-02-28T08:47:06Z: Loading startup files...
2024-02-28T08:47:07.805Z [container-agent] verbose: 2024-02-28T08:47:06Z: Loading plugins...
2024-02-28T08:47:07.805Z [container-agent] verbose: 2024-02-28T08:47:06Z: Connecting to MySQL at: serverconfig@horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local
2024-02-28T08:47:07.805Z [container-agent] Traceback (most recent call last):
2024-02-28T08:47:07.805Z [container-agent]   File "<string>", line 1, in <module>
2024-02-28T08:47:07.805Z [container-agent] mysqlsh.DBError: MySQL Error (2003): Shell.connect: Can't connect to MySQL server on 'horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local:3306' (111)
2024-02-28T08:47:07.805Z [container-agent] 
2024-02-28T08:48:14.563Z [container-agent] 2024-02-28 08:48:14 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-02-28T08:48:14.593Z [container-agent] 2024-02-28 08:48:14 INFO juju-log Attempting reboot from complete outage.
2024-02-28T08:48:22.327Z [container-agent] 2024-02-28 08:48:22 WARNING juju-log No relation: certificates
2024-02-28T08:48:22.671Z [container-agent] 2024-02-28 08:48:22 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T08:52:04.800Z [container-agent] 2024-02-28 08:52:04 ERROR juju-log Uncaught exception while in charm code:
2024-02-28T08:52:04.800Z [container-agent] Traceback (most recent call last):
2024-02-28T08:52:04.800Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/./src/charm.py", line 786, in <module>
2024-02-28T08:52:04.800Z [container-agent]     main(MySQLOperatorCharm)
2024-02-28T08:52:04.800Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 436, in main
2024-02-28T08:52:04.800Z [container-agent]     _emit_charm_event(charm, dispatcher.event_name)
2024-02-28T08:52:04.800Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 144, in _emit_charm_event
2024-02-28T08:52:04.800Z [container-agent]     event_to_emit.emit(*args, **kwargs)
2024-02-28T08:52:04.800Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 351, in emit
2024-02-28T08:52:04.800Z [container-agent]     framework._emit(event)
2024-02-28T08:52:04.800Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 853, in _emit
2024-02-28T08:52:04.800Z [container-agent]     self._reemit(event_path)
2024-02-28T08:52:04.800Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 942, in _reemit
2024-02-28T08:52:04.800Z [container-agent]     custom_handler(event)
2024-02-28T08:52:04.800Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/rotate_mysql_logs.py", line 55, in _rotate_mysql_logs
2024-02-28T08:52:04.800Z [container-agent]     self.charm._mysql.flush_mysql_logs(MySQLTextLogs.ERROR)
2024-02-28T08:52:04.800Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-02-28T08:52:04.800Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-02-28T08:52:04.800Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 665, in _run_mysqlsh_script
2024-02-28T08:52:04.800Z [container-agent]     process = self.container.exec(cmd, timeout=timeout)
2024-02-28T08:52:04.800Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/model.py", line 2688, in exec
2024-02-28T08:52:04.800Z [container-agent]     return self._pebble.exec(
2024-02-28T08:52:04.800Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 2501, in exec
2024-02-28T08:52:04.800Z [container-agent]     resp = self._request('POST', '/v1/exec', body=body)
2024-02-28T08:52:04.800Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1655, in _request
2024-02-28T08:52:04.800Z [container-agent]     response = self._request_raw(method, path, query, headers, data)
2024-02-28T08:52:04.800Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1690, in _request_raw
2024-02-28T08:52:04.800Z [container-agent]     response = self.opener.open(request, timeout=self.timeout)
2024-02-28T08:52:04.800Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 519, in open
2024-02-28T08:52:04.800Z [container-agent]     response = self._open(req, data)
2024-02-28T08:52:04.800Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 536, in _open
2024-02-28T08:52:04.800Z [container-agent]     result = self._call_chain(self.handle_open, protocol, protocol +
2024-02-28T08:52:04.800Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 496, in _call_chain
2024-02-28T08:52:04.800Z [container-agent]     result = func(*args)
2024-02-28T08:52:04.800Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 311, in http_open
2024-02-28T08:52:04.800Z [container-agent]     return self.do_open(_UnixSocketConnection, req,  # type:ignore
2024-02-28T08:52:04.800Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 1352, in do_open
2024-02-28T08:52:04.800Z [container-agent]     r = h.getresponse()
2024-02-28T08:52:04.800Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1375, in getresponse
2024-02-28T08:52:04.800Z [container-agent]     response.begin()
2024-02-28T08:52:04.800Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 318, in begin
2024-02-28T08:52:04.800Z [container-agent]     version, status, reason = self._read_status()
2024-02-28T08:52:04.800Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 287, in _read_status
2024-02-28T08:52:04.800Z [container-agent]     raise RemoteDisconnected("Remote end closed connection without"
2024-02-28T08:52:04.800Z [container-agent] http.client.RemoteDisconnected: Remote end closed connection without response
2024-02-28T08:52:04.944Z [container-agent] 2024-02-28 08:52:04 INFO juju.util.exec exec.go:209 run result: exit status 1
2024-02-28T08:52:06.899Z [container-agent] 2024-02-28 08:52:06 INFO juju-log Setting up the logrotate configurations
2024-02-28T08:52:06.964Z [container-agent] 2024-02-28 08:52:06 INFO juju-log Adding pebble layer
2024-02-28T08:52:13.756Z [container-agent] 2024-02-28 08:52:13 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-02-28T08:52:13.778Z [container-agent] 2024-02-28 08:52:13 INFO juju-log Attempting reboot from complete outage.
2024-02-28T08:52:16.842Z [container-agent] 2024-02-28 08:52:16 INFO juju.worker.uniter.operation runhook.go:186 ran "mysql-pebble-ready" hook (via hook dispatching script: dispatch)
2024-02-28T08:52:28.418Z [container-agent] 2024-02-28 08:52:28 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T08:52:30.573Z [container-agent] 2024-02-28 08:52:30 WARNING juju-log No relation: certificates
2024-02-28T08:52:30.941Z [container-agent] 2024-02-28 08:52:30 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T08:58:23.611Z [container-agent] 2024-02-28 08:58:23 INFO juju-log Starting the log rotate manager
2024-02-28T08:58:23.626Z [container-agent] 2024-02-28 08:58:23 INFO juju-log Started log rotate manager process with PID 123991
2024-02-28T08:58:23.910Z [container-agent] 2024-02-28 08:58:23 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T08:58:25.835Z [container-agent] 2024-02-28 08:58:25 WARNING juju-log No relation: certificates
2024-02-28T08:58:26.194Z [container-agent] 2024-02-28 08:58:26 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T09:02:58.170Z [container-agent] 2024-02-28 09:02:58 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T09:03:00.615Z [container-agent] 2024-02-28 09:03:00 WARNING juju-log No relation: certificates
2024-02-28T09:03:00.967Z [container-agent] 2024-02-28 09:03:00 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T09:07:06.222Z [container-agent] 2024-02-28 09:07:06 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T09:07:08.968Z [container-agent] 2024-02-28 09:07:08 WARNING juju-log No relation: certificates
2024-02-28T09:07:09.320Z [container-agent] 2024-02-28 09:07:09 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T09:12:35.436Z [container-agent] 2024-02-28 09:12:35 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T09:12:38.795Z [container-agent] 2024-02-28 09:12:38 WARNING juju-log No relation: certificates
2024-02-28T09:12:39.133Z [container-agent] 2024-02-28 09:12:39 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T09:18:16.724Z [container-agent] 2024-02-28 09:18:16 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T09:18:20.809Z [container-agent] 2024-02-28 09:18:20 WARNING juju-log No relation: certificates
2024-02-28T09:18:21.153Z [container-agent] 2024-02-28 09:18:21 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T09:23:12.550Z [container-agent] 2024-02-28 09:23:12 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T09:23:16.956Z [container-agent] 2024-02-28 09:23:16 WARNING juju-log No relation: certificates
2024-02-28T09:23:17.296Z [container-agent] 2024-02-28 09:23:17 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T09:28:25.106Z [container-agent] 2024-02-28 09:28:25 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T09:28:29.965Z [container-agent] 2024-02-28 09:28:29 WARNING juju-log No relation: certificates
2024-02-28T09:28:30.327Z [container-agent] 2024-02-28 09:28:30 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T09:33:12.070Z [container-agent] 2024-02-28 09:33:12 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T09:33:17.365Z [container-agent] 2024-02-28 09:33:17 WARNING juju-log No relation: certificates
2024-02-28T09:33:17.744Z [container-agent] 2024-02-28 09:33:17 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T09:39:11.082Z [container-agent] 2024-02-28 09:39:11 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T09:39:17.172Z [container-agent] 2024-02-28 09:39:17 WARNING juju-log No relation: certificates
2024-02-28T09:39:17.525Z [container-agent] 2024-02-28 09:39:17 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T09:43:37.859Z [container-agent] 2024-02-28 09:43:37 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T09:43:44.142Z [container-agent] 2024-02-28 09:43:44 WARNING juju-log No relation: certificates
2024-02-28T09:43:44.487Z [container-agent] 2024-02-28 09:43:44 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T09:49:35.668Z [container-agent] 2024-02-28 09:49:35 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T09:49:42.387Z [container-agent] 2024-02-28 09:49:42 WARNING juju-log No relation: certificates
2024-02-28T09:49:42.737Z [container-agent] 2024-02-28 09:49:42 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T09:53:57.968Z [container-agent] 2024-02-28 09:53:57 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T09:54:04.755Z [container-agent] 2024-02-28 09:54:04 WARNING juju-log No relation: certificates
2024-02-28T09:54:05.115Z [container-agent] 2024-02-28 09:54:05 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T09:58:01.074Z [container-agent] 2024-02-28 09:58:01 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T09:58:08.251Z [container-agent] 2024-02-28 09:58:08 WARNING juju-log No relation: certificates
2024-02-28T09:58:08.648Z [container-agent] 2024-02-28 09:58:08 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T10:02:34.403Z [container-agent] 2024-02-28 10:02:34 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T10:02:41.730Z [container-agent] 2024-02-28 10:02:41 WARNING juju-log No relation: certificates
2024-02-28T10:02:42.074Z [container-agent] 2024-02-28 10:02:42 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T10:08:04.785Z [container-agent] 2024-02-28 10:08:04 ERROR juju-log Uncaught exception while in charm code:
2024-02-28T10:08:04.785Z [container-agent] Traceback (most recent call last):
2024-02-28T10:08:04.785Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 2507, in exec
2024-02-28T10:08:04.785Z [container-agent]     control_ws = self._connect_websocket(task_id, 'control')
2024-02-28T10:08:04.785Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 2589, in _connect_websocket
2024-02-28T10:08:04.785Z [container-agent]     ws.connect(url, socket=sock)
2024-02-28T10:08:04.785Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_core.py", line 253, in connect
2024-02-28T10:08:04.785Z [container-agent]     self.handshake_response = handshake(self.sock, url, *addrs, **options)
2024-02-28T10:08:04.785Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_handshake.py", line 57, in handshake
2024-02-28T10:08:04.785Z [container-agent]     status, resp = _get_resp_headers(sock)
2024-02-28T10:08:04.785Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_handshake.py", line 143, in _get_resp_headers
2024-02-28T10:08:04.785Z [container-agent]     status, resp_headers, status_message = read_headers(sock)
2024-02-28T10:08:04.785Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_http.py", line 312, in read_headers
2024-02-28T10:08:04.785Z [container-agent]     line = recv_line(sock)
2024-02-28T10:08:04.785Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_socket.py", line 131, in recv_line
2024-02-28T10:08:04.785Z [container-agent]     c = recv(sock, 1)
2024-02-28T10:08:04.785Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_socket.py", line 122, in recv
2024-02-28T10:08:04.785Z [container-agent]     raise WebSocketConnectionClosedException(
2024-02-28T10:08:04.785Z [container-agent] websocket._exceptions.WebSocketConnectionClosedException: Connection to remote host was lost.
2024-02-28T10:08:04.785Z [container-agent] 
2024-02-28T10:08:04.785Z [container-agent] During handling of the above exception, another exception occurred:
2024-02-28T10:08:04.785Z [container-agent] 
2024-02-28T10:08:04.785Z [container-agent] Traceback (most recent call last):
2024-02-28T10:08:04.785Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 1348, in do_open
2024-02-28T10:08:04.785Z [container-agent]     h.request(req.get_method(), req.selector, req.data, headers,
2024-02-28T10:08:04.785Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1283, in request
2024-02-28T10:08:04.785Z [container-agent]     self._send_request(method, url, body, headers, encode_chunked)
2024-02-28T10:08:04.785Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1329, in _send_request
2024-02-28T10:08:04.785Z [container-agent]     self.endheaders(body, encode_chunked=encode_chunked)
2024-02-28T10:08:04.785Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1278, in endheaders
2024-02-28T10:08:04.785Z [container-agent]     self._send_output(message_body, encode_chunked=encode_chunked)
2024-02-28T10:08:04.785Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1038, in _send_output
2024-02-28T10:08:04.785Z [container-agent]     self.send(msg)
2024-02-28T10:08:04.785Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 976, in send
2024-02-28T10:08:04.785Z [container-agent]     self.connect()
2024-02-28T10:08:04.785Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 297, in connect
2024-02-28T10:08:04.785Z [container-agent]     self.sock.connect(self.socket_path)
2024-02-28T10:08:04.785Z [container-agent] ConnectionRefusedError: [Errno 111] Connection refused
2024-02-28T10:08:04.785Z [container-agent] 
2024-02-28T10:08:04.785Z [container-agent] During handling of the above exception, another exception occurred:
2024-02-28T10:08:04.785Z [container-agent] 
2024-02-28T10:08:04.785Z [container-agent] Traceback (most recent call last):
2024-02-28T10:08:04.785Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1690, in _request_raw
2024-02-28T10:08:04.785Z [container-agent]     response = self.opener.open(request, timeout=self.timeout)
2024-02-28T10:08:04.785Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 519, in open
2024-02-28T10:08:04.785Z [container-agent]     response = self._open(req, data)
2024-02-28T10:08:04.785Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 536, in _open
2024-02-28T10:08:04.785Z [container-agent]     result = self._call_chain(self.handle_open, protocol, protocol +
2024-02-28T10:08:04.785Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 496, in _call_chain
2024-02-28T10:08:04.785Z [container-agent]     result = func(*args)
2024-02-28T10:08:04.785Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 311, in http_open
2024-02-28T10:08:04.785Z [container-agent]     return self.do_open(_UnixSocketConnection, req,  # type:ignore
2024-02-28T10:08:04.785Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 1351, in do_open
2024-02-28T10:08:04.785Z [container-agent]     raise URLError(err)
2024-02-28T10:08:04.785Z [container-agent] urllib.error.URLError: <urlopen error [Errno 111] Connection refused>
2024-02-28T10:08:04.785Z [container-agent] 
2024-02-28T10:08:04.785Z [container-agent] The above exception was the direct cause of the following exception:
2024-02-28T10:08:04.785Z [container-agent] 
2024-02-28T10:08:04.785Z [container-agent] Traceback (most recent call last):
2024-02-28T10:08:04.785Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/./src/charm.py", line 786, in <module>
2024-02-28T10:08:04.785Z [container-agent]     main(MySQLOperatorCharm)
2024-02-28T10:08:04.785Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 436, in main
2024-02-28T10:08:04.785Z [container-agent]     _emit_charm_event(charm, dispatcher.event_name)
2024-02-28T10:08:04.785Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 144, in _emit_charm_event
2024-02-28T10:08:04.785Z [container-agent]     event_to_emit.emit(*args, **kwargs)
2024-02-28T10:08:04.785Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 351, in emit
2024-02-28T10:08:04.785Z [container-agent]     framework._emit(event)
2024-02-28T10:08:04.785Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 853, in _emit
2024-02-28T10:08:04.785Z [container-agent]     self._reemit(event_path)
2024-02-28T10:08:04.785Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 942, in _reemit
2024-02-28T10:08:04.785Z [container-agent]     custom_handler(event)
2024-02-28T10:08:04.785Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/rotate_mysql_logs.py", line 55, in _rotate_mysql_logs
2024-02-28T10:08:04.785Z [container-agent]     self.charm._mysql.flush_mysql_logs(MySQLTextLogs.ERROR)
2024-02-28T10:08:04.785Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-02-28T10:08:04.785Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-02-28T10:08:04.785Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 665, in _run_mysqlsh_script
2024-02-28T10:08:04.785Z [container-agent]     process = self.container.exec(cmd, timeout=timeout)
2024-02-28T10:08:04.785Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/model.py", line 2688, in exec
2024-02-28T10:08:04.785Z [container-agent]     return self._pebble.exec(
2024-02-28T10:08:04.785Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 2514, in exec
2024-02-28T10:08:04.785Z [container-agent]     change = self.wait_change(ChangeID(change_id))
2024-02-28T10:08:04.785Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1893, in wait_change
2024-02-28T10:08:04.785Z [container-agent]     return self._wait_change_using_wait(change_id, timeout)
2024-02-28T10:08:04.785Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1914, in _wait_change_using_wait
2024-02-28T10:08:04.785Z [container-agent]     return self._wait_change(change_id, this_timeout)
2024-02-28T10:08:04.785Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1928, in _wait_change
2024-02-28T10:08:04.785Z [container-agent]     resp = self._request('GET', f'/v1/changes/{change_id}/wait', query)
2024-02-28T10:08:04.785Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1655, in _request
2024-02-28T10:08:04.785Z [container-agent]     response = self._request_raw(method, path, query, headers, data)
2024-02-28T10:08:04.785Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1707, in _request_raw
2024-02-28T10:08:04.785Z [container-agent]     raise ConnectionError(e.reason) from e
2024-02-28T10:08:04.785Z [container-agent] ops.pebble.ConnectionError: [Errno 111] Connection refused
2024-02-28T10:08:04.960Z [container-agent] 2024-02-28 10:08:04 INFO juju.util.exec exec.go:209 run result: exit status 1
2024-02-28T10:08:06.166Z [container-agent] 2024-02-28 10:08:06 INFO juju-log Starting the log rotate manager
2024-02-28T10:08:06.179Z [container-agent] 2024-02-28 10:08:06 INFO juju-log Started log rotate manager process with PID 130699
2024-02-28T10:08:06.944Z [container-agent] 2024-02-28 10:08:06 ERROR juju-log Failed to get cluster status for cluster-7573647c1e26466beeeac32b95ab8f91
2024-02-28T10:08:07.008Z [container-agent] 2024-02-28 10:08:07 INFO juju-log Kubernetes pod label error created
2024-02-28T10:08:07.019Z [container-agent] 2024-02-28 10:08:07 WARNING juju-log No relation: certificates
2024-02-28T10:08:07.353Z [container-agent] 2024-02-28 10:08:07 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T10:08:10.161Z [container-agent] 2024-02-28 10:08:10 INFO juju-log Setting up the logrotate configurations
2024-02-28T10:08:10.218Z [container-agent] 2024-02-28 10:08:10 INFO juju-log Adding pebble layer
2024-02-28T10:08:16.898Z [container-agent] 2024-02-28 10:08:16 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-02-28T10:08:16.929Z [container-agent] 2024-02-28 10:08:16 INFO juju-log Attempting reboot from complete outage.
2024-02-28T10:08:20.039Z [container-agent] 2024-02-28 10:08:20 INFO juju.worker.uniter.operation runhook.go:186 ran "mysql-pebble-ready" hook (via hook dispatching script: dispatch)
2024-02-28T10:13:15.484Z [container-agent] 2024-02-28 10:13:15 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T10:13:18.121Z [container-agent] 2024-02-28 10:13:18 INFO juju-log Kubernetes pod label primary created
2024-02-28T10:13:18.135Z [container-agent] 2024-02-28 10:13:18 WARNING juju-log No relation: certificates
2024-02-28T10:13:18.509Z [container-agent] 2024-02-28 10:13:18 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T10:18:43.496Z [container-agent] 2024-02-28 10:18:43 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T10:18:46.430Z [container-agent] 2024-02-28 10:18:46 WARNING juju-log No relation: certificates
2024-02-28T10:18:46.788Z [container-agent] 2024-02-28 10:18:46 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T10:22:50.180Z [container-agent] 2024-02-28 10:22:50 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T10:22:53.298Z [container-agent] 2024-02-28 10:22:53 WARNING juju-log No relation: certificates
2024-02-28T10:22:53.688Z [container-agent] 2024-02-28 10:22:53 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T10:28:21.442Z [container-agent] 2024-02-28 10:28:21 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T10:28:25.009Z [container-agent] 2024-02-28 10:28:25 WARNING juju-log No relation: certificates
2024-02-28T10:28:25.365Z [container-agent] 2024-02-28 10:28:25 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T10:33:15.323Z [container-agent] 2024-02-28 10:33:15 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T10:33:19.694Z [container-agent] 2024-02-28 10:33:19 WARNING juju-log No relation: certificates
2024-02-28T10:33:20.045Z [container-agent] 2024-02-28 10:33:20 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T10:37:28.338Z [container-agent] 2024-02-28 10:37:28 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T10:37:33.009Z [container-agent] 2024-02-28 10:37:33 WARNING juju-log No relation: certificates
2024-02-28T10:37:33.364Z [container-agent] 2024-02-28 10:37:33 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T10:42:45.786Z [container-agent] 2024-02-28 10:42:45 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T10:42:50.713Z [container-agent] 2024-02-28 10:42:50 WARNING juju-log No relation: certificates
2024-02-28T10:42:51.053Z [container-agent] 2024-02-28 10:42:51 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T10:47:30.639Z [container-agent] 2024-02-28 10:47:30 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T10:47:36.356Z [container-agent] 2024-02-28 10:47:36 WARNING juju-log No relation: certificates
2024-02-28T10:47:36.693Z [container-agent] 2024-02-28 10:47:36 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T10:52:12.798Z [container-agent] 2024-02-28 10:52:12 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T10:52:18.865Z [container-agent] 2024-02-28 10:52:18 WARNING juju-log No relation: certificates
2024-02-28T10:52:19.211Z [container-agent] 2024-02-28 10:52:19 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T10:57:43.217Z [container-agent] 2024-02-28 10:57:43 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T10:57:49.758Z [container-agent] 2024-02-28 10:57:49 WARNING juju-log No relation: certificates
2024-02-28T10:57:50.095Z [container-agent] 2024-02-28 10:57:50 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T11:03:29.756Z [container-agent] 2024-02-28 11:03:29 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T11:03:36.707Z [container-agent] 2024-02-28 11:03:36 WARNING juju-log No relation: certificates
2024-02-28T11:03:37.067Z [container-agent] 2024-02-28 11:03:37 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T11:08:14.077Z [container-agent] 2024-02-28 11:08:14 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T11:08:21.514Z [container-agent] 2024-02-28 11:08:21 WARNING juju-log No relation: certificates
2024-02-28T11:08:21.882Z [container-agent] 2024-02-28 11:08:21 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T11:13:48.869Z [container-agent] 2024-02-28 11:13:48 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T11:13:56.587Z [container-agent] 2024-02-28 11:13:56 WARNING juju-log No relation: certificates
2024-02-28T11:13:56.948Z [container-agent] 2024-02-28 11:13:56 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T11:19:47.758Z [container-agent] 2024-02-28 11:19:47 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T11:19:56.176Z [container-agent] 2024-02-28 11:19:56 WARNING juju-log No relation: certificates
2024-02-28T11:19:56.504Z [container-agent] 2024-02-28 11:19:56 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T11:20:12.917Z [container-agent] 2024-02-28 11:20:12 ERROR juju-log Failed to flush SLOW LOGS logs.
2024-02-28T11:20:12.917Z [container-agent] Traceback (most recent call last):
2024-02-28T11:20:12.917Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 666, in _run_mysqlsh_script
2024-02-28T11:20:12.917Z [container-agent]     stdout, _ = process.wait_output()
2024-02-28T11:20:12.917Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1441, in wait_output
2024-02-28T11:20:12.917Z [container-agent]     raise ExecError[AnyStr](self._command, exit_code, out_value, err_value)
2024-02-28T11:20:12.917Z [container-agent] ops.pebble.ExecError: non-zero exit code 1 executing ['/usr/bin/mysqlsh', '--no-wizard', '--python', '--verbose=1', '-f', '/tmp/script.py', ';', 'rm', '/tmp/script.py'], stdout='', stderr='Cannot set LC_ALL to locale en_US.UTF-8: No such file or directory\nverbose: 2024-02-28T11:20:11Z: Loading startup files...\nverbose: 2024-02-28T11:20:11Z: Loading plugins...\nverbose: 2024-02-28T11:20:11Z: Connecting to MySQL at: serverconfig@horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local\nTraceback (most recent call last):\n  File "<string>", line 1, in <module>\nmysqlsh.DBError: MySQL Error (2003): Shell.connect: Can\'t connect to MySQL server on \'horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local:3306\' (111)\n'
2024-02-28T11:20:12.917Z [container-agent] 
2024-02-28T11:20:12.917Z [container-agent] During handling of the above exception, another exception occurred:
2024-02-28T11:20:12.917Z [container-agent] 
2024-02-28T11:20:12.917Z [container-agent] Traceback (most recent call last):
2024-02-28T11:20:12.917Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-02-28T11:20:12.917Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-02-28T11:20:12.917Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 669, in _run_mysqlsh_script
2024-02-28T11:20:12.917Z [container-agent]     raise MySQLClientError(e.stderr)
2024-02-28T11:20:12.917Z [container-agent] charms.mysql.v0.mysql.MySQLClientError: Cannot set LC_ALL to locale en_US.UTF-8: No such file or directory
2024-02-28T11:20:12.917Z [container-agent] verbose: 2024-02-28T11:20:11Z: Loading startup files...
2024-02-28T11:20:12.917Z [container-agent] verbose: 2024-02-28T11:20:11Z: Loading plugins...
2024-02-28T11:20:12.917Z [container-agent] verbose: 2024-02-28T11:20:11Z: Connecting to MySQL at: serverconfig@horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local
2024-02-28T11:20:12.917Z [container-agent] Traceback (most recent call last):
2024-02-28T11:20:12.917Z [container-agent]   File "<string>", line 1, in <module>
2024-02-28T11:20:12.917Z [container-agent] mysqlsh.DBError: MySQL Error (2003): Shell.connect: Can't connect to MySQL server on 'horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local:3306' (111)
2024-02-28T11:20:12.917Z [container-agent] 
2024-02-28T11:24:38.557Z [container-agent] 2024-02-28 11:24:38 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-02-28T11:24:38.603Z [container-agent] 2024-02-28 11:24:38 INFO juju-log Attempting reboot from complete outage.
2024-02-28T11:24:46.513Z [container-agent] 2024-02-28 11:24:46 WARNING juju-log No relation: certificates
2024-02-28T11:24:46.861Z [container-agent] 2024-02-28 11:24:46 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T11:25:09.801Z [container-agent] 2024-02-28 11:25:09 ERROR juju-log Uncaught exception while in charm code:
2024-02-28T11:25:09.801Z [container-agent] Traceback (most recent call last):
2024-02-28T11:25:09.801Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 2507, in exec
2024-02-28T11:25:09.801Z [container-agent]     control_ws = self._connect_websocket(task_id, 'control')
2024-02-28T11:25:09.801Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 2589, in _connect_websocket
2024-02-28T11:25:09.801Z [container-agent]     ws.connect(url, socket=sock)
2024-02-28T11:25:09.801Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_core.py", line 253, in connect
2024-02-28T11:25:09.801Z [container-agent]     self.handshake_response = handshake(self.sock, url, *addrs, **options)
2024-02-28T11:25:09.801Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_handshake.py", line 57, in handshake
2024-02-28T11:25:09.801Z [container-agent]     status, resp = _get_resp_headers(sock)
2024-02-28T11:25:09.801Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_handshake.py", line 143, in _get_resp_headers
2024-02-28T11:25:09.801Z [container-agent]     status, resp_headers, status_message = read_headers(sock)
2024-02-28T11:25:09.801Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_http.py", line 312, in read_headers
2024-02-28T11:25:09.801Z [container-agent]     line = recv_line(sock)
2024-02-28T11:25:09.801Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_socket.py", line 131, in recv_line
2024-02-28T11:25:09.801Z [container-agent]     c = recv(sock, 1)
2024-02-28T11:25:09.801Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_socket.py", line 122, in recv
2024-02-28T11:25:09.801Z [container-agent]     raise WebSocketConnectionClosedException(
2024-02-28T11:25:09.801Z [container-agent] websocket._exceptions.WebSocketConnectionClosedException: Connection to remote host was lost.
2024-02-28T11:25:09.801Z [container-agent] 
2024-02-28T11:25:09.801Z [container-agent] During handling of the above exception, another exception occurred:
2024-02-28T11:25:09.801Z [container-agent] 
2024-02-28T11:25:09.801Z [container-agent] Traceback (most recent call last):
2024-02-28T11:25:09.801Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 1348, in do_open
2024-02-28T11:25:09.801Z [container-agent]     h.request(req.get_method(), req.selector, req.data, headers,
2024-02-28T11:25:09.801Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1283, in request
2024-02-28T11:25:09.801Z [container-agent]     self._send_request(method, url, body, headers, encode_chunked)
2024-02-28T11:25:09.801Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1329, in _send_request
2024-02-28T11:25:09.801Z [container-agent]     self.endheaders(body, encode_chunked=encode_chunked)
2024-02-28T11:25:09.801Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1278, in endheaders
2024-02-28T11:25:09.801Z [container-agent]     self._send_output(message_body, encode_chunked=encode_chunked)
2024-02-28T11:25:09.801Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1038, in _send_output
2024-02-28T11:25:09.801Z [container-agent]     self.send(msg)
2024-02-28T11:25:09.801Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 976, in send
2024-02-28T11:25:09.801Z [container-agent]     self.connect()
2024-02-28T11:25:09.801Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 297, in connect
2024-02-28T11:25:09.801Z [container-agent]     self.sock.connect(self.socket_path)
2024-02-28T11:25:09.801Z [container-agent] ConnectionRefusedError: [Errno 111] Connection refused
2024-02-28T11:25:09.801Z [container-agent] 
2024-02-28T11:25:09.801Z [container-agent] During handling of the above exception, another exception occurred:
2024-02-28T11:25:09.801Z [container-agent] 
2024-02-28T11:25:09.801Z [container-agent] Traceback (most recent call last):
2024-02-28T11:25:09.801Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1690, in _request_raw
2024-02-28T11:25:09.801Z [container-agent]     response = self.opener.open(request, timeout=self.timeout)
2024-02-28T11:25:09.801Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 519, in open
2024-02-28T11:25:09.801Z [container-agent]     response = self._open(req, data)
2024-02-28T11:25:09.801Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 536, in _open
2024-02-28T11:25:09.801Z [container-agent]     result = self._call_chain(self.handle_open, protocol, protocol +
2024-02-28T11:25:09.801Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 496, in _call_chain
2024-02-28T11:25:09.801Z [container-agent]     result = func(*args)
2024-02-28T11:25:09.801Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 311, in http_open
2024-02-28T11:25:09.801Z [container-agent]     return self.do_open(_UnixSocketConnection, req,  # type:ignore
2024-02-28T11:25:09.801Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 1351, in do_open
2024-02-28T11:25:09.801Z [container-agent]     raise URLError(err)
2024-02-28T11:25:09.801Z [container-agent] urllib.error.URLError: <urlopen error [Errno 111] Connection refused>
2024-02-28T11:25:09.801Z [container-agent] 
2024-02-28T11:25:09.801Z [container-agent] The above exception was the direct cause of the following exception:
2024-02-28T11:25:09.801Z [container-agent] 
2024-02-28T11:25:09.801Z [container-agent] Traceback (most recent call last):
2024-02-28T11:25:09.801Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/./src/charm.py", line 786, in <module>
2024-02-28T11:25:09.801Z [container-agent]     main(MySQLOperatorCharm)
2024-02-28T11:25:09.801Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 436, in main
2024-02-28T11:25:09.801Z [container-agent]     _emit_charm_event(charm, dispatcher.event_name)
2024-02-28T11:25:09.801Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 144, in _emit_charm_event
2024-02-28T11:25:09.801Z [container-agent]     event_to_emit.emit(*args, **kwargs)
2024-02-28T11:25:09.801Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 351, in emit
2024-02-28T11:25:09.801Z [container-agent]     framework._emit(event)
2024-02-28T11:25:09.801Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 853, in _emit
2024-02-28T11:25:09.801Z [container-agent]     self._reemit(event_path)
2024-02-28T11:25:09.801Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 942, in _reemit
2024-02-28T11:25:09.801Z [container-agent]     custom_handler(event)
2024-02-28T11:25:09.801Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/rotate_mysql_logs.py", line 57, in _rotate_mysql_logs
2024-02-28T11:25:09.801Z [container-agent]     self.charm._mysql.flush_mysql_logs(MySQLTextLogs.SLOW)
2024-02-28T11:25:09.801Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-02-28T11:25:09.801Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-02-28T11:25:09.801Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 665, in _run_mysqlsh_script
2024-02-28T11:25:09.801Z [container-agent]     process = self.container.exec(cmd, timeout=timeout)
2024-02-28T11:25:09.801Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/model.py", line 2688, in exec
2024-02-28T11:25:09.801Z [container-agent]     return self._pebble.exec(
2024-02-28T11:25:09.801Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 2514, in exec
2024-02-28T11:25:09.801Z [container-agent]     change = self.wait_change(ChangeID(change_id))
2024-02-28T11:25:09.801Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1893, in wait_change
2024-02-28T11:25:09.801Z [container-agent]     return self._wait_change_using_wait(change_id, timeout)
2024-02-28T11:25:09.801Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1914, in _wait_change_using_wait
2024-02-28T11:25:09.801Z [container-agent]     return self._wait_change(change_id, this_timeout)
2024-02-28T11:25:09.801Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1928, in _wait_change
2024-02-28T11:25:09.801Z [container-agent]     resp = self._request('GET', f'/v1/changes/{change_id}/wait', query)
2024-02-28T11:25:09.801Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1655, in _request
2024-02-28T11:25:09.801Z [container-agent]     response = self._request_raw(method, path, query, headers, data)
2024-02-28T11:25:09.801Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1707, in _request_raw
2024-02-28T11:25:09.801Z [container-agent]     raise ConnectionError(e.reason) from e
2024-02-28T11:25:09.801Z [container-agent] ops.pebble.ConnectionError: [Errno 111] Connection refused
2024-02-28T11:25:09.950Z [container-agent] 2024-02-28 11:25:09 INFO juju.util.exec exec.go:209 run result: exit status 1
2024-02-28T11:25:13.048Z [container-agent] 2024-02-28 11:25:13 INFO juju-log Setting up the logrotate configurations
2024-02-28T11:25:13.116Z [container-agent] 2024-02-28 11:25:13 INFO juju-log Adding pebble layer
2024-02-28T11:25:19.924Z [container-agent] 2024-02-28 11:25:19 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-02-28T11:25:19.949Z [container-agent] 2024-02-28 11:25:19 INFO juju-log Attempting reboot from complete outage.
2024-02-28T11:25:22.843Z [container-agent] 2024-02-28 11:25:22 INFO juju.worker.uniter.operation runhook.go:186 ran "mysql-pebble-ready" hook (via hook dispatching script: dispatch)
2024-02-28T11:28:45.609Z [container-agent] 2024-02-28 11:28:45 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T11:28:47.604Z [container-agent] 2024-02-28 11:28:47 WARNING juju-log No relation: certificates
2024-02-28T11:28:47.952Z [container-agent] 2024-02-28 11:28:47 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T11:34:07.421Z [container-agent] 2024-02-28 11:34:07 INFO juju-log Starting the log rotate manager
2024-02-28T11:34:07.438Z [container-agent] 2024-02-28 11:34:07 INFO juju-log Started log rotate manager process with PID 138675
2024-02-28T11:34:07.871Z [container-agent] 2024-02-28 11:34:07 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T11:34:09.935Z [container-agent] 2024-02-28 11:34:09 WARNING juju-log No relation: certificates
2024-02-28T11:34:10.302Z [container-agent] 2024-02-28 11:34:10 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T11:38:35.077Z [container-agent] 2024-02-28 11:38:35 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T11:38:37.528Z [container-agent] 2024-02-28 11:38:37 WARNING juju-log No relation: certificates
2024-02-28T11:38:37.915Z [container-agent] 2024-02-28 11:38:37 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T11:42:37.724Z [container-agent] 2024-02-28 11:42:37 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T11:42:40.412Z [container-agent] 2024-02-28 11:42:40 WARNING juju-log No relation: certificates
2024-02-28T11:42:40.813Z [container-agent] 2024-02-28 11:42:40 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T11:47:46.845Z [container-agent] 2024-02-28 11:47:46 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T11:47:50.416Z [container-agent] 2024-02-28 11:47:50 WARNING juju-log No relation: certificates
2024-02-28T11:47:50.757Z [container-agent] 2024-02-28 11:47:50 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T11:53:00.831Z [container-agent] 2024-02-28 11:53:00 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T11:53:04.401Z [container-agent] 2024-02-28 11:53:04 WARNING juju-log No relation: certificates
2024-02-28T11:53:04.734Z [container-agent] 2024-02-28 11:53:04 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T11:58:30.160Z [container-agent] 2024-02-28 11:58:30 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T11:58:34.442Z [container-agent] 2024-02-28 11:58:34 WARNING juju-log No relation: certificates
2024-02-28T11:58:34.785Z [container-agent] 2024-02-28 11:58:34 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T12:04:02.820Z [container-agent] 2024-02-28 12:04:02 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T12:04:07.340Z [container-agent] 2024-02-28 12:04:07 WARNING juju-log No relation: certificates
2024-02-28T12:04:07.718Z [container-agent] 2024-02-28 12:04:07 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T12:09:18.985Z [container-agent] 2024-02-28 12:09:18 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T12:09:24.101Z [container-agent] 2024-02-28 12:09:24 WARNING juju-log No relation: certificates
2024-02-28T12:09:24.458Z [container-agent] 2024-02-28 12:09:24 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T12:13:45.209Z [container-agent] 2024-02-28 12:13:45 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T12:13:50.889Z [container-agent] 2024-02-28 12:13:50 WARNING juju-log No relation: certificates
2024-02-28T12:13:51.260Z [container-agent] 2024-02-28 12:13:51 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T12:18:14.250Z [container-agent] 2024-02-28 12:18:14 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T12:18:20.355Z [container-agent] 2024-02-28 12:18:20 WARNING juju-log No relation: certificates
2024-02-28T12:18:20.735Z [container-agent] 2024-02-28 12:18:20 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T12:24:11.865Z [container-agent] 2024-02-28 12:24:11 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T12:24:18.871Z [container-agent] 2024-02-28 12:24:18 WARNING juju-log No relation: certificates
2024-02-28T12:24:19.237Z [container-agent] 2024-02-28 12:24:19 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T12:28:29.186Z [container-agent] 2024-02-28 12:28:29 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T12:28:36.224Z [container-agent] 2024-02-28 12:28:36 WARNING juju-log No relation: certificates
2024-02-28T12:28:36.604Z [container-agent] 2024-02-28 12:28:36 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T12:33:22.895Z [container-agent] 2024-02-28 12:33:22 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T12:33:29.778Z [container-agent] 2024-02-28 12:33:29 WARNING juju-log No relation: certificates
2024-02-28T12:33:30.164Z [container-agent] 2024-02-28 12:33:30 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T12:38:49.288Z [container-agent] 2024-02-28 12:38:49 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T12:38:57.081Z [container-agent] 2024-02-28 12:38:57 WARNING juju-log No relation: certificates
2024-02-28T12:38:57.439Z [container-agent] 2024-02-28 12:38:57 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T12:40:15.563Z [container-agent] 2024-02-28 12:40:15 ERROR juju-log Failed to flush SLOW LOGS logs.
2024-02-28T12:40:15.563Z [container-agent] Traceback (most recent call last):
2024-02-28T12:40:15.563Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 666, in _run_mysqlsh_script
2024-02-28T12:40:15.563Z [container-agent]     stdout, _ = process.wait_output()
2024-02-28T12:40:15.563Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1441, in wait_output
2024-02-28T12:40:15.563Z [container-agent]     raise ExecError[AnyStr](self._command, exit_code, out_value, err_value)
2024-02-28T12:40:15.563Z [container-agent] ops.pebble.ExecError: non-zero exit code 1 executing ['/usr/bin/mysqlsh', '--no-wizard', '--python', '--verbose=1', '-f', '/tmp/script.py', ';', 'rm', '/tmp/script.py'], stdout='', stderr='Cannot set LC_ALL to locale en_US.UTF-8: No such file or directory\nverbose: 2024-02-28T12:40:11Z: Loading startup files...\nverbose: 2024-02-28T12:40:11Z: Loading plugins...\nverbose: 2024-02-28T12:40:11Z: Connecting to MySQL at: serverconfig@horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local\nTraceback (most recent call last):\n  File "<string>", line 1, in <module>\nmysqlsh.DBError: MySQL Error (2013): Shell.connect: Lost connection to MySQL server at \'reading initial communication packet\', system error: 104\n'
2024-02-28T12:40:15.563Z [container-agent] 
2024-02-28T12:40:15.563Z [container-agent] During handling of the above exception, another exception occurred:
2024-02-28T12:40:15.563Z [container-agent] 
2024-02-28T12:40:15.563Z [container-agent] Traceback (most recent call last):
2024-02-28T12:40:15.563Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-02-28T12:40:15.563Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-02-28T12:40:15.563Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 669, in _run_mysqlsh_script
2024-02-28T12:40:15.563Z [container-agent]     raise MySQLClientError(e.stderr)
2024-02-28T12:40:15.563Z [container-agent] charms.mysql.v0.mysql.MySQLClientError: Cannot set LC_ALL to locale en_US.UTF-8: No such file or directory
2024-02-28T12:40:15.563Z [container-agent] verbose: 2024-02-28T12:40:11Z: Loading startup files...
2024-02-28T12:40:15.563Z [container-agent] verbose: 2024-02-28T12:40:11Z: Loading plugins...
2024-02-28T12:40:15.563Z [container-agent] verbose: 2024-02-28T12:40:11Z: Connecting to MySQL at: serverconfig@horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local
2024-02-28T12:40:15.563Z [container-agent] Traceback (most recent call last):
2024-02-28T12:40:15.563Z [container-agent]   File "<string>", line 1, in <module>
2024-02-28T12:40:15.563Z [container-agent] mysqlsh.DBError: MySQL Error (2013): Shell.connect: Lost connection to MySQL server at 'reading initial communication packet', system error: 104
2024-02-28T12:40:15.563Z [container-agent] 
2024-02-28T12:40:21.166Z [container-agent] 2024-02-28 12:40:21 INFO juju-log Setting up the logrotate configurations
2024-02-28T12:40:21.230Z [container-agent] 2024-02-28 12:40:21 INFO juju-log Adding pebble layer
2024-02-28T12:40:27.956Z [container-agent] 2024-02-28 12:40:27 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-02-28T12:40:27.989Z [container-agent] 2024-02-28 12:40:27 INFO juju-log Attempting reboot from complete outage.
2024-02-28T12:40:31.057Z [container-agent] 2024-02-28 12:40:31 INFO juju.worker.uniter.operation runhook.go:186 ran "mysql-pebble-ready" hook (via hook dispatching script: dispatch)
2024-02-28T12:42:57.991Z [container-agent] 2024-02-28 12:42:57 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T12:42:59.916Z [container-agent] 2024-02-28 12:42:59 WARNING juju-log No relation: certificates
2024-02-28T12:43:00.251Z [container-agent] 2024-02-28 12:43:00 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T12:47:30.481Z [container-agent] 2024-02-28 12:47:30 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T12:47:33.084Z [container-agent] 2024-02-28 12:47:33 WARNING juju-log No relation: certificates
2024-02-28T12:47:33.443Z [container-agent] 2024-02-28 12:47:33 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T12:53:12.922Z [container-agent] 2024-02-28 12:53:12 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T12:53:16.011Z [container-agent] 2024-02-28 12:53:16 WARNING juju-log No relation: certificates
2024-02-28T12:53:16.358Z [container-agent] 2024-02-28 12:53:16 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T12:59:07.986Z [container-agent] 2024-02-28 12:59:07 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T12:59:11.479Z [container-agent] 2024-02-28 12:59:11 WARNING juju-log No relation: certificates
2024-02-28T12:59:11.803Z [container-agent] 2024-02-28 12:59:11 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T13:03:38.235Z [container-agent] 2024-02-28 13:03:38 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T13:03:41.997Z [container-agent] 2024-02-28 13:03:41 WARNING juju-log No relation: certificates
2024-02-28T13:03:42.329Z [container-agent] 2024-02-28 13:03:42 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T13:07:49.406Z [container-agent] 2024-02-28 13:07:49 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T13:07:53.707Z [container-agent] 2024-02-28 13:07:53 WARNING juju-log No relation: certificates
2024-02-28T13:07:54.074Z [container-agent] 2024-02-28 13:07:54 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T13:12:08.566Z [container-agent] 2024-02-28 13:12:08 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T13:12:13.128Z [container-agent] 2024-02-28 13:12:13 WARNING juju-log No relation: certificates
2024-02-28T13:12:13.471Z [container-agent] 2024-02-28 13:12:13 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T13:17:12.010Z [container-agent] 2024-02-28 13:17:12 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T13:17:17.200Z [container-agent] 2024-02-28 13:17:17 WARNING juju-log No relation: certificates
2024-02-28T13:17:17.549Z [container-agent] 2024-02-28 13:17:17 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T13:23:10.531Z [container-agent] 2024-02-28 13:23:10 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T13:23:16.866Z [container-agent] 2024-02-28 13:23:16 WARNING juju-log No relation: certificates
2024-02-28T13:23:17.215Z [container-agent] 2024-02-28 13:23:17 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T13:27:17.046Z [container-agent] 2024-02-28 13:27:17 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T13:27:23.286Z [container-agent] 2024-02-28 13:27:23 WARNING juju-log No relation: certificates
2024-02-28T13:27:23.652Z [container-agent] 2024-02-28 13:27:23 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T13:31:36.685Z [container-agent] 2024-02-28 13:31:36 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T13:31:42.544Z [container-agent] 2024-02-28 13:31:42 WARNING juju-log No relation: certificates
2024-02-28T13:31:42.897Z [container-agent] 2024-02-28 13:31:42 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T13:36:51.229Z [container-agent] 2024-02-28 13:36:51 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T13:36:57.614Z [container-agent] 2024-02-28 13:36:57 WARNING juju-log No relation: certificates
2024-02-28T13:36:57.966Z [container-agent] 2024-02-28 13:36:57 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T13:41:46.144Z [container-agent] 2024-02-28 13:41:46 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T13:41:53.514Z [container-agent] 2024-02-28 13:41:53 WARNING juju-log No relation: certificates
2024-02-28T13:41:53.868Z [container-agent] 2024-02-28 13:41:53 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T13:46:33.266Z [container-agent] 2024-02-28 13:46:33 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T13:46:40.772Z [container-agent] 2024-02-28 13:46:40 WARNING juju-log No relation: certificates
2024-02-28T13:46:41.099Z [container-agent] 2024-02-28 13:46:41 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T13:49:10.967Z [container-agent] 2024-02-28 13:49:10 ERROR juju-log Failed to flush ERROR LOGS logs.
2024-02-28T13:49:10.967Z [container-agent] Traceback (most recent call last):
2024-02-28T13:49:10.967Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 666, in _run_mysqlsh_script
2024-02-28T13:49:10.967Z [container-agent]     stdout, _ = process.wait_output()
2024-02-28T13:49:10.967Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1441, in wait_output
2024-02-28T13:49:10.967Z [container-agent]     raise ExecError[AnyStr](self._command, exit_code, out_value, err_value)
2024-02-28T13:49:10.967Z [container-agent] ops.pebble.ExecError: non-zero exit code 1 executing ['/usr/bin/mysqlsh', '--no-wizard', '--python', '--verbose=1', '-f', '/tmp/script.py', ';', 'rm', '/tmp/script.py'], stdout='', stderr='Cannot set LC_ALL to locale en_US.UTF-8: No such file or directory\nverbose: 2024-02-28T13:49:06Z: Loading startup files...\nverbose: 2024-02-28T13:49:06Z: Loading plugins...\nverbose: 2024-02-28T13:49:06Z: Connecting to MySQL at: serverconfig@horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local\nTraceback (most recent call last):\n  File "<string>", line 1, in <module>\nmysqlsh.DBError: MySQL Error (2013): Shell.connect: Lost connection to MySQL server at \'reading initial communication packet\', system error: 104\n'
2024-02-28T13:49:10.967Z [container-agent] 
2024-02-28T13:49:10.967Z [container-agent] During handling of the above exception, another exception occurred:
2024-02-28T13:49:10.967Z [container-agent] 
2024-02-28T13:49:10.967Z [container-agent] Traceback (most recent call last):
2024-02-28T13:49:10.967Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-02-28T13:49:10.967Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-02-28T13:49:10.967Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 669, in _run_mysqlsh_script
2024-02-28T13:49:10.967Z [container-agent]     raise MySQLClientError(e.stderr)
2024-02-28T13:49:10.967Z [container-agent] charms.mysql.v0.mysql.MySQLClientError: Cannot set LC_ALL to locale en_US.UTF-8: No such file or directory
2024-02-28T13:49:10.967Z [container-agent] verbose: 2024-02-28T13:49:06Z: Loading startup files...
2024-02-28T13:49:10.967Z [container-agent] verbose: 2024-02-28T13:49:06Z: Loading plugins...
2024-02-28T13:49:10.967Z [container-agent] verbose: 2024-02-28T13:49:06Z: Connecting to MySQL at: serverconfig@horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local
2024-02-28T13:49:10.967Z [container-agent] Traceback (most recent call last):
2024-02-28T13:49:10.967Z [container-agent]   File "<string>", line 1, in <module>
2024-02-28T13:49:10.967Z [container-agent] mysqlsh.DBError: MySQL Error (2013): Shell.connect: Lost connection to MySQL server at 'reading initial communication packet', system error: 104
2024-02-28T13:49:10.967Z [container-agent] 
2024-02-28T13:49:14.085Z [container-agent] 2024-02-28 13:49:14 ERROR juju-log Uncaught exception while in charm code:
2024-02-28T13:49:14.085Z [container-agent] Traceback (most recent call last):
2024-02-28T13:49:14.085Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/./src/charm.py", line 786, in <module>
2024-02-28T13:49:14.085Z [container-agent]     main(MySQLOperatorCharm)
2024-02-28T13:49:14.085Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 436, in main
2024-02-28T13:49:14.085Z [container-agent]     _emit_charm_event(charm, dispatcher.event_name)
2024-02-28T13:49:14.085Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 144, in _emit_charm_event
2024-02-28T13:49:14.085Z [container-agent]     event_to_emit.emit(*args, **kwargs)
2024-02-28T13:49:14.085Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 351, in emit
2024-02-28T13:49:14.085Z [container-agent]     framework._emit(event)
2024-02-28T13:49:14.085Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 853, in _emit
2024-02-28T13:49:14.085Z [container-agent]     self._reemit(event_path)
2024-02-28T13:49:14.085Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 942, in _reemit
2024-02-28T13:49:14.085Z [container-agent]     custom_handler(event)
2024-02-28T13:49:14.086Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/rotate_mysql_logs.py", line 56, in _rotate_mysql_logs
2024-02-28T13:49:14.086Z [container-agent]     self.charm._mysql.flush_mysql_logs(MySQLTextLogs.GENERAL)
2024-02-28T13:49:14.086Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-02-28T13:49:14.086Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-02-28T13:49:14.086Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 666, in _run_mysqlsh_script
2024-02-28T13:49:14.086Z [container-agent]     stdout, _ = process.wait_output()
2024-02-28T13:49:14.086Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1436, in wait_output
2024-02-28T13:49:14.086Z [container-agent]     exit_code: int = self._wait()
2024-02-28T13:49:14.086Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1375, in _wait
2024-02-28T13:49:14.086Z [container-agent]     change = self._client.wait_change(self._change_id, timeout=timeout)
2024-02-28T13:49:14.086Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1893, in wait_change
2024-02-28T13:49:14.086Z [container-agent]     return self._wait_change_using_wait(change_id, timeout)
2024-02-28T13:49:14.086Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1914, in _wait_change_using_wait
2024-02-28T13:49:14.086Z [container-agent]     return self._wait_change(change_id, this_timeout)
2024-02-28T13:49:14.086Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1928, in _wait_change
2024-02-28T13:49:14.086Z [container-agent]     resp = self._request('GET', f'/v1/changes/{change_id}/wait', query)
2024-02-28T13:49:14.086Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1655, in _request
2024-02-28T13:49:14.086Z [container-agent]     response = self._request_raw(method, path, query, headers, data)
2024-02-28T13:49:14.086Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1690, in _request_raw
2024-02-28T13:49:14.086Z [container-agent]     response = self.opener.open(request, timeout=self.timeout)
2024-02-28T13:49:14.086Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 519, in open
2024-02-28T13:49:14.086Z [container-agent]     response = self._open(req, data)
2024-02-28T13:49:14.086Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 536, in _open
2024-02-28T13:49:14.086Z [container-agent]     result = self._call_chain(self.handle_open, protocol, protocol +
2024-02-28T13:49:14.086Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 496, in _call_chain
2024-02-28T13:49:14.086Z [container-agent]     result = func(*args)
2024-02-28T13:49:14.086Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 311, in http_open
2024-02-28T13:49:14.086Z [container-agent]     return self.do_open(_UnixSocketConnection, req,  # type:ignore
2024-02-28T13:49:14.086Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 1352, in do_open
2024-02-28T13:49:14.086Z [container-agent]     r = h.getresponse()
2024-02-28T13:49:14.086Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1375, in getresponse
2024-02-28T13:49:14.086Z [container-agent]     response.begin()
2024-02-28T13:49:14.086Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 318, in begin
2024-02-28T13:49:14.086Z [container-agent]     version, status, reason = self._read_status()
2024-02-28T13:49:14.086Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 287, in _read_status
2024-02-28T13:49:14.086Z [container-agent]     raise RemoteDisconnected("Remote end closed connection without"
2024-02-28T13:49:14.086Z [container-agent] http.client.RemoteDisconnected: Remote end closed connection without response
2024-02-28T13:49:14.254Z [container-agent] 2024-02-28 13:49:14 INFO juju.util.exec exec.go:209 run result: exit status 1
2024-02-28T13:49:15.850Z [container-agent] 2024-02-28 13:49:15 INFO juju-log Starting the log rotate manager
2024-02-28T13:49:15.864Z [container-agent] 2024-02-28 13:49:15 INFO juju-log Started log rotate manager process with PID 151994
2024-02-28T13:49:16.111Z [container-agent] 2024-02-28 13:49:16 INFO juju-log Setting up the logrotate configurations
2024-02-28T13:49:16.171Z [container-agent] 2024-02-28 13:49:16 INFO juju-log Adding pebble layer
2024-02-28T13:49:22.910Z [container-agent] 2024-02-28 13:49:22 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-02-28T13:49:22.942Z [container-agent] 2024-02-28 13:49:22 INFO juju-log Attempting reboot from complete outage.
2024-02-28T13:49:25.879Z [container-agent] 2024-02-28 13:49:25 INFO juju.worker.uniter.operation runhook.go:186 ran "mysql-pebble-ready" hook (via hook dispatching script: dispatch)
2024-02-28T13:51:58.157Z [container-agent] 2024-02-28 13:51:58 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T13:52:00.383Z [container-agent] 2024-02-28 13:52:00 WARNING juju-log No relation: certificates
2024-02-28T13:52:00.769Z [container-agent] 2024-02-28 13:52:00 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T13:57:05.572Z [container-agent] 2024-02-28 13:57:05 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T13:57:08.158Z [container-agent] 2024-02-28 13:57:08 WARNING juju-log No relation: certificates
2024-02-28T13:57:08.482Z [container-agent] 2024-02-28 13:57:08 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T14:02:20.532Z [container-agent] 2024-02-28 14:02:20 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T14:02:23.480Z [container-agent] 2024-02-28 14:02:23 WARNING juju-log No relation: certificates
2024-02-28T14:02:23.821Z [container-agent] 2024-02-28 14:02:23 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T14:06:24.604Z [container-agent] 2024-02-28 14:06:24 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T14:06:28.006Z [container-agent] 2024-02-28 14:06:28 WARNING juju-log No relation: certificates
2024-02-28T14:06:28.353Z [container-agent] 2024-02-28 14:06:28 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T14:11:50.019Z [container-agent] 2024-02-28 14:11:50 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T14:11:54.049Z [container-agent] 2024-02-28 14:11:54 WARNING juju-log No relation: certificates
2024-02-28T14:11:54.412Z [container-agent] 2024-02-28 14:11:54 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T14:16:20.372Z [container-agent] 2024-02-28 14:16:20 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T14:16:24.377Z [container-agent] 2024-02-28 14:16:24 WARNING juju-log No relation: certificates
2024-02-28T14:16:24.703Z [container-agent] 2024-02-28 14:16:24 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T14:21:16.442Z [container-agent] 2024-02-28 14:21:16 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T14:21:21.436Z [container-agent] 2024-02-28 14:21:21 WARNING juju-log No relation: certificates
2024-02-28T14:21:21.797Z [container-agent] 2024-02-28 14:21:21 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T14:25:29.144Z [container-agent] 2024-02-28 14:25:29 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T14:25:34.487Z [container-agent] 2024-02-28 14:25:34 WARNING juju-log No relation: certificates
2024-02-28T14:25:34.835Z [container-agent] 2024-02-28 14:25:34 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T14:29:34.235Z [container-agent] 2024-02-28 14:29:34 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T14:29:39.924Z [container-agent] 2024-02-28 14:29:39 WARNING juju-log No relation: certificates
2024-02-28T14:29:40.279Z [container-agent] 2024-02-28 14:29:40 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T14:34:16.838Z [container-agent] 2024-02-28 14:34:16 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T14:34:22.299Z [container-agent] 2024-02-28 14:34:22 WARNING juju-log No relation: certificates
2024-02-28T14:34:22.670Z [container-agent] 2024-02-28 14:34:22 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T14:40:15.304Z [container-agent] 2024-02-28 14:40:15 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T14:40:22.059Z [container-agent] 2024-02-28 14:40:22 WARNING juju-log No relation: certificates
2024-02-28T14:40:22.419Z [container-agent] 2024-02-28 14:40:22 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T14:46:12.305Z [container-agent] 2024-02-28 14:46:12 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T14:46:19.570Z [container-agent] 2024-02-28 14:46:19 WARNING juju-log No relation: certificates
2024-02-28T14:46:19.936Z [container-agent] 2024-02-28 14:46:19 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T14:51:48.560Z [container-agent] 2024-02-28 14:51:48 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T14:51:56.168Z [container-agent] 2024-02-28 14:51:56 WARNING juju-log No relation: certificates
2024-02-28T14:51:56.526Z [container-agent] 2024-02-28 14:51:56 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T14:52:09.427Z [container-agent] 2024-02-28 14:52:09 ERROR juju-log Failed to flush GENERAL LOGS logs.
2024-02-28T14:52:09.427Z [container-agent] Traceback (most recent call last):
2024-02-28T14:52:09.427Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 666, in _run_mysqlsh_script
2024-02-28T14:52:09.427Z [container-agent]     stdout, _ = process.wait_output()
2024-02-28T14:52:09.427Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1441, in wait_output
2024-02-28T14:52:09.427Z [container-agent]     raise ExecError[AnyStr](self._command, exit_code, out_value, err_value)
2024-02-28T14:52:09.427Z [container-agent] ops.pebble.ExecError: non-zero exit code 1 executing ['/usr/bin/mysqlsh', '--no-wizard', '--python', '--verbose=1', '-f', '/tmp/script.py', ';', 'rm', '/tmp/script.py'], stdout='', stderr='Cannot set LC_ALL to locale en_US.UTF-8: No such file or directory\nverbose: 2024-02-28T14:52:08Z: Loading startup files...\nverbose: 2024-02-28T14:52:08Z: Loading plugins...\nverbose: 2024-02-28T14:52:08Z: Connecting to MySQL at: serverconfig@horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local\nTraceback (most recent call last):\n  File "<string>", line 1, in <module>\nmysqlsh.DBError: MySQL Error (2003): Shell.connect: Can\'t connect to MySQL server on \'horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local:3306\' (111)\n'
2024-02-28T14:52:09.427Z [container-agent] 
2024-02-28T14:52:09.427Z [container-agent] During handling of the above exception, another exception occurred:
2024-02-28T14:52:09.427Z [container-agent] 
2024-02-28T14:52:09.427Z [container-agent] Traceback (most recent call last):
2024-02-28T14:52:09.427Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-02-28T14:52:09.427Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-02-28T14:52:09.427Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 669, in _run_mysqlsh_script
2024-02-28T14:52:09.427Z [container-agent]     raise MySQLClientError(e.stderr)
2024-02-28T14:52:09.427Z [container-agent] charms.mysql.v0.mysql.MySQLClientError: Cannot set LC_ALL to locale en_US.UTF-8: No such file or directory
2024-02-28T14:52:09.427Z [container-agent] verbose: 2024-02-28T14:52:08Z: Loading startup files...
2024-02-28T14:52:09.427Z [container-agent] verbose: 2024-02-28T14:52:08Z: Loading plugins...
2024-02-28T14:52:09.427Z [container-agent] verbose: 2024-02-28T14:52:08Z: Connecting to MySQL at: serverconfig@horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local
2024-02-28T14:52:09.427Z [container-agent] Traceback (most recent call last):
2024-02-28T14:52:09.427Z [container-agent]   File "<string>", line 1, in <module>
2024-02-28T14:52:09.427Z [container-agent] mysqlsh.DBError: MySQL Error (2003): Shell.connect: Can't connect to MySQL server on 'horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local:3306' (111)
2024-02-28T14:52:09.427Z [container-agent] 
2024-02-28T14:57:04.765Z [container-agent] 2024-02-28 14:57:04 ERROR juju-log Uncaught exception while in charm code:
2024-02-28T14:57:04.765Z [container-agent] Traceback (most recent call last):
2024-02-28T14:57:04.765Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/./src/charm.py", line 786, in <module>
2024-02-28T14:57:04.765Z [container-agent]     main(MySQLOperatorCharm)
2024-02-28T14:57:04.765Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 436, in main
2024-02-28T14:57:04.765Z [container-agent]     _emit_charm_event(charm, dispatcher.event_name)
2024-02-28T14:57:04.765Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 144, in _emit_charm_event
2024-02-28T14:57:04.765Z [container-agent]     event_to_emit.emit(*args, **kwargs)
2024-02-28T14:57:04.765Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 351, in emit
2024-02-28T14:57:04.765Z [container-agent]     framework._emit(event)
2024-02-28T14:57:04.765Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 853, in _emit
2024-02-28T14:57:04.765Z [container-agent]     self._reemit(event_path)
2024-02-28T14:57:04.765Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 942, in _reemit
2024-02-28T14:57:04.765Z [container-agent]     custom_handler(event)
2024-02-28T14:57:04.765Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/rotate_mysql_logs.py", line 55, in _rotate_mysql_logs
2024-02-28T14:57:04.765Z [container-agent]     self.charm._mysql.flush_mysql_logs(MySQLTextLogs.ERROR)
2024-02-28T14:57:04.765Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-02-28T14:57:04.765Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-02-28T14:57:04.765Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 666, in _run_mysqlsh_script
2024-02-28T14:57:04.765Z [container-agent]     stdout, _ = process.wait_output()
2024-02-28T14:57:04.765Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1436, in wait_output
2024-02-28T14:57:04.765Z [container-agent]     exit_code: int = self._wait()
2024-02-28T14:57:04.765Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1375, in _wait
2024-02-28T14:57:04.765Z [container-agent]     change = self._client.wait_change(self._change_id, timeout=timeout)
2024-02-28T14:57:04.765Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1893, in wait_change
2024-02-28T14:57:04.765Z [container-agent]     return self._wait_change_using_wait(change_id, timeout)
2024-02-28T14:57:04.765Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1914, in _wait_change_using_wait
2024-02-28T14:57:04.765Z [container-agent]     return self._wait_change(change_id, this_timeout)
2024-02-28T14:57:04.765Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1928, in _wait_change
2024-02-28T14:57:04.765Z [container-agent]     resp = self._request('GET', f'/v1/changes/{change_id}/wait', query)
2024-02-28T14:57:04.765Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1655, in _request
2024-02-28T14:57:04.765Z [container-agent]     response = self._request_raw(method, path, query, headers, data)
2024-02-28T14:57:04.765Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1690, in _request_raw
2024-02-28T14:57:04.765Z [container-agent]     response = self.opener.open(request, timeout=self.timeout)
2024-02-28T14:57:04.765Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 519, in open
2024-02-28T14:57:04.765Z [container-agent]     response = self._open(req, data)
2024-02-28T14:57:04.765Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 536, in _open
2024-02-28T14:57:04.765Z [container-agent]     result = self._call_chain(self.handle_open, protocol, protocol +
2024-02-28T14:57:04.765Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 496, in _call_chain
2024-02-28T14:57:04.765Z [container-agent]     result = func(*args)
2024-02-28T14:57:04.765Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 311, in http_open
2024-02-28T14:57:04.765Z [container-agent]     return self.do_open(_UnixSocketConnection, req,  # type:ignore
2024-02-28T14:57:04.765Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 1352, in do_open
2024-02-28T14:57:04.765Z [container-agent]     r = h.getresponse()
2024-02-28T14:57:04.765Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1375, in getresponse
2024-02-28T14:57:04.765Z [container-agent]     response.begin()
2024-02-28T14:57:04.765Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 318, in begin
2024-02-28T14:57:04.765Z [container-agent]     version, status, reason = self._read_status()
2024-02-28T14:57:04.765Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 287, in _read_status
2024-02-28T14:57:04.765Z [container-agent]     raise RemoteDisconnected("Remote end closed connection without"
2024-02-28T14:57:04.765Z [container-agent] http.client.RemoteDisconnected: Remote end closed connection without response
2024-02-28T14:57:04.967Z [container-agent] 2024-02-28 14:57:04 INFO juju.util.exec exec.go:209 run result: exit status 1
2024-02-28T14:57:06.122Z [container-agent] 2024-02-28 14:57:06 INFO juju-log Starting the log rotate manager
2024-02-28T14:57:06.133Z [container-agent] 2024-02-28 14:57:06 INFO juju-log Started log rotate manager process with PID 158644
2024-02-28T14:57:06.807Z [container-agent] 2024-02-28 14:57:06 ERROR juju-log Failed to get cluster status for cluster-7573647c1e26466beeeac32b95ab8f91
2024-02-28T14:57:06.880Z [container-agent] 2024-02-28 14:57:06 INFO juju-log Kubernetes pod label error created
2024-02-28T14:57:06.894Z [container-agent] 2024-02-28 14:57:06 WARNING juju-log No relation: certificates
2024-02-28T14:57:07.273Z [container-agent] 2024-02-28 14:57:07 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T14:57:08.575Z [container-agent] 2024-02-28 14:57:08 INFO juju-log Setting up the logrotate configurations
2024-02-28T14:57:08.623Z [container-agent] 2024-02-28 14:57:08 INFO juju-log Adding pebble layer
2024-02-28T14:57:15.333Z [container-agent] 2024-02-28 14:57:15 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-02-28T14:57:15.367Z [container-agent] 2024-02-28 14:57:15 INFO juju-log Attempting reboot from complete outage.
2024-02-28T14:57:18.417Z [container-agent] 2024-02-28 14:57:18 INFO juju.worker.uniter.operation runhook.go:186 ran "mysql-pebble-ready" hook (via hook dispatching script: dispatch)
2024-02-28T15:01:09.197Z [container-agent] 2024-02-28 15:01:09 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T15:01:11.447Z [container-agent] 2024-02-28 15:01:11 INFO juju-log Kubernetes pod label primary created
2024-02-28T15:01:11.462Z [container-agent] 2024-02-28 15:01:11 WARNING juju-log No relation: certificates
2024-02-28T15:01:11.863Z [container-agent] 2024-02-28 15:01:11 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T15:07:07.542Z [container-agent] 2024-02-28 15:07:07 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T15:07:10.460Z [container-agent] 2024-02-28 15:07:10 WARNING juju-log No relation: certificates
2024-02-28T15:07:10.794Z [container-agent] 2024-02-28 15:07:10 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T15:12:47.223Z [container-agent] 2024-02-28 15:12:47 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T15:12:50.559Z [container-agent] 2024-02-28 15:12:50 WARNING juju-log No relation: certificates
2024-02-28T15:12:50.895Z [container-agent] 2024-02-28 15:12:50 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T15:17:00.082Z [container-agent] 2024-02-28 15:17:00 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T15:17:03.925Z [container-agent] 2024-02-28 15:17:03 WARNING juju-log No relation: certificates
2024-02-28T15:17:04.290Z [container-agent] 2024-02-28 15:17:04 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T15:22:55.856Z [container-agent] 2024-02-28 15:22:55 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T15:23:00.224Z [container-agent] 2024-02-28 15:23:00 WARNING juju-log No relation: certificates
2024-02-28T15:23:00.573Z [container-agent] 2024-02-28 15:23:00 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T15:28:31.860Z [container-agent] 2024-02-28 15:28:31 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T15:28:36.924Z [container-agent] 2024-02-28 15:28:36 WARNING juju-log No relation: certificates
2024-02-28T15:28:37.262Z [container-agent] 2024-02-28 15:28:37 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T15:33:13.192Z [container-agent] 2024-02-28 15:33:13 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T15:33:18.196Z [container-agent] 2024-02-28 15:33:18 WARNING juju-log No relation: certificates
2024-02-28T15:33:18.547Z [container-agent] 2024-02-28 15:33:18 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T15:37:27.410Z [container-agent] 2024-02-28 15:37:27 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T15:37:33.022Z [container-agent] 2024-02-28 15:37:33 WARNING juju-log No relation: certificates
2024-02-28T15:37:33.376Z [container-agent] 2024-02-28 15:37:33 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T15:42:14.216Z [container-agent] 2024-02-28 15:42:14 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T15:42:20.389Z [container-agent] 2024-02-28 15:42:20 WARNING juju-log No relation: certificates
2024-02-28T15:42:20.725Z [container-agent] 2024-02-28 15:42:20 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T15:47:11.285Z [container-agent] 2024-02-28 15:47:11 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T15:47:17.297Z [container-agent] 2024-02-28 15:47:17 WARNING juju-log No relation: certificates
2024-02-28T15:47:17.645Z [container-agent] 2024-02-28 15:47:17 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T15:53:12.756Z [container-agent] 2024-02-28 15:53:12 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T15:53:19.898Z [container-agent] 2024-02-28 15:53:19 WARNING juju-log No relation: certificates
2024-02-28T15:53:20.235Z [container-agent] 2024-02-28 15:53:20 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T15:58:30.611Z [container-agent] 2024-02-28 15:58:30 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T15:58:38.253Z [container-agent] 2024-02-28 15:58:38 WARNING juju-log No relation: certificates
2024-02-28T15:58:38.669Z [container-agent] 2024-02-28 15:58:38 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T16:03:23.450Z [container-agent] 2024-02-28 16:03:23 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T16:03:31.153Z [container-agent] 2024-02-28 16:03:31 WARNING juju-log No relation: certificates
2024-02-28T16:03:31.523Z [container-agent] 2024-02-28 16:03:31 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T16:07:51.245Z [container-agent] 2024-02-28 16:07:51 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T16:07:59.582Z [container-agent] 2024-02-28 16:07:59 WARNING juju-log No relation: certificates
2024-02-28T16:07:59.930Z [container-agent] 2024-02-28 16:07:59 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T16:10:08.072Z [container-agent] 2024-02-28 16:10:08 ERROR juju-log Failed to flush ERROR LOGS logs.
2024-02-28T16:10:08.072Z [container-agent] Traceback (most recent call last):
2024-02-28T16:10:08.072Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 666, in _run_mysqlsh_script
2024-02-28T16:10:08.072Z [container-agent]     stdout, _ = process.wait_output()
2024-02-28T16:10:08.072Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1441, in wait_output
2024-02-28T16:10:08.072Z [container-agent]     raise ExecError[AnyStr](self._command, exit_code, out_value, err_value)
2024-02-28T16:10:08.072Z [container-agent] ops.pebble.ExecError: non-zero exit code 1 executing ['/usr/bin/mysqlsh', '--no-wizard', '--python', '--verbose=1', '-f', '/tmp/script.py', ';', 'rm', '/tmp/script.py'], stdout='', stderr='Cannot set LC_ALL to locale en_US.UTF-8: No such file or directory\nverbose: 2024-02-28T16:10:06Z: Loading startup files...\nverbose: 2024-02-28T16:10:06Z: Loading plugins...\nverbose: 2024-02-28T16:10:06Z: Connecting to MySQL at: serverconfig@horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local\nTraceback (most recent call last):\n  File "<string>", line 1, in <module>\nmysqlsh.DBError: MySQL Error (2003): Shell.connect: Can\'t connect to MySQL server on \'horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local:3306\' (111)\n'
2024-02-28T16:10:08.072Z [container-agent] 
2024-02-28T16:10:08.072Z [container-agent] During handling of the above exception, another exception occurred:
2024-02-28T16:10:08.072Z [container-agent] 
2024-02-28T16:10:08.072Z [container-agent] Traceback (most recent call last):
2024-02-28T16:10:08.072Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-02-28T16:10:08.072Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-02-28T16:10:08.072Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 669, in _run_mysqlsh_script
2024-02-28T16:10:08.072Z [container-agent]     raise MySQLClientError(e.stderr)
2024-02-28T16:10:08.072Z [container-agent] charms.mysql.v0.mysql.MySQLClientError: Cannot set LC_ALL to locale en_US.UTF-8: No such file or directory
2024-02-28T16:10:08.072Z [container-agent] verbose: 2024-02-28T16:10:06Z: Loading startup files...
2024-02-28T16:10:08.072Z [container-agent] verbose: 2024-02-28T16:10:06Z: Loading plugins...
2024-02-28T16:10:08.072Z [container-agent] verbose: 2024-02-28T16:10:06Z: Connecting to MySQL at: serverconfig@horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local
2024-02-28T16:10:08.072Z [container-agent] Traceback (most recent call last):
2024-02-28T16:10:08.072Z [container-agent]   File "<string>", line 1, in <module>
2024-02-28T16:10:08.072Z [container-agent] mysqlsh.DBError: MySQL Error (2003): Shell.connect: Can't connect to MySQL server on 'horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local:3306' (111)
2024-02-28T16:10:08.072Z [container-agent] 
2024-02-28T16:12:16.819Z [container-agent] 2024-02-28 16:12:16 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-02-28T16:12:16.842Z [container-agent] 2024-02-28 16:12:16 INFO juju-log Attempting reboot from complete outage.
2024-02-28T16:12:24.800Z [container-agent] 2024-02-28 16:12:24 WARNING juju-log No relation: certificates
2024-02-28T16:12:25.157Z [container-agent] 2024-02-28 16:12:25 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T16:15:04.895Z [container-agent] 2024-02-28 16:15:04 ERROR juju-log Uncaught exception while in charm code:
2024-02-28T16:15:04.895Z [container-agent] Traceback (most recent call last):
2024-02-28T16:15:04.895Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 2507, in exec
2024-02-28T16:15:04.895Z [container-agent]     control_ws = self._connect_websocket(task_id, 'control')
2024-02-28T16:15:04.895Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 2589, in _connect_websocket
2024-02-28T16:15:04.895Z [container-agent]     ws.connect(url, socket=sock)
2024-02-28T16:15:04.895Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_core.py", line 253, in connect
2024-02-28T16:15:04.895Z [container-agent]     self.handshake_response = handshake(self.sock, url, *addrs, **options)
2024-02-28T16:15:04.895Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_handshake.py", line 57, in handshake
2024-02-28T16:15:04.895Z [container-agent]     status, resp = _get_resp_headers(sock)
2024-02-28T16:15:04.895Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_handshake.py", line 143, in _get_resp_headers
2024-02-28T16:15:04.895Z [container-agent]     status, resp_headers, status_message = read_headers(sock)
2024-02-28T16:15:04.895Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_http.py", line 312, in read_headers
2024-02-28T16:15:04.895Z [container-agent]     line = recv_line(sock)
2024-02-28T16:15:04.895Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_socket.py", line 131, in recv_line
2024-02-28T16:15:04.895Z [container-agent]     c = recv(sock, 1)
2024-02-28T16:15:04.895Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_socket.py", line 122, in recv
2024-02-28T16:15:04.895Z [container-agent]     raise WebSocketConnectionClosedException(
2024-02-28T16:15:04.895Z [container-agent] websocket._exceptions.WebSocketConnectionClosedException: Connection to remote host was lost.
2024-02-28T16:15:04.895Z [container-agent] 
2024-02-28T16:15:04.895Z [container-agent] During handling of the above exception, another exception occurred:
2024-02-28T16:15:04.895Z [container-agent] 
2024-02-28T16:15:04.895Z [container-agent] Traceback (most recent call last):
2024-02-28T16:15:04.895Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 1348, in do_open
2024-02-28T16:15:04.895Z [container-agent]     h.request(req.get_method(), req.selector, req.data, headers,
2024-02-28T16:15:04.895Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1283, in request
2024-02-28T16:15:04.895Z [container-agent]     self._send_request(method, url, body, headers, encode_chunked)
2024-02-28T16:15:04.895Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1329, in _send_request
2024-02-28T16:15:04.895Z [container-agent]     self.endheaders(body, encode_chunked=encode_chunked)
2024-02-28T16:15:04.895Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1278, in endheaders
2024-02-28T16:15:04.895Z [container-agent]     self._send_output(message_body, encode_chunked=encode_chunked)
2024-02-28T16:15:04.895Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1038, in _send_output
2024-02-28T16:15:04.895Z [container-agent]     self.send(msg)
2024-02-28T16:15:04.895Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 976, in send
2024-02-28T16:15:04.895Z [container-agent]     self.connect()
2024-02-28T16:15:04.895Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 297, in connect
2024-02-28T16:15:04.895Z [container-agent]     self.sock.connect(self.socket_path)
2024-02-28T16:15:04.895Z [container-agent] ConnectionRefusedError: [Errno 111] Connection refused
2024-02-28T16:15:04.895Z [container-agent] 
2024-02-28T16:15:04.895Z [container-agent] During handling of the above exception, another exception occurred:
2024-02-28T16:15:04.895Z [container-agent] 
2024-02-28T16:15:04.895Z [container-agent] Traceback (most recent call last):
2024-02-28T16:15:04.895Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1690, in _request_raw
2024-02-28T16:15:04.895Z [container-agent]     response = self.opener.open(request, timeout=self.timeout)
2024-02-28T16:15:04.896Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 519, in open
2024-02-28T16:15:04.896Z [container-agent]     response = self._open(req, data)
2024-02-28T16:15:04.896Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 536, in _open
2024-02-28T16:15:04.896Z [container-agent]     result = self._call_chain(self.handle_open, protocol, protocol +
2024-02-28T16:15:04.896Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 496, in _call_chain
2024-02-28T16:15:04.896Z [container-agent]     result = func(*args)
2024-02-28T16:15:04.896Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 311, in http_open
2024-02-28T16:15:04.896Z [container-agent]     return self.do_open(_UnixSocketConnection, req,  # type:ignore
2024-02-28T16:15:04.896Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 1351, in do_open
2024-02-28T16:15:04.896Z [container-agent]     raise URLError(err)
2024-02-28T16:15:04.896Z [container-agent] urllib.error.URLError: <urlopen error [Errno 111] Connection refused>
2024-02-28T16:15:04.896Z [container-agent] 
2024-02-28T16:15:04.896Z [container-agent] The above exception was the direct cause of the following exception:
2024-02-28T16:15:04.896Z [container-agent] 
2024-02-28T16:15:04.896Z [container-agent] Traceback (most recent call last):
2024-02-28T16:15:04.896Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/./src/charm.py", line 786, in <module>
2024-02-28T16:15:04.896Z [container-agent]     main(MySQLOperatorCharm)
2024-02-28T16:15:04.896Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 436, in main
2024-02-28T16:15:04.896Z [container-agent]     _emit_charm_event(charm, dispatcher.event_name)
2024-02-28T16:15:04.896Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 144, in _emit_charm_event
2024-02-28T16:15:04.896Z [container-agent]     event_to_emit.emit(*args, **kwargs)
2024-02-28T16:15:04.896Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 351, in emit
2024-02-28T16:15:04.896Z [container-agent]     framework._emit(event)
2024-02-28T16:15:04.896Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 853, in _emit
2024-02-28T16:15:04.896Z [container-agent]     self._reemit(event_path)
2024-02-28T16:15:04.896Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 942, in _reemit
2024-02-28T16:15:04.896Z [container-agent]     custom_handler(event)
2024-02-28T16:15:04.896Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/rotate_mysql_logs.py", line 55, in _rotate_mysql_logs
2024-02-28T16:15:04.896Z [container-agent]     self.charm._mysql.flush_mysql_logs(MySQLTextLogs.ERROR)
2024-02-28T16:15:04.896Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-02-28T16:15:04.896Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-02-28T16:15:04.896Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 665, in _run_mysqlsh_script
2024-02-28T16:15:04.896Z [container-agent]     process = self.container.exec(cmd, timeout=timeout)
2024-02-28T16:15:04.896Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/model.py", line 2688, in exec
2024-02-28T16:15:04.896Z [container-agent]     return self._pebble.exec(
2024-02-28T16:15:04.896Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 2514, in exec
2024-02-28T16:15:04.896Z [container-agent]     change = self.wait_change(ChangeID(change_id))
2024-02-28T16:15:04.896Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1893, in wait_change
2024-02-28T16:15:04.896Z [container-agent]     return self._wait_change_using_wait(change_id, timeout)
2024-02-28T16:15:04.896Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1914, in _wait_change_using_wait
2024-02-28T16:15:04.896Z [container-agent]     return self._wait_change(change_id, this_timeout)
2024-02-28T16:15:04.896Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1928, in _wait_change
2024-02-28T16:15:04.896Z [container-agent]     resp = self._request('GET', f'/v1/changes/{change_id}/wait', query)
2024-02-28T16:15:04.896Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1655, in _request
2024-02-28T16:15:04.896Z [container-agent]     response = self._request_raw(method, path, query, headers, data)
2024-02-28T16:15:04.896Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1707, in _request_raw
2024-02-28T16:15:04.896Z [container-agent]     raise ConnectionError(e.reason) from e
2024-02-28T16:15:04.896Z [container-agent] ops.pebble.ConnectionError: [Errno 111] Connection refused
2024-02-28T16:15:05.040Z [container-agent] 2024-02-28 16:15:05 INFO juju.util.exec exec.go:209 run result: exit status 1
2024-02-28T16:15:06.968Z [container-agent] 2024-02-28 16:15:06 INFO juju-log Setting up the logrotate configurations
2024-02-28T16:15:07.031Z [container-agent] 2024-02-28 16:15:07 INFO juju-log Adding pebble layer
2024-02-28T16:15:13.850Z [container-agent] 2024-02-28 16:15:13 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-02-28T16:15:13.873Z [container-agent] 2024-02-28 16:15:13 INFO juju-log Attempting reboot from complete outage.
2024-02-28T16:15:16.929Z [container-agent] 2024-02-28 16:15:16 INFO juju.worker.uniter.operation runhook.go:186 ran "mysql-pebble-ready" hook (via hook dispatching script: dispatch)
2024-02-28T16:16:36.037Z [container-agent] 2024-02-28 16:16:36 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T16:16:37.997Z [container-agent] 2024-02-28 16:16:37 WARNING juju-log No relation: certificates
2024-02-28T16:16:38.340Z [container-agent] 2024-02-28 16:16:38 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T16:21:03.748Z [container-agent] 2024-02-28 16:21:03 INFO juju-log Starting the log rotate manager
2024-02-28T16:21:03.761Z [container-agent] 2024-02-28 16:21:03 INFO juju-log Started log rotate manager process with PID 166731
2024-02-28T16:21:04.166Z [container-agent] 2024-02-28 16:21:04 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T16:21:06.418Z [container-agent] 2024-02-28 16:21:06 WARNING juju-log No relation: certificates
2024-02-28T16:21:06.796Z [container-agent] 2024-02-28 16:21:06 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T16:26:35.173Z [container-agent] 2024-02-28 16:26:35 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T16:26:37.526Z [container-agent] 2024-02-28 16:26:37 WARNING juju-log No relation: certificates
2024-02-28T16:26:37.852Z [container-agent] 2024-02-28 16:26:37 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T16:31:15.211Z [container-agent] 2024-02-28 16:31:15 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T16:31:18.128Z [container-agent] 2024-02-28 16:31:18 WARNING juju-log No relation: certificates
2024-02-28T16:31:18.544Z [container-agent] 2024-02-28 16:31:18 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T16:36:07.298Z [container-agent] 2024-02-28 16:36:07 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T16:36:10.676Z [container-agent] 2024-02-28 16:36:10 WARNING juju-log No relation: certificates
2024-02-28T16:36:11.034Z [container-agent] 2024-02-28 16:36:11 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T16:40:34.409Z [container-agent] 2024-02-28 16:40:34 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T16:40:38.232Z [container-agent] 2024-02-28 16:40:38 WARNING juju-log No relation: certificates
2024-02-28T16:40:38.597Z [container-agent] 2024-02-28 16:40:38 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T16:45:16.885Z [container-agent] 2024-02-28 16:45:16 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T16:45:21.214Z [container-agent] 2024-02-28 16:45:21 WARNING juju-log No relation: certificates
2024-02-28T16:45:21.568Z [container-agent] 2024-02-28 16:45:21 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T16:51:11.294Z [container-agent] 2024-02-28 16:51:11 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T16:51:16.389Z [container-agent] 2024-02-28 16:51:16 WARNING juju-log No relation: certificates
2024-02-28T16:51:16.730Z [container-agent] 2024-02-28 16:51:16 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T16:56:20.757Z [container-agent] 2024-02-28 16:56:20 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T16:56:26.007Z [container-agent] 2024-02-28 16:56:26 WARNING juju-log No relation: certificates
2024-02-28T16:56:26.353Z [container-agent] 2024-02-28 16:56:26 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T17:00:48.056Z [container-agent] 2024-02-28 17:00:48 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T17:00:53.800Z [container-agent] 2024-02-28 17:00:53 WARNING juju-log No relation: certificates
2024-02-28T17:00:54.145Z [container-agent] 2024-02-28 17:00:54 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T17:06:44.325Z [container-agent] 2024-02-28 17:06:44 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T17:06:50.477Z [container-agent] 2024-02-28 17:06:50 WARNING juju-log No relation: certificates
2024-02-28T17:06:50.830Z [container-agent] 2024-02-28 17:06:50 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T17:12:37.942Z [container-agent] 2024-02-28 17:12:37 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T17:12:44.681Z [container-agent] 2024-02-28 17:12:44 WARNING juju-log No relation: certificates
2024-02-28T17:12:45.046Z [container-agent] 2024-02-28 17:12:45 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T17:17:01.579Z [container-agent] 2024-02-28 17:17:01 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T17:17:08.417Z [container-agent] 2024-02-28 17:17:08 WARNING juju-log No relation: certificates
2024-02-28T17:17:08.757Z [container-agent] 2024-02-28 17:17:08 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T17:22:34.759Z [container-agent] 2024-02-28 17:22:34 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T17:22:42.271Z [container-agent] 2024-02-28 17:22:42 WARNING juju-log No relation: certificates
2024-02-28T17:22:42.609Z [container-agent] 2024-02-28 17:22:42 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T17:27:14.175Z [container-agent] 2024-02-28 17:27:14 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T17:27:22.340Z [container-agent] 2024-02-28 17:27:22 WARNING juju-log No relation: certificates
2024-02-28T17:27:22.679Z [container-agent] 2024-02-28 17:27:22 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T17:31:34.014Z [container-agent] 2024-02-28 17:31:34 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T17:31:42.526Z [container-agent] 2024-02-28 17:31:42 WARNING juju-log No relation: certificates
2024-02-28T17:31:42.876Z [container-agent] 2024-02-28 17:31:42 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T17:37:09.745Z [container-agent] 2024-02-28 17:37:09 ERROR juju-log Uncaught exception while in charm code:
2024-02-28T17:37:09.745Z [container-agent] Traceback (most recent call last):
2024-02-28T17:37:09.745Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/./src/charm.py", line 786, in <module>
2024-02-28T17:37:09.745Z [container-agent]     main(MySQLOperatorCharm)
2024-02-28T17:37:09.745Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 436, in main
2024-02-28T17:37:09.745Z [container-agent]     _emit_charm_event(charm, dispatcher.event_name)
2024-02-28T17:37:09.745Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 144, in _emit_charm_event
2024-02-28T17:37:09.745Z [container-agent]     event_to_emit.emit(*args, **kwargs)
2024-02-28T17:37:09.745Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 351, in emit
2024-02-28T17:37:09.745Z [container-agent]     framework._emit(event)
2024-02-28T17:37:09.745Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 853, in _emit
2024-02-28T17:37:09.745Z [container-agent]     self._reemit(event_path)
2024-02-28T17:37:09.745Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 942, in _reemit
2024-02-28T17:37:09.745Z [container-agent]     custom_handler(event)
2024-02-28T17:37:09.745Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/rotate_mysql_logs.py", line 57, in _rotate_mysql_logs
2024-02-28T17:37:09.745Z [container-agent]     self.charm._mysql.flush_mysql_logs(MySQLTextLogs.SLOW)
2024-02-28T17:37:09.745Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-02-28T17:37:09.745Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-02-28T17:37:09.745Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 665, in _run_mysqlsh_script
2024-02-28T17:37:09.745Z [container-agent]     process = self.container.exec(cmd, timeout=timeout)
2024-02-28T17:37:09.745Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/model.py", line 2688, in exec
2024-02-28T17:37:09.745Z [container-agent]     return self._pebble.exec(
2024-02-28T17:37:09.745Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 2501, in exec
2024-02-28T17:37:09.745Z [container-agent]     resp = self._request('POST', '/v1/exec', body=body)
2024-02-28T17:37:09.745Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1655, in _request
2024-02-28T17:37:09.745Z [container-agent]     response = self._request_raw(method, path, query, headers, data)
2024-02-28T17:37:09.745Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1690, in _request_raw
2024-02-28T17:37:09.745Z [container-agent]     response = self.opener.open(request, timeout=self.timeout)
2024-02-28T17:37:09.745Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 519, in open
2024-02-28T17:37:09.745Z [container-agent]     response = self._open(req, data)
2024-02-28T17:37:09.745Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 536, in _open
2024-02-28T17:37:09.745Z [container-agent]     result = self._call_chain(self.handle_open, protocol, protocol +
2024-02-28T17:37:09.745Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 496, in _call_chain
2024-02-28T17:37:09.745Z [container-agent]     result = func(*args)
2024-02-28T17:37:09.745Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 311, in http_open
2024-02-28T17:37:09.745Z [container-agent]     return self.do_open(_UnixSocketConnection, req,  # type:ignore
2024-02-28T17:37:09.745Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 1352, in do_open
2024-02-28T17:37:09.745Z [container-agent]     r = h.getresponse()
2024-02-28T17:37:09.745Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1375, in getresponse
2024-02-28T17:37:09.745Z [container-agent]     response.begin()
2024-02-28T17:37:09.745Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 318, in begin
2024-02-28T17:37:09.745Z [container-agent]     version, status, reason = self._read_status()
2024-02-28T17:37:09.745Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 287, in _read_status
2024-02-28T17:37:09.745Z [container-agent]     raise RemoteDisconnected("Remote end closed connection without"
2024-02-28T17:37:09.745Z [container-agent] http.client.RemoteDisconnected: Remote end closed connection without response
2024-02-28T17:37:09.904Z [container-agent] 2024-02-28 17:37:09 INFO juju.util.exec exec.go:209 run result: exit status 1
2024-02-28T17:37:14.429Z [container-agent] 2024-02-28 17:37:14 INFO juju-log Starting the log rotate manager
2024-02-28T17:37:14.441Z [container-agent] 2024-02-28 17:37:14 INFO juju-log Started log rotate manager process with PID 174042
2024-02-28T17:37:15.028Z [container-agent] 2024-02-28 17:37:15 ERROR juju-log Failed to get cluster status for cluster-7573647c1e26466beeeac32b95ab8f91
2024-02-28T17:37:15.084Z [container-agent] 2024-02-28 17:37:15 INFO juju-log Kubernetes pod label error created
2024-02-28T17:37:15.096Z [container-agent] 2024-02-28 17:37:15 WARNING juju-log No relation: certificates
2024-02-28T17:37:15.451Z [container-agent] 2024-02-28 17:37:15 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T17:37:17.025Z [container-agent] 2024-02-28 17:37:17 INFO juju-log Setting up the logrotate configurations
2024-02-28T17:37:17.097Z [container-agent] 2024-02-28 17:37:17 INFO juju-log Adding pebble layer
2024-02-28T17:37:23.798Z [container-agent] 2024-02-28 17:37:23 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-02-28T17:37:23.837Z [container-agent] 2024-02-28 17:37:23 INFO juju-log Attempting reboot from complete outage.
2024-02-28T17:37:26.727Z [container-agent] 2024-02-28 17:37:26 INFO juju.worker.uniter.operation runhook.go:186 ran "mysql-pebble-ready" hook (via hook dispatching script: dispatch)
2024-02-28T17:41:49.386Z [container-agent] 2024-02-28 17:41:49 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T17:41:51.687Z [container-agent] 2024-02-28 17:41:51 INFO juju-log Kubernetes pod label primary created
2024-02-28T17:41:51.701Z [container-agent] 2024-02-28 17:41:51 WARNING juju-log No relation: certificates
2024-02-28T17:41:52.050Z [container-agent] 2024-02-28 17:41:52 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T17:46:25.229Z [container-agent] 2024-02-28 17:46:25 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T17:46:27.746Z [container-agent] 2024-02-28 17:46:27 WARNING juju-log No relation: certificates
2024-02-28T17:46:28.070Z [container-agent] 2024-02-28 17:46:28 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T17:50:57.278Z [container-agent] 2024-02-28 17:50:57 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T17:51:00.441Z [container-agent] 2024-02-28 17:51:00 WARNING juju-log No relation: certificates
2024-02-28T17:51:00.796Z [container-agent] 2024-02-28 17:51:00 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T17:55:49.606Z [container-agent] 2024-02-28 17:55:49 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T17:55:53.214Z [container-agent] 2024-02-28 17:55:53 WARNING juju-log No relation: certificates
2024-02-28T17:55:53.562Z [container-agent] 2024-02-28 17:55:53 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T18:01:23.959Z [container-agent] 2024-02-28 18:01:23 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T18:01:27.911Z [container-agent] 2024-02-28 18:01:27 WARNING juju-log No relation: certificates
2024-02-28T18:01:28.253Z [container-agent] 2024-02-28 18:01:28 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T18:05:44.645Z [container-agent] 2024-02-28 18:05:44 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T18:05:49.480Z [container-agent] 2024-02-28 18:05:49 WARNING juju-log No relation: certificates
2024-02-28T18:05:49.842Z [container-agent] 2024-02-28 18:05:49 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T18:10:49.618Z [container-agent] 2024-02-28 18:10:49 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T18:10:54.750Z [container-agent] 2024-02-28 18:10:54 WARNING juju-log No relation: certificates
2024-02-28T18:10:55.089Z [container-agent] 2024-02-28 18:10:55 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T18:16:12.333Z [container-agent] 2024-02-28 18:16:12 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T18:16:17.951Z [container-agent] 2024-02-28 18:16:17 WARNING juju-log No relation: certificates
2024-02-28T18:16:18.298Z [container-agent] 2024-02-28 18:16:18 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T18:21:20.207Z [container-agent] 2024-02-28 18:21:20 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T18:21:26.262Z [container-agent] 2024-02-28 18:21:26 WARNING juju-log No relation: certificates
2024-02-28T18:21:26.620Z [container-agent] 2024-02-28 18:21:26 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T18:25:49.993Z [container-agent] 2024-02-28 18:25:49 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T18:25:56.528Z [container-agent] 2024-02-28 18:25:56 WARNING juju-log No relation: certificates
2024-02-28T18:25:56.916Z [container-agent] 2024-02-28 18:25:56 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T18:30:11.823Z [container-agent] 2024-02-28 18:30:11 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T18:30:18.692Z [container-agent] 2024-02-28 18:30:18 WARNING juju-log No relation: certificates
2024-02-28T18:30:19.049Z [container-agent] 2024-02-28 18:30:19 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T18:34:26.783Z [container-agent] 2024-02-28 18:34:26 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T18:34:34.178Z [container-agent] 2024-02-28 18:34:34 WARNING juju-log No relation: certificates
2024-02-28T18:34:34.519Z [container-agent] 2024-02-28 18:34:34 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T18:38:40.833Z [container-agent] 2024-02-28 18:38:40 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T18:38:48.470Z [container-agent] 2024-02-28 18:38:48 WARNING juju-log No relation: certificates
2024-02-28T18:38:48.849Z [container-agent] 2024-02-28 18:38:48 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T18:43:27.368Z [container-agent] 2024-02-28 18:43:27 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T18:43:35.763Z [container-agent] 2024-02-28 18:43:35 WARNING juju-log No relation: certificates
2024-02-28T18:43:36.094Z [container-agent] 2024-02-28 18:43:36 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T18:47:42.560Z [container-agent] 2024-02-28 18:47:42 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-02-28T18:47:42.594Z [container-agent] 2024-02-28 18:47:42 INFO juju-log Attempting reboot from complete outage.
2024-02-28T18:47:50.596Z [container-agent] 2024-02-28 18:47:50 WARNING juju-log No relation: certificates
2024-02-28T18:47:50.969Z [container-agent] 2024-02-28 18:47:50 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T18:52:04.810Z [container-agent] 2024-02-28 18:52:04 ERROR juju-log Uncaught exception while in charm code:
2024-02-28T18:52:04.810Z [container-agent] Traceback (most recent call last):
2024-02-28T18:52:04.810Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 2507, in exec
2024-02-28T18:52:04.810Z [container-agent]     control_ws = self._connect_websocket(task_id, 'control')
2024-02-28T18:52:04.810Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 2589, in _connect_websocket
2024-02-28T18:52:04.810Z [container-agent]     ws.connect(url, socket=sock)
2024-02-28T18:52:04.810Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_core.py", line 253, in connect
2024-02-28T18:52:04.810Z [container-agent]     self.handshake_response = handshake(self.sock, url, *addrs, **options)
2024-02-28T18:52:04.810Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_handshake.py", line 57, in handshake
2024-02-28T18:52:04.810Z [container-agent]     status, resp = _get_resp_headers(sock)
2024-02-28T18:52:04.810Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_handshake.py", line 143, in _get_resp_headers
2024-02-28T18:52:04.810Z [container-agent]     status, resp_headers, status_message = read_headers(sock)
2024-02-28T18:52:04.810Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_http.py", line 312, in read_headers
2024-02-28T18:52:04.810Z [container-agent]     line = recv_line(sock)
2024-02-28T18:52:04.810Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_socket.py", line 131, in recv_line
2024-02-28T18:52:04.810Z [container-agent]     c = recv(sock, 1)
2024-02-28T18:52:04.810Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_socket.py", line 122, in recv
2024-02-28T18:52:04.810Z [container-agent]     raise WebSocketConnectionClosedException(
2024-02-28T18:52:04.810Z [container-agent] websocket._exceptions.WebSocketConnectionClosedException: Connection to remote host was lost.
2024-02-28T18:52:04.810Z [container-agent] 
2024-02-28T18:52:04.810Z [container-agent] During handling of the above exception, another exception occurred:
2024-02-28T18:52:04.810Z [container-agent] 
2024-02-28T18:52:04.810Z [container-agent] Traceback (most recent call last):
2024-02-28T18:52:04.810Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 1348, in do_open
2024-02-28T18:52:04.810Z [container-agent]     h.request(req.get_method(), req.selector, req.data, headers,
2024-02-28T18:52:04.810Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1283, in request
2024-02-28T18:52:04.810Z [container-agent]     self._send_request(method, url, body, headers, encode_chunked)
2024-02-28T18:52:04.810Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1329, in _send_request
2024-02-28T18:52:04.810Z [container-agent]     self.endheaders(body, encode_chunked=encode_chunked)
2024-02-28T18:52:04.810Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1278, in endheaders
2024-02-28T18:52:04.810Z [container-agent]     self._send_output(message_body, encode_chunked=encode_chunked)
2024-02-28T18:52:04.810Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1038, in _send_output
2024-02-28T18:52:04.810Z [container-agent]     self.send(msg)
2024-02-28T18:52:04.810Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 976, in send
2024-02-28T18:52:04.810Z [container-agent]     self.connect()
2024-02-28T18:52:04.810Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 297, in connect
2024-02-28T18:52:04.810Z [container-agent]     self.sock.connect(self.socket_path)
2024-02-28T18:52:04.810Z [container-agent] ConnectionRefusedError: [Errno 111] Connection refused
2024-02-28T18:52:04.810Z [container-agent] 
2024-02-28T18:52:04.810Z [container-agent] During handling of the above exception, another exception occurred:
2024-02-28T18:52:04.810Z [container-agent] 
2024-02-28T18:52:04.810Z [container-agent] Traceback (most recent call last):
2024-02-28T18:52:04.810Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1690, in _request_raw
2024-02-28T18:52:04.810Z [container-agent]     response = self.opener.open(request, timeout=self.timeout)
2024-02-28T18:52:04.810Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 519, in open
2024-02-28T18:52:04.810Z [container-agent]     response = self._open(req, data)
2024-02-28T18:52:04.810Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 536, in _open
2024-02-28T18:52:04.810Z [container-agent]     result = self._call_chain(self.handle_open, protocol, protocol +
2024-02-28T18:52:04.810Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 496, in _call_chain
2024-02-28T18:52:04.810Z [container-agent]     result = func(*args)
2024-02-28T18:52:04.810Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 311, in http_open
2024-02-28T18:52:04.810Z [container-agent]     return self.do_open(_UnixSocketConnection, req,  # type:ignore
2024-02-28T18:52:04.810Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 1351, in do_open
2024-02-28T18:52:04.810Z [container-agent]     raise URLError(err)
2024-02-28T18:52:04.810Z [container-agent] urllib.error.URLError: <urlopen error [Errno 111] Connection refused>
2024-02-28T18:52:04.810Z [container-agent] 
2024-02-28T18:52:04.810Z [container-agent] The above exception was the direct cause of the following exception:
2024-02-28T18:52:04.810Z [container-agent] 
2024-02-28T18:52:04.810Z [container-agent] Traceback (most recent call last):
2024-02-28T18:52:04.810Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/./src/charm.py", line 786, in <module>
2024-02-28T18:52:04.810Z [container-agent]     main(MySQLOperatorCharm)
2024-02-28T18:52:04.810Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 436, in main
2024-02-28T18:52:04.810Z [container-agent]     _emit_charm_event(charm, dispatcher.event_name)
2024-02-28T18:52:04.810Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 144, in _emit_charm_event
2024-02-28T18:52:04.810Z [container-agent]     event_to_emit.emit(*args, **kwargs)
2024-02-28T18:52:04.810Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 351, in emit
2024-02-28T18:52:04.810Z [container-agent]     framework._emit(event)
2024-02-28T18:52:04.810Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 853, in _emit
2024-02-28T18:52:04.810Z [container-agent]     self._reemit(event_path)
2024-02-28T18:52:04.810Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 942, in _reemit
2024-02-28T18:52:04.810Z [container-agent]     custom_handler(event)
2024-02-28T18:52:04.810Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/rotate_mysql_logs.py", line 55, in _rotate_mysql_logs
2024-02-28T18:52:04.810Z [container-agent]     self.charm._mysql.flush_mysql_logs(MySQLTextLogs.ERROR)
2024-02-28T18:52:04.810Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-02-28T18:52:04.810Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-02-28T18:52:04.810Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 665, in _run_mysqlsh_script
2024-02-28T18:52:04.810Z [container-agent]     process = self.container.exec(cmd, timeout=timeout)
2024-02-28T18:52:04.810Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/model.py", line 2688, in exec
2024-02-28T18:52:04.810Z [container-agent]     return self._pebble.exec(
2024-02-28T18:52:04.810Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 2514, in exec
2024-02-28T18:52:04.810Z [container-agent]     change = self.wait_change(ChangeID(change_id))
2024-02-28T18:52:04.810Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1893, in wait_change
2024-02-28T18:52:04.810Z [container-agent]     return self._wait_change_using_wait(change_id, timeout)
2024-02-28T18:52:04.810Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1914, in _wait_change_using_wait
2024-02-28T18:52:04.810Z [container-agent]     return self._wait_change(change_id, this_timeout)
2024-02-28T18:52:04.810Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1928, in _wait_change
2024-02-28T18:52:04.810Z [container-agent]     resp = self._request('GET', f'/v1/changes/{change_id}/wait', query)
2024-02-28T18:52:04.810Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1655, in _request
2024-02-28T18:52:04.810Z [container-agent]     response = self._request_raw(method, path, query, headers, data)
2024-02-28T18:52:04.810Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1707, in _request_raw
2024-02-28T18:52:04.810Z [container-agent]     raise ConnectionError(e.reason) from e
2024-02-28T18:52:04.810Z [container-agent] ops.pebble.ConnectionError: [Errno 111] Connection refused
2024-02-28T18:52:04.940Z [container-agent] 2024-02-28 18:52:04 INFO juju.util.exec exec.go:209 run result: exit status 1
2024-02-28T18:52:09.781Z [container-agent] 2024-02-28 18:52:09 INFO juju-log Setting up the logrotate configurations
2024-02-28T18:52:09.845Z [container-agent] 2024-02-28 18:52:09 INFO juju-log Adding pebble layer
2024-02-28T18:52:16.571Z [container-agent] 2024-02-28 18:52:16 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-02-28T18:52:16.595Z [container-agent] 2024-02-28 18:52:16 INFO juju-log Attempting reboot from complete outage.
2024-02-28T18:52:19.605Z [container-agent] 2024-02-28 18:52:19 INFO juju.worker.uniter.operation runhook.go:186 ran "mysql-pebble-ready" hook (via hook dispatching script: dispatch)
2024-02-28T18:53:23.299Z [container-agent] 2024-02-28 18:53:23 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T18:53:25.082Z [container-agent] 2024-02-28 18:53:25 WARNING juju-log No relation: certificates
2024-02-28T18:53:25.453Z [container-agent] 2024-02-28 18:53:25 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T18:59:11.685Z [container-agent] 2024-02-28 18:59:11 INFO juju-log Starting the log rotate manager
2024-02-28T18:59:11.698Z [container-agent] 2024-02-28 18:59:11 INFO juju-log Started log rotate manager process with PID 181879
2024-02-28T18:59:11.991Z [container-agent] 2024-02-28 18:59:11 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T18:59:14.050Z [container-agent] 2024-02-28 18:59:14 WARNING juju-log No relation: certificates
2024-02-28T18:59:14.410Z [container-agent] 2024-02-28 18:59:14 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T19:05:06.189Z [container-agent] 2024-02-28 19:05:06 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T19:05:08.789Z [container-agent] 2024-02-28 19:05:08 WARNING juju-log No relation: certificates
2024-02-28T19:05:09.139Z [container-agent] 2024-02-28 19:05:09 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T19:10:01.844Z [container-agent] 2024-02-28 19:10:01 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T19:10:05.169Z [container-agent] 2024-02-28 19:10:05 WARNING juju-log No relation: certificates
2024-02-28T19:10:05.618Z [container-agent] 2024-02-28 19:10:05 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T19:15:07.165Z [container-agent] 2024-02-28 19:15:07 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T19:15:10.606Z [container-agent] 2024-02-28 19:15:10 WARNING juju-log No relation: certificates
2024-02-28T19:15:10.976Z [container-agent] 2024-02-28 19:15:10 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T19:19:54.876Z [container-agent] 2024-02-28 19:19:54 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T19:19:58.876Z [container-agent] 2024-02-28 19:19:58 WARNING juju-log No relation: certificates
2024-02-28T19:19:59.241Z [container-agent] 2024-02-28 19:19:59 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T19:24:08.714Z [container-agent] 2024-02-28 19:24:08 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T19:24:12.813Z [container-agent] 2024-02-28 19:24:12 WARNING juju-log No relation: certificates
2024-02-28T19:24:13.155Z [container-agent] 2024-02-28 19:24:13 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T19:29:09.695Z [container-agent] 2024-02-28 19:29:09 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T19:29:14.641Z [container-agent] 2024-02-28 19:29:14 WARNING juju-log No relation: certificates
2024-02-28T19:29:15.008Z [container-agent] 2024-02-28 19:29:15 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T19:33:20.552Z [container-agent] 2024-02-28 19:33:20 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T19:33:25.665Z [container-agent] 2024-02-28 19:33:25 WARNING juju-log No relation: certificates
2024-02-28T19:33:26.007Z [container-agent] 2024-02-28 19:33:26 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T19:37:58.238Z [container-agent] 2024-02-28 19:37:58 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T19:38:03.949Z [container-agent] 2024-02-28 19:38:03 WARNING juju-log No relation: certificates
2024-02-28T19:38:04.295Z [container-agent] 2024-02-28 19:38:04 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T19:43:26.307Z [container-agent] 2024-02-28 19:43:26 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T19:43:32.629Z [container-agent] 2024-02-28 19:43:32 WARNING juju-log No relation: certificates
2024-02-28T19:43:32.975Z [container-agent] 2024-02-28 19:43:32 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T19:47:42.923Z [container-agent] 2024-02-28 19:47:42 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T19:47:49.605Z [container-agent] 2024-02-28 19:47:49 WARNING juju-log No relation: certificates
2024-02-28T19:47:49.969Z [container-agent] 2024-02-28 19:47:49 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T19:51:50.447Z [container-agent] 2024-02-28 19:51:50 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T19:51:57.109Z [container-agent] 2024-02-28 19:51:57 WARNING juju-log No relation: certificates
2024-02-28T19:51:57.455Z [container-agent] 2024-02-28 19:51:57 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T19:56:02.801Z [container-agent] 2024-02-28 19:56:02 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T19:56:09.352Z [container-agent] 2024-02-28 19:56:09 WARNING juju-log No relation: certificates
2024-02-28T19:56:09.737Z [container-agent] 2024-02-28 19:56:09 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T20:00:20.718Z [container-agent] 2024-02-28 20:00:20 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T20:00:28.495Z [container-agent] 2024-02-28 20:00:28 WARNING juju-log No relation: certificates
2024-02-28T20:00:28.839Z [container-agent] 2024-02-28 20:00:28 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T20:04:55.439Z [container-agent] 2024-02-28 20:04:55 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T20:05:03.162Z [container-agent] 2024-02-28 20:05:03 WARNING juju-log No relation: certificates
2024-02-28T20:05:03.501Z [container-agent] 2024-02-28 20:05:03 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T20:08:12.792Z [container-agent] 2024-02-28 20:08:12 ERROR juju-log Failed to flush SLOW LOGS logs.
2024-02-28T20:08:12.792Z [container-agent] Traceback (most recent call last):
2024-02-28T20:08:12.792Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 666, in _run_mysqlsh_script
2024-02-28T20:08:12.792Z [container-agent]     stdout, _ = process.wait_output()
2024-02-28T20:08:12.792Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1441, in wait_output
2024-02-28T20:08:12.792Z [container-agent]     raise ExecError[AnyStr](self._command, exit_code, out_value, err_value)
2024-02-28T20:08:12.792Z [container-agent] ops.pebble.ExecError: non-zero exit code 1 executing ['/usr/bin/mysqlsh', '--no-wizard', '--python', '--verbose=1', '-f', '/tmp/script.py', ';', 'rm', '/tmp/script.py'], stdout='', stderr='Cannot set LC_ALL to locale en_US.UTF-8: No such file or directory\nverbose: 2024-02-28T20:08:11Z: Loading startup files...\nverbose: 2024-02-28T20:08:11Z: Loading plugins...\nverbose: 2024-02-28T20:08:11Z: Connecting to MySQL at: serverconfig@horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local\nTraceback (most recent call last):\n  File "<string>", line 1, in <module>\nmysqlsh.DBError: MySQL Error (2003): Shell.connect: Can\'t connect to MySQL server on \'horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local:3306\' (111)\n'
2024-02-28T20:08:12.792Z [container-agent] 
2024-02-28T20:08:12.792Z [container-agent] During handling of the above exception, another exception occurred:
2024-02-28T20:08:12.792Z [container-agent] 
2024-02-28T20:08:12.792Z [container-agent] Traceback (most recent call last):
2024-02-28T20:08:12.792Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-02-28T20:08:12.792Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-02-28T20:08:12.792Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 669, in _run_mysqlsh_script
2024-02-28T20:08:12.792Z [container-agent]     raise MySQLClientError(e.stderr)
2024-02-28T20:08:12.792Z [container-agent] charms.mysql.v0.mysql.MySQLClientError: Cannot set LC_ALL to locale en_US.UTF-8: No such file or directory
2024-02-28T20:08:12.792Z [container-agent] verbose: 2024-02-28T20:08:11Z: Loading startup files...
2024-02-28T20:08:12.792Z [container-agent] verbose: 2024-02-28T20:08:11Z: Loading plugins...
2024-02-28T20:08:12.792Z [container-agent] verbose: 2024-02-28T20:08:11Z: Connecting to MySQL at: serverconfig@horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local
2024-02-28T20:08:12.792Z [container-agent] Traceback (most recent call last):
2024-02-28T20:08:12.792Z [container-agent]   File "<string>", line 1, in <module>
2024-02-28T20:08:12.792Z [container-agent] mysqlsh.DBError: MySQL Error (2003): Shell.connect: Can't connect to MySQL server on 'horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local:3306' (111)
2024-02-28T20:08:12.792Z [container-agent] 
2024-02-28T20:09:46.265Z [container-agent] 2024-02-28 20:09:46 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-02-28T20:09:46.296Z [container-agent] 2024-02-28 20:09:46 INFO juju-log Attempting reboot from complete outage.
2024-02-28T20:09:53.537Z [container-agent] 2024-02-28 20:09:53 WARNING juju-log No relation: certificates
2024-02-28T20:09:53.868Z [container-agent] 2024-02-28 20:09:53 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T20:13:09.788Z [container-agent] 2024-02-28 20:13:09 ERROR juju-log Uncaught exception while in charm code:
2024-02-28T20:13:09.788Z [container-agent] Traceback (most recent call last):
2024-02-28T20:13:09.788Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/./src/charm.py", line 786, in <module>
2024-02-28T20:13:09.788Z [container-agent]     main(MySQLOperatorCharm)
2024-02-28T20:13:09.788Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 436, in main
2024-02-28T20:13:09.788Z [container-agent]     _emit_charm_event(charm, dispatcher.event_name)
2024-02-28T20:13:09.788Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 144, in _emit_charm_event
2024-02-28T20:13:09.788Z [container-agent]     event_to_emit.emit(*args, **kwargs)
2024-02-28T20:13:09.788Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 351, in emit
2024-02-28T20:13:09.788Z [container-agent]     framework._emit(event)
2024-02-28T20:13:09.788Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 853, in _emit
2024-02-28T20:13:09.788Z [container-agent]     self._reemit(event_path)
2024-02-28T20:13:09.788Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 942, in _reemit
2024-02-28T20:13:09.788Z [container-agent]     custom_handler(event)
2024-02-28T20:13:09.788Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/rotate_mysql_logs.py", line 57, in _rotate_mysql_logs
2024-02-28T20:13:09.788Z [container-agent]     self.charm._mysql.flush_mysql_logs(MySQLTextLogs.SLOW)
2024-02-28T20:13:09.788Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-02-28T20:13:09.788Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-02-28T20:13:09.788Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 665, in _run_mysqlsh_script
2024-02-28T20:13:09.788Z [container-agent]     process = self.container.exec(cmd, timeout=timeout)
2024-02-28T20:13:09.788Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/model.py", line 2688, in exec
2024-02-28T20:13:09.788Z [container-agent]     return self._pebble.exec(
2024-02-28T20:13:09.788Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 2501, in exec
2024-02-28T20:13:09.788Z [container-agent]     resp = self._request('POST', '/v1/exec', body=body)
2024-02-28T20:13:09.788Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1655, in _request
2024-02-28T20:13:09.788Z [container-agent]     response = self._request_raw(method, path, query, headers, data)
2024-02-28T20:13:09.788Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1690, in _request_raw
2024-02-28T20:13:09.788Z [container-agent]     response = self.opener.open(request, timeout=self.timeout)
2024-02-28T20:13:09.788Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 519, in open
2024-02-28T20:13:09.788Z [container-agent]     response = self._open(req, data)
2024-02-28T20:13:09.788Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 536, in _open
2024-02-28T20:13:09.788Z [container-agent]     result = self._call_chain(self.handle_open, protocol, protocol +
2024-02-28T20:13:09.788Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 496, in _call_chain
2024-02-28T20:13:09.788Z [container-agent]     result = func(*args)
2024-02-28T20:13:09.788Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 311, in http_open
2024-02-28T20:13:09.788Z [container-agent]     return self.do_open(_UnixSocketConnection, req,  # type:ignore
2024-02-28T20:13:09.788Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 1352, in do_open
2024-02-28T20:13:09.788Z [container-agent]     r = h.getresponse()
2024-02-28T20:13:09.788Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1375, in getresponse
2024-02-28T20:13:09.788Z [container-agent]     response.begin()
2024-02-28T20:13:09.788Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 318, in begin
2024-02-28T20:13:09.788Z [container-agent]     version, status, reason = self._read_status()
2024-02-28T20:13:09.788Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 287, in _read_status
2024-02-28T20:13:09.788Z [container-agent]     raise RemoteDisconnected("Remote end closed connection without"
2024-02-28T20:13:09.788Z [container-agent] http.client.RemoteDisconnected: Remote end closed connection without response
2024-02-28T20:13:09.954Z [container-agent] 2024-02-28 20:13:09 INFO juju.util.exec exec.go:209 run result: exit status 1
2024-02-28T20:13:12.853Z [container-agent] 2024-02-28 20:13:12 INFO juju-log Setting up the logrotate configurations
2024-02-28T20:13:12.920Z [container-agent] 2024-02-28 20:13:12 INFO juju-log Adding pebble layer
2024-02-28T20:13:19.668Z [container-agent] 2024-02-28 20:13:19 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-02-28T20:13:19.689Z [container-agent] 2024-02-28 20:13:19 INFO juju-log Attempting reboot from complete outage.
2024-02-28T20:13:22.598Z [container-agent] 2024-02-28 20:13:22 INFO juju.worker.uniter.operation runhook.go:186 ran "mysql-pebble-ready" hook (via hook dispatching script: dispatch)
2024-02-28T20:15:29.677Z [container-agent] 2024-02-28 20:15:29 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T20:15:31.680Z [container-agent] 2024-02-28 20:15:31 WARNING juju-log No relation: certificates
2024-02-28T20:15:32.059Z [container-agent] 2024-02-28 20:15:32 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T20:20:56.699Z [container-agent] 2024-02-28 20:20:56 INFO juju-log Starting the log rotate manager
2024-02-28T20:20:56.713Z [container-agent] 2024-02-28 20:20:56 INFO juju-log Started log rotate manager process with PID 189527
2024-02-28T20:20:56.995Z [container-agent] 2024-02-28 20:20:56 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T20:20:58.965Z [container-agent] 2024-02-28 20:20:58 WARNING juju-log No relation: certificates
2024-02-28T20:20:59.325Z [container-agent] 2024-02-28 20:20:59 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T20:25:12.145Z [container-agent] 2024-02-28 20:25:12 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T20:25:14.794Z [container-agent] 2024-02-28 20:25:14 WARNING juju-log No relation: certificates
2024-02-28T20:25:15.140Z [container-agent] 2024-02-28 20:25:15 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T20:29:48.229Z [container-agent] 2024-02-28 20:29:48 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T20:29:51.074Z [container-agent] 2024-02-28 20:29:51 WARNING juju-log No relation: certificates
2024-02-28T20:29:51.433Z [container-agent] 2024-02-28 20:29:51 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T20:34:58.744Z [container-agent] 2024-02-28 20:34:58 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T20:35:02.242Z [container-agent] 2024-02-28 20:35:02 WARNING juju-log No relation: certificates
2024-02-28T20:35:02.580Z [container-agent] 2024-02-28 20:35:02 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T20:39:18.675Z [container-agent] 2024-02-28 20:39:18 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T20:39:22.657Z [container-agent] 2024-02-28 20:39:22 WARNING juju-log No relation: certificates
2024-02-28T20:39:23.017Z [container-agent] 2024-02-28 20:39:23 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T20:44:43.903Z [container-agent] 2024-02-28 20:44:43 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T20:44:48.349Z [container-agent] 2024-02-28 20:44:48 WARNING juju-log No relation: certificates
2024-02-28T20:44:48.742Z [container-agent] 2024-02-28 20:44:48 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T20:49:12.475Z [container-agent] 2024-02-28 20:49:12 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T20:49:17.355Z [container-agent] 2024-02-28 20:49:17 WARNING juju-log No relation: certificates
2024-02-28T20:49:17.710Z [container-agent] 2024-02-28 20:49:17 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T20:54:37.900Z [container-agent] 2024-02-28 20:54:37 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T20:54:43.124Z [container-agent] 2024-02-28 20:54:43 WARNING juju-log No relation: certificates
2024-02-28T20:54:43.478Z [container-agent] 2024-02-28 20:54:43 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T21:00:25.412Z [container-agent] 2024-02-28 21:00:25 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T21:00:31.177Z [container-agent] 2024-02-28 21:00:31 WARNING juju-log No relation: certificates
2024-02-28T21:00:31.521Z [container-agent] 2024-02-28 21:00:31 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T21:06:11.836Z [container-agent] 2024-02-28 21:06:11 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T21:06:18.262Z [container-agent] 2024-02-28 21:06:18 WARNING juju-log No relation: certificates
2024-02-28T21:06:18.600Z [container-agent] 2024-02-28 21:06:18 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T21:10:17.074Z [container-agent] 2024-02-28 21:10:17 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T21:10:23.765Z [container-agent] 2024-02-28 21:10:23 WARNING juju-log No relation: certificates
2024-02-28T21:10:24.129Z [container-agent] 2024-02-28 21:10:24 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T21:14:54.296Z [container-agent] 2024-02-28 21:14:54 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T21:15:01.300Z [container-agent] 2024-02-28 21:15:01 WARNING juju-log No relation: certificates
2024-02-28T21:15:01.728Z [container-agent] 2024-02-28 21:15:01 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T21:20:15.767Z [container-agent] 2024-02-28 21:20:15 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T21:20:23.417Z [container-agent] 2024-02-28 21:20:23 WARNING juju-log No relation: certificates
2024-02-28T21:20:23.780Z [container-agent] 2024-02-28 21:20:23 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T21:25:14.217Z [container-agent] 2024-02-28 21:25:14 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T21:25:22.534Z [container-agent] 2024-02-28 21:25:22 WARNING juju-log No relation: certificates
2024-02-28T21:25:22.877Z [container-agent] 2024-02-28 21:25:22 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T21:29:27.130Z [container-agent] 2024-02-28 21:29:27 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T21:29:35.598Z [container-agent] 2024-02-28 21:29:35 WARNING juju-log No relation: certificates
2024-02-28T21:29:35.962Z [container-agent] 2024-02-28 21:29:35 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T21:34:15.508Z [container-agent] 2024-02-28 21:34:15 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-02-28T21:34:15.540Z [container-agent] 2024-02-28 21:34:15 INFO juju-log Attempting reboot from complete outage.
2024-02-28T21:34:23.779Z [container-agent] 2024-02-28 21:34:23 WARNING juju-log No relation: certificates
2024-02-28T21:34:24.145Z [container-agent] 2024-02-28 21:34:24 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T21:36:09.793Z [container-agent] 2024-02-28 21:36:09 ERROR juju-log Uncaught exception while in charm code:
2024-02-28T21:36:09.793Z [container-agent] Traceback (most recent call last):
2024-02-28T21:36:09.793Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 2507, in exec
2024-02-28T21:36:09.793Z [container-agent]     control_ws = self._connect_websocket(task_id, 'control')
2024-02-28T21:36:09.793Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 2589, in _connect_websocket
2024-02-28T21:36:09.793Z [container-agent]     ws.connect(url, socket=sock)
2024-02-28T21:36:09.793Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_core.py", line 253, in connect
2024-02-28T21:36:09.793Z [container-agent]     self.handshake_response = handshake(self.sock, url, *addrs, **options)
2024-02-28T21:36:09.793Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_handshake.py", line 57, in handshake
2024-02-28T21:36:09.793Z [container-agent]     status, resp = _get_resp_headers(sock)
2024-02-28T21:36:09.793Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_handshake.py", line 143, in _get_resp_headers
2024-02-28T21:36:09.793Z [container-agent]     status, resp_headers, status_message = read_headers(sock)
2024-02-28T21:36:09.793Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_http.py", line 312, in read_headers
2024-02-28T21:36:09.793Z [container-agent]     line = recv_line(sock)
2024-02-28T21:36:09.793Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_socket.py", line 131, in recv_line
2024-02-28T21:36:09.793Z [container-agent]     c = recv(sock, 1)
2024-02-28T21:36:09.793Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_socket.py", line 122, in recv
2024-02-28T21:36:09.793Z [container-agent]     raise WebSocketConnectionClosedException(
2024-02-28T21:36:09.793Z [container-agent] websocket._exceptions.WebSocketConnectionClosedException: Connection to remote host was lost.
2024-02-28T21:36:09.793Z [container-agent] 
2024-02-28T21:36:09.793Z [container-agent] During handling of the above exception, another exception occurred:
2024-02-28T21:36:09.793Z [container-agent] 
2024-02-28T21:36:09.793Z [container-agent] Traceback (most recent call last):
2024-02-28T21:36:09.793Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 1348, in do_open
2024-02-28T21:36:09.793Z [container-agent]     h.request(req.get_method(), req.selector, req.data, headers,
2024-02-28T21:36:09.793Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1283, in request
2024-02-28T21:36:09.793Z [container-agent]     self._send_request(method, url, body, headers, encode_chunked)
2024-02-28T21:36:09.793Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1329, in _send_request
2024-02-28T21:36:09.793Z [container-agent]     self.endheaders(body, encode_chunked=encode_chunked)
2024-02-28T21:36:09.793Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1278, in endheaders
2024-02-28T21:36:09.793Z [container-agent]     self._send_output(message_body, encode_chunked=encode_chunked)
2024-02-28T21:36:09.793Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1038, in _send_output
2024-02-28T21:36:09.793Z [container-agent]     self.send(msg)
2024-02-28T21:36:09.793Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 976, in send
2024-02-28T21:36:09.793Z [container-agent]     self.connect()
2024-02-28T21:36:09.793Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 297, in connect
2024-02-28T21:36:09.793Z [container-agent]     self.sock.connect(self.socket_path)
2024-02-28T21:36:09.794Z [container-agent] ConnectionRefusedError: [Errno 111] Connection refused
2024-02-28T21:36:09.794Z [container-agent] 
2024-02-28T21:36:09.794Z [container-agent] During handling of the above exception, another exception occurred:
2024-02-28T21:36:09.794Z [container-agent] 
2024-02-28T21:36:09.794Z [container-agent] Traceback (most recent call last):
2024-02-28T21:36:09.794Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1690, in _request_raw
2024-02-28T21:36:09.794Z [container-agent]     response = self.opener.open(request, timeout=self.timeout)
2024-02-28T21:36:09.794Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 519, in open
2024-02-28T21:36:09.794Z [container-agent]     response = self._open(req, data)
2024-02-28T21:36:09.794Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 536, in _open
2024-02-28T21:36:09.794Z [container-agent]     result = self._call_chain(self.handle_open, protocol, protocol +
2024-02-28T21:36:09.794Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 496, in _call_chain
2024-02-28T21:36:09.794Z [container-agent]     result = func(*args)
2024-02-28T21:36:09.794Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 311, in http_open
2024-02-28T21:36:09.794Z [container-agent]     return self.do_open(_UnixSocketConnection, req,  # type:ignore
2024-02-28T21:36:09.794Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 1351, in do_open
2024-02-28T21:36:09.794Z [container-agent]     raise URLError(err)
2024-02-28T21:36:09.794Z [container-agent] urllib.error.URLError: <urlopen error [Errno 111] Connection refused>
2024-02-28T21:36:09.794Z [container-agent] 
2024-02-28T21:36:09.794Z [container-agent] The above exception was the direct cause of the following exception:
2024-02-28T21:36:09.794Z [container-agent] 
2024-02-28T21:36:09.794Z [container-agent] Traceback (most recent call last):
2024-02-28T21:36:09.794Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/./src/charm.py", line 786, in <module>
2024-02-28T21:36:09.794Z [container-agent]     main(MySQLOperatorCharm)
2024-02-28T21:36:09.794Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 436, in main
2024-02-28T21:36:09.794Z [container-agent]     _emit_charm_event(charm, dispatcher.event_name)
2024-02-28T21:36:09.794Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 144, in _emit_charm_event
2024-02-28T21:36:09.794Z [container-agent]     event_to_emit.emit(*args, **kwargs)
2024-02-28T21:36:09.794Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 351, in emit
2024-02-28T21:36:09.794Z [container-agent]     framework._emit(event)
2024-02-28T21:36:09.794Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 853, in _emit
2024-02-28T21:36:09.794Z [container-agent]     self._reemit(event_path)
2024-02-28T21:36:09.794Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 942, in _reemit
2024-02-28T21:36:09.794Z [container-agent]     custom_handler(event)
2024-02-28T21:36:09.794Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/rotate_mysql_logs.py", line 57, in _rotate_mysql_logs
2024-02-28T21:36:09.794Z [container-agent]     self.charm._mysql.flush_mysql_logs(MySQLTextLogs.SLOW)
2024-02-28T21:36:09.794Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-02-28T21:36:09.794Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-02-28T21:36:09.794Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 665, in _run_mysqlsh_script
2024-02-28T21:36:09.794Z [container-agent]     process = self.container.exec(cmd, timeout=timeout)
2024-02-28T21:36:09.794Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/model.py", line 2688, in exec
2024-02-28T21:36:09.794Z [container-agent]     return self._pebble.exec(
2024-02-28T21:36:09.794Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 2514, in exec
2024-02-28T21:36:09.794Z [container-agent]     change = self.wait_change(ChangeID(change_id))
2024-02-28T21:36:09.794Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1893, in wait_change
2024-02-28T21:36:09.794Z [container-agent]     return self._wait_change_using_wait(change_id, timeout)
2024-02-28T21:36:09.794Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1914, in _wait_change_using_wait
2024-02-28T21:36:09.794Z [container-agent]     return self._wait_change(change_id, this_timeout)
2024-02-28T21:36:09.794Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1928, in _wait_change
2024-02-28T21:36:09.794Z [container-agent]     resp = self._request('GET', f'/v1/changes/{change_id}/wait', query)
2024-02-28T21:36:09.794Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1655, in _request
2024-02-28T21:36:09.794Z [container-agent]     response = self._request_raw(method, path, query, headers, data)
2024-02-28T21:36:09.794Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1707, in _request_raw
2024-02-28T21:36:09.794Z [container-agent]     raise ConnectionError(e.reason) from e
2024-02-28T21:36:09.794Z [container-agent] ops.pebble.ConnectionError: [Errno 111] Connection refused
2024-02-28T21:36:09.917Z [container-agent] 2024-02-28 21:36:09 INFO juju.util.exec exec.go:209 run result: exit status 1
2024-02-28T21:36:11.736Z [container-agent] 2024-02-28 21:36:11 INFO juju-log Setting up the logrotate configurations
2024-02-28T21:36:11.804Z [container-agent] 2024-02-28 21:36:11 INFO juju-log Adding pebble layer
2024-02-28T21:36:18.597Z [container-agent] 2024-02-28 21:36:18 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-02-28T21:36:18.623Z [container-agent] 2024-02-28 21:36:18 INFO juju-log Attempting reboot from complete outage.
2024-02-28T21:36:21.619Z [container-agent] 2024-02-28 21:36:21 INFO juju.worker.uniter.operation runhook.go:186 ran "mysql-pebble-ready" hook (via hook dispatching script: dispatch)
2024-02-28T21:38:37.248Z [container-agent] 2024-02-28 21:38:37 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T21:38:39.164Z [container-agent] 2024-02-28 21:38:39 WARNING juju-log No relation: certificates
2024-02-28T21:38:39.519Z [container-agent] 2024-02-28 21:38:39 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T21:44:11.926Z [container-agent] 2024-02-28 21:44:11 INFO juju-log Starting the log rotate manager
2024-02-28T21:44:11.939Z [container-agent] 2024-02-28 21:44:11 INFO juju-log Started log rotate manager process with PID 197320
2024-02-28T21:44:12.244Z [container-agent] 2024-02-28 21:44:12 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T21:44:14.297Z [container-agent] 2024-02-28 21:44:14 WARNING juju-log No relation: certificates
2024-02-28T21:44:14.652Z [container-agent] 2024-02-28 21:44:14 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T21:48:28.899Z [container-agent] 2024-02-28 21:48:28 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T21:48:31.511Z [container-agent] 2024-02-28 21:48:31 WARNING juju-log No relation: certificates
2024-02-28T21:48:31.870Z [container-agent] 2024-02-28 21:48:31 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T21:52:40.830Z [container-agent] 2024-02-28 21:52:40 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T21:52:43.615Z [container-agent] 2024-02-28 21:52:43 WARNING juju-log No relation: certificates
2024-02-28T21:52:44.001Z [container-agent] 2024-02-28 21:52:44 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T21:58:21.680Z [container-agent] 2024-02-28 21:58:21 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T21:58:24.940Z [container-agent] 2024-02-28 21:58:24 WARNING juju-log No relation: certificates
2024-02-28T21:58:25.293Z [container-agent] 2024-02-28 21:58:25 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T22:03:33.776Z [container-agent] 2024-02-28 22:03:33 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T22:03:37.580Z [container-agent] 2024-02-28 22:03:37 WARNING juju-log No relation: certificates
2024-02-28T22:03:37.923Z [container-agent] 2024-02-28 22:03:37 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T22:07:57.750Z [container-agent] 2024-02-28 22:07:57 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T22:08:02.047Z [container-agent] 2024-02-28 22:08:02 WARNING juju-log No relation: certificates
2024-02-28T22:08:02.462Z [container-agent] 2024-02-28 22:08:02 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T22:13:13.243Z [container-agent] 2024-02-28 22:13:13 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T22:13:18.115Z [container-agent] 2024-02-28 22:13:18 WARNING juju-log No relation: certificates
2024-02-28T22:13:18.513Z [container-agent] 2024-02-28 22:13:18 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T22:19:09.905Z [container-agent] 2024-02-28 22:19:09 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T22:19:15.145Z [container-agent] 2024-02-28 22:19:15 WARNING juju-log No relation: certificates
2024-02-28T22:19:15.513Z [container-agent] 2024-02-28 22:19:15 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T22:24:14.211Z [container-agent] 2024-02-28 22:24:14 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T22:24:19.858Z [container-agent] 2024-02-28 22:24:19 WARNING juju-log No relation: certificates
2024-02-28T22:24:20.201Z [container-agent] 2024-02-28 22:24:20 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T22:29:57.831Z [container-agent] 2024-02-28 22:29:57 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T22:30:03.727Z [container-agent] 2024-02-28 22:30:03 WARNING juju-log No relation: certificates
2024-02-28T22:30:04.087Z [container-agent] 2024-02-28 22:30:04 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T22:34:13.344Z [container-agent] 2024-02-28 22:34:13 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T22:34:20.437Z [container-agent] 2024-02-28 22:34:20 WARNING juju-log No relation: certificates
2024-02-28T22:34:20.784Z [container-agent] 2024-02-28 22:34:20 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T22:40:12.619Z [container-agent] 2024-02-28 22:40:12 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T22:40:19.987Z [container-agent] 2024-02-28 22:40:19 WARNING juju-log No relation: certificates
2024-02-28T22:40:20.369Z [container-agent] 2024-02-28 22:40:20 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T22:44:16.542Z [container-agent] 2024-02-28 22:44:16 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T22:44:24.339Z [container-agent] 2024-02-28 22:44:24 WARNING juju-log No relation: certificates
2024-02-28T22:44:24.677Z [container-agent] 2024-02-28 22:44:24 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T22:49:18.170Z [container-agent] 2024-02-28 22:49:18 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T22:49:26.147Z [container-agent] 2024-02-28 22:49:26 WARNING juju-log No relation: certificates
2024-02-28T22:49:26.522Z [container-agent] 2024-02-28 22:49:26 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T22:54:14.832Z [container-agent] 2024-02-28 22:54:14 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T22:54:23.365Z [container-agent] 2024-02-28 22:54:23 WARNING juju-log No relation: certificates
2024-02-28T22:54:23.696Z [container-agent] 2024-02-28 22:54:23 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T22:58:15.904Z [container-agent] 2024-02-28 22:58:15 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T22:58:25.025Z [container-agent] 2024-02-28 22:58:25 WARNING juju-log No relation: certificates
2024-02-28T22:58:25.374Z [container-agent] 2024-02-28 22:58:25 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T23:04:16.326Z [container-agent] 2024-02-28 23:04:16 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T23:04:25.440Z [container-agent] 2024-02-28 23:04:25 WARNING juju-log No relation: certificates
2024-02-28T23:04:25.837Z [container-agent] 2024-02-28 23:04:25 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T23:09:18.043Z [container-agent] 2024-02-28 23:09:18 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T23:09:28.259Z [container-agent] 2024-02-28 23:09:28 WARNING juju-log No relation: certificates
2024-02-28T23:09:28.601Z [container-agent] 2024-02-28 23:09:28 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T23:14:44.286Z [container-agent] 2024-02-28 23:14:44 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-02-28T23:14:44.320Z [container-agent] 2024-02-28 23:14:44 INFO juju-log Attempting reboot from complete outage.
2024-02-28T23:14:53.534Z [container-agent] 2024-02-28 23:14:53 WARNING juju-log No relation: certificates
2024-02-28T23:14:53.919Z [container-agent] 2024-02-28 23:14:53 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T23:19:09.814Z [container-agent] 2024-02-28 23:19:09 ERROR juju-log Uncaught exception while in charm code:
2024-02-28T23:19:09.814Z [container-agent] Traceback (most recent call last):
2024-02-28T23:19:09.814Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/./src/charm.py", line 786, in <module>
2024-02-28T23:19:09.814Z [container-agent]     main(MySQLOperatorCharm)
2024-02-28T23:19:09.814Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 436, in main
2024-02-28T23:19:09.814Z [container-agent]     _emit_charm_event(charm, dispatcher.event_name)
2024-02-28T23:19:09.814Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 144, in _emit_charm_event
2024-02-28T23:19:09.814Z [container-agent]     event_to_emit.emit(*args, **kwargs)
2024-02-28T23:19:09.814Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 351, in emit
2024-02-28T23:19:09.814Z [container-agent]     framework._emit(event)
2024-02-28T23:19:09.814Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 853, in _emit
2024-02-28T23:19:09.814Z [container-agent]     self._reemit(event_path)
2024-02-28T23:19:09.814Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 942, in _reemit
2024-02-28T23:19:09.814Z [container-agent]     custom_handler(event)
2024-02-28T23:19:09.814Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/rotate_mysql_logs.py", line 56, in _rotate_mysql_logs
2024-02-28T23:19:09.814Z [container-agent]     self.charm._mysql.flush_mysql_logs(MySQLTextLogs.GENERAL)
2024-02-28T23:19:09.814Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-02-28T23:19:09.814Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-02-28T23:19:09.814Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 666, in _run_mysqlsh_script
2024-02-28T23:19:09.814Z [container-agent]     stdout, _ = process.wait_output()
2024-02-28T23:19:09.814Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1436, in wait_output
2024-02-28T23:19:09.814Z [container-agent]     exit_code: int = self._wait()
2024-02-28T23:19:09.814Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1375, in _wait
2024-02-28T23:19:09.814Z [container-agent]     change = self._client.wait_change(self._change_id, timeout=timeout)
2024-02-28T23:19:09.814Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1893, in wait_change
2024-02-28T23:19:09.814Z [container-agent]     return self._wait_change_using_wait(change_id, timeout)
2024-02-28T23:19:09.814Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1914, in _wait_change_using_wait
2024-02-28T23:19:09.814Z [container-agent]     return self._wait_change(change_id, this_timeout)
2024-02-28T23:19:09.814Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1928, in _wait_change
2024-02-28T23:19:09.814Z [container-agent]     resp = self._request('GET', f'/v1/changes/{change_id}/wait', query)
2024-02-28T23:19:09.814Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1655, in _request
2024-02-28T23:19:09.814Z [container-agent]     response = self._request_raw(method, path, query, headers, data)
2024-02-28T23:19:09.814Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1690, in _request_raw
2024-02-28T23:19:09.814Z [container-agent]     response = self.opener.open(request, timeout=self.timeout)
2024-02-28T23:19:09.814Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 519, in open
2024-02-28T23:19:09.814Z [container-agent]     response = self._open(req, data)
2024-02-28T23:19:09.814Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 536, in _open
2024-02-28T23:19:09.814Z [container-agent]     result = self._call_chain(self.handle_open, protocol, protocol +
2024-02-28T23:19:09.814Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 496, in _call_chain
2024-02-28T23:19:09.814Z [container-agent]     result = func(*args)
2024-02-28T23:19:09.814Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 311, in http_open
2024-02-28T23:19:09.814Z [container-agent]     return self.do_open(_UnixSocketConnection, req,  # type:ignore
2024-02-28T23:19:09.814Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 1352, in do_open
2024-02-28T23:19:09.814Z [container-agent]     r = h.getresponse()
2024-02-28T23:19:09.814Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1375, in getresponse
2024-02-28T23:19:09.814Z [container-agent]     response.begin()
2024-02-28T23:19:09.814Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 318, in begin
2024-02-28T23:19:09.814Z [container-agent]     version, status, reason = self._read_status()
2024-02-28T23:19:09.814Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 287, in _read_status
2024-02-28T23:19:09.814Z [container-agent]     raise RemoteDisconnected("Remote end closed connection without"
2024-02-28T23:19:09.814Z [container-agent] http.client.RemoteDisconnected: Remote end closed connection without response
2024-02-28T23:19:09.940Z [container-agent] 2024-02-28 23:19:09 INFO juju.util.exec exec.go:209 run result: exit status 1
2024-02-28T23:19:11.720Z [container-agent] 2024-02-28 23:19:11 INFO juju-log Setting up the logrotate configurations
2024-02-28T23:19:11.774Z [container-agent] 2024-02-28 23:19:11 INFO juju-log Adding pebble layer
2024-02-28T23:19:18.516Z [container-agent] 2024-02-28 23:19:18 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-02-28T23:19:18.542Z [container-agent] 2024-02-28 23:19:18 INFO juju-log Attempting reboot from complete outage.
2024-02-28T23:19:21.619Z [container-agent] 2024-02-28 23:19:21 INFO juju.worker.uniter.operation runhook.go:186 ran "mysql-pebble-ready" hook (via hook dispatching script: dispatch)
2024-02-28T23:20:16.401Z [container-agent] 2024-02-28 23:20:16 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T23:20:18.463Z [container-agent] 2024-02-28 23:20:18 WARNING juju-log No relation: certificates
2024-02-28T23:20:18.846Z [container-agent] 2024-02-28 23:20:18 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T23:24:50.373Z [container-agent] 2024-02-28 23:24:50 INFO juju-log Starting the log rotate manager
2024-02-28T23:24:50.386Z [container-agent] 2024-02-28 23:24:50 INFO juju-log Started log rotate manager process with PID 206817
2024-02-28T23:24:50.743Z [container-agent] 2024-02-28 23:24:50 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T23:24:52.730Z [container-agent] 2024-02-28 23:24:52 WARNING juju-log No relation: certificates
2024-02-28T23:24:53.072Z [container-agent] 2024-02-28 23:24:53 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T23:29:33.941Z [container-agent] 2024-02-28 23:29:33 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T23:29:36.335Z [container-agent] 2024-02-28 23:29:36 WARNING juju-log No relation: certificates
2024-02-28T23:29:36.730Z [container-agent] 2024-02-28 23:29:36 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T23:35:23.194Z [container-agent] 2024-02-28 23:35:23 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T23:35:26.023Z [container-agent] 2024-02-28 23:35:26 WARNING juju-log No relation: certificates
2024-02-28T23:35:26.352Z [container-agent] 2024-02-28 23:35:26 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T23:39:43.130Z [container-agent] 2024-02-28 23:39:43 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T23:39:46.587Z [container-agent] 2024-02-28 23:39:46 WARNING juju-log No relation: certificates
2024-02-28T23:39:46.961Z [container-agent] 2024-02-28 23:39:46 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T23:44:56.509Z [container-agent] 2024-02-28 23:44:56 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T23:45:00.278Z [container-agent] 2024-02-28 23:45:00 WARNING juju-log No relation: certificates
2024-02-28T23:45:00.689Z [container-agent] 2024-02-28 23:45:00 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T23:48:58.360Z [container-agent] 2024-02-28 23:48:58 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T23:49:02.646Z [container-agent] 2024-02-28 23:49:02 WARNING juju-log No relation: certificates
2024-02-28T23:49:03.008Z [container-agent] 2024-02-28 23:49:03 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T23:53:13.065Z [container-agent] 2024-02-28 23:53:13 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T23:53:17.805Z [container-agent] 2024-02-28 23:53:17 WARNING juju-log No relation: certificates
2024-02-28T23:53:18.149Z [container-agent] 2024-02-28 23:53:18 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-28T23:58:21.383Z [container-agent] 2024-02-28 23:58:21 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-28T23:58:25.901Z [container-agent] 2024-02-28 23:58:25 WARNING juju-log No relation: certificates
2024-02-28T23:58:26.256Z [container-agent] 2024-02-28 23:58:26 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T00:02:48.482Z [container-agent] 2024-02-29 00:02:48 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T00:02:53.990Z [container-agent] 2024-02-29 00:02:53 WARNING juju-log No relation: certificates
2024-02-29T00:02:54.337Z [container-agent] 2024-02-29 00:02:54 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T00:06:51.599Z [container-agent] 2024-02-29 00:06:51 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T00:06:57.604Z [container-agent] 2024-02-29 00:06:57 WARNING juju-log No relation: certificates
2024-02-29T00:06:57.990Z [container-agent] 2024-02-29 00:06:57 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T00:11:11.525Z [container-agent] 2024-02-29 00:11:11 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T00:11:18.035Z [container-agent] 2024-02-29 00:11:18 WARNING juju-log No relation: certificates
2024-02-29T00:11:18.376Z [container-agent] 2024-02-29 00:11:18 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T00:15:45.793Z [container-agent] 2024-02-29 00:15:45 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T00:15:52.108Z [container-agent] 2024-02-29 00:15:52 WARNING juju-log No relation: certificates
2024-02-29T00:15:52.475Z [container-agent] 2024-02-29 00:15:52 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T00:19:47.787Z [container-agent] 2024-02-29 00:19:47 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T00:19:54.911Z [container-agent] 2024-02-29 00:19:54 WARNING juju-log No relation: certificates
2024-02-29T00:19:55.268Z [container-agent] 2024-02-29 00:19:55 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T00:23:07.248Z [container-agent] 2024-02-29 00:23:07 ERROR juju-log Failed to flush ERROR LOGS logs.
2024-02-29T00:23:07.248Z [container-agent] Traceback (most recent call last):
2024-02-29T00:23:07.248Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 666, in _run_mysqlsh_script
2024-02-29T00:23:07.248Z [container-agent]     stdout, _ = process.wait_output()
2024-02-29T00:23:07.248Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1441, in wait_output
2024-02-29T00:23:07.248Z [container-agent]     raise ExecError[AnyStr](self._command, exit_code, out_value, err_value)
2024-02-29T00:23:07.248Z [container-agent] ops.pebble.ExecError: non-zero exit code 1 executing ['/usr/bin/mysqlsh', '--no-wizard', '--python', '--verbose=1', '-f', '/tmp/script.py', ';', 'rm', '/tmp/script.py'], stdout='', stderr='Cannot set LC_ALL to locale en_US.UTF-8: No such file or directory\nverbose: 2024-02-29T00:23:06Z: Loading startup files...\nverbose: 2024-02-29T00:23:06Z: Loading plugins...\nverbose: 2024-02-29T00:23:06Z: Connecting to MySQL at: serverconfig@horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local\nTraceback (most recent call last):\n  File "<string>", line 1, in <module>\nmysqlsh.DBError: MySQL Error (2003): Shell.connect: Can\'t connect to MySQL server on \'horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local:3306\' (111)\n'
2024-02-29T00:23:07.248Z [container-agent] 
2024-02-29T00:23:07.248Z [container-agent] During handling of the above exception, another exception occurred:
2024-02-29T00:23:07.248Z [container-agent] 
2024-02-29T00:23:07.248Z [container-agent] Traceback (most recent call last):
2024-02-29T00:23:07.248Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-02-29T00:23:07.248Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-02-29T00:23:07.248Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 669, in _run_mysqlsh_script
2024-02-29T00:23:07.248Z [container-agent]     raise MySQLClientError(e.stderr)
2024-02-29T00:23:07.248Z [container-agent] charms.mysql.v0.mysql.MySQLClientError: Cannot set LC_ALL to locale en_US.UTF-8: No such file or directory
2024-02-29T00:23:07.248Z [container-agent] verbose: 2024-02-29T00:23:06Z: Loading startup files...
2024-02-29T00:23:07.248Z [container-agent] verbose: 2024-02-29T00:23:06Z: Loading plugins...
2024-02-29T00:23:07.248Z [container-agent] verbose: 2024-02-29T00:23:06Z: Connecting to MySQL at: serverconfig@horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local
2024-02-29T00:23:07.248Z [container-agent] Traceback (most recent call last):
2024-02-29T00:23:07.248Z [container-agent]   File "<string>", line 1, in <module>
2024-02-29T00:23:07.248Z [container-agent] mysqlsh.DBError: MySQL Error (2003): Shell.connect: Can't connect to MySQL server on 'horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local:3306' (111)
2024-02-29T00:23:07.248Z [container-agent] 
2024-02-29T00:25:15.448Z [container-agent] 2024-02-29 00:25:15 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-02-29T00:25:15.481Z [container-agent] 2024-02-29 00:25:15 INFO juju-log Attempting reboot from complete outage.
2024-02-29T00:25:22.727Z [container-agent] 2024-02-29 00:25:22 WARNING juju-log No relation: certificates
2024-02-29T00:25:23.097Z [container-agent] 2024-02-29 00:25:23 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T00:28:04.814Z [container-agent] 2024-02-29 00:28:04 ERROR juju-log Uncaught exception while in charm code:
2024-02-29T00:28:04.814Z [container-agent] Traceback (most recent call last):
2024-02-29T00:28:04.814Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/./src/charm.py", line 786, in <module>
2024-02-29T00:28:04.814Z [container-agent]     main(MySQLOperatorCharm)
2024-02-29T00:28:04.814Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 436, in main
2024-02-29T00:28:04.814Z [container-agent]     _emit_charm_event(charm, dispatcher.event_name)
2024-02-29T00:28:04.814Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 144, in _emit_charm_event
2024-02-29T00:28:04.814Z [container-agent]     event_to_emit.emit(*args, **kwargs)
2024-02-29T00:28:04.814Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 351, in emit
2024-02-29T00:28:04.814Z [container-agent]     framework._emit(event)
2024-02-29T00:28:04.814Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 853, in _emit
2024-02-29T00:28:04.814Z [container-agent]     self._reemit(event_path)
2024-02-29T00:28:04.814Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 942, in _reemit
2024-02-29T00:28:04.814Z [container-agent]     custom_handler(event)
2024-02-29T00:28:04.814Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/rotate_mysql_logs.py", line 55, in _rotate_mysql_logs
2024-02-29T00:28:04.814Z [container-agent]     self.charm._mysql.flush_mysql_logs(MySQLTextLogs.ERROR)
2024-02-29T00:28:04.814Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-02-29T00:28:04.814Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-02-29T00:28:04.814Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 665, in _run_mysqlsh_script
2024-02-29T00:28:04.814Z [container-agent]     process = self.container.exec(cmd, timeout=timeout)
2024-02-29T00:28:04.814Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/model.py", line 2688, in exec
2024-02-29T00:28:04.814Z [container-agent]     return self._pebble.exec(
2024-02-29T00:28:04.814Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 2501, in exec
2024-02-29T00:28:04.814Z [container-agent]     resp = self._request('POST', '/v1/exec', body=body)
2024-02-29T00:28:04.814Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1655, in _request
2024-02-29T00:28:04.814Z [container-agent]     response = self._request_raw(method, path, query, headers, data)
2024-02-29T00:28:04.814Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1690, in _request_raw
2024-02-29T00:28:04.814Z [container-agent]     response = self.opener.open(request, timeout=self.timeout)
2024-02-29T00:28:04.814Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 519, in open
2024-02-29T00:28:04.814Z [container-agent]     response = self._open(req, data)
2024-02-29T00:28:04.814Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 536, in _open
2024-02-29T00:28:04.814Z [container-agent]     result = self._call_chain(self.handle_open, protocol, protocol +
2024-02-29T00:28:04.814Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 496, in _call_chain
2024-02-29T00:28:04.814Z [container-agent]     result = func(*args)
2024-02-29T00:28:04.814Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 311, in http_open
2024-02-29T00:28:04.814Z [container-agent]     return self.do_open(_UnixSocketConnection, req,  # type:ignore
2024-02-29T00:28:04.814Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 1352, in do_open
2024-02-29T00:28:04.814Z [container-agent]     r = h.getresponse()
2024-02-29T00:28:04.814Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1375, in getresponse
2024-02-29T00:28:04.814Z [container-agent]     response.begin()
2024-02-29T00:28:04.814Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 318, in begin
2024-02-29T00:28:04.814Z [container-agent]     version, status, reason = self._read_status()
2024-02-29T00:28:04.814Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 287, in _read_status
2024-02-29T00:28:04.814Z [container-agent]     raise RemoteDisconnected("Remote end closed connection without"
2024-02-29T00:28:04.814Z [container-agent] http.client.RemoteDisconnected: Remote end closed connection without response
2024-02-29T00:28:04.979Z [container-agent] 2024-02-29 00:28:04 INFO juju.util.exec exec.go:209 run result: exit status 1
2024-02-29T00:28:09.697Z [container-agent] 2024-02-29 00:28:09 INFO juju-log Setting up the logrotate configurations
2024-02-29T00:28:09.763Z [container-agent] 2024-02-29 00:28:09 INFO juju-log Adding pebble layer
2024-02-29T00:28:16.527Z [container-agent] 2024-02-29 00:28:16 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-02-29T00:28:16.555Z [container-agent] 2024-02-29 00:28:16 INFO juju-log Attempting reboot from complete outage.
2024-02-29T00:28:19.550Z [container-agent] 2024-02-29 00:28:19 INFO juju.worker.uniter.operation runhook.go:186 ran "mysql-pebble-ready" hook (via hook dispatching script: dispatch)
2024-02-29T00:30:42.526Z [container-agent] 2024-02-29 00:30:42 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T00:30:44.541Z [container-agent] 2024-02-29 00:30:44 WARNING juju-log No relation: certificates
2024-02-29T00:30:44.889Z [container-agent] 2024-02-29 00:30:44 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T00:35:59.024Z [container-agent] 2024-02-29 00:35:59 INFO juju-log Starting the log rotate manager
2024-02-29T00:35:59.036Z [container-agent] 2024-02-29 00:35:59 INFO juju-log Started log rotate manager process with PID 213516
2024-02-29T00:35:59.315Z [container-agent] 2024-02-29 00:35:59 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T00:36:01.170Z [container-agent] 2024-02-29 00:36:01 WARNING juju-log No relation: certificates
2024-02-29T00:36:01.531Z [container-agent] 2024-02-29 00:36:01 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T00:41:55.699Z [container-agent] 2024-02-29 00:41:55 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T00:41:58.253Z [container-agent] 2024-02-29 00:41:58 WARNING juju-log No relation: certificates
2024-02-29T00:41:58.605Z [container-agent] 2024-02-29 00:41:58 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T00:46:18.849Z [container-agent] 2024-02-29 00:46:18 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T00:46:21.907Z [container-agent] 2024-02-29 00:46:21 WARNING juju-log No relation: certificates
2024-02-29T00:46:22.275Z [container-agent] 2024-02-29 00:46:22 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T00:52:07.096Z [container-agent] 2024-02-29 00:52:07 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T00:52:10.722Z [container-agent] 2024-02-29 00:52:10 WARNING juju-log No relation: certificates
2024-02-29T00:52:11.054Z [container-agent] 2024-02-29 00:52:11 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T00:56:28.189Z [container-agent] 2024-02-29 00:56:28 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T00:56:32.173Z [container-agent] 2024-02-29 00:56:32 WARNING juju-log No relation: certificates
2024-02-29T00:56:32.520Z [container-agent] 2024-02-29 00:56:32 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T01:02:11.470Z [container-agent] 2024-02-29 01:02:11 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T01:02:16.009Z [container-agent] 2024-02-29 01:02:16 WARNING juju-log No relation: certificates
2024-02-29T01:02:16.372Z [container-agent] 2024-02-29 01:02:16 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T01:08:09.953Z [container-agent] 2024-02-29 01:08:09 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T01:08:15.219Z [container-agent] 2024-02-29 01:08:15 WARNING juju-log No relation: certificates
2024-02-29T01:08:15.575Z [container-agent] 2024-02-29 01:08:15 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T01:13:10.926Z [container-agent] 2024-02-29 01:13:10 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T01:13:16.614Z [container-agent] 2024-02-29 01:13:16 WARNING juju-log No relation: certificates
2024-02-29T01:13:16.969Z [container-agent] 2024-02-29 01:13:16 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T01:17:10.509Z [container-agent] 2024-02-29 01:17:10 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T01:17:16.596Z [container-agent] 2024-02-29 01:17:16 WARNING juju-log No relation: certificates
2024-02-29T01:17:17.008Z [container-agent] 2024-02-29 01:17:17 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T01:22:48.680Z [container-agent] 2024-02-29 01:22:48 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T01:22:54.994Z [container-agent] 2024-02-29 01:22:54 WARNING juju-log No relation: certificates
2024-02-29T01:22:55.345Z [container-agent] 2024-02-29 01:22:55 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T01:28:47.932Z [container-agent] 2024-02-29 01:28:47 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T01:28:54.807Z [container-agent] 2024-02-29 01:28:54 WARNING juju-log No relation: certificates
2024-02-29T01:28:55.153Z [container-agent] 2024-02-29 01:28:55 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T01:33:35.239Z [container-agent] 2024-02-29 01:33:35 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T01:33:42.701Z [container-agent] 2024-02-29 01:33:42 WARNING juju-log No relation: certificates
2024-02-29T01:33:43.047Z [container-agent] 2024-02-29 01:33:43 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T01:39:19.854Z [container-agent] 2024-02-29 01:39:19 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T01:39:27.733Z [container-agent] 2024-02-29 01:39:27 WARNING juju-log No relation: certificates
2024-02-29T01:39:28.093Z [container-agent] 2024-02-29 01:39:28 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T01:43:51.712Z [container-agent] 2024-02-29 01:43:51 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-02-29T01:43:51.738Z [container-agent] 2024-02-29 01:43:51 INFO juju-log Attempting reboot from complete outage.
2024-02-29T01:43:59.478Z [container-agent] 2024-02-29 01:43:59 WARNING juju-log No relation: certificates
2024-02-29T01:43:59.836Z [container-agent] 2024-02-29 01:43:59 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T01:45:04.777Z [container-agent] 2024-02-29 01:45:04 ERROR juju-log Uncaught exception while in charm code:
2024-02-29T01:45:04.777Z [container-agent] Traceback (most recent call last):
2024-02-29T01:45:04.777Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 2507, in exec
2024-02-29T01:45:04.777Z [container-agent]     control_ws = self._connect_websocket(task_id, 'control')
2024-02-29T01:45:04.777Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 2589, in _connect_websocket
2024-02-29T01:45:04.777Z [container-agent]     ws.connect(url, socket=sock)
2024-02-29T01:45:04.777Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_core.py", line 253, in connect
2024-02-29T01:45:04.777Z [container-agent]     self.handshake_response = handshake(self.sock, url, *addrs, **options)
2024-02-29T01:45:04.777Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_handshake.py", line 57, in handshake
2024-02-29T01:45:04.777Z [container-agent]     status, resp = _get_resp_headers(sock)
2024-02-29T01:45:04.777Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_handshake.py", line 143, in _get_resp_headers
2024-02-29T01:45:04.777Z [container-agent]     status, resp_headers, status_message = read_headers(sock)
2024-02-29T01:45:04.777Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_http.py", line 312, in read_headers
2024-02-29T01:45:04.777Z [container-agent]     line = recv_line(sock)
2024-02-29T01:45:04.777Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_socket.py", line 131, in recv_line
2024-02-29T01:45:04.777Z [container-agent]     c = recv(sock, 1)
2024-02-29T01:45:04.777Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_socket.py", line 122, in recv
2024-02-29T01:45:04.777Z [container-agent]     raise WebSocketConnectionClosedException(
2024-02-29T01:45:04.777Z [container-agent] websocket._exceptions.WebSocketConnectionClosedException: Connection to remote host was lost.
2024-02-29T01:45:04.777Z [container-agent] 
2024-02-29T01:45:04.777Z [container-agent] During handling of the above exception, another exception occurred:
2024-02-29T01:45:04.777Z [container-agent] 
2024-02-29T01:45:04.777Z [container-agent] Traceback (most recent call last):
2024-02-29T01:45:04.777Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 1348, in do_open
2024-02-29T01:45:04.777Z [container-agent]     h.request(req.get_method(), req.selector, req.data, headers,
2024-02-29T01:45:04.777Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1283, in request
2024-02-29T01:45:04.777Z [container-agent]     self._send_request(method, url, body, headers, encode_chunked)
2024-02-29T01:45:04.777Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1329, in _send_request
2024-02-29T01:45:04.777Z [container-agent]     self.endheaders(body, encode_chunked=encode_chunked)
2024-02-29T01:45:04.777Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1278, in endheaders
2024-02-29T01:45:04.777Z [container-agent]     self._send_output(message_body, encode_chunked=encode_chunked)
2024-02-29T01:45:04.777Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1038, in _send_output
2024-02-29T01:45:04.777Z [container-agent]     self.send(msg)
2024-02-29T01:45:04.777Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 976, in send
2024-02-29T01:45:04.777Z [container-agent]     self.connect()
2024-02-29T01:45:04.777Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 297, in connect
2024-02-29T01:45:04.777Z [container-agent]     self.sock.connect(self.socket_path)
2024-02-29T01:45:04.777Z [container-agent] ConnectionRefusedError: [Errno 111] Connection refused
2024-02-29T01:45:04.777Z [container-agent] 
2024-02-29T01:45:04.777Z [container-agent] During handling of the above exception, another exception occurred:
2024-02-29T01:45:04.777Z [container-agent] 
2024-02-29T01:45:04.777Z [container-agent] Traceback (most recent call last):
2024-02-29T01:45:04.777Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1690, in _request_raw
2024-02-29T01:45:04.777Z [container-agent]     response = self.opener.open(request, timeout=self.timeout)
2024-02-29T01:45:04.777Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 519, in open
2024-02-29T01:45:04.777Z [container-agent]     response = self._open(req, data)
2024-02-29T01:45:04.777Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 536, in _open
2024-02-29T01:45:04.777Z [container-agent]     result = self._call_chain(self.handle_open, protocol, protocol +
2024-02-29T01:45:04.777Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 496, in _call_chain
2024-02-29T01:45:04.777Z [container-agent]     result = func(*args)
2024-02-29T01:45:04.777Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 311, in http_open
2024-02-29T01:45:04.777Z [container-agent]     return self.do_open(_UnixSocketConnection, req,  # type:ignore
2024-02-29T01:45:04.777Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 1351, in do_open
2024-02-29T01:45:04.777Z [container-agent]     raise URLError(err)
2024-02-29T01:45:04.777Z [container-agent] urllib.error.URLError: <urlopen error [Errno 111] Connection refused>
2024-02-29T01:45:04.777Z [container-agent] 
2024-02-29T01:45:04.777Z [container-agent] The above exception was the direct cause of the following exception:
2024-02-29T01:45:04.777Z [container-agent] 
2024-02-29T01:45:04.777Z [container-agent] Traceback (most recent call last):
2024-02-29T01:45:04.777Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/./src/charm.py", line 786, in <module>
2024-02-29T01:45:04.777Z [container-agent]     main(MySQLOperatorCharm)
2024-02-29T01:45:04.777Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 436, in main
2024-02-29T01:45:04.777Z [container-agent]     _emit_charm_event(charm, dispatcher.event_name)
2024-02-29T01:45:04.777Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 144, in _emit_charm_event
2024-02-29T01:45:04.777Z [container-agent]     event_to_emit.emit(*args, **kwargs)
2024-02-29T01:45:04.777Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 351, in emit
2024-02-29T01:45:04.777Z [container-agent]     framework._emit(event)
2024-02-29T01:45:04.777Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 853, in _emit
2024-02-29T01:45:04.777Z [container-agent]     self._reemit(event_path)
2024-02-29T01:45:04.777Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 942, in _reemit
2024-02-29T01:45:04.777Z [container-agent]     custom_handler(event)
2024-02-29T01:45:04.777Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/rotate_mysql_logs.py", line 55, in _rotate_mysql_logs
2024-02-29T01:45:04.777Z [container-agent]     self.charm._mysql.flush_mysql_logs(MySQLTextLogs.ERROR)
2024-02-29T01:45:04.777Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-02-29T01:45:04.777Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-02-29T01:45:04.777Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 665, in _run_mysqlsh_script
2024-02-29T01:45:04.777Z [container-agent]     process = self.container.exec(cmd, timeout=timeout)
2024-02-29T01:45:04.777Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/model.py", line 2688, in exec
2024-02-29T01:45:04.777Z [container-agent]     return self._pebble.exec(
2024-02-29T01:45:04.777Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 2514, in exec
2024-02-29T01:45:04.777Z [container-agent]     change = self.wait_change(ChangeID(change_id))
2024-02-29T01:45:04.777Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1893, in wait_change
2024-02-29T01:45:04.777Z [container-agent]     return self._wait_change_using_wait(change_id, timeout)
2024-02-29T01:45:04.777Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1914, in _wait_change_using_wait
2024-02-29T01:45:04.777Z [container-agent]     return self._wait_change(change_id, this_timeout)
2024-02-29T01:45:04.777Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1928, in _wait_change
2024-02-29T01:45:04.777Z [container-agent]     resp = self._request('GET', f'/v1/changes/{change_id}/wait', query)
2024-02-29T01:45:04.777Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1655, in _request
2024-02-29T01:45:04.777Z [container-agent]     response = self._request_raw(method, path, query, headers, data)
2024-02-29T01:45:04.777Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1707, in _request_raw
2024-02-29T01:45:04.777Z [container-agent]     raise ConnectionError(e.reason) from e
2024-02-29T01:45:04.777Z [container-agent] ops.pebble.ConnectionError: [Errno 111] Connection refused
2024-02-29T01:45:04.893Z [container-agent] 2024-02-29 01:45:04 INFO juju.util.exec exec.go:209 run result: exit status 1
2024-02-29T01:45:07.428Z [container-agent] 2024-02-29 01:45:07 INFO juju-log Setting up the logrotate configurations
2024-02-29T01:45:07.524Z [container-agent] 2024-02-29 01:45:07 INFO juju-log Adding pebble layer
2024-02-29T01:45:14.429Z [container-agent] 2024-02-29 01:45:14 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-02-29T01:45:14.454Z [container-agent] 2024-02-29 01:45:14 INFO juju-log Attempting reboot from complete outage.
2024-02-29T01:45:17.551Z [container-agent] 2024-02-29 01:45:17 INFO juju.worker.uniter.operation runhook.go:186 ran "mysql-pebble-ready" hook (via hook dispatching script: dispatch)
2024-02-29T01:49:13.896Z [container-agent] 2024-02-29 01:49:13 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T01:49:15.986Z [container-agent] 2024-02-29 01:49:15 WARNING juju-log No relation: certificates
2024-02-29T01:49:16.342Z [container-agent] 2024-02-29 01:49:16 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T01:54:53.125Z [container-agent] 2024-02-29 01:54:53 INFO juju-log Starting the log rotate manager
2024-02-29T01:54:53.139Z [container-agent] 2024-02-29 01:54:53 INFO juju-log Started log rotate manager process with PID 220606
2024-02-29T01:54:53.406Z [container-agent] 2024-02-29 01:54:53 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T01:54:55.362Z [container-agent] 2024-02-29 01:54:55 WARNING juju-log No relation: certificates
2024-02-29T01:54:55.709Z [container-agent] 2024-02-29 01:54:55 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T02:00:53.370Z [container-agent] 2024-02-29 02:00:53 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T02:00:55.775Z [container-agent] 2024-02-29 02:00:55 WARNING juju-log No relation: certificates
2024-02-29T02:00:56.116Z [container-agent] 2024-02-29 02:00:56 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T02:06:26.856Z [container-agent] 2024-02-29 02:06:26 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T02:06:29.976Z [container-agent] 2024-02-29 02:06:29 WARNING juju-log No relation: certificates
2024-02-29T02:06:30.354Z [container-agent] 2024-02-29 02:06:30 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T02:12:01.354Z [container-agent] 2024-02-29 02:12:01 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T02:12:05.007Z [container-agent] 2024-02-29 02:12:05 WARNING juju-log No relation: certificates
2024-02-29T02:12:05.343Z [container-agent] 2024-02-29 02:12:05 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T02:16:14.265Z [container-agent] 2024-02-29 02:16:14 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T02:16:18.494Z [container-agent] 2024-02-29 02:16:18 WARNING juju-log No relation: certificates
2024-02-29T02:16:18.838Z [container-agent] 2024-02-29 02:16:18 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T02:22:08.569Z [container-agent] 2024-02-29 02:22:08 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T02:22:12.732Z [container-agent] 2024-02-29 02:22:12 WARNING juju-log No relation: certificates
2024-02-29T02:22:13.076Z [container-agent] 2024-02-29 02:22:13 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T02:26:28.878Z [container-agent] 2024-02-29 02:26:28 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T02:26:33.933Z [container-agent] 2024-02-29 02:26:33 WARNING juju-log No relation: certificates
2024-02-29T02:26:34.292Z [container-agent] 2024-02-29 02:26:34 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T02:32:12.285Z [container-agent] 2024-02-29 02:32:12 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T02:32:17.989Z [container-agent] 2024-02-29 02:32:17 WARNING juju-log No relation: certificates
2024-02-29T02:32:18.342Z [container-agent] 2024-02-29 02:32:18 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T02:36:47.453Z [container-agent] 2024-02-29 02:36:47 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T02:36:53.309Z [container-agent] 2024-02-29 02:36:53 WARNING juju-log No relation: certificates
2024-02-29T02:36:53.684Z [container-agent] 2024-02-29 02:36:53 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T02:42:11.314Z [container-agent] 2024-02-29 02:42:11 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T02:42:17.660Z [container-agent] 2024-02-29 02:42:17 WARNING juju-log No relation: certificates
2024-02-29T02:42:18.011Z [container-agent] 2024-02-29 02:42:18 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T02:47:12.473Z [container-agent] 2024-02-29 02:47:12 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T02:47:19.536Z [container-agent] 2024-02-29 02:47:19 WARNING juju-log No relation: certificates
2024-02-29T02:47:19.889Z [container-agent] 2024-02-29 02:47:19 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T02:51:46.293Z [container-agent] 2024-02-29 02:51:46 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T02:51:53.431Z [container-agent] 2024-02-29 02:51:53 WARNING juju-log No relation: certificates
2024-02-29T02:51:53.796Z [container-agent] 2024-02-29 02:51:53 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T02:57:45.548Z [container-agent] 2024-02-29 02:57:45 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T02:57:53.309Z [container-agent] 2024-02-29 02:57:53 WARNING juju-log No relation: certificates
2024-02-29T02:57:53.649Z [container-agent] 2024-02-29 02:57:53 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T03:02:43.434Z [container-agent] 2024-02-29 03:02:43 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T03:02:50.858Z [container-agent] 2024-02-29 03:02:50 WARNING juju-log No relation: certificates
2024-02-29T03:02:51.240Z [container-agent] 2024-02-29 03:02:51 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T03:07:16.043Z [container-agent] 2024-02-29 03:07:16 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T03:07:24.205Z [container-agent] 2024-02-29 03:07:24 WARNING juju-log No relation: certificates
2024-02-29T03:07:24.533Z [container-agent] 2024-02-29 03:07:24 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T03:12:58.964Z [container-agent] 2024-02-29 03:12:58 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-02-29T03:12:58.992Z [container-agent] 2024-02-29 03:12:58 INFO juju-log Attempting reboot from complete outage.
2024-02-29T03:13:07.212Z [container-agent] 2024-02-29 03:13:07 WARNING juju-log No relation: certificates
2024-02-29T03:13:07.619Z [container-agent] 2024-02-29 03:13:07 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T03:14:09.816Z [container-agent] 2024-02-29 03:14:09 ERROR juju-log Uncaught exception while in charm code:
2024-02-29T03:14:09.816Z [container-agent] Traceback (most recent call last):
2024-02-29T03:14:09.816Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/./src/charm.py", line 786, in <module>
2024-02-29T03:14:09.816Z [container-agent]     main(MySQLOperatorCharm)
2024-02-29T03:14:09.816Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 436, in main
2024-02-29T03:14:09.816Z [container-agent]     _emit_charm_event(charm, dispatcher.event_name)
2024-02-29T03:14:09.816Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 144, in _emit_charm_event
2024-02-29T03:14:09.816Z [container-agent]     event_to_emit.emit(*args, **kwargs)
2024-02-29T03:14:09.816Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 351, in emit
2024-02-29T03:14:09.816Z [container-agent]     framework._emit(event)
2024-02-29T03:14:09.816Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 853, in _emit
2024-02-29T03:14:09.816Z [container-agent]     self._reemit(event_path)
2024-02-29T03:14:09.816Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 942, in _reemit
2024-02-29T03:14:09.816Z [container-agent]     custom_handler(event)
2024-02-29T03:14:09.816Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/rotate_mysql_logs.py", line 57, in _rotate_mysql_logs
2024-02-29T03:14:09.816Z [container-agent]     self.charm._mysql.flush_mysql_logs(MySQLTextLogs.SLOW)
2024-02-29T03:14:09.816Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-02-29T03:14:09.816Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-02-29T03:14:09.816Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 665, in _run_mysqlsh_script
2024-02-29T03:14:09.816Z [container-agent]     process = self.container.exec(cmd, timeout=timeout)
2024-02-29T03:14:09.816Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/model.py", line 2688, in exec
2024-02-29T03:14:09.816Z [container-agent]     return self._pebble.exec(
2024-02-29T03:14:09.816Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 2501, in exec
2024-02-29T03:14:09.816Z [container-agent]     resp = self._request('POST', '/v1/exec', body=body)
2024-02-29T03:14:09.816Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1655, in _request
2024-02-29T03:14:09.816Z [container-agent]     response = self._request_raw(method, path, query, headers, data)
2024-02-29T03:14:09.816Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1690, in _request_raw
2024-02-29T03:14:09.816Z [container-agent]     response = self.opener.open(request, timeout=self.timeout)
2024-02-29T03:14:09.816Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 519, in open
2024-02-29T03:14:09.816Z [container-agent]     response = self._open(req, data)
2024-02-29T03:14:09.816Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 536, in _open
2024-02-29T03:14:09.816Z [container-agent]     result = self._call_chain(self.handle_open, protocol, protocol +
2024-02-29T03:14:09.816Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 496, in _call_chain
2024-02-29T03:14:09.816Z [container-agent]     result = func(*args)
2024-02-29T03:14:09.816Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 311, in http_open
2024-02-29T03:14:09.816Z [container-agent]     return self.do_open(_UnixSocketConnection, req,  # type:ignore
2024-02-29T03:14:09.816Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 1352, in do_open
2024-02-29T03:14:09.816Z [container-agent]     r = h.getresponse()
2024-02-29T03:14:09.816Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1375, in getresponse
2024-02-29T03:14:09.816Z [container-agent]     response.begin()
2024-02-29T03:14:09.816Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 318, in begin
2024-02-29T03:14:09.816Z [container-agent]     version, status, reason = self._read_status()
2024-02-29T03:14:09.816Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 287, in _read_status
2024-02-29T03:14:09.816Z [container-agent]     raise RemoteDisconnected("Remote end closed connection without"
2024-02-29T03:14:09.816Z [container-agent] http.client.RemoteDisconnected: Remote end closed connection without response
2024-02-29T03:14:09.955Z [container-agent] 2024-02-29 03:14:09 INFO juju.util.exec exec.go:209 run result: exit status 1
2024-02-29T03:14:11.902Z [container-agent] 2024-02-29 03:14:11 INFO juju-log Setting up the logrotate configurations
2024-02-29T03:14:11.967Z [container-agent] 2024-02-29 03:14:11 INFO juju-log Adding pebble layer
2024-02-29T03:14:18.738Z [container-agent] 2024-02-29 03:14:18 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-02-29T03:14:18.764Z [container-agent] 2024-02-29 03:14:18 INFO juju-log Attempting reboot from complete outage.
2024-02-29T03:14:21.810Z [container-agent] 2024-02-29 03:14:21 INFO juju.worker.uniter.operation runhook.go:186 ran "mysql-pebble-ready" hook (via hook dispatching script: dispatch)
2024-02-29T03:17:25.229Z [container-agent] 2024-02-29 03:17:25 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T03:17:27.216Z [container-agent] 2024-02-29 03:17:27 WARNING juju-log No relation: certificates
2024-02-29T03:17:27.580Z [container-agent] 2024-02-29 03:17:27 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T03:21:31.052Z [container-agent] 2024-02-29 03:21:31 INFO juju-log Starting the log rotate manager
2024-02-29T03:21:31.067Z [container-agent] 2024-02-29 03:21:31 INFO juju-log Started log rotate manager process with PID 228668
2024-02-29T03:21:31.351Z [container-agent] 2024-02-29 03:21:31 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T03:21:33.396Z [container-agent] 2024-02-29 03:21:33 WARNING juju-log No relation: certificates
2024-02-29T03:21:33.740Z [container-agent] 2024-02-29 03:21:33 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T03:27:26.168Z [container-agent] 2024-02-29 03:27:26 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T03:27:28.424Z [container-agent] 2024-02-29 03:27:28 WARNING juju-log No relation: certificates
2024-02-29T03:27:28.757Z [container-agent] 2024-02-29 03:27:28 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T03:31:40.221Z [container-agent] 2024-02-29 03:31:40 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T03:31:43.045Z [container-agent] 2024-02-29 03:31:43 WARNING juju-log No relation: certificates
2024-02-29T03:31:43.400Z [container-agent] 2024-02-29 03:31:43 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T03:37:12.323Z [container-agent] 2024-02-29 03:37:12 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T03:37:15.810Z [container-agent] 2024-02-29 03:37:15 WARNING juju-log No relation: certificates
2024-02-29T03:37:16.145Z [container-agent] 2024-02-29 03:37:16 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T03:43:01.643Z [container-agent] 2024-02-29 03:43:01 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T03:43:05.689Z [container-agent] 2024-02-29 03:43:05 WARNING juju-log No relation: certificates
2024-02-29T03:43:06.086Z [container-agent] 2024-02-29 03:43:06 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T03:47:38.816Z [container-agent] 2024-02-29 03:47:38 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T03:47:42.937Z [container-agent] 2024-02-29 03:47:42 WARNING juju-log No relation: certificates
2024-02-29T03:47:43.340Z [container-agent] 2024-02-29 03:47:43 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T03:51:44.275Z [container-agent] 2024-02-29 03:51:44 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T03:51:49.223Z [container-agent] 2024-02-29 03:51:49 WARNING juju-log No relation: certificates
2024-02-29T03:51:49.583Z [container-agent] 2024-02-29 03:51:49 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T03:57:09.931Z [container-agent] 2024-02-29 03:57:09 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T03:57:15.137Z [container-agent] 2024-02-29 03:57:15 WARNING juju-log No relation: certificates
2024-02-29T03:57:15.495Z [container-agent] 2024-02-29 03:57:15 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T04:02:11.670Z [container-agent] 2024-02-29 04:02:11 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T04:02:17.619Z [container-agent] 2024-02-29 04:02:17 WARNING juju-log No relation: certificates
2024-02-29T04:02:17.981Z [container-agent] 2024-02-29 04:02:17 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T04:06:20.585Z [container-agent] 2024-02-29 04:06:20 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T04:06:26.603Z [container-agent] 2024-02-29 04:06:26 WARNING juju-log No relation: certificates
2024-02-29T04:06:26.943Z [container-agent] 2024-02-29 04:06:26 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T04:11:29.410Z [container-agent] 2024-02-29 04:11:29 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T04:11:36.176Z [container-agent] 2024-02-29 04:11:36 WARNING juju-log No relation: certificates
2024-02-29T04:11:36.522Z [container-agent] 2024-02-29 04:11:36 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T04:15:36.176Z [container-agent] 2024-02-29 04:15:36 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T04:15:43.131Z [container-agent] 2024-02-29 04:15:43 WARNING juju-log No relation: certificates
2024-02-29T04:15:43.524Z [container-agent] 2024-02-29 04:15:43 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T04:17:09.201Z [container-agent] 2024-02-29 04:17:09 ERROR juju-log Failed to flush GENERAL LOGS logs.
2024-02-29T04:17:09.201Z [container-agent] Traceback (most recent call last):
2024-02-29T04:17:09.201Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 666, in _run_mysqlsh_script
2024-02-29T04:17:09.201Z [container-agent]     stdout, _ = process.wait_output()
2024-02-29T04:17:09.201Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1441, in wait_output
2024-02-29T04:17:09.201Z [container-agent]     raise ExecError[AnyStr](self._command, exit_code, out_value, err_value)
2024-02-29T04:17:09.201Z [container-agent] ops.pebble.ExecError: non-zero exit code 1 executing ['/usr/bin/mysqlsh', '--no-wizard', '--python', '--verbose=1', '-f', '/tmp/script.py', ';', 'rm', '/tmp/script.py'], stdout='', stderr='Cannot set LC_ALL to locale en_US.UTF-8: No such file or directory\nverbose: 2024-02-29T04:17:08Z: Loading startup files...\nverbose: 2024-02-29T04:17:08Z: Loading plugins...\nverbose: 2024-02-29T04:17:08Z: Connecting to MySQL at: serverconfig@horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local\nTraceback (most recent call last):\n  File "<string>", line 1, in <module>\nmysqlsh.DBError: MySQL Error (2003): Shell.connect: Can\'t connect to MySQL server on \'horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local:3306\' (111)\n'
2024-02-29T04:17:09.201Z [container-agent] 
2024-02-29T04:17:09.201Z [container-agent] During handling of the above exception, another exception occurred:
2024-02-29T04:17:09.201Z [container-agent] 
2024-02-29T04:17:09.201Z [container-agent] Traceback (most recent call last):
2024-02-29T04:17:09.201Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-02-29T04:17:09.201Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-02-29T04:17:09.201Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 669, in _run_mysqlsh_script
2024-02-29T04:17:09.201Z [container-agent]     raise MySQLClientError(e.stderr)
2024-02-29T04:17:09.201Z [container-agent] charms.mysql.v0.mysql.MySQLClientError: Cannot set LC_ALL to locale en_US.UTF-8: No such file or directory
2024-02-29T04:17:09.201Z [container-agent] verbose: 2024-02-29T04:17:08Z: Loading startup files...
2024-02-29T04:17:09.201Z [container-agent] verbose: 2024-02-29T04:17:08Z: Loading plugins...
2024-02-29T04:17:09.201Z [container-agent] verbose: 2024-02-29T04:17:08Z: Connecting to MySQL at: serverconfig@horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local
2024-02-29T04:17:09.201Z [container-agent] Traceback (most recent call last):
2024-02-29T04:17:09.201Z [container-agent]   File "<string>", line 1, in <module>
2024-02-29T04:17:09.201Z [container-agent] mysqlsh.DBError: MySQL Error (2003): Shell.connect: Can't connect to MySQL server on 'horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local:3306' (111)
2024-02-29T04:17:09.201Z [container-agent] 
2024-02-29T04:21:23.930Z [container-agent] 2024-02-29 04:21:23 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-02-29T04:21:23.952Z [container-agent] 2024-02-29 04:21:23 INFO juju-log Attempting reboot from complete outage.
2024-02-29T04:21:31.451Z [container-agent] 2024-02-29 04:21:31 WARNING juju-log No relation: certificates
2024-02-29T04:21:31.818Z [container-agent] 2024-02-29 04:21:31 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T04:22:04.756Z [container-agent] 2024-02-29 04:22:04 ERROR juju-log Uncaught exception while in charm code:
2024-02-29T04:22:04.757Z [container-agent] Traceback (most recent call last):
2024-02-29T04:22:04.757Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 2507, in exec
2024-02-29T04:22:04.757Z [container-agent]     control_ws = self._connect_websocket(task_id, 'control')
2024-02-29T04:22:04.757Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 2589, in _connect_websocket
2024-02-29T04:22:04.757Z [container-agent]     ws.connect(url, socket=sock)
2024-02-29T04:22:04.757Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_core.py", line 253, in connect
2024-02-29T04:22:04.757Z [container-agent]     self.handshake_response = handshake(self.sock, url, *addrs, **options)
2024-02-29T04:22:04.757Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_handshake.py", line 57, in handshake
2024-02-29T04:22:04.757Z [container-agent]     status, resp = _get_resp_headers(sock)
2024-02-29T04:22:04.757Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_handshake.py", line 143, in _get_resp_headers
2024-02-29T04:22:04.757Z [container-agent]     status, resp_headers, status_message = read_headers(sock)
2024-02-29T04:22:04.757Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_http.py", line 312, in read_headers
2024-02-29T04:22:04.757Z [container-agent]     line = recv_line(sock)
2024-02-29T04:22:04.757Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_socket.py", line 131, in recv_line
2024-02-29T04:22:04.757Z [container-agent]     c = recv(sock, 1)
2024-02-29T04:22:04.757Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_socket.py", line 122, in recv
2024-02-29T04:22:04.757Z [container-agent]     raise WebSocketConnectionClosedException(
2024-02-29T04:22:04.757Z [container-agent] websocket._exceptions.WebSocketConnectionClosedException: Connection to remote host was lost.
2024-02-29T04:22:04.757Z [container-agent] 
2024-02-29T04:22:04.757Z [container-agent] During handling of the above exception, another exception occurred:
2024-02-29T04:22:04.757Z [container-agent] 
2024-02-29T04:22:04.757Z [container-agent] Traceback (most recent call last):
2024-02-29T04:22:04.757Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 1348, in do_open
2024-02-29T04:22:04.757Z [container-agent]     h.request(req.get_method(), req.selector, req.data, headers,
2024-02-29T04:22:04.757Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1283, in request
2024-02-29T04:22:04.757Z [container-agent]     self._send_request(method, url, body, headers, encode_chunked)
2024-02-29T04:22:04.757Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1329, in _send_request
2024-02-29T04:22:04.757Z [container-agent]     self.endheaders(body, encode_chunked=encode_chunked)
2024-02-29T04:22:04.757Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1278, in endheaders
2024-02-29T04:22:04.757Z [container-agent]     self._send_output(message_body, encode_chunked=encode_chunked)
2024-02-29T04:22:04.757Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1038, in _send_output
2024-02-29T04:22:04.757Z [container-agent]     self.send(msg)
2024-02-29T04:22:04.757Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 976, in send
2024-02-29T04:22:04.757Z [container-agent]     self.connect()
2024-02-29T04:22:04.757Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 297, in connect
2024-02-29T04:22:04.757Z [container-agent]     self.sock.connect(self.socket_path)
2024-02-29T04:22:04.757Z [container-agent] ConnectionRefusedError: [Errno 111] Connection refused
2024-02-29T04:22:04.757Z [container-agent] 
2024-02-29T04:22:04.757Z [container-agent] During handling of the above exception, another exception occurred:
2024-02-29T04:22:04.757Z [container-agent] 
2024-02-29T04:22:04.757Z [container-agent] Traceback (most recent call last):
2024-02-29T04:22:04.757Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1690, in _request_raw
2024-02-29T04:22:04.757Z [container-agent]     response = self.opener.open(request, timeout=self.timeout)
2024-02-29T04:22:04.757Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 519, in open
2024-02-29T04:22:04.757Z [container-agent]     response = self._open(req, data)
2024-02-29T04:22:04.757Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 536, in _open
2024-02-29T04:22:04.757Z [container-agent]     result = self._call_chain(self.handle_open, protocol, protocol +
2024-02-29T04:22:04.757Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 496, in _call_chain
2024-02-29T04:22:04.757Z [container-agent]     result = func(*args)
2024-02-29T04:22:04.757Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 311, in http_open
2024-02-29T04:22:04.757Z [container-agent]     return self.do_open(_UnixSocketConnection, req,  # type:ignore
2024-02-29T04:22:04.757Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 1351, in do_open
2024-02-29T04:22:04.757Z [container-agent]     raise URLError(err)
2024-02-29T04:22:04.757Z [container-agent] urllib.error.URLError: <urlopen error [Errno 111] Connection refused>
2024-02-29T04:22:04.757Z [container-agent] 
2024-02-29T04:22:04.757Z [container-agent] The above exception was the direct cause of the following exception:
2024-02-29T04:22:04.757Z [container-agent] 
2024-02-29T04:22:04.757Z [container-agent] Traceback (most recent call last):
2024-02-29T04:22:04.757Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/./src/charm.py", line 786, in <module>
2024-02-29T04:22:04.757Z [container-agent]     main(MySQLOperatorCharm)
2024-02-29T04:22:04.757Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 436, in main
2024-02-29T04:22:04.757Z [container-agent]     _emit_charm_event(charm, dispatcher.event_name)
2024-02-29T04:22:04.757Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 144, in _emit_charm_event
2024-02-29T04:22:04.757Z [container-agent]     event_to_emit.emit(*args, **kwargs)
2024-02-29T04:22:04.757Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 351, in emit
2024-02-29T04:22:04.757Z [container-agent]     framework._emit(event)
2024-02-29T04:22:04.757Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 853, in _emit
2024-02-29T04:22:04.757Z [container-agent]     self._reemit(event_path)
2024-02-29T04:22:04.757Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 942, in _reemit
2024-02-29T04:22:04.757Z [container-agent]     custom_handler(event)
2024-02-29T04:22:04.757Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/rotate_mysql_logs.py", line 55, in _rotate_mysql_logs
2024-02-29T04:22:04.757Z [container-agent]     self.charm._mysql.flush_mysql_logs(MySQLTextLogs.ERROR)
2024-02-29T04:22:04.757Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-02-29T04:22:04.757Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-02-29T04:22:04.757Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 665, in _run_mysqlsh_script
2024-02-29T04:22:04.757Z [container-agent]     process = self.container.exec(cmd, timeout=timeout)
2024-02-29T04:22:04.757Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/model.py", line 2688, in exec
2024-02-29T04:22:04.757Z [container-agent]     return self._pebble.exec(
2024-02-29T04:22:04.757Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 2514, in exec
2024-02-29T04:22:04.757Z [container-agent]     change = self.wait_change(ChangeID(change_id))
2024-02-29T04:22:04.757Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1893, in wait_change
2024-02-29T04:22:04.757Z [container-agent]     return self._wait_change_using_wait(change_id, timeout)
2024-02-29T04:22:04.757Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1914, in _wait_change_using_wait
2024-02-29T04:22:04.757Z [container-agent]     return self._wait_change(change_id, this_timeout)
2024-02-29T04:22:04.757Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1928, in _wait_change
2024-02-29T04:22:04.757Z [container-agent]     resp = self._request('GET', f'/v1/changes/{change_id}/wait', query)
2024-02-29T04:22:04.757Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1655, in _request
2024-02-29T04:22:04.757Z [container-agent]     response = self._request_raw(method, path, query, headers, data)
2024-02-29T04:22:04.757Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1707, in _request_raw
2024-02-29T04:22:04.757Z [container-agent]     raise ConnectionError(e.reason) from e
2024-02-29T04:22:04.757Z [container-agent] ops.pebble.ConnectionError: [Errno 111] Connection refused
2024-02-29T04:22:04.937Z [container-agent] 2024-02-29 04:22:04 INFO juju.util.exec exec.go:209 run result: exit status 1
2024-02-29T04:22:09.710Z [container-agent] 2024-02-29 04:22:09 INFO juju-log Setting up the logrotate configurations
2024-02-29T04:22:09.771Z [container-agent] 2024-02-29 04:22:09 INFO juju-log Adding pebble layer
2024-02-29T04:22:16.505Z [container-agent] 2024-02-29 04:22:16 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-02-29T04:22:16.529Z [container-agent] 2024-02-29 04:22:16 INFO juju-log Attempting reboot from complete outage.
2024-02-29T04:22:19.653Z [container-agent] 2024-02-29 04:22:19 INFO juju.worker.uniter.operation runhook.go:186 ran "mysql-pebble-ready" hook (via hook dispatching script: dispatch)
2024-02-29T04:26:12.094Z [container-agent] 2024-02-29 04:26:12 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T04:26:14.100Z [container-agent] 2024-02-29 04:26:14 WARNING juju-log No relation: certificates
2024-02-29T04:26:14.500Z [container-agent] 2024-02-29 04:26:14 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T04:30:35.699Z [container-agent] 2024-02-29 04:30:35 INFO juju-log Starting the log rotate manager
2024-02-29T04:30:35.714Z [container-agent] 2024-02-29 04:30:35 INFO juju-log Started log rotate manager process with PID 234932
2024-02-29T04:30:35.996Z [container-agent] 2024-02-29 04:30:35 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T04:30:37.844Z [container-agent] 2024-02-29 04:30:37 WARNING juju-log No relation: certificates
2024-02-29T04:30:38.235Z [container-agent] 2024-02-29 04:30:38 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T04:35:41.544Z [container-agent] 2024-02-29 04:35:41 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T04:35:43.854Z [container-agent] 2024-02-29 04:35:43 WARNING juju-log No relation: certificates
2024-02-29T04:35:44.243Z [container-agent] 2024-02-29 04:35:44 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T04:39:54.210Z [container-agent] 2024-02-29 04:39:54 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T04:39:57.070Z [container-agent] 2024-02-29 04:39:57 WARNING juju-log No relation: certificates
2024-02-29T04:39:57.414Z [container-agent] 2024-02-29 04:39:57 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T04:44:07.272Z [container-agent] 2024-02-29 04:44:07 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T04:44:10.396Z [container-agent] 2024-02-29 04:44:10 WARNING juju-log No relation: certificates
2024-02-29T04:44:10.738Z [container-agent] 2024-02-29 04:44:10 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T04:48:12.486Z [container-agent] 2024-02-29 04:48:12 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T04:48:16.430Z [container-agent] 2024-02-29 04:48:16 WARNING juju-log No relation: certificates
2024-02-29T04:48:16.770Z [container-agent] 2024-02-29 04:48:16 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T04:53:08.281Z [container-agent] 2024-02-29 04:53:08 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T04:53:12.543Z [container-agent] 2024-02-29 04:53:12 WARNING juju-log No relation: certificates
2024-02-29T04:53:12.903Z [container-agent] 2024-02-29 04:53:12 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T04:58:33.407Z [container-agent] 2024-02-29 04:58:33 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T04:58:38.213Z [container-agent] 2024-02-29 04:58:38 WARNING juju-log No relation: certificates
2024-02-29T04:58:38.558Z [container-agent] 2024-02-29 04:58:38 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T05:02:49.515Z [container-agent] 2024-02-29 05:02:49 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T05:02:54.609Z [container-agent] 2024-02-29 05:02:54 WARNING juju-log No relation: certificates
2024-02-29T05:02:54.957Z [container-agent] 2024-02-29 05:02:54 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T05:07:27.555Z [container-agent] 2024-02-29 05:07:27 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T05:07:32.971Z [container-agent] 2024-02-29 05:07:32 WARNING juju-log No relation: certificates
2024-02-29T05:07:33.338Z [container-agent] 2024-02-29 05:07:33 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T05:12:26.253Z [container-agent] 2024-02-29 05:12:26 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T05:12:32.319Z [container-agent] 2024-02-29 05:12:32 WARNING juju-log No relation: certificates
2024-02-29T05:12:32.702Z [container-agent] 2024-02-29 05:12:32 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T05:16:27.799Z [container-agent] 2024-02-29 05:16:27 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T05:16:34.397Z [container-agent] 2024-02-29 05:16:34 WARNING juju-log No relation: certificates
2024-02-29T05:16:34.792Z [container-agent] 2024-02-29 05:16:34 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T05:21:38.676Z [container-agent] 2024-02-29 05:21:38 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T05:21:45.516Z [container-agent] 2024-02-29 05:21:45 WARNING juju-log No relation: certificates
2024-02-29T05:21:45.890Z [container-agent] 2024-02-29 05:21:45 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T05:26:01.455Z [container-agent] 2024-02-29 05:26:01 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T05:26:08.257Z [container-agent] 2024-02-29 05:26:08 WARNING juju-log No relation: certificates
2024-02-29T05:26:08.590Z [container-agent] 2024-02-29 05:26:08 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T05:27:12.485Z [container-agent] 2024-02-29 05:27:12 ERROR juju-log Failed to flush SLOW LOGS logs.
2024-02-29T05:27:12.485Z [container-agent] Traceback (most recent call last):
2024-02-29T05:27:12.485Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 666, in _run_mysqlsh_script
2024-02-29T05:27:12.485Z [container-agent]     stdout, _ = process.wait_output()
2024-02-29T05:27:12.485Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1441, in wait_output
2024-02-29T05:27:12.485Z [container-agent]     raise ExecError[AnyStr](self._command, exit_code, out_value, err_value)
2024-02-29T05:27:12.485Z [container-agent] ops.pebble.ExecError: non-zero exit code 1 executing ['/usr/bin/mysqlsh', '--no-wizard', '--python', '--verbose=1', '-f', '/tmp/script.py', ';', 'rm', '/tmp/script.py'], stdout='', stderr='Cannot set LC_ALL to locale en_US.UTF-8: No such file or directory\nverbose: 2024-02-29T05:27:11Z: Loading startup files...\nverbose: 2024-02-29T05:27:11Z: Loading plugins...\nverbose: 2024-02-29T05:27:11Z: Connecting to MySQL at: serverconfig@horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local\nTraceback (most recent call last):\n  File "<string>", line 1, in <module>\nmysqlsh.DBError: MySQL Error (2003): Shell.connect: Can\'t connect to MySQL server on \'horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local:3306\' (111)\n'
2024-02-29T05:27:12.485Z [container-agent] 
2024-02-29T05:27:12.485Z [container-agent] During handling of the above exception, another exception occurred:
2024-02-29T05:27:12.485Z [container-agent] 
2024-02-29T05:27:12.485Z [container-agent] Traceback (most recent call last):
2024-02-29T05:27:12.485Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-02-29T05:27:12.485Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-02-29T05:27:12.485Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 669, in _run_mysqlsh_script
2024-02-29T05:27:12.485Z [container-agent]     raise MySQLClientError(e.stderr)
2024-02-29T05:27:12.485Z [container-agent] charms.mysql.v0.mysql.MySQLClientError: Cannot set LC_ALL to locale en_US.UTF-8: No such file or directory
2024-02-29T05:27:12.485Z [container-agent] verbose: 2024-02-29T05:27:11Z: Loading startup files...
2024-02-29T05:27:12.485Z [container-agent] verbose: 2024-02-29T05:27:11Z: Loading plugins...
2024-02-29T05:27:12.485Z [container-agent] verbose: 2024-02-29T05:27:11Z: Connecting to MySQL at: serverconfig@horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local
2024-02-29T05:27:12.485Z [container-agent] Traceback (most recent call last):
2024-02-29T05:27:12.485Z [container-agent]   File "<string>", line 1, in <module>
2024-02-29T05:27:12.485Z [container-agent] mysqlsh.DBError: MySQL Error (2003): Shell.connect: Can't connect to MySQL server on 'horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local:3306' (111)
2024-02-29T05:27:12.485Z [container-agent] 
2024-02-29T05:31:13.354Z [container-agent] 2024-02-29 05:31:13 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-02-29T05:31:13.384Z [container-agent] 2024-02-29 05:31:13 INFO juju-log Attempting reboot from complete outage.
2024-02-29T05:31:21.006Z [container-agent] 2024-02-29 05:31:21 WARNING juju-log No relation: certificates
2024-02-29T05:31:21.362Z [container-agent] 2024-02-29 05:31:21 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T05:32:09.805Z [container-agent] 2024-02-29 05:32:09 ERROR juju-log Uncaught exception while in charm code:
2024-02-29T05:32:09.805Z [container-agent] Traceback (most recent call last):
2024-02-29T05:32:09.805Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/./src/charm.py", line 786, in <module>
2024-02-29T05:32:09.805Z [container-agent]     main(MySQLOperatorCharm)
2024-02-29T05:32:09.805Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 436, in main
2024-02-29T05:32:09.805Z [container-agent]     _emit_charm_event(charm, dispatcher.event_name)
2024-02-29T05:32:09.805Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 144, in _emit_charm_event
2024-02-29T05:32:09.805Z [container-agent]     event_to_emit.emit(*args, **kwargs)
2024-02-29T05:32:09.805Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 351, in emit
2024-02-29T05:32:09.805Z [container-agent]     framework._emit(event)
2024-02-29T05:32:09.805Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 853, in _emit
2024-02-29T05:32:09.805Z [container-agent]     self._reemit(event_path)
2024-02-29T05:32:09.805Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 942, in _reemit
2024-02-29T05:32:09.805Z [container-agent]     custom_handler(event)
2024-02-29T05:32:09.805Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/rotate_mysql_logs.py", line 57, in _rotate_mysql_logs
2024-02-29T05:32:09.805Z [container-agent]     self.charm._mysql.flush_mysql_logs(MySQLTextLogs.SLOW)
2024-02-29T05:32:09.805Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-02-29T05:32:09.805Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-02-29T05:32:09.805Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 666, in _run_mysqlsh_script
2024-02-29T05:32:09.805Z [container-agent]     stdout, _ = process.wait_output()
2024-02-29T05:32:09.805Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1436, in wait_output
2024-02-29T05:32:09.805Z [container-agent]     exit_code: int = self._wait()
2024-02-29T05:32:09.805Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1375, in _wait
2024-02-29T05:32:09.805Z [container-agent]     change = self._client.wait_change(self._change_id, timeout=timeout)
2024-02-29T05:32:09.805Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1893, in wait_change
2024-02-29T05:32:09.805Z [container-agent]     return self._wait_change_using_wait(change_id, timeout)
2024-02-29T05:32:09.805Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1914, in _wait_change_using_wait
2024-02-29T05:32:09.805Z [container-agent]     return self._wait_change(change_id, this_timeout)
2024-02-29T05:32:09.805Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1928, in _wait_change
2024-02-29T05:32:09.805Z [container-agent]     resp = self._request('GET', f'/v1/changes/{change_id}/wait', query)
2024-02-29T05:32:09.805Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1655, in _request
2024-02-29T05:32:09.805Z [container-agent]     response = self._request_raw(method, path, query, headers, data)
2024-02-29T05:32:09.805Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1690, in _request_raw
2024-02-29T05:32:09.805Z [container-agent]     response = self.opener.open(request, timeout=self.timeout)
2024-02-29T05:32:09.805Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 519, in open
2024-02-29T05:32:09.805Z [container-agent]     response = self._open(req, data)
2024-02-29T05:32:09.805Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 536, in _open
2024-02-29T05:32:09.805Z [container-agent]     result = self._call_chain(self.handle_open, protocol, protocol +
2024-02-29T05:32:09.805Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 496, in _call_chain
2024-02-29T05:32:09.805Z [container-agent]     result = func(*args)
2024-02-29T05:32:09.805Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 311, in http_open
2024-02-29T05:32:09.805Z [container-agent]     return self.do_open(_UnixSocketConnection, req,  # type:ignore
2024-02-29T05:32:09.805Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 1352, in do_open
2024-02-29T05:32:09.805Z [container-agent]     r = h.getresponse()
2024-02-29T05:32:09.805Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1375, in getresponse
2024-02-29T05:32:09.805Z [container-agent]     response.begin()
2024-02-29T05:32:09.805Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 318, in begin
2024-02-29T05:32:09.805Z [container-agent]     version, status, reason = self._read_status()
2024-02-29T05:32:09.805Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 287, in _read_status
2024-02-29T05:32:09.805Z [container-agent]     raise RemoteDisconnected("Remote end closed connection without"
2024-02-29T05:32:09.805Z [container-agent] http.client.RemoteDisconnected: Remote end closed connection without response
2024-02-29T05:32:09.986Z [container-agent] 2024-02-29 05:32:09 INFO juju.util.exec exec.go:209 run result: exit status 1
2024-02-29T05:32:12.509Z [container-agent] 2024-02-29 05:32:12 INFO juju-log Setting up the logrotate configurations
2024-02-29T05:32:12.580Z [container-agent] 2024-02-29 05:32:12 INFO juju-log Adding pebble layer
2024-02-29T05:32:19.341Z [container-agent] 2024-02-29 05:32:19 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-02-29T05:32:19.363Z [container-agent] 2024-02-29 05:32:19 INFO juju-log Attempting reboot from complete outage.
2024-02-29T05:32:22.238Z [container-agent] 2024-02-29 05:32:22 INFO juju.worker.uniter.operation runhook.go:186 ran "mysql-pebble-ready" hook (via hook dispatching script: dispatch)
2024-02-29T05:36:18.700Z [container-agent] 2024-02-29 05:36:18 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T05:36:20.579Z [container-agent] 2024-02-29 05:36:20 WARNING juju-log No relation: certificates
2024-02-29T05:36:20.936Z [container-agent] 2024-02-29 05:36:20 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T05:41:49.998Z [container-agent] 2024-02-29 05:41:49 INFO juju-log Starting the log rotate manager
2024-02-29T05:41:50.011Z [container-agent] 2024-02-29 05:41:50 INFO juju-log Started log rotate manager process with PID 241476
2024-02-29T05:41:50.281Z [container-agent] 2024-02-29 05:41:50 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T05:41:52.175Z [container-agent] 2024-02-29 05:41:52 WARNING juju-log No relation: certificates
2024-02-29T05:41:52.516Z [container-agent] 2024-02-29 05:41:52 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T05:46:46.425Z [container-agent] 2024-02-29 05:46:46 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T05:46:48.698Z [container-agent] 2024-02-29 05:46:48 WARNING juju-log No relation: certificates
2024-02-29T05:46:49.047Z [container-agent] 2024-02-29 05:46:49 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T05:50:51.431Z [container-agent] 2024-02-29 05:50:51 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T05:50:54.088Z [container-agent] 2024-02-29 05:50:54 WARNING juju-log No relation: certificates
2024-02-29T05:50:54.425Z [container-agent] 2024-02-29 05:50:54 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T05:56:36.390Z [container-agent] 2024-02-29 05:56:36 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T05:56:39.751Z [container-agent] 2024-02-29 05:56:39 WARNING juju-log No relation: certificates
2024-02-29T05:56:40.099Z [container-agent] 2024-02-29 05:56:40 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T06:02:15.147Z [container-agent] 2024-02-29 06:02:15 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T06:02:19.250Z [container-agent] 2024-02-29 06:02:19 WARNING juju-log No relation: certificates
2024-02-29T06:02:19.587Z [container-agent] 2024-02-29 06:02:19 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T06:08:08.754Z [container-agent] 2024-02-29 06:08:08 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T06:08:13.108Z [container-agent] 2024-02-29 06:08:13 WARNING juju-log No relation: certificates
2024-02-29T06:08:13.446Z [container-agent] 2024-02-29 06:08:13 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T06:13:35.957Z [container-agent] 2024-02-29 06:13:35 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T06:13:40.538Z [container-agent] 2024-02-29 06:13:40 WARNING juju-log No relation: certificates
2024-02-29T06:13:40.912Z [container-agent] 2024-02-29 06:13:40 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T06:19:16.067Z [container-agent] 2024-02-29 06:19:16 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T06:19:21.788Z [container-agent] 2024-02-29 06:19:21 WARNING juju-log No relation: certificates
2024-02-29T06:19:22.123Z [container-agent] 2024-02-29 06:19:22 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T06:23:42.934Z [container-agent] 2024-02-29 06:23:42 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T06:23:48.829Z [container-agent] 2024-02-29 06:23:48 WARNING juju-log No relation: certificates
2024-02-29T06:23:49.178Z [container-agent] 2024-02-29 06:23:49 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T06:28:52.920Z [container-agent] 2024-02-29 06:28:52 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T06:28:59.390Z [container-agent] 2024-02-29 06:28:59 WARNING juju-log No relation: certificates
2024-02-29T06:28:59.761Z [container-agent] 2024-02-29 06:28:59 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T06:34:00.711Z [container-agent] 2024-02-29 06:34:00 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T06:34:06.966Z [container-agent] 2024-02-29 06:34:06 WARNING juju-log No relation: certificates
2024-02-29T06:34:07.364Z [container-agent] 2024-02-29 06:34:07 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T06:39:38.188Z [container-agent] 2024-02-29 06:39:38 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T06:39:45.306Z [container-agent] 2024-02-29 06:39:45 WARNING juju-log No relation: certificates
2024-02-29T06:39:45.648Z [container-agent] 2024-02-29 06:39:45 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T06:45:00.895Z [container-agent] 2024-02-29 06:45:00 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T06:45:08.175Z [container-agent] 2024-02-29 06:45:08 WARNING juju-log No relation: certificates
2024-02-29T06:45:08.534Z [container-agent] 2024-02-29 06:45:08 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T06:49:15.079Z [container-agent] 2024-02-29 06:49:15 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T06:49:23.380Z [container-agent] 2024-02-29 06:49:23 WARNING juju-log No relation: certificates
2024-02-29T06:49:23.739Z [container-agent] 2024-02-29 06:49:23 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T06:50:07.756Z [container-agent] 2024-02-29 06:50:07 ERROR juju-log Failed to flush ERROR LOGS logs.
2024-02-29T06:50:07.756Z [container-agent] Traceback (most recent call last):
2024-02-29T06:50:07.756Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 666, in _run_mysqlsh_script
2024-02-29T06:50:07.756Z [container-agent]     stdout, _ = process.wait_output()
2024-02-29T06:50:07.756Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1441, in wait_output
2024-02-29T06:50:07.756Z [container-agent]     raise ExecError[AnyStr](self._command, exit_code, out_value, err_value)
2024-02-29T06:50:07.756Z [container-agent] ops.pebble.ExecError: non-zero exit code 1 executing ['/usr/bin/mysqlsh', '--no-wizard', '--python', '--verbose=1', '-f', '/tmp/script.py', ';', 'rm', '/tmp/script.py'], stdout='', stderr='Cannot set LC_ALL to locale en_US.UTF-8: No such file or directory\nverbose: 2024-02-29T06:50:06Z: Loading startup files...\nverbose: 2024-02-29T06:50:06Z: Loading plugins...\nverbose: 2024-02-29T06:50:06Z: Connecting to MySQL at: serverconfig@horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local\nTraceback (most recent call last):\n  File "<string>", line 1, in <module>\nmysqlsh.DBError: MySQL Error (2003): Shell.connect: Can\'t connect to MySQL server on \'horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local:3306\' (111)\n'
2024-02-29T06:50:07.756Z [container-agent] 
2024-02-29T06:50:07.756Z [container-agent] During handling of the above exception, another exception occurred:
2024-02-29T06:50:07.756Z [container-agent] 
2024-02-29T06:50:07.756Z [container-agent] Traceback (most recent call last):
2024-02-29T06:50:07.756Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-02-29T06:50:07.757Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-02-29T06:50:07.757Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 669, in _run_mysqlsh_script
2024-02-29T06:50:07.757Z [container-agent]     raise MySQLClientError(e.stderr)
2024-02-29T06:50:07.757Z [container-agent] charms.mysql.v0.mysql.MySQLClientError: Cannot set LC_ALL to locale en_US.UTF-8: No such file or directory
2024-02-29T06:50:07.757Z [container-agent] verbose: 2024-02-29T06:50:06Z: Loading startup files...
2024-02-29T06:50:07.757Z [container-agent] verbose: 2024-02-29T06:50:06Z: Loading plugins...
2024-02-29T06:50:07.757Z [container-agent] verbose: 2024-02-29T06:50:06Z: Connecting to MySQL at: serverconfig@horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local
2024-02-29T06:50:07.757Z [container-agent] Traceback (most recent call last):
2024-02-29T06:50:07.757Z [container-agent]   File "<string>", line 1, in <module>
2024-02-29T06:50:07.757Z [container-agent] mysqlsh.DBError: MySQL Error (2003): Shell.connect: Can't connect to MySQL server on 'horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local:3306' (111)
2024-02-29T06:50:07.757Z [container-agent] 
2024-02-29T06:54:31.684Z [container-agent] 2024-02-29 06:54:31 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-02-29T06:54:31.717Z [container-agent] 2024-02-29 06:54:31 INFO juju-log Attempting reboot from complete outage.
2024-02-29T06:54:39.823Z [container-agent] 2024-02-29 06:54:39 WARNING juju-log No relation: certificates
2024-02-29T06:54:40.194Z [container-agent] 2024-02-29 06:54:40 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T06:55:04.801Z [container-agent] 2024-02-29 06:55:04 ERROR juju-log Uncaught exception while in charm code:
2024-02-29T06:55:04.801Z [container-agent] Traceback (most recent call last):
2024-02-29T06:55:04.801Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 2507, in exec
2024-02-29T06:55:04.801Z [container-agent]     control_ws = self._connect_websocket(task_id, 'control')
2024-02-29T06:55:04.801Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 2589, in _connect_websocket
2024-02-29T06:55:04.801Z [container-agent]     ws.connect(url, socket=sock)
2024-02-29T06:55:04.801Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_core.py", line 253, in connect
2024-02-29T06:55:04.801Z [container-agent]     self.handshake_response = handshake(self.sock, url, *addrs, **options)
2024-02-29T06:55:04.801Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_handshake.py", line 57, in handshake
2024-02-29T06:55:04.801Z [container-agent]     status, resp = _get_resp_headers(sock)
2024-02-29T06:55:04.801Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_handshake.py", line 143, in _get_resp_headers
2024-02-29T06:55:04.801Z [container-agent]     status, resp_headers, status_message = read_headers(sock)
2024-02-29T06:55:04.801Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_http.py", line 312, in read_headers
2024-02-29T06:55:04.801Z [container-agent]     line = recv_line(sock)
2024-02-29T06:55:04.801Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_socket.py", line 131, in recv_line
2024-02-29T06:55:04.801Z [container-agent]     c = recv(sock, 1)
2024-02-29T06:55:04.801Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_socket.py", line 122, in recv
2024-02-29T06:55:04.801Z [container-agent]     raise WebSocketConnectionClosedException(
2024-02-29T06:55:04.801Z [container-agent] websocket._exceptions.WebSocketConnectionClosedException: Connection to remote host was lost.
2024-02-29T06:55:04.801Z [container-agent] 
2024-02-29T06:55:04.801Z [container-agent] During handling of the above exception, another exception occurred:
2024-02-29T06:55:04.801Z [container-agent] 
2024-02-29T06:55:04.801Z [container-agent] Traceback (most recent call last):
2024-02-29T06:55:04.801Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 1348, in do_open
2024-02-29T06:55:04.801Z [container-agent]     h.request(req.get_method(), req.selector, req.data, headers,
2024-02-29T06:55:04.801Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1283, in request
2024-02-29T06:55:04.801Z [container-agent]     self._send_request(method, url, body, headers, encode_chunked)
2024-02-29T06:55:04.801Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1329, in _send_request
2024-02-29T06:55:04.801Z [container-agent]     self.endheaders(body, encode_chunked=encode_chunked)
2024-02-29T06:55:04.801Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1278, in endheaders
2024-02-29T06:55:04.801Z [container-agent]     self._send_output(message_body, encode_chunked=encode_chunked)
2024-02-29T06:55:04.801Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1038, in _send_output
2024-02-29T06:55:04.801Z [container-agent]     self.send(msg)
2024-02-29T06:55:04.801Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 976, in send
2024-02-29T06:55:04.801Z [container-agent]     self.connect()
2024-02-29T06:55:04.801Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 297, in connect
2024-02-29T06:55:04.801Z [container-agent]     self.sock.connect(self.socket_path)
2024-02-29T06:55:04.801Z [container-agent] ConnectionRefusedError: [Errno 111] Connection refused
2024-02-29T06:55:04.801Z [container-agent] 
2024-02-29T06:55:04.801Z [container-agent] During handling of the above exception, another exception occurred:
2024-02-29T06:55:04.801Z [container-agent] 
2024-02-29T06:55:04.801Z [container-agent] Traceback (most recent call last):
2024-02-29T06:55:04.801Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1690, in _request_raw
2024-02-29T06:55:04.801Z [container-agent]     response = self.opener.open(request, timeout=self.timeout)
2024-02-29T06:55:04.801Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 519, in open
2024-02-29T06:55:04.801Z [container-agent]     response = self._open(req, data)
2024-02-29T06:55:04.801Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 536, in _open
2024-02-29T06:55:04.801Z [container-agent]     result = self._call_chain(self.handle_open, protocol, protocol +
2024-02-29T06:55:04.801Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 496, in _call_chain
2024-02-29T06:55:04.801Z [container-agent]     result = func(*args)
2024-02-29T06:55:04.801Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 311, in http_open
2024-02-29T06:55:04.801Z [container-agent]     return self.do_open(_UnixSocketConnection, req,  # type:ignore
2024-02-29T06:55:04.801Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 1351, in do_open
2024-02-29T06:55:04.801Z [container-agent]     raise URLError(err)
2024-02-29T06:55:04.801Z [container-agent] urllib.error.URLError: <urlopen error [Errno 111] Connection refused>
2024-02-29T06:55:04.801Z [container-agent] 
2024-02-29T06:55:04.801Z [container-agent] The above exception was the direct cause of the following exception:
2024-02-29T06:55:04.801Z [container-agent] 
2024-02-29T06:55:04.801Z [container-agent] Traceback (most recent call last):
2024-02-29T06:55:04.801Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/./src/charm.py", line 786, in <module>
2024-02-29T06:55:04.801Z [container-agent]     main(MySQLOperatorCharm)
2024-02-29T06:55:04.801Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 436, in main
2024-02-29T06:55:04.801Z [container-agent]     _emit_charm_event(charm, dispatcher.event_name)
2024-02-29T06:55:04.801Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 144, in _emit_charm_event
2024-02-29T06:55:04.801Z [container-agent]     event_to_emit.emit(*args, **kwargs)
2024-02-29T06:55:04.801Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 351, in emit
2024-02-29T06:55:04.801Z [container-agent]     framework._emit(event)
2024-02-29T06:55:04.801Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 853, in _emit
2024-02-29T06:55:04.801Z [container-agent]     self._reemit(event_path)
2024-02-29T06:55:04.801Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 942, in _reemit
2024-02-29T06:55:04.801Z [container-agent]     custom_handler(event)
2024-02-29T06:55:04.801Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/rotate_mysql_logs.py", line 55, in _rotate_mysql_logs
2024-02-29T06:55:04.801Z [container-agent]     self.charm._mysql.flush_mysql_logs(MySQLTextLogs.ERROR)
2024-02-29T06:55:04.801Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-02-29T06:55:04.801Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-02-29T06:55:04.801Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 665, in _run_mysqlsh_script
2024-02-29T06:55:04.801Z [container-agent]     process = self.container.exec(cmd, timeout=timeout)
2024-02-29T06:55:04.801Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/model.py", line 2688, in exec
2024-02-29T06:55:04.801Z [container-agent]     return self._pebble.exec(
2024-02-29T06:55:04.801Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 2514, in exec
2024-02-29T06:55:04.801Z [container-agent]     change = self.wait_change(ChangeID(change_id))
2024-02-29T06:55:04.801Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1893, in wait_change
2024-02-29T06:55:04.801Z [container-agent]     return self._wait_change_using_wait(change_id, timeout)
2024-02-29T06:55:04.801Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1914, in _wait_change_using_wait
2024-02-29T06:55:04.801Z [container-agent]     return self._wait_change(change_id, this_timeout)
2024-02-29T06:55:04.801Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1928, in _wait_change
2024-02-29T06:55:04.801Z [container-agent]     resp = self._request('GET', f'/v1/changes/{change_id}/wait', query)
2024-02-29T06:55:04.801Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1655, in _request
2024-02-29T06:55:04.801Z [container-agent]     response = self._request_raw(method, path, query, headers, data)
2024-02-29T06:55:04.801Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1707, in _request_raw
2024-02-29T06:55:04.801Z [container-agent]     raise ConnectionError(e.reason) from e
2024-02-29T06:55:04.801Z [container-agent] ops.pebble.ConnectionError: [Errno 111] Connection refused
2024-02-29T06:55:05.001Z [container-agent] 2024-02-29 06:55:05 INFO juju.util.exec exec.go:209 run result: exit status 1
2024-02-29T06:55:06.956Z [container-agent] 2024-02-29 06:55:06 INFO juju-log Setting up the logrotate configurations
2024-02-29T06:55:07.028Z [container-agent] 2024-02-29 06:55:07 INFO juju-log Adding pebble layer
2024-02-29T06:55:13.842Z [container-agent] 2024-02-29 06:55:13 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-02-29T06:55:13.864Z [container-agent] 2024-02-29 06:55:13 INFO juju-log Attempting reboot from complete outage.
2024-02-29T06:55:16.970Z [container-agent] 2024-02-29 06:55:16 INFO juju.worker.uniter.operation runhook.go:186 ran "mysql-pebble-ready" hook (via hook dispatching script: dispatch)
2024-02-29T06:58:57.126Z [container-agent] 2024-02-29 06:58:57 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T06:58:59.045Z [container-agent] 2024-02-29 06:58:59 WARNING juju-log No relation: certificates
2024-02-29T06:58:59.431Z [container-agent] 2024-02-29 06:58:59 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T07:03:52.891Z [container-agent] 2024-02-29 07:03:52 INFO juju-log Starting the log rotate manager
2024-02-29T07:03:52.903Z [container-agent] 2024-02-29 07:03:52 INFO juju-log Started log rotate manager process with PID 249032
2024-02-29T07:03:53.188Z [container-agent] 2024-02-29 07:03:53 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T07:03:55.097Z [container-agent] 2024-02-29 07:03:55 WARNING juju-log No relation: certificates
2024-02-29T07:03:55.436Z [container-agent] 2024-02-29 07:03:55 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T07:08:41.265Z [container-agent] 2024-02-29 07:08:41 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T07:08:43.775Z [container-agent] 2024-02-29 07:08:43 WARNING juju-log No relation: certificates
2024-02-29T07:08:44.182Z [container-agent] 2024-02-29 07:08:44 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T07:13:06.550Z [container-agent] 2024-02-29 07:13:06 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T07:13:09.439Z [container-agent] 2024-02-29 07:13:09 WARNING juju-log No relation: certificates
2024-02-29T07:13:09.791Z [container-agent] 2024-02-29 07:13:09 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T07:18:26.943Z [container-agent] 2024-02-29 07:18:26 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T07:18:30.456Z [container-agent] 2024-02-29 07:18:30 WARNING juju-log No relation: certificates
2024-02-29T07:18:30.810Z [container-agent] 2024-02-29 07:18:30 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T07:22:54.115Z [container-agent] 2024-02-29 07:22:54 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T07:22:57.731Z [container-agent] 2024-02-29 07:22:57 WARNING juju-log No relation: certificates
2024-02-29T07:22:58.065Z [container-agent] 2024-02-29 07:22:58 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T07:27:20.910Z [container-agent] 2024-02-29 07:27:20 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T07:27:25.220Z [container-agent] 2024-02-29 07:27:25 WARNING juju-log No relation: certificates
2024-02-29T07:27:25.565Z [container-agent] 2024-02-29 07:27:25 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T07:31:34.500Z [container-agent] 2024-02-29 07:31:34 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T07:31:39.239Z [container-agent] 2024-02-29 07:31:39 WARNING juju-log No relation: certificates
2024-02-29T07:31:39.603Z [container-agent] 2024-02-29 07:31:39 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T07:36:12.519Z [container-agent] 2024-02-29 07:36:12 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T07:36:17.868Z [container-agent] 2024-02-29 07:36:17 WARNING juju-log No relation: certificates
2024-02-29T07:36:18.219Z [container-agent] 2024-02-29 07:36:18 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T07:41:58.225Z [container-agent] 2024-02-29 07:41:58 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T07:42:03.620Z [container-agent] 2024-02-29 07:42:03 WARNING juju-log No relation: certificates
2024-02-29T07:42:03.969Z [container-agent] 2024-02-29 07:42:03 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T07:46:46.087Z [container-agent] 2024-02-29 07:46:46 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T07:46:52.386Z [container-agent] 2024-02-29 07:46:52 WARNING juju-log No relation: certificates
2024-02-29T07:46:52.737Z [container-agent] 2024-02-29 07:46:52 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T07:52:12.176Z [container-agent] 2024-02-29 07:52:12 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T07:52:18.869Z [container-agent] 2024-02-29 07:52:18 WARNING juju-log No relation: certificates
2024-02-29T07:52:19.221Z [container-agent] 2024-02-29 07:52:19 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T07:56:29.798Z [container-agent] 2024-02-29 07:56:29 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T07:56:36.745Z [container-agent] 2024-02-29 07:56:36 WARNING juju-log No relation: certificates
2024-02-29T07:56:37.083Z [container-agent] 2024-02-29 07:56:37 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T08:00:32.975Z [container-agent] 2024-02-29 08:00:32 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T08:00:40.265Z [container-agent] 2024-02-29 08:00:40 WARNING juju-log No relation: certificates
2024-02-29T08:00:40.636Z [container-agent] 2024-02-29 08:00:40 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T08:05:14.186Z [container-agent] 2024-02-29 08:05:14 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T08:05:22.184Z [container-agent] 2024-02-29 08:05:22 WARNING juju-log No relation: certificates
2024-02-29T08:05:22.564Z [container-agent] 2024-02-29 08:05:22 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T08:09:42.978Z [container-agent] 2024-02-29 08:09:42 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-02-29T08:09:43.006Z [container-agent] 2024-02-29 08:09:43 INFO juju-log Attempting reboot from complete outage.
2024-02-29T08:09:50.785Z [container-agent] 2024-02-29 08:09:50 WARNING juju-log No relation: certificates
2024-02-29T08:09:51.134Z [container-agent] 2024-02-29 08:09:51 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T08:13:04.816Z [container-agent] 2024-02-29 08:13:04 ERROR juju-log Uncaught exception while in charm code:
2024-02-29T08:13:04.816Z [container-agent] Traceback (most recent call last):
2024-02-29T08:13:04.816Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 2507, in exec
2024-02-29T08:13:04.816Z [container-agent]     control_ws = self._connect_websocket(task_id, 'control')
2024-02-29T08:13:04.816Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 2589, in _connect_websocket
2024-02-29T08:13:04.816Z [container-agent]     ws.connect(url, socket=sock)
2024-02-29T08:13:04.816Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_core.py", line 253, in connect
2024-02-29T08:13:04.816Z [container-agent]     self.handshake_response = handshake(self.sock, url, *addrs, **options)
2024-02-29T08:13:04.816Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_handshake.py", line 57, in handshake
2024-02-29T08:13:04.816Z [container-agent]     status, resp = _get_resp_headers(sock)
2024-02-29T08:13:04.816Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_handshake.py", line 143, in _get_resp_headers
2024-02-29T08:13:04.816Z [container-agent]     status, resp_headers, status_message = read_headers(sock)
2024-02-29T08:13:04.816Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_http.py", line 312, in read_headers
2024-02-29T08:13:04.816Z [container-agent]     line = recv_line(sock)
2024-02-29T08:13:04.816Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_socket.py", line 131, in recv_line
2024-02-29T08:13:04.816Z [container-agent]     c = recv(sock, 1)
2024-02-29T08:13:04.816Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_socket.py", line 122, in recv
2024-02-29T08:13:04.816Z [container-agent]     raise WebSocketConnectionClosedException(
2024-02-29T08:13:04.816Z [container-agent] websocket._exceptions.WebSocketConnectionClosedException: Connection to remote host was lost.
2024-02-29T08:13:04.816Z [container-agent] 
2024-02-29T08:13:04.816Z [container-agent] During handling of the above exception, another exception occurred:
2024-02-29T08:13:04.816Z [container-agent] 
2024-02-29T08:13:04.816Z [container-agent] Traceback (most recent call last):
2024-02-29T08:13:04.816Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 1348, in do_open
2024-02-29T08:13:04.816Z [container-agent]     h.request(req.get_method(), req.selector, req.data, headers,
2024-02-29T08:13:04.816Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1283, in request
2024-02-29T08:13:04.816Z [container-agent]     self._send_request(method, url, body, headers, encode_chunked)
2024-02-29T08:13:04.816Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1329, in _send_request
2024-02-29T08:13:04.816Z [container-agent]     self.endheaders(body, encode_chunked=encode_chunked)
2024-02-29T08:13:04.816Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1278, in endheaders
2024-02-29T08:13:04.816Z [container-agent]     self._send_output(message_body, encode_chunked=encode_chunked)
2024-02-29T08:13:04.816Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1038, in _send_output
2024-02-29T08:13:04.816Z [container-agent]     self.send(msg)
2024-02-29T08:13:04.816Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 976, in send
2024-02-29T08:13:04.816Z [container-agent]     self.connect()
2024-02-29T08:13:04.816Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 297, in connect
2024-02-29T08:13:04.816Z [container-agent]     self.sock.connect(self.socket_path)
2024-02-29T08:13:04.816Z [container-agent] ConnectionRefusedError: [Errno 111] Connection refused
2024-02-29T08:13:04.816Z [container-agent] 
2024-02-29T08:13:04.816Z [container-agent] During handling of the above exception, another exception occurred:
2024-02-29T08:13:04.816Z [container-agent] 
2024-02-29T08:13:04.816Z [container-agent] Traceback (most recent call last):
2024-02-29T08:13:04.816Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1690, in _request_raw
2024-02-29T08:13:04.816Z [container-agent]     response = self.opener.open(request, timeout=self.timeout)
2024-02-29T08:13:04.816Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 519, in open
2024-02-29T08:13:04.816Z [container-agent]     response = self._open(req, data)
2024-02-29T08:13:04.816Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 536, in _open
2024-02-29T08:13:04.816Z [container-agent]     result = self._call_chain(self.handle_open, protocol, protocol +
2024-02-29T08:13:04.816Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 496, in _call_chain
2024-02-29T08:13:04.816Z [container-agent]     result = func(*args)
2024-02-29T08:13:04.816Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 311, in http_open
2024-02-29T08:13:04.816Z [container-agent]     return self.do_open(_UnixSocketConnection, req,  # type:ignore
2024-02-29T08:13:04.816Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 1351, in do_open
2024-02-29T08:13:04.816Z [container-agent]     raise URLError(err)
2024-02-29T08:13:04.816Z [container-agent] urllib.error.URLError: <urlopen error [Errno 111] Connection refused>
2024-02-29T08:13:04.816Z [container-agent] 
2024-02-29T08:13:04.816Z [container-agent] The above exception was the direct cause of the following exception:
2024-02-29T08:13:04.816Z [container-agent] 
2024-02-29T08:13:04.816Z [container-agent] Traceback (most recent call last):
2024-02-29T08:13:04.816Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/./src/charm.py", line 786, in <module>
2024-02-29T08:13:04.816Z [container-agent]     main(MySQLOperatorCharm)
2024-02-29T08:13:04.816Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 436, in main
2024-02-29T08:13:04.816Z [container-agent]     _emit_charm_event(charm, dispatcher.event_name)
2024-02-29T08:13:04.816Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 144, in _emit_charm_event
2024-02-29T08:13:04.816Z [container-agent]     event_to_emit.emit(*args, **kwargs)
2024-02-29T08:13:04.816Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 351, in emit
2024-02-29T08:13:04.816Z [container-agent]     framework._emit(event)
2024-02-29T08:13:04.816Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 853, in _emit
2024-02-29T08:13:04.816Z [container-agent]     self._reemit(event_path)
2024-02-29T08:13:04.816Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 942, in _reemit
2024-02-29T08:13:04.816Z [container-agent]     custom_handler(event)
2024-02-29T08:13:04.816Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/rotate_mysql_logs.py", line 55, in _rotate_mysql_logs
2024-02-29T08:13:04.816Z [container-agent]     self.charm._mysql.flush_mysql_logs(MySQLTextLogs.ERROR)
2024-02-29T08:13:04.816Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-02-29T08:13:04.816Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-02-29T08:13:04.816Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 665, in _run_mysqlsh_script
2024-02-29T08:13:04.816Z [container-agent]     process = self.container.exec(cmd, timeout=timeout)
2024-02-29T08:13:04.816Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/model.py", line 2688, in exec
2024-02-29T08:13:04.816Z [container-agent]     return self._pebble.exec(
2024-02-29T08:13:04.817Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 2514, in exec
2024-02-29T08:13:04.817Z [container-agent]     change = self.wait_change(ChangeID(change_id))
2024-02-29T08:13:04.817Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1893, in wait_change
2024-02-29T08:13:04.817Z [container-agent]     return self._wait_change_using_wait(change_id, timeout)
2024-02-29T08:13:04.817Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1914, in _wait_change_using_wait
2024-02-29T08:13:04.817Z [container-agent]     return self._wait_change(change_id, this_timeout)
2024-02-29T08:13:04.817Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1928, in _wait_change
2024-02-29T08:13:04.817Z [container-agent]     resp = self._request('GET', f'/v1/changes/{change_id}/wait', query)
2024-02-29T08:13:04.817Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1655, in _request
2024-02-29T08:13:04.817Z [container-agent]     response = self._request_raw(method, path, query, headers, data)
2024-02-29T08:13:04.817Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1707, in _request_raw
2024-02-29T08:13:04.817Z [container-agent]     raise ConnectionError(e.reason) from e
2024-02-29T08:13:04.817Z [container-agent] ops.pebble.ConnectionError: [Errno 111] Connection refused
2024-02-29T08:13:04.958Z [container-agent] 2024-02-29 08:13:04 INFO juju.util.exec exec.go:209 run result: exit status 1
2024-02-29T08:13:09.940Z [container-agent] 2024-02-29 08:13:09 INFO juju-log Setting up the logrotate configurations
2024-02-29T08:13:10.000Z [container-agent] 2024-02-29 08:13:10 INFO juju-log Adding pebble layer
2024-02-29T08:13:16.796Z [container-agent] 2024-02-29 08:13:16 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-02-29T08:13:16.821Z [container-agent] 2024-02-29 08:13:16 INFO juju-log Attempting reboot from complete outage.
2024-02-29T08:13:19.900Z [container-agent] 2024-02-29 08:13:19 INFO juju.worker.uniter.operation runhook.go:186 ran "mysql-pebble-ready" hook (via hook dispatching script: dispatch)
2024-02-29T08:13:55.982Z [container-agent] 2024-02-29 08:13:55 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T08:13:57.826Z [container-agent] 2024-02-29 08:13:57 WARNING juju-log No relation: certificates
2024-02-29T08:13:58.194Z [container-agent] 2024-02-29 08:13:58 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T08:17:57.017Z [container-agent] 2024-02-29 08:17:57 INFO juju-log Starting the log rotate manager
2024-02-29T08:17:57.029Z [container-agent] 2024-02-29 08:17:57 INFO juju-log Started log rotate manager process with PID 256272
2024-02-29T08:17:57.316Z [container-agent] 2024-02-29 08:17:57 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T08:17:59.182Z [container-agent] 2024-02-29 08:17:59 WARNING juju-log No relation: certificates
2024-02-29T08:17:59.559Z [container-agent] 2024-02-29 08:17:59 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T08:23:33.398Z [container-agent] 2024-02-29 08:23:33 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T08:23:36.027Z [container-agent] 2024-02-29 08:23:36 WARNING juju-log No relation: certificates
2024-02-29T08:23:36.378Z [container-agent] 2024-02-29 08:23:36 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T08:28:30.845Z [container-agent] 2024-02-29 08:28:30 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T08:28:34.037Z [container-agent] 2024-02-29 08:28:34 WARNING juju-log No relation: certificates
2024-02-29T08:28:34.437Z [container-agent] 2024-02-29 08:28:34 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T08:33:55.126Z [container-agent] 2024-02-29 08:33:55 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T08:33:58.568Z [container-agent] 2024-02-29 08:33:58 WARNING juju-log No relation: certificates
2024-02-29T08:33:58.932Z [container-agent] 2024-02-29 08:33:58 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T08:39:52.067Z [container-agent] 2024-02-29 08:39:52 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T08:39:55.783Z [container-agent] 2024-02-29 08:39:55 WARNING juju-log No relation: certificates
2024-02-29T08:39:56.135Z [container-agent] 2024-02-29 08:39:56 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T08:43:57.486Z [container-agent] 2024-02-29 08:43:57 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T08:44:01.768Z [container-agent] 2024-02-29 08:44:01 WARNING juju-log No relation: certificates
2024-02-29T08:44:02.113Z [container-agent] 2024-02-29 08:44:02 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T08:49:09.410Z [container-agent] 2024-02-29 08:49:09 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T08:49:14.747Z [container-agent] 2024-02-29 08:49:14 WARNING juju-log No relation: certificates
2024-02-29T08:49:15.134Z [container-agent] 2024-02-29 08:49:15 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T08:53:09.682Z [container-agent] 2024-02-29 08:53:09 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T08:53:15.086Z [container-agent] 2024-02-29 08:53:15 WARNING juju-log No relation: certificates
2024-02-29T08:53:15.441Z [container-agent] 2024-02-29 08:53:15 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T08:58:35.724Z [container-agent] 2024-02-29 08:58:35 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T08:58:40.981Z [container-agent] 2024-02-29 08:58:40 WARNING juju-log No relation: certificates
2024-02-29T08:58:41.311Z [container-agent] 2024-02-29 08:58:41 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T09:03:29.997Z [container-agent] 2024-02-29 09:03:29 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T09:03:36.260Z [container-agent] 2024-02-29 09:03:36 WARNING juju-log No relation: certificates
2024-02-29T09:03:36.634Z [container-agent] 2024-02-29 09:03:36 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T09:07:40.256Z [container-agent] 2024-02-29 09:07:40 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T09:07:46.868Z [container-agent] 2024-02-29 09:07:46 WARNING juju-log No relation: certificates
2024-02-29T09:07:47.228Z [container-agent] 2024-02-29 09:07:47 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T09:12:55.304Z [container-agent] 2024-02-29 09:12:55 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T09:13:01.710Z [container-agent] 2024-02-29 09:13:01 WARNING juju-log No relation: certificates
2024-02-29T09:13:02.090Z [container-agent] 2024-02-29 09:13:02 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T09:18:41.199Z [container-agent] 2024-02-29 09:18:41 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T09:18:49.013Z [container-agent] 2024-02-29 09:18:49 WARNING juju-log No relation: certificates
2024-02-29T09:18:49.415Z [container-agent] 2024-02-29 09:18:49 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T09:23:29.301Z [container-agent] 2024-02-29 09:23:29 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-02-29T09:23:29.334Z [container-agent] 2024-02-29 09:23:29 INFO juju-log Attempting reboot from complete outage.
2024-02-29T09:23:37.263Z [container-agent] 2024-02-29 09:23:37 WARNING juju-log No relation: certificates
2024-02-29T09:23:37.634Z [container-agent] 2024-02-29 09:23:37 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T09:28:04.747Z [container-agent] 2024-02-29 09:28:04 ERROR juju-log Uncaught exception while in charm code:
2024-02-29T09:28:04.747Z [container-agent] Traceback (most recent call last):
2024-02-29T09:28:04.747Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 2507, in exec
2024-02-29T09:28:04.747Z [container-agent]     control_ws = self._connect_websocket(task_id, 'control')
2024-02-29T09:28:04.747Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 2589, in _connect_websocket
2024-02-29T09:28:04.747Z [container-agent]     ws.connect(url, socket=sock)
2024-02-29T09:28:04.747Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_core.py", line 253, in connect
2024-02-29T09:28:04.747Z [container-agent]     self.handshake_response = handshake(self.sock, url, *addrs, **options)
2024-02-29T09:28:04.747Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_handshake.py", line 57, in handshake
2024-02-29T09:28:04.747Z [container-agent]     status, resp = _get_resp_headers(sock)
2024-02-29T09:28:04.747Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_handshake.py", line 143, in _get_resp_headers
2024-02-29T09:28:04.747Z [container-agent]     status, resp_headers, status_message = read_headers(sock)
2024-02-29T09:28:04.747Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_http.py", line 312, in read_headers
2024-02-29T09:28:04.747Z [container-agent]     line = recv_line(sock)
2024-02-29T09:28:04.747Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_socket.py", line 131, in recv_line
2024-02-29T09:28:04.747Z [container-agent]     c = recv(sock, 1)
2024-02-29T09:28:04.747Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_socket.py", line 122, in recv
2024-02-29T09:28:04.747Z [container-agent]     raise WebSocketConnectionClosedException(
2024-02-29T09:28:04.747Z [container-agent] websocket._exceptions.WebSocketConnectionClosedException: Connection to remote host was lost.
2024-02-29T09:28:04.747Z [container-agent] 
2024-02-29T09:28:04.747Z [container-agent] During handling of the above exception, another exception occurred:
2024-02-29T09:28:04.747Z [container-agent] 
2024-02-29T09:28:04.747Z [container-agent] Traceback (most recent call last):
2024-02-29T09:28:04.747Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 1348, in do_open
2024-02-29T09:28:04.747Z [container-agent]     h.request(req.get_method(), req.selector, req.data, headers,
2024-02-29T09:28:04.747Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1283, in request
2024-02-29T09:28:04.747Z [container-agent]     self._send_request(method, url, body, headers, encode_chunked)
2024-02-29T09:28:04.747Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1329, in _send_request
2024-02-29T09:28:04.747Z [container-agent]     self.endheaders(body, encode_chunked=encode_chunked)
2024-02-29T09:28:04.747Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1278, in endheaders
2024-02-29T09:28:04.747Z [container-agent]     self._send_output(message_body, encode_chunked=encode_chunked)
2024-02-29T09:28:04.747Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1038, in _send_output
2024-02-29T09:28:04.747Z [container-agent]     self.send(msg)
2024-02-29T09:28:04.747Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 976, in send
2024-02-29T09:28:04.747Z [container-agent]     self.connect()
2024-02-29T09:28:04.747Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 297, in connect
2024-02-29T09:28:04.747Z [container-agent]     self.sock.connect(self.socket_path)
2024-02-29T09:28:04.747Z [container-agent] ConnectionRefusedError: [Errno 111] Connection refused
2024-02-29T09:28:04.747Z [container-agent] 
2024-02-29T09:28:04.747Z [container-agent] During handling of the above exception, another exception occurred:
2024-02-29T09:28:04.747Z [container-agent] 
2024-02-29T09:28:04.747Z [container-agent] Traceback (most recent call last):
2024-02-29T09:28:04.747Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1690, in _request_raw
2024-02-29T09:28:04.747Z [container-agent]     response = self.opener.open(request, timeout=self.timeout)
2024-02-29T09:28:04.747Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 519, in open
2024-02-29T09:28:04.747Z [container-agent]     response = self._open(req, data)
2024-02-29T09:28:04.747Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 536, in _open
2024-02-29T09:28:04.747Z [container-agent]     result = self._call_chain(self.handle_open, protocol, protocol +
2024-02-29T09:28:04.747Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 496, in _call_chain
2024-02-29T09:28:04.747Z [container-agent]     result = func(*args)
2024-02-29T09:28:04.747Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 311, in http_open
2024-02-29T09:28:04.747Z [container-agent]     return self.do_open(_UnixSocketConnection, req,  # type:ignore
2024-02-29T09:28:04.747Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 1351, in do_open
2024-02-29T09:28:04.747Z [container-agent]     raise URLError(err)
2024-02-29T09:28:04.747Z [container-agent] urllib.error.URLError: <urlopen error [Errno 111] Connection refused>
2024-02-29T09:28:04.747Z [container-agent] 
2024-02-29T09:28:04.747Z [container-agent] The above exception was the direct cause of the following exception:
2024-02-29T09:28:04.747Z [container-agent] 
2024-02-29T09:28:04.747Z [container-agent] Traceback (most recent call last):
2024-02-29T09:28:04.747Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/./src/charm.py", line 786, in <module>
2024-02-29T09:28:04.747Z [container-agent]     main(MySQLOperatorCharm)
2024-02-29T09:28:04.747Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 436, in main
2024-02-29T09:28:04.747Z [container-agent]     _emit_charm_event(charm, dispatcher.event_name)
2024-02-29T09:28:04.747Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 144, in _emit_charm_event
2024-02-29T09:28:04.747Z [container-agent]     event_to_emit.emit(*args, **kwargs)
2024-02-29T09:28:04.747Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 351, in emit
2024-02-29T09:28:04.747Z [container-agent]     framework._emit(event)
2024-02-29T09:28:04.747Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 853, in _emit
2024-02-29T09:28:04.747Z [container-agent]     self._reemit(event_path)
2024-02-29T09:28:04.747Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 942, in _reemit
2024-02-29T09:28:04.747Z [container-agent]     custom_handler(event)
2024-02-29T09:28:04.747Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/rotate_mysql_logs.py", line 55, in _rotate_mysql_logs
2024-02-29T09:28:04.747Z [container-agent]     self.charm._mysql.flush_mysql_logs(MySQLTextLogs.ERROR)
2024-02-29T09:28:04.747Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-02-29T09:28:04.747Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-02-29T09:28:04.747Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 665, in _run_mysqlsh_script
2024-02-29T09:28:04.747Z [container-agent]     process = self.container.exec(cmd, timeout=timeout)
2024-02-29T09:28:04.747Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/model.py", line 2688, in exec
2024-02-29T09:28:04.747Z [container-agent]     return self._pebble.exec(
2024-02-29T09:28:04.747Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 2514, in exec
2024-02-29T09:28:04.747Z [container-agent]     change = self.wait_change(ChangeID(change_id))
2024-02-29T09:28:04.747Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1893, in wait_change
2024-02-29T09:28:04.747Z [container-agent]     return self._wait_change_using_wait(change_id, timeout)
2024-02-29T09:28:04.747Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1914, in _wait_change_using_wait
2024-02-29T09:28:04.747Z [container-agent]     return self._wait_change(change_id, this_timeout)
2024-02-29T09:28:04.747Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1928, in _wait_change
2024-02-29T09:28:04.747Z [container-agent]     resp = self._request('GET', f'/v1/changes/{change_id}/wait', query)
2024-02-29T09:28:04.747Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1655, in _request
2024-02-29T09:28:04.747Z [container-agent]     response = self._request_raw(method, path, query, headers, data)
2024-02-29T09:28:04.747Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1707, in _request_raw
2024-02-29T09:28:04.747Z [container-agent]     raise ConnectionError(e.reason) from e
2024-02-29T09:28:04.747Z [container-agent] ops.pebble.ConnectionError: [Errno 111] Connection refused
2024-02-29T09:28:04.974Z [container-agent] 2024-02-29 09:28:04 INFO juju.util.exec exec.go:209 run result: exit status 1
2024-02-29T09:28:07.420Z [container-agent] 2024-02-29 09:28:07 INFO juju-log Setting up the logrotate configurations
2024-02-29T09:28:07.498Z [container-agent] 2024-02-29 09:28:07 INFO juju-log Adding pebble layer
2024-02-29T09:28:14.252Z [container-agent] 2024-02-29 09:28:14 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-02-29T09:28:14.283Z [container-agent] 2024-02-29 09:28:14 INFO juju-log Attempting reboot from complete outage.
2024-02-29T09:28:17.315Z [container-agent] 2024-02-29 09:28:17 INFO juju.worker.uniter.operation runhook.go:186 ran "mysql-pebble-ready" hook (via hook dispatching script: dispatch)
2024-02-29T09:28:30.619Z [container-agent] 2024-02-29 09:28:30 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T09:28:32.428Z [container-agent] 2024-02-29 09:28:32 WARNING juju-log No relation: certificates
2024-02-29T09:28:32.847Z [container-agent] 2024-02-29 09:28:32 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T09:33:34.628Z [container-agent] 2024-02-29 09:33:34 INFO juju-log Starting the log rotate manager
2024-02-29T09:33:34.644Z [container-agent] 2024-02-29 09:33:34 INFO juju-log Started log rotate manager process with PID 263481
2024-02-29T09:33:34.913Z [container-agent] 2024-02-29 09:33:34 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T09:33:36.880Z [container-agent] 2024-02-29 09:33:36 WARNING juju-log No relation: certificates
2024-02-29T09:33:37.223Z [container-agent] 2024-02-29 09:33:37 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T09:37:37.708Z [container-agent] 2024-02-29 09:37:37 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T09:37:39.974Z [container-agent] 2024-02-29 09:37:39 WARNING juju-log No relation: certificates
2024-02-29T09:37:40.346Z [container-agent] 2024-02-29 09:37:40 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T09:42:49.684Z [container-agent] 2024-02-29 09:42:49 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T09:42:52.349Z [container-agent] 2024-02-29 09:42:52 WARNING juju-log No relation: certificates
2024-02-29T09:42:52.686Z [container-agent] 2024-02-29 09:42:52 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T09:47:40.895Z [container-agent] 2024-02-29 09:47:40 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T09:47:44.249Z [container-agent] 2024-02-29 09:47:44 WARNING juju-log No relation: certificates
2024-02-29T09:47:44.603Z [container-agent] 2024-02-29 09:47:44 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T09:52:02.048Z [container-agent] 2024-02-29 09:52:02 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T09:52:05.642Z [container-agent] 2024-02-29 09:52:05 WARNING juju-log No relation: certificates
2024-02-29T09:52:05.969Z [container-agent] 2024-02-29 09:52:05 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T09:56:52.739Z [container-agent] 2024-02-29 09:56:52 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T09:56:56.865Z [container-agent] 2024-02-29 09:56:56 WARNING juju-log No relation: certificates
2024-02-29T09:56:57.210Z [container-agent] 2024-02-29 09:56:57 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T10:02:13.925Z [container-agent] 2024-02-29 10:02:13 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T10:02:18.842Z [container-agent] 2024-02-29 10:02:18 WARNING juju-log No relation: certificates
2024-02-29T10:02:19.194Z [container-agent] 2024-02-29 10:02:19 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T10:06:43.482Z [container-agent] 2024-02-29 10:06:43 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T10:06:48.117Z [container-agent] 2024-02-29 10:06:48 WARNING juju-log No relation: certificates
2024-02-29T10:06:48.469Z [container-agent] 2024-02-29 10:06:48 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T10:10:45.571Z [container-agent] 2024-02-29 10:10:45 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T10:10:51.184Z [container-agent] 2024-02-29 10:10:51 WARNING juju-log No relation: certificates
2024-02-29T10:10:51.550Z [container-agent] 2024-02-29 10:10:51 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T10:15:29.983Z [container-agent] 2024-02-29 10:15:29 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T10:15:35.743Z [container-agent] 2024-02-29 10:15:35 WARNING juju-log No relation: certificates
2024-02-29T10:15:36.117Z [container-agent] 2024-02-29 10:15:36 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T10:21:26.755Z [container-agent] 2024-02-29 10:21:26 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T10:21:33.017Z [container-agent] 2024-02-29 10:21:33 WARNING juju-log No relation: certificates
2024-02-29T10:21:33.364Z [container-agent] 2024-02-29 10:21:33 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T10:26:18.517Z [container-agent] 2024-02-29 10:26:18 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T10:26:25.458Z [container-agent] 2024-02-29 10:26:25 WARNING juju-log No relation: certificates
2024-02-29T10:26:25.807Z [container-agent] 2024-02-29 10:26:25 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T10:30:28.814Z [container-agent] 2024-02-29 10:30:28 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T10:30:36.307Z [container-agent] 2024-02-29 10:30:36 WARNING juju-log No relation: certificates
2024-02-29T10:30:36.667Z [container-agent] 2024-02-29 10:30:36 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T10:35:00.990Z [container-agent] 2024-02-29 10:35:00 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T10:35:08.513Z [container-agent] 2024-02-29 10:35:08 WARNING juju-log No relation: certificates
2024-02-29T10:35:08.872Z [container-agent] 2024-02-29 10:35:08 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T10:40:13.854Z [container-agent] 2024-02-29 10:40:13 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T10:40:22.102Z [container-agent] 2024-02-29 10:40:22 WARNING juju-log No relation: certificates
2024-02-29T10:40:22.446Z [container-agent] 2024-02-29 10:40:22 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T10:44:55.517Z [container-agent] 2024-02-29 10:44:55 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-02-29T10:44:55.540Z [container-agent] 2024-02-29 10:44:55 INFO juju-log Attempting reboot from complete outage.
2024-02-29T10:45:02.944Z [container-agent] 2024-02-29 10:45:02 WARNING juju-log No relation: certificates
2024-02-29T10:45:03.284Z [container-agent] 2024-02-29 10:45:03 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T10:46:04.774Z [container-agent] 2024-02-29 10:46:04 ERROR juju-log Uncaught exception while in charm code:
2024-02-29T10:46:04.774Z [container-agent] Traceback (most recent call last):
2024-02-29T10:46:04.774Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 2507, in exec
2024-02-29T10:46:04.774Z [container-agent]     control_ws = self._connect_websocket(task_id, 'control')
2024-02-29T10:46:04.774Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 2589, in _connect_websocket
2024-02-29T10:46:04.774Z [container-agent]     ws.connect(url, socket=sock)
2024-02-29T10:46:04.774Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_core.py", line 253, in connect
2024-02-29T10:46:04.774Z [container-agent]     self.handshake_response = handshake(self.sock, url, *addrs, **options)
2024-02-29T10:46:04.774Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_handshake.py", line 57, in handshake
2024-02-29T10:46:04.774Z [container-agent]     status, resp = _get_resp_headers(sock)
2024-02-29T10:46:04.774Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_handshake.py", line 143, in _get_resp_headers
2024-02-29T10:46:04.774Z [container-agent]     status, resp_headers, status_message = read_headers(sock)
2024-02-29T10:46:04.774Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_http.py", line 312, in read_headers
2024-02-29T10:46:04.774Z [container-agent]     line = recv_line(sock)
2024-02-29T10:46:04.774Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_socket.py", line 131, in recv_line
2024-02-29T10:46:04.774Z [container-agent]     c = recv(sock, 1)
2024-02-29T10:46:04.774Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_socket.py", line 122, in recv
2024-02-29T10:46:04.774Z [container-agent]     raise WebSocketConnectionClosedException(
2024-02-29T10:46:04.774Z [container-agent] websocket._exceptions.WebSocketConnectionClosedException: Connection to remote host was lost.
2024-02-29T10:46:04.774Z [container-agent] 
2024-02-29T10:46:04.774Z [container-agent] During handling of the above exception, another exception occurred:
2024-02-29T10:46:04.774Z [container-agent] 
2024-02-29T10:46:04.774Z [container-agent] Traceback (most recent call last):
2024-02-29T10:46:04.774Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 1348, in do_open
2024-02-29T10:46:04.774Z [container-agent]     h.request(req.get_method(), req.selector, req.data, headers,
2024-02-29T10:46:04.774Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1283, in request
2024-02-29T10:46:04.774Z [container-agent]     self._send_request(method, url, body, headers, encode_chunked)
2024-02-29T10:46:04.774Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1329, in _send_request
2024-02-29T10:46:04.774Z [container-agent]     self.endheaders(body, encode_chunked=encode_chunked)
2024-02-29T10:46:04.774Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1278, in endheaders
2024-02-29T10:46:04.774Z [container-agent]     self._send_output(message_body, encode_chunked=encode_chunked)
2024-02-29T10:46:04.774Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1038, in _send_output
2024-02-29T10:46:04.774Z [container-agent]     self.send(msg)
2024-02-29T10:46:04.774Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 976, in send
2024-02-29T10:46:04.774Z [container-agent]     self.connect()
2024-02-29T10:46:04.774Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 297, in connect
2024-02-29T10:46:04.774Z [container-agent]     self.sock.connect(self.socket_path)
2024-02-29T10:46:04.774Z [container-agent] ConnectionRefusedError: [Errno 111] Connection refused
2024-02-29T10:46:04.774Z [container-agent] 
2024-02-29T10:46:04.774Z [container-agent] During handling of the above exception, another exception occurred:
2024-02-29T10:46:04.774Z [container-agent] 
2024-02-29T10:46:04.774Z [container-agent] Traceback (most recent call last):
2024-02-29T10:46:04.774Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1690, in _request_raw
2024-02-29T10:46:04.774Z [container-agent]     response = self.opener.open(request, timeout=self.timeout)
2024-02-29T10:46:04.774Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 519, in open
2024-02-29T10:46:04.774Z [container-agent]     response = self._open(req, data)
2024-02-29T10:46:04.774Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 536, in _open
2024-02-29T10:46:04.774Z [container-agent]     result = self._call_chain(self.handle_open, protocol, protocol +
2024-02-29T10:46:04.774Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 496, in _call_chain
2024-02-29T10:46:04.774Z [container-agent]     result = func(*args)
2024-02-29T10:46:04.774Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 311, in http_open
2024-02-29T10:46:04.774Z [container-agent]     return self.do_open(_UnixSocketConnection, req,  # type:ignore
2024-02-29T10:46:04.774Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 1351, in do_open
2024-02-29T10:46:04.774Z [container-agent]     raise URLError(err)
2024-02-29T10:46:04.774Z [container-agent] urllib.error.URLError: <urlopen error [Errno 111] Connection refused>
2024-02-29T10:46:04.774Z [container-agent] 
2024-02-29T10:46:04.774Z [container-agent] The above exception was the direct cause of the following exception:
2024-02-29T10:46:04.774Z [container-agent] 
2024-02-29T10:46:04.774Z [container-agent] Traceback (most recent call last):
2024-02-29T10:46:04.774Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/./src/charm.py", line 786, in <module>
2024-02-29T10:46:04.774Z [container-agent]     main(MySQLOperatorCharm)
2024-02-29T10:46:04.774Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 436, in main
2024-02-29T10:46:04.774Z [container-agent]     _emit_charm_event(charm, dispatcher.event_name)
2024-02-29T10:46:04.774Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 144, in _emit_charm_event
2024-02-29T10:46:04.774Z [container-agent]     event_to_emit.emit(*args, **kwargs)
2024-02-29T10:46:04.774Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 351, in emit
2024-02-29T10:46:04.774Z [container-agent]     framework._emit(event)
2024-02-29T10:46:04.774Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 853, in _emit
2024-02-29T10:46:04.774Z [container-agent]     self._reemit(event_path)
2024-02-29T10:46:04.774Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 942, in _reemit
2024-02-29T10:46:04.774Z [container-agent]     custom_handler(event)
2024-02-29T10:46:04.774Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/rotate_mysql_logs.py", line 55, in _rotate_mysql_logs
2024-02-29T10:46:04.774Z [container-agent]     self.charm._mysql.flush_mysql_logs(MySQLTextLogs.ERROR)
2024-02-29T10:46:04.774Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-02-29T10:46:04.774Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-02-29T10:46:04.774Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 665, in _run_mysqlsh_script
2024-02-29T10:46:04.774Z [container-agent]     process = self.container.exec(cmd, timeout=timeout)
2024-02-29T10:46:04.774Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/model.py", line 2688, in exec
2024-02-29T10:46:04.774Z [container-agent]     return self._pebble.exec(
2024-02-29T10:46:04.774Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 2514, in exec
2024-02-29T10:46:04.774Z [container-agent]     change = self.wait_change(ChangeID(change_id))
2024-02-29T10:46:04.774Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1893, in wait_change
2024-02-29T10:46:04.774Z [container-agent]     return self._wait_change_using_wait(change_id, timeout)
2024-02-29T10:46:04.774Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1914, in _wait_change_using_wait
2024-02-29T10:46:04.774Z [container-agent]     return self._wait_change(change_id, this_timeout)
2024-02-29T10:46:04.774Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1928, in _wait_change
2024-02-29T10:46:04.774Z [container-agent]     resp = self._request('GET', f'/v1/changes/{change_id}/wait', query)
2024-02-29T10:46:04.774Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1655, in _request
2024-02-29T10:46:04.774Z [container-agent]     response = self._request_raw(method, path, query, headers, data)
2024-02-29T10:46:04.774Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1707, in _request_raw
2024-02-29T10:46:04.774Z [container-agent]     raise ConnectionError(e.reason) from e
2024-02-29T10:46:04.774Z [container-agent] ops.pebble.ConnectionError: [Errno 111] Connection refused
2024-02-29T10:46:04.977Z [container-agent] 2024-02-29 10:46:04 INFO juju.util.exec exec.go:209 run result: exit status 1
2024-02-29T10:46:06.837Z [container-agent] 2024-02-29 10:46:06 INFO juju-log Setting up the logrotate configurations
2024-02-29T10:46:06.912Z [container-agent] 2024-02-29 10:46:06 INFO juju-log Adding pebble layer
2024-02-29T10:46:13.606Z [container-agent] 2024-02-29 10:46:13 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-02-29T10:46:13.630Z [container-agent] 2024-02-29 10:46:13 INFO juju-log Attempting reboot from complete outage.
2024-02-29T10:46:16.442Z [container-agent] 2024-02-29 10:46:16 INFO juju.worker.uniter.operation runhook.go:186 ran "mysql-pebble-ready" hook (via hook dispatching script: dispatch)
2024-02-29T10:50:00.363Z [container-agent] 2024-02-29 10:50:00 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T10:50:02.239Z [container-agent] 2024-02-29 10:50:02 WARNING juju-log No relation: certificates
2024-02-29T10:50:02.704Z [container-agent] 2024-02-29 10:50:02 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T10:55:51.563Z [container-agent] 2024-02-29 10:55:51 INFO juju-log Starting the log rotate manager
2024-02-29T10:55:51.577Z [container-agent] 2024-02-29 10:55:51 INFO juju-log Started log rotate manager process with PID 271078
2024-02-29T10:55:51.856Z [container-agent] 2024-02-29 10:55:51 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T10:55:53.863Z [container-agent] 2024-02-29 10:55:53 WARNING juju-log No relation: certificates
2024-02-29T10:55:54.203Z [container-agent] 2024-02-29 10:55:54 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T11:01:22.144Z [container-agent] 2024-02-29 11:01:22 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T11:01:24.653Z [container-agent] 2024-02-29 11:01:24 WARNING juju-log No relation: certificates
2024-02-29T11:01:25.007Z [container-agent] 2024-02-29 11:01:25 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T11:05:58.205Z [container-agent] 2024-02-29 11:05:58 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T11:06:01.293Z [container-agent] 2024-02-29 11:06:01 WARNING juju-log No relation: certificates
2024-02-29T11:06:01.655Z [container-agent] 2024-02-29 11:06:01 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T11:10:07.043Z [container-agent] 2024-02-29 11:10:07 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T11:10:10.563Z [container-agent] 2024-02-29 11:10:10 WARNING juju-log No relation: certificates
2024-02-29T11:10:10.931Z [container-agent] 2024-02-29 11:10:10 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T11:15:35.390Z [container-agent] 2024-02-29 11:15:35 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T11:15:38.969Z [container-agent] 2024-02-29 11:15:38 WARNING juju-log No relation: certificates
2024-02-29T11:15:39.302Z [container-agent] 2024-02-29 11:15:39 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T11:20:08.741Z [container-agent] 2024-02-29 11:20:08 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T11:20:12.755Z [container-agent] 2024-02-29 11:20:12 WARNING juju-log No relation: certificates
2024-02-29T11:20:13.085Z [container-agent] 2024-02-29 11:20:13 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T11:25:58.485Z [container-agent] 2024-02-29 11:25:58 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T11:26:03.016Z [container-agent] 2024-02-29 11:26:03 WARNING juju-log No relation: certificates
2024-02-29T11:26:03.348Z [container-agent] 2024-02-29 11:26:03 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T11:30:29.478Z [container-agent] 2024-02-29 11:30:29 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T11:30:34.823Z [container-agent] 2024-02-29 11:30:34 WARNING juju-log No relation: certificates
2024-02-29T11:30:35.187Z [container-agent] 2024-02-29 11:30:35 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T11:35:52.194Z [container-agent] 2024-02-29 11:35:52 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T11:35:57.923Z [container-agent] 2024-02-29 11:35:57 WARNING juju-log No relation: certificates
2024-02-29T11:35:58.276Z [container-agent] 2024-02-29 11:35:58 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T11:40:11.140Z [container-agent] 2024-02-29 11:40:11 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T11:40:17.339Z [container-agent] 2024-02-29 11:40:17 WARNING juju-log No relation: certificates
2024-02-29T11:40:17.706Z [container-agent] 2024-02-29 11:40:17 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T11:44:25.691Z [container-agent] 2024-02-29 11:44:25 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T11:44:31.959Z [container-agent] 2024-02-29 11:44:31 WARNING juju-log No relation: certificates
2024-02-29T11:44:32.331Z [container-agent] 2024-02-29 11:44:32 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T11:50:12.733Z [container-agent] 2024-02-29 11:50:12 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T11:50:19.885Z [container-agent] 2024-02-29 11:50:19 WARNING juju-log No relation: certificates
2024-02-29T11:50:20.241Z [container-agent] 2024-02-29 11:50:20 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T11:55:35.428Z [container-agent] 2024-02-29 11:55:35 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T11:55:42.871Z [container-agent] 2024-02-29 11:55:42 WARNING juju-log No relation: certificates
2024-02-29T11:55:43.224Z [container-agent] 2024-02-29 11:55:43 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T12:00:53.344Z [container-agent] 2024-02-29 12:00:53 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T12:01:00.842Z [container-agent] 2024-02-29 12:01:00 WARNING juju-log No relation: certificates
2024-02-29T12:01:01.217Z [container-agent] 2024-02-29 12:01:01 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T12:05:43.427Z [container-agent] 2024-02-29 12:05:43 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-02-29T12:05:43.457Z [container-agent] 2024-02-29 12:05:43 INFO juju-log Attempting reboot from complete outage.
2024-02-29T12:05:51.496Z [container-agent] 2024-02-29 12:05:51 WARNING juju-log No relation: certificates
2024-02-29T12:05:51.863Z [container-agent] 2024-02-29 12:05:51 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T12:08:04.823Z [container-agent] 2024-02-29 12:08:04 ERROR juju-log Uncaught exception while in charm code:
2024-02-29T12:08:04.823Z [container-agent] Traceback (most recent call last):
2024-02-29T12:08:04.823Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 2507, in exec
2024-02-29T12:08:04.823Z [container-agent]     control_ws = self._connect_websocket(task_id, 'control')
2024-02-29T12:08:04.823Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 2589, in _connect_websocket
2024-02-29T12:08:04.823Z [container-agent]     ws.connect(url, socket=sock)
2024-02-29T12:08:04.823Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_core.py", line 253, in connect
2024-02-29T12:08:04.823Z [container-agent]     self.handshake_response = handshake(self.sock, url, *addrs, **options)
2024-02-29T12:08:04.823Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_handshake.py", line 57, in handshake
2024-02-29T12:08:04.823Z [container-agent]     status, resp = _get_resp_headers(sock)
2024-02-29T12:08:04.823Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_handshake.py", line 143, in _get_resp_headers
2024-02-29T12:08:04.823Z [container-agent]     status, resp_headers, status_message = read_headers(sock)
2024-02-29T12:08:04.823Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_http.py", line 312, in read_headers
2024-02-29T12:08:04.823Z [container-agent]     line = recv_line(sock)
2024-02-29T12:08:04.823Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_socket.py", line 131, in recv_line
2024-02-29T12:08:04.823Z [container-agent]     c = recv(sock, 1)
2024-02-29T12:08:04.823Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_socket.py", line 122, in recv
2024-02-29T12:08:04.823Z [container-agent]     raise WebSocketConnectionClosedException(
2024-02-29T12:08:04.823Z [container-agent] websocket._exceptions.WebSocketConnectionClosedException: Connection to remote host was lost.
2024-02-29T12:08:04.823Z [container-agent] 
2024-02-29T12:08:04.823Z [container-agent] During handling of the above exception, another exception occurred:
2024-02-29T12:08:04.823Z [container-agent] 
2024-02-29T12:08:04.823Z [container-agent] Traceback (most recent call last):
2024-02-29T12:08:04.823Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 1348, in do_open
2024-02-29T12:08:04.823Z [container-agent]     h.request(req.get_method(), req.selector, req.data, headers,
2024-02-29T12:08:04.823Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1283, in request
2024-02-29T12:08:04.823Z [container-agent]     self._send_request(method, url, body, headers, encode_chunked)
2024-02-29T12:08:04.823Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1329, in _send_request
2024-02-29T12:08:04.823Z [container-agent]     self.endheaders(body, encode_chunked=encode_chunked)
2024-02-29T12:08:04.823Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1278, in endheaders
2024-02-29T12:08:04.823Z [container-agent]     self._send_output(message_body, encode_chunked=encode_chunked)
2024-02-29T12:08:04.823Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1038, in _send_output
2024-02-29T12:08:04.823Z [container-agent]     self.send(msg)
2024-02-29T12:08:04.823Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 976, in send
2024-02-29T12:08:04.823Z [container-agent]     self.connect()
2024-02-29T12:08:04.823Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 297, in connect
2024-02-29T12:08:04.823Z [container-agent]     self.sock.connect(self.socket_path)
2024-02-29T12:08:04.823Z [container-agent] ConnectionRefusedError: [Errno 111] Connection refused
2024-02-29T12:08:04.823Z [container-agent] 
2024-02-29T12:08:04.823Z [container-agent] During handling of the above exception, another exception occurred:
2024-02-29T12:08:04.823Z [container-agent] 
2024-02-29T12:08:04.823Z [container-agent] Traceback (most recent call last):
2024-02-29T12:08:04.823Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1690, in _request_raw
2024-02-29T12:08:04.823Z [container-agent]     response = self.opener.open(request, timeout=self.timeout)
2024-02-29T12:08:04.823Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 519, in open
2024-02-29T12:08:04.823Z [container-agent]     response = self._open(req, data)
2024-02-29T12:08:04.823Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 536, in _open
2024-02-29T12:08:04.823Z [container-agent]     result = self._call_chain(self.handle_open, protocol, protocol +
2024-02-29T12:08:04.823Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 496, in _call_chain
2024-02-29T12:08:04.823Z [container-agent]     result = func(*args)
2024-02-29T12:08:04.823Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 311, in http_open
2024-02-29T12:08:04.823Z [container-agent]     return self.do_open(_UnixSocketConnection, req,  # type:ignore
2024-02-29T12:08:04.823Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 1351, in do_open
2024-02-29T12:08:04.823Z [container-agent]     raise URLError(err)
2024-02-29T12:08:04.823Z [container-agent] urllib.error.URLError: <urlopen error [Errno 111] Connection refused>
2024-02-29T12:08:04.823Z [container-agent] 
2024-02-29T12:08:04.823Z [container-agent] The above exception was the direct cause of the following exception:
2024-02-29T12:08:04.823Z [container-agent] 
2024-02-29T12:08:04.823Z [container-agent] Traceback (most recent call last):
2024-02-29T12:08:04.823Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/./src/charm.py", line 786, in <module>
2024-02-29T12:08:04.823Z [container-agent]     main(MySQLOperatorCharm)
2024-02-29T12:08:04.823Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 436, in main
2024-02-29T12:08:04.823Z [container-agent]     _emit_charm_event(charm, dispatcher.event_name)
2024-02-29T12:08:04.823Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 144, in _emit_charm_event
2024-02-29T12:08:04.823Z [container-agent]     event_to_emit.emit(*args, **kwargs)
2024-02-29T12:08:04.823Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 351, in emit
2024-02-29T12:08:04.823Z [container-agent]     framework._emit(event)
2024-02-29T12:08:04.823Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 853, in _emit
2024-02-29T12:08:04.823Z [container-agent]     self._reemit(event_path)
2024-02-29T12:08:04.823Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 942, in _reemit
2024-02-29T12:08:04.824Z [container-agent]     custom_handler(event)
2024-02-29T12:08:04.824Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/rotate_mysql_logs.py", line 55, in _rotate_mysql_logs
2024-02-29T12:08:04.824Z [container-agent]     self.charm._mysql.flush_mysql_logs(MySQLTextLogs.ERROR)
2024-02-29T12:08:04.824Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-02-29T12:08:04.824Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-02-29T12:08:04.824Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 665, in _run_mysqlsh_script
2024-02-29T12:08:04.824Z [container-agent]     process = self.container.exec(cmd, timeout=timeout)
2024-02-29T12:08:04.824Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/model.py", line 2688, in exec
2024-02-29T12:08:04.824Z [container-agent]     return self._pebble.exec(
2024-02-29T12:08:04.824Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 2514, in exec
2024-02-29T12:08:04.824Z [container-agent]     change = self.wait_change(ChangeID(change_id))
2024-02-29T12:08:04.824Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1893, in wait_change
2024-02-29T12:08:04.824Z [container-agent]     return self._wait_change_using_wait(change_id, timeout)
2024-02-29T12:08:04.824Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1914, in _wait_change_using_wait
2024-02-29T12:08:04.824Z [container-agent]     return self._wait_change(change_id, this_timeout)
2024-02-29T12:08:04.824Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1928, in _wait_change
2024-02-29T12:08:04.824Z [container-agent]     resp = self._request('GET', f'/v1/changes/{change_id}/wait', query)
2024-02-29T12:08:04.824Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1655, in _request
2024-02-29T12:08:04.824Z [container-agent]     response = self._request_raw(method, path, query, headers, data)
2024-02-29T12:08:04.824Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1707, in _request_raw
2024-02-29T12:08:04.824Z [container-agent]     raise ConnectionError(e.reason) from e
2024-02-29T12:08:04.824Z [container-agent] ops.pebble.ConnectionError: [Errno 111] Connection refused
2024-02-29T12:08:05.004Z [container-agent] 2024-02-29 12:08:05 INFO juju.util.exec exec.go:209 run result: exit status 1
2024-02-29T12:08:09.436Z [container-agent] 2024-02-29 12:08:09 INFO juju-log Setting up the logrotate configurations
2024-02-29T12:08:09.494Z [container-agent] 2024-02-29 12:08:09 INFO juju-log Adding pebble layer
2024-02-29T12:08:16.302Z [container-agent] 2024-02-29 12:08:16 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-02-29T12:08:16.328Z [container-agent] 2024-02-29 12:08:16 INFO juju-log Attempting reboot from complete outage.
2024-02-29T12:08:19.339Z [container-agent] 2024-02-29 12:08:19 INFO juju.worker.uniter.operation runhook.go:186 ran "mysql-pebble-ready" hook (via hook dispatching script: dispatch)
2024-02-29T12:09:42.373Z [container-agent] 2024-02-29 12:09:42 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T12:09:44.208Z [container-agent] 2024-02-29 12:09:44 WARNING juju-log No relation: certificates
2024-02-29T12:09:44.573Z [container-agent] 2024-02-29 12:09:44 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T12:13:58.224Z [container-agent] 2024-02-29 12:13:58 INFO juju-log Starting the log rotate manager
2024-02-29T12:13:58.237Z [container-agent] 2024-02-29 12:13:58 INFO juju-log Started log rotate manager process with PID 278518
2024-02-29T12:13:58.505Z [container-agent] 2024-02-29 12:13:58 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T12:14:00.407Z [container-agent] 2024-02-29 12:14:00 WARNING juju-log No relation: certificates
2024-02-29T12:14:00.784Z [container-agent] 2024-02-29 12:14:00 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T12:18:48.834Z [container-agent] 2024-02-29 12:18:48 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T12:18:51.644Z [container-agent] 2024-02-29 12:18:51 WARNING juju-log No relation: certificates
2024-02-29T12:18:51.977Z [container-agent] 2024-02-29 12:18:51 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T12:23:58.751Z [container-agent] 2024-02-29 12:23:58 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T12:24:01.748Z [container-agent] 2024-02-29 12:24:01 WARNING juju-log No relation: certificates
2024-02-29T12:24:02.073Z [container-agent] 2024-02-29 12:24:02 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T12:28:35.431Z [container-agent] 2024-02-29 12:28:35 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T12:28:38.910Z [container-agent] 2024-02-29 12:28:38 WARNING juju-log No relation: certificates
2024-02-29T12:28:39.366Z [container-agent] 2024-02-29 12:28:39 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T12:34:16.993Z [container-agent] 2024-02-29 12:34:16 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T12:34:21.004Z [container-agent] 2024-02-29 12:34:21 WARNING juju-log No relation: certificates
2024-02-29T12:34:21.415Z [container-agent] 2024-02-29 12:34:21 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T12:39:25.786Z [container-agent] 2024-02-29 12:39:25 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T12:39:29.854Z [container-agent] 2024-02-29 12:39:29 WARNING juju-log No relation: certificates
2024-02-29T12:39:30.191Z [container-agent] 2024-02-29 12:39:30 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T12:44:59.954Z [container-agent] 2024-02-29 12:44:59 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T12:45:04.558Z [container-agent] 2024-02-29 12:45:04 WARNING juju-log No relation: certificates
2024-02-29T12:45:04.920Z [container-agent] 2024-02-29 12:45:04 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T12:49:51.819Z [container-agent] 2024-02-29 12:49:51 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T12:49:56.450Z [container-agent] 2024-02-29 12:49:56 WARNING juju-log No relation: certificates
2024-02-29T12:49:56.788Z [container-agent] 2024-02-29 12:49:56 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T12:55:25.274Z [container-agent] 2024-02-29 12:55:25 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T12:55:30.971Z [container-agent] 2024-02-29 12:55:30 WARNING juju-log No relation: certificates
2024-02-29T12:55:31.381Z [container-agent] 2024-02-29 12:55:31 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T13:01:14.014Z [container-agent] 2024-02-29 13:01:14 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T13:01:20.631Z [container-agent] 2024-02-29 13:01:20 WARNING juju-log No relation: certificates
2024-02-29T13:01:20.968Z [container-agent] 2024-02-29 13:01:20 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T13:06:12.526Z [container-agent] 2024-02-29 13:06:12 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T13:06:19.397Z [container-agent] 2024-02-29 13:06:19 WARNING juju-log No relation: certificates
2024-02-29T13:06:19.744Z [container-agent] 2024-02-29 13:06:19 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T13:11:55.424Z [container-agent] 2024-02-29 13:11:55 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T13:12:02.630Z [container-agent] 2024-02-29 13:12:02 WARNING juju-log No relation: certificates
2024-02-29T13:12:02.994Z [container-agent] 2024-02-29 13:12:02 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T13:16:55.083Z [container-agent] 2024-02-29 13:16:55 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T13:17:02.907Z [container-agent] 2024-02-29 13:17:02 WARNING juju-log No relation: certificates
2024-02-29T13:17:03.248Z [container-agent] 2024-02-29 13:17:03 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T13:22:22.795Z [container-agent] 2024-02-29 13:22:22 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-02-29T13:22:22.818Z [container-agent] 2024-02-29 13:22:22 INFO juju-log Attempting reboot from complete outage.
2024-02-29T13:22:30.772Z [container-agent] 2024-02-29 13:22:30 WARNING juju-log No relation: certificates
2024-02-29T13:22:31.171Z [container-agent] 2024-02-29 13:22:31 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T13:26:04.868Z [container-agent] 2024-02-29 13:26:04 ERROR juju-log Uncaught exception while in charm code:
2024-02-29T13:26:04.869Z [container-agent] Traceback (most recent call last):
2024-02-29T13:26:04.869Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 2507, in exec
2024-02-29T13:26:04.869Z [container-agent]     control_ws = self._connect_websocket(task_id, 'control')
2024-02-29T13:26:04.869Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 2589, in _connect_websocket
2024-02-29T13:26:04.869Z [container-agent]     ws.connect(url, socket=sock)
2024-02-29T13:26:04.869Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_core.py", line 253, in connect
2024-02-29T13:26:04.869Z [container-agent]     self.handshake_response = handshake(self.sock, url, *addrs, **options)
2024-02-29T13:26:04.869Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_handshake.py", line 57, in handshake
2024-02-29T13:26:04.869Z [container-agent]     status, resp = _get_resp_headers(sock)
2024-02-29T13:26:04.869Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_handshake.py", line 143, in _get_resp_headers
2024-02-29T13:26:04.869Z [container-agent]     status, resp_headers, status_message = read_headers(sock)
2024-02-29T13:26:04.869Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_http.py", line 312, in read_headers
2024-02-29T13:26:04.869Z [container-agent]     line = recv_line(sock)
2024-02-29T13:26:04.869Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_socket.py", line 131, in recv_line
2024-02-29T13:26:04.869Z [container-agent]     c = recv(sock, 1)
2024-02-29T13:26:04.869Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_socket.py", line 122, in recv
2024-02-29T13:26:04.869Z [container-agent]     raise WebSocketConnectionClosedException(
2024-02-29T13:26:04.869Z [container-agent] websocket._exceptions.WebSocketConnectionClosedException: Connection to remote host was lost.
2024-02-29T13:26:04.869Z [container-agent] 
2024-02-29T13:26:04.869Z [container-agent] During handling of the above exception, another exception occurred:
2024-02-29T13:26:04.869Z [container-agent] 
2024-02-29T13:26:04.869Z [container-agent] Traceback (most recent call last):
2024-02-29T13:26:04.869Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 1348, in do_open
2024-02-29T13:26:04.869Z [container-agent]     h.request(req.get_method(), req.selector, req.data, headers,
2024-02-29T13:26:04.869Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1283, in request
2024-02-29T13:26:04.869Z [container-agent]     self._send_request(method, url, body, headers, encode_chunked)
2024-02-29T13:26:04.869Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1329, in _send_request
2024-02-29T13:26:04.869Z [container-agent]     self.endheaders(body, encode_chunked=encode_chunked)
2024-02-29T13:26:04.869Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1278, in endheaders
2024-02-29T13:26:04.869Z [container-agent]     self._send_output(message_body, encode_chunked=encode_chunked)
2024-02-29T13:26:04.869Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1038, in _send_output
2024-02-29T13:26:04.869Z [container-agent]     self.send(msg)
2024-02-29T13:26:04.869Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 976, in send
2024-02-29T13:26:04.869Z [container-agent]     self.connect()
2024-02-29T13:26:04.869Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 297, in connect
2024-02-29T13:26:04.869Z [container-agent]     self.sock.connect(self.socket_path)
2024-02-29T13:26:04.869Z [container-agent] ConnectionRefusedError: [Errno 111] Connection refused
2024-02-29T13:26:04.869Z [container-agent] 
2024-02-29T13:26:04.869Z [container-agent] During handling of the above exception, another exception occurred:
2024-02-29T13:26:04.869Z [container-agent] 
2024-02-29T13:26:04.869Z [container-agent] Traceback (most recent call last):
2024-02-29T13:26:04.869Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1690, in _request_raw
2024-02-29T13:26:04.869Z [container-agent]     response = self.opener.open(request, timeout=self.timeout)
2024-02-29T13:26:04.869Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 519, in open
2024-02-29T13:26:04.869Z [container-agent]     response = self._open(req, data)
2024-02-29T13:26:04.869Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 536, in _open
2024-02-29T13:26:04.869Z [container-agent]     result = self._call_chain(self.handle_open, protocol, protocol +
2024-02-29T13:26:04.869Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 496, in _call_chain
2024-02-29T13:26:04.869Z [container-agent]     result = func(*args)
2024-02-29T13:26:04.869Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 311, in http_open
2024-02-29T13:26:04.869Z [container-agent]     return self.do_open(_UnixSocketConnection, req,  # type:ignore
2024-02-29T13:26:04.869Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 1351, in do_open
2024-02-29T13:26:04.869Z [container-agent]     raise URLError(err)
2024-02-29T13:26:04.869Z [container-agent] urllib.error.URLError: <urlopen error [Errno 111] Connection refused>
2024-02-29T13:26:04.869Z [container-agent] 
2024-02-29T13:26:04.869Z [container-agent] The above exception was the direct cause of the following exception:
2024-02-29T13:26:04.869Z [container-agent] 
2024-02-29T13:26:04.869Z [container-agent] Traceback (most recent call last):
2024-02-29T13:26:04.869Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/./src/charm.py", line 786, in <module>
2024-02-29T13:26:04.869Z [container-agent]     main(MySQLOperatorCharm)
2024-02-29T13:26:04.869Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 436, in main
2024-02-29T13:26:04.869Z [container-agent]     _emit_charm_event(charm, dispatcher.event_name)
2024-02-29T13:26:04.869Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 144, in _emit_charm_event
2024-02-29T13:26:04.869Z [container-agent]     event_to_emit.emit(*args, **kwargs)
2024-02-29T13:26:04.869Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 351, in emit
2024-02-29T13:26:04.869Z [container-agent]     framework._emit(event)
2024-02-29T13:26:04.869Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 853, in _emit
2024-02-29T13:26:04.869Z [container-agent]     self._reemit(event_path)
2024-02-29T13:26:04.869Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 942, in _reemit
2024-02-29T13:26:04.869Z [container-agent]     custom_handler(event)
2024-02-29T13:26:04.869Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/rotate_mysql_logs.py", line 55, in _rotate_mysql_logs
2024-02-29T13:26:04.869Z [container-agent]     self.charm._mysql.flush_mysql_logs(MySQLTextLogs.ERROR)
2024-02-29T13:26:04.869Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-02-29T13:26:04.869Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-02-29T13:26:04.869Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 665, in _run_mysqlsh_script
2024-02-29T13:26:04.869Z [container-agent]     process = self.container.exec(cmd, timeout=timeout)
2024-02-29T13:26:04.869Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/model.py", line 2688, in exec
2024-02-29T13:26:04.869Z [container-agent]     return self._pebble.exec(
2024-02-29T13:26:04.869Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 2514, in exec
2024-02-29T13:26:04.869Z [container-agent]     change = self.wait_change(ChangeID(change_id))
2024-02-29T13:26:04.869Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1893, in wait_change
2024-02-29T13:26:04.869Z [container-agent]     return self._wait_change_using_wait(change_id, timeout)
2024-02-29T13:26:04.869Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1914, in _wait_change_using_wait
2024-02-29T13:26:04.869Z [container-agent]     return self._wait_change(change_id, this_timeout)
2024-02-29T13:26:04.869Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1928, in _wait_change
2024-02-29T13:26:04.869Z [container-agent]     resp = self._request('GET', f'/v1/changes/{change_id}/wait', query)
2024-02-29T13:26:04.869Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1655, in _request
2024-02-29T13:26:04.869Z [container-agent]     response = self._request_raw(method, path, query, headers, data)
2024-02-29T13:26:04.869Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1707, in _request_raw
2024-02-29T13:26:04.869Z [container-agent]     raise ConnectionError(e.reason) from e
2024-02-29T13:26:04.869Z [container-agent] ops.pebble.ConnectionError: [Errno 111] Connection refused
2024-02-29T13:26:05.063Z [container-agent] 2024-02-29 13:26:05 INFO juju.util.exec exec.go:209 run result: exit status 1
2024-02-29T13:26:07.197Z [container-agent] 2024-02-29 13:26:07 INFO juju-log Setting up the logrotate configurations
2024-02-29T13:26:07.266Z [container-agent] 2024-02-29 13:26:07 INFO juju-log Adding pebble layer
2024-02-29T13:26:14.088Z [container-agent] 2024-02-29 13:26:14 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-02-29T13:26:14.112Z [container-agent] 2024-02-29 13:26:14 INFO juju-log Attempting reboot from complete outage.
2024-02-29T13:26:17.172Z [container-agent] 2024-02-29 13:26:17 INFO juju.worker.uniter.operation runhook.go:186 ran "mysql-pebble-ready" hook (via hook dispatching script: dispatch)
2024-02-29T13:28:00.271Z [container-agent] 2024-02-29 13:28:00 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T13:28:02.396Z [container-agent] 2024-02-29 13:28:02 WARNING juju-log No relation: certificates
2024-02-29T13:28:02.882Z [container-agent] 2024-02-29 13:28:02 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T13:33:41.364Z [container-agent] 2024-02-29 13:33:41 INFO juju-log Starting the log rotate manager
2024-02-29T13:33:41.377Z [container-agent] 2024-02-29 13:33:41 INFO juju-log Started log rotate manager process with PID 285851
2024-02-29T13:33:41.650Z [container-agent] 2024-02-29 13:33:41 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T13:33:43.690Z [container-agent] 2024-02-29 13:33:43 WARNING juju-log No relation: certificates
2024-02-29T13:33:44.106Z [container-agent] 2024-02-29 13:33:44 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T13:38:31.984Z [container-agent] 2024-02-29 13:38:31 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T13:38:34.475Z [container-agent] 2024-02-29 13:38:34 WARNING juju-log No relation: certificates
2024-02-29T13:38:34.827Z [container-agent] 2024-02-29 13:38:34 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T13:43:06.169Z [container-agent] 2024-02-29 13:43:06 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T13:43:09.252Z [container-agent] 2024-02-29 13:43:09 WARNING juju-log No relation: certificates
2024-02-29T13:43:09.608Z [container-agent] 2024-02-29 13:43:09 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T13:47:50.271Z [container-agent] 2024-02-29 13:47:50 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T13:47:53.438Z [container-agent] 2024-02-29 13:47:53 WARNING juju-log No relation: certificates
2024-02-29T13:47:53.756Z [container-agent] 2024-02-29 13:47:53 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T13:52:20.592Z [container-agent] 2024-02-29 13:52:20 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T13:52:24.367Z [container-agent] 2024-02-29 13:52:24 WARNING juju-log No relation: certificates
2024-02-29T13:52:24.720Z [container-agent] 2024-02-29 13:52:24 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T13:57:44.947Z [container-agent] 2024-02-29 13:57:44 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T13:57:49.213Z [container-agent] 2024-02-29 13:57:49 WARNING juju-log No relation: certificates
2024-02-29T13:57:49.557Z [container-agent] 2024-02-29 13:57:49 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T14:02:02.475Z [container-agent] 2024-02-29 14:02:02 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T14:02:06.883Z [container-agent] 2024-02-29 14:02:06 WARNING juju-log No relation: certificates
2024-02-29T14:02:07.248Z [container-agent] 2024-02-29 14:02:07 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T14:06:10.119Z [container-agent] 2024-02-29 14:06:10 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T14:06:15.437Z [container-agent] 2024-02-29 14:06:15 WARNING juju-log No relation: certificates
2024-02-29T14:06:15.816Z [container-agent] 2024-02-29 14:06:15 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T14:10:58.324Z [container-agent] 2024-02-29 14:10:58 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T14:11:03.620Z [container-agent] 2024-02-29 14:11:03 WARNING juju-log No relation: certificates
2024-02-29T14:11:03.984Z [container-agent] 2024-02-29 14:11:03 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T14:15:11.198Z [container-agent] 2024-02-29 14:15:11 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T14:15:17.193Z [container-agent] 2024-02-29 14:15:17 WARNING juju-log No relation: certificates
2024-02-29T14:15:17.541Z [container-agent] 2024-02-29 14:15:17 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T14:20:35.273Z [container-agent] 2024-02-29 14:20:35 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T14:20:41.252Z [container-agent] 2024-02-29 14:20:41 WARNING juju-log No relation: certificates
2024-02-29T14:20:41.587Z [container-agent] 2024-02-29 14:20:41 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T14:24:37.868Z [container-agent] 2024-02-29 14:24:37 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T14:24:44.679Z [container-agent] 2024-02-29 14:24:44 WARNING juju-log No relation: certificates
2024-02-29T14:24:45.029Z [container-agent] 2024-02-29 14:24:45 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T14:29:12.987Z [container-agent] 2024-02-29 14:29:12 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T14:29:20.243Z [container-agent] 2024-02-29 14:29:20 WARNING juju-log No relation: certificates
2024-02-29T14:29:20.577Z [container-agent] 2024-02-29 14:29:20 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T14:32:09.183Z [container-agent] 2024-02-29 14:32:09 ERROR juju-log Failed to flush GENERAL LOGS logs.
2024-02-29T14:32:09.184Z [container-agent] Traceback (most recent call last):
2024-02-29T14:32:09.184Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 666, in _run_mysqlsh_script
2024-02-29T14:32:09.184Z [container-agent]     stdout, _ = process.wait_output()
2024-02-29T14:32:09.184Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1441, in wait_output
2024-02-29T14:32:09.184Z [container-agent]     raise ExecError[AnyStr](self._command, exit_code, out_value, err_value)
2024-02-29T14:32:09.184Z [container-agent] ops.pebble.ExecError: non-zero exit code 1 executing ['/usr/bin/mysqlsh', '--no-wizard', '--python', '--verbose=1', '-f', '/tmp/script.py', ';', 'rm', '/tmp/script.py'], stdout='', stderr='Cannot set LC_ALL to locale en_US.UTF-8: No such file or directory\nverbose: 2024-02-29T14:32:07Z: Loading startup files...\nverbose: 2024-02-29T14:32:07Z: Loading plugins...\nverbose: 2024-02-29T14:32:08Z: Connecting to MySQL at: serverconfig@horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local\nTraceback (most recent call last):\n  File "<string>", line 1, in <module>\nmysqlsh.DBError: MySQL Error (2003): Shell.connect: Can\'t connect to MySQL server on \'horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local:3306\' (111)\n'
2024-02-29T14:32:09.184Z [container-agent] 
2024-02-29T14:32:09.184Z [container-agent] During handling of the above exception, another exception occurred:
2024-02-29T14:32:09.184Z [container-agent] 
2024-02-29T14:32:09.184Z [container-agent] Traceback (most recent call last):
2024-02-29T14:32:09.184Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-02-29T14:32:09.184Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-02-29T14:32:09.184Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 669, in _run_mysqlsh_script
2024-02-29T14:32:09.184Z [container-agent]     raise MySQLClientError(e.stderr)
2024-02-29T14:32:09.184Z [container-agent] charms.mysql.v0.mysql.MySQLClientError: Cannot set LC_ALL to locale en_US.UTF-8: No such file or directory
2024-02-29T14:32:09.184Z [container-agent] verbose: 2024-02-29T14:32:07Z: Loading startup files...
2024-02-29T14:32:09.184Z [container-agent] verbose: 2024-02-29T14:32:07Z: Loading plugins...
2024-02-29T14:32:09.184Z [container-agent] verbose: 2024-02-29T14:32:08Z: Connecting to MySQL at: serverconfig@horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local
2024-02-29T14:32:09.184Z [container-agent] Traceback (most recent call last):
2024-02-29T14:32:09.184Z [container-agent]   File "<string>", line 1, in <module>
2024-02-29T14:32:09.184Z [container-agent] mysqlsh.DBError: MySQL Error (2003): Shell.connect: Can't connect to MySQL server on 'horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local:3306' (111)
2024-02-29T14:32:09.184Z [container-agent] 
2024-02-29T14:34:31.722Z [container-agent] 2024-02-29 14:34:31 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-02-29T14:34:31.786Z [container-agent] 2024-02-29 14:34:31 INFO juju-log Attempting reboot from complete outage.
2024-02-29T14:34:39.093Z [container-agent] 2024-02-29 14:34:39 WARNING juju-log No relation: certificates
2024-02-29T14:34:39.460Z [container-agent] 2024-02-29 14:34:39 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T14:37:04.779Z [container-agent] 2024-02-29 14:37:04 ERROR juju-log Uncaught exception while in charm code:
2024-02-29T14:37:04.779Z [container-agent] Traceback (most recent call last):
2024-02-29T14:37:04.779Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 2507, in exec
2024-02-29T14:37:04.779Z [container-agent]     control_ws = self._connect_websocket(task_id, 'control')
2024-02-29T14:37:04.779Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 2589, in _connect_websocket
2024-02-29T14:37:04.779Z [container-agent]     ws.connect(url, socket=sock)
2024-02-29T14:37:04.779Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_core.py", line 253, in connect
2024-02-29T14:37:04.779Z [container-agent]     self.handshake_response = handshake(self.sock, url, *addrs, **options)
2024-02-29T14:37:04.779Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_handshake.py", line 57, in handshake
2024-02-29T14:37:04.779Z [container-agent]     status, resp = _get_resp_headers(sock)
2024-02-29T14:37:04.779Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_handshake.py", line 143, in _get_resp_headers
2024-02-29T14:37:04.779Z [container-agent]     status, resp_headers, status_message = read_headers(sock)
2024-02-29T14:37:04.779Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_http.py", line 312, in read_headers
2024-02-29T14:37:04.779Z [container-agent]     line = recv_line(sock)
2024-02-29T14:37:04.779Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_socket.py", line 131, in recv_line
2024-02-29T14:37:04.779Z [container-agent]     c = recv(sock, 1)
2024-02-29T14:37:04.779Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_socket.py", line 122, in recv
2024-02-29T14:37:04.779Z [container-agent]     raise WebSocketConnectionClosedException(
2024-02-29T14:37:04.779Z [container-agent] websocket._exceptions.WebSocketConnectionClosedException: Connection to remote host was lost.
2024-02-29T14:37:04.779Z [container-agent] 
2024-02-29T14:37:04.779Z [container-agent] During handling of the above exception, another exception occurred:
2024-02-29T14:37:04.779Z [container-agent] 
2024-02-29T14:37:04.779Z [container-agent] Traceback (most recent call last):
2024-02-29T14:37:04.779Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 1348, in do_open
2024-02-29T14:37:04.779Z [container-agent]     h.request(req.get_method(), req.selector, req.data, headers,
2024-02-29T14:37:04.779Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1283, in request
2024-02-29T14:37:04.779Z [container-agent]     self._send_request(method, url, body, headers, encode_chunked)
2024-02-29T14:37:04.779Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1329, in _send_request
2024-02-29T14:37:04.779Z [container-agent]     self.endheaders(body, encode_chunked=encode_chunked)
2024-02-29T14:37:04.779Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1278, in endheaders
2024-02-29T14:37:04.780Z [container-agent]     self._send_output(message_body, encode_chunked=encode_chunked)
2024-02-29T14:37:04.780Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1038, in _send_output
2024-02-29T14:37:04.780Z [container-agent]     self.send(msg)
2024-02-29T14:37:04.780Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 976, in send
2024-02-29T14:37:04.780Z [container-agent]     self.connect()
2024-02-29T14:37:04.780Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 297, in connect
2024-02-29T14:37:04.780Z [container-agent]     self.sock.connect(self.socket_path)
2024-02-29T14:37:04.780Z [container-agent] ConnectionRefusedError: [Errno 111] Connection refused
2024-02-29T14:37:04.780Z [container-agent] 
2024-02-29T14:37:04.780Z [container-agent] During handling of the above exception, another exception occurred:
2024-02-29T14:37:04.780Z [container-agent] 
2024-02-29T14:37:04.780Z [container-agent] Traceback (most recent call last):
2024-02-29T14:37:04.780Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1690, in _request_raw
2024-02-29T14:37:04.780Z [container-agent]     response = self.opener.open(request, timeout=self.timeout)
2024-02-29T14:37:04.780Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 519, in open
2024-02-29T14:37:04.780Z [container-agent]     response = self._open(req, data)
2024-02-29T14:37:04.780Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 536, in _open
2024-02-29T14:37:04.780Z [container-agent]     result = self._call_chain(self.handle_open, protocol, protocol +
2024-02-29T14:37:04.780Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 496, in _call_chain
2024-02-29T14:37:04.780Z [container-agent]     result = func(*args)
2024-02-29T14:37:04.780Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 311, in http_open
2024-02-29T14:37:04.780Z [container-agent]     return self.do_open(_UnixSocketConnection, req,  # type:ignore
2024-02-29T14:37:04.780Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 1351, in do_open
2024-02-29T14:37:04.780Z [container-agent]     raise URLError(err)
2024-02-29T14:37:04.780Z [container-agent] urllib.error.URLError: <urlopen error [Errno 111] Connection refused>
2024-02-29T14:37:04.780Z [container-agent] 
2024-02-29T14:37:04.780Z [container-agent] The above exception was the direct cause of the following exception:
2024-02-29T14:37:04.780Z [container-agent] 
2024-02-29T14:37:04.780Z [container-agent] Traceback (most recent call last):
2024-02-29T14:37:04.780Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/./src/charm.py", line 786, in <module>
2024-02-29T14:37:04.780Z [container-agent]     main(MySQLOperatorCharm)
2024-02-29T14:37:04.780Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 436, in main
2024-02-29T14:37:04.780Z [container-agent]     _emit_charm_event(charm, dispatcher.event_name)
2024-02-29T14:37:04.780Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 144, in _emit_charm_event
2024-02-29T14:37:04.780Z [container-agent]     event_to_emit.emit(*args, **kwargs)
2024-02-29T14:37:04.780Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 351, in emit
2024-02-29T14:37:04.780Z [container-agent]     framework._emit(event)
2024-02-29T14:37:04.780Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 853, in _emit
2024-02-29T14:37:04.780Z [container-agent]     self._reemit(event_path)
2024-02-29T14:37:04.780Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 942, in _reemit
2024-02-29T14:37:04.780Z [container-agent]     custom_handler(event)
2024-02-29T14:37:04.780Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/rotate_mysql_logs.py", line 55, in _rotate_mysql_logs
2024-02-29T14:37:04.780Z [container-agent]     self.charm._mysql.flush_mysql_logs(MySQLTextLogs.ERROR)
2024-02-29T14:37:04.780Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-02-29T14:37:04.780Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-02-29T14:37:04.780Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 665, in _run_mysqlsh_script
2024-02-29T14:37:04.780Z [container-agent]     process = self.container.exec(cmd, timeout=timeout)
2024-02-29T14:37:04.780Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/model.py", line 2688, in exec
2024-02-29T14:37:04.780Z [container-agent]     return self._pebble.exec(
2024-02-29T14:37:04.780Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 2514, in exec
2024-02-29T14:37:04.780Z [container-agent]     change = self.wait_change(ChangeID(change_id))
2024-02-29T14:37:04.780Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1893, in wait_change
2024-02-29T14:37:04.780Z [container-agent]     return self._wait_change_using_wait(change_id, timeout)
2024-02-29T14:37:04.780Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1914, in _wait_change_using_wait
2024-02-29T14:37:04.780Z [container-agent]     return self._wait_change(change_id, this_timeout)
2024-02-29T14:37:04.780Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1928, in _wait_change
2024-02-29T14:37:04.780Z [container-agent]     resp = self._request('GET', f'/v1/changes/{change_id}/wait', query)
2024-02-29T14:37:04.780Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1655, in _request
2024-02-29T14:37:04.780Z [container-agent]     response = self._request_raw(method, path, query, headers, data)
2024-02-29T14:37:04.780Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1707, in _request_raw
2024-02-29T14:37:04.780Z [container-agent]     raise ConnectionError(e.reason) from e
2024-02-29T14:37:04.780Z [container-agent] ops.pebble.ConnectionError: [Errno 111] Connection refused
2024-02-29T14:37:04.938Z [container-agent] 2024-02-29 14:37:04 INFO juju.util.exec exec.go:209 run result: exit status 1
2024-02-29T14:37:10.142Z [container-agent] 2024-02-29 14:37:10 INFO juju-log Setting up the logrotate configurations
2024-02-29T14:37:10.245Z [container-agent] 2024-02-29 14:37:10 INFO juju-log Adding pebble layer
2024-02-29T14:37:17.013Z [container-agent] 2024-02-29 14:37:17 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-02-29T14:37:17.038Z [container-agent] 2024-02-29 14:37:17 INFO juju-log Attempting reboot from complete outage.
2024-02-29T14:37:20.054Z [container-agent] 2024-02-29 14:37:20 INFO juju.worker.uniter.operation runhook.go:186 ran "mysql-pebble-ready" hook (via hook dispatching script: dispatch)
2024-02-29T14:39:11.195Z [container-agent] 2024-02-29 14:39:11 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T14:39:13.053Z [container-agent] 2024-02-29 14:39:13 WARNING juju-log No relation: certificates
2024-02-29T14:39:13.398Z [container-agent] 2024-02-29 14:39:13 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T14:44:55.042Z [container-agent] 2024-02-29 14:44:55 INFO juju-log Starting the log rotate manager
2024-02-29T14:44:55.055Z [container-agent] 2024-02-29 14:44:55 INFO juju-log Started log rotate manager process with PID 292460
2024-02-29T14:44:55.359Z [container-agent] 2024-02-29 14:44:55 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T14:44:57.251Z [container-agent] 2024-02-29 14:44:57 WARNING juju-log No relation: certificates
2024-02-29T14:44:57.621Z [container-agent] 2024-02-29 14:44:57 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T14:50:45.475Z [container-agent] 2024-02-29 14:50:45 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T14:50:47.949Z [container-agent] 2024-02-29 14:50:47 WARNING juju-log No relation: certificates
2024-02-29T14:50:48.327Z [container-agent] 2024-02-29 14:50:48 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T14:56:15.369Z [container-agent] 2024-02-29 14:56:15 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T14:56:18.726Z [container-agent] 2024-02-29 14:56:18 WARNING juju-log No relation: certificates
2024-02-29T14:56:19.074Z [container-agent] 2024-02-29 14:56:19 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T15:01:20.184Z [container-agent] 2024-02-29 15:01:20 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T15:01:23.742Z [container-agent] 2024-02-29 15:01:23 WARNING juju-log No relation: certificates
2024-02-29T15:01:24.117Z [container-agent] 2024-02-29 15:01:24 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T15:06:15.605Z [container-agent] 2024-02-29 15:06:15 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T15:06:19.521Z [container-agent] 2024-02-29 15:06:19 WARNING juju-log No relation: certificates
2024-02-29T15:06:19.906Z [container-agent] 2024-02-29 15:06:19 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T15:11:31.164Z [container-agent] 2024-02-29 15:11:31 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T15:11:35.639Z [container-agent] 2024-02-29 15:11:35 WARNING juju-log No relation: certificates
2024-02-29T15:11:35.982Z [container-agent] 2024-02-29 15:11:35 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T15:16:41.100Z [container-agent] 2024-02-29 15:16:41 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T15:16:46.394Z [container-agent] 2024-02-29 15:16:46 WARNING juju-log No relation: certificates
2024-02-29T15:16:46.751Z [container-agent] 2024-02-29 15:16:46 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T15:21:35.857Z [container-agent] 2024-02-29 15:21:35 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T15:21:41.034Z [container-agent] 2024-02-29 15:21:41 WARNING juju-log No relation: certificates
2024-02-29T15:21:41.377Z [container-agent] 2024-02-29 15:21:41 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T15:26:58.688Z [container-agent] 2024-02-29 15:26:58 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T15:27:04.558Z [container-agent] 2024-02-29 15:27:04 WARNING juju-log No relation: certificates
2024-02-29T15:27:04.898Z [container-agent] 2024-02-29 15:27:04 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T15:32:26.818Z [container-agent] 2024-02-29 15:32:26 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T15:32:33.275Z [container-agent] 2024-02-29 15:32:33 WARNING juju-log No relation: certificates
2024-02-29T15:32:33.631Z [container-agent] 2024-02-29 15:32:33 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T15:37:16.051Z [container-agent] 2024-02-29 15:37:16 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T15:37:22.431Z [container-agent] 2024-02-29 15:37:22 WARNING juju-log No relation: certificates
2024-02-29T15:37:22.783Z [container-agent] 2024-02-29 15:37:22 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T15:41:36.653Z [container-agent] 2024-02-29 15:41:36 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T15:41:44.095Z [container-agent] 2024-02-29 15:41:44 WARNING juju-log No relation: certificates
2024-02-29T15:41:44.441Z [container-agent] 2024-02-29 15:41:44 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T15:47:17.640Z [container-agent] 2024-02-29 15:47:17 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T15:47:25.002Z [container-agent] 2024-02-29 15:47:25 WARNING juju-log No relation: certificates
2024-02-29T15:47:25.381Z [container-agent] 2024-02-29 15:47:25 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T15:50:07.749Z [container-agent] 2024-02-29 15:50:07 ERROR juju-log Failed to flush ERROR LOGS logs.
2024-02-29T15:50:07.749Z [container-agent] Traceback (most recent call last):
2024-02-29T15:50:07.749Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 666, in _run_mysqlsh_script
2024-02-29T15:50:07.749Z [container-agent]     stdout, _ = process.wait_output()
2024-02-29T15:50:07.749Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1441, in wait_output
2024-02-29T15:50:07.749Z [container-agent]     raise ExecError[AnyStr](self._command, exit_code, out_value, err_value)
2024-02-29T15:50:07.749Z [container-agent] ops.pebble.ExecError: non-zero exit code 1 executing ['/usr/bin/mysqlsh', '--no-wizard', '--python', '--verbose=1', '-f', '/tmp/script.py', ';', 'rm', '/tmp/script.py'], stdout='', stderr='Cannot set LC_ALL to locale en_US.UTF-8: No such file or directory\nverbose: 2024-02-29T15:50:06Z: Loading startup files...\nverbose: 2024-02-29T15:50:06Z: Loading plugins...\nverbose: 2024-02-29T15:50:06Z: Connecting to MySQL at: serverconfig@horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local\nTraceback (most recent call last):\n  File "<string>", line 1, in <module>\nmysqlsh.DBError: MySQL Error (2003): Shell.connect: Can\'t connect to MySQL server on \'horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local:3306\' (111)\n'
2024-02-29T15:50:07.749Z [container-agent] 
2024-02-29T15:50:07.749Z [container-agent] During handling of the above exception, another exception occurred:
2024-02-29T15:50:07.749Z [container-agent] 
2024-02-29T15:50:07.749Z [container-agent] Traceback (most recent call last):
2024-02-29T15:50:07.749Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-02-29T15:50:07.749Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-02-29T15:50:07.749Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 669, in _run_mysqlsh_script
2024-02-29T15:50:07.749Z [container-agent]     raise MySQLClientError(e.stderr)
2024-02-29T15:50:07.749Z [container-agent] charms.mysql.v0.mysql.MySQLClientError: Cannot set LC_ALL to locale en_US.UTF-8: No such file or directory
2024-02-29T15:50:07.749Z [container-agent] verbose: 2024-02-29T15:50:06Z: Loading startup files...
2024-02-29T15:50:07.749Z [container-agent] verbose: 2024-02-29T15:50:06Z: Loading plugins...
2024-02-29T15:50:07.749Z [container-agent] verbose: 2024-02-29T15:50:06Z: Connecting to MySQL at: serverconfig@horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local
2024-02-29T15:50:07.749Z [container-agent] Traceback (most recent call last):
2024-02-29T15:50:07.749Z [container-agent]   File "<string>", line 1, in <module>
2024-02-29T15:50:07.749Z [container-agent] mysqlsh.DBError: MySQL Error (2003): Shell.connect: Can't connect to MySQL server on 'horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local:3306' (111)
2024-02-29T15:50:07.749Z [container-agent] 
2024-02-29T15:52:34.687Z [container-agent] 2024-02-29 15:52:34 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-02-29T15:52:34.713Z [container-agent] 2024-02-29 15:52:34 INFO juju-log Attempting reboot from complete outage.
2024-02-29T15:52:41.904Z [container-agent] 2024-02-29 15:52:41 WARNING juju-log No relation: certificates
2024-02-29T15:52:42.260Z [container-agent] 2024-02-29 15:52:42 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T15:55:04.790Z [container-agent] 2024-02-29 15:55:04 ERROR juju-log Uncaught exception while in charm code:
2024-02-29T15:55:04.790Z [container-agent] Traceback (most recent call last):
2024-02-29T15:55:04.790Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 2507, in exec
2024-02-29T15:55:04.790Z [container-agent]     control_ws = self._connect_websocket(task_id, 'control')
2024-02-29T15:55:04.790Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 2589, in _connect_websocket
2024-02-29T15:55:04.790Z [container-agent]     ws.connect(url, socket=sock)
2024-02-29T15:55:04.790Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_core.py", line 253, in connect
2024-02-29T15:55:04.790Z [container-agent]     self.handshake_response = handshake(self.sock, url, *addrs, **options)
2024-02-29T15:55:04.790Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_handshake.py", line 57, in handshake
2024-02-29T15:55:04.790Z [container-agent]     status, resp = _get_resp_headers(sock)
2024-02-29T15:55:04.790Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_handshake.py", line 143, in _get_resp_headers
2024-02-29T15:55:04.790Z [container-agent]     status, resp_headers, status_message = read_headers(sock)
2024-02-29T15:55:04.790Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_http.py", line 312, in read_headers
2024-02-29T15:55:04.790Z [container-agent]     line = recv_line(sock)
2024-02-29T15:55:04.790Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_socket.py", line 131, in recv_line
2024-02-29T15:55:04.790Z [container-agent]     c = recv(sock, 1)
2024-02-29T15:55:04.790Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_socket.py", line 122, in recv
2024-02-29T15:55:04.790Z [container-agent]     raise WebSocketConnectionClosedException(
2024-02-29T15:55:04.790Z [container-agent] websocket._exceptions.WebSocketConnectionClosedException: Connection to remote host was lost.
2024-02-29T15:55:04.790Z [container-agent] 
2024-02-29T15:55:04.790Z [container-agent] During handling of the above exception, another exception occurred:
2024-02-29T15:55:04.790Z [container-agent] 
2024-02-29T15:55:04.790Z [container-agent] Traceback (most recent call last):
2024-02-29T15:55:04.790Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 1348, in do_open
2024-02-29T15:55:04.790Z [container-agent]     h.request(req.get_method(), req.selector, req.data, headers,
2024-02-29T15:55:04.790Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1283, in request
2024-02-29T15:55:04.790Z [container-agent]     self._send_request(method, url, body, headers, encode_chunked)
2024-02-29T15:55:04.790Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1329, in _send_request
2024-02-29T15:55:04.790Z [container-agent]     self.endheaders(body, encode_chunked=encode_chunked)
2024-02-29T15:55:04.790Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1278, in endheaders
2024-02-29T15:55:04.790Z [container-agent]     self._send_output(message_body, encode_chunked=encode_chunked)
2024-02-29T15:55:04.790Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1038, in _send_output
2024-02-29T15:55:04.790Z [container-agent]     self.send(msg)
2024-02-29T15:55:04.790Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 976, in send
2024-02-29T15:55:04.790Z [container-agent]     self.connect()
2024-02-29T15:55:04.790Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 297, in connect
2024-02-29T15:55:04.790Z [container-agent]     self.sock.connect(self.socket_path)
2024-02-29T15:55:04.790Z [container-agent] ConnectionRefusedError: [Errno 111] Connection refused
2024-02-29T15:55:04.790Z [container-agent] 
2024-02-29T15:55:04.790Z [container-agent] During handling of the above exception, another exception occurred:
2024-02-29T15:55:04.790Z [container-agent] 
2024-02-29T15:55:04.790Z [container-agent] Traceback (most recent call last):
2024-02-29T15:55:04.790Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1690, in _request_raw
2024-02-29T15:55:04.790Z [container-agent]     response = self.opener.open(request, timeout=self.timeout)
2024-02-29T15:55:04.790Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 519, in open
2024-02-29T15:55:04.790Z [container-agent]     response = self._open(req, data)
2024-02-29T15:55:04.790Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 536, in _open
2024-02-29T15:55:04.790Z [container-agent]     result = self._call_chain(self.handle_open, protocol, protocol +
2024-02-29T15:55:04.790Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 496, in _call_chain
2024-02-29T15:55:04.790Z [container-agent]     result = func(*args)
2024-02-29T15:55:04.790Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 311, in http_open
2024-02-29T15:55:04.790Z [container-agent]     return self.do_open(_UnixSocketConnection, req,  # type:ignore
2024-02-29T15:55:04.790Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 1351, in do_open
2024-02-29T15:55:04.790Z [container-agent]     raise URLError(err)
2024-02-29T15:55:04.790Z [container-agent] urllib.error.URLError: <urlopen error [Errno 111] Connection refused>
2024-02-29T15:55:04.790Z [container-agent] 
2024-02-29T15:55:04.790Z [container-agent] The above exception was the direct cause of the following exception:
2024-02-29T15:55:04.790Z [container-agent] 
2024-02-29T15:55:04.790Z [container-agent] Traceback (most recent call last):
2024-02-29T15:55:04.790Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/./src/charm.py", line 786, in <module>
2024-02-29T15:55:04.790Z [container-agent]     main(MySQLOperatorCharm)
2024-02-29T15:55:04.790Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 436, in main
2024-02-29T15:55:04.790Z [container-agent]     _emit_charm_event(charm, dispatcher.event_name)
2024-02-29T15:55:04.790Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 144, in _emit_charm_event
2024-02-29T15:55:04.790Z [container-agent]     event_to_emit.emit(*args, **kwargs)
2024-02-29T15:55:04.790Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 351, in emit
2024-02-29T15:55:04.790Z [container-agent]     framework._emit(event)
2024-02-29T15:55:04.790Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 853, in _emit
2024-02-29T15:55:04.790Z [container-agent]     self._reemit(event_path)
2024-02-29T15:55:04.790Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 942, in _reemit
2024-02-29T15:55:04.790Z [container-agent]     custom_handler(event)
2024-02-29T15:55:04.790Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/rotate_mysql_logs.py", line 55, in _rotate_mysql_logs
2024-02-29T15:55:04.790Z [container-agent]     self.charm._mysql.flush_mysql_logs(MySQLTextLogs.ERROR)
2024-02-29T15:55:04.790Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-02-29T15:55:04.790Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-02-29T15:55:04.790Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 665, in _run_mysqlsh_script
2024-02-29T15:55:04.790Z [container-agent]     process = self.container.exec(cmd, timeout=timeout)
2024-02-29T15:55:04.790Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/model.py", line 2688, in exec
2024-02-29T15:55:04.790Z [container-agent]     return self._pebble.exec(
2024-02-29T15:55:04.790Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 2514, in exec
2024-02-29T15:55:04.790Z [container-agent]     change = self.wait_change(ChangeID(change_id))
2024-02-29T15:55:04.790Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1893, in wait_change
2024-02-29T15:55:04.790Z [container-agent]     return self._wait_change_using_wait(change_id, timeout)
2024-02-29T15:55:04.790Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1914, in _wait_change_using_wait
2024-02-29T15:55:04.790Z [container-agent]     return self._wait_change(change_id, this_timeout)
2024-02-29T15:55:04.790Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1928, in _wait_change
2024-02-29T15:55:04.790Z [container-agent]     resp = self._request('GET', f'/v1/changes/{change_id}/wait', query)
2024-02-29T15:55:04.790Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1655, in _request
2024-02-29T15:55:04.790Z [container-agent]     response = self._request_raw(method, path, query, headers, data)
2024-02-29T15:55:04.790Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1707, in _request_raw
2024-02-29T15:55:04.790Z [container-agent]     raise ConnectionError(e.reason) from e
2024-02-29T15:55:04.790Z [container-agent] ops.pebble.ConnectionError: [Errno 111] Connection refused
2024-02-29T15:55:04.935Z [container-agent] 2024-02-29 15:55:04 INFO juju.util.exec exec.go:209 run result: exit status 1
2024-02-29T15:55:07.754Z [container-agent] 2024-02-29 15:55:07 INFO juju-log Setting up the logrotate configurations
2024-02-29T15:55:07.817Z [container-agent] 2024-02-29 15:55:07 INFO juju-log Adding pebble layer
2024-02-29T15:55:14.668Z [container-agent] 2024-02-29 15:55:14 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-02-29T15:55:14.695Z [container-agent] 2024-02-29 15:55:14 INFO juju-log Attempting reboot from complete outage.
2024-02-29T15:55:17.725Z [container-agent] 2024-02-29 15:55:17 INFO juju.worker.uniter.operation runhook.go:186 ran "mysql-pebble-ready" hook (via hook dispatching script: dispatch)
2024-02-29T15:57:08.922Z [container-agent] 2024-02-29 15:57:08 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T15:57:10.830Z [container-agent] 2024-02-29 15:57:10 WARNING juju-log No relation: certificates
2024-02-29T15:57:11.243Z [container-agent] 2024-02-29 15:57:11 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T16:01:50.380Z [container-agent] 2024-02-29 16:01:50 INFO juju-log Starting the log rotate manager
2024-02-29T16:01:50.395Z [container-agent] 2024-02-29 16:01:50 INFO juju-log Started log rotate manager process with PID 299627
2024-02-29T16:01:50.679Z [container-agent] 2024-02-29 16:01:50 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T16:01:52.618Z [container-agent] 2024-02-29 16:01:52 WARNING juju-log No relation: certificates
2024-02-29T16:01:52.968Z [container-agent] 2024-02-29 16:01:52 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T16:06:06.617Z [container-agent] 2024-02-29 16:06:06 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T16:06:09.323Z [container-agent] 2024-02-29 16:06:09 WARNING juju-log No relation: certificates
2024-02-29T16:06:09.689Z [container-agent] 2024-02-29 16:06:09 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T16:12:06.700Z [container-agent] 2024-02-29 16:12:06 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T16:12:09.582Z [container-agent] 2024-02-29 16:12:09 WARNING juju-log No relation: certificates
2024-02-29T16:12:09.945Z [container-agent] 2024-02-29 16:12:09 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T16:17:30.050Z [container-agent] 2024-02-29 16:17:30 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T16:17:33.716Z [container-agent] 2024-02-29 16:17:33 WARNING juju-log No relation: certificates
2024-02-29T16:17:34.065Z [container-agent] 2024-02-29 16:17:34 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T16:22:25.053Z [container-agent] 2024-02-29 16:22:25 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T16:22:29.163Z [container-agent] 2024-02-29 16:22:29 WARNING juju-log No relation: certificates
2024-02-29T16:22:29.518Z [container-agent] 2024-02-29 16:22:29 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T16:27:25.200Z [container-agent] 2024-02-29 16:27:25 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T16:27:29.644Z [container-agent] 2024-02-29 16:27:29 WARNING juju-log No relation: certificates
2024-02-29T16:27:30.046Z [container-agent] 2024-02-29 16:27:30 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T16:33:19.226Z [container-agent] 2024-02-29 16:33:19 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T16:33:24.331Z [container-agent] 2024-02-29 16:33:24 WARNING juju-log No relation: certificates
2024-02-29T16:33:24.692Z [container-agent] 2024-02-29 16:33:24 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T16:38:14.434Z [container-agent] 2024-02-29 16:38:14 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T16:38:19.927Z [container-agent] 2024-02-29 16:38:19 WARNING juju-log No relation: certificates
2024-02-29T16:38:20.272Z [container-agent] 2024-02-29 16:38:20 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T16:42:37.507Z [container-agent] 2024-02-29 16:42:37 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T16:42:43.182Z [container-agent] 2024-02-29 16:42:43 WARNING juju-log No relation: certificates
2024-02-29T16:42:43.526Z [container-agent] 2024-02-29 16:42:43 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T16:46:50.976Z [container-agent] 2024-02-29 16:46:50 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T16:46:56.935Z [container-agent] 2024-02-29 16:46:56 WARNING juju-log No relation: certificates
2024-02-29T16:46:57.281Z [container-agent] 2024-02-29 16:46:57 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T16:52:42.760Z [container-agent] 2024-02-29 16:52:42 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T16:52:49.454Z [container-agent] 2024-02-29 16:52:49 WARNING juju-log No relation: certificates
2024-02-29T16:52:49.801Z [container-agent] 2024-02-29 16:52:49 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T16:56:54.860Z [container-agent] 2024-02-29 16:56:54 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T16:57:01.518Z [container-agent] 2024-02-29 16:57:01 WARNING juju-log No relation: certificates
2024-02-29T16:57:01.876Z [container-agent] 2024-02-29 16:57:01 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T17:01:02.470Z [container-agent] 2024-02-29 17:01:02 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T17:01:09.305Z [container-agent] 2024-02-29 17:01:09 WARNING juju-log No relation: certificates
2024-02-29T17:01:09.631Z [container-agent] 2024-02-29 17:01:09 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T17:06:45.074Z [container-agent] 2024-02-29 17:06:45 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T17:06:53.136Z [container-agent] 2024-02-29 17:06:53 WARNING juju-log No relation: certificates
2024-02-29T17:06:53.471Z [container-agent] 2024-02-29 17:06:53 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T17:11:56.638Z [container-agent] 2024-02-29 17:11:56 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T17:12:04.886Z [container-agent] 2024-02-29 17:12:04 WARNING juju-log No relation: certificates
2024-02-29T17:12:05.284Z [container-agent] 2024-02-29 17:12:05 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T17:16:23.089Z [container-agent] 2024-02-29 17:16:23 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T17:16:32.120Z [container-agent] 2024-02-29 17:16:32 WARNING juju-log No relation: certificates
2024-02-29T17:16:32.477Z [container-agent] 2024-02-29 17:16:32 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T17:17:07.853Z [container-agent] 2024-02-29 17:17:07 ERROR juju-log Failed to flush ERROR LOGS logs.
2024-02-29T17:17:07.853Z [container-agent] Traceback (most recent call last):
2024-02-29T17:17:07.853Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 666, in _run_mysqlsh_script
2024-02-29T17:17:07.853Z [container-agent]     stdout, _ = process.wait_output()
2024-02-29T17:17:07.853Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1441, in wait_output
2024-02-29T17:17:07.853Z [container-agent]     raise ExecError[AnyStr](self._command, exit_code, out_value, err_value)
2024-02-29T17:17:07.853Z [container-agent] ops.pebble.ExecError: non-zero exit code 1 executing ['/usr/bin/mysqlsh', '--no-wizard', '--python', '--verbose=1', '-f', '/tmp/script.py', ';', 'rm', '/tmp/script.py'], stdout='', stderr='Cannot set LC_ALL to locale en_US.UTF-8: No such file or directory\nverbose: 2024-02-29T17:17:06Z: Loading startup files...\nverbose: 2024-02-29T17:17:06Z: Loading plugins...\nverbose: 2024-02-29T17:17:06Z: Connecting to MySQL at: serverconfig@horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local\nTraceback (most recent call last):\n  File "<string>", line 1, in <module>\nmysqlsh.DBError: MySQL Error (2003): Shell.connect: Can\'t connect to MySQL server on \'horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local:3306\' (111)\n'
2024-02-29T17:17:07.853Z [container-agent] 
2024-02-29T17:17:07.853Z [container-agent] During handling of the above exception, another exception occurred:
2024-02-29T17:17:07.853Z [container-agent] 
2024-02-29T17:17:07.853Z [container-agent] Traceback (most recent call last):
2024-02-29T17:17:07.853Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-02-29T17:17:07.853Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-02-29T17:17:07.853Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 669, in _run_mysqlsh_script
2024-02-29T17:17:07.853Z [container-agent]     raise MySQLClientError(e.stderr)
2024-02-29T17:17:07.853Z [container-agent] charms.mysql.v0.mysql.MySQLClientError: Cannot set LC_ALL to locale en_US.UTF-8: No such file or directory
2024-02-29T17:17:07.853Z [container-agent] verbose: 2024-02-29T17:17:06Z: Loading startup files...
2024-02-29T17:17:07.853Z [container-agent] verbose: 2024-02-29T17:17:06Z: Loading plugins...
2024-02-29T17:17:07.853Z [container-agent] verbose: 2024-02-29T17:17:06Z: Connecting to MySQL at: serverconfig@horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local
2024-02-29T17:17:07.853Z [container-agent] Traceback (most recent call last):
2024-02-29T17:17:07.853Z [container-agent]   File "<string>", line 1, in <module>
2024-02-29T17:17:07.853Z [container-agent] mysqlsh.DBError: MySQL Error (2003): Shell.connect: Can't connect to MySQL server on 'horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local:3306' (111)
2024-02-29T17:17:07.853Z [container-agent] 
2024-02-29T17:21:16.063Z [container-agent] 2024-02-29 17:21:16 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-02-29T17:21:16.093Z [container-agent] 2024-02-29 17:21:16 INFO juju-log Attempting reboot from complete outage.
2024-02-29T17:21:24.590Z [container-agent] 2024-02-29 17:21:24 WARNING juju-log No relation: certificates
2024-02-29T17:21:24.973Z [container-agent] 2024-02-29 17:21:24 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T17:22:04.780Z [container-agent] 2024-02-29 17:22:04 ERROR juju-log Uncaught exception while in charm code:
2024-02-29T17:22:04.780Z [container-agent] Traceback (most recent call last):
2024-02-29T17:22:04.780Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 2507, in exec
2024-02-29T17:22:04.780Z [container-agent]     control_ws = self._connect_websocket(task_id, 'control')
2024-02-29T17:22:04.780Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 2589, in _connect_websocket
2024-02-29T17:22:04.780Z [container-agent]     ws.connect(url, socket=sock)
2024-02-29T17:22:04.780Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_core.py", line 253, in connect
2024-02-29T17:22:04.780Z [container-agent]     self.handshake_response = handshake(self.sock, url, *addrs, **options)
2024-02-29T17:22:04.780Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_handshake.py", line 57, in handshake
2024-02-29T17:22:04.780Z [container-agent]     status, resp = _get_resp_headers(sock)
2024-02-29T17:22:04.780Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_handshake.py", line 143, in _get_resp_headers
2024-02-29T17:22:04.780Z [container-agent]     status, resp_headers, status_message = read_headers(sock)
2024-02-29T17:22:04.780Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_http.py", line 312, in read_headers
2024-02-29T17:22:04.780Z [container-agent]     line = recv_line(sock)
2024-02-29T17:22:04.780Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_socket.py", line 131, in recv_line
2024-02-29T17:22:04.780Z [container-agent]     c = recv(sock, 1)
2024-02-29T17:22:04.780Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_socket.py", line 122, in recv
2024-02-29T17:22:04.780Z [container-agent]     raise WebSocketConnectionClosedException(
2024-02-29T17:22:04.780Z [container-agent] websocket._exceptions.WebSocketConnectionClosedException: Connection to remote host was lost.
2024-02-29T17:22:04.780Z [container-agent] 
2024-02-29T17:22:04.780Z [container-agent] During handling of the above exception, another exception occurred:
2024-02-29T17:22:04.780Z [container-agent] 
2024-02-29T17:22:04.780Z [container-agent] Traceback (most recent call last):
2024-02-29T17:22:04.780Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 1348, in do_open
2024-02-29T17:22:04.780Z [container-agent]     h.request(req.get_method(), req.selector, req.data, headers,
2024-02-29T17:22:04.780Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1283, in request
2024-02-29T17:22:04.780Z [container-agent]     self._send_request(method, url, body, headers, encode_chunked)
2024-02-29T17:22:04.780Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1329, in _send_request
2024-02-29T17:22:04.780Z [container-agent]     self.endheaders(body, encode_chunked=encode_chunked)
2024-02-29T17:22:04.780Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1278, in endheaders
2024-02-29T17:22:04.780Z [container-agent]     self._send_output(message_body, encode_chunked=encode_chunked)
2024-02-29T17:22:04.780Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1038, in _send_output
2024-02-29T17:22:04.780Z [container-agent]     self.send(msg)
2024-02-29T17:22:04.780Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 976, in send
2024-02-29T17:22:04.780Z [container-agent]     self.connect()
2024-02-29T17:22:04.780Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 297, in connect
2024-02-29T17:22:04.780Z [container-agent]     self.sock.connect(self.socket_path)
2024-02-29T17:22:04.780Z [container-agent] ConnectionRefusedError: [Errno 111] Connection refused
2024-02-29T17:22:04.780Z [container-agent] 
2024-02-29T17:22:04.780Z [container-agent] During handling of the above exception, another exception occurred:
2024-02-29T17:22:04.780Z [container-agent] 
2024-02-29T17:22:04.780Z [container-agent] Traceback (most recent call last):
2024-02-29T17:22:04.780Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1690, in _request_raw
2024-02-29T17:22:04.780Z [container-agent]     response = self.opener.open(request, timeout=self.timeout)
2024-02-29T17:22:04.780Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 519, in open
2024-02-29T17:22:04.780Z [container-agent]     response = self._open(req, data)
2024-02-29T17:22:04.780Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 536, in _open
2024-02-29T17:22:04.780Z [container-agent]     result = self._call_chain(self.handle_open, protocol, protocol +
2024-02-29T17:22:04.780Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 496, in _call_chain
2024-02-29T17:22:04.780Z [container-agent]     result = func(*args)
2024-02-29T17:22:04.780Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 311, in http_open
2024-02-29T17:22:04.780Z [container-agent]     return self.do_open(_UnixSocketConnection, req,  # type:ignore
2024-02-29T17:22:04.780Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 1351, in do_open
2024-02-29T17:22:04.780Z [container-agent]     raise URLError(err)
2024-02-29T17:22:04.780Z [container-agent] urllib.error.URLError: <urlopen error [Errno 111] Connection refused>
2024-02-29T17:22:04.780Z [container-agent] 
2024-02-29T17:22:04.780Z [container-agent] The above exception was the direct cause of the following exception:
2024-02-29T17:22:04.780Z [container-agent] 
2024-02-29T17:22:04.780Z [container-agent] Traceback (most recent call last):
2024-02-29T17:22:04.780Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/./src/charm.py", line 786, in <module>
2024-02-29T17:22:04.780Z [container-agent]     main(MySQLOperatorCharm)
2024-02-29T17:22:04.780Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 436, in main
2024-02-29T17:22:04.780Z [container-agent]     _emit_charm_event(charm, dispatcher.event_name)
2024-02-29T17:22:04.780Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 144, in _emit_charm_event
2024-02-29T17:22:04.780Z [container-agent]     event_to_emit.emit(*args, **kwargs)
2024-02-29T17:22:04.780Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 351, in emit
2024-02-29T17:22:04.780Z [container-agent]     framework._emit(event)
2024-02-29T17:22:04.780Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 853, in _emit
2024-02-29T17:22:04.780Z [container-agent]     self._reemit(event_path)
2024-02-29T17:22:04.780Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 942, in _reemit
2024-02-29T17:22:04.780Z [container-agent]     custom_handler(event)
2024-02-29T17:22:04.780Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/rotate_mysql_logs.py", line 55, in _rotate_mysql_logs
2024-02-29T17:22:04.780Z [container-agent]     self.charm._mysql.flush_mysql_logs(MySQLTextLogs.ERROR)
2024-02-29T17:22:04.780Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-02-29T17:22:04.781Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-02-29T17:22:04.781Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 665, in _run_mysqlsh_script
2024-02-29T17:22:04.781Z [container-agent]     process = self.container.exec(cmd, timeout=timeout)
2024-02-29T17:22:04.781Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/model.py", line 2688, in exec
2024-02-29T17:22:04.781Z [container-agent]     return self._pebble.exec(
2024-02-29T17:22:04.781Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 2514, in exec
2024-02-29T17:22:04.781Z [container-agent]     change = self.wait_change(ChangeID(change_id))
2024-02-29T17:22:04.781Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1893, in wait_change
2024-02-29T17:22:04.781Z [container-agent]     return self._wait_change_using_wait(change_id, timeout)
2024-02-29T17:22:04.781Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1914, in _wait_change_using_wait
2024-02-29T17:22:04.781Z [container-agent]     return self._wait_change(change_id, this_timeout)
2024-02-29T17:22:04.781Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1928, in _wait_change
2024-02-29T17:22:04.781Z [container-agent]     resp = self._request('GET', f'/v1/changes/{change_id}/wait', query)
2024-02-29T17:22:04.781Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1655, in _request
2024-02-29T17:22:04.781Z [container-agent]     response = self._request_raw(method, path, query, headers, data)
2024-02-29T17:22:04.781Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1707, in _request_raw
2024-02-29T17:22:04.781Z [container-agent]     raise ConnectionError(e.reason) from e
2024-02-29T17:22:04.781Z [container-agent] ops.pebble.ConnectionError: [Errno 111] Connection refused
2024-02-29T17:22:04.935Z [container-agent] 2024-02-29 17:22:04 INFO juju.util.exec exec.go:209 run result: exit status 1
2024-02-29T17:22:06.722Z [container-agent] 2024-02-29 17:22:06 INFO juju-log Setting up the logrotate configurations
2024-02-29T17:22:06.780Z [container-agent] 2024-02-29 17:22:06 INFO juju-log Adding pebble layer
2024-02-29T17:22:13.504Z [container-agent] 2024-02-29 17:22:13 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-02-29T17:22:13.529Z [container-agent] 2024-02-29 17:22:13 INFO juju-log Attempting reboot from complete outage.
2024-02-29T17:22:16.587Z [container-agent] 2024-02-29 17:22:16 INFO juju.worker.uniter.operation runhook.go:186 ran "mysql-pebble-ready" hook (via hook dispatching script: dispatch)
2024-02-29T17:25:31.505Z [container-agent] 2024-02-29 17:25:31 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T17:25:33.472Z [container-agent] 2024-02-29 17:25:33 WARNING juju-log No relation: certificates
2024-02-29T17:25:33.846Z [container-agent] 2024-02-29 17:25:33 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T17:30:10.251Z [container-agent] 2024-02-29 17:30:10 INFO juju-log Starting the log rotate manager
2024-02-29T17:30:10.263Z [container-agent] 2024-02-29 17:30:10 INFO juju-log Started log rotate manager process with PID 307898
2024-02-29T17:30:10.572Z [container-agent] 2024-02-29 17:30:10 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T17:30:12.500Z [container-agent] 2024-02-29 17:30:12 WARNING juju-log No relation: certificates
2024-02-29T17:30:12.840Z [container-agent] 2024-02-29 17:30:12 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T17:34:37.088Z [container-agent] 2024-02-29 17:34:37 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T17:34:39.351Z [container-agent] 2024-02-29 17:34:39 WARNING juju-log No relation: certificates
2024-02-29T17:34:39.747Z [container-agent] 2024-02-29 17:34:39 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T17:39:00.284Z [container-agent] 2024-02-29 17:39:00 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T17:39:03.224Z [container-agent] 2024-02-29 17:39:03 WARNING juju-log No relation: certificates
2024-02-29T17:39:03.552Z [container-agent] 2024-02-29 17:39:03 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T17:44:00.452Z [container-agent] 2024-02-29 17:44:00 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T17:44:03.566Z [container-agent] 2024-02-29 17:44:03 WARNING juju-log No relation: certificates
2024-02-29T17:44:03.925Z [container-agent] 2024-02-29 17:44:03 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T17:49:26.216Z [container-agent] 2024-02-29 17:49:26 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T17:49:30.019Z [container-agent] 2024-02-29 17:49:30 WARNING juju-log No relation: certificates
2024-02-29T17:49:30.378Z [container-agent] 2024-02-29 17:49:30 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T17:53:55.259Z [container-agent] 2024-02-29 17:53:55 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T17:53:59.543Z [container-agent] 2024-02-29 17:53:59 WARNING juju-log No relation: certificates
2024-02-29T17:53:59.886Z [container-agent] 2024-02-29 17:53:59 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T17:59:10.418Z [container-agent] 2024-02-29 17:59:10 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T17:59:15.251Z [container-agent] 2024-02-29 17:59:15 WARNING juju-log No relation: certificates
2024-02-29T17:59:15.653Z [container-agent] 2024-02-29 17:59:15 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T18:04:16.314Z [container-agent] 2024-02-29 18:04:16 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T18:04:21.543Z [container-agent] 2024-02-29 18:04:21 WARNING juju-log No relation: certificates
2024-02-29T18:04:21.907Z [container-agent] 2024-02-29 18:04:21 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T18:08:20.855Z [container-agent] 2024-02-29 18:08:20 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T18:08:26.083Z [container-agent] 2024-02-29 18:08:26 WARNING juju-log No relation: certificates
2024-02-29T18:08:26.471Z [container-agent] 2024-02-29 18:08:26 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T18:13:13.538Z [container-agent] 2024-02-29 18:13:13 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T18:13:19.734Z [container-agent] 2024-02-29 18:13:19 WARNING juju-log No relation: certificates
2024-02-29T18:13:20.084Z [container-agent] 2024-02-29 18:13:20 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T18:19:02.008Z [container-agent] 2024-02-29 18:19:02 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T18:19:08.385Z [container-agent] 2024-02-29 18:19:08 WARNING juju-log No relation: certificates
2024-02-29T18:19:08.733Z [container-agent] 2024-02-29 18:19:08 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T18:23:31.964Z [container-agent] 2024-02-29 18:23:31 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T18:23:38.803Z [container-agent] 2024-02-29 18:23:38 WARNING juju-log No relation: certificates
2024-02-29T18:23:39.169Z [container-agent] 2024-02-29 18:23:39 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T18:28:19.971Z [container-agent] 2024-02-29 18:28:19 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T18:28:27.359Z [container-agent] 2024-02-29 18:28:27 WARNING juju-log No relation: certificates
2024-02-29T18:28:27.703Z [container-agent] 2024-02-29 18:28:27 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T18:33:25.274Z [container-agent] 2024-02-29 18:33:25 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T18:33:33.157Z [container-agent] 2024-02-29 18:33:33 WARNING juju-log No relation: certificates
2024-02-29T18:33:33.508Z [container-agent] 2024-02-29 18:33:33 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T18:37:24.626Z [container-agent] 2024-02-29 18:37:24 INFO juju-log Setting up the logrotate configurations
2024-02-29T18:37:24.699Z [container-agent] 2024-02-29 18:37:24 INFO juju-log Adding pebble layer
2024-02-29T18:37:31.434Z [container-agent] 2024-02-29 18:37:31 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-02-29T18:37:31.465Z [container-agent] 2024-02-29 18:37:31 INFO juju-log Attempting reboot from complete outage.
2024-02-29T18:37:34.487Z [container-agent] 2024-02-29 18:37:34 INFO juju.worker.uniter.operation runhook.go:186 ran "mysql-pebble-ready" hook (via hook dispatching script: dispatch)
2024-02-29T18:38:18.864Z [container-agent] 2024-02-29 18:38:18 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T18:38:20.692Z [container-agent] 2024-02-29 18:38:20 WARNING juju-log No relation: certificates
2024-02-29T18:38:21.021Z [container-agent] 2024-02-29 18:38:21 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T18:43:45.901Z [container-agent] 2024-02-29 18:43:45 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T18:43:48.532Z [container-agent] 2024-02-29 18:43:48 WARNING juju-log No relation: certificates
2024-02-29T18:43:48.880Z [container-agent] 2024-02-29 18:43:48 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T18:49:01.866Z [container-agent] 2024-02-29 18:49:01 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T18:49:04.800Z [container-agent] 2024-02-29 18:49:04 WARNING juju-log No relation: certificates
2024-02-29T18:49:05.151Z [container-agent] 2024-02-29 18:49:05 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T18:53:31.446Z [container-agent] 2024-02-29 18:53:31 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T18:53:34.971Z [container-agent] 2024-02-29 18:53:34 WARNING juju-log No relation: certificates
2024-02-29T18:53:35.316Z [container-agent] 2024-02-29 18:53:35 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T18:59:08.368Z [container-agent] 2024-02-29 18:59:08 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T18:59:12.031Z [container-agent] 2024-02-29 18:59:12 WARNING juju-log No relation: certificates
2024-02-29T18:59:12.403Z [container-agent] 2024-02-29 18:59:12 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T19:04:44.151Z [container-agent] 2024-02-29 19:04:44 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T19:04:48.786Z [container-agent] 2024-02-29 19:04:48 WARNING juju-log No relation: certificates
2024-02-29T19:04:49.151Z [container-agent] 2024-02-29 19:04:49 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T19:09:09.004Z [container-agent] 2024-02-29 19:09:09 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T19:09:13.395Z [container-agent] 2024-02-29 19:09:13 WARNING juju-log No relation: certificates
2024-02-29T19:09:13.736Z [container-agent] 2024-02-29 19:09:13 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T19:13:50.598Z [container-agent] 2024-02-29 19:13:50 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T19:13:55.226Z [container-agent] 2024-02-29 19:13:55 WARNING juju-log No relation: certificates
2024-02-29T19:13:55.561Z [container-agent] 2024-02-29 19:13:55 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T19:18:34.013Z [container-agent] 2024-02-29 19:18:34 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T19:18:39.145Z [container-agent] 2024-02-29 19:18:39 WARNING juju-log No relation: certificates
2024-02-29T19:18:39.482Z [container-agent] 2024-02-29 19:18:39 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T19:23:11.425Z [container-agent] 2024-02-29 19:23:11 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T19:23:17.576Z [container-agent] 2024-02-29 19:23:17 WARNING juju-log No relation: certificates
2024-02-29T19:23:17.920Z [container-agent] 2024-02-29 19:23:17 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T19:28:55.715Z [container-agent] 2024-02-29 19:28:55 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T19:29:02.056Z [container-agent] 2024-02-29 19:29:02 WARNING juju-log No relation: certificates
2024-02-29T19:29:02.421Z [container-agent] 2024-02-29 19:29:02 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T19:33:24.237Z [container-agent] 2024-02-29 19:33:24 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T19:33:30.944Z [container-agent] 2024-02-29 19:33:30 WARNING juju-log No relation: certificates
2024-02-29T19:33:31.292Z [container-agent] 2024-02-29 19:33:31 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T19:39:15.178Z [container-agent] 2024-02-29 19:39:15 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T19:39:22.604Z [container-agent] 2024-02-29 19:39:22 WARNING juju-log No relation: certificates
2024-02-29T19:39:22.965Z [container-agent] 2024-02-29 19:39:22 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T19:44:23.457Z [container-agent] 2024-02-29 19:44:23 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-02-29T19:44:23.486Z [container-agent] 2024-02-29 19:44:23 INFO juju-log Attempting reboot from complete outage.
2024-02-29T19:44:30.640Z [container-agent] 2024-02-29 19:44:30 WARNING juju-log No relation: certificates
2024-02-29T19:44:31.007Z [container-agent] 2024-02-29 19:44:31 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T19:48:04.815Z [container-agent] 2024-02-29 19:48:04 ERROR juju-log Uncaught exception while in charm code:
2024-02-29T19:48:04.815Z [container-agent] Traceback (most recent call last):
2024-02-29T19:48:04.815Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/./src/charm.py", line 786, in <module>
2024-02-29T19:48:04.815Z [container-agent]     main(MySQLOperatorCharm)
2024-02-29T19:48:04.815Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 436, in main
2024-02-29T19:48:04.815Z [container-agent]     _emit_charm_event(charm, dispatcher.event_name)
2024-02-29T19:48:04.815Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 144, in _emit_charm_event
2024-02-29T19:48:04.815Z [container-agent]     event_to_emit.emit(*args, **kwargs)
2024-02-29T19:48:04.815Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 351, in emit
2024-02-29T19:48:04.815Z [container-agent]     framework._emit(event)
2024-02-29T19:48:04.815Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 853, in _emit
2024-02-29T19:48:04.815Z [container-agent]     self._reemit(event_path)
2024-02-29T19:48:04.815Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 942, in _reemit
2024-02-29T19:48:04.815Z [container-agent]     custom_handler(event)
2024-02-29T19:48:04.815Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/rotate_mysql_logs.py", line 55, in _rotate_mysql_logs
2024-02-29T19:48:04.815Z [container-agent]     self.charm._mysql.flush_mysql_logs(MySQLTextLogs.ERROR)
2024-02-29T19:48:04.815Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-02-29T19:48:04.815Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-02-29T19:48:04.815Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 665, in _run_mysqlsh_script
2024-02-29T19:48:04.815Z [container-agent]     process = self.container.exec(cmd, timeout=timeout)
2024-02-29T19:48:04.815Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/model.py", line 2688, in exec
2024-02-29T19:48:04.815Z [container-agent]     return self._pebble.exec(
2024-02-29T19:48:04.815Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 2501, in exec
2024-02-29T19:48:04.815Z [container-agent]     resp = self._request('POST', '/v1/exec', body=body)
2024-02-29T19:48:04.815Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1655, in _request
2024-02-29T19:48:04.815Z [container-agent]     response = self._request_raw(method, path, query, headers, data)
2024-02-29T19:48:04.815Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1690, in _request_raw
2024-02-29T19:48:04.815Z [container-agent]     response = self.opener.open(request, timeout=self.timeout)
2024-02-29T19:48:04.815Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 519, in open
2024-02-29T19:48:04.815Z [container-agent]     response = self._open(req, data)
2024-02-29T19:48:04.815Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 536, in _open
2024-02-29T19:48:04.815Z [container-agent]     result = self._call_chain(self.handle_open, protocol, protocol +
2024-02-29T19:48:04.815Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 496, in _call_chain
2024-02-29T19:48:04.815Z [container-agent]     result = func(*args)
2024-02-29T19:48:04.815Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 311, in http_open
2024-02-29T19:48:04.815Z [container-agent]     return self.do_open(_UnixSocketConnection, req,  # type:ignore
2024-02-29T19:48:04.815Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 1352, in do_open
2024-02-29T19:48:04.815Z [container-agent]     r = h.getresponse()
2024-02-29T19:48:04.815Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1375, in getresponse
2024-02-29T19:48:04.815Z [container-agent]     response.begin()
2024-02-29T19:48:04.815Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 318, in begin
2024-02-29T19:48:04.815Z [container-agent]     version, status, reason = self._read_status()
2024-02-29T19:48:04.815Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 287, in _read_status
2024-02-29T19:48:04.815Z [container-agent]     raise RemoteDisconnected("Remote end closed connection without"
2024-02-29T19:48:04.815Z [container-agent] http.client.RemoteDisconnected: Remote end closed connection without response
2024-02-29T19:48:05.008Z [container-agent] 2024-02-29 19:48:05 INFO juju.util.exec exec.go:209 run result: exit status 1
2024-02-29T19:48:07.263Z [container-agent] 2024-02-29 19:48:07 INFO juju-log Setting up the logrotate configurations
2024-02-29T19:48:07.328Z [container-agent] 2024-02-29 19:48:07 INFO juju-log Adding pebble layer
2024-02-29T19:48:14.085Z [container-agent] 2024-02-29 19:48:14 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-02-29T19:48:14.106Z [container-agent] 2024-02-29 19:48:14 INFO juju-log Attempting reboot from complete outage.
2024-02-29T19:48:17.110Z [container-agent] 2024-02-29 19:48:17 INFO juju.worker.uniter.operation runhook.go:186 ran "mysql-pebble-ready" hook (via hook dispatching script: dispatch)
2024-02-29T19:48:22.220Z [container-agent] 2024-02-29 19:48:22 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T19:48:24.084Z [container-agent] 2024-02-29 19:48:24 WARNING juju-log No relation: certificates
2024-02-29T19:48:24.459Z [container-agent] 2024-02-29 19:48:24 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T19:53:17.155Z [container-agent] 2024-02-29 19:53:17 INFO juju-log Starting the log rotate manager
2024-02-29T19:53:17.168Z [container-agent] 2024-02-29 19:53:17 INFO juju-log Started log rotate manager process with PID 321762
2024-02-29T19:53:17.446Z [container-agent] 2024-02-29 19:53:17 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T19:53:19.459Z [container-agent] 2024-02-29 19:53:19 WARNING juju-log No relation: certificates
2024-02-29T19:53:19.817Z [container-agent] 2024-02-29 19:53:19 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T19:57:33.670Z [container-agent] 2024-02-29 19:57:33 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T19:57:36.101Z [container-agent] 2024-02-29 19:57:36 WARNING juju-log No relation: certificates
2024-02-29T19:57:36.440Z [container-agent] 2024-02-29 19:57:36 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T20:03:06.476Z [container-agent] 2024-02-29 20:03:06 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T20:03:09.479Z [container-agent] 2024-02-29 20:03:09 WARNING juju-log No relation: certificates
2024-02-29T20:03:09.855Z [container-agent] 2024-02-29 20:03:09 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T20:08:46.588Z [container-agent] 2024-02-29 20:08:46 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T20:08:50.221Z [container-agent] 2024-02-29 20:08:50 WARNING juju-log No relation: certificates
2024-02-29T20:08:50.589Z [container-agent] 2024-02-29 20:08:50 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T20:14:07.928Z [container-agent] 2024-02-29 20:14:07 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T20:14:11.741Z [container-agent] 2024-02-29 20:14:11 WARNING juju-log No relation: certificates
2024-02-29T20:14:12.120Z [container-agent] 2024-02-29 20:14:12 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T20:20:02.208Z [container-agent] 2024-02-29 20:20:02 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T20:20:06.403Z [container-agent] 2024-02-29 20:20:06 WARNING juju-log No relation: certificates
2024-02-29T20:20:06.769Z [container-agent] 2024-02-29 20:20:06 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T20:24:09.379Z [container-agent] 2024-02-29 20:24:09 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T20:24:14.259Z [container-agent] 2024-02-29 20:24:14 WARNING juju-log No relation: certificates
2024-02-29T20:24:14.600Z [container-agent] 2024-02-29 20:24:14 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T20:28:51.405Z [container-agent] 2024-02-29 20:28:51 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T20:28:56.668Z [container-agent] 2024-02-29 20:28:56 WARNING juju-log No relation: certificates
2024-02-29T20:28:57.013Z [container-agent] 2024-02-29 20:28:57 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T20:34:47.277Z [container-agent] 2024-02-29 20:34:47 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T20:34:53.185Z [container-agent] 2024-02-29 20:34:53 WARNING juju-log No relation: certificates
2024-02-29T20:34:53.552Z [container-agent] 2024-02-29 20:34:53 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T20:39:11.306Z [container-agent] 2024-02-29 20:39:11 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T20:39:17.655Z [container-agent] 2024-02-29 20:39:17 WARNING juju-log No relation: certificates
2024-02-29T20:39:18.034Z [container-agent] 2024-02-29 20:39:18 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T20:45:00.003Z [container-agent] 2024-02-29 20:45:00 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T20:45:06.512Z [container-agent] 2024-02-29 20:45:06 WARNING juju-log No relation: certificates
2024-02-29T20:45:06.859Z [container-agent] 2024-02-29 20:45:06 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T20:50:27.347Z [container-agent] 2024-02-29 20:50:27 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T20:50:34.394Z [container-agent] 2024-02-29 20:50:34 WARNING juju-log No relation: certificates
2024-02-29T20:50:34.792Z [container-agent] 2024-02-29 20:50:34 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T20:56:13.397Z [container-agent] 2024-02-29 20:56:13 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T20:56:21.416Z [container-agent] 2024-02-29 20:56:21 WARNING juju-log No relation: certificates
2024-02-29T20:56:21.764Z [container-agent] 2024-02-29 20:56:21 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T21:01:15.284Z [container-agent] 2024-02-29 21:01:15 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-02-29T21:01:15.318Z [container-agent] 2024-02-29 21:01:15 INFO juju-log Attempting reboot from complete outage.
2024-02-29T21:01:23.179Z [container-agent] 2024-02-29 21:01:23 WARNING juju-log No relation: certificates
2024-02-29T21:01:23.535Z [container-agent] 2024-02-29 21:01:23 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T21:03:04.801Z [container-agent] 2024-02-29 21:03:04 ERROR juju-log Uncaught exception while in charm code:
2024-02-29T21:03:04.801Z [container-agent] Traceback (most recent call last):
2024-02-29T21:03:04.801Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 2507, in exec
2024-02-29T21:03:04.801Z [container-agent]     control_ws = self._connect_websocket(task_id, 'control')
2024-02-29T21:03:04.801Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 2589, in _connect_websocket
2024-02-29T21:03:04.801Z [container-agent]     ws.connect(url, socket=sock)
2024-02-29T21:03:04.801Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_core.py", line 253, in connect
2024-02-29T21:03:04.801Z [container-agent]     self.handshake_response = handshake(self.sock, url, *addrs, **options)
2024-02-29T21:03:04.801Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_handshake.py", line 57, in handshake
2024-02-29T21:03:04.801Z [container-agent]     status, resp = _get_resp_headers(sock)
2024-02-29T21:03:04.801Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_handshake.py", line 143, in _get_resp_headers
2024-02-29T21:03:04.801Z [container-agent]     status, resp_headers, status_message = read_headers(sock)
2024-02-29T21:03:04.801Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_http.py", line 312, in read_headers
2024-02-29T21:03:04.801Z [container-agent]     line = recv_line(sock)
2024-02-29T21:03:04.801Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_socket.py", line 131, in recv_line
2024-02-29T21:03:04.801Z [container-agent]     c = recv(sock, 1)
2024-02-29T21:03:04.801Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_socket.py", line 122, in recv
2024-02-29T21:03:04.801Z [container-agent]     raise WebSocketConnectionClosedException(
2024-02-29T21:03:04.801Z [container-agent] websocket._exceptions.WebSocketConnectionClosedException: Connection to remote host was lost.
2024-02-29T21:03:04.801Z [container-agent] 
2024-02-29T21:03:04.801Z [container-agent] During handling of the above exception, another exception occurred:
2024-02-29T21:03:04.801Z [container-agent] 
2024-02-29T21:03:04.801Z [container-agent] Traceback (most recent call last):
2024-02-29T21:03:04.801Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 1348, in do_open
2024-02-29T21:03:04.801Z [container-agent]     h.request(req.get_method(), req.selector, req.data, headers,
2024-02-29T21:03:04.801Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1283, in request
2024-02-29T21:03:04.801Z [container-agent]     self._send_request(method, url, body, headers, encode_chunked)
2024-02-29T21:03:04.801Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1329, in _send_request
2024-02-29T21:03:04.801Z [container-agent]     self.endheaders(body, encode_chunked=encode_chunked)
2024-02-29T21:03:04.801Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1278, in endheaders
2024-02-29T21:03:04.801Z [container-agent]     self._send_output(message_body, encode_chunked=encode_chunked)
2024-02-29T21:03:04.801Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1038, in _send_output
2024-02-29T21:03:04.801Z [container-agent]     self.send(msg)
2024-02-29T21:03:04.801Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 976, in send
2024-02-29T21:03:04.801Z [container-agent]     self.connect()
2024-02-29T21:03:04.801Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 297, in connect
2024-02-29T21:03:04.801Z [container-agent]     self.sock.connect(self.socket_path)
2024-02-29T21:03:04.801Z [container-agent] ConnectionRefusedError: [Errno 111] Connection refused
2024-02-29T21:03:04.801Z [container-agent] 
2024-02-29T21:03:04.801Z [container-agent] During handling of the above exception, another exception occurred:
2024-02-29T21:03:04.801Z [container-agent] 
2024-02-29T21:03:04.801Z [container-agent] Traceback (most recent call last):
2024-02-29T21:03:04.801Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1690, in _request_raw
2024-02-29T21:03:04.801Z [container-agent]     response = self.opener.open(request, timeout=self.timeout)
2024-02-29T21:03:04.801Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 519, in open
2024-02-29T21:03:04.801Z [container-agent]     response = self._open(req, data)
2024-02-29T21:03:04.801Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 536, in _open
2024-02-29T21:03:04.801Z [container-agent]     result = self._call_chain(self.handle_open, protocol, protocol +
2024-02-29T21:03:04.801Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 496, in _call_chain
2024-02-29T21:03:04.801Z [container-agent]     result = func(*args)
2024-02-29T21:03:04.801Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 311, in http_open
2024-02-29T21:03:04.801Z [container-agent]     return self.do_open(_UnixSocketConnection, req,  # type:ignore
2024-02-29T21:03:04.801Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 1351, in do_open
2024-02-29T21:03:04.801Z [container-agent]     raise URLError(err)
2024-02-29T21:03:04.801Z [container-agent] urllib.error.URLError: <urlopen error [Errno 111] Connection refused>
2024-02-29T21:03:04.801Z [container-agent] 
2024-02-29T21:03:04.801Z [container-agent] The above exception was the direct cause of the following exception:
2024-02-29T21:03:04.801Z [container-agent] 
2024-02-29T21:03:04.801Z [container-agent] Traceback (most recent call last):
2024-02-29T21:03:04.801Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/./src/charm.py", line 786, in <module>
2024-02-29T21:03:04.801Z [container-agent]     main(MySQLOperatorCharm)
2024-02-29T21:03:04.801Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 436, in main
2024-02-29T21:03:04.801Z [container-agent]     _emit_charm_event(charm, dispatcher.event_name)
2024-02-29T21:03:04.801Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 144, in _emit_charm_event
2024-02-29T21:03:04.801Z [container-agent]     event_to_emit.emit(*args, **kwargs)
2024-02-29T21:03:04.801Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 351, in emit
2024-02-29T21:03:04.801Z [container-agent]     framework._emit(event)
2024-02-29T21:03:04.801Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 853, in _emit
2024-02-29T21:03:04.801Z [container-agent]     self._reemit(event_path)
2024-02-29T21:03:04.801Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 942, in _reemit
2024-02-29T21:03:04.801Z [container-agent]     custom_handler(event)
2024-02-29T21:03:04.801Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/rotate_mysql_logs.py", line 55, in _rotate_mysql_logs
2024-02-29T21:03:04.801Z [container-agent]     self.charm._mysql.flush_mysql_logs(MySQLTextLogs.ERROR)
2024-02-29T21:03:04.801Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-02-29T21:03:04.801Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-02-29T21:03:04.801Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 665, in _run_mysqlsh_script
2024-02-29T21:03:04.801Z [container-agent]     process = self.container.exec(cmd, timeout=timeout)
2024-02-29T21:03:04.801Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/model.py", line 2688, in exec
2024-02-29T21:03:04.801Z [container-agent]     return self._pebble.exec(
2024-02-29T21:03:04.801Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 2514, in exec
2024-02-29T21:03:04.801Z [container-agent]     change = self.wait_change(ChangeID(change_id))
2024-02-29T21:03:04.801Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1893, in wait_change
2024-02-29T21:03:04.801Z [container-agent]     return self._wait_change_using_wait(change_id, timeout)
2024-02-29T21:03:04.801Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1914, in _wait_change_using_wait
2024-02-29T21:03:04.801Z [container-agent]     return self._wait_change(change_id, this_timeout)
2024-02-29T21:03:04.801Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1928, in _wait_change
2024-02-29T21:03:04.801Z [container-agent]     resp = self._request('GET', f'/v1/changes/{change_id}/wait', query)
2024-02-29T21:03:04.801Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1655, in _request
2024-02-29T21:03:04.801Z [container-agent]     response = self._request_raw(method, path, query, headers, data)
2024-02-29T21:03:04.801Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1707, in _request_raw
2024-02-29T21:03:04.801Z [container-agent]     raise ConnectionError(e.reason) from e
2024-02-29T21:03:04.801Z [container-agent] ops.pebble.ConnectionError: [Errno 111] Connection refused
2024-02-29T21:03:04.995Z [container-agent] 2024-02-29 21:03:04 INFO juju.util.exec exec.go:209 run result: exit status 1
2024-02-29T21:03:09.860Z [container-agent] 2024-02-29 21:03:09 INFO juju-log Setting up the logrotate configurations
2024-02-29T21:03:09.914Z [container-agent] 2024-02-29 21:03:09 INFO juju-log Adding pebble layer
2024-02-29T21:03:16.646Z [container-agent] 2024-02-29 21:03:16 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-02-29T21:03:16.673Z [container-agent] 2024-02-29 21:03:16 INFO juju-log Attempting reboot from complete outage.
2024-02-29T21:03:19.711Z [container-agent] 2024-02-29 21:03:19 INFO juju.worker.uniter.operation runhook.go:186 ran "mysql-pebble-ready" hook (via hook dispatching script: dispatch)
2024-02-29T21:06:55.558Z [container-agent] 2024-02-29 21:06:55 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T21:06:57.437Z [container-agent] 2024-02-29 21:06:57 WARNING juju-log No relation: certificates
2024-02-29T21:06:57.804Z [container-agent] 2024-02-29 21:06:57 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T21:11:19.442Z [container-agent] 2024-02-29 21:11:19 INFO juju-log Starting the log rotate manager
2024-02-29T21:11:19.457Z [container-agent] 2024-02-29 21:11:19 INFO juju-log Started log rotate manager process with PID 328868
2024-02-29T21:11:19.727Z [container-agent] 2024-02-29 21:11:19 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T21:11:21.717Z [container-agent] 2024-02-29 21:11:21 WARNING juju-log No relation: certificates
2024-02-29T21:11:22.103Z [container-agent] 2024-02-29 21:11:22 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T21:15:48.913Z [container-agent] 2024-02-29 21:15:48 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T21:15:51.250Z [container-agent] 2024-02-29 21:15:51 WARNING juju-log No relation: certificates
2024-02-29T21:15:51.573Z [container-agent] 2024-02-29 21:15:51 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T21:20:31.228Z [container-agent] 2024-02-29 21:20:31 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T21:20:34.295Z [container-agent] 2024-02-29 21:20:34 WARNING juju-log No relation: certificates
2024-02-29T21:20:34.664Z [container-agent] 2024-02-29 21:20:34 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T21:24:42.425Z [container-agent] 2024-02-29 21:24:42 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T21:24:45.731Z [container-agent] 2024-02-29 21:24:45 WARNING juju-log No relation: certificates
2024-02-29T21:24:46.072Z [container-agent] 2024-02-29 21:24:46 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T21:30:02.304Z [container-agent] 2024-02-29 21:30:02 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T21:30:06.064Z [container-agent] 2024-02-29 21:30:06 WARNING juju-log No relation: certificates
2024-02-29T21:30:06.446Z [container-agent] 2024-02-29 21:30:06 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T21:35:45.800Z [container-agent] 2024-02-29 21:35:45 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T21:35:50.312Z [container-agent] 2024-02-29 21:35:50 WARNING juju-log No relation: certificates
2024-02-29T21:35:50.652Z [container-agent] 2024-02-29 21:35:50 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T21:40:14.004Z [container-agent] 2024-02-29 21:40:14 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T21:40:19.006Z [container-agent] 2024-02-29 21:40:19 WARNING juju-log No relation: certificates
2024-02-29T21:40:19.347Z [container-agent] 2024-02-29 21:40:19 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T21:45:09.924Z [container-agent] 2024-02-29 21:45:09 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T21:45:15.032Z [container-agent] 2024-02-29 21:45:15 WARNING juju-log No relation: certificates
2024-02-29T21:45:15.394Z [container-agent] 2024-02-29 21:45:15 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T21:50:14.689Z [container-agent] 2024-02-29 21:50:14 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T21:50:20.269Z [container-agent] 2024-02-29 21:50:20 WARNING juju-log No relation: certificates
2024-02-29T21:50:20.605Z [container-agent] 2024-02-29 21:50:20 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T21:55:34.766Z [container-agent] 2024-02-29 21:55:34 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T21:55:40.854Z [container-agent] 2024-02-29 21:55:40 WARNING juju-log No relation: certificates
2024-02-29T21:55:41.202Z [container-agent] 2024-02-29 21:55:41 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T22:01:28.106Z [container-agent] 2024-02-29 22:01:28 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T22:01:34.865Z [container-agent] 2024-02-29 22:01:34 WARNING juju-log No relation: certificates
2024-02-29T22:01:35.213Z [container-agent] 2024-02-29 22:01:35 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T22:06:15.189Z [container-agent] 2024-02-29 22:06:15 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T22:06:22.464Z [container-agent] 2024-02-29 22:06:22 WARNING juju-log No relation: certificates
2024-02-29T22:06:22.842Z [container-agent] 2024-02-29 22:06:22 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T22:11:13.272Z [container-agent] 2024-02-29 22:11:13 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T22:11:20.927Z [container-agent] 2024-02-29 22:11:20 WARNING juju-log No relation: certificates
2024-02-29T22:11:21.272Z [container-agent] 2024-02-29 22:11:21 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T22:15:34.719Z [container-agent] 2024-02-29 22:15:34 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T22:15:42.557Z [container-agent] 2024-02-29 22:15:42 WARNING juju-log No relation: certificates
2024-02-29T22:15:42.905Z [container-agent] 2024-02-29 22:15:42 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T22:19:37.050Z [container-agent] 2024-02-29 22:19:37 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T22:19:44.283Z [container-agent] 2024-02-29 22:19:44 WARNING juju-log No relation: certificates
2024-02-29T22:19:44.674Z [container-agent] 2024-02-29 22:19:44 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T22:24:03.717Z [container-agent] 2024-02-29 22:24:03 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-02-29T22:24:03.743Z [container-agent] 2024-02-29 22:24:03 INFO juju-log Attempting reboot from complete outage.
2024-02-29T22:24:10.879Z [container-agent] 2024-02-29 22:24:10 WARNING juju-log No relation: certificates
2024-02-29T22:24:11.220Z [container-agent] 2024-02-29 22:24:11 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T22:26:04.783Z [container-agent] 2024-02-29 22:26:04 ERROR juju-log Uncaught exception while in charm code:
2024-02-29T22:26:04.783Z [container-agent] Traceback (most recent call last):
2024-02-29T22:26:04.783Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 2507, in exec
2024-02-29T22:26:04.783Z [container-agent]     control_ws = self._connect_websocket(task_id, 'control')
2024-02-29T22:26:04.783Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 2589, in _connect_websocket
2024-02-29T22:26:04.783Z [container-agent]     ws.connect(url, socket=sock)
2024-02-29T22:26:04.783Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_core.py", line 253, in connect
2024-02-29T22:26:04.783Z [container-agent]     self.handshake_response = handshake(self.sock, url, *addrs, **options)
2024-02-29T22:26:04.783Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_handshake.py", line 57, in handshake
2024-02-29T22:26:04.783Z [container-agent]     status, resp = _get_resp_headers(sock)
2024-02-29T22:26:04.783Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_handshake.py", line 143, in _get_resp_headers
2024-02-29T22:26:04.783Z [container-agent]     status, resp_headers, status_message = read_headers(sock)
2024-02-29T22:26:04.783Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_http.py", line 312, in read_headers
2024-02-29T22:26:04.783Z [container-agent]     line = recv_line(sock)
2024-02-29T22:26:04.783Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_socket.py", line 131, in recv_line
2024-02-29T22:26:04.783Z [container-agent]     c = recv(sock, 1)
2024-02-29T22:26:04.783Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_socket.py", line 122, in recv
2024-02-29T22:26:04.783Z [container-agent]     raise WebSocketConnectionClosedException(
2024-02-29T22:26:04.783Z [container-agent] websocket._exceptions.WebSocketConnectionClosedException: Connection to remote host was lost.
2024-02-29T22:26:04.783Z [container-agent] 
2024-02-29T22:26:04.783Z [container-agent] During handling of the above exception, another exception occurred:
2024-02-29T22:26:04.783Z [container-agent] 
2024-02-29T22:26:04.783Z [container-agent] Traceback (most recent call last):
2024-02-29T22:26:04.783Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 1348, in do_open
2024-02-29T22:26:04.783Z [container-agent]     h.request(req.get_method(), req.selector, req.data, headers,
2024-02-29T22:26:04.783Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1283, in request
2024-02-29T22:26:04.783Z [container-agent]     self._send_request(method, url, body, headers, encode_chunked)
2024-02-29T22:26:04.783Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1329, in _send_request
2024-02-29T22:26:04.783Z [container-agent]     self.endheaders(body, encode_chunked=encode_chunked)
2024-02-29T22:26:04.783Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1278, in endheaders
2024-02-29T22:26:04.783Z [container-agent]     self._send_output(message_body, encode_chunked=encode_chunked)
2024-02-29T22:26:04.783Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1038, in _send_output
2024-02-29T22:26:04.783Z [container-agent]     self.send(msg)
2024-02-29T22:26:04.783Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 976, in send
2024-02-29T22:26:04.783Z [container-agent]     self.connect()
2024-02-29T22:26:04.783Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 297, in connect
2024-02-29T22:26:04.783Z [container-agent]     self.sock.connect(self.socket_path)
2024-02-29T22:26:04.783Z [container-agent] ConnectionRefusedError: [Errno 111] Connection refused
2024-02-29T22:26:04.783Z [container-agent] 
2024-02-29T22:26:04.783Z [container-agent] During handling of the above exception, another exception occurred:
2024-02-29T22:26:04.783Z [container-agent] 
2024-02-29T22:26:04.783Z [container-agent] Traceback (most recent call last):
2024-02-29T22:26:04.783Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1690, in _request_raw
2024-02-29T22:26:04.783Z [container-agent]     response = self.opener.open(request, timeout=self.timeout)
2024-02-29T22:26:04.783Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 519, in open
2024-02-29T22:26:04.783Z [container-agent]     response = self._open(req, data)
2024-02-29T22:26:04.783Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 536, in _open
2024-02-29T22:26:04.783Z [container-agent]     result = self._call_chain(self.handle_open, protocol, protocol +
2024-02-29T22:26:04.783Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 496, in _call_chain
2024-02-29T22:26:04.783Z [container-agent]     result = func(*args)
2024-02-29T22:26:04.783Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 311, in http_open
2024-02-29T22:26:04.783Z [container-agent]     return self.do_open(_UnixSocketConnection, req,  # type:ignore
2024-02-29T22:26:04.783Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 1351, in do_open
2024-02-29T22:26:04.783Z [container-agent]     raise URLError(err)
2024-02-29T22:26:04.783Z [container-agent] urllib.error.URLError: <urlopen error [Errno 111] Connection refused>
2024-02-29T22:26:04.783Z [container-agent] 
2024-02-29T22:26:04.783Z [container-agent] The above exception was the direct cause of the following exception:
2024-02-29T22:26:04.783Z [container-agent] 
2024-02-29T22:26:04.783Z [container-agent] Traceback (most recent call last):
2024-02-29T22:26:04.783Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/./src/charm.py", line 786, in <module>
2024-02-29T22:26:04.783Z [container-agent]     main(MySQLOperatorCharm)
2024-02-29T22:26:04.783Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 436, in main
2024-02-29T22:26:04.783Z [container-agent]     _emit_charm_event(charm, dispatcher.event_name)
2024-02-29T22:26:04.783Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 144, in _emit_charm_event
2024-02-29T22:26:04.783Z [container-agent]     event_to_emit.emit(*args, **kwargs)
2024-02-29T22:26:04.783Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 351, in emit
2024-02-29T22:26:04.783Z [container-agent]     framework._emit(event)
2024-02-29T22:26:04.783Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 853, in _emit
2024-02-29T22:26:04.783Z [container-agent]     self._reemit(event_path)
2024-02-29T22:26:04.783Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 942, in _reemit
2024-02-29T22:26:04.783Z [container-agent]     custom_handler(event)
2024-02-29T22:26:04.783Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/rotate_mysql_logs.py", line 55, in _rotate_mysql_logs
2024-02-29T22:26:04.783Z [container-agent]     self.charm._mysql.flush_mysql_logs(MySQLTextLogs.ERROR)
2024-02-29T22:26:04.783Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-02-29T22:26:04.783Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-02-29T22:26:04.783Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 665, in _run_mysqlsh_script
2024-02-29T22:26:04.783Z [container-agent]     process = self.container.exec(cmd, timeout=timeout)
2024-02-29T22:26:04.783Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/model.py", line 2688, in exec
2024-02-29T22:26:04.783Z [container-agent]     return self._pebble.exec(
2024-02-29T22:26:04.783Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 2514, in exec
2024-02-29T22:26:04.783Z [container-agent]     change = self.wait_change(ChangeID(change_id))
2024-02-29T22:26:04.783Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1893, in wait_change
2024-02-29T22:26:04.783Z [container-agent]     return self._wait_change_using_wait(change_id, timeout)
2024-02-29T22:26:04.783Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1914, in _wait_change_using_wait
2024-02-29T22:26:04.783Z [container-agent]     return self._wait_change(change_id, this_timeout)
2024-02-29T22:26:04.783Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1928, in _wait_change
2024-02-29T22:26:04.783Z [container-agent]     resp = self._request('GET', f'/v1/changes/{change_id}/wait', query)
2024-02-29T22:26:04.783Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1655, in _request
2024-02-29T22:26:04.783Z [container-agent]     response = self._request_raw(method, path, query, headers, data)
2024-02-29T22:26:04.783Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1707, in _request_raw
2024-02-29T22:26:04.783Z [container-agent]     raise ConnectionError(e.reason) from e
2024-02-29T22:26:04.783Z [container-agent] ops.pebble.ConnectionError: [Errno 111] Connection refused
2024-02-29T22:26:04.929Z [container-agent] 2024-02-29 22:26:04 INFO juju.util.exec exec.go:209 run result: exit status 1
2024-02-29T22:26:07.580Z [container-agent] 2024-02-29 22:26:07 INFO juju-log Setting up the logrotate configurations
2024-02-29T22:26:07.642Z [container-agent] 2024-02-29 22:26:07 INFO juju-log Adding pebble layer
2024-02-29T22:26:14.359Z [container-agent] 2024-02-29 22:26:14 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-02-29T22:26:14.382Z [container-agent] 2024-02-29 22:26:14 INFO juju-log Attempting reboot from complete outage.
2024-02-29T22:26:17.371Z [container-agent] 2024-02-29 22:26:17 INFO juju.worker.uniter.operation runhook.go:186 ran "mysql-pebble-ready" hook (via hook dispatching script: dispatch)
2024-02-29T22:28:13.336Z [container-agent] 2024-02-29 22:28:13 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T22:28:15.338Z [container-agent] 2024-02-29 22:28:15 WARNING juju-log No relation: certificates
2024-02-29T22:28:15.689Z [container-agent] 2024-02-29 22:28:15 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T22:32:32.452Z [container-agent] 2024-02-29 22:32:32 INFO juju-log Starting the log rotate manager
2024-02-29T22:32:32.464Z [container-agent] 2024-02-29 22:32:32 INFO juju-log Started log rotate manager process with PID 336641
2024-02-29T22:32:32.736Z [container-agent] 2024-02-29 22:32:32 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T22:32:34.799Z [container-agent] 2024-02-29 22:32:34 WARNING juju-log No relation: certificates
2024-02-29T22:32:35.188Z [container-agent] 2024-02-29 22:32:35 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T22:36:58.832Z [container-agent] 2024-02-29 22:36:58 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T22:37:01.172Z [container-agent] 2024-02-29 22:37:01 WARNING juju-log No relation: certificates
2024-02-29T22:37:01.498Z [container-agent] 2024-02-29 22:37:01 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T22:41:08.072Z [container-agent] 2024-02-29 22:41:08 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T22:41:10.995Z [container-agent] 2024-02-29 22:41:10 WARNING juju-log No relation: certificates
2024-02-29T22:41:11.314Z [container-agent] 2024-02-29 22:41:11 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T22:45:41.647Z [container-agent] 2024-02-29 22:45:41 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T22:45:44.927Z [container-agent] 2024-02-29 22:45:44 WARNING juju-log No relation: certificates
2024-02-29T22:45:45.290Z [container-agent] 2024-02-29 22:45:45 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T22:51:23.462Z [container-agent] 2024-02-29 22:51:23 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T22:51:27.144Z [container-agent] 2024-02-29 22:51:27 WARNING juju-log No relation: certificates
2024-02-29T22:51:27.488Z [container-agent] 2024-02-29 22:51:27 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T22:57:16.602Z [container-agent] 2024-02-29 22:57:16 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T22:57:21.064Z [container-agent] 2024-02-29 22:57:21 WARNING juju-log No relation: certificates
2024-02-29T22:57:21.407Z [container-agent] 2024-02-29 22:57:21 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T23:02:42.797Z [container-agent] 2024-02-29 23:02:42 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T23:02:47.781Z [container-agent] 2024-02-29 23:02:47 WARNING juju-log No relation: certificates
2024-02-29T23:02:48.126Z [container-agent] 2024-02-29 23:02:48 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T23:08:33.447Z [container-agent] 2024-02-29 23:08:33 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T23:08:38.995Z [container-agent] 2024-02-29 23:08:38 WARNING juju-log No relation: certificates
2024-02-29T23:08:39.336Z [container-agent] 2024-02-29 23:08:39 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T23:13:54.191Z [container-agent] 2024-02-29 23:13:54 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T23:13:59.558Z [container-agent] 2024-02-29 23:13:59 WARNING juju-log No relation: certificates
2024-02-29T23:13:59.912Z [container-agent] 2024-02-29 23:13:59 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T23:18:33.727Z [container-agent] 2024-02-29 23:18:33 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T23:18:39.729Z [container-agent] 2024-02-29 23:18:39 WARNING juju-log No relation: certificates
2024-02-29T23:18:40.073Z [container-agent] 2024-02-29 23:18:40 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T23:23:28.427Z [container-agent] 2024-02-29 23:23:28 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T23:23:35.015Z [container-agent] 2024-02-29 23:23:35 WARNING juju-log No relation: certificates
2024-02-29T23:23:35.360Z [container-agent] 2024-02-29 23:23:35 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T23:28:58.908Z [container-agent] 2024-02-29 23:28:58 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T23:29:05.812Z [container-agent] 2024-02-29 23:29:05 WARNING juju-log No relation: certificates
2024-02-29T23:29:06.155Z [container-agent] 2024-02-29 23:29:06 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T23:34:37.263Z [container-agent] 2024-02-29 23:34:37 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T23:34:44.374Z [container-agent] 2024-02-29 23:34:44 WARNING juju-log No relation: certificates
2024-02-29T23:34:44.730Z [container-agent] 2024-02-29 23:34:44 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T23:39:18.822Z [container-agent] 2024-02-29 23:39:18 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T23:39:26.878Z [container-agent] 2024-02-29 23:39:26 WARNING juju-log No relation: certificates
2024-02-29T23:39:27.255Z [container-agent] 2024-02-29 23:39:27 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T23:44:14.053Z [container-agent] 2024-02-29 23:44:14 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T23:44:22.647Z [container-agent] 2024-02-29 23:44:22 WARNING juju-log No relation: certificates
2024-02-29T23:44:22.985Z [container-agent] 2024-02-29 23:44:22 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T23:48:52.148Z [container-agent] 2024-02-29 23:48:52 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T23:48:59.063Z [container-agent] 2024-02-29 23:48:59 WARNING juju-log Failed to get cluster primary addresses
2024-02-29T23:48:59.063Z [container-agent] Traceback (most recent call last):
2024-02-29T23:48:59.063Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 666, in _run_mysqlsh_script
2024-02-29T23:48:59.063Z [container-agent]     stdout, _ = process.wait_output()
2024-02-29T23:48:59.063Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1441, in wait_output
2024-02-29T23:48:59.063Z [container-agent]     raise ExecError[AnyStr](self._command, exit_code, out_value, err_value)
2024-02-29T23:48:59.063Z [container-agent] ops.pebble.ExecError: non-zero exit code 1 executing ['/usr/bin/mysqlsh', '--no-wizard', '--python', '--verbose=1', '-f', '/tmp/script.py', ';', 'rm', '/tmp/script.py'], stdout='', stderr='Cannot set LC_ALL to locale en_US.UTF-8: No such file or directory\nverbose: 2024-02-29T23:48:57Z: Loading startup files...\nverbose: 2024-02-29T23:48:57Z: Loading plugins...\nverbose: 2024-02-29T23:48:57Z: Connecting to MySQL at: clusteradmin@horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local\nverbose: 2024-02-29T23:48:57Z: Shell.connect_to_primary: tid=12: CONNECTED: horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local\nverbose: 2024-02-29T23:48:57Z: Redirecting session from \'mysqlx://clusteradmin@horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local:33060\' to a PRIMARY of an InnoDB cluster or ReplicaSet...\nTraceback (most recent call last):\n  File "<string>", line 1, in <module>\nRuntimeError: Shell.connect_to_primary: No primaries found in an InnoDB cluster\n\n'
2024-02-29T23:48:59.063Z [container-agent] 
2024-02-29T23:48:59.063Z [container-agent] During handling of the above exception, another exception occurred:
2024-02-29T23:48:59.063Z [container-agent] 
2024-02-29T23:48:59.063Z [container-agent] Traceback (most recent call last):
2024-02-29T23:48:59.063Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 1767, in get_cluster_primary_address
2024-02-29T23:48:59.063Z [container-agent]     output = self._run_mysqlsh_script("\n".join(get_cluster_primary_commands))
2024-02-29T23:48:59.063Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 669, in _run_mysqlsh_script
2024-02-29T23:48:59.063Z [container-agent]     raise MySQLClientError(e.stderr)
2024-02-29T23:48:59.063Z [container-agent] charms.mysql.v0.mysql.MySQLClientError: Cannot set LC_ALL to locale en_US.UTF-8: No such file or directory
2024-02-29T23:48:59.063Z [container-agent] verbose: 2024-02-29T23:48:57Z: Loading startup files...
2024-02-29T23:48:59.063Z [container-agent] verbose: 2024-02-29T23:48:57Z: Loading plugins...
2024-02-29T23:48:59.063Z [container-agent] verbose: 2024-02-29T23:48:57Z: Connecting to MySQL at: clusteradmin@horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local
2024-02-29T23:48:59.063Z [container-agent] verbose: 2024-02-29T23:48:57Z: Shell.connect_to_primary: tid=12: CONNECTED: horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local
2024-02-29T23:48:59.063Z [container-agent] verbose: 2024-02-29T23:48:57Z: Redirecting session from 'mysqlx://clusteradmin@horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local:33060' to a PRIMARY of an InnoDB cluster or ReplicaSet...
2024-02-29T23:48:59.063Z [container-agent] Traceback (most recent call last):
2024-02-29T23:48:59.063Z [container-agent]   File "<string>", line 1, in <module>
2024-02-29T23:48:59.063Z [container-agent] RuntimeError: Shell.connect_to_primary: No primaries found in an InnoDB cluster
2024-02-29T23:48:59.064Z [container-agent] 
2024-02-29T23:48:59.064Z [container-agent] 
2024-02-29T23:49:01.720Z [container-agent] 2024-02-29 23:49:01 ERROR juju-log Failed to get cluster status for cluster-7573647c1e26466beeeac32b95ab8f91
2024-02-29T23:49:01.817Z [container-agent] 2024-02-29 23:49:01 INFO juju-log Kubernetes pod label error created
2024-02-29T23:49:01.828Z [container-agent] 2024-02-29 23:49:01 WARNING juju-log No relation: certificates
2024-02-29T23:49:02.203Z [container-agent] 2024-02-29 23:49:02 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T23:53:16.281Z [container-agent] 2024-02-29 23:53:16 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-02-29T23:53:16.310Z [container-agent] 2024-02-29 23:53:16 INFO juju-log Attempting reboot from complete outage.
2024-02-29T23:53:24.701Z [container-agent] 2024-02-29 23:53:24 INFO juju-log Kubernetes pod label primary created
2024-02-29T23:53:24.716Z [container-agent] 2024-02-29 23:53:24 WARNING juju-log No relation: certificates
2024-02-29T23:53:25.090Z [container-agent] 2024-02-29 23:53:25 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-02-29T23:54:00.750Z [container-agent] 2024-02-29 23:54:00 INFO juju-log Setting up the logrotate configurations
2024-02-29T23:54:00.810Z [container-agent] 2024-02-29 23:54:00 INFO juju-log Adding pebble layer
2024-02-29T23:54:07.630Z [container-agent] 2024-02-29 23:54:07 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-02-29T23:54:07.652Z [container-agent] 2024-02-29 23:54:07 INFO juju-log Attempting reboot from complete outage.
2024-02-29T23:54:10.283Z [container-agent] 2024-02-29 23:54:10 INFO juju.worker.uniter.operation runhook.go:186 ran "mysql-pebble-ready" hook (via hook dispatching script: dispatch)
2024-02-29T23:58:53.418Z [container-agent] 2024-02-29 23:58:53 INFO juju-log Unit workload member-state is online with member-role primary
2024-02-29T23:58:55.651Z [container-agent] 2024-02-29 23:58:55 WARNING juju-log No relation: certificates
2024-02-29T23:58:56.013Z [container-agent] 2024-02-29 23:58:56 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T00:03:37.831Z [container-agent] 2024-03-01 00:03:37 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T00:03:40.518Z [container-agent] 2024-03-01 00:03:40 WARNING juju-log No relation: certificates
2024-03-01T00:03:40.875Z [container-agent] 2024-03-01 00:03:40 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T00:08:25.646Z [container-agent] 2024-03-01 00:08:25 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T00:08:28.629Z [container-agent] 2024-03-01 00:08:28 WARNING juju-log No relation: certificates
2024-03-01T00:08:29.019Z [container-agent] 2024-03-01 00:08:29 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T00:13:54.293Z [container-agent] 2024-03-01 00:13:54 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T00:13:57.797Z [container-agent] 2024-03-01 00:13:57 WARNING juju-log No relation: certificates
2024-03-01T00:13:58.150Z [container-agent] 2024-03-01 00:13:58 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T00:19:46.968Z [container-agent] 2024-03-01 00:19:46 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T00:19:51.260Z [container-agent] 2024-03-01 00:19:51 WARNING juju-log No relation: certificates
2024-03-01T00:19:51.639Z [container-agent] 2024-03-01 00:19:51 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T00:25:40.719Z [container-agent] 2024-03-01 00:25:40 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T00:25:45.420Z [container-agent] 2024-03-01 00:25:45 WARNING juju-log No relation: certificates
2024-03-01T00:25:45.789Z [container-agent] 2024-03-01 00:25:45 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T00:30:25.245Z [container-agent] 2024-03-01 00:30:25 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T00:30:30.465Z [container-agent] 2024-03-01 00:30:30 WARNING juju-log No relation: certificates
2024-03-01T00:30:30.810Z [container-agent] 2024-03-01 00:30:30 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T00:35:10.947Z [container-agent] 2024-03-01 00:35:10 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T00:35:16.658Z [container-agent] 2024-03-01 00:35:16 WARNING juju-log No relation: certificates
2024-03-01T00:35:17.010Z [container-agent] 2024-03-01 00:35:17 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T00:39:26.470Z [container-agent] 2024-03-01 00:39:26 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T00:39:32.719Z [container-agent] 2024-03-01 00:39:32 WARNING juju-log No relation: certificates
2024-03-01T00:39:33.093Z [container-agent] 2024-03-01 00:39:33 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T00:45:11.824Z [container-agent] 2024-03-01 00:45:11 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T00:45:17.864Z [container-agent] 2024-03-01 00:45:17 WARNING juju-log No relation: certificates
2024-03-01T00:45:18.233Z [container-agent] 2024-03-01 00:45:18 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T00:49:12.155Z [container-agent] 2024-03-01 00:49:12 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T00:49:19.534Z [container-agent] 2024-03-01 00:49:19 WARNING juju-log No relation: certificates
2024-03-01T00:49:19.896Z [container-agent] 2024-03-01 00:49:19 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T00:54:58.357Z [container-agent] 2024-03-01 00:54:58 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T00:55:05.766Z [container-agent] 2024-03-01 00:55:05 WARNING juju-log No relation: certificates
2024-03-01T00:55:06.129Z [container-agent] 2024-03-01 00:55:06 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T00:59:18.417Z [container-agent] 2024-03-01 00:59:18 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T00:59:26.251Z [container-agent] 2024-03-01 00:59:26 WARNING juju-log No relation: certificates
2024-03-01T00:59:26.590Z [container-agent] 2024-03-01 00:59:26 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T01:03:21.017Z [container-agent] 2024-03-01 01:03:21 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T01:03:28.659Z [container-agent] 2024-03-01 01:03:28 WARNING juju-log No relation: certificates
2024-03-01T01:03:29.029Z [container-agent] 2024-03-01 01:03:29 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T01:05:08.057Z [container-agent] 2024-03-01 01:05:08 ERROR juju-log Failed to flush ERROR LOGS logs.
2024-03-01T01:05:08.057Z [container-agent] Traceback (most recent call last):
2024-03-01T01:05:08.057Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 666, in _run_mysqlsh_script
2024-03-01T01:05:08.057Z [container-agent]     stdout, _ = process.wait_output()
2024-03-01T01:05:08.057Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1441, in wait_output
2024-03-01T01:05:08.057Z [container-agent]     raise ExecError[AnyStr](self._command, exit_code, out_value, err_value)
2024-03-01T01:05:08.057Z [container-agent] ops.pebble.ExecError: non-zero exit code 1 executing ['/usr/bin/mysqlsh', '--no-wizard', '--python', '--verbose=1', '-f', '/tmp/script.py', ';', 'rm', '/tmp/script.py'], stdout='', stderr='Cannot set LC_ALL to locale en_US.UTF-8: No such file or directory\nverbose: 2024-03-01T01:05:06Z: Loading startup files...\nverbose: 2024-03-01T01:05:06Z: Loading plugins...\nverbose: 2024-03-01T01:05:06Z: Connecting to MySQL at: serverconfig@horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local\nTraceback (most recent call last):\n  File "<string>", line 1, in <module>\nmysqlsh.DBError: MySQL Error (2003): Shell.connect: Can\'t connect to MySQL server on \'horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local:3306\' (111)\n'
2024-03-01T01:05:08.057Z [container-agent] 
2024-03-01T01:05:08.057Z [container-agent] During handling of the above exception, another exception occurred:
2024-03-01T01:05:08.057Z [container-agent] 
2024-03-01T01:05:08.057Z [container-agent] Traceback (most recent call last):
2024-03-01T01:05:08.057Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-03-01T01:05:08.057Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-03-01T01:05:08.057Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 669, in _run_mysqlsh_script
2024-03-01T01:05:08.057Z [container-agent]     raise MySQLClientError(e.stderr)
2024-03-01T01:05:08.057Z [container-agent] charms.mysql.v0.mysql.MySQLClientError: Cannot set LC_ALL to locale en_US.UTF-8: No such file or directory
2024-03-01T01:05:08.057Z [container-agent] verbose: 2024-03-01T01:05:06Z: Loading startup files...
2024-03-01T01:05:08.057Z [container-agent] verbose: 2024-03-01T01:05:06Z: Loading plugins...
2024-03-01T01:05:08.057Z [container-agent] verbose: 2024-03-01T01:05:06Z: Connecting to MySQL at: serverconfig@horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local
2024-03-01T01:05:08.057Z [container-agent] Traceback (most recent call last):
2024-03-01T01:05:08.057Z [container-agent]   File "<string>", line 1, in <module>
2024-03-01T01:05:08.057Z [container-agent] mysqlsh.DBError: MySQL Error (2003): Shell.connect: Can't connect to MySQL server on 'horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local:3306' (111)
2024-03-01T01:05:08.057Z [container-agent] 
2024-03-01T01:08:14.754Z [container-agent] 2024-03-01 01:08:14 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-03-01T01:08:14.781Z [container-agent] 2024-03-01 01:08:14 INFO juju-log Attempting reboot from complete outage.
2024-03-01T01:08:22.733Z [container-agent] 2024-03-01 01:08:22 WARNING juju-log No relation: certificates
2024-03-01T01:08:23.085Z [container-agent] 2024-03-01 01:08:23 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T01:10:04.814Z [container-agent] 2024-03-01 01:10:04 ERROR juju-log Uncaught exception while in charm code:
2024-03-01T01:10:04.814Z [container-agent] Traceback (most recent call last):
2024-03-01T01:10:04.814Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 2507, in exec
2024-03-01T01:10:04.814Z [container-agent]     control_ws = self._connect_websocket(task_id, 'control')
2024-03-01T01:10:04.814Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 2589, in _connect_websocket
2024-03-01T01:10:04.814Z [container-agent]     ws.connect(url, socket=sock)
2024-03-01T01:10:04.814Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_core.py", line 253, in connect
2024-03-01T01:10:04.814Z [container-agent]     self.handshake_response = handshake(self.sock, url, *addrs, **options)
2024-03-01T01:10:04.814Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_handshake.py", line 57, in handshake
2024-03-01T01:10:04.814Z [container-agent]     status, resp = _get_resp_headers(sock)
2024-03-01T01:10:04.814Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_handshake.py", line 143, in _get_resp_headers
2024-03-01T01:10:04.814Z [container-agent]     status, resp_headers, status_message = read_headers(sock)
2024-03-01T01:10:04.814Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_http.py", line 312, in read_headers
2024-03-01T01:10:04.814Z [container-agent]     line = recv_line(sock)
2024-03-01T01:10:04.814Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_socket.py", line 131, in recv_line
2024-03-01T01:10:04.814Z [container-agent]     c = recv(sock, 1)
2024-03-01T01:10:04.814Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_socket.py", line 122, in recv
2024-03-01T01:10:04.814Z [container-agent]     raise WebSocketConnectionClosedException(
2024-03-01T01:10:04.814Z [container-agent] websocket._exceptions.WebSocketConnectionClosedException: Connection to remote host was lost.
2024-03-01T01:10:04.814Z [container-agent] 
2024-03-01T01:10:04.814Z [container-agent] During handling of the above exception, another exception occurred:
2024-03-01T01:10:04.814Z [container-agent] 
2024-03-01T01:10:04.814Z [container-agent] Traceback (most recent call last):
2024-03-01T01:10:04.814Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 1348, in do_open
2024-03-01T01:10:04.814Z [container-agent]     h.request(req.get_method(), req.selector, req.data, headers,
2024-03-01T01:10:04.814Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1283, in request
2024-03-01T01:10:04.814Z [container-agent]     self._send_request(method, url, body, headers, encode_chunked)
2024-03-01T01:10:04.814Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1329, in _send_request
2024-03-01T01:10:04.814Z [container-agent]     self.endheaders(body, encode_chunked=encode_chunked)
2024-03-01T01:10:04.814Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1278, in endheaders
2024-03-01T01:10:04.814Z [container-agent]     self._send_output(message_body, encode_chunked=encode_chunked)
2024-03-01T01:10:04.814Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1038, in _send_output
2024-03-01T01:10:04.814Z [container-agent]     self.send(msg)
2024-03-01T01:10:04.814Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 976, in send
2024-03-01T01:10:04.814Z [container-agent]     self.connect()
2024-03-01T01:10:04.814Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 297, in connect
2024-03-01T01:10:04.814Z [container-agent]     self.sock.connect(self.socket_path)
2024-03-01T01:10:04.814Z [container-agent] ConnectionRefusedError: [Errno 111] Connection refused
2024-03-01T01:10:04.814Z [container-agent] 
2024-03-01T01:10:04.814Z [container-agent] During handling of the above exception, another exception occurred:
2024-03-01T01:10:04.814Z [container-agent] 
2024-03-01T01:10:04.814Z [container-agent] Traceback (most recent call last):
2024-03-01T01:10:04.814Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1690, in _request_raw
2024-03-01T01:10:04.814Z [container-agent]     response = self.opener.open(request, timeout=self.timeout)
2024-03-01T01:10:04.814Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 519, in open
2024-03-01T01:10:04.814Z [container-agent]     response = self._open(req, data)
2024-03-01T01:10:04.814Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 536, in _open
2024-03-01T01:10:04.814Z [container-agent]     result = self._call_chain(self.handle_open, protocol, protocol +
2024-03-01T01:10:04.814Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 496, in _call_chain
2024-03-01T01:10:04.814Z [container-agent]     result = func(*args)
2024-03-01T01:10:04.814Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 311, in http_open
2024-03-01T01:10:04.814Z [container-agent]     return self.do_open(_UnixSocketConnection, req,  # type:ignore
2024-03-01T01:10:04.814Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 1351, in do_open
2024-03-01T01:10:04.814Z [container-agent]     raise URLError(err)
2024-03-01T01:10:04.814Z [container-agent] urllib.error.URLError: <urlopen error [Errno 111] Connection refused>
2024-03-01T01:10:04.814Z [container-agent] 
2024-03-01T01:10:04.814Z [container-agent] The above exception was the direct cause of the following exception:
2024-03-01T01:10:04.814Z [container-agent] 
2024-03-01T01:10:04.814Z [container-agent] Traceback (most recent call last):
2024-03-01T01:10:04.814Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/./src/charm.py", line 786, in <module>
2024-03-01T01:10:04.814Z [container-agent]     main(MySQLOperatorCharm)
2024-03-01T01:10:04.814Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 436, in main
2024-03-01T01:10:04.814Z [container-agent]     _emit_charm_event(charm, dispatcher.event_name)
2024-03-01T01:10:04.814Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 144, in _emit_charm_event
2024-03-01T01:10:04.814Z [container-agent]     event_to_emit.emit(*args, **kwargs)
2024-03-01T01:10:04.814Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 351, in emit
2024-03-01T01:10:04.814Z [container-agent]     framework._emit(event)
2024-03-01T01:10:04.814Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 853, in _emit
2024-03-01T01:10:04.814Z [container-agent]     self._reemit(event_path)
2024-03-01T01:10:04.814Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 942, in _reemit
2024-03-01T01:10:04.814Z [container-agent]     custom_handler(event)
2024-03-01T01:10:04.814Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/rotate_mysql_logs.py", line 55, in _rotate_mysql_logs
2024-03-01T01:10:04.814Z [container-agent]     self.charm._mysql.flush_mysql_logs(MySQLTextLogs.ERROR)
2024-03-01T01:10:04.814Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-03-01T01:10:04.814Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-03-01T01:10:04.814Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 665, in _run_mysqlsh_script
2024-03-01T01:10:04.814Z [container-agent]     process = self.container.exec(cmd, timeout=timeout)
2024-03-01T01:10:04.814Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/model.py", line 2688, in exec
2024-03-01T01:10:04.814Z [container-agent]     return self._pebble.exec(
2024-03-01T01:10:04.814Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 2514, in exec
2024-03-01T01:10:04.814Z [container-agent]     change = self.wait_change(ChangeID(change_id))
2024-03-01T01:10:04.814Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1893, in wait_change
2024-03-01T01:10:04.814Z [container-agent]     return self._wait_change_using_wait(change_id, timeout)
2024-03-01T01:10:04.814Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1914, in _wait_change_using_wait
2024-03-01T01:10:04.814Z [container-agent]     return self._wait_change(change_id, this_timeout)
2024-03-01T01:10:04.814Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1928, in _wait_change
2024-03-01T01:10:04.814Z [container-agent]     resp = self._request('GET', f'/v1/changes/{change_id}/wait', query)
2024-03-01T01:10:04.814Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1655, in _request
2024-03-01T01:10:04.814Z [container-agent]     response = self._request_raw(method, path, query, headers, data)
2024-03-01T01:10:04.814Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1707, in _request_raw
2024-03-01T01:10:04.814Z [container-agent]     raise ConnectionError(e.reason) from e
2024-03-01T01:10:04.814Z [container-agent] ops.pebble.ConnectionError: [Errno 111] Connection refused
2024-03-01T01:10:05.033Z [container-agent] 2024-03-01 01:10:05 INFO juju.util.exec exec.go:209 run result: exit status 1
2024-03-01T01:10:07.993Z [container-agent] 2024-03-01 01:10:07 INFO juju-log Setting up the logrotate configurations
2024-03-01T01:10:08.053Z [container-agent] 2024-03-01 01:10:08 INFO juju-log Adding pebble layer
2024-03-01T01:10:14.866Z [container-agent] 2024-03-01 01:10:14 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-03-01T01:10:14.892Z [container-agent] 2024-03-01 01:10:14 INFO juju-log Attempting reboot from complete outage.
2024-03-01T01:10:17.942Z [container-agent] 2024-03-01 01:10:17 INFO juju.worker.uniter.operation runhook.go:186 ran "mysql-pebble-ready" hook (via hook dispatching script: dispatch)
2024-03-01T01:12:35.882Z [container-agent] 2024-03-01 01:12:35 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T01:12:37.679Z [container-agent] 2024-03-01 01:12:37 WARNING juju-log No relation: certificates
2024-03-01T01:12:38.011Z [container-agent] 2024-03-01 01:12:38 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T01:16:40.514Z [container-agent] 2024-03-01 01:16:40 INFO juju-log Starting the log rotate manager
2024-03-01T01:16:40.528Z [container-agent] 2024-03-01 01:16:40 INFO juju-log Started log rotate manager process with PID 352460
2024-03-01T01:16:40.807Z [container-agent] 2024-03-01 01:16:40 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T01:16:42.743Z [container-agent] 2024-03-01 01:16:42 WARNING juju-log No relation: certificates
2024-03-01T01:16:43.077Z [container-agent] 2024-03-01 01:16:43 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T01:21:16.574Z [container-agent] 2024-03-01 01:21:16 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T01:21:19.033Z [container-agent] 2024-03-01 01:21:19 WARNING juju-log No relation: certificates
2024-03-01T01:21:19.384Z [container-agent] 2024-03-01 01:21:19 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T01:26:06.576Z [container-agent] 2024-03-01 01:26:06 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T01:26:09.738Z [container-agent] 2024-03-01 01:26:09 WARNING juju-log No relation: certificates
2024-03-01T01:26:10.097Z [container-agent] 2024-03-01 01:26:10 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T01:30:16.998Z [container-agent] 2024-03-01 01:30:16 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T01:30:20.328Z [container-agent] 2024-03-01 01:30:20 WARNING juju-log No relation: certificates
2024-03-01T01:30:20.678Z [container-agent] 2024-03-01 01:30:20 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T01:35:23.595Z [container-agent] 2024-03-01 01:35:23 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T01:35:27.077Z [container-agent] 2024-03-01 01:35:27 WARNING juju-log No relation: certificates
2024-03-01T01:35:27.440Z [container-agent] 2024-03-01 01:35:27 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T01:39:56.183Z [container-agent] 2024-03-01 01:39:56 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T01:40:00.325Z [container-agent] 2024-03-01 01:40:00 WARNING juju-log No relation: certificates
2024-03-01T01:40:00.695Z [container-agent] 2024-03-01 01:40:00 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T01:44:08.991Z [container-agent] 2024-03-01 01:44:08 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T01:44:13.674Z [container-agent] 2024-03-01 01:44:13 WARNING juju-log No relation: certificates
2024-03-01T01:44:14.076Z [container-agent] 2024-03-01 01:44:14 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T01:49:33.165Z [container-agent] 2024-03-01 01:49:33 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T01:49:38.275Z [container-agent] 2024-03-01 01:49:38 WARNING juju-log No relation: certificates
2024-03-01T01:49:38.671Z [container-agent] 2024-03-01 01:49:38 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T01:54:37.102Z [container-agent] 2024-03-01 01:54:37 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T01:54:42.539Z [container-agent] 2024-03-01 01:54:42 WARNING juju-log No relation: certificates
2024-03-01T01:54:42.904Z [container-agent] 2024-03-01 01:54:42 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T01:59:17.656Z [container-agent] 2024-03-01 01:59:17 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T01:59:23.770Z [container-agent] 2024-03-01 01:59:23 WARNING juju-log No relation: certificates
2024-03-01T01:59:24.150Z [container-agent] 2024-03-01 01:59:24 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T02:04:39.767Z [container-agent] 2024-03-01 02:04:39 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T02:04:46.168Z [container-agent] 2024-03-01 02:04:46 WARNING juju-log No relation: certificates
2024-03-01T02:04:46.528Z [container-agent] 2024-03-01 02:04:46 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T02:10:30.245Z [container-agent] 2024-03-01 02:10:30 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T02:10:37.391Z [container-agent] 2024-03-01 02:10:37 WARNING juju-log No relation: certificates
2024-03-01T02:10:37.751Z [container-agent] 2024-03-01 02:10:37 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T02:14:50.408Z [container-agent] 2024-03-01 02:14:50 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T02:14:57.922Z [container-agent] 2024-03-01 02:14:57 WARNING juju-log No relation: certificates
2024-03-01T02:14:58.275Z [container-agent] 2024-03-01 02:14:58 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T02:20:49.582Z [container-agent] 2024-03-01 02:20:49 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T02:20:57.328Z [container-agent] 2024-03-01 02:20:57 WARNING juju-log No relation: certificates
2024-03-01T02:20:57.687Z [container-agent] 2024-03-01 02:20:57 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T02:24:50.991Z [container-agent] 2024-03-01 02:24:50 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T02:24:59.323Z [container-agent] 2024-03-01 02:24:59 WARNING juju-log No relation: certificates
2024-03-01T02:24:59.672Z [container-agent] 2024-03-01 02:24:59 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T02:29:29.251Z [container-agent] 2024-03-01 02:29:29 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T02:29:39.569Z [container-agent] 2024-03-01 02:29:39 ERROR juju-log Failed to get cluster status for cluster-7573647c1e26466beeeac32b95ab8f91
2024-03-01T02:29:39.765Z [container-agent] 2024-03-01 02:29:39 INFO juju-log Kubernetes pod label error created
2024-03-01T02:29:39.779Z [container-agent] 2024-03-01 02:29:39 WARNING juju-log No relation: certificates
2024-03-01T02:29:40.144Z [container-agent] 2024-03-01 02:29:40 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T02:34:35.799Z [container-agent] 2024-03-01 02:34:35 ERROR juju-log Failed to get cluster status for cluster-7573647c1e26466beeeac32b95ab8f91
2024-03-01T02:34:35.848Z [container-agent] 2024-03-01 02:34:35 WARNING juju-log No relation: certificates
2024-03-01T02:34:36.209Z [container-agent] 2024-03-01 02:34:36 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T02:34:37.604Z [container-agent] 2024-03-01 02:34:37 INFO juju-log Setting up the logrotate configurations
2024-03-01T02:34:37.670Z [container-agent] 2024-03-01 02:34:37 INFO juju-log Adding pebble layer
2024-03-01T02:34:44.380Z [container-agent] 2024-03-01 02:34:44 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-03-01T02:34:44.414Z [container-agent] 2024-03-01 02:34:44 INFO juju-log Attempting reboot from complete outage.
2024-03-01T02:34:47.511Z [container-agent] 2024-03-01 02:34:47 INFO juju.worker.uniter.operation runhook.go:186 ran "mysql-pebble-ready" hook (via hook dispatching script: dispatch)
2024-03-01T02:39:05.292Z [container-agent] 2024-03-01 02:39:05 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T02:39:07.895Z [container-agent] 2024-03-01 02:39:07 INFO juju-log Kubernetes pod label primary created
2024-03-01T02:39:07.909Z [container-agent] 2024-03-01 02:39:07 WARNING juju-log No relation: certificates
2024-03-01T02:39:08.254Z [container-agent] 2024-03-01 02:39:08 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T02:44:31.463Z [container-agent] 2024-03-01 02:44:31 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T02:44:34.408Z [container-agent] 2024-03-01 02:44:34 WARNING juju-log No relation: certificates
2024-03-01T02:44:34.765Z [container-agent] 2024-03-01 02:44:34 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T02:49:11.383Z [container-agent] 2024-03-01 02:49:11 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T02:49:14.607Z [container-agent] 2024-03-01 02:49:14 WARNING juju-log No relation: certificates
2024-03-01T02:49:14.949Z [container-agent] 2024-03-01 02:49:14 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T02:54:34.425Z [container-agent] 2024-03-01 02:54:34 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T02:54:38.374Z [container-agent] 2024-03-01 02:54:38 WARNING juju-log No relation: certificates
2024-03-01T02:54:38.714Z [container-agent] 2024-03-01 02:54:38 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T02:58:42.379Z [container-agent] 2024-03-01 02:58:42 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T02:58:46.728Z [container-agent] 2024-03-01 02:58:46 WARNING juju-log No relation: certificates
2024-03-01T02:58:47.083Z [container-agent] 2024-03-01 02:58:47 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T03:03:49.247Z [container-agent] 2024-03-01 03:03:49 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T03:03:53.786Z [container-agent] 2024-03-01 03:03:53 WARNING juju-log No relation: certificates
2024-03-01T03:03:54.145Z [container-agent] 2024-03-01 03:03:54 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T03:08:26.808Z [container-agent] 2024-03-01 03:08:26 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T03:08:31.873Z [container-agent] 2024-03-01 03:08:31 WARNING juju-log No relation: certificates
2024-03-01T03:08:32.221Z [container-agent] 2024-03-01 03:08:32 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T03:13:51.118Z [container-agent] 2024-03-01 03:13:51 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T03:13:56.031Z [container-agent] 2024-03-01 03:13:56 WARNING juju-log No relation: certificates
2024-03-01T03:13:56.376Z [container-agent] 2024-03-01 03:13:56 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T03:18:55.964Z [container-agent] 2024-03-01 03:18:55 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T03:19:01.724Z [container-agent] 2024-03-01 03:19:01 WARNING juju-log No relation: certificates
2024-03-01T03:19:02.065Z [container-agent] 2024-03-01 03:19:02 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T03:23:17.477Z [container-agent] 2024-03-01 03:23:17 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T03:23:23.823Z [container-agent] 2024-03-01 03:23:23 WARNING juju-log No relation: certificates
2024-03-01T03:23:24.188Z [container-agent] 2024-03-01 03:23:24 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T03:29:12.001Z [container-agent] 2024-03-01 03:29:12 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T03:29:18.922Z [container-agent] 2024-03-01 03:29:18 WARNING juju-log No relation: certificates
2024-03-01T03:29:19.299Z [container-agent] 2024-03-01 03:29:19 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T03:34:56.429Z [container-agent] 2024-03-01 03:34:56 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T03:35:03.628Z [container-agent] 2024-03-01 03:35:03 WARNING juju-log No relation: certificates
2024-03-01T03:35:04.018Z [container-agent] 2024-03-01 03:35:04 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T03:38:09.073Z [container-agent] 2024-03-01 03:38:09 ERROR juju-log Failed to flush GENERAL LOGS logs.
2024-03-01T03:38:09.073Z [container-agent] Traceback (most recent call last):
2024-03-01T03:38:09.073Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 666, in _run_mysqlsh_script
2024-03-01T03:38:09.073Z [container-agent]     stdout, _ = process.wait_output()
2024-03-01T03:38:09.073Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1441, in wait_output
2024-03-01T03:38:09.073Z [container-agent]     raise ExecError[AnyStr](self._command, exit_code, out_value, err_value)
2024-03-01T03:38:09.073Z [container-agent] ops.pebble.ExecError: non-zero exit code 1 executing ['/usr/bin/mysqlsh', '--no-wizard', '--python', '--verbose=1', '-f', '/tmp/script.py', ';', 'rm', '/tmp/script.py'], stdout='', stderr='Cannot set LC_ALL to locale en_US.UTF-8: No such file or directory\nverbose: 2024-03-01T03:38:07Z: Loading startup files...\nverbose: 2024-03-01T03:38:07Z: Loading plugins...\nverbose: 2024-03-01T03:38:08Z: Connecting to MySQL at: serverconfig@horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local\nTraceback (most recent call last):\n  File "<string>", line 1, in <module>\nmysqlsh.DBError: MySQL Error (2003): Shell.connect: Can\'t connect to MySQL server on \'horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local:3306\' (111)\n'
2024-03-01T03:38:09.073Z [container-agent] 
2024-03-01T03:38:09.073Z [container-agent] During handling of the above exception, another exception occurred:
2024-03-01T03:38:09.073Z [container-agent] 
2024-03-01T03:38:09.073Z [container-agent] Traceback (most recent call last):
2024-03-01T03:38:09.073Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-03-01T03:38:09.073Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-03-01T03:38:09.073Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 669, in _run_mysqlsh_script
2024-03-01T03:38:09.073Z [container-agent]     raise MySQLClientError(e.stderr)
2024-03-01T03:38:09.073Z [container-agent] charms.mysql.v0.mysql.MySQLClientError: Cannot set LC_ALL to locale en_US.UTF-8: No such file or directory
2024-03-01T03:38:09.073Z [container-agent] verbose: 2024-03-01T03:38:07Z: Loading startup files...
2024-03-01T03:38:09.073Z [container-agent] verbose: 2024-03-01T03:38:07Z: Loading plugins...
2024-03-01T03:38:09.073Z [container-agent] verbose: 2024-03-01T03:38:08Z: Connecting to MySQL at: serverconfig@horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local
2024-03-01T03:38:09.073Z [container-agent] Traceback (most recent call last):
2024-03-01T03:38:09.073Z [container-agent]   File "<string>", line 1, in <module>
2024-03-01T03:38:09.073Z [container-agent] mysqlsh.DBError: MySQL Error (2003): Shell.connect: Can't connect to MySQL server on 'horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local:3306' (111)
2024-03-01T03:38:09.073Z [container-agent] 
2024-03-01T03:39:45.542Z [container-agent] 2024-03-01 03:39:45 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-03-01T03:39:45.576Z [container-agent] 2024-03-01 03:39:45 INFO juju-log Attempting reboot from complete outage.
2024-03-01T03:39:52.918Z [container-agent] 2024-03-01 03:39:52 WARNING juju-log No relation: certificates
2024-03-01T03:39:53.264Z [container-agent] 2024-03-01 03:39:53 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T03:43:04.790Z [container-agent] 2024-03-01 03:43:04 ERROR juju-log Uncaught exception while in charm code:
2024-03-01T03:43:04.790Z [container-agent] Traceback (most recent call last):
2024-03-01T03:43:04.790Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/./src/charm.py", line 786, in <module>
2024-03-01T03:43:04.790Z [container-agent]     main(MySQLOperatorCharm)
2024-03-01T03:43:04.790Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 436, in main
2024-03-01T03:43:04.790Z [container-agent]     _emit_charm_event(charm, dispatcher.event_name)
2024-03-01T03:43:04.790Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 144, in _emit_charm_event
2024-03-01T03:43:04.790Z [container-agent]     event_to_emit.emit(*args, **kwargs)
2024-03-01T03:43:04.790Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 351, in emit
2024-03-01T03:43:04.790Z [container-agent]     framework._emit(event)
2024-03-01T03:43:04.790Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 853, in _emit
2024-03-01T03:43:04.790Z [container-agent]     self._reemit(event_path)
2024-03-01T03:43:04.790Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 942, in _reemit
2024-03-01T03:43:04.790Z [container-agent]     custom_handler(event)
2024-03-01T03:43:04.790Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/rotate_mysql_logs.py", line 55, in _rotate_mysql_logs
2024-03-01T03:43:04.790Z [container-agent]     self.charm._mysql.flush_mysql_logs(MySQLTextLogs.ERROR)
2024-03-01T03:43:04.790Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-03-01T03:43:04.790Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-03-01T03:43:04.790Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 666, in _run_mysqlsh_script
2024-03-01T03:43:04.790Z [container-agent]     stdout, _ = process.wait_output()
2024-03-01T03:43:04.790Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1436, in wait_output
2024-03-01T03:43:04.790Z [container-agent]     exit_code: int = self._wait()
2024-03-01T03:43:04.790Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1375, in _wait
2024-03-01T03:43:04.790Z [container-agent]     change = self._client.wait_change(self._change_id, timeout=timeout)
2024-03-01T03:43:04.790Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1893, in wait_change
2024-03-01T03:43:04.790Z [container-agent]     return self._wait_change_using_wait(change_id, timeout)
2024-03-01T03:43:04.790Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1914, in _wait_change_using_wait
2024-03-01T03:43:04.790Z [container-agent]     return self._wait_change(change_id, this_timeout)
2024-03-01T03:43:04.790Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1928, in _wait_change
2024-03-01T03:43:04.790Z [container-agent]     resp = self._request('GET', f'/v1/changes/{change_id}/wait', query)
2024-03-01T03:43:04.790Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1655, in _request
2024-03-01T03:43:04.790Z [container-agent]     response = self._request_raw(method, path, query, headers, data)
2024-03-01T03:43:04.790Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1690, in _request_raw
2024-03-01T03:43:04.790Z [container-agent]     response = self.opener.open(request, timeout=self.timeout)
2024-03-01T03:43:04.790Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 519, in open
2024-03-01T03:43:04.790Z [container-agent]     response = self._open(req, data)
2024-03-01T03:43:04.790Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 536, in _open
2024-03-01T03:43:04.790Z [container-agent]     result = self._call_chain(self.handle_open, protocol, protocol +
2024-03-01T03:43:04.790Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 496, in _call_chain
2024-03-01T03:43:04.790Z [container-agent]     result = func(*args)
2024-03-01T03:43:04.790Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 311, in http_open
2024-03-01T03:43:04.790Z [container-agent]     return self.do_open(_UnixSocketConnection, req,  # type:ignore
2024-03-01T03:43:04.790Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 1352, in do_open
2024-03-01T03:43:04.790Z [container-agent]     r = h.getresponse()
2024-03-01T03:43:04.790Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1375, in getresponse
2024-03-01T03:43:04.790Z [container-agent]     response.begin()
2024-03-01T03:43:04.790Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 318, in begin
2024-03-01T03:43:04.790Z [container-agent]     version, status, reason = self._read_status()
2024-03-01T03:43:04.790Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 287, in _read_status
2024-03-01T03:43:04.790Z [container-agent]     raise RemoteDisconnected("Remote end closed connection without"
2024-03-01T03:43:04.790Z [container-agent] http.client.RemoteDisconnected: Remote end closed connection without response
2024-03-01T03:43:04.964Z [container-agent] 2024-03-01 03:43:04 INFO juju.util.exec exec.go:209 run result: exit status 1
2024-03-01T03:43:10.134Z [container-agent] 2024-03-01 03:43:10 INFO juju-log Setting up the logrotate configurations
2024-03-01T03:43:10.192Z [container-agent] 2024-03-01 03:43:10 INFO juju-log Adding pebble layer
2024-03-01T03:43:16.893Z [container-agent] 2024-03-01 03:43:16 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-03-01T03:43:16.914Z [container-agent] 2024-03-01 03:43:16 INFO juju-log Attempting reboot from complete outage.
2024-03-01T03:43:19.736Z [container-agent] 2024-03-01 03:43:19 INFO juju.worker.uniter.operation runhook.go:186 ran "mysql-pebble-ready" hook (via hook dispatching script: dispatch)
2024-03-01T03:45:29.307Z [container-agent] 2024-03-01 03:45:29 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T03:45:31.247Z [container-agent] 2024-03-01 03:45:31 WARNING juju-log No relation: certificates
2024-03-01T03:45:31.607Z [container-agent] 2024-03-01 03:45:31 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T03:49:41.990Z [container-agent] 2024-03-01 03:49:41 INFO juju-log Starting the log rotate manager
2024-03-01T03:49:42.003Z [container-agent] 2024-03-01 03:49:42 INFO juju-log Started log rotate manager process with PID 367220
2024-03-01T03:49:42.275Z [container-agent] 2024-03-01 03:49:42 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T03:49:44.209Z [container-agent] 2024-03-01 03:49:44 WARNING juju-log No relation: certificates
2024-03-01T03:49:44.628Z [container-agent] 2024-03-01 03:49:44 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T03:55:33.476Z [container-agent] 2024-03-01 03:55:33 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T03:55:36.109Z [container-agent] 2024-03-01 03:55:36 WARNING juju-log No relation: certificates
2024-03-01T03:55:36.466Z [container-agent] 2024-03-01 03:55:36 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T04:00:09.634Z [container-agent] 2024-03-01 04:00:09 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T04:00:12.580Z [container-agent] 2024-03-01 04:00:12 WARNING juju-log No relation: certificates
2024-03-01T04:00:12.948Z [container-agent] 2024-03-01 04:00:12 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T04:04:27.313Z [container-agent] 2024-03-01 04:04:27 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T04:04:30.694Z [container-agent] 2024-03-01 04:04:30 WARNING juju-log No relation: certificates
2024-03-01T04:04:31.052Z [container-agent] 2024-03-01 04:04:31 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T04:10:07.736Z [container-agent] 2024-03-01 04:10:07 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T04:10:11.627Z [container-agent] 2024-03-01 04:10:11 WARNING juju-log No relation: certificates
2024-03-01T04:10:11.980Z [container-agent] 2024-03-01 04:10:11 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T04:15:09.564Z [container-agent] 2024-03-01 04:15:09 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T04:15:13.835Z [container-agent] 2024-03-01 04:15:13 WARNING juju-log No relation: certificates
2024-03-01T04:15:14.196Z [container-agent] 2024-03-01 04:15:14 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T04:19:29.940Z [container-agent] 2024-03-01 04:19:29 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T04:19:35.059Z [container-agent] 2024-03-01 04:19:35 WARNING juju-log No relation: certificates
2024-03-01T04:19:35.415Z [container-agent] 2024-03-01 04:19:35 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T04:23:39.344Z [container-agent] 2024-03-01 04:23:39 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T04:23:44.571Z [container-agent] 2024-03-01 04:23:44 WARNING juju-log No relation: certificates
2024-03-01T04:23:44.942Z [container-agent] 2024-03-01 04:23:44 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T04:28:13.771Z [container-agent] 2024-03-01 04:28:13 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T04:28:19.553Z [container-agent] 2024-03-01 04:28:19 WARNING juju-log No relation: certificates
2024-03-01T04:28:19.922Z [container-agent] 2024-03-01 04:28:19 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T04:32:59.963Z [container-agent] 2024-03-01 04:32:59 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T04:33:05.321Z [container-agent] 2024-03-01 04:33:05 WARNING juju-log No relation: certificates
2024-03-01T04:33:05.650Z [container-agent] 2024-03-01 04:33:05 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T04:37:43.133Z [container-agent] 2024-03-01 04:37:43 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T04:37:49.716Z [container-agent] 2024-03-01 04:37:49 WARNING juju-log No relation: certificates
2024-03-01T04:37:50.051Z [container-agent] 2024-03-01 04:37:50 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T04:41:55.487Z [container-agent] 2024-03-01 04:41:55 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T04:42:01.866Z [container-agent] 2024-03-01 04:42:01 WARNING juju-log No relation: certificates
2024-03-01T04:42:02.236Z [container-agent] 2024-03-01 04:42:02 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T04:47:42.760Z [container-agent] 2024-03-01 04:47:42 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T04:47:50.331Z [container-agent] 2024-03-01 04:47:50 WARNING juju-log No relation: certificates
2024-03-01T04:47:50.697Z [container-agent] 2024-03-01 04:47:50 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T04:53:41.471Z [container-agent] 2024-03-01 04:53:41 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T04:53:48.930Z [container-agent] 2024-03-01 04:53:48 WARNING juju-log No relation: certificates
2024-03-01T04:53:49.297Z [container-agent] 2024-03-01 04:53:49 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T04:58:01.582Z [container-agent] 2024-03-01 04:58:01 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T04:58:09.473Z [container-agent] 2024-03-01 04:58:09 WARNING juju-log No relation: certificates
2024-03-01T04:58:09.841Z [container-agent] 2024-03-01 04:58:09 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T04:59:08.273Z [container-agent] 2024-03-01 04:59:08 ERROR juju-log Failed to flush ERROR LOGS logs.
2024-03-01T04:59:08.273Z [container-agent] Traceback (most recent call last):
2024-03-01T04:59:08.273Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 666, in _run_mysqlsh_script
2024-03-01T04:59:08.273Z [container-agent]     stdout, _ = process.wait_output()
2024-03-01T04:59:08.273Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1441, in wait_output
2024-03-01T04:59:08.273Z [container-agent]     raise ExecError[AnyStr](self._command, exit_code, out_value, err_value)
2024-03-01T04:59:08.273Z [container-agent] ops.pebble.ExecError: non-zero exit code 1 executing ['/usr/bin/mysqlsh', '--no-wizard', '--python', '--verbose=1', '-f', '/tmp/script.py', ';', 'rm', '/tmp/script.py'], stdout='', stderr='Cannot set LC_ALL to locale en_US.UTF-8: No such file or directory\nverbose: 2024-03-01T04:59:06Z: Loading startup files...\nverbose: 2024-03-01T04:59:06Z: Loading plugins...\nverbose: 2024-03-01T04:59:06Z: Connecting to MySQL at: serverconfig@horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local\nTraceback (most recent call last):\n  File "<string>", line 1, in <module>\nmysqlsh.DBError: MySQL Error (2003): Shell.connect: Can\'t connect to MySQL server on \'horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local:3306\' (111)\n'
2024-03-01T04:59:08.273Z [container-agent] 
2024-03-01T04:59:08.273Z [container-agent] During handling of the above exception, another exception occurred:
2024-03-01T04:59:08.273Z [container-agent] 
2024-03-01T04:59:08.273Z [container-agent] Traceback (most recent call last):
2024-03-01T04:59:08.273Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-03-01T04:59:08.273Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-03-01T04:59:08.273Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 669, in _run_mysqlsh_script
2024-03-01T04:59:08.273Z [container-agent]     raise MySQLClientError(e.stderr)
2024-03-01T04:59:08.273Z [container-agent] charms.mysql.v0.mysql.MySQLClientError: Cannot set LC_ALL to locale en_US.UTF-8: No such file or directory
2024-03-01T04:59:08.273Z [container-agent] verbose: 2024-03-01T04:59:06Z: Loading startup files...
2024-03-01T04:59:08.273Z [container-agent] verbose: 2024-03-01T04:59:06Z: Loading plugins...
2024-03-01T04:59:08.273Z [container-agent] verbose: 2024-03-01T04:59:06Z: Connecting to MySQL at: serverconfig@horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local
2024-03-01T04:59:08.273Z [container-agent] Traceback (most recent call last):
2024-03-01T04:59:08.273Z [container-agent]   File "<string>", line 1, in <module>
2024-03-01T04:59:08.273Z [container-agent] mysqlsh.DBError: MySQL Error (2003): Shell.connect: Can't connect to MySQL server on 'horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local:3306' (111)
2024-03-01T04:59:08.273Z [container-agent] 
2024-03-01T05:03:15.798Z [container-agent] 2024-03-01 05:03:15 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-03-01T05:03:15.830Z [container-agent] 2024-03-01 05:03:15 INFO juju-log Attempting reboot from complete outage.
2024-03-01T05:03:23.875Z [container-agent] 2024-03-01 05:03:23 WARNING juju-log No relation: certificates
2024-03-01T05:03:24.227Z [container-agent] 2024-03-01 05:03:24 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T05:04:04.799Z [container-agent] 2024-03-01 05:04:04 ERROR juju-log Uncaught exception while in charm code:
2024-03-01T05:04:04.799Z [container-agent] Traceback (most recent call last):
2024-03-01T05:04:04.799Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 2507, in exec
2024-03-01T05:04:04.799Z [container-agent]     control_ws = self._connect_websocket(task_id, 'control')
2024-03-01T05:04:04.799Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 2589, in _connect_websocket
2024-03-01T05:04:04.799Z [container-agent]     ws.connect(url, socket=sock)
2024-03-01T05:04:04.799Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_core.py", line 253, in connect
2024-03-01T05:04:04.799Z [container-agent]     self.handshake_response = handshake(self.sock, url, *addrs, **options)
2024-03-01T05:04:04.799Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_handshake.py", line 57, in handshake
2024-03-01T05:04:04.799Z [container-agent]     status, resp = _get_resp_headers(sock)
2024-03-01T05:04:04.799Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_handshake.py", line 143, in _get_resp_headers
2024-03-01T05:04:04.799Z [container-agent]     status, resp_headers, status_message = read_headers(sock)
2024-03-01T05:04:04.799Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_http.py", line 312, in read_headers
2024-03-01T05:04:04.799Z [container-agent]     line = recv_line(sock)
2024-03-01T05:04:04.799Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_socket.py", line 131, in recv_line
2024-03-01T05:04:04.799Z [container-agent]     c = recv(sock, 1)
2024-03-01T05:04:04.799Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_socket.py", line 122, in recv
2024-03-01T05:04:04.799Z [container-agent]     raise WebSocketConnectionClosedException(
2024-03-01T05:04:04.799Z [container-agent] websocket._exceptions.WebSocketConnectionClosedException: Connection to remote host was lost.
2024-03-01T05:04:04.799Z [container-agent] 
2024-03-01T05:04:04.799Z [container-agent] During handling of the above exception, another exception occurred:
2024-03-01T05:04:04.799Z [container-agent] 
2024-03-01T05:04:04.799Z [container-agent] Traceback (most recent call last):
2024-03-01T05:04:04.799Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 1348, in do_open
2024-03-01T05:04:04.799Z [container-agent]     h.request(req.get_method(), req.selector, req.data, headers,
2024-03-01T05:04:04.799Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1283, in request
2024-03-01T05:04:04.799Z [container-agent]     self._send_request(method, url, body, headers, encode_chunked)
2024-03-01T05:04:04.799Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1329, in _send_request
2024-03-01T05:04:04.799Z [container-agent]     self.endheaders(body, encode_chunked=encode_chunked)
2024-03-01T05:04:04.799Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1278, in endheaders
2024-03-01T05:04:04.799Z [container-agent]     self._send_output(message_body, encode_chunked=encode_chunked)
2024-03-01T05:04:04.799Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1038, in _send_output
2024-03-01T05:04:04.799Z [container-agent]     self.send(msg)
2024-03-01T05:04:04.799Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 976, in send
2024-03-01T05:04:04.799Z [container-agent]     self.connect()
2024-03-01T05:04:04.799Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 297, in connect
2024-03-01T05:04:04.799Z [container-agent]     self.sock.connect(self.socket_path)
2024-03-01T05:04:04.799Z [container-agent] ConnectionRefusedError: [Errno 111] Connection refused
2024-03-01T05:04:04.799Z [container-agent] 
2024-03-01T05:04:04.799Z [container-agent] During handling of the above exception, another exception occurred:
2024-03-01T05:04:04.799Z [container-agent] 
2024-03-01T05:04:04.799Z [container-agent] Traceback (most recent call last):
2024-03-01T05:04:04.799Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1690, in _request_raw
2024-03-01T05:04:04.799Z [container-agent]     response = self.opener.open(request, timeout=self.timeout)
2024-03-01T05:04:04.799Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 519, in open
2024-03-01T05:04:04.799Z [container-agent]     response = self._open(req, data)
2024-03-01T05:04:04.799Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 536, in _open
2024-03-01T05:04:04.799Z [container-agent]     result = self._call_chain(self.handle_open, protocol, protocol +
2024-03-01T05:04:04.799Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 496, in _call_chain
2024-03-01T05:04:04.799Z [container-agent]     result = func(*args)
2024-03-01T05:04:04.799Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 311, in http_open
2024-03-01T05:04:04.799Z [container-agent]     return self.do_open(_UnixSocketConnection, req,  # type:ignore
2024-03-01T05:04:04.799Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 1351, in do_open
2024-03-01T05:04:04.799Z [container-agent]     raise URLError(err)
2024-03-01T05:04:04.799Z [container-agent] urllib.error.URLError: <urlopen error [Errno 111] Connection refused>
2024-03-01T05:04:04.799Z [container-agent] 
2024-03-01T05:04:04.799Z [container-agent] The above exception was the direct cause of the following exception:
2024-03-01T05:04:04.799Z [container-agent] 
2024-03-01T05:04:04.799Z [container-agent] Traceback (most recent call last):
2024-03-01T05:04:04.799Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/./src/charm.py", line 786, in <module>
2024-03-01T05:04:04.799Z [container-agent]     main(MySQLOperatorCharm)
2024-03-01T05:04:04.799Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 436, in main
2024-03-01T05:04:04.799Z [container-agent]     _emit_charm_event(charm, dispatcher.event_name)
2024-03-01T05:04:04.799Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 144, in _emit_charm_event
2024-03-01T05:04:04.799Z [container-agent]     event_to_emit.emit(*args, **kwargs)
2024-03-01T05:04:04.799Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 351, in emit
2024-03-01T05:04:04.799Z [container-agent]     framework._emit(event)
2024-03-01T05:04:04.799Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 853, in _emit
2024-03-01T05:04:04.799Z [container-agent]     self._reemit(event_path)
2024-03-01T05:04:04.799Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 942, in _reemit
2024-03-01T05:04:04.799Z [container-agent]     custom_handler(event)
2024-03-01T05:04:04.799Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/rotate_mysql_logs.py", line 55, in _rotate_mysql_logs
2024-03-01T05:04:04.799Z [container-agent]     self.charm._mysql.flush_mysql_logs(MySQLTextLogs.ERROR)
2024-03-01T05:04:04.799Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-03-01T05:04:04.799Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-03-01T05:04:04.799Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 665, in _run_mysqlsh_script
2024-03-01T05:04:04.799Z [container-agent]     process = self.container.exec(cmd, timeout=timeout)
2024-03-01T05:04:04.799Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/model.py", line 2688, in exec
2024-03-01T05:04:04.799Z [container-agent]     return self._pebble.exec(
2024-03-01T05:04:04.799Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 2514, in exec
2024-03-01T05:04:04.799Z [container-agent]     change = self.wait_change(ChangeID(change_id))
2024-03-01T05:04:04.799Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1893, in wait_change
2024-03-01T05:04:04.799Z [container-agent]     return self._wait_change_using_wait(change_id, timeout)
2024-03-01T05:04:04.799Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1914, in _wait_change_using_wait
2024-03-01T05:04:04.799Z [container-agent]     return self._wait_change(change_id, this_timeout)
2024-03-01T05:04:04.799Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1928, in _wait_change
2024-03-01T05:04:04.799Z [container-agent]     resp = self._request('GET', f'/v1/changes/{change_id}/wait', query)
2024-03-01T05:04:04.799Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1655, in _request
2024-03-01T05:04:04.799Z [container-agent]     response = self._request_raw(method, path, query, headers, data)
2024-03-01T05:04:04.799Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1707, in _request_raw
2024-03-01T05:04:04.799Z [container-agent]     raise ConnectionError(e.reason) from e
2024-03-01T05:04:04.799Z [container-agent] ops.pebble.ConnectionError: [Errno 111] Connection refused
2024-03-01T05:04:04.995Z [container-agent] 2024-03-01 05:04:04 INFO juju.util.exec exec.go:209 run result: exit status 1
2024-03-01T05:04:08.075Z [container-agent] 2024-03-01 05:04:08 INFO juju-log Setting up the logrotate configurations
2024-03-01T05:04:08.139Z [container-agent] 2024-03-01 05:04:08 INFO juju-log Adding pebble layer
2024-03-01T05:04:14.966Z [container-agent] 2024-03-01 05:04:14 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-03-01T05:04:14.994Z [container-agent] 2024-03-01 05:04:14 INFO juju-log Attempting reboot from complete outage.
2024-03-01T05:04:18.075Z [container-agent] 2024-03-01 05:04:18 INFO juju.worker.uniter.operation runhook.go:186 ran "mysql-pebble-ready" hook (via hook dispatching script: dispatch)
2024-03-01T05:07:44.904Z [container-agent] 2024-03-01 05:07:44 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T05:07:46.762Z [container-agent] 2024-03-01 05:07:46 WARNING juju-log No relation: certificates
2024-03-01T05:07:47.114Z [container-agent] 2024-03-01 05:07:47 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T05:13:18.662Z [container-agent] 2024-03-01 05:13:18 INFO juju-log Starting the log rotate manager
2024-03-01T05:13:18.675Z [container-agent] 2024-03-01 05:13:18 INFO juju-log Started log rotate manager process with PID 374938
2024-03-01T05:13:18.960Z [container-agent] 2024-03-01 05:13:18 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T05:13:21.062Z [container-agent] 2024-03-01 05:13:21 WARNING juju-log No relation: certificates
2024-03-01T05:13:21.456Z [container-agent] 2024-03-01 05:13:21 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T05:17:27.854Z [container-agent] 2024-03-01 05:17:27 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T05:17:30.279Z [container-agent] 2024-03-01 05:17:30 WARNING juju-log No relation: certificates
2024-03-01T05:17:30.621Z [container-agent] 2024-03-01 05:17:30 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T05:22:06.553Z [container-agent] 2024-03-01 05:22:06 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T05:22:09.420Z [container-agent] 2024-03-01 05:22:09 WARNING juju-log No relation: certificates
2024-03-01T05:22:09.759Z [container-agent] 2024-03-01 05:22:09 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T05:26:07.999Z [container-agent] 2024-03-01 05:26:07 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T05:26:11.125Z [container-agent] 2024-03-01 05:26:11 WARNING juju-log No relation: certificates
2024-03-01T05:26:11.483Z [container-agent] 2024-03-01 05:26:11 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T05:30:13.128Z [container-agent] 2024-03-01 05:30:13 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T05:30:16.936Z [container-agent] 2024-03-01 05:30:16 WARNING juju-log No relation: certificates
2024-03-01T05:30:17.293Z [container-agent] 2024-03-01 05:30:17 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T05:36:00.771Z [container-agent] 2024-03-01 05:36:00 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T05:36:04.817Z [container-agent] 2024-03-01 05:36:04 WARNING juju-log No relation: certificates
2024-03-01T05:36:05.220Z [container-agent] 2024-03-01 05:36:05 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T05:41:36.941Z [container-agent] 2024-03-01 05:41:36 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T05:41:41.267Z [container-agent] 2024-03-01 05:41:41 WARNING juju-log No relation: certificates
2024-03-01T05:41:41.631Z [container-agent] 2024-03-01 05:41:41 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T05:46:31.024Z [container-agent] 2024-03-01 05:46:31 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T05:46:36.395Z [container-agent] 2024-03-01 05:46:36 WARNING juju-log No relation: certificates
2024-03-01T05:46:36.800Z [container-agent] 2024-03-01 05:46:36 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T05:51:00.275Z [container-agent] 2024-03-01 05:51:00 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T05:51:05.396Z [container-agent] 2024-03-01 05:51:05 WARNING juju-log No relation: certificates
2024-03-01T05:51:05.760Z [container-agent] 2024-03-01 05:51:05 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T05:55:36.154Z [container-agent] 2024-03-01 05:55:36 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T05:55:41.454Z [container-agent] 2024-03-01 05:55:41 WARNING juju-log No relation: certificates
2024-03-01T05:55:41.775Z [container-agent] 2024-03-01 05:55:41 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T06:01:11.247Z [container-agent] 2024-03-01 06:01:11 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T06:01:17.959Z [container-agent] 2024-03-01 06:01:17 WARNING juju-log No relation: certificates
2024-03-01T06:01:18.326Z [container-agent] 2024-03-01 06:01:18 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T06:06:11.565Z [container-agent] 2024-03-01 06:06:11 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T06:06:18.560Z [container-agent] 2024-03-01 06:06:18 WARNING juju-log No relation: certificates
2024-03-01T06:06:18.900Z [container-agent] 2024-03-01 06:06:18 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T06:10:40.464Z [container-agent] 2024-03-01 06:10:40 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T06:10:47.445Z [container-agent] 2024-03-01 06:10:47 WARNING juju-log No relation: certificates
2024-03-01T06:10:47.782Z [container-agent] 2024-03-01 06:10:47 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T06:15:14.028Z [container-agent] 2024-03-01 06:15:14 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T06:15:21.833Z [container-agent] 2024-03-01 06:15:21 WARNING juju-log No relation: certificates
2024-03-01T06:15:22.177Z [container-agent] 2024-03-01 06:15:22 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T06:19:07.773Z [container-agent] 2024-03-01 06:19:07 ERROR juju-log Failed to flush ERROR LOGS logs.
2024-03-01T06:19:07.773Z [container-agent] Traceback (most recent call last):
2024-03-01T06:19:07.773Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 666, in _run_mysqlsh_script
2024-03-01T06:19:07.773Z [container-agent]     stdout, _ = process.wait_output()
2024-03-01T06:19:07.773Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1441, in wait_output
2024-03-01T06:19:07.773Z [container-agent]     raise ExecError[AnyStr](self._command, exit_code, out_value, err_value)
2024-03-01T06:19:07.773Z [container-agent] ops.pebble.ExecError: non-zero exit code 1 executing ['/usr/bin/mysqlsh', '--no-wizard', '--python', '--verbose=1', '-f', '/tmp/script.py', ';', 'rm', '/tmp/script.py'], stdout='', stderr='Cannot set LC_ALL to locale en_US.UTF-8: No such file or directory\nverbose: 2024-03-01T06:19:06Z: Loading startup files...\nverbose: 2024-03-01T06:19:06Z: Loading plugins...\nverbose: 2024-03-01T06:19:06Z: Connecting to MySQL at: serverconfig@horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local\nTraceback (most recent call last):\n  File "<string>", line 1, in <module>\nmysqlsh.DBError: MySQL Error (2003): Shell.connect: Can\'t connect to MySQL server on \'horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local:3306\' (111)\n'
2024-03-01T06:19:07.773Z [container-agent] 
2024-03-01T06:19:07.773Z [container-agent] During handling of the above exception, another exception occurred:
2024-03-01T06:19:07.773Z [container-agent] 
2024-03-01T06:19:07.773Z [container-agent] Traceback (most recent call last):
2024-03-01T06:19:07.773Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-03-01T06:19:07.773Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-03-01T06:19:07.773Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 669, in _run_mysqlsh_script
2024-03-01T06:19:07.773Z [container-agent]     raise MySQLClientError(e.stderr)
2024-03-01T06:19:07.773Z [container-agent] charms.mysql.v0.mysql.MySQLClientError: Cannot set LC_ALL to locale en_US.UTF-8: No such file or directory
2024-03-01T06:19:07.773Z [container-agent] verbose: 2024-03-01T06:19:06Z: Loading startup files...
2024-03-01T06:19:07.773Z [container-agent] verbose: 2024-03-01T06:19:06Z: Loading plugins...
2024-03-01T06:19:07.773Z [container-agent] verbose: 2024-03-01T06:19:06Z: Connecting to MySQL at: serverconfig@horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local
2024-03-01T06:19:07.773Z [container-agent] Traceback (most recent call last):
2024-03-01T06:19:07.773Z [container-agent]   File "<string>", line 1, in <module>
2024-03-01T06:19:07.773Z [container-agent] mysqlsh.DBError: MySQL Error (2003): Shell.connect: Can't connect to MySQL server on 'horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local:3306' (111)
2024-03-01T06:19:07.773Z [container-agent] 
2024-03-01T06:20:35.593Z [container-agent] 2024-03-01 06:20:35 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-03-01T06:20:35.621Z [container-agent] 2024-03-01 06:20:35 INFO juju-log Attempting reboot from complete outage.
2024-03-01T06:20:43.037Z [container-agent] 2024-03-01 06:20:43 WARNING juju-log No relation: certificates
2024-03-01T06:20:43.389Z [container-agent] 2024-03-01 06:20:43 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T06:24:04.807Z [container-agent] 2024-03-01 06:24:04 ERROR juju-log Uncaught exception while in charm code:
2024-03-01T06:24:04.807Z [container-agent] Traceback (most recent call last):
2024-03-01T06:24:04.807Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 2507, in exec
2024-03-01T06:24:04.807Z [container-agent]     control_ws = self._connect_websocket(task_id, 'control')
2024-03-01T06:24:04.807Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 2589, in _connect_websocket
2024-03-01T06:24:04.807Z [container-agent]     ws.connect(url, socket=sock)
2024-03-01T06:24:04.807Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_core.py", line 253, in connect
2024-03-01T06:24:04.807Z [container-agent]     self.handshake_response = handshake(self.sock, url, *addrs, **options)
2024-03-01T06:24:04.807Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_handshake.py", line 57, in handshake
2024-03-01T06:24:04.807Z [container-agent]     status, resp = _get_resp_headers(sock)
2024-03-01T06:24:04.807Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_handshake.py", line 143, in _get_resp_headers
2024-03-01T06:24:04.807Z [container-agent]     status, resp_headers, status_message = read_headers(sock)
2024-03-01T06:24:04.807Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_http.py", line 312, in read_headers
2024-03-01T06:24:04.807Z [container-agent]     line = recv_line(sock)
2024-03-01T06:24:04.807Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_socket.py", line 131, in recv_line
2024-03-01T06:24:04.807Z [container-agent]     c = recv(sock, 1)
2024-03-01T06:24:04.807Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_socket.py", line 122, in recv
2024-03-01T06:24:04.807Z [container-agent]     raise WebSocketConnectionClosedException(
2024-03-01T06:24:04.807Z [container-agent] websocket._exceptions.WebSocketConnectionClosedException: Connection to remote host was lost.
2024-03-01T06:24:04.807Z [container-agent] 
2024-03-01T06:24:04.807Z [container-agent] During handling of the above exception, another exception occurred:
2024-03-01T06:24:04.807Z [container-agent] 
2024-03-01T06:24:04.807Z [container-agent] Traceback (most recent call last):
2024-03-01T06:24:04.807Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 1348, in do_open
2024-03-01T06:24:04.807Z [container-agent]     h.request(req.get_method(), req.selector, req.data, headers,
2024-03-01T06:24:04.807Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1283, in request
2024-03-01T06:24:04.807Z [container-agent]     self._send_request(method, url, body, headers, encode_chunked)
2024-03-01T06:24:04.807Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1329, in _send_request
2024-03-01T06:24:04.807Z [container-agent]     self.endheaders(body, encode_chunked=encode_chunked)
2024-03-01T06:24:04.807Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1278, in endheaders
2024-03-01T06:24:04.807Z [container-agent]     self._send_output(message_body, encode_chunked=encode_chunked)
2024-03-01T06:24:04.807Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1038, in _send_output
2024-03-01T06:24:04.807Z [container-agent]     self.send(msg)
2024-03-01T06:24:04.807Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 976, in send
2024-03-01T06:24:04.807Z [container-agent]     self.connect()
2024-03-01T06:24:04.807Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 297, in connect
2024-03-01T06:24:04.807Z [container-agent]     self.sock.connect(self.socket_path)
2024-03-01T06:24:04.807Z [container-agent] ConnectionRefusedError: [Errno 111] Connection refused
2024-03-01T06:24:04.807Z [container-agent] 
2024-03-01T06:24:04.807Z [container-agent] During handling of the above exception, another exception occurred:
2024-03-01T06:24:04.807Z [container-agent] 
2024-03-01T06:24:04.807Z [container-agent] Traceback (most recent call last):
2024-03-01T06:24:04.807Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1690, in _request_raw
2024-03-01T06:24:04.807Z [container-agent]     response = self.opener.open(request, timeout=self.timeout)
2024-03-01T06:24:04.807Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 519, in open
2024-03-01T06:24:04.807Z [container-agent]     response = self._open(req, data)
2024-03-01T06:24:04.807Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 536, in _open
2024-03-01T06:24:04.807Z [container-agent]     result = self._call_chain(self.handle_open, protocol, protocol +
2024-03-01T06:24:04.807Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 496, in _call_chain
2024-03-01T06:24:04.807Z [container-agent]     result = func(*args)
2024-03-01T06:24:04.807Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 311, in http_open
2024-03-01T06:24:04.807Z [container-agent]     return self.do_open(_UnixSocketConnection, req,  # type:ignore
2024-03-01T06:24:04.807Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 1351, in do_open
2024-03-01T06:24:04.807Z [container-agent]     raise URLError(err)
2024-03-01T06:24:04.807Z [container-agent] urllib.error.URLError: <urlopen error [Errno 111] Connection refused>
2024-03-01T06:24:04.807Z [container-agent] 
2024-03-01T06:24:04.807Z [container-agent] The above exception was the direct cause of the following exception:
2024-03-01T06:24:04.807Z [container-agent] 
2024-03-01T06:24:04.807Z [container-agent] Traceback (most recent call last):
2024-03-01T06:24:04.807Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/./src/charm.py", line 786, in <module>
2024-03-01T06:24:04.807Z [container-agent]     main(MySQLOperatorCharm)
2024-03-01T06:24:04.807Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 436, in main
2024-03-01T06:24:04.807Z [container-agent]     _emit_charm_event(charm, dispatcher.event_name)
2024-03-01T06:24:04.807Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 144, in _emit_charm_event
2024-03-01T06:24:04.807Z [container-agent]     event_to_emit.emit(*args, **kwargs)
2024-03-01T06:24:04.807Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 351, in emit
2024-03-01T06:24:04.807Z [container-agent]     framework._emit(event)
2024-03-01T06:24:04.807Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 853, in _emit
2024-03-01T06:24:04.807Z [container-agent]     self._reemit(event_path)
2024-03-01T06:24:04.807Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 942, in _reemit
2024-03-01T06:24:04.807Z [container-agent]     custom_handler(event)
2024-03-01T06:24:04.807Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/rotate_mysql_logs.py", line 55, in _rotate_mysql_logs
2024-03-01T06:24:04.807Z [container-agent]     self.charm._mysql.flush_mysql_logs(MySQLTextLogs.ERROR)
2024-03-01T06:24:04.807Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-03-01T06:24:04.807Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-03-01T06:24:04.807Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 665, in _run_mysqlsh_script
2024-03-01T06:24:04.807Z [container-agent]     process = self.container.exec(cmd, timeout=timeout)
2024-03-01T06:24:04.807Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/model.py", line 2688, in exec
2024-03-01T06:24:04.807Z [container-agent]     return self._pebble.exec(
2024-03-01T06:24:04.807Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 2514, in exec
2024-03-01T06:24:04.807Z [container-agent]     change = self.wait_change(ChangeID(change_id))
2024-03-01T06:24:04.807Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1893, in wait_change
2024-03-01T06:24:04.807Z [container-agent]     return self._wait_change_using_wait(change_id, timeout)
2024-03-01T06:24:04.807Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1914, in _wait_change_using_wait
2024-03-01T06:24:04.807Z [container-agent]     return self._wait_change(change_id, this_timeout)
2024-03-01T06:24:04.807Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1928, in _wait_change
2024-03-01T06:24:04.807Z [container-agent]     resp = self._request('GET', f'/v1/changes/{change_id}/wait', query)
2024-03-01T06:24:04.807Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1655, in _request
2024-03-01T06:24:04.807Z [container-agent]     response = self._request_raw(method, path, query, headers, data)
2024-03-01T06:24:04.807Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1707, in _request_raw
2024-03-01T06:24:04.807Z [container-agent]     raise ConnectionError(e.reason) from e
2024-03-01T06:24:04.807Z [container-agent] ops.pebble.ConnectionError: [Errno 111] Connection refused
2024-03-01T06:24:04.960Z [container-agent] 2024-03-01 06:24:04 INFO juju.util.exec exec.go:209 run result: exit status 1
2024-03-01T06:24:06.777Z [container-agent] 2024-03-01 06:24:06 INFO juju-log Setting up the logrotate configurations
2024-03-01T06:24:06.841Z [container-agent] 2024-03-01 06:24:06 INFO juju-log Adding pebble layer
2024-03-01T06:24:13.615Z [container-agent] 2024-03-01 06:24:13 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-03-01T06:24:13.642Z [container-agent] 2024-03-01 06:24:13 INFO juju-log Attempting reboot from complete outage.
2024-03-01T06:24:16.680Z [container-agent] 2024-03-01 06:24:16 INFO juju.worker.uniter.operation runhook.go:186 ran "mysql-pebble-ready" hook (via hook dispatching script: dispatch)
2024-03-01T06:25:00.273Z [container-agent] 2024-03-01 06:25:00 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T06:25:02.250Z [container-agent] 2024-03-01 06:25:02 WARNING juju-log No relation: certificates
2024-03-01T06:25:02.596Z [container-agent] 2024-03-01 06:25:02 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T06:30:22.641Z [container-agent] 2024-03-01 06:30:22 INFO juju-log Starting the log rotate manager
2024-03-01T06:30:22.656Z [container-agent] 2024-03-01 06:30:22 INFO juju-log Started log rotate manager process with PID 382210
2024-03-01T06:30:22.952Z [container-agent] 2024-03-01 06:30:22 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T06:30:24.954Z [container-agent] 2024-03-01 06:30:24 WARNING juju-log No relation: certificates
2024-03-01T06:30:25.289Z [container-agent] 2024-03-01 06:30:25 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T06:36:14.396Z [container-agent] 2024-03-01 06:36:14 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T06:36:17.109Z [container-agent] 2024-03-01 06:36:17 WARNING juju-log No relation: certificates
2024-03-01T06:36:17.465Z [container-agent] 2024-03-01 06:36:17 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T06:40:25.362Z [container-agent] 2024-03-01 06:40:25 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T06:40:28.352Z [container-agent] 2024-03-01 06:40:28 WARNING juju-log No relation: certificates
2024-03-01T06:40:28.760Z [container-agent] 2024-03-01 06:40:28 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T06:46:20.583Z [container-agent] 2024-03-01 06:46:20 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T06:46:23.974Z [container-agent] 2024-03-01 06:46:23 WARNING juju-log No relation: certificates
2024-03-01T06:46:24.322Z [container-agent] 2024-03-01 06:46:24 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T06:52:10.705Z [container-agent] 2024-03-01 06:52:10 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T06:52:14.710Z [container-agent] 2024-03-01 06:52:14 WARNING juju-log No relation: certificates
2024-03-01T06:52:15.080Z [container-agent] 2024-03-01 06:52:15 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T06:57:53.893Z [container-agent] 2024-03-01 06:57:53 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T06:57:58.405Z [container-agent] 2024-03-01 06:57:58 WARNING juju-log No relation: certificates
2024-03-01T06:57:58.805Z [container-agent] 2024-03-01 06:57:58 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T07:03:44.609Z [container-agent] 2024-03-01 07:03:44 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T07:03:49.763Z [container-agent] 2024-03-01 07:03:49 WARNING juju-log No relation: certificates
2024-03-01T07:03:50.123Z [container-agent] 2024-03-01 07:03:50 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T07:08:10.213Z [container-agent] 2024-03-01 07:08:10 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T07:08:15.843Z [container-agent] 2024-03-01 07:08:15 WARNING juju-log No relation: certificates
2024-03-01T07:08:16.194Z [container-agent] 2024-03-01 07:08:16 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T07:12:11.145Z [container-agent] 2024-03-01 07:12:11 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T07:12:17.358Z [container-agent] 2024-03-01 07:12:17 WARNING juju-log No relation: certificates
2024-03-01T07:12:17.743Z [container-agent] 2024-03-01 07:12:17 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T07:18:00.808Z [container-agent] 2024-03-01 07:18:00 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T07:18:06.750Z [container-agent] 2024-03-01 07:18:06 WARNING juju-log No relation: certificates
2024-03-01T07:18:07.143Z [container-agent] 2024-03-01 07:18:07 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T07:23:39.972Z [container-agent] 2024-03-01 07:23:39 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T07:23:47.080Z [container-agent] 2024-03-01 07:23:47 WARNING juju-log No relation: certificates
2024-03-01T07:23:47.421Z [container-agent] 2024-03-01 07:23:47 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T07:28:12.909Z [container-agent] 2024-03-01 07:28:12 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T07:28:20.451Z [container-agent] 2024-03-01 07:28:20 WARNING juju-log No relation: certificates
2024-03-01T07:28:20.815Z [container-agent] 2024-03-01 07:28:20 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T07:32:59.340Z [container-agent] 2024-03-01 07:32:59 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T07:33:07.221Z [container-agent] 2024-03-01 07:33:07 WARNING juju-log No relation: certificates
2024-03-01T07:33:07.569Z [container-agent] 2024-03-01 07:33:07 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T07:37:24.205Z [container-agent] 2024-03-01 07:37:24 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T07:37:32.189Z [container-agent] 2024-03-01 07:37:32 WARNING juju-log No relation: certificates
2024-03-01T07:37:32.530Z [container-agent] 2024-03-01 07:37:32 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T07:41:11.229Z [container-agent] 2024-03-01 07:41:11 ERROR juju-log Failed to flush GENERAL LOGS logs.
2024-03-01T07:41:11.229Z [container-agent] Traceback (most recent call last):
2024-03-01T07:41:11.229Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 666, in _run_mysqlsh_script
2024-03-01T07:41:11.229Z [container-agent]     stdout, _ = process.wait_output()
2024-03-01T07:41:11.229Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1441, in wait_output
2024-03-01T07:41:11.229Z [container-agent]     raise ExecError[AnyStr](self._command, exit_code, out_value, err_value)
2024-03-01T07:41:11.229Z [container-agent] ops.pebble.ExecError: non-zero exit code 1 executing ['/usr/bin/mysqlsh', '--no-wizard', '--python', '--verbose=1', '-f', '/tmp/script.py', ';', 'rm', '/tmp/script.py'], stdout='', stderr='Cannot set LC_ALL to locale en_US.UTF-8: No such file or directory\nverbose: 2024-03-01T07:41:08Z: Loading startup files...\nverbose: 2024-03-01T07:41:08Z: Loading plugins...\nverbose: 2024-03-01T07:41:08Z: Connecting to MySQL at: serverconfig@horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local\nTraceback (most recent call last):\n  File "<string>", line 1, in <module>\nmysqlsh.DBError: MySQL Error (2013): Shell.connect: Lost connection to MySQL server at \'reading initial communication packet\', system error: 104\n'
2024-03-01T07:41:11.229Z [container-agent] 
2024-03-01T07:41:11.229Z [container-agent] During handling of the above exception, another exception occurred:
2024-03-01T07:41:11.229Z [container-agent] 
2024-03-01T07:41:11.229Z [container-agent] Traceback (most recent call last):
2024-03-01T07:41:11.229Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-03-01T07:41:11.229Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-03-01T07:41:11.229Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 669, in _run_mysqlsh_script
2024-03-01T07:41:11.229Z [container-agent]     raise MySQLClientError(e.stderr)
2024-03-01T07:41:11.229Z [container-agent] charms.mysql.v0.mysql.MySQLClientError: Cannot set LC_ALL to locale en_US.UTF-8: No such file or directory
2024-03-01T07:41:11.229Z [container-agent] verbose: 2024-03-01T07:41:08Z: Loading startup files...
2024-03-01T07:41:11.229Z [container-agent] verbose: 2024-03-01T07:41:08Z: Loading plugins...
2024-03-01T07:41:11.229Z [container-agent] verbose: 2024-03-01T07:41:08Z: Connecting to MySQL at: serverconfig@horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local
2024-03-01T07:41:11.229Z [container-agent] Traceback (most recent call last):
2024-03-01T07:41:11.229Z [container-agent]   File "<string>", line 1, in <module>
2024-03-01T07:41:11.229Z [container-agent] mysqlsh.DBError: MySQL Error (2013): Shell.connect: Lost connection to MySQL server at 'reading initial communication packet', system error: 104
2024-03-01T07:41:11.229Z [container-agent] 
2024-03-01T07:41:14.419Z [container-agent] 2024-03-01 07:41:14 ERROR juju-log Failed to flush SLOW LOGS logs.
2024-03-01T07:41:14.419Z [container-agent] Traceback (most recent call last):
2024-03-01T07:41:14.419Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 666, in _run_mysqlsh_script
2024-03-01T07:41:14.419Z [container-agent]     stdout, _ = process.wait_output()
2024-03-01T07:41:14.419Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1441, in wait_output
2024-03-01T07:41:14.419Z [container-agent]     raise ExecError[AnyStr](self._command, exit_code, out_value, err_value)
2024-03-01T07:41:14.419Z [container-agent] ops.pebble.ExecError: non-zero exit code 1 executing ['/usr/bin/mysqlsh', '--no-wizard', '--python', '--verbose=1', '-f', '/tmp/script.py', ';', 'rm', '/tmp/script.py'], stdout='', stderr='Cannot set LC_ALL to locale en_US.UTF-8: No such file or directory\nverbose: 2024-03-01T07:41:12Z: Loading startup files...\nverbose: 2024-03-01T07:41:12Z: Loading plugins...\nverbose: 2024-03-01T07:41:12Z: Connecting to MySQL at: serverconfig@horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local\nTraceback (most recent call last):\n  File "<string>", line 1, in <module>\nmysqlsh.DBError: MySQL Error (2003): Shell.connect: Can\'t connect to MySQL server on \'horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local:3306\' (111)\n'
2024-03-01T07:41:14.419Z [container-agent] 
2024-03-01T07:41:14.419Z [container-agent] During handling of the above exception, another exception occurred:
2024-03-01T07:41:14.419Z [container-agent] 
2024-03-01T07:41:14.419Z [container-agent] Traceback (most recent call last):
2024-03-01T07:41:14.419Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-03-01T07:41:14.419Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-03-01T07:41:14.419Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 669, in _run_mysqlsh_script
2024-03-01T07:41:14.419Z [container-agent]     raise MySQLClientError(e.stderr)
2024-03-01T07:41:14.419Z [container-agent] charms.mysql.v0.mysql.MySQLClientError: Cannot set LC_ALL to locale en_US.UTF-8: No such file or directory
2024-03-01T07:41:14.419Z [container-agent] verbose: 2024-03-01T07:41:12Z: Loading startup files...
2024-03-01T07:41:14.419Z [container-agent] verbose: 2024-03-01T07:41:12Z: Loading plugins...
2024-03-01T07:41:14.419Z [container-agent] verbose: 2024-03-01T07:41:12Z: Connecting to MySQL at: serverconfig@horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local
2024-03-01T07:41:14.419Z [container-agent] Traceback (most recent call last):
2024-03-01T07:41:14.419Z [container-agent]   File "<string>", line 1, in <module>
2024-03-01T07:41:14.419Z [container-agent] mysqlsh.DBError: MySQL Error (2003): Shell.connect: Can't connect to MySQL server on 'horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local:3306' (111)
2024-03-01T07:41:14.419Z [container-agent] 
2024-03-01T07:41:19.585Z [container-agent] 2024-03-01 07:41:19 INFO juju-log Setting up the logrotate configurations
2024-03-01T07:41:19.665Z [container-agent] 2024-03-01 07:41:19 INFO juju-log Adding pebble layer
2024-03-01T07:41:26.389Z [container-agent] 2024-03-01 07:41:26 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-03-01T07:41:26.418Z [container-agent] 2024-03-01 07:41:26 INFO juju-log Attempting reboot from complete outage.
2024-03-01T07:41:29.429Z [container-agent] 2024-03-01 07:41:29 INFO juju.worker.uniter.operation runhook.go:186 ran "mysql-pebble-ready" hook (via hook dispatching script: dispatch)
2024-03-01T07:42:20.389Z [container-agent] 2024-03-01 07:42:20 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T07:42:22.237Z [container-agent] 2024-03-01 07:42:22 WARNING juju-log No relation: certificates
2024-03-01T07:42:22.597Z [container-agent] 2024-03-01 07:42:22 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T07:46:35.849Z [container-agent] 2024-03-01 07:46:35 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T07:46:38.052Z [container-agent] 2024-03-01 07:46:38 WARNING juju-log No relation: certificates
2024-03-01T07:46:38.432Z [container-agent] 2024-03-01 07:46:38 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T07:51:07.707Z [container-agent] 2024-03-01 07:51:07 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T07:51:10.499Z [container-agent] 2024-03-01 07:51:10 WARNING juju-log No relation: certificates
2024-03-01T07:51:10.856Z [container-agent] 2024-03-01 07:51:10 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T07:55:17.261Z [container-agent] 2024-03-01 07:55:17 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T07:55:20.806Z [container-agent] 2024-03-01 07:55:20 WARNING juju-log No relation: certificates
2024-03-01T07:55:21.151Z [container-agent] 2024-03-01 07:55:21 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T07:59:53.128Z [container-agent] 2024-03-01 07:59:53 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T07:59:56.557Z [container-agent] 2024-03-01 07:59:56 WARNING juju-log No relation: certificates
2024-03-01T07:59:56.938Z [container-agent] 2024-03-01 07:59:56 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T08:05:12.755Z [container-agent] 2024-03-01 08:05:12 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T08:05:16.971Z [container-agent] 2024-03-01 08:05:16 WARNING juju-log No relation: certificates
2024-03-01T08:05:17.344Z [container-agent] 2024-03-01 08:05:17 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T08:11:09.470Z [container-agent] 2024-03-01 08:11:09 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T08:11:14.176Z [container-agent] 2024-03-01 08:11:14 WARNING juju-log No relation: certificates
2024-03-01T08:11:14.541Z [container-agent] 2024-03-01 08:11:14 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T08:15:45.238Z [container-agent] 2024-03-01 08:15:45 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T08:15:50.310Z [container-agent] 2024-03-01 08:15:50 WARNING juju-log No relation: certificates
2024-03-01T08:15:50.662Z [container-agent] 2024-03-01 08:15:50 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T08:20:52.609Z [container-agent] 2024-03-01 08:20:52 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T08:20:57.828Z [container-agent] 2024-03-01 08:20:57 WARNING juju-log No relation: certificates
2024-03-01T08:20:58.188Z [container-agent] 2024-03-01 08:20:58 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T08:25:28.634Z [container-agent] 2024-03-01 08:25:28 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T08:25:34.879Z [container-agent] 2024-03-01 08:25:34 WARNING juju-log No relation: certificates
2024-03-01T08:25:35.223Z [container-agent] 2024-03-01 08:25:35 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T08:30:21.921Z [container-agent] 2024-03-01 08:30:21 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T08:30:28.329Z [container-agent] 2024-03-01 08:30:28 WARNING juju-log No relation: certificates
2024-03-01T08:30:28.688Z [container-agent] 2024-03-01 08:30:28 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T08:34:34.714Z [container-agent] 2024-03-01 08:34:34 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T08:34:41.562Z [container-agent] 2024-03-01 08:34:41 WARNING juju-log No relation: certificates
2024-03-01T08:34:41.924Z [container-agent] 2024-03-01 08:34:41 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T08:38:40.217Z [container-agent] 2024-03-01 08:38:40 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T08:38:47.452Z [container-agent] 2024-03-01 08:38:47 WARNING juju-log No relation: certificates
2024-03-01T08:38:47.822Z [container-agent] 2024-03-01 08:38:47 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T08:44:13.953Z [container-agent] 2024-03-01 08:44:13 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T08:44:21.838Z [container-agent] 2024-03-01 08:44:21 WARNING juju-log No relation: certificates
2024-03-01T08:44:22.218Z [container-agent] 2024-03-01 08:44:22 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T08:47:15.743Z [container-agent] 2024-03-01 08:47:15 ERROR juju-log Failed to flush SLOW LOGS logs.
2024-03-01T08:47:15.743Z [container-agent] Traceback (most recent call last):
2024-03-01T08:47:15.743Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 666, in _run_mysqlsh_script
2024-03-01T08:47:15.743Z [container-agent]     stdout, _ = process.wait_output()
2024-03-01T08:47:15.743Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1441, in wait_output
2024-03-01T08:47:15.743Z [container-agent]     raise ExecError[AnyStr](self._command, exit_code, out_value, err_value)
2024-03-01T08:47:15.743Z [container-agent] ops.pebble.ExecError: non-zero exit code 1 executing ['/usr/bin/mysqlsh', '--no-wizard', '--python', '--verbose=1', '-f', '/tmp/script.py', ';', 'rm', '/tmp/script.py'], stdout='', stderr='Cannot set LC_ALL to locale en_US.UTF-8: No such file or directory\nverbose: 2024-03-01T08:47:11Z: Loading startup files...\nverbose: 2024-03-01T08:47:11Z: Loading plugins...\nverbose: 2024-03-01T08:47:11Z: Connecting to MySQL at: serverconfig@horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local\nTraceback (most recent call last):\n  File "<string>", line 1, in <module>\nmysqlsh.DBError: MySQL Error (2013): Shell.connect: Lost connection to MySQL server at \'reading initial communication packet\', system error: 104\n'
2024-03-01T08:47:15.743Z [container-agent] 
2024-03-01T08:47:15.743Z [container-agent] During handling of the above exception, another exception occurred:
2024-03-01T08:47:15.743Z [container-agent] 
2024-03-01T08:47:15.743Z [container-agent] Traceback (most recent call last):
2024-03-01T08:47:15.743Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-03-01T08:47:15.743Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-03-01T08:47:15.743Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 669, in _run_mysqlsh_script
2024-03-01T08:47:15.743Z [container-agent]     raise MySQLClientError(e.stderr)
2024-03-01T08:47:15.743Z [container-agent] charms.mysql.v0.mysql.MySQLClientError: Cannot set LC_ALL to locale en_US.UTF-8: No such file or directory
2024-03-01T08:47:15.743Z [container-agent] verbose: 2024-03-01T08:47:11Z: Loading startup files...
2024-03-01T08:47:15.743Z [container-agent] verbose: 2024-03-01T08:47:11Z: Loading plugins...
2024-03-01T08:47:15.743Z [container-agent] verbose: 2024-03-01T08:47:11Z: Connecting to MySQL at: serverconfig@horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local
2024-03-01T08:47:15.743Z [container-agent] Traceback (most recent call last):
2024-03-01T08:47:15.743Z [container-agent]   File "<string>", line 1, in <module>
2024-03-01T08:47:15.743Z [container-agent] mysqlsh.DBError: MySQL Error (2013): Shell.connect: Lost connection to MySQL server at 'reading initial communication packet', system error: 104
2024-03-01T08:47:15.743Z [container-agent] 
2024-03-01T08:47:22.528Z [container-agent] 2024-03-01 08:47:22 INFO juju-log Setting up the logrotate configurations
2024-03-01T08:47:22.590Z [container-agent] 2024-03-01 08:47:22 INFO juju-log Adding pebble layer
2024-03-01T08:47:29.340Z [container-agent] 2024-03-01 08:47:29 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-03-01T08:47:29.371Z [container-agent] 2024-03-01 08:47:29 INFO juju-log Attempting reboot from complete outage.
2024-03-01T08:47:32.446Z [container-agent] 2024-03-01 08:47:32 INFO juju.worker.uniter.operation runhook.go:186 ran "mysql-pebble-ready" hook (via hook dispatching script: dispatch)
2024-03-01T08:49:25.549Z [container-agent] 2024-03-01 08:49:25 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T08:49:27.607Z [container-agent] 2024-03-01 08:49:27 WARNING juju-log No relation: certificates
2024-03-01T08:49:27.964Z [container-agent] 2024-03-01 08:49:27 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T08:55:09.528Z [container-agent] 2024-03-01 08:55:09 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T08:55:12.161Z [container-agent] 2024-03-01 08:55:12 WARNING juju-log No relation: certificates
2024-03-01T08:55:12.527Z [container-agent] 2024-03-01 08:55:12 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T08:59:47.374Z [container-agent] 2024-03-01 08:59:47 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T08:59:50.495Z [container-agent] 2024-03-01 08:59:50 WARNING juju-log No relation: certificates
2024-03-01T08:59:50.844Z [container-agent] 2024-03-01 08:59:50 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T09:05:41.818Z [container-agent] 2024-03-01 09:05:41 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T09:05:45.378Z [container-agent] 2024-03-01 09:05:45 WARNING juju-log No relation: certificates
2024-03-01T09:05:45.730Z [container-agent] 2024-03-01 09:05:45 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T09:10:36.529Z [container-agent] 2024-03-01 09:10:36 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T09:10:40.400Z [container-agent] 2024-03-01 09:10:40 WARNING juju-log No relation: certificates
2024-03-01T09:10:40.749Z [container-agent] 2024-03-01 09:10:40 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T09:15:51.150Z [container-agent] 2024-03-01 09:15:51 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T09:15:55.154Z [container-agent] 2024-03-01 09:15:55 WARNING juju-log No relation: certificates
2024-03-01T09:15:55.486Z [container-agent] 2024-03-01 09:15:55 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T09:20:59.413Z [container-agent] 2024-03-01 09:20:59 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T09:21:04.142Z [container-agent] 2024-03-01 09:21:04 WARNING juju-log No relation: certificates
2024-03-01T09:21:04.473Z [container-agent] 2024-03-01 09:21:04 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T09:25:39.224Z [container-agent] 2024-03-01 09:25:39 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T09:25:44.321Z [container-agent] 2024-03-01 09:25:44 WARNING juju-log No relation: certificates
2024-03-01T09:25:44.691Z [container-agent] 2024-03-01 09:25:44 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T09:29:56.341Z [container-agent] 2024-03-01 09:29:56 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T09:30:01.875Z [container-agent] 2024-03-01 09:30:01 WARNING juju-log No relation: certificates
2024-03-01T09:30:02.271Z [container-agent] 2024-03-01 09:30:02 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T09:35:30.592Z [container-agent] 2024-03-01 09:35:30 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T09:35:37.016Z [container-agent] 2024-03-01 09:35:37 WARNING juju-log No relation: certificates
2024-03-01T09:35:37.359Z [container-agent] 2024-03-01 09:35:37 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T09:40:24.638Z [container-agent] 2024-03-01 09:40:24 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T09:40:31.012Z [container-agent] 2024-03-01 09:40:31 WARNING juju-log No relation: certificates
2024-03-01T09:40:31.365Z [container-agent] 2024-03-01 09:40:31 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T09:45:01.604Z [container-agent] 2024-03-01 09:45:01 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T09:45:08.682Z [container-agent] 2024-03-01 09:45:08 WARNING juju-log No relation: certificates
2024-03-01T09:45:09.026Z [container-agent] 2024-03-01 09:45:09 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T09:49:54.026Z [container-agent] 2024-03-01 09:49:54 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T09:50:00.799Z [container-agent] 2024-03-01 09:50:00 WARNING juju-log No relation: certificates
2024-03-01T09:50:01.151Z [container-agent] 2024-03-01 09:50:01 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T09:54:35.262Z [container-agent] 2024-03-01 09:54:35 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T09:54:43.157Z [container-agent] 2024-03-01 09:54:43 WARNING juju-log No relation: certificates
2024-03-01T09:54:43.530Z [container-agent] 2024-03-01 09:54:43 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T09:59:57.071Z [container-agent] 2024-03-01 09:59:57 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T10:00:05.393Z [container-agent] 2024-03-01 10:00:05 WARNING juju-log No relation: certificates
2024-03-01T10:00:05.764Z [container-agent] 2024-03-01 10:00:05 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T10:04:02.270Z [container-agent] 2024-03-01 10:04:02 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T10:04:10.198Z [container-agent] 2024-03-01 10:04:10 WARNING juju-log No relation: certificates
2024-03-01T10:04:10.521Z [container-agent] 2024-03-01 10:04:10 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T10:08:21.521Z [container-agent] 2024-03-01 10:08:21 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T10:08:30.869Z [container-agent] 2024-03-01 10:08:30 WARNING juju-log No relation: certificates
2024-03-01T10:08:31.221Z [container-agent] 2024-03-01 10:08:31 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T10:12:50.110Z [container-agent] 2024-03-01 10:12:50 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-03-01T10:12:50.140Z [container-agent] 2024-03-01 10:12:50 INFO juju-log Attempting reboot from complete outage.
2024-03-01T10:12:58.485Z [container-agent] 2024-03-01 10:12:58 WARNING juju-log No relation: certificates
2024-03-01T10:12:58.890Z [container-agent] 2024-03-01 10:12:58 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T10:16:09.805Z [container-agent] 2024-03-01 10:16:09 ERROR juju-log Uncaught exception while in charm code:
2024-03-01T10:16:09.805Z [container-agent] Traceback (most recent call last):
2024-03-01T10:16:09.805Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/./src/charm.py", line 786, in <module>
2024-03-01T10:16:09.805Z [container-agent]     main(MySQLOperatorCharm)
2024-03-01T10:16:09.805Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 436, in main
2024-03-01T10:16:09.805Z [container-agent]     _emit_charm_event(charm, dispatcher.event_name)
2024-03-01T10:16:09.805Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 144, in _emit_charm_event
2024-03-01T10:16:09.805Z [container-agent]     event_to_emit.emit(*args, **kwargs)
2024-03-01T10:16:09.805Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 351, in emit
2024-03-01T10:16:09.805Z [container-agent]     framework._emit(event)
2024-03-01T10:16:09.805Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 853, in _emit
2024-03-01T10:16:09.805Z [container-agent]     self._reemit(event_path)
2024-03-01T10:16:09.805Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 942, in _reemit
2024-03-01T10:16:09.805Z [container-agent]     custom_handler(event)
2024-03-01T10:16:09.805Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/rotate_mysql_logs.py", line 56, in _rotate_mysql_logs
2024-03-01T10:16:09.805Z [container-agent]     self.charm._mysql.flush_mysql_logs(MySQLTextLogs.GENERAL)
2024-03-01T10:16:09.805Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-03-01T10:16:09.805Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-03-01T10:16:09.805Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 666, in _run_mysqlsh_script
2024-03-01T10:16:09.805Z [container-agent]     stdout, _ = process.wait_output()
2024-03-01T10:16:09.805Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1436, in wait_output
2024-03-01T10:16:09.805Z [container-agent]     exit_code: int = self._wait()
2024-03-01T10:16:09.805Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1375, in _wait
2024-03-01T10:16:09.805Z [container-agent]     change = self._client.wait_change(self._change_id, timeout=timeout)
2024-03-01T10:16:09.805Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1893, in wait_change
2024-03-01T10:16:09.805Z [container-agent]     return self._wait_change_using_wait(change_id, timeout)
2024-03-01T10:16:09.805Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1914, in _wait_change_using_wait
2024-03-01T10:16:09.805Z [container-agent]     return self._wait_change(change_id, this_timeout)
2024-03-01T10:16:09.805Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1928, in _wait_change
2024-03-01T10:16:09.805Z [container-agent]     resp = self._request('GET', f'/v1/changes/{change_id}/wait', query)
2024-03-01T10:16:09.805Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1655, in _request
2024-03-01T10:16:09.805Z [container-agent]     response = self._request_raw(method, path, query, headers, data)
2024-03-01T10:16:09.805Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1690, in _request_raw
2024-03-01T10:16:09.805Z [container-agent]     response = self.opener.open(request, timeout=self.timeout)
2024-03-01T10:16:09.805Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 519, in open
2024-03-01T10:16:09.805Z [container-agent]     response = self._open(req, data)
2024-03-01T10:16:09.805Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 536, in _open
2024-03-01T10:16:09.805Z [container-agent]     result = self._call_chain(self.handle_open, protocol, protocol +
2024-03-01T10:16:09.805Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 496, in _call_chain
2024-03-01T10:16:09.805Z [container-agent]     result = func(*args)
2024-03-01T10:16:09.805Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 311, in http_open
2024-03-01T10:16:09.805Z [container-agent]     return self.do_open(_UnixSocketConnection, req,  # type:ignore
2024-03-01T10:16:09.805Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 1352, in do_open
2024-03-01T10:16:09.805Z [container-agent]     r = h.getresponse()
2024-03-01T10:16:09.805Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1375, in getresponse
2024-03-01T10:16:09.805Z [container-agent]     response.begin()
2024-03-01T10:16:09.805Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 318, in begin
2024-03-01T10:16:09.805Z [container-agent]     version, status, reason = self._read_status()
2024-03-01T10:16:09.805Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 287, in _read_status
2024-03-01T10:16:09.805Z [container-agent]     raise RemoteDisconnected("Remote end closed connection without"
2024-03-01T10:16:09.805Z [container-agent] http.client.RemoteDisconnected: Remote end closed connection without response
2024-03-01T10:16:09.912Z [container-agent] 2024-03-01 10:16:09 INFO juju.util.exec exec.go:209 run result: exit status 1
2024-03-01T10:16:11.686Z [container-agent] 2024-03-01 10:16:11 INFO juju-log Setting up the logrotate configurations
2024-03-01T10:16:11.752Z [container-agent] 2024-03-01 10:16:11 INFO juju-log Adding pebble layer
2024-03-01T10:16:18.509Z [container-agent] 2024-03-01 10:16:18 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-03-01T10:16:18.532Z [container-agent] 2024-03-01 10:16:18 INFO juju-log Attempting reboot from complete outage.
2024-03-01T10:16:21.381Z [container-agent] 2024-03-01 10:16:21 INFO juju.worker.uniter.operation runhook.go:186 ran "mysql-pebble-ready" hook (via hook dispatching script: dispatch)
2024-03-01T10:17:58.453Z [container-agent] 2024-03-01 10:17:58 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T10:18:00.467Z [container-agent] 2024-03-01 10:18:00 WARNING juju-log No relation: certificates
2024-03-01T10:18:00.828Z [container-agent] 2024-03-01 10:18:00 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T10:23:49.922Z [container-agent] 2024-03-01 10:23:49 INFO juju-log Starting the log rotate manager
2024-03-01T10:23:49.937Z [container-agent] 2024-03-01 10:23:49 INFO juju-log Started log rotate manager process with PID 404722
2024-03-01T10:23:50.220Z [container-agent] 2024-03-01 10:23:50 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T10:23:52.077Z [container-agent] 2024-03-01 10:23:52 WARNING juju-log No relation: certificates
2024-03-01T10:23:52.413Z [container-agent] 2024-03-01 10:23:52 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T10:29:36.638Z [container-agent] 2024-03-01 10:29:36 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T10:29:39.149Z [container-agent] 2024-03-01 10:29:39 WARNING juju-log No relation: certificates
2024-03-01T10:29:39.493Z [container-agent] 2024-03-01 10:29:39 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T10:35:08.805Z [container-agent] 2024-03-01 10:35:08 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T10:35:11.692Z [container-agent] 2024-03-01 10:35:11 WARNING juju-log No relation: certificates
2024-03-01T10:35:12.051Z [container-agent] 2024-03-01 10:35:12 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T10:40:45.095Z [container-agent] 2024-03-01 10:40:45 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T10:40:48.766Z [container-agent] 2024-03-01 10:40:48 WARNING juju-log No relation: certificates
2024-03-01T10:40:49.153Z [container-agent] 2024-03-01 10:40:49 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T10:44:46.994Z [container-agent] 2024-03-01 10:44:46 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T10:44:51.030Z [container-agent] 2024-03-01 10:44:51 WARNING juju-log No relation: certificates
2024-03-01T10:44:51.372Z [container-agent] 2024-03-01 10:44:51 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T10:49:37.844Z [container-agent] 2024-03-01 10:49:37 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T10:49:41.675Z [container-agent] 2024-03-01 10:49:41 WARNING juju-log No relation: certificates
2024-03-01T10:49:42.033Z [container-agent] 2024-03-01 10:49:42 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T10:55:16.344Z [container-agent] 2024-03-01 10:55:16 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T10:55:21.495Z [container-agent] 2024-03-01 10:55:21 WARNING juju-log No relation: certificates
2024-03-01T10:55:21.840Z [container-agent] 2024-03-01 10:55:21 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T10:59:29.395Z [container-agent] 2024-03-01 10:59:29 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T10:59:34.876Z [container-agent] 2024-03-01 10:59:34 WARNING juju-log No relation: certificates
2024-03-01T10:59:35.218Z [container-agent] 2024-03-01 10:59:35 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T11:04:02.004Z [container-agent] 2024-03-01 11:04:02 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T11:04:07.766Z [container-agent] 2024-03-01 11:04:07 WARNING juju-log No relation: certificates
2024-03-01T11:04:08.091Z [container-agent] 2024-03-01 11:04:08 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T11:08:17.268Z [container-agent] 2024-03-01 11:08:17 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T11:08:23.293Z [container-agent] 2024-03-01 11:08:23 WARNING juju-log No relation: certificates
2024-03-01T11:08:23.649Z [container-agent] 2024-03-01 11:08:23 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T11:13:31.471Z [container-agent] 2024-03-01 11:13:31 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T11:13:38.252Z [container-agent] 2024-03-01 11:13:38 WARNING juju-log No relation: certificates
2024-03-01T11:13:38.610Z [container-agent] 2024-03-01 11:13:38 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T11:18:36.602Z [container-agent] 2024-03-01 11:18:36 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T11:18:42.795Z [container-agent] 2024-03-01 11:18:42 WARNING juju-log No relation: certificates
2024-03-01T11:18:43.141Z [container-agent] 2024-03-01 11:18:43 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T11:23:13.128Z [container-agent] 2024-03-01 11:23:13 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T11:23:20.959Z [container-agent] 2024-03-01 11:23:20 WARNING juju-log No relation: certificates
2024-03-01T11:23:21.294Z [container-agent] 2024-03-01 11:23:21 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T11:28:22.518Z [container-agent] 2024-03-01 11:28:22 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T11:28:29.680Z [container-agent] 2024-03-01 11:28:29 WARNING juju-log No relation: certificates
2024-03-01T11:28:30.071Z [container-agent] 2024-03-01 11:28:30 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T11:33:23.372Z [container-agent] 2024-03-01 11:33:23 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T11:33:31.266Z [container-agent] 2024-03-01 11:33:31 WARNING juju-log No relation: certificates
2024-03-01T11:33:31.633Z [container-agent] 2024-03-01 11:33:31 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T11:35:12.996Z [container-agent] 2024-03-01 11:35:12 ERROR juju-log Failed to flush SLOW LOGS logs.
2024-03-01T11:35:12.996Z [container-agent] Traceback (most recent call last):
2024-03-01T11:35:12.996Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 666, in _run_mysqlsh_script
2024-03-01T11:35:12.996Z [container-agent]     stdout, _ = process.wait_output()
2024-03-01T11:35:12.996Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1441, in wait_output
2024-03-01T11:35:12.996Z [container-agent]     raise ExecError[AnyStr](self._command, exit_code, out_value, err_value)
2024-03-01T11:35:12.996Z [container-agent] ops.pebble.ExecError: non-zero exit code 1 executing ['/usr/bin/mysqlsh', '--no-wizard', '--python', '--verbose=1', '-f', '/tmp/script.py', ';', 'rm', '/tmp/script.py'], stdout='', stderr='Cannot set LC_ALL to locale en_US.UTF-8: No such file or directory\nverbose: 2024-03-01T11:35:11Z: Loading startup files...\nverbose: 2024-03-01T11:35:11Z: Loading plugins...\nverbose: 2024-03-01T11:35:11Z: Connecting to MySQL at: serverconfig@horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local\nTraceback (most recent call last):\n  File "<string>", line 1, in <module>\nmysqlsh.DBError: MySQL Error (2003): Shell.connect: Can\'t connect to MySQL server on \'horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local:3306\' (111)\n'
2024-03-01T11:35:12.996Z [container-agent] 
2024-03-01T11:35:12.996Z [container-agent] During handling of the above exception, another exception occurred:
2024-03-01T11:35:12.996Z [container-agent] 
2024-03-01T11:35:12.996Z [container-agent] Traceback (most recent call last):
2024-03-01T11:35:12.996Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-03-01T11:35:12.996Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-03-01T11:35:12.996Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 669, in _run_mysqlsh_script
2024-03-01T11:35:12.996Z [container-agent]     raise MySQLClientError(e.stderr)
2024-03-01T11:35:12.996Z [container-agent] charms.mysql.v0.mysql.MySQLClientError: Cannot set LC_ALL to locale en_US.UTF-8: No such file or directory
2024-03-01T11:35:12.996Z [container-agent] verbose: 2024-03-01T11:35:11Z: Loading startup files...
2024-03-01T11:35:12.996Z [container-agent] verbose: 2024-03-01T11:35:11Z: Loading plugins...
2024-03-01T11:35:12.996Z [container-agent] verbose: 2024-03-01T11:35:11Z: Connecting to MySQL at: serverconfig@horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local
2024-03-01T11:35:12.996Z [container-agent] Traceback (most recent call last):
2024-03-01T11:35:12.996Z [container-agent]   File "<string>", line 1, in <module>
2024-03-01T11:35:12.996Z [container-agent] mysqlsh.DBError: MySQL Error (2003): Shell.connect: Can't connect to MySQL server on 'horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local:3306' (111)
2024-03-01T11:35:12.996Z [container-agent] 
2024-03-01T11:39:01.768Z [container-agent] 2024-03-01 11:39:01 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-03-01T11:39:01.792Z [container-agent] 2024-03-01 11:39:01 INFO juju-log Attempting reboot from complete outage.
2024-03-01T11:39:09.394Z [container-agent] 2024-03-01 11:39:09 WARNING juju-log No relation: certificates
2024-03-01T11:39:09.732Z [container-agent] 2024-03-01 11:39:09 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T11:40:09.794Z [container-agent] 2024-03-01 11:40:09 ERROR juju-log Uncaught exception while in charm code:
2024-03-01T11:40:09.794Z [container-agent] Traceback (most recent call last):
2024-03-01T11:40:09.794Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/./src/charm.py", line 786, in <module>
2024-03-01T11:40:09.794Z [container-agent]     main(MySQLOperatorCharm)
2024-03-01T11:40:09.794Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 436, in main
2024-03-01T11:40:09.794Z [container-agent]     _emit_charm_event(charm, dispatcher.event_name)
2024-03-01T11:40:09.794Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 144, in _emit_charm_event
2024-03-01T11:40:09.794Z [container-agent]     event_to_emit.emit(*args, **kwargs)
2024-03-01T11:40:09.794Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 351, in emit
2024-03-01T11:40:09.794Z [container-agent]     framework._emit(event)
2024-03-01T11:40:09.794Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 853, in _emit
2024-03-01T11:40:09.794Z [container-agent]     self._reemit(event_path)
2024-03-01T11:40:09.794Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 942, in _reemit
2024-03-01T11:40:09.794Z [container-agent]     custom_handler(event)
2024-03-01T11:40:09.794Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/rotate_mysql_logs.py", line 56, in _rotate_mysql_logs
2024-03-01T11:40:09.794Z [container-agent]     self.charm._mysql.flush_mysql_logs(MySQLTextLogs.GENERAL)
2024-03-01T11:40:09.794Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-03-01T11:40:09.794Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-03-01T11:40:09.794Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 666, in _run_mysqlsh_script
2024-03-01T11:40:09.794Z [container-agent]     stdout, _ = process.wait_output()
2024-03-01T11:40:09.794Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1436, in wait_output
2024-03-01T11:40:09.794Z [container-agent]     exit_code: int = self._wait()
2024-03-01T11:40:09.794Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1375, in _wait
2024-03-01T11:40:09.794Z [container-agent]     change = self._client.wait_change(self._change_id, timeout=timeout)
2024-03-01T11:40:09.794Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1893, in wait_change
2024-03-01T11:40:09.794Z [container-agent]     return self._wait_change_using_wait(change_id, timeout)
2024-03-01T11:40:09.794Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1914, in _wait_change_using_wait
2024-03-01T11:40:09.794Z [container-agent]     return self._wait_change(change_id, this_timeout)
2024-03-01T11:40:09.794Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1928, in _wait_change
2024-03-01T11:40:09.794Z [container-agent]     resp = self._request('GET', f'/v1/changes/{change_id}/wait', query)
2024-03-01T11:40:09.794Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1655, in _request
2024-03-01T11:40:09.794Z [container-agent]     response = self._request_raw(method, path, query, headers, data)
2024-03-01T11:40:09.794Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1690, in _request_raw
2024-03-01T11:40:09.794Z [container-agent]     response = self.opener.open(request, timeout=self.timeout)
2024-03-01T11:40:09.794Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 519, in open
2024-03-01T11:40:09.794Z [container-agent]     response = self._open(req, data)
2024-03-01T11:40:09.794Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 536, in _open
2024-03-01T11:40:09.794Z [container-agent]     result = self._call_chain(self.handle_open, protocol, protocol +
2024-03-01T11:40:09.794Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 496, in _call_chain
2024-03-01T11:40:09.794Z [container-agent]     result = func(*args)
2024-03-01T11:40:09.794Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 311, in http_open
2024-03-01T11:40:09.794Z [container-agent]     return self.do_open(_UnixSocketConnection, req,  # type:ignore
2024-03-01T11:40:09.794Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 1352, in do_open
2024-03-01T11:40:09.794Z [container-agent]     r = h.getresponse()
2024-03-01T11:40:09.794Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1375, in getresponse
2024-03-01T11:40:09.794Z [container-agent]     response.begin()
2024-03-01T11:40:09.794Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 318, in begin
2024-03-01T11:40:09.794Z [container-agent]     version, status, reason = self._read_status()
2024-03-01T11:40:09.794Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 287, in _read_status
2024-03-01T11:40:09.794Z [container-agent]     raise RemoteDisconnected("Remote end closed connection without"
2024-03-01T11:40:09.794Z [container-agent] http.client.RemoteDisconnected: Remote end closed connection without response
2024-03-01T11:40:09.937Z [container-agent] 2024-03-01 11:40:09 INFO juju.util.exec exec.go:209 run result: exit status 1
2024-03-01T11:40:14.466Z [container-agent] 2024-03-01 11:40:14 INFO juju-log Setting up the logrotate configurations
2024-03-01T11:40:14.545Z [container-agent] 2024-03-01 11:40:14 INFO juju-log Adding pebble layer
2024-03-01T11:40:21.274Z [container-agent] 2024-03-01 11:40:21 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-03-01T11:40:21.302Z [container-agent] 2024-03-01 11:40:21 INFO juju-log Attempting reboot from complete outage.
2024-03-01T11:40:24.277Z [container-agent] 2024-03-01 11:40:24 INFO juju.worker.uniter.operation runhook.go:186 ran "mysql-pebble-ready" hook (via hook dispatching script: dispatch)
2024-03-01T11:44:41.346Z [container-agent] 2024-03-01 11:44:41 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T11:44:43.181Z [container-agent] 2024-03-01 11:44:43 WARNING juju-log No relation: certificates
2024-03-01T11:44:43.550Z [container-agent] 2024-03-01 11:44:43 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T11:48:57.418Z [container-agent] 2024-03-01 11:48:57 INFO juju-log Starting the log rotate manager
2024-03-01T11:48:57.431Z [container-agent] 2024-03-01 11:48:57 INFO juju-log Started log rotate manager process with PID 412679
2024-03-01T11:48:57.734Z [container-agent] 2024-03-01 11:48:57 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T11:48:59.717Z [container-agent] 2024-03-01 11:48:59 WARNING juju-log No relation: certificates
2024-03-01T11:49:00.076Z [container-agent] 2024-03-01 11:49:00 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T11:53:06.016Z [container-agent] 2024-03-01 11:53:06 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T11:53:08.461Z [container-agent] 2024-03-01 11:53:08 WARNING juju-log No relation: certificates
2024-03-01T11:53:08.786Z [container-agent] 2024-03-01 11:53:08 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T11:57:50.947Z [container-agent] 2024-03-01 11:57:50 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T11:57:53.837Z [container-agent] 2024-03-01 11:57:53 WARNING juju-log No relation: certificates
2024-03-01T11:57:54.166Z [container-agent] 2024-03-01 11:57:54 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T12:03:35.656Z [container-agent] 2024-03-01 12:03:35 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T12:03:38.928Z [container-agent] 2024-03-01 12:03:38 WARNING juju-log No relation: certificates
2024-03-01T12:03:39.266Z [container-agent] 2024-03-01 12:03:39 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T12:07:42.175Z [container-agent] 2024-03-01 12:07:42 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T12:07:46.118Z [container-agent] 2024-03-01 12:07:46 WARNING juju-log No relation: certificates
2024-03-01T12:07:46.522Z [container-agent] 2024-03-01 12:07:46 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T12:11:45.904Z [container-agent] 2024-03-01 12:11:45 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T12:11:50.408Z [container-agent] 2024-03-01 12:11:50 WARNING juju-log No relation: certificates
2024-03-01T12:11:50.785Z [container-agent] 2024-03-01 12:11:50 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T12:17:43.328Z [container-agent] 2024-03-01 12:17:43 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T12:17:48.049Z [container-agent] 2024-03-01 12:17:48 WARNING juju-log No relation: certificates
2024-03-01T12:17:48.389Z [container-agent] 2024-03-01 12:17:48 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T12:22:24.575Z [container-agent] 2024-03-01 12:22:24 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T12:22:29.748Z [container-agent] 2024-03-01 12:22:29 WARNING juju-log No relation: certificates
2024-03-01T12:22:30.087Z [container-agent] 2024-03-01 12:22:30 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T12:26:38.765Z [container-agent] 2024-03-01 12:26:38 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T12:26:44.519Z [container-agent] 2024-03-01 12:26:44 WARNING juju-log No relation: certificates
2024-03-01T12:26:44.865Z [container-agent] 2024-03-01 12:26:44 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T12:31:10.563Z [container-agent] 2024-03-01 12:31:10 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T12:31:16.862Z [container-agent] 2024-03-01 12:31:16 WARNING juju-log No relation: certificates
2024-03-01T12:31:17.237Z [container-agent] 2024-03-01 12:31:17 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T12:36:11.272Z [container-agent] 2024-03-01 12:36:11 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T12:36:17.890Z [container-agent] 2024-03-01 12:36:17 WARNING juju-log No relation: certificates
2024-03-01T12:36:18.263Z [container-agent] 2024-03-01 12:36:18 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T12:41:13.002Z [container-agent] 2024-03-01 12:41:13 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T12:41:20.117Z [container-agent] 2024-03-01 12:41:20 WARNING juju-log No relation: certificates
2024-03-01T12:41:20.458Z [container-agent] 2024-03-01 12:41:20 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T12:46:41.928Z [container-agent] 2024-03-01 12:46:41 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T12:46:49.362Z [container-agent] 2024-03-01 12:46:49 WARNING juju-log No relation: certificates
2024-03-01T12:46:49.747Z [container-agent] 2024-03-01 12:46:49 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T12:50:07.782Z [container-agent] 2024-03-01 12:50:07 ERROR juju-log Failed to flush ERROR LOGS logs.
2024-03-01T12:50:07.782Z [container-agent] Traceback (most recent call last):
2024-03-01T12:50:07.782Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 666, in _run_mysqlsh_script
2024-03-01T12:50:07.782Z [container-agent]     stdout, _ = process.wait_output()
2024-03-01T12:50:07.782Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1441, in wait_output
2024-03-01T12:50:07.782Z [container-agent]     raise ExecError[AnyStr](self._command, exit_code, out_value, err_value)
2024-03-01T12:50:07.782Z [container-agent] ops.pebble.ExecError: non-zero exit code 1 executing ['/usr/bin/mysqlsh', '--no-wizard', '--python', '--verbose=1', '-f', '/tmp/script.py', ';', 'rm', '/tmp/script.py'], stdout='', stderr='Cannot set LC_ALL to locale en_US.UTF-8: No such file or directory\nverbose: 2024-03-01T12:50:06Z: Loading startup files...\nverbose: 2024-03-01T12:50:06Z: Loading plugins...\nverbose: 2024-03-01T12:50:06Z: Connecting to MySQL at: serverconfig@horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local\nTraceback (most recent call last):\n  File "<string>", line 1, in <module>\nmysqlsh.DBError: MySQL Error (2003): Shell.connect: Can\'t connect to MySQL server on \'horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local:3306\' (111)\n'
2024-03-01T12:50:07.782Z [container-agent] 
2024-03-01T12:50:07.782Z [container-agent] During handling of the above exception, another exception occurred:
2024-03-01T12:50:07.782Z [container-agent] 
2024-03-01T12:50:07.782Z [container-agent] Traceback (most recent call last):
2024-03-01T12:50:07.782Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-03-01T12:50:07.782Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-03-01T12:50:07.782Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 669, in _run_mysqlsh_script
2024-03-01T12:50:07.782Z [container-agent]     raise MySQLClientError(e.stderr)
2024-03-01T12:50:07.782Z [container-agent] charms.mysql.v0.mysql.MySQLClientError: Cannot set LC_ALL to locale en_US.UTF-8: No such file or directory
2024-03-01T12:50:07.782Z [container-agent] verbose: 2024-03-01T12:50:06Z: Loading startup files...
2024-03-01T12:50:07.782Z [container-agent] verbose: 2024-03-01T12:50:06Z: Loading plugins...
2024-03-01T12:50:07.782Z [container-agent] verbose: 2024-03-01T12:50:06Z: Connecting to MySQL at: serverconfig@horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local
2024-03-01T12:50:07.782Z [container-agent] Traceback (most recent call last):
2024-03-01T12:50:07.782Z [container-agent]   File "<string>", line 1, in <module>
2024-03-01T12:50:07.782Z [container-agent] mysqlsh.DBError: MySQL Error (2003): Shell.connect: Can't connect to MySQL server on 'horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local:3306' (111)
2024-03-01T12:50:07.782Z [container-agent] 
2024-03-01T12:51:35.081Z [container-agent] 2024-03-01 12:51:35 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-03-01T12:51:35.109Z [container-agent] 2024-03-01 12:51:35 INFO juju-log Attempting reboot from complete outage.
2024-03-01T12:51:42.526Z [container-agent] 2024-03-01 12:51:42 WARNING juju-log No relation: certificates
2024-03-01T12:51:42.868Z [container-agent] 2024-03-01 12:51:42 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T12:55:04.805Z [container-agent] 2024-03-01 12:55:04 ERROR juju-log Uncaught exception while in charm code:
2024-03-01T12:55:04.805Z [container-agent] Traceback (most recent call last):
2024-03-01T12:55:04.805Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/./src/charm.py", line 786, in <module>
2024-03-01T12:55:04.805Z [container-agent]     main(MySQLOperatorCharm)
2024-03-01T12:55:04.805Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 436, in main
2024-03-01T12:55:04.805Z [container-agent]     _emit_charm_event(charm, dispatcher.event_name)
2024-03-01T12:55:04.805Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 144, in _emit_charm_event
2024-03-01T12:55:04.805Z [container-agent]     event_to_emit.emit(*args, **kwargs)
2024-03-01T12:55:04.805Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 351, in emit
2024-03-01T12:55:04.805Z [container-agent]     framework._emit(event)
2024-03-01T12:55:04.805Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 853, in _emit
2024-03-01T12:55:04.805Z [container-agent]     self._reemit(event_path)
2024-03-01T12:55:04.805Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 942, in _reemit
2024-03-01T12:55:04.805Z [container-agent]     custom_handler(event)
2024-03-01T12:55:04.805Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/rotate_mysql_logs.py", line 55, in _rotate_mysql_logs
2024-03-01T12:55:04.805Z [container-agent]     self.charm._mysql.flush_mysql_logs(MySQLTextLogs.ERROR)
2024-03-01T12:55:04.805Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-03-01T12:55:04.805Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-03-01T12:55:04.805Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 666, in _run_mysqlsh_script
2024-03-01T12:55:04.805Z [container-agent]     stdout, _ = process.wait_output()
2024-03-01T12:55:04.805Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1436, in wait_output
2024-03-01T12:55:04.805Z [container-agent]     exit_code: int = self._wait()
2024-03-01T12:55:04.805Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1375, in _wait
2024-03-01T12:55:04.805Z [container-agent]     change = self._client.wait_change(self._change_id, timeout=timeout)
2024-03-01T12:55:04.805Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1893, in wait_change
2024-03-01T12:55:04.805Z [container-agent]     return self._wait_change_using_wait(change_id, timeout)
2024-03-01T12:55:04.805Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1914, in _wait_change_using_wait
2024-03-01T12:55:04.805Z [container-agent]     return self._wait_change(change_id, this_timeout)
2024-03-01T12:55:04.805Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1928, in _wait_change
2024-03-01T12:55:04.805Z [container-agent]     resp = self._request('GET', f'/v1/changes/{change_id}/wait', query)
2024-03-01T12:55:04.805Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1655, in _request
2024-03-01T12:55:04.805Z [container-agent]     response = self._request_raw(method, path, query, headers, data)
2024-03-01T12:55:04.805Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1690, in _request_raw
2024-03-01T12:55:04.805Z [container-agent]     response = self.opener.open(request, timeout=self.timeout)
2024-03-01T12:55:04.805Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 519, in open
2024-03-01T12:55:04.805Z [container-agent]     response = self._open(req, data)
2024-03-01T12:55:04.805Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 536, in _open
2024-03-01T12:55:04.805Z [container-agent]     result = self._call_chain(self.handle_open, protocol, protocol +
2024-03-01T12:55:04.805Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 496, in _call_chain
2024-03-01T12:55:04.805Z [container-agent]     result = func(*args)
2024-03-01T12:55:04.805Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 311, in http_open
2024-03-01T12:55:04.805Z [container-agent]     return self.do_open(_UnixSocketConnection, req,  # type:ignore
2024-03-01T12:55:04.805Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 1352, in do_open
2024-03-01T12:55:04.805Z [container-agent]     r = h.getresponse()
2024-03-01T12:55:04.805Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1375, in getresponse
2024-03-01T12:55:04.805Z [container-agent]     response.begin()
2024-03-01T12:55:04.805Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 318, in begin
2024-03-01T12:55:04.805Z [container-agent]     version, status, reason = self._read_status()
2024-03-01T12:55:04.805Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 287, in _read_status
2024-03-01T12:55:04.805Z [container-agent]     raise RemoteDisconnected("Remote end closed connection without"
2024-03-01T12:55:04.805Z [container-agent] http.client.RemoteDisconnected: Remote end closed connection without response
2024-03-01T12:55:04.927Z [container-agent] 2024-03-01 12:55:04 INFO juju.util.exec exec.go:209 run result: exit status 1
2024-03-01T12:55:07.384Z [container-agent] 2024-03-01 12:55:07 INFO juju-log Setting up the logrotate configurations
2024-03-01T12:55:07.447Z [container-agent] 2024-03-01 12:55:07 INFO juju-log Adding pebble layer
2024-03-01T12:55:14.303Z [container-agent] 2024-03-01 12:55:14 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-03-01T12:55:14.332Z [container-agent] 2024-03-01 12:55:14 INFO juju-log Attempting reboot from complete outage.
2024-03-01T12:55:17.190Z [container-agent] 2024-03-01 12:55:17 INFO juju.worker.uniter.operation runhook.go:186 ran "mysql-pebble-ready" hook (via hook dispatching script: dispatch)
2024-03-01T12:56:18.148Z [container-agent] 2024-03-01 12:56:18 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T12:56:20.021Z [container-agent] 2024-03-01 12:56:20 WARNING juju-log No relation: certificates
2024-03-01T12:56:20.396Z [container-agent] 2024-03-01 12:56:20 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T13:01:02.340Z [container-agent] 2024-03-01 13:01:02 INFO juju-log Starting the log rotate manager
2024-03-01T13:01:02.355Z [container-agent] 2024-03-01 13:01:02 INFO juju-log Started log rotate manager process with PID 419654
2024-03-01T13:01:02.688Z [container-agent] 2024-03-01 13:01:02 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T13:01:05.120Z [container-agent] 2024-03-01 13:01:05 WARNING juju-log No relation: certificates
2024-03-01T13:01:05.505Z [container-agent] 2024-03-01 13:01:05 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T13:06:11.533Z [container-agent] 2024-03-01 13:06:11 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T13:06:13.907Z [container-agent] 2024-03-01 13:06:13 WARNING juju-log No relation: certificates
2024-03-01T13:06:14.281Z [container-agent] 2024-03-01 13:06:14 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T13:10:16.041Z [container-agent] 2024-03-01 13:10:16 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T13:10:19.178Z [container-agent] 2024-03-01 13:10:19 WARNING juju-log No relation: certificates
2024-03-01T13:10:19.535Z [container-agent] 2024-03-01 13:10:19 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T13:14:22.257Z [container-agent] 2024-03-01 13:14:22 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T13:14:25.286Z [container-agent] 2024-03-01 13:14:25 WARNING juju-log No relation: certificates
2024-03-01T13:14:25.637Z [container-agent] 2024-03-01 13:14:25 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T13:20:11.003Z [container-agent] 2024-03-01 13:20:11 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T13:20:14.953Z [container-agent] 2024-03-01 13:20:14 WARNING juju-log No relation: certificates
2024-03-01T13:20:15.346Z [container-agent] 2024-03-01 13:20:15 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T13:25:14.450Z [container-agent] 2024-03-01 13:25:14 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T13:25:18.652Z [container-agent] 2024-03-01 13:25:18 WARNING juju-log No relation: certificates
2024-03-01T13:25:18.992Z [container-agent] 2024-03-01 13:25:18 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T13:31:11.569Z [container-agent] 2024-03-01 13:31:11 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T13:31:16.390Z [container-agent] 2024-03-01 13:31:16 WARNING juju-log No relation: certificates
2024-03-01T13:31:16.750Z [container-agent] 2024-03-01 13:31:16 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T13:36:12.387Z [container-agent] 2024-03-01 13:36:12 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T13:36:17.439Z [container-agent] 2024-03-01 13:36:17 WARNING juju-log No relation: certificates
2024-03-01T13:36:17.793Z [container-agent] 2024-03-01 13:36:17 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T13:40:38.017Z [container-agent] 2024-03-01 13:40:38 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T13:40:43.707Z [container-agent] 2024-03-01 13:40:43 WARNING juju-log No relation: certificates
2024-03-01T13:40:44.070Z [container-agent] 2024-03-01 13:40:44 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T13:46:33.840Z [container-agent] 2024-03-01 13:46:33 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T13:46:40.081Z [container-agent] 2024-03-01 13:46:40 WARNING juju-log No relation: certificates
2024-03-01T13:46:40.422Z [container-agent] 2024-03-01 13:46:40 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T13:51:20.811Z [container-agent] 2024-03-01 13:51:20 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T13:51:27.554Z [container-agent] 2024-03-01 13:51:27 WARNING juju-log No relation: certificates
2024-03-01T13:51:27.917Z [container-agent] 2024-03-01 13:51:27 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T13:55:57.525Z [container-agent] 2024-03-01 13:55:57 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T13:56:04.565Z [container-agent] 2024-03-01 13:56:04 WARNING juju-log No relation: certificates
2024-03-01T13:56:04.915Z [container-agent] 2024-03-01 13:56:04 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T14:01:37.999Z [container-agent] 2024-03-01 14:01:37 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T14:01:45.534Z [container-agent] 2024-03-01 14:01:45 WARNING juju-log No relation: certificates
2024-03-01T14:01:45.893Z [container-agent] 2024-03-01 14:01:45 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T14:06:42.631Z [container-agent] 2024-03-01 14:06:42 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-03-01T14:06:42.661Z [container-agent] 2024-03-01 14:06:42 INFO juju-log Attempting reboot from complete outage.
2024-03-01T14:06:50.351Z [container-agent] 2024-03-01 14:06:50 WARNING juju-log No relation: certificates
2024-03-01T14:06:50.694Z [container-agent] 2024-03-01 14:06:50 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T14:10:04.789Z [container-agent] 2024-03-01 14:10:04 ERROR juju-log Uncaught exception while in charm code:
2024-03-01T14:10:04.789Z [container-agent] Traceback (most recent call last):
2024-03-01T14:10:04.789Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/./src/charm.py", line 786, in <module>
2024-03-01T14:10:04.789Z [container-agent]     main(MySQLOperatorCharm)
2024-03-01T14:10:04.789Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 436, in main
2024-03-01T14:10:04.789Z [container-agent]     _emit_charm_event(charm, dispatcher.event_name)
2024-03-01T14:10:04.789Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 144, in _emit_charm_event
2024-03-01T14:10:04.789Z [container-agent]     event_to_emit.emit(*args, **kwargs)
2024-03-01T14:10:04.789Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 351, in emit
2024-03-01T14:10:04.789Z [container-agent]     framework._emit(event)
2024-03-01T14:10:04.789Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 853, in _emit
2024-03-01T14:10:04.789Z [container-agent]     self._reemit(event_path)
2024-03-01T14:10:04.789Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 942, in _reemit
2024-03-01T14:10:04.789Z [container-agent]     custom_handler(event)
2024-03-01T14:10:04.789Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/rotate_mysql_logs.py", line 55, in _rotate_mysql_logs
2024-03-01T14:10:04.789Z [container-agent]     self.charm._mysql.flush_mysql_logs(MySQLTextLogs.ERROR)
2024-03-01T14:10:04.789Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-03-01T14:10:04.789Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-03-01T14:10:04.789Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 666, in _run_mysqlsh_script
2024-03-01T14:10:04.789Z [container-agent]     stdout, _ = process.wait_output()
2024-03-01T14:10:04.789Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1436, in wait_output
2024-03-01T14:10:04.789Z [container-agent]     exit_code: int = self._wait()
2024-03-01T14:10:04.789Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1375, in _wait
2024-03-01T14:10:04.789Z [container-agent]     change = self._client.wait_change(self._change_id, timeout=timeout)
2024-03-01T14:10:04.789Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1893, in wait_change
2024-03-01T14:10:04.789Z [container-agent]     return self._wait_change_using_wait(change_id, timeout)
2024-03-01T14:10:04.789Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1914, in _wait_change_using_wait
2024-03-01T14:10:04.789Z [container-agent]     return self._wait_change(change_id, this_timeout)
2024-03-01T14:10:04.789Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1928, in _wait_change
2024-03-01T14:10:04.789Z [container-agent]     resp = self._request('GET', f'/v1/changes/{change_id}/wait', query)
2024-03-01T14:10:04.789Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1655, in _request
2024-03-01T14:10:04.789Z [container-agent]     response = self._request_raw(method, path, query, headers, data)
2024-03-01T14:10:04.789Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1690, in _request_raw
2024-03-01T14:10:04.789Z [container-agent]     response = self.opener.open(request, timeout=self.timeout)
2024-03-01T14:10:04.789Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 519, in open
2024-03-01T14:10:04.789Z [container-agent]     response = self._open(req, data)
2024-03-01T14:10:04.789Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 536, in _open
2024-03-01T14:10:04.789Z [container-agent]     result = self._call_chain(self.handle_open, protocol, protocol +
2024-03-01T14:10:04.789Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 496, in _call_chain
2024-03-01T14:10:04.789Z [container-agent]     result = func(*args)
2024-03-01T14:10:04.789Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 311, in http_open
2024-03-01T14:10:04.789Z [container-agent]     return self.do_open(_UnixSocketConnection, req,  # type:ignore
2024-03-01T14:10:04.789Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 1352, in do_open
2024-03-01T14:10:04.789Z [container-agent]     r = h.getresponse()
2024-03-01T14:10:04.789Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1375, in getresponse
2024-03-01T14:10:04.789Z [container-agent]     response.begin()
2024-03-01T14:10:04.789Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 318, in begin
2024-03-01T14:10:04.789Z [container-agent]     version, status, reason = self._read_status()
2024-03-01T14:10:04.789Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 287, in _read_status
2024-03-01T14:10:04.789Z [container-agent]     raise RemoteDisconnected("Remote end closed connection without"
2024-03-01T14:10:04.789Z [container-agent] http.client.RemoteDisconnected: Remote end closed connection without response
2024-03-01T14:10:04.903Z [container-agent] 2024-03-01 14:10:04 INFO juju.util.exec exec.go:209 run result: exit status 1
2024-03-01T14:10:10.358Z [container-agent] 2024-03-01 14:10:10 INFO juju-log Setting up the logrotate configurations
2024-03-01T14:10:10.430Z [container-agent] 2024-03-01 14:10:10 INFO juju-log Adding pebble layer
2024-03-01T14:10:17.154Z [container-agent] 2024-03-01 14:10:17 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-03-01T14:10:17.177Z [container-agent] 2024-03-01 14:10:17 INFO juju-log Attempting reboot from complete outage.
2024-03-01T14:10:19.872Z [container-agent] 2024-03-01 14:10:19 INFO juju.worker.uniter.operation runhook.go:186 ran "mysql-pebble-ready" hook (via hook dispatching script: dispatch)
2024-03-01T14:12:17.394Z [container-agent] 2024-03-01 14:12:17 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T14:12:19.459Z [container-agent] 2024-03-01 14:12:19 WARNING juju-log No relation: certificates
2024-03-01T14:12:19.803Z [container-agent] 2024-03-01 14:12:19 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T14:16:24.195Z [container-agent] 2024-03-01 14:16:24 INFO juju-log Starting the log rotate manager
2024-03-01T14:16:24.213Z [container-agent] 2024-03-01 14:16:24 INFO juju-log Started log rotate manager process with PID 426757
2024-03-01T14:16:24.497Z [container-agent] 2024-03-01 14:16:24 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T14:16:26.397Z [container-agent] 2024-03-01 14:16:26 WARNING juju-log No relation: certificates
2024-03-01T14:16:26.726Z [container-agent] 2024-03-01 14:16:26 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T14:22:10.414Z [container-agent] 2024-03-01 14:22:10 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T14:22:12.965Z [container-agent] 2024-03-01 14:22:12 WARNING juju-log No relation: certificates
2024-03-01T14:22:13.310Z [container-agent] 2024-03-01 14:22:13 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T14:26:38.355Z [container-agent] 2024-03-01 14:26:38 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T14:26:41.222Z [container-agent] 2024-03-01 14:26:41 WARNING juju-log No relation: certificates
2024-03-01T14:26:41.573Z [container-agent] 2024-03-01 14:26:41 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T14:32:26.265Z [container-agent] 2024-03-01 14:32:26 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T14:32:29.374Z [container-agent] 2024-03-01 14:32:29 WARNING juju-log No relation: certificates
2024-03-01T14:32:29.706Z [container-agent] 2024-03-01 14:32:29 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T14:38:19.082Z [container-agent] 2024-03-01 14:38:19 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T14:38:23.154Z [container-agent] 2024-03-01 14:38:23 WARNING juju-log No relation: certificates
2024-03-01T14:38:23.507Z [container-agent] 2024-03-01 14:38:23 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T14:42:26.697Z [container-agent] 2024-03-01 14:42:26 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T14:42:30.956Z [container-agent] 2024-03-01 14:42:30 WARNING juju-log No relation: certificates
2024-03-01T14:42:31.300Z [container-agent] 2024-03-01 14:42:31 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T14:48:02.420Z [container-agent] 2024-03-01 14:48:02 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T14:48:06.922Z [container-agent] 2024-03-01 14:48:06 WARNING juju-log No relation: certificates
2024-03-01T14:48:07.245Z [container-agent] 2024-03-01 14:48:07 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T14:53:29.446Z [container-agent] 2024-03-01 14:53:29 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T14:53:34.851Z [container-agent] 2024-03-01 14:53:34 WARNING juju-log No relation: certificates
2024-03-01T14:53:35.211Z [container-agent] 2024-03-01 14:53:35 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T14:58:11.897Z [container-agent] 2024-03-01 14:58:11 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T14:58:17.903Z [container-agent] 2024-03-01 14:58:17 WARNING juju-log No relation: certificates
2024-03-01T14:58:18.259Z [container-agent] 2024-03-01 14:58:18 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T15:04:01.122Z [container-agent] 2024-03-01 15:04:01 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T15:04:07.137Z [container-agent] 2024-03-01 15:04:07 WARNING juju-log No relation: certificates
2024-03-01T15:04:07.455Z [container-agent] 2024-03-01 15:04:07 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T15:08:12.015Z [container-agent] 2024-03-01 15:08:12 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T15:08:18.772Z [container-agent] 2024-03-01 15:08:18 WARNING juju-log No relation: certificates
2024-03-01T15:08:19.163Z [container-agent] 2024-03-01 15:08:19 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T15:12:43.677Z [container-agent] 2024-03-01 15:12:43 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T15:12:50.759Z [container-agent] 2024-03-01 15:12:50 WARNING juju-log No relation: certificates
2024-03-01T15:12:51.108Z [container-agent] 2024-03-01 15:12:51 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T15:17:55.521Z [container-agent] 2024-03-01 15:17:55 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T15:18:02.715Z [container-agent] 2024-03-01 15:18:02 WARNING juju-log No relation: certificates
2024-03-01T15:18:03.077Z [container-agent] 2024-03-01 15:18:03 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T15:22:54.410Z [container-agent] 2024-03-01 15:22:54 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T15:23:02.044Z [container-agent] 2024-03-01 15:23:02 WARNING juju-log No relation: certificates
2024-03-01T15:23:02.437Z [container-agent] 2024-03-01 15:23:02 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T15:27:43.907Z [container-agent] 2024-03-01 15:27:43 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T15:27:52.130Z [container-agent] 2024-03-01 15:27:52 WARNING juju-log No relation: certificates
2024-03-01T15:27:52.510Z [container-agent] 2024-03-01 15:27:52 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T15:32:01.421Z [container-agent] 2024-03-01 15:32:01 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-03-01T15:32:01.449Z [container-agent] 2024-03-01 15:32:01 INFO juju-log Attempting reboot from complete outage.
2024-03-01T15:32:08.987Z [container-agent] 2024-03-01 15:32:08 WARNING juju-log No relation: certificates
2024-03-01T15:32:09.362Z [container-agent] 2024-03-01 15:32:09 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T15:36:09.775Z [container-agent] 2024-03-01 15:36:09 ERROR juju-log Uncaught exception while in charm code:
2024-03-01T15:36:09.775Z [container-agent] Traceback (most recent call last):
2024-03-01T15:36:09.775Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 2507, in exec
2024-03-01T15:36:09.775Z [container-agent]     control_ws = self._connect_websocket(task_id, 'control')
2024-03-01T15:36:09.775Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 2589, in _connect_websocket
2024-03-01T15:36:09.775Z [container-agent]     ws.connect(url, socket=sock)
2024-03-01T15:36:09.775Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_core.py", line 253, in connect
2024-03-01T15:36:09.775Z [container-agent]     self.handshake_response = handshake(self.sock, url, *addrs, **options)
2024-03-01T15:36:09.775Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_handshake.py", line 57, in handshake
2024-03-01T15:36:09.775Z [container-agent]     status, resp = _get_resp_headers(sock)
2024-03-01T15:36:09.775Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_handshake.py", line 143, in _get_resp_headers
2024-03-01T15:36:09.775Z [container-agent]     status, resp_headers, status_message = read_headers(sock)
2024-03-01T15:36:09.775Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_http.py", line 312, in read_headers
2024-03-01T15:36:09.775Z [container-agent]     line = recv_line(sock)
2024-03-01T15:36:09.775Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_socket.py", line 131, in recv_line
2024-03-01T15:36:09.775Z [container-agent]     c = recv(sock, 1)
2024-03-01T15:36:09.775Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_socket.py", line 122, in recv
2024-03-01T15:36:09.775Z [container-agent]     raise WebSocketConnectionClosedException(
2024-03-01T15:36:09.775Z [container-agent] websocket._exceptions.WebSocketConnectionClosedException: Connection to remote host was lost.
2024-03-01T15:36:09.775Z [container-agent] 
2024-03-01T15:36:09.775Z [container-agent] During handling of the above exception, another exception occurred:
2024-03-01T15:36:09.775Z [container-agent] 
2024-03-01T15:36:09.775Z [container-agent] Traceback (most recent call last):
2024-03-01T15:36:09.775Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 1348, in do_open
2024-03-01T15:36:09.775Z [container-agent]     h.request(req.get_method(), req.selector, req.data, headers,
2024-03-01T15:36:09.775Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1283, in request
2024-03-01T15:36:09.775Z [container-agent]     self._send_request(method, url, body, headers, encode_chunked)
2024-03-01T15:36:09.775Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1329, in _send_request
2024-03-01T15:36:09.776Z [container-agent]     self.endheaders(body, encode_chunked=encode_chunked)
2024-03-01T15:36:09.776Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1278, in endheaders
2024-03-01T15:36:09.776Z [container-agent]     self._send_output(message_body, encode_chunked=encode_chunked)
2024-03-01T15:36:09.776Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1038, in _send_output
2024-03-01T15:36:09.776Z [container-agent]     self.send(msg)
2024-03-01T15:36:09.776Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 976, in send
2024-03-01T15:36:09.776Z [container-agent]     self.connect()
2024-03-01T15:36:09.776Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 297, in connect
2024-03-01T15:36:09.776Z [container-agent]     self.sock.connect(self.socket_path)
2024-03-01T15:36:09.776Z [container-agent] ConnectionRefusedError: [Errno 111] Connection refused
2024-03-01T15:36:09.776Z [container-agent] 
2024-03-01T15:36:09.776Z [container-agent] During handling of the above exception, another exception occurred:
2024-03-01T15:36:09.776Z [container-agent] 
2024-03-01T15:36:09.776Z [container-agent] Traceback (most recent call last):
2024-03-01T15:36:09.776Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1690, in _request_raw
2024-03-01T15:36:09.776Z [container-agent]     response = self.opener.open(request, timeout=self.timeout)
2024-03-01T15:36:09.776Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 519, in open
2024-03-01T15:36:09.776Z [container-agent]     response = self._open(req, data)
2024-03-01T15:36:09.776Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 536, in _open
2024-03-01T15:36:09.776Z [container-agent]     result = self._call_chain(self.handle_open, protocol, protocol +
2024-03-01T15:36:09.776Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 496, in _call_chain
2024-03-01T15:36:09.776Z [container-agent]     result = func(*args)
2024-03-01T15:36:09.776Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 311, in http_open
2024-03-01T15:36:09.776Z [container-agent]     return self.do_open(_UnixSocketConnection, req,  # type:ignore
2024-03-01T15:36:09.776Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 1351, in do_open
2024-03-01T15:36:09.776Z [container-agent]     raise URLError(err)
2024-03-01T15:36:09.776Z [container-agent] urllib.error.URLError: <urlopen error [Errno 111] Connection refused>
2024-03-01T15:36:09.776Z [container-agent] 
2024-03-01T15:36:09.776Z [container-agent] The above exception was the direct cause of the following exception:
2024-03-01T15:36:09.776Z [container-agent] 
2024-03-01T15:36:09.776Z [container-agent] Traceback (most recent call last):
2024-03-01T15:36:09.776Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/./src/charm.py", line 786, in <module>
2024-03-01T15:36:09.776Z [container-agent]     main(MySQLOperatorCharm)
2024-03-01T15:36:09.776Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 436, in main
2024-03-01T15:36:09.776Z [container-agent]     _emit_charm_event(charm, dispatcher.event_name)
2024-03-01T15:36:09.776Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 144, in _emit_charm_event
2024-03-01T15:36:09.776Z [container-agent]     event_to_emit.emit(*args, **kwargs)
2024-03-01T15:36:09.776Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 351, in emit
2024-03-01T15:36:09.776Z [container-agent]     framework._emit(event)
2024-03-01T15:36:09.776Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 853, in _emit
2024-03-01T15:36:09.776Z [container-agent]     self._reemit(event_path)
2024-03-01T15:36:09.776Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 942, in _reemit
2024-03-01T15:36:09.776Z [container-agent]     custom_handler(event)
2024-03-01T15:36:09.776Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/rotate_mysql_logs.py", line 57, in _rotate_mysql_logs
2024-03-01T15:36:09.776Z [container-agent]     self.charm._mysql.flush_mysql_logs(MySQLTextLogs.SLOW)
2024-03-01T15:36:09.776Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-03-01T15:36:09.776Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-03-01T15:36:09.776Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 665, in _run_mysqlsh_script
2024-03-01T15:36:09.776Z [container-agent]     process = self.container.exec(cmd, timeout=timeout)
2024-03-01T15:36:09.776Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/model.py", line 2688, in exec
2024-03-01T15:36:09.776Z [container-agent]     return self._pebble.exec(
2024-03-01T15:36:09.776Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 2514, in exec
2024-03-01T15:36:09.776Z [container-agent]     change = self.wait_change(ChangeID(change_id))
2024-03-01T15:36:09.776Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1893, in wait_change
2024-03-01T15:36:09.776Z [container-agent]     return self._wait_change_using_wait(change_id, timeout)
2024-03-01T15:36:09.776Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1914, in _wait_change_using_wait
2024-03-01T15:36:09.776Z [container-agent]     return self._wait_change(change_id, this_timeout)
2024-03-01T15:36:09.776Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1928, in _wait_change
2024-03-01T15:36:09.776Z [container-agent]     resp = self._request('GET', f'/v1/changes/{change_id}/wait', query)
2024-03-01T15:36:09.776Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1655, in _request
2024-03-01T15:36:09.776Z [container-agent]     response = self._request_raw(method, path, query, headers, data)
2024-03-01T15:36:09.776Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1707, in _request_raw
2024-03-01T15:36:09.776Z [container-agent]     raise ConnectionError(e.reason) from e
2024-03-01T15:36:09.776Z [container-agent] ops.pebble.ConnectionError: [Errno 111] Connection refused
2024-03-01T15:36:09.900Z [container-agent] 2024-03-01 15:36:09 INFO juju.util.exec exec.go:209 run result: exit status 1
2024-03-01T15:36:13.379Z [container-agent] 2024-03-01 15:36:13 INFO juju-log Setting up the logrotate configurations
2024-03-01T15:36:13.456Z [container-agent] 2024-03-01 15:36:13 INFO juju-log Adding pebble layer
2024-03-01T15:36:20.241Z [container-agent] 2024-03-01 15:36:20 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-03-01T15:36:20.269Z [container-agent] 2024-03-01 15:36:20 INFO juju-log Attempting reboot from complete outage.
2024-03-01T15:36:23.191Z [container-agent] 2024-03-01 15:36:23 INFO juju.worker.uniter.operation runhook.go:186 ran "mysql-pebble-ready" hook (via hook dispatching script: dispatch)
2024-03-01T15:36:45.120Z [container-agent] 2024-03-01 15:36:45 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T15:36:47.174Z [container-agent] 2024-03-01 15:36:47 WARNING juju-log No relation: certificates
2024-03-01T15:36:47.517Z [container-agent] 2024-03-01 15:36:47 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T15:41:53.845Z [container-agent] 2024-03-01 15:41:53 INFO juju-log Starting the log rotate manager
2024-03-01T15:41:53.859Z [container-agent] 2024-03-01 15:41:53 INFO juju-log Started log rotate manager process with PID 434826
2024-03-01T15:41:54.140Z [container-agent] 2024-03-01 15:41:54 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T15:41:56.125Z [container-agent] 2024-03-01 15:41:56 WARNING juju-log No relation: certificates
2024-03-01T15:41:56.506Z [container-agent] 2024-03-01 15:41:56 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T15:47:15.206Z [container-agent] 2024-03-01 15:47:15 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T15:47:17.892Z [container-agent] 2024-03-01 15:47:17 WARNING juju-log No relation: certificates
2024-03-01T15:47:18.285Z [container-agent] 2024-03-01 15:47:18 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T15:51:38.140Z [container-agent] 2024-03-01 15:51:38 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T15:51:40.950Z [container-agent] 2024-03-01 15:51:40 WARNING juju-log No relation: certificates
2024-03-01T15:51:41.267Z [container-agent] 2024-03-01 15:51:41 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T15:57:22.648Z [container-agent] 2024-03-01 15:57:22 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T15:57:25.898Z [container-agent] 2024-03-01 15:57:25 WARNING juju-log No relation: certificates
2024-03-01T15:57:26.283Z [container-agent] 2024-03-01 15:57:26 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T16:01:52.515Z [container-agent] 2024-03-01 16:01:52 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T16:01:56.179Z [container-agent] 2024-03-01 16:01:56 WARNING juju-log No relation: certificates
2024-03-01T16:01:56.531Z [container-agent] 2024-03-01 16:01:56 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T16:07:33.634Z [container-agent] 2024-03-01 16:07:33 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T16:07:38.141Z [container-agent] 2024-03-01 16:07:38 WARNING juju-log No relation: certificates
2024-03-01T16:07:38.492Z [container-agent] 2024-03-01 16:07:38 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T16:12:44.069Z [container-agent] 2024-03-01 16:12:44 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T16:12:48.995Z [container-agent] 2024-03-01 16:12:48 WARNING juju-log No relation: certificates
2024-03-01T16:12:49.370Z [container-agent] 2024-03-01 16:12:49 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T16:17:52.528Z [container-agent] 2024-03-01 16:17:52 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T16:17:57.169Z [container-agent] 2024-03-01 16:17:57 WARNING juju-log No relation: certificates
2024-03-01T16:17:57.526Z [container-agent] 2024-03-01 16:17:57 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T16:23:48.434Z [container-agent] 2024-03-01 16:23:48 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T16:23:54.308Z [container-agent] 2024-03-01 16:23:54 WARNING juju-log No relation: certificates
2024-03-01T16:23:54.671Z [container-agent] 2024-03-01 16:23:54 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T16:29:13.595Z [container-agent] 2024-03-01 16:29:13 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T16:29:20.013Z [container-agent] 2024-03-01 16:29:20 WARNING juju-log No relation: certificates
2024-03-01T16:29:20.368Z [container-agent] 2024-03-01 16:29:20 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T16:34:41.335Z [container-agent] 2024-03-01 16:34:41 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T16:34:47.983Z [container-agent] 2024-03-01 16:34:47 WARNING juju-log No relation: certificates
2024-03-01T16:34:48.330Z [container-agent] 2024-03-01 16:34:48 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T16:39:52.170Z [container-agent] 2024-03-01 16:39:52 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T16:39:59.674Z [container-agent] 2024-03-01 16:39:59 WARNING juju-log No relation: certificates
2024-03-01T16:40:00.053Z [container-agent] 2024-03-01 16:40:00 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T16:45:22.361Z [container-agent] 2024-03-01 16:45:22 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T16:45:29.475Z [container-agent] 2024-03-01 16:45:29 WARNING juju-log No relation: certificates
2024-03-01T16:45:29.832Z [container-agent] 2024-03-01 16:45:29 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T16:50:57.116Z [container-agent] 2024-03-01 16:50:57 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-03-01T16:50:57.141Z [container-agent] 2024-03-01 16:50:57 INFO juju-log Attempting reboot from complete outage.
2024-03-01T16:51:04.695Z [container-agent] 2024-03-01 16:51:04 WARNING juju-log No relation: certificates
2024-03-01T16:51:05.026Z [container-agent] 2024-03-01 16:51:05 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T16:54:04.758Z [container-agent] 2024-03-01 16:54:04 ERROR juju-log Uncaught exception while in charm code:
2024-03-01T16:54:04.758Z [container-agent] Traceback (most recent call last):
2024-03-01T16:54:04.758Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 2507, in exec
2024-03-01T16:54:04.758Z [container-agent]     control_ws = self._connect_websocket(task_id, 'control')
2024-03-01T16:54:04.758Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 2589, in _connect_websocket
2024-03-01T16:54:04.758Z [container-agent]     ws.connect(url, socket=sock)
2024-03-01T16:54:04.758Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_core.py", line 253, in connect
2024-03-01T16:54:04.758Z [container-agent]     self.handshake_response = handshake(self.sock, url, *addrs, **options)
2024-03-01T16:54:04.758Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_handshake.py", line 57, in handshake
2024-03-01T16:54:04.758Z [container-agent]     status, resp = _get_resp_headers(sock)
2024-03-01T16:54:04.758Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_handshake.py", line 143, in _get_resp_headers
2024-03-01T16:54:04.758Z [container-agent]     status, resp_headers, status_message = read_headers(sock)
2024-03-01T16:54:04.758Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_http.py", line 312, in read_headers
2024-03-01T16:54:04.758Z [container-agent]     line = recv_line(sock)
2024-03-01T16:54:04.758Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_socket.py", line 131, in recv_line
2024-03-01T16:54:04.758Z [container-agent]     c = recv(sock, 1)
2024-03-01T16:54:04.758Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_socket.py", line 122, in recv
2024-03-01T16:54:04.758Z [container-agent]     raise WebSocketConnectionClosedException(
2024-03-01T16:54:04.758Z [container-agent] websocket._exceptions.WebSocketConnectionClosedException: Connection to remote host was lost.
2024-03-01T16:54:04.758Z [container-agent] 
2024-03-01T16:54:04.758Z [container-agent] During handling of the above exception, another exception occurred:
2024-03-01T16:54:04.758Z [container-agent] 
2024-03-01T16:54:04.758Z [container-agent] Traceback (most recent call last):
2024-03-01T16:54:04.758Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 1348, in do_open
2024-03-01T16:54:04.758Z [container-agent]     h.request(req.get_method(), req.selector, req.data, headers,
2024-03-01T16:54:04.758Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1283, in request
2024-03-01T16:54:04.758Z [container-agent]     self._send_request(method, url, body, headers, encode_chunked)
2024-03-01T16:54:04.758Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1329, in _send_request
2024-03-01T16:54:04.758Z [container-agent]     self.endheaders(body, encode_chunked=encode_chunked)
2024-03-01T16:54:04.758Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1278, in endheaders
2024-03-01T16:54:04.758Z [container-agent]     self._send_output(message_body, encode_chunked=encode_chunked)
2024-03-01T16:54:04.758Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1038, in _send_output
2024-03-01T16:54:04.758Z [container-agent]     self.send(msg)
2024-03-01T16:54:04.758Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 976, in send
2024-03-01T16:54:04.758Z [container-agent]     self.connect()
2024-03-01T16:54:04.758Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 297, in connect
2024-03-01T16:54:04.758Z [container-agent]     self.sock.connect(self.socket_path)
2024-03-01T16:54:04.758Z [container-agent] ConnectionRefusedError: [Errno 111] Connection refused
2024-03-01T16:54:04.758Z [container-agent] 
2024-03-01T16:54:04.758Z [container-agent] During handling of the above exception, another exception occurred:
2024-03-01T16:54:04.758Z [container-agent] 
2024-03-01T16:54:04.758Z [container-agent] Traceback (most recent call last):
2024-03-01T16:54:04.758Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1690, in _request_raw
2024-03-01T16:54:04.758Z [container-agent]     response = self.opener.open(request, timeout=self.timeout)
2024-03-01T16:54:04.758Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 519, in open
2024-03-01T16:54:04.758Z [container-agent]     response = self._open(req, data)
2024-03-01T16:54:04.758Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 536, in _open
2024-03-01T16:54:04.758Z [container-agent]     result = self._call_chain(self.handle_open, protocol, protocol +
2024-03-01T16:54:04.758Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 496, in _call_chain
2024-03-01T16:54:04.758Z [container-agent]     result = func(*args)
2024-03-01T16:54:04.758Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 311, in http_open
2024-03-01T16:54:04.758Z [container-agent]     return self.do_open(_UnixSocketConnection, req,  # type:ignore
2024-03-01T16:54:04.758Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 1351, in do_open
2024-03-01T16:54:04.758Z [container-agent]     raise URLError(err)
2024-03-01T16:54:04.758Z [container-agent] urllib.error.URLError: <urlopen error [Errno 111] Connection refused>
2024-03-01T16:54:04.758Z [container-agent] 
2024-03-01T16:54:04.758Z [container-agent] The above exception was the direct cause of the following exception:
2024-03-01T16:54:04.758Z [container-agent] 
2024-03-01T16:54:04.758Z [container-agent] Traceback (most recent call last):
2024-03-01T16:54:04.758Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/./src/charm.py", line 786, in <module>
2024-03-01T16:54:04.758Z [container-agent]     main(MySQLOperatorCharm)
2024-03-01T16:54:04.758Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 436, in main
2024-03-01T16:54:04.758Z [container-agent]     _emit_charm_event(charm, dispatcher.event_name)
2024-03-01T16:54:04.758Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 144, in _emit_charm_event
2024-03-01T16:54:04.758Z [container-agent]     event_to_emit.emit(*args, **kwargs)
2024-03-01T16:54:04.758Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 351, in emit
2024-03-01T16:54:04.758Z [container-agent]     framework._emit(event)
2024-03-01T16:54:04.758Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 853, in _emit
2024-03-01T16:54:04.758Z [container-agent]     self._reemit(event_path)
2024-03-01T16:54:04.758Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 942, in _reemit
2024-03-01T16:54:04.758Z [container-agent]     custom_handler(event)
2024-03-01T16:54:04.758Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/rotate_mysql_logs.py", line 55, in _rotate_mysql_logs
2024-03-01T16:54:04.758Z [container-agent]     self.charm._mysql.flush_mysql_logs(MySQLTextLogs.ERROR)
2024-03-01T16:54:04.758Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-03-01T16:54:04.758Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-03-01T16:54:04.758Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 665, in _run_mysqlsh_script
2024-03-01T16:54:04.758Z [container-agent]     process = self.container.exec(cmd, timeout=timeout)
2024-03-01T16:54:04.758Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/model.py", line 2688, in exec
2024-03-01T16:54:04.758Z [container-agent]     return self._pebble.exec(
2024-03-01T16:54:04.758Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 2514, in exec
2024-03-01T16:54:04.758Z [container-agent]     change = self.wait_change(ChangeID(change_id))
2024-03-01T16:54:04.758Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1893, in wait_change
2024-03-01T16:54:04.758Z [container-agent]     return self._wait_change_using_wait(change_id, timeout)
2024-03-01T16:54:04.758Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1914, in _wait_change_using_wait
2024-03-01T16:54:04.758Z [container-agent]     return self._wait_change(change_id, this_timeout)
2024-03-01T16:54:04.758Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1928, in _wait_change
2024-03-01T16:54:04.758Z [container-agent]     resp = self._request('GET', f'/v1/changes/{change_id}/wait', query)
2024-03-01T16:54:04.758Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1655, in _request
2024-03-01T16:54:04.758Z [container-agent]     response = self._request_raw(method, path, query, headers, data)
2024-03-01T16:54:04.758Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1707, in _request_raw
2024-03-01T16:54:04.758Z [container-agent]     raise ConnectionError(e.reason) from e
2024-03-01T16:54:04.758Z [container-agent] ops.pebble.ConnectionError: [Errno 111] Connection refused
2024-03-01T16:54:04.952Z [container-agent] 2024-03-01 16:54:04 INFO juju.util.exec exec.go:209 run result: exit status 1
2024-03-01T16:54:06.567Z [container-agent] 2024-03-01 16:54:06 INFO juju-log Setting up the logrotate configurations
2024-03-01T16:54:06.630Z [container-agent] 2024-03-01 16:54:06 INFO juju-log Adding pebble layer
2024-03-01T16:54:13.406Z [container-agent] 2024-03-01 16:54:13 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-03-01T16:54:13.428Z [container-agent] 2024-03-01 16:54:13 INFO juju-log Attempting reboot from complete outage.
2024-03-01T16:54:16.298Z [container-agent] 2024-03-01 16:54:16 INFO juju.worker.uniter.operation runhook.go:186 ran "mysql-pebble-ready" hook (via hook dispatching script: dispatch)
2024-03-01T16:55:16.465Z [container-agent] 2024-03-01 16:55:16 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T16:55:18.397Z [container-agent] 2024-03-01 16:55:18 WARNING juju-log No relation: certificates
2024-03-01T16:55:18.745Z [container-agent] 2024-03-01 16:55:18 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T17:00:11.852Z [container-agent] 2024-03-01 17:00:11 INFO juju-log Starting the log rotate manager
2024-03-01T17:00:11.866Z [container-agent] 2024-03-01 17:00:11 INFO juju-log Started log rotate manager process with PID 442092
2024-03-01T17:00:12.179Z [container-agent] 2024-03-01 17:00:12 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T17:00:14.212Z [container-agent] 2024-03-01 17:00:14 WARNING juju-log No relation: certificates
2024-03-01T17:00:14.556Z [container-agent] 2024-03-01 17:00:14 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T17:04:15.089Z [container-agent] 2024-03-01 17:04:15 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T17:04:17.543Z [container-agent] 2024-03-01 17:04:17 WARNING juju-log No relation: certificates
2024-03-01T17:04:17.887Z [container-agent] 2024-03-01 17:04:17 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T17:09:27.816Z [container-agent] 2024-03-01 17:09:27 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T17:09:30.618Z [container-agent] 2024-03-01 17:09:30 WARNING juju-log No relation: certificates
2024-03-01T17:09:30.968Z [container-agent] 2024-03-01 17:09:30 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T17:15:07.089Z [container-agent] 2024-03-01 17:15:07 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T17:15:10.585Z [container-agent] 2024-03-01 17:15:10 WARNING juju-log No relation: certificates
2024-03-01T17:15:10.964Z [container-agent] 2024-03-01 17:15:10 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T17:20:10.949Z [container-agent] 2024-03-01 17:20:10 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T17:20:14.798Z [container-agent] 2024-03-01 17:20:14 WARNING juju-log No relation: certificates
2024-03-01T17:20:15.142Z [container-agent] 2024-03-01 17:20:15 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T17:24:22.829Z [container-agent] 2024-03-01 17:24:22 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T17:24:26.683Z [container-agent] 2024-03-01 17:24:26 WARNING juju-log No relation: certificates
2024-03-01T17:24:27.006Z [container-agent] 2024-03-01 17:24:27 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T17:29:30.961Z [container-agent] 2024-03-01 17:29:30 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T17:29:35.720Z [container-agent] 2024-03-01 17:29:35 WARNING juju-log No relation: certificates
2024-03-01T17:29:36.077Z [container-agent] 2024-03-01 17:29:36 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T17:35:27.267Z [container-agent] 2024-03-01 17:35:27 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T17:35:32.509Z [container-agent] 2024-03-01 17:35:32 WARNING juju-log No relation: certificates
2024-03-01T17:35:32.918Z [container-agent] 2024-03-01 17:35:32 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T17:40:26.909Z [container-agent] 2024-03-01 17:40:26 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T17:40:32.381Z [container-agent] 2024-03-01 17:40:32 WARNING juju-log No relation: certificates
2024-03-01T17:40:32.738Z [container-agent] 2024-03-01 17:40:32 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T17:45:11.051Z [container-agent] 2024-03-01 17:45:11 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T17:45:17.170Z [container-agent] 2024-03-01 17:45:17 WARNING juju-log No relation: certificates
2024-03-01T17:45:17.532Z [container-agent] 2024-03-01 17:45:17 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T17:49:17.639Z [container-agent] 2024-03-01 17:49:17 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T17:49:24.277Z [container-agent] 2024-03-01 17:49:24 WARNING juju-log No relation: certificates
2024-03-01T17:49:24.620Z [container-agent] 2024-03-01 17:49:24 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T17:54:54.526Z [container-agent] 2024-03-01 17:54:54 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T17:55:01.106Z [container-agent] 2024-03-01 17:55:01 WARNING juju-log No relation: certificates
2024-03-01T17:55:01.455Z [container-agent] 2024-03-01 17:55:01 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T17:59:23.119Z [container-agent] 2024-03-01 17:59:23 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T17:59:30.072Z [container-agent] 2024-03-01 17:59:30 WARNING juju-log No relation: certificates
2024-03-01T17:59:30.420Z [container-agent] 2024-03-01 17:59:30 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T18:05:21.562Z [container-agent] 2024-03-01 18:05:21 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T18:05:29.462Z [container-agent] 2024-03-01 18:05:29 WARNING juju-log No relation: certificates
2024-03-01T18:05:29.844Z [container-agent] 2024-03-01 18:05:29 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T18:10:11.358Z [container-agent] 2024-03-01 18:10:11 ERROR juju-log Failed to flush GENERAL LOGS logs.
2024-03-01T18:10:11.358Z [container-agent] Traceback (most recent call last):
2024-03-01T18:10:11.358Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 666, in _run_mysqlsh_script
2024-03-01T18:10:11.358Z [container-agent]     stdout, _ = process.wait_output()
2024-03-01T18:10:11.358Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1441, in wait_output
2024-03-01T18:10:11.358Z [container-agent]     raise ExecError[AnyStr](self._command, exit_code, out_value, err_value)
2024-03-01T18:10:11.358Z [container-agent] ops.pebble.ExecError: non-zero exit code 1 executing ['/usr/bin/mysqlsh', '--no-wizard', '--python', '--verbose=1', '-f', '/tmp/script.py', ';', 'rm', '/tmp/script.py'], stdout='', stderr='Cannot set LC_ALL to locale en_US.UTF-8: No such file or directory\nverbose: 2024-03-01T18:10:08Z: Loading startup files...\nverbose: 2024-03-01T18:10:08Z: Loading plugins...\nverbose: 2024-03-01T18:10:08Z: Connecting to MySQL at: serverconfig@horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local\nTraceback (most recent call last):\n  File "<string>", line 1, in <module>\nmysqlsh.DBError: MySQL Error (2013): Shell.connect: Lost connection to MySQL server at \'reading initial communication packet\', system error: 104\n'
2024-03-01T18:10:11.359Z [container-agent] 
2024-03-01T18:10:11.359Z [container-agent] During handling of the above exception, another exception occurred:
2024-03-01T18:10:11.359Z [container-agent] 
2024-03-01T18:10:11.359Z [container-agent] Traceback (most recent call last):
2024-03-01T18:10:11.359Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-03-01T18:10:11.359Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-03-01T18:10:11.359Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 669, in _run_mysqlsh_script
2024-03-01T18:10:11.359Z [container-agent]     raise MySQLClientError(e.stderr)
2024-03-01T18:10:11.359Z [container-agent] charms.mysql.v0.mysql.MySQLClientError: Cannot set LC_ALL to locale en_US.UTF-8: No such file or directory
2024-03-01T18:10:11.359Z [container-agent] verbose: 2024-03-01T18:10:08Z: Loading startup files...
2024-03-01T18:10:11.359Z [container-agent] verbose: 2024-03-01T18:10:08Z: Loading plugins...
2024-03-01T18:10:11.359Z [container-agent] verbose: 2024-03-01T18:10:08Z: Connecting to MySQL at: serverconfig@horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local
2024-03-01T18:10:11.359Z [container-agent] Traceback (most recent call last):
2024-03-01T18:10:11.359Z [container-agent]   File "<string>", line 1, in <module>
2024-03-01T18:10:11.359Z [container-agent] mysqlsh.DBError: MySQL Error (2013): Shell.connect: Lost connection to MySQL server at 'reading initial communication packet', system error: 104
2024-03-01T18:10:11.359Z [container-agent] 
2024-03-01T18:10:13.472Z [container-agent] 2024-03-01 18:10:13 ERROR juju-log Failed to flush SLOW LOGS logs.
2024-03-01T18:10:13.472Z [container-agent] Traceback (most recent call last):
2024-03-01T18:10:13.472Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 666, in _run_mysqlsh_script
2024-03-01T18:10:13.472Z [container-agent]     stdout, _ = process.wait_output()
2024-03-01T18:10:13.472Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1441, in wait_output
2024-03-01T18:10:13.472Z [container-agent]     raise ExecError[AnyStr](self._command, exit_code, out_value, err_value)
2024-03-01T18:10:13.472Z [container-agent] ops.pebble.ExecError: non-zero exit code 1 executing ['/usr/bin/mysqlsh', '--no-wizard', '--python', '--verbose=1', '-f', '/tmp/script.py', ';', 'rm', '/tmp/script.py'], stdout='', stderr='Cannot set LC_ALL to locale en_US.UTF-8: No such file or directory\nverbose: 2024-03-01T18:10:12Z: Loading startup files...\nverbose: 2024-03-01T18:10:12Z: Loading plugins...\nverbose: 2024-03-01T18:10:12Z: Connecting to MySQL at: serverconfig@horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local\nTraceback (most recent call last):\n  File "<string>", line 1, in <module>\nmysqlsh.DBError: MySQL Error (2003): Shell.connect: Can\'t connect to MySQL server on \'horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local:3306\' (111)\n'
2024-03-01T18:10:13.472Z [container-agent] 
2024-03-01T18:10:13.472Z [container-agent] During handling of the above exception, another exception occurred:
2024-03-01T18:10:13.472Z [container-agent] 
2024-03-01T18:10:13.472Z [container-agent] Traceback (most recent call last):
2024-03-01T18:10:13.472Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-03-01T18:10:13.472Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-03-01T18:10:13.472Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 669, in _run_mysqlsh_script
2024-03-01T18:10:13.472Z [container-agent]     raise MySQLClientError(e.stderr)
2024-03-01T18:10:13.472Z [container-agent] charms.mysql.v0.mysql.MySQLClientError: Cannot set LC_ALL to locale en_US.UTF-8: No such file or directory
2024-03-01T18:10:13.472Z [container-agent] verbose: 2024-03-01T18:10:12Z: Loading startup files...
2024-03-01T18:10:13.472Z [container-agent] verbose: 2024-03-01T18:10:12Z: Loading plugins...
2024-03-01T18:10:13.472Z [container-agent] verbose: 2024-03-01T18:10:12Z: Connecting to MySQL at: serverconfig@horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local
2024-03-01T18:10:13.472Z [container-agent] Traceback (most recent call last):
2024-03-01T18:10:13.472Z [container-agent]   File "<string>", line 1, in <module>
2024-03-01T18:10:13.472Z [container-agent] mysqlsh.DBError: MySQL Error (2003): Shell.connect: Can't connect to MySQL server on 'horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local:3306' (111)
2024-03-01T18:10:13.472Z [container-agent] 
2024-03-01T18:10:19.483Z [container-agent] 2024-03-01 18:10:19 INFO juju-log Setting up the logrotate configurations
2024-03-01T18:10:19.561Z [container-agent] 2024-03-01 18:10:19 INFO juju-log Adding pebble layer
2024-03-01T18:10:26.278Z [container-agent] 2024-03-01 18:10:26 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-03-01T18:10:26.308Z [container-agent] 2024-03-01 18:10:26 INFO juju-log Attempting reboot from complete outage.
2024-03-01T18:10:29.043Z [container-agent] 2024-03-01 18:10:29 INFO juju.worker.uniter.operation runhook.go:186 ran "mysql-pebble-ready" hook (via hook dispatching script: dispatch)
2024-03-01T18:10:41.359Z [container-agent] 2024-03-01 18:10:41 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T18:10:43.206Z [container-agent] 2024-03-01 18:10:43 WARNING juju-log No relation: certificates
2024-03-01T18:10:43.547Z [container-agent] 2024-03-01 18:10:43 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T18:15:42.942Z [container-agent] 2024-03-01 18:15:42 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T18:15:45.382Z [container-agent] 2024-03-01 18:15:45 WARNING juju-log No relation: certificates
2024-03-01T18:15:45.747Z [container-agent] 2024-03-01 18:15:45 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T18:21:16.781Z [container-agent] 2024-03-01 18:21:16 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T18:21:19.909Z [container-agent] 2024-03-01 18:21:19 WARNING juju-log No relation: certificates
2024-03-01T18:21:20.265Z [container-agent] 2024-03-01 18:21:20 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T18:25:38.400Z [container-agent] 2024-03-01 18:25:38 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T18:25:41.567Z [container-agent] 2024-03-01 18:25:41 WARNING juju-log No relation: certificates
2024-03-01T18:25:41.932Z [container-agent] 2024-03-01 18:25:41 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T18:29:45.314Z [container-agent] 2024-03-01 18:29:45 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T18:29:49.307Z [container-agent] 2024-03-01 18:29:49 WARNING juju-log No relation: certificates
2024-03-01T18:29:49.653Z [container-agent] 2024-03-01 18:29:49 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T18:34:15.244Z [container-agent] 2024-03-01 18:34:15 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T18:34:19.514Z [container-agent] 2024-03-01 18:34:19 WARNING juju-log No relation: certificates
2024-03-01T18:34:19.883Z [container-agent] 2024-03-01 18:34:19 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T18:39:10.248Z [container-agent] 2024-03-01 18:39:10 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T18:39:14.678Z [container-agent] 2024-03-01 18:39:14 WARNING juju-log No relation: certificates
2024-03-01T18:39:15.033Z [container-agent] 2024-03-01 18:39:15 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T18:43:40.865Z [container-agent] 2024-03-01 18:43:40 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T18:43:45.811Z [container-agent] 2024-03-01 18:43:45 WARNING juju-log No relation: certificates
2024-03-01T18:43:46.157Z [container-agent] 2024-03-01 18:43:46 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T18:48:10.096Z [container-agent] 2024-03-01 18:48:10 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T18:48:15.548Z [container-agent] 2024-03-01 18:48:15 WARNING juju-log No relation: certificates
2024-03-01T18:48:15.904Z [container-agent] 2024-03-01 18:48:15 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T18:52:51.059Z [container-agent] 2024-03-01 18:52:51 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T18:52:56.749Z [container-agent] 2024-03-01 18:52:56 WARNING juju-log No relation: certificates
2024-03-01T18:52:57.100Z [container-agent] 2024-03-01 18:52:57 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T18:56:52.132Z [container-agent] 2024-03-01 18:56:52 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T18:56:58.311Z [container-agent] 2024-03-01 18:56:58 WARNING juju-log No relation: certificates
2024-03-01T18:56:58.672Z [container-agent] 2024-03-01 18:56:58 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T19:01:27.295Z [container-agent] 2024-03-01 19:01:27 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T19:01:34.169Z [container-agent] 2024-03-01 19:01:34 WARNING juju-log No relation: certificates
2024-03-01T19:01:34.529Z [container-agent] 2024-03-01 19:01:34 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T19:05:50.127Z [container-agent] 2024-03-01 19:05:50 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T19:05:57.290Z [container-agent] 2024-03-01 19:05:57 WARNING juju-log No relation: certificates
2024-03-01T19:05:57.688Z [container-agent] 2024-03-01 19:05:57 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T19:10:46.039Z [container-agent] 2024-03-01 19:10:46 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T19:10:53.672Z [container-agent] 2024-03-01 19:10:53 WARNING juju-log No relation: certificates
2024-03-01T19:10:54.029Z [container-agent] 2024-03-01 19:10:54 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T19:16:35.893Z [container-agent] 2024-03-01 19:16:35 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T19:16:44.245Z [container-agent] 2024-03-01 19:16:44 WARNING juju-log No relation: certificates
2024-03-01T19:16:44.611Z [container-agent] 2024-03-01 19:16:44 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T19:21:14.852Z [container-agent] 2024-03-01 19:21:14 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T19:21:23.386Z [container-agent] 2024-03-01 19:21:23 WARNING juju-log No relation: certificates
2024-03-01T19:21:23.736Z [container-agent] 2024-03-01 19:21:23 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T19:22:07.869Z [container-agent] 2024-03-01 19:22:07 ERROR juju-log Failed to flush ERROR LOGS logs.
2024-03-01T19:22:07.869Z [container-agent] Traceback (most recent call last):
2024-03-01T19:22:07.869Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 666, in _run_mysqlsh_script
2024-03-01T19:22:07.869Z [container-agent]     stdout, _ = process.wait_output()
2024-03-01T19:22:07.869Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1441, in wait_output
2024-03-01T19:22:07.869Z [container-agent]     raise ExecError[AnyStr](self._command, exit_code, out_value, err_value)
2024-03-01T19:22:07.869Z [container-agent] ops.pebble.ExecError: non-zero exit code 1 executing ['/usr/bin/mysqlsh', '--no-wizard', '--python', '--verbose=1', '-f', '/tmp/script.py', ';', 'rm', '/tmp/script.py'], stdout='', stderr='Cannot set LC_ALL to locale en_US.UTF-8: No such file or directory\nverbose: 2024-03-01T19:22:06Z: Loading startup files...\nverbose: 2024-03-01T19:22:06Z: Loading plugins...\nverbose: 2024-03-01T19:22:06Z: Connecting to MySQL at: serverconfig@horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local\nTraceback (most recent call last):\n  File "<string>", line 1, in <module>\nmysqlsh.DBError: MySQL Error (2003): Shell.connect: Can\'t connect to MySQL server on \'horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local:3306\' (111)\n'
2024-03-01T19:22:07.869Z [container-agent] 
2024-03-01T19:22:07.869Z [container-agent] During handling of the above exception, another exception occurred:
2024-03-01T19:22:07.869Z [container-agent] 
2024-03-01T19:22:07.869Z [container-agent] Traceback (most recent call last):
2024-03-01T19:22:07.869Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-03-01T19:22:07.869Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-03-01T19:22:07.869Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 669, in _run_mysqlsh_script
2024-03-01T19:22:07.869Z [container-agent]     raise MySQLClientError(e.stderr)
2024-03-01T19:22:07.869Z [container-agent] charms.mysql.v0.mysql.MySQLClientError: Cannot set LC_ALL to locale en_US.UTF-8: No such file or directory
2024-03-01T19:22:07.869Z [container-agent] verbose: 2024-03-01T19:22:06Z: Loading startup files...
2024-03-01T19:22:07.869Z [container-agent] verbose: 2024-03-01T19:22:06Z: Loading plugins...
2024-03-01T19:22:07.869Z [container-agent] verbose: 2024-03-01T19:22:06Z: Connecting to MySQL at: serverconfig@horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local
2024-03-01T19:22:07.869Z [container-agent] Traceback (most recent call last):
2024-03-01T19:22:07.869Z [container-agent]   File "<string>", line 1, in <module>
2024-03-01T19:22:07.869Z [container-agent] mysqlsh.DBError: MySQL Error (2003): Shell.connect: Can't connect to MySQL server on 'horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local:3306' (111)
2024-03-01T19:22:07.869Z [container-agent] 
2024-03-01T19:27:04.823Z [container-agent] 2024-03-01 19:27:04 ERROR juju-log Uncaught exception while in charm code:
2024-03-01T19:27:04.823Z [container-agent] Traceback (most recent call last):
2024-03-01T19:27:04.823Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 2507, in exec
2024-03-01T19:27:04.823Z [container-agent]     control_ws = self._connect_websocket(task_id, 'control')
2024-03-01T19:27:04.823Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 2589, in _connect_websocket
2024-03-01T19:27:04.823Z [container-agent]     ws.connect(url, socket=sock)
2024-03-01T19:27:04.823Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_core.py", line 253, in connect
2024-03-01T19:27:04.823Z [container-agent]     self.handshake_response = handshake(self.sock, url, *addrs, **options)
2024-03-01T19:27:04.823Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_handshake.py", line 57, in handshake
2024-03-01T19:27:04.823Z [container-agent]     status, resp = _get_resp_headers(sock)
2024-03-01T19:27:04.823Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_handshake.py", line 143, in _get_resp_headers
2024-03-01T19:27:04.823Z [container-agent]     status, resp_headers, status_message = read_headers(sock)
2024-03-01T19:27:04.823Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_http.py", line 312, in read_headers
2024-03-01T19:27:04.823Z [container-agent]     line = recv_line(sock)
2024-03-01T19:27:04.823Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_socket.py", line 131, in recv_line
2024-03-01T19:27:04.823Z [container-agent]     c = recv(sock, 1)
2024-03-01T19:27:04.823Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_socket.py", line 122, in recv
2024-03-01T19:27:04.823Z [container-agent]     raise WebSocketConnectionClosedException(
2024-03-01T19:27:04.823Z [container-agent] websocket._exceptions.WebSocketConnectionClosedException: Connection to remote host was lost.
2024-03-01T19:27:04.823Z [container-agent] 
2024-03-01T19:27:04.823Z [container-agent] During handling of the above exception, another exception occurred:
2024-03-01T19:27:04.823Z [container-agent] 
2024-03-01T19:27:04.823Z [container-agent] Traceback (most recent call last):
2024-03-01T19:27:04.823Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 1348, in do_open
2024-03-01T19:27:04.823Z [container-agent]     h.request(req.get_method(), req.selector, req.data, headers,
2024-03-01T19:27:04.823Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1283, in request
2024-03-01T19:27:04.823Z [container-agent]     self._send_request(method, url, body, headers, encode_chunked)
2024-03-01T19:27:04.823Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1329, in _send_request
2024-03-01T19:27:04.823Z [container-agent]     self.endheaders(body, encode_chunked=encode_chunked)
2024-03-01T19:27:04.823Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1278, in endheaders
2024-03-01T19:27:04.823Z [container-agent]     self._send_output(message_body, encode_chunked=encode_chunked)
2024-03-01T19:27:04.823Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1038, in _send_output
2024-03-01T19:27:04.823Z [container-agent]     self.send(msg)
2024-03-01T19:27:04.823Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 976, in send
2024-03-01T19:27:04.823Z [container-agent]     self.connect()
2024-03-01T19:27:04.823Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 297, in connect
2024-03-01T19:27:04.823Z [container-agent]     self.sock.connect(self.socket_path)
2024-03-01T19:27:04.823Z [container-agent] ConnectionRefusedError: [Errno 111] Connection refused
2024-03-01T19:27:04.823Z [container-agent] 
2024-03-01T19:27:04.823Z [container-agent] During handling of the above exception, another exception occurred:
2024-03-01T19:27:04.823Z [container-agent] 
2024-03-01T19:27:04.823Z [container-agent] Traceback (most recent call last):
2024-03-01T19:27:04.823Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1690, in _request_raw
2024-03-01T19:27:04.823Z [container-agent]     response = self.opener.open(request, timeout=self.timeout)
2024-03-01T19:27:04.823Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 519, in open
2024-03-01T19:27:04.823Z [container-agent]     response = self._open(req, data)
2024-03-01T19:27:04.823Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 536, in _open
2024-03-01T19:27:04.823Z [container-agent]     result = self._call_chain(self.handle_open, protocol, protocol +
2024-03-01T19:27:04.823Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 496, in _call_chain
2024-03-01T19:27:04.823Z [container-agent]     result = func(*args)
2024-03-01T19:27:04.823Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 311, in http_open
2024-03-01T19:27:04.823Z [container-agent]     return self.do_open(_UnixSocketConnection, req,  # type:ignore
2024-03-01T19:27:04.823Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 1351, in do_open
2024-03-01T19:27:04.823Z [container-agent]     raise URLError(err)
2024-03-01T19:27:04.823Z [container-agent] urllib.error.URLError: <urlopen error [Errno 111] Connection refused>
2024-03-01T19:27:04.823Z [container-agent] 
2024-03-01T19:27:04.823Z [container-agent] The above exception was the direct cause of the following exception:
2024-03-01T19:27:04.823Z [container-agent] 
2024-03-01T19:27:04.823Z [container-agent] Traceback (most recent call last):
2024-03-01T19:27:04.823Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/./src/charm.py", line 786, in <module>
2024-03-01T19:27:04.823Z [container-agent]     main(MySQLOperatorCharm)
2024-03-01T19:27:04.823Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 436, in main
2024-03-01T19:27:04.823Z [container-agent]     _emit_charm_event(charm, dispatcher.event_name)
2024-03-01T19:27:04.823Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 144, in _emit_charm_event
2024-03-01T19:27:04.823Z [container-agent]     event_to_emit.emit(*args, **kwargs)
2024-03-01T19:27:04.823Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 351, in emit
2024-03-01T19:27:04.823Z [container-agent]     framework._emit(event)
2024-03-01T19:27:04.823Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 853, in _emit
2024-03-01T19:27:04.823Z [container-agent]     self._reemit(event_path)
2024-03-01T19:27:04.823Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 942, in _reemit
2024-03-01T19:27:04.823Z [container-agent]     custom_handler(event)
2024-03-01T19:27:04.823Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/rotate_mysql_logs.py", line 55, in _rotate_mysql_logs
2024-03-01T19:27:04.823Z [container-agent]     self.charm._mysql.flush_mysql_logs(MySQLTextLogs.ERROR)
2024-03-01T19:27:04.823Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-03-01T19:27:04.823Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-03-01T19:27:04.823Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 665, in _run_mysqlsh_script
2024-03-01T19:27:04.823Z [container-agent]     process = self.container.exec(cmd, timeout=timeout)
2024-03-01T19:27:04.823Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/model.py", line 2688, in exec
2024-03-01T19:27:04.823Z [container-agent]     return self._pebble.exec(
2024-03-01T19:27:04.823Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 2514, in exec
2024-03-01T19:27:04.823Z [container-agent]     change = self.wait_change(ChangeID(change_id))
2024-03-01T19:27:04.823Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1893, in wait_change
2024-03-01T19:27:04.823Z [container-agent]     return self._wait_change_using_wait(change_id, timeout)
2024-03-01T19:27:04.823Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1914, in _wait_change_using_wait
2024-03-01T19:27:04.823Z [container-agent]     return self._wait_change(change_id, this_timeout)
2024-03-01T19:27:04.823Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1928, in _wait_change
2024-03-01T19:27:04.823Z [container-agent]     resp = self._request('GET', f'/v1/changes/{change_id}/wait', query)
2024-03-01T19:27:04.823Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1655, in _request
2024-03-01T19:27:04.823Z [container-agent]     response = self._request_raw(method, path, query, headers, data)
2024-03-01T19:27:04.823Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1707, in _request_raw
2024-03-01T19:27:04.823Z [container-agent]     raise ConnectionError(e.reason) from e
2024-03-01T19:27:04.823Z [container-agent] ops.pebble.ConnectionError: [Errno 111] Connection refused
2024-03-01T19:27:05.006Z [container-agent] 2024-03-01 19:27:05 INFO juju.util.exec exec.go:209 run result: exit status 1
2024-03-01T19:27:06.299Z [container-agent] 2024-03-01 19:27:06 INFO juju-log Starting the log rotate manager
2024-03-01T19:27:06.313Z [container-agent] 2024-03-01 19:27:06 INFO juju-log Started log rotate manager process with PID 456650
2024-03-01T19:27:07.003Z [container-agent] 2024-03-01 19:27:07 ERROR juju-log Failed to get cluster status for cluster-7573647c1e26466beeeac32b95ab8f91
2024-03-01T19:27:07.070Z [container-agent] 2024-03-01 19:27:07 INFO juju-log Kubernetes pod label error created
2024-03-01T19:27:07.084Z [container-agent] 2024-03-01 19:27:07 WARNING juju-log No relation: certificates
2024-03-01T19:27:07.502Z [container-agent] 2024-03-01 19:27:07 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T19:27:09.095Z [container-agent] 2024-03-01 19:27:09 INFO juju-log Setting up the logrotate configurations
2024-03-01T19:27:09.157Z [container-agent] 2024-03-01 19:27:09 INFO juju-log Adding pebble layer
2024-03-01T19:27:15.969Z [container-agent] 2024-03-01 19:27:15 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-03-01T19:27:15.996Z [container-agent] 2024-03-01 19:27:15 INFO juju-log Attempting reboot from complete outage.
2024-03-01T19:27:18.883Z [container-agent] 2024-03-01 19:27:18 INFO juju.worker.uniter.operation runhook.go:186 ran "mysql-pebble-ready" hook (via hook dispatching script: dispatch)
2024-03-01T19:32:44.736Z [container-agent] 2024-03-01 19:32:44 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T19:32:47.538Z [container-agent] 2024-03-01 19:32:47 INFO juju-log Kubernetes pod label primary created
2024-03-01T19:32:47.566Z [container-agent] 2024-03-01 19:32:47 WARNING juju-log No relation: certificates
2024-03-01T19:32:47.920Z [container-agent] 2024-03-01 19:32:47 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T19:37:35.718Z [container-agent] 2024-03-01 19:37:35 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T19:37:38.495Z [container-agent] 2024-03-01 19:37:38 WARNING juju-log No relation: certificates
2024-03-01T19:37:38.840Z [container-agent] 2024-03-01 19:37:38 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T19:42:26.122Z [container-agent] 2024-03-01 19:42:26 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T19:42:29.521Z [container-agent] 2024-03-01 19:42:29 WARNING juju-log No relation: certificates
2024-03-01T19:42:29.863Z [container-agent] 2024-03-01 19:42:29 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T19:48:13.139Z [container-agent] 2024-03-01 19:48:13 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T19:48:17.005Z [container-agent] 2024-03-01 19:48:17 WARNING juju-log No relation: certificates
2024-03-01T19:48:17.345Z [container-agent] 2024-03-01 19:48:17 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T19:53:30.875Z [container-agent] 2024-03-01 19:53:30 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T19:53:35.017Z [container-agent] 2024-03-01 19:53:35 WARNING juju-log No relation: certificates
2024-03-01T19:53:35.348Z [container-agent] 2024-03-01 19:53:35 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T19:58:14.257Z [container-agent] 2024-03-01 19:58:14 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T19:58:19.083Z [container-agent] 2024-03-01 19:58:19 WARNING juju-log No relation: certificates
2024-03-01T19:58:19.435Z [container-agent] 2024-03-01 19:58:19 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T20:03:10.393Z [container-agent] 2024-03-01 20:03:10 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T20:03:15.249Z [container-agent] 2024-03-01 20:03:15 WARNING juju-log No relation: certificates
2024-03-01T20:03:15.627Z [container-agent] 2024-03-01 20:03:15 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T20:07:10.526Z [container-agent] 2024-03-01 20:07:10 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T20:07:16.214Z [container-agent] 2024-03-01 20:07:16 WARNING juju-log No relation: certificates
2024-03-01T20:07:16.575Z [container-agent] 2024-03-01 20:07:16 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T20:12:42.239Z [container-agent] 2024-03-01 20:12:42 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T20:12:48.106Z [container-agent] 2024-03-01 20:12:48 WARNING juju-log No relation: certificates
2024-03-01T20:12:48.445Z [container-agent] 2024-03-01 20:12:48 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T20:17:21.110Z [container-agent] 2024-03-01 20:17:21 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T20:17:27.077Z [container-agent] 2024-03-01 20:17:27 WARNING juju-log No relation: certificates
2024-03-01T20:17:27.424Z [container-agent] 2024-03-01 20:17:27 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T20:21:53.584Z [container-agent] 2024-03-01 20:21:53 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T20:22:00.760Z [container-agent] 2024-03-01 20:22:00 WARNING juju-log No relation: certificates
2024-03-01T20:22:01.107Z [container-agent] 2024-03-01 20:22:01 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T20:26:21.154Z [container-agent] 2024-03-01 20:26:21 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T20:26:28.403Z [container-agent] 2024-03-01 20:26:28 WARNING juju-log No relation: certificates
2024-03-01T20:26:28.753Z [container-agent] 2024-03-01 20:26:28 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T20:31:27.363Z [container-agent] 2024-03-01 20:31:27 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T20:31:34.787Z [container-agent] 2024-03-01 20:31:34 WARNING juju-log No relation: certificates
2024-03-01T20:31:35.139Z [container-agent] 2024-03-01 20:31:35 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T20:37:17.708Z [container-agent] 2024-03-01 20:37:17 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T20:37:25.262Z [container-agent] 2024-03-01 20:37:25 WARNING juju-log No relation: certificates
2024-03-01T20:37:25.627Z [container-agent] 2024-03-01 20:37:25 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T20:42:24.980Z [container-agent] 2024-03-01 20:42:24 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T20:42:33.943Z [container-agent] 2024-03-01 20:42:33 WARNING juju-log No relation: certificates
2024-03-01T20:42:34.328Z [container-agent] 2024-03-01 20:42:34 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T20:47:50.638Z [container-agent] 2024-03-01 20:47:50 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T20:47:59.594Z [container-agent] 2024-03-01 20:47:59 WARNING juju-log No relation: certificates
2024-03-01T20:47:59.939Z [container-agent] 2024-03-01 20:47:59 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T20:53:22.602Z [container-agent] 2024-03-01 20:53:22 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T20:53:32.601Z [container-agent] 2024-03-01 20:53:32 WARNING juju-log No relation: certificates
2024-03-01T20:53:32.978Z [container-agent] 2024-03-01 20:53:32 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T20:58:04.060Z [container-agent] 2024-03-01 20:58:04 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-03-01T20:58:04.089Z [container-agent] 2024-03-01 20:58:04 INFO juju-log Attempting reboot from complete outage.
2024-03-01T20:58:12.309Z [container-agent] 2024-03-01 20:58:12 WARNING juju-log No relation: certificates
2024-03-01T20:58:12.663Z [container-agent] 2024-03-01 20:58:12 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T20:59:09.806Z [container-agent] 2024-03-01 20:59:09 ERROR juju-log Uncaught exception while in charm code:
2024-03-01T20:59:09.806Z [container-agent] Traceback (most recent call last):
2024-03-01T20:59:09.806Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/./src/charm.py", line 786, in <module>
2024-03-01T20:59:09.806Z [container-agent]     main(MySQLOperatorCharm)
2024-03-01T20:59:09.806Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 436, in main
2024-03-01T20:59:09.806Z [container-agent]     _emit_charm_event(charm, dispatcher.event_name)
2024-03-01T20:59:09.806Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 144, in _emit_charm_event
2024-03-01T20:59:09.806Z [container-agent]     event_to_emit.emit(*args, **kwargs)
2024-03-01T20:59:09.806Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 351, in emit
2024-03-01T20:59:09.806Z [container-agent]     framework._emit(event)
2024-03-01T20:59:09.806Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 853, in _emit
2024-03-01T20:59:09.806Z [container-agent]     self._reemit(event_path)
2024-03-01T20:59:09.806Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 942, in _reemit
2024-03-01T20:59:09.806Z [container-agent]     custom_handler(event)
2024-03-01T20:59:09.806Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/rotate_mysql_logs.py", line 56, in _rotate_mysql_logs
2024-03-01T20:59:09.806Z [container-agent]     self.charm._mysql.flush_mysql_logs(MySQLTextLogs.GENERAL)
2024-03-01T20:59:09.806Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-03-01T20:59:09.806Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-03-01T20:59:09.806Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 666, in _run_mysqlsh_script
2024-03-01T20:59:09.806Z [container-agent]     stdout, _ = process.wait_output()
2024-03-01T20:59:09.806Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1436, in wait_output
2024-03-01T20:59:09.806Z [container-agent]     exit_code: int = self._wait()
2024-03-01T20:59:09.806Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1375, in _wait
2024-03-01T20:59:09.806Z [container-agent]     change = self._client.wait_change(self._change_id, timeout=timeout)
2024-03-01T20:59:09.806Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1893, in wait_change
2024-03-01T20:59:09.806Z [container-agent]     return self._wait_change_using_wait(change_id, timeout)
2024-03-01T20:59:09.806Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1914, in _wait_change_using_wait
2024-03-01T20:59:09.806Z [container-agent]     return self._wait_change(change_id, this_timeout)
2024-03-01T20:59:09.806Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1928, in _wait_change
2024-03-01T20:59:09.806Z [container-agent]     resp = self._request('GET', f'/v1/changes/{change_id}/wait', query)
2024-03-01T20:59:09.806Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1655, in _request
2024-03-01T20:59:09.806Z [container-agent]     response = self._request_raw(method, path, query, headers, data)
2024-03-01T20:59:09.806Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1690, in _request_raw
2024-03-01T20:59:09.806Z [container-agent]     response = self.opener.open(request, timeout=self.timeout)
2024-03-01T20:59:09.806Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 519, in open
2024-03-01T20:59:09.806Z [container-agent]     response = self._open(req, data)
2024-03-01T20:59:09.806Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 536, in _open
2024-03-01T20:59:09.806Z [container-agent]     result = self._call_chain(self.handle_open, protocol, protocol +
2024-03-01T20:59:09.806Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 496, in _call_chain
2024-03-01T20:59:09.806Z [container-agent]     result = func(*args)
2024-03-01T20:59:09.806Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 311, in http_open
2024-03-01T20:59:09.806Z [container-agent]     return self.do_open(_UnixSocketConnection, req,  # type:ignore
2024-03-01T20:59:09.806Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 1352, in do_open
2024-03-01T20:59:09.806Z [container-agent]     r = h.getresponse()
2024-03-01T20:59:09.806Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1375, in getresponse
2024-03-01T20:59:09.806Z [container-agent]     response.begin()
2024-03-01T20:59:09.806Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 318, in begin
2024-03-01T20:59:09.806Z [container-agent]     version, status, reason = self._read_status()
2024-03-01T20:59:09.806Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 287, in _read_status
2024-03-01T20:59:09.806Z [container-agent]     raise RemoteDisconnected("Remote end closed connection without"
2024-03-01T20:59:09.806Z [container-agent] http.client.RemoteDisconnected: Remote end closed connection without response
2024-03-01T20:59:09.946Z [container-agent] 2024-03-01 20:59:09 INFO juju.util.exec exec.go:209 run result: exit status 1
2024-03-01T20:59:12.470Z [container-agent] 2024-03-01 20:59:12 INFO juju-log Setting up the logrotate configurations
2024-03-01T20:59:12.532Z [container-agent] 2024-03-01 20:59:12 INFO juju-log Adding pebble layer
2024-03-01T20:59:19.351Z [container-agent] 2024-03-01 20:59:19 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-03-01T20:59:19.377Z [container-agent] 2024-03-01 20:59:19 INFO juju-log Attempting reboot from complete outage.
2024-03-01T20:59:22.355Z [container-agent] 2024-03-01 20:59:22 INFO juju.worker.uniter.operation runhook.go:186 ran "mysql-pebble-ready" hook (via hook dispatching script: dispatch)
2024-03-01T21:03:56.664Z [container-agent] 2024-03-01 21:03:56 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T21:03:58.678Z [container-agent] 2024-03-01 21:03:58 WARNING juju-log No relation: certificates
2024-03-01T21:03:59.024Z [container-agent] 2024-03-01 21:03:59 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T21:08:03.926Z [container-agent] 2024-03-01 21:08:03 INFO juju-log Starting the log rotate manager
2024-03-01T21:08:03.944Z [container-agent] 2024-03-01 21:08:03 INFO juju-log Started log rotate manager process with PID 466086
2024-03-01T21:08:04.496Z [container-agent] 2024-03-01 21:08:04 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T21:08:06.874Z [container-agent] 2024-03-01 21:08:06 WARNING juju-log No relation: certificates
2024-03-01T21:08:07.311Z [container-agent] 2024-03-01 21:08:07 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T21:12:20.616Z [container-agent] 2024-03-01 21:12:20 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T21:12:23.100Z [container-agent] 2024-03-01 21:12:23 WARNING juju-log No relation: certificates
2024-03-01T21:12:23.431Z [container-agent] 2024-03-01 21:12:23 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T21:18:12.055Z [container-agent] 2024-03-01 21:18:12 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T21:18:15.035Z [container-agent] 2024-03-01 21:18:15 WARNING juju-log No relation: certificates
2024-03-01T21:18:15.408Z [container-agent] 2024-03-01 21:18:15 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T21:23:46.993Z [container-agent] 2024-03-01 21:23:46 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T21:23:50.591Z [container-agent] 2024-03-01 21:23:50 WARNING juju-log No relation: certificates
2024-03-01T21:23:50.935Z [container-agent] 2024-03-01 21:23:50 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T21:28:20.364Z [container-agent] 2024-03-01 21:28:20 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T21:28:24.065Z [container-agent] 2024-03-01 21:28:24 WARNING juju-log No relation: certificates
2024-03-01T21:28:24.417Z [container-agent] 2024-03-01 21:28:24 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T21:32:25.247Z [container-agent] 2024-03-01 21:32:25 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T21:32:29.676Z [container-agent] 2024-03-01 21:32:29 WARNING juju-log No relation: certificates
2024-03-01T21:32:30.023Z [container-agent] 2024-03-01 21:32:30 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T21:36:47.699Z [container-agent] 2024-03-01 21:36:47 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T21:36:52.373Z [container-agent] 2024-03-01 21:36:52 WARNING juju-log No relation: certificates
2024-03-01T21:36:52.754Z [container-agent] 2024-03-01 21:36:52 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T21:41:38.698Z [container-agent] 2024-03-01 21:41:38 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T21:41:43.870Z [container-agent] 2024-03-01 21:41:43 WARNING juju-log No relation: certificates
2024-03-01T21:41:44.211Z [container-agent] 2024-03-01 21:41:44 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T21:46:20.564Z [container-agent] 2024-03-01 21:46:20 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T21:46:25.539Z [container-agent] 2024-03-01 21:46:25 WARNING juju-log No relation: certificates
2024-03-01T21:46:25.880Z [container-agent] 2024-03-01 21:46:25 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T21:50:27.615Z [container-agent] 2024-03-01 21:50:27 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T21:50:33.714Z [container-agent] 2024-03-01 21:50:33 WARNING juju-log No relation: certificates
2024-03-01T21:50:34.103Z [container-agent] 2024-03-01 21:50:34 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T21:56:22.340Z [container-agent] 2024-03-01 21:56:22 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T21:56:28.876Z [container-agent] 2024-03-01 21:56:28 WARNING juju-log No relation: certificates
2024-03-01T21:56:29.232Z [container-agent] 2024-03-01 21:56:29 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T22:00:43.090Z [container-agent] 2024-03-01 22:00:43 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T22:00:49.955Z [container-agent] 2024-03-01 22:00:49 WARNING juju-log No relation: certificates
2024-03-01T22:00:50.321Z [container-agent] 2024-03-01 22:00:50 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T22:05:16.709Z [container-agent] 2024-03-01 22:05:16 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T22:05:24.067Z [container-agent] 2024-03-01 22:05:24 WARNING juju-log No relation: certificates
2024-03-01T22:05:24.410Z [container-agent] 2024-03-01 22:05:24 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T22:10:16.025Z [container-agent] 2024-03-01 22:10:16 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T22:10:23.742Z [container-agent] 2024-03-01 22:10:23 WARNING juju-log No relation: certificates
2024-03-01T22:10:24.103Z [container-agent] 2024-03-01 22:10:24 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T22:15:35.968Z [container-agent] 2024-03-01 22:15:35 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T22:15:44.248Z [container-agent] 2024-03-01 22:15:44 WARNING juju-log No relation: certificates
2024-03-01T22:15:44.618Z [container-agent] 2024-03-01 22:15:44 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T22:16:11.221Z [container-agent] 2024-03-01 22:16:11 ERROR juju-log Failed to flush GENERAL LOGS logs.
2024-03-01T22:16:11.221Z [container-agent] Traceback (most recent call last):
2024-03-01T22:16:11.221Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 666, in _run_mysqlsh_script
2024-03-01T22:16:11.221Z [container-agent]     stdout, _ = process.wait_output()
2024-03-01T22:16:11.221Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1441, in wait_output
2024-03-01T22:16:11.221Z [container-agent]     raise ExecError[AnyStr](self._command, exit_code, out_value, err_value)
2024-03-01T22:16:11.221Z [container-agent] ops.pebble.ExecError: non-zero exit code 1 executing ['/usr/bin/mysqlsh', '--no-wizard', '--python', '--verbose=1', '-f', '/tmp/script.py', ';', 'rm', '/tmp/script.py'], stdout='', stderr='Cannot set LC_ALL to locale en_US.UTF-8: No such file or directory\nverbose: 2024-03-01T22:16:08Z: Loading startup files...\nverbose: 2024-03-01T22:16:08Z: Loading plugins...\nverbose: 2024-03-01T22:16:08Z: Connecting to MySQL at: serverconfig@horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local\nTraceback (most recent call last):\n  File "<string>", line 1, in <module>\nmysqlsh.DBError: MySQL Error (2013): Shell.connect: Lost connection to MySQL server at \'reading initial communication packet\', system error: 104\n'
2024-03-01T22:16:11.221Z [container-agent] 
2024-03-01T22:16:11.221Z [container-agent] During handling of the above exception, another exception occurred:
2024-03-01T22:16:11.221Z [container-agent] 
2024-03-01T22:16:11.221Z [container-agent] Traceback (most recent call last):
2024-03-01T22:16:11.221Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-03-01T22:16:11.221Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-03-01T22:16:11.221Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 669, in _run_mysqlsh_script
2024-03-01T22:16:11.221Z [container-agent]     raise MySQLClientError(e.stderr)
2024-03-01T22:16:11.221Z [container-agent] charms.mysql.v0.mysql.MySQLClientError: Cannot set LC_ALL to locale en_US.UTF-8: No such file or directory
2024-03-01T22:16:11.221Z [container-agent] verbose: 2024-03-01T22:16:08Z: Loading startup files...
2024-03-01T22:16:11.221Z [container-agent] verbose: 2024-03-01T22:16:08Z: Loading plugins...
2024-03-01T22:16:11.221Z [container-agent] verbose: 2024-03-01T22:16:08Z: Connecting to MySQL at: serverconfig@horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local
2024-03-01T22:16:11.221Z [container-agent] Traceback (most recent call last):
2024-03-01T22:16:11.221Z [container-agent]   File "<string>", line 1, in <module>
2024-03-01T22:16:11.221Z [container-agent] mysqlsh.DBError: MySQL Error (2013): Shell.connect: Lost connection to MySQL server at 'reading initial communication packet', system error: 104
2024-03-01T22:16:11.221Z [container-agent] 
2024-03-01T22:16:14.351Z [container-agent] 2024-03-01 22:16:14 ERROR juju-log Failed to flush SLOW LOGS logs.
2024-03-01T22:16:14.351Z [container-agent] Traceback (most recent call last):
2024-03-01T22:16:14.351Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 666, in _run_mysqlsh_script
2024-03-01T22:16:14.351Z [container-agent]     stdout, _ = process.wait_output()
2024-03-01T22:16:14.351Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1441, in wait_output
2024-03-01T22:16:14.351Z [container-agent]     raise ExecError[AnyStr](self._command, exit_code, out_value, err_value)
2024-03-01T22:16:14.351Z [container-agent] ops.pebble.ExecError: non-zero exit code 1 executing ['/usr/bin/mysqlsh', '--no-wizard', '--python', '--verbose=1', '-f', '/tmp/script.py', ';', 'rm', '/tmp/script.py'], stdout='', stderr='Cannot set LC_ALL to locale en_US.UTF-8: No such file or directory\nverbose: 2024-03-01T22:16:12Z: Loading startup files...\nverbose: 2024-03-01T22:16:12Z: Loading plugins...\nverbose: 2024-03-01T22:16:12Z: Connecting to MySQL at: serverconfig@horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local\nTraceback (most recent call last):\n  File "<string>", line 1, in <module>\nmysqlsh.DBError: MySQL Error (2003): Shell.connect: Can\'t connect to MySQL server on \'horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local:3306\' (111)\n'
2024-03-01T22:16:14.351Z [container-agent] 
2024-03-01T22:16:14.351Z [container-agent] During handling of the above exception, another exception occurred:
2024-03-01T22:16:14.351Z [container-agent] 
2024-03-01T22:16:14.351Z [container-agent] Traceback (most recent call last):
2024-03-01T22:16:14.351Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-03-01T22:16:14.351Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-03-01T22:16:14.351Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 669, in _run_mysqlsh_script
2024-03-01T22:16:14.351Z [container-agent]     raise MySQLClientError(e.stderr)
2024-03-01T22:16:14.351Z [container-agent] charms.mysql.v0.mysql.MySQLClientError: Cannot set LC_ALL to locale en_US.UTF-8: No such file or directory
2024-03-01T22:16:14.351Z [container-agent] verbose: 2024-03-01T22:16:12Z: Loading startup files...
2024-03-01T22:16:14.351Z [container-agent] verbose: 2024-03-01T22:16:12Z: Loading plugins...
2024-03-01T22:16:14.351Z [container-agent] verbose: 2024-03-01T22:16:12Z: Connecting to MySQL at: serverconfig@horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local
2024-03-01T22:16:14.351Z [container-agent] Traceback (most recent call last):
2024-03-01T22:16:14.351Z [container-agent]   File "<string>", line 1, in <module>
2024-03-01T22:16:14.351Z [container-agent] mysqlsh.DBError: MySQL Error (2003): Shell.connect: Can't connect to MySQL server on 'horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local:3306' (111)
2024-03-01T22:16:14.351Z [container-agent] 
2024-03-01T22:16:16.461Z [container-agent] 2024-03-01 22:16:16 INFO juju-log Setting up the logrotate configurations
2024-03-01T22:16:16.533Z [container-agent] 2024-03-01 22:16:16 INFO juju-log Adding pebble layer
2024-03-01T22:16:23.289Z [container-agent] 2024-03-01 22:16:23 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-03-01T22:16:23.321Z [container-agent] 2024-03-01 22:16:23 INFO juju-log Attempting reboot from complete outage.
2024-03-01T22:16:26.255Z [container-agent] 2024-03-01 22:16:26 INFO juju.worker.uniter.operation runhook.go:186 ran "mysql-pebble-ready" hook (via hook dispatching script: dispatch)
2024-03-01T22:19:55.873Z [container-agent] 2024-03-01 22:19:55 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T22:19:57.963Z [container-agent] 2024-03-01 22:19:57 WARNING juju-log No relation: certificates
2024-03-01T22:19:58.306Z [container-agent] 2024-03-01 22:19:58 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T22:23:57.733Z [container-agent] 2024-03-01 22:23:57 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T22:24:00.587Z [container-agent] 2024-03-01 22:24:00 WARNING juju-log No relation: certificates
2024-03-01T22:24:00.968Z [container-agent] 2024-03-01 22:24:00 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T22:29:55.305Z [container-agent] 2024-03-01 22:29:55 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T22:29:58.487Z [container-agent] 2024-03-01 22:29:58 WARNING juju-log No relation: certificates
2024-03-01T22:29:58.849Z [container-agent] 2024-03-01 22:29:58 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T22:35:37.719Z [container-agent] 2024-03-01 22:35:37 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T22:35:41.228Z [container-agent] 2024-03-01 22:35:41 WARNING juju-log No relation: certificates
2024-03-01T22:35:41.617Z [container-agent] 2024-03-01 22:35:41 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T22:41:01.683Z [container-agent] 2024-03-01 22:41:01 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T22:41:05.719Z [container-agent] 2024-03-01 22:41:05 WARNING juju-log No relation: certificates
2024-03-01T22:41:06.051Z [container-agent] 2024-03-01 22:41:06 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T22:45:40.569Z [container-agent] 2024-03-01 22:45:40 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T22:45:44.873Z [container-agent] 2024-03-01 22:45:44 WARNING juju-log No relation: certificates
2024-03-01T22:45:45.212Z [container-agent] 2024-03-01 22:45:45 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T22:50:36.369Z [container-agent] 2024-03-01 22:50:36 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T22:50:41.209Z [container-agent] 2024-03-01 22:50:41 WARNING juju-log No relation: certificates
2024-03-01T22:50:41.593Z [container-agent] 2024-03-01 22:50:41 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T22:55:54.832Z [container-agent] 2024-03-01 22:55:54 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T22:56:00.395Z [container-agent] 2024-03-01 22:56:00 WARNING juju-log No relation: certificates
2024-03-01T22:56:00.744Z [container-agent] 2024-03-01 22:56:00 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T23:01:10.918Z [container-agent] 2024-03-01 23:01:10 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T23:01:16.999Z [container-agent] 2024-03-01 23:01:16 WARNING juju-log No relation: certificates
2024-03-01T23:01:17.360Z [container-agent] 2024-03-01 23:01:17 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T23:05:15.373Z [container-agent] 2024-03-01 23:05:15 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T23:05:21.770Z [container-agent] 2024-03-01 23:05:21 WARNING juju-log No relation: certificates
2024-03-01T23:05:22.119Z [container-agent] 2024-03-01 23:05:22 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T23:10:12.650Z [container-agent] 2024-03-01 23:10:12 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T23:10:19.329Z [container-agent] 2024-03-01 23:10:19 WARNING juju-log No relation: certificates
2024-03-01T23:10:19.650Z [container-agent] 2024-03-01 23:10:19 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T23:15:23.439Z [container-agent] 2024-03-01 23:15:23 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T23:15:30.653Z [container-agent] 2024-03-01 23:15:30 WARNING juju-log No relation: certificates
2024-03-01T23:15:31.045Z [container-agent] 2024-03-01 23:15:31 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T23:20:24.921Z [container-agent] 2024-03-01 23:20:24 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T23:20:31.768Z [container-agent] 2024-03-01 23:20:31 WARNING juju-log No relation: certificates
2024-03-01T23:20:32.143Z [container-agent] 2024-03-01 23:20:32 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T23:24:48.901Z [container-agent] 2024-03-01 23:24:48 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T23:24:57.299Z [container-agent] 2024-03-01 23:24:57 WARNING juju-log No relation: certificates
2024-03-01T23:24:57.671Z [container-agent] 2024-03-01 23:24:57 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T23:28:10.438Z [container-agent] 2024-03-01 23:28:10 ERROR juju-log Failed to flush ERROR LOGS logs.
2024-03-01T23:28:10.438Z [container-agent] Traceback (most recent call last):
2024-03-01T23:28:10.438Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 666, in _run_mysqlsh_script
2024-03-01T23:28:10.438Z [container-agent]     stdout, _ = process.wait_output()
2024-03-01T23:28:10.438Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1441, in wait_output
2024-03-01T23:28:10.438Z [container-agent]     raise ExecError[AnyStr](self._command, exit_code, out_value, err_value)
2024-03-01T23:28:10.438Z [container-agent] ops.pebble.ExecError: non-zero exit code 1 executing ['/usr/bin/mysqlsh', '--no-wizard', '--python', '--verbose=1', '-f', '/tmp/script.py', ';', 'rm', '/tmp/script.py'], stdout='', stderr='Cannot set LC_ALL to locale en_US.UTF-8: No such file or directory\nverbose: 2024-03-01T23:28:06Z: Loading startup files...\nverbose: 2024-03-01T23:28:06Z: Loading plugins...\nverbose: 2024-03-01T23:28:06Z: Connecting to MySQL at: serverconfig@horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local\nTraceback (most recent call last):\n  File "<string>", line 1, in <module>\nmysqlsh.DBError: MySQL Error (2013): Shell.connect: Lost connection to MySQL server at \'reading initial communication packet\', system error: 104\n'
2024-03-01T23:28:10.438Z [container-agent] 
2024-03-01T23:28:10.438Z [container-agent] During handling of the above exception, another exception occurred:
2024-03-01T23:28:10.438Z [container-agent] 
2024-03-01T23:28:10.438Z [container-agent] Traceback (most recent call last):
2024-03-01T23:28:10.438Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-03-01T23:28:10.438Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-03-01T23:28:10.438Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 669, in _run_mysqlsh_script
2024-03-01T23:28:10.438Z [container-agent]     raise MySQLClientError(e.stderr)
2024-03-01T23:28:10.438Z [container-agent] charms.mysql.v0.mysql.MySQLClientError: Cannot set LC_ALL to locale en_US.UTF-8: No such file or directory
2024-03-01T23:28:10.438Z [container-agent] verbose: 2024-03-01T23:28:06Z: Loading startup files...
2024-03-01T23:28:10.438Z [container-agent] verbose: 2024-03-01T23:28:06Z: Loading plugins...
2024-03-01T23:28:10.438Z [container-agent] verbose: 2024-03-01T23:28:06Z: Connecting to MySQL at: serverconfig@horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local
2024-03-01T23:28:10.438Z [container-agent] Traceback (most recent call last):
2024-03-01T23:28:10.438Z [container-agent]   File "<string>", line 1, in <module>
2024-03-01T23:28:10.438Z [container-agent] mysqlsh.DBError: MySQL Error (2013): Shell.connect: Lost connection to MySQL server at 'reading initial communication packet', system error: 104
2024-03-01T23:28:10.438Z [container-agent] 
2024-03-01T23:28:12.887Z [container-agent] 2024-03-01 23:28:12 ERROR juju-log Failed to flush GENERAL LOGS logs.
2024-03-01T23:28:12.887Z [container-agent] Traceback (most recent call last):
2024-03-01T23:28:12.887Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 666, in _run_mysqlsh_script
2024-03-01T23:28:12.887Z [container-agent]     stdout, _ = process.wait_output()
2024-03-01T23:28:12.887Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1441, in wait_output
2024-03-01T23:28:12.887Z [container-agent]     raise ExecError[AnyStr](self._command, exit_code, out_value, err_value)
2024-03-01T23:28:12.887Z [container-agent] ops.pebble.ExecError: non-zero exit code 1 executing ['/usr/bin/mysqlsh', '--no-wizard', '--python', '--verbose=1', '-f', '/tmp/script.py', ';', 'rm', '/tmp/script.py'], stdout='', stderr='Cannot set LC_ALL to locale en_US.UTF-8: No such file or directory\nverbose: 2024-03-01T23:28:11Z: Loading startup files...\nverbose: 2024-03-01T23:28:11Z: Loading plugins...\nverbose: 2024-03-01T23:28:11Z: Connecting to MySQL at: serverconfig@horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local\nTraceback (most recent call last):\n  File "<string>", line 1, in <module>\nmysqlsh.DBError: MySQL Error (2003): Shell.connect: Can\'t connect to MySQL server on \'horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local:3306\' (111)\n'
2024-03-01T23:28:12.887Z [container-agent] 
2024-03-01T23:28:12.887Z [container-agent] During handling of the above exception, another exception occurred:
2024-03-01T23:28:12.887Z [container-agent] 
2024-03-01T23:28:12.887Z [container-agent] Traceback (most recent call last):
2024-03-01T23:28:12.887Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-03-01T23:28:12.887Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-03-01T23:28:12.887Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 669, in _run_mysqlsh_script
2024-03-01T23:28:12.887Z [container-agent]     raise MySQLClientError(e.stderr)
2024-03-01T23:28:12.887Z [container-agent] charms.mysql.v0.mysql.MySQLClientError: Cannot set LC_ALL to locale en_US.UTF-8: No such file or directory
2024-03-01T23:28:12.887Z [container-agent] verbose: 2024-03-01T23:28:11Z: Loading startup files...
2024-03-01T23:28:12.887Z [container-agent] verbose: 2024-03-01T23:28:11Z: Loading plugins...
2024-03-01T23:28:12.887Z [container-agent] verbose: 2024-03-01T23:28:11Z: Connecting to MySQL at: serverconfig@horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local
2024-03-01T23:28:12.887Z [container-agent] Traceback (most recent call last):
2024-03-01T23:28:12.887Z [container-agent]   File "<string>", line 1, in <module>
2024-03-01T23:28:12.887Z [container-agent] mysqlsh.DBError: MySQL Error (2003): Shell.connect: Can't connect to MySQL server on 'horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local:3306' (111)
2024-03-01T23:28:12.887Z [container-agent] 
2024-03-01T23:28:14.594Z [container-agent] 2024-03-01 23:28:14 ERROR juju-log Uncaught exception while in charm code:
2024-03-01T23:28:14.594Z [container-agent] Traceback (most recent call last):
2024-03-01T23:28:14.594Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/./src/charm.py", line 786, in <module>
2024-03-01T23:28:14.594Z [container-agent]     main(MySQLOperatorCharm)
2024-03-01T23:28:14.594Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 436, in main
2024-03-01T23:28:14.594Z [container-agent]     _emit_charm_event(charm, dispatcher.event_name)
2024-03-01T23:28:14.594Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 144, in _emit_charm_event
2024-03-01T23:28:14.594Z [container-agent]     event_to_emit.emit(*args, **kwargs)
2024-03-01T23:28:14.594Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 351, in emit
2024-03-01T23:28:14.594Z [container-agent]     framework._emit(event)
2024-03-01T23:28:14.594Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 853, in _emit
2024-03-01T23:28:14.594Z [container-agent]     self._reemit(event_path)
2024-03-01T23:28:14.594Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 942, in _reemit
2024-03-01T23:28:14.594Z [container-agent]     custom_handler(event)
2024-03-01T23:28:14.594Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/rotate_mysql_logs.py", line 57, in _rotate_mysql_logs
2024-03-01T23:28:14.594Z [container-agent]     self.charm._mysql.flush_mysql_logs(MySQLTextLogs.SLOW)
2024-03-01T23:28:14.594Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-03-01T23:28:14.594Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-03-01T23:28:14.594Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 665, in _run_mysqlsh_script
2024-03-01T23:28:14.594Z [container-agent]     process = self.container.exec(cmd, timeout=timeout)
2024-03-01T23:28:14.594Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/model.py", line 2688, in exec
2024-03-01T23:28:14.594Z [container-agent]     return self._pebble.exec(
2024-03-01T23:28:14.594Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 2508, in exec
2024-03-01T23:28:14.594Z [container-agent]     stdio_ws = self._connect_websocket(task_id, 'stdio')
2024-03-01T23:28:14.594Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 2586, in _connect_websocket
2024-03-01T23:28:14.594Z [container-agent]     sock.connect(self.socket_path)
2024-03-01T23:28:14.594Z [container-agent] FileNotFoundError: [Errno 2] No such file or directory
2024-03-01T23:28:14.729Z [container-agent] 2024-03-01 23:28:14 INFO juju.util.exec exec.go:209 run result: exit status 1
2024-03-01T23:28:19.071Z [container-agent] 2024-03-01 23:28:19 INFO juju-log Starting the log rotate manager
2024-03-01T23:28:19.083Z [container-agent] 2024-03-01 23:28:19 INFO juju-log Started log rotate manager process with PID 480005
2024-03-01T23:28:19.263Z [container-agent] 2024-03-01 23:28:19 INFO juju-log Setting up the logrotate configurations
2024-03-01T23:28:19.325Z [container-agent] 2024-03-01 23:28:19 INFO juju-log Adding pebble layer
2024-03-01T23:28:26.072Z [container-agent] 2024-03-01 23:28:26 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-03-01T23:28:26.101Z [container-agent] 2024-03-01 23:28:26 INFO juju-log Attempting reboot from complete outage.
2024-03-01T23:28:29.029Z [container-agent] 2024-03-01 23:28:29 INFO juju.worker.uniter.operation runhook.go:186 ran "mysql-pebble-ready" hook (via hook dispatching script: dispatch)
2024-03-01T23:29:10.667Z [container-agent] 2024-03-01 23:29:10 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T23:29:12.604Z [container-agent] 2024-03-01 23:29:12 WARNING juju-log No relation: certificates
2024-03-01T23:29:12.963Z [container-agent] 2024-03-01 23:29:12 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T23:34:06.206Z [container-agent] 2024-03-01 23:34:06 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T23:34:08.593Z [container-agent] 2024-03-01 23:34:08 WARNING juju-log No relation: certificates
2024-03-01T23:34:08.913Z [container-agent] 2024-03-01 23:34:08 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T23:39:52.256Z [container-agent] 2024-03-01 23:39:52 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T23:39:55.076Z [container-agent] 2024-03-01 23:39:55 WARNING juju-log No relation: certificates
2024-03-01T23:39:55.404Z [container-agent] 2024-03-01 23:39:55 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T23:43:55.225Z [container-agent] 2024-03-01 23:43:55 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T23:43:58.475Z [container-agent] 2024-03-01 23:43:58 WARNING juju-log No relation: certificates
2024-03-01T23:43:58.853Z [container-agent] 2024-03-01 23:43:58 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T23:48:25.300Z [container-agent] 2024-03-01 23:48:25 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T23:48:29.183Z [container-agent] 2024-03-01 23:48:29 WARNING juju-log No relation: certificates
2024-03-01T23:48:29.527Z [container-agent] 2024-03-01 23:48:29 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T23:52:36.076Z [container-agent] 2024-03-01 23:52:36 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T23:52:40.204Z [container-agent] 2024-03-01 23:52:40 WARNING juju-log No relation: certificates
2024-03-01T23:52:40.548Z [container-agent] 2024-03-01 23:52:40 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-01T23:58:25.388Z [container-agent] 2024-03-01 23:58:25 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-01T23:58:30.281Z [container-agent] 2024-03-01 23:58:30 WARNING juju-log No relation: certificates
2024-03-01T23:58:30.630Z [container-agent] 2024-03-01 23:58:30 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T00:02:47.994Z [container-agent] 2024-03-02 00:02:47 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T00:02:53.005Z [container-agent] 2024-03-02 00:02:53 WARNING juju-log No relation: certificates
2024-03-02T00:02:53.362Z [container-agent] 2024-03-02 00:02:53 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T00:08:19.193Z [container-agent] 2024-03-02 00:08:19 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T00:08:25.394Z [container-agent] 2024-03-02 00:08:25 WARNING juju-log No relation: certificates
2024-03-02T00:08:25.745Z [container-agent] 2024-03-02 00:08:25 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T00:12:45.955Z [container-agent] 2024-03-02 00:12:45 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T00:12:51.922Z [container-agent] 2024-03-02 00:12:51 WARNING juju-log No relation: certificates
2024-03-02T00:12:52.277Z [container-agent] 2024-03-02 00:12:52 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T00:18:01.067Z [container-agent] 2024-03-02 00:18:01 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T00:18:07.124Z [container-agent] 2024-03-02 00:18:07 WARNING juju-log No relation: certificates
2024-03-02T00:18:07.455Z [container-agent] 2024-03-02 00:18:07 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T00:23:28.427Z [container-agent] 2024-03-02 00:23:28 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T00:23:35.556Z [container-agent] 2024-03-02 00:23:35 WARNING juju-log No relation: certificates
2024-03-02T00:23:35.916Z [container-agent] 2024-03-02 00:23:35 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T00:28:29.142Z [container-agent] 2024-03-02 00:28:29 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T00:28:36.515Z [container-agent] 2024-03-02 00:28:36 WARNING juju-log No relation: certificates
2024-03-02T00:28:36.871Z [container-agent] 2024-03-02 00:28:36 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T00:34:14.148Z [container-agent] 2024-03-02 00:34:14 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T00:34:22.179Z [container-agent] 2024-03-02 00:34:22 WARNING juju-log No relation: certificates
2024-03-02T00:34:22.535Z [container-agent] 2024-03-02 00:34:22 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T00:39:19.615Z [container-agent] 2024-03-02 00:39:19 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-03-02T00:39:19.643Z [container-agent] 2024-03-02 00:39:19 INFO juju-log Attempting reboot from complete outage.
2024-03-02T00:39:27.195Z [container-agent] 2024-03-02 00:39:27 WARNING juju-log No relation: certificates
2024-03-02T00:39:27.554Z [container-agent] 2024-03-02 00:39:27 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T00:40:04.798Z [container-agent] 2024-03-02 00:40:04 ERROR juju-log Uncaught exception while in charm code:
2024-03-02T00:40:04.798Z [container-agent] Traceback (most recent call last):
2024-03-02T00:40:04.798Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 2507, in exec
2024-03-02T00:40:04.798Z [container-agent]     control_ws = self._connect_websocket(task_id, 'control')
2024-03-02T00:40:04.798Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 2589, in _connect_websocket
2024-03-02T00:40:04.798Z [container-agent]     ws.connect(url, socket=sock)
2024-03-02T00:40:04.798Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_core.py", line 253, in connect
2024-03-02T00:40:04.798Z [container-agent]     self.handshake_response = handshake(self.sock, url, *addrs, **options)
2024-03-02T00:40:04.798Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_handshake.py", line 57, in handshake
2024-03-02T00:40:04.798Z [container-agent]     status, resp = _get_resp_headers(sock)
2024-03-02T00:40:04.798Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_handshake.py", line 143, in _get_resp_headers
2024-03-02T00:40:04.798Z [container-agent]     status, resp_headers, status_message = read_headers(sock)
2024-03-02T00:40:04.798Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_http.py", line 312, in read_headers
2024-03-02T00:40:04.798Z [container-agent]     line = recv_line(sock)
2024-03-02T00:40:04.798Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_socket.py", line 131, in recv_line
2024-03-02T00:40:04.798Z [container-agent]     c = recv(sock, 1)
2024-03-02T00:40:04.798Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_socket.py", line 122, in recv
2024-03-02T00:40:04.798Z [container-agent]     raise WebSocketConnectionClosedException(
2024-03-02T00:40:04.798Z [container-agent] websocket._exceptions.WebSocketConnectionClosedException: Connection to remote host was lost.
2024-03-02T00:40:04.798Z [container-agent] 
2024-03-02T00:40:04.798Z [container-agent] During handling of the above exception, another exception occurred:
2024-03-02T00:40:04.798Z [container-agent] 
2024-03-02T00:40:04.798Z [container-agent] Traceback (most recent call last):
2024-03-02T00:40:04.798Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 1348, in do_open
2024-03-02T00:40:04.798Z [container-agent]     h.request(req.get_method(), req.selector, req.data, headers,
2024-03-02T00:40:04.798Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1283, in request
2024-03-02T00:40:04.798Z [container-agent]     self._send_request(method, url, body, headers, encode_chunked)
2024-03-02T00:40:04.798Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1329, in _send_request
2024-03-02T00:40:04.798Z [container-agent]     self.endheaders(body, encode_chunked=encode_chunked)
2024-03-02T00:40:04.798Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1278, in endheaders
2024-03-02T00:40:04.798Z [container-agent]     self._send_output(message_body, encode_chunked=encode_chunked)
2024-03-02T00:40:04.798Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1038, in _send_output
2024-03-02T00:40:04.798Z [container-agent]     self.send(msg)
2024-03-02T00:40:04.798Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 976, in send
2024-03-02T00:40:04.798Z [container-agent]     self.connect()
2024-03-02T00:40:04.798Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 297, in connect
2024-03-02T00:40:04.798Z [container-agent]     self.sock.connect(self.socket_path)
2024-03-02T00:40:04.798Z [container-agent] ConnectionRefusedError: [Errno 111] Connection refused
2024-03-02T00:40:04.798Z [container-agent] 
2024-03-02T00:40:04.798Z [container-agent] During handling of the above exception, another exception occurred:
2024-03-02T00:40:04.798Z [container-agent] 
2024-03-02T00:40:04.798Z [container-agent] Traceback (most recent call last):
2024-03-02T00:40:04.798Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1690, in _request_raw
2024-03-02T00:40:04.798Z [container-agent]     response = self.opener.open(request, timeout=self.timeout)
2024-03-02T00:40:04.798Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 519, in open
2024-03-02T00:40:04.798Z [container-agent]     response = self._open(req, data)
2024-03-02T00:40:04.798Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 536, in _open
2024-03-02T00:40:04.798Z [container-agent]     result = self._call_chain(self.handle_open, protocol, protocol +
2024-03-02T00:40:04.798Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 496, in _call_chain
2024-03-02T00:40:04.798Z [container-agent]     result = func(*args)
2024-03-02T00:40:04.798Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 311, in http_open
2024-03-02T00:40:04.798Z [container-agent]     return self.do_open(_UnixSocketConnection, req,  # type:ignore
2024-03-02T00:40:04.798Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 1351, in do_open
2024-03-02T00:40:04.798Z [container-agent]     raise URLError(err)
2024-03-02T00:40:04.798Z [container-agent] urllib.error.URLError: <urlopen error [Errno 111] Connection refused>
2024-03-02T00:40:04.798Z [container-agent] 
2024-03-02T00:40:04.798Z [container-agent] The above exception was the direct cause of the following exception:
2024-03-02T00:40:04.798Z [container-agent] 
2024-03-02T00:40:04.798Z [container-agent] Traceback (most recent call last):
2024-03-02T00:40:04.798Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/./src/charm.py", line 786, in <module>
2024-03-02T00:40:04.798Z [container-agent]     main(MySQLOperatorCharm)
2024-03-02T00:40:04.798Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 436, in main
2024-03-02T00:40:04.798Z [container-agent]     _emit_charm_event(charm, dispatcher.event_name)
2024-03-02T00:40:04.798Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 144, in _emit_charm_event
2024-03-02T00:40:04.798Z [container-agent]     event_to_emit.emit(*args, **kwargs)
2024-03-02T00:40:04.798Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 351, in emit
2024-03-02T00:40:04.798Z [container-agent]     framework._emit(event)
2024-03-02T00:40:04.798Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 853, in _emit
2024-03-02T00:40:04.798Z [container-agent]     self._reemit(event_path)
2024-03-02T00:40:04.798Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 942, in _reemit
2024-03-02T00:40:04.798Z [container-agent]     custom_handler(event)
2024-03-02T00:40:04.798Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/rotate_mysql_logs.py", line 55, in _rotate_mysql_logs
2024-03-02T00:40:04.798Z [container-agent]     self.charm._mysql.flush_mysql_logs(MySQLTextLogs.ERROR)
2024-03-02T00:40:04.798Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-03-02T00:40:04.798Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-03-02T00:40:04.798Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 665, in _run_mysqlsh_script
2024-03-02T00:40:04.798Z [container-agent]     process = self.container.exec(cmd, timeout=timeout)
2024-03-02T00:40:04.798Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/model.py", line 2688, in exec
2024-03-02T00:40:04.798Z [container-agent]     return self._pebble.exec(
2024-03-02T00:40:04.798Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 2514, in exec
2024-03-02T00:40:04.798Z [container-agent]     change = self.wait_change(ChangeID(change_id))
2024-03-02T00:40:04.798Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1893, in wait_change
2024-03-02T00:40:04.798Z [container-agent]     return self._wait_change_using_wait(change_id, timeout)
2024-03-02T00:40:04.798Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1914, in _wait_change_using_wait
2024-03-02T00:40:04.798Z [container-agent]     return self._wait_change(change_id, this_timeout)
2024-03-02T00:40:04.798Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1928, in _wait_change
2024-03-02T00:40:04.798Z [container-agent]     resp = self._request('GET', f'/v1/changes/{change_id}/wait', query)
2024-03-02T00:40:04.798Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1655, in _request
2024-03-02T00:40:04.799Z [container-agent]     response = self._request_raw(method, path, query, headers, data)
2024-03-02T00:40:04.799Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1707, in _request_raw
2024-03-02T00:40:04.799Z [container-agent]     raise ConnectionError(e.reason) from e
2024-03-02T00:40:04.799Z [container-agent] ops.pebble.ConnectionError: [Errno 111] Connection refused
2024-03-02T00:40:04.949Z [container-agent] 2024-03-02 00:40:04 INFO juju.util.exec exec.go:209 run result: exit status 1
2024-03-02T00:40:07.083Z [container-agent] 2024-03-02 00:40:07 INFO juju-log Setting up the logrotate configurations
2024-03-02T00:40:07.152Z [container-agent] 2024-03-02 00:40:07 INFO juju-log Adding pebble layer
2024-03-02T00:40:13.873Z [container-agent] 2024-03-02 00:40:13 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-03-02T00:40:13.901Z [container-agent] 2024-03-02 00:40:13 INFO juju-log Attempting reboot from complete outage.
2024-03-02T00:40:16.977Z [container-agent] 2024-03-02 00:40:16 INFO juju.worker.uniter.operation runhook.go:186 ran "mysql-pebble-ready" hook (via hook dispatching script: dispatch)
2024-03-02T00:43:23.382Z [container-agent] 2024-03-02 00:43:23 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T00:43:25.228Z [container-agent] 2024-03-02 00:43:25 WARNING juju-log No relation: certificates
2024-03-02T00:43:25.615Z [container-agent] 2024-03-02 00:43:25 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T00:48:23.211Z [container-agent] 2024-03-02 00:48:23 INFO juju-log Starting the log rotate manager
2024-03-02T00:48:23.225Z [container-agent] 2024-03-02 00:48:23 INFO juju-log Started log rotate manager process with PID 487520
2024-03-02T00:48:23.532Z [container-agent] 2024-03-02 00:48:23 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T00:48:25.468Z [container-agent] 2024-03-02 00:48:25 WARNING juju-log No relation: certificates
2024-03-02T00:48:25.827Z [container-agent] 2024-03-02 00:48:25 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T00:52:28.197Z [container-agent] 2024-03-02 00:52:28 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T00:52:30.680Z [container-agent] 2024-03-02 00:52:30 WARNING juju-log No relation: certificates
2024-03-02T00:52:31.020Z [container-agent] 2024-03-02 00:52:31 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T00:58:14.684Z [container-agent] 2024-03-02 00:58:14 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T00:58:17.893Z [container-agent] 2024-03-02 00:58:17 WARNING juju-log No relation: certificates
2024-03-02T00:58:18.258Z [container-agent] 2024-03-02 00:58:18 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T01:02:25.590Z [container-agent] 2024-03-02 01:02:25 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T01:02:28.865Z [container-agent] 2024-03-02 01:02:28 WARNING juju-log No relation: certificates
2024-03-02T01:02:29.220Z [container-agent] 2024-03-02 01:02:29 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T01:08:24.781Z [container-agent] 2024-03-02 01:08:24 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T01:08:28.888Z [container-agent] 2024-03-02 01:08:28 WARNING juju-log No relation: certificates
2024-03-02T01:08:29.240Z [container-agent] 2024-03-02 01:08:29 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T01:13:30.013Z [container-agent] 2024-03-02 01:13:30 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T01:13:34.600Z [container-agent] 2024-03-02 01:13:34 WARNING juju-log No relation: certificates
2024-03-02T01:13:34.950Z [container-agent] 2024-03-02 01:13:34 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T01:17:52.452Z [container-agent] 2024-03-02 01:17:52 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T01:17:57.110Z [container-agent] 2024-03-02 01:17:57 WARNING juju-log No relation: certificates
2024-03-02T01:17:57.485Z [container-agent] 2024-03-02 01:17:57 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T01:22:11.716Z [container-agent] 2024-03-02 01:22:11 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T01:22:16.943Z [container-agent] 2024-03-02 01:22:16 WARNING juju-log No relation: certificates
2024-03-02T01:22:17.310Z [container-agent] 2024-03-02 01:22:17 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T01:27:10.551Z [container-agent] 2024-03-02 01:27:10 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T01:27:16.272Z [container-agent] 2024-03-02 01:27:16 WARNING juju-log No relation: certificates
2024-03-02T01:27:16.614Z [container-agent] 2024-03-02 01:27:16 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T01:32:34.980Z [container-agent] 2024-03-02 01:32:34 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T01:32:40.727Z [container-agent] 2024-03-02 01:32:40 WARNING juju-log No relation: certificates
2024-03-02T01:32:41.082Z [container-agent] 2024-03-02 01:32:41 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T01:37:12.478Z [container-agent] 2024-03-02 01:37:12 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T01:37:19.151Z [container-agent] 2024-03-02 01:37:19 WARNING juju-log No relation: certificates
2024-03-02T01:37:19.524Z [container-agent] 2024-03-02 01:37:19 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T01:42:17.931Z [container-agent] 2024-03-02 01:42:17 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T01:42:25.039Z [container-agent] 2024-03-02 01:42:25 WARNING juju-log No relation: certificates
2024-03-02T01:42:25.416Z [container-agent] 2024-03-02 01:42:25 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T01:48:02.157Z [container-agent] 2024-03-02 01:48:02 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T01:48:09.159Z [container-agent] 2024-03-02 01:48:09 WARNING juju-log No relation: certificates
2024-03-02T01:48:09.477Z [container-agent] 2024-03-02 01:48:09 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T01:53:03.489Z [container-agent] 2024-03-02 01:53:03 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T01:53:11.021Z [container-agent] 2024-03-02 01:53:11 WARNING juju-log No relation: certificates
2024-03-02T01:53:11.377Z [container-agent] 2024-03-02 01:53:11 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T01:57:26.336Z [container-agent] 2024-03-02 01:57:26 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T01:57:36.058Z [container-agent] 2024-03-02 01:57:36 WARNING juju-log No relation: certificates
2024-03-02T01:57:36.464Z [container-agent] 2024-03-02 01:57:36 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T02:02:15.768Z [container-agent] 2024-03-02 02:02:15 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-03-02T02:02:15.801Z [container-agent] 2024-03-02 02:02:15 INFO juju-log Attempting reboot from complete outage.
2024-03-02T02:02:23.947Z [container-agent] 2024-03-02 02:02:23 WARNING juju-log No relation: certificates
2024-03-02T02:02:24.301Z [container-agent] 2024-03-02 02:02:24 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T02:02:40.389Z [container-agent] 2024-03-02 02:02:40 INFO juju-log Setting up the logrotate configurations
2024-03-02T02:02:40.468Z [container-agent] 2024-03-02 02:02:40 INFO juju-log Adding pebble layer
2024-03-02T02:02:47.178Z [container-agent] 2024-03-02 02:02:47 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-03-02T02:02:47.202Z [container-agent] 2024-03-02 02:02:47 INFO juju-log Attempting reboot from complete outage.
2024-03-02T02:02:50.086Z [container-agent] 2024-03-02 02:02:50 INFO juju.worker.uniter.operation runhook.go:186 ran "mysql-pebble-ready" hook (via hook dispatching script: dispatch)
2024-03-02T02:07:52.014Z [container-agent] 2024-03-02 02:07:52 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T02:07:54.332Z [container-agent] 2024-03-02 02:07:54 WARNING juju-log No relation: certificates
2024-03-02T02:07:54.723Z [container-agent] 2024-03-02 02:07:54 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T02:13:51.555Z [container-agent] 2024-03-02 02:13:51 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T02:13:54.331Z [container-agent] 2024-03-02 02:13:54 WARNING juju-log No relation: certificates
2024-03-02T02:13:54.660Z [container-agent] 2024-03-02 02:13:54 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T02:18:23.047Z [container-agent] 2024-03-02 02:18:23 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T02:18:26.465Z [container-agent] 2024-03-02 02:18:26 WARNING juju-log No relation: certificates
2024-03-02T02:18:26.819Z [container-agent] 2024-03-02 02:18:26 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T02:22:54.150Z [container-agent] 2024-03-02 02:22:54 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T02:22:57.881Z [container-agent] 2024-03-02 02:22:57 WARNING juju-log No relation: certificates
2024-03-02T02:22:58.269Z [container-agent] 2024-03-02 02:22:58 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T02:27:21.725Z [container-agent] 2024-03-02 02:27:21 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T02:27:25.611Z [container-agent] 2024-03-02 02:27:25 WARNING juju-log No relation: certificates
2024-03-02T02:27:25.952Z [container-agent] 2024-03-02 02:27:25 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T02:32:42.647Z [container-agent] 2024-03-02 02:32:42 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T02:32:47.404Z [container-agent] 2024-03-02 02:32:47 WARNING juju-log No relation: certificates
2024-03-02T02:32:47.757Z [container-agent] 2024-03-02 02:32:47 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T02:38:18.479Z [container-agent] 2024-03-02 02:38:18 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T02:38:23.714Z [container-agent] 2024-03-02 02:38:23 WARNING juju-log No relation: certificates
2024-03-02T02:38:24.060Z [container-agent] 2024-03-02 02:38:24 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T02:43:37.104Z [container-agent] 2024-03-02 02:43:37 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T02:43:42.091Z [container-agent] 2024-03-02 02:43:42 WARNING juju-log No relation: certificates
2024-03-02T02:43:42.428Z [container-agent] 2024-03-02 02:43:42 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T02:48:38.467Z [container-agent] 2024-03-02 02:48:38 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T02:48:44.050Z [container-agent] 2024-03-02 02:48:44 WARNING juju-log No relation: certificates
2024-03-02T02:48:44.403Z [container-agent] 2024-03-02 02:48:44 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T02:52:55.725Z [container-agent] 2024-03-02 02:52:55 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T02:53:02.341Z [container-agent] 2024-03-02 02:53:02 WARNING juju-log No relation: certificates
2024-03-02T02:53:02.683Z [container-agent] 2024-03-02 02:53:02 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T02:57:27.649Z [container-agent] 2024-03-02 02:57:27 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T02:57:34.602Z [container-agent] 2024-03-02 02:57:34 WARNING juju-log No relation: certificates
2024-03-02T02:57:34.947Z [container-agent] 2024-03-02 02:57:34 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T03:01:50.557Z [container-agent] 2024-03-02 03:01:50 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T03:01:57.939Z [container-agent] 2024-03-02 03:01:57 WARNING juju-log No relation: certificates
2024-03-02T03:01:58.337Z [container-agent] 2024-03-02 03:01:58 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T03:06:57.540Z [container-agent] 2024-03-02 03:06:57 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T03:07:04.893Z [container-agent] 2024-03-02 03:07:04 WARNING juju-log No relation: certificates
2024-03-02T03:07:05.217Z [container-agent] 2024-03-02 03:07:05 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T03:11:24.025Z [container-agent] 2024-03-02 03:11:24 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T03:11:32.229Z [container-agent] 2024-03-02 03:11:32 WARNING juju-log No relation: certificates
2024-03-02T03:11:32.582Z [container-agent] 2024-03-02 03:11:32 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T03:15:32.788Z [container-agent] 2024-03-02 03:15:32 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-03-02T03:15:32.820Z [container-agent] 2024-03-02 03:15:32 INFO juju-log Attempting reboot from complete outage.
2024-03-02T03:15:41.003Z [container-agent] 2024-03-02 03:15:41 WARNING juju-log No relation: certificates
2024-03-02T03:15:41.395Z [container-agent] 2024-03-02 03:15:41 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T03:17:04.775Z [container-agent] 2024-03-02 03:17:04 ERROR juju-log Uncaught exception while in charm code:
2024-03-02T03:17:04.775Z [container-agent] Traceback (most recent call last):
2024-03-02T03:17:04.775Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 2507, in exec
2024-03-02T03:17:04.775Z [container-agent]     control_ws = self._connect_websocket(task_id, 'control')
2024-03-02T03:17:04.775Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 2589, in _connect_websocket
2024-03-02T03:17:04.775Z [container-agent]     ws.connect(url, socket=sock)
2024-03-02T03:17:04.775Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_core.py", line 253, in connect
2024-03-02T03:17:04.775Z [container-agent]     self.handshake_response = handshake(self.sock, url, *addrs, **options)
2024-03-02T03:17:04.775Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_handshake.py", line 57, in handshake
2024-03-02T03:17:04.775Z [container-agent]     status, resp = _get_resp_headers(sock)
2024-03-02T03:17:04.775Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_handshake.py", line 143, in _get_resp_headers
2024-03-02T03:17:04.775Z [container-agent]     status, resp_headers, status_message = read_headers(sock)
2024-03-02T03:17:04.775Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_http.py", line 312, in read_headers
2024-03-02T03:17:04.775Z [container-agent]     line = recv_line(sock)
2024-03-02T03:17:04.775Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_socket.py", line 131, in recv_line
2024-03-02T03:17:04.775Z [container-agent]     c = recv(sock, 1)
2024-03-02T03:17:04.775Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_socket.py", line 122, in recv
2024-03-02T03:17:04.775Z [container-agent]     raise WebSocketConnectionClosedException(
2024-03-02T03:17:04.775Z [container-agent] websocket._exceptions.WebSocketConnectionClosedException: Connection to remote host was lost.
2024-03-02T03:17:04.775Z [container-agent] 
2024-03-02T03:17:04.775Z [container-agent] During handling of the above exception, another exception occurred:
2024-03-02T03:17:04.775Z [container-agent] 
2024-03-02T03:17:04.775Z [container-agent] Traceback (most recent call last):
2024-03-02T03:17:04.775Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 1348, in do_open
2024-03-02T03:17:04.775Z [container-agent]     h.request(req.get_method(), req.selector, req.data, headers,
2024-03-02T03:17:04.775Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1283, in request
2024-03-02T03:17:04.775Z [container-agent]     self._send_request(method, url, body, headers, encode_chunked)
2024-03-02T03:17:04.775Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1329, in _send_request
2024-03-02T03:17:04.775Z [container-agent]     self.endheaders(body, encode_chunked=encode_chunked)
2024-03-02T03:17:04.775Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1278, in endheaders
2024-03-02T03:17:04.775Z [container-agent]     self._send_output(message_body, encode_chunked=encode_chunked)
2024-03-02T03:17:04.775Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1038, in _send_output
2024-03-02T03:17:04.775Z [container-agent]     self.send(msg)
2024-03-02T03:17:04.775Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 976, in send
2024-03-02T03:17:04.775Z [container-agent]     self.connect()
2024-03-02T03:17:04.775Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 297, in connect
2024-03-02T03:17:04.775Z [container-agent]     self.sock.connect(self.socket_path)
2024-03-02T03:17:04.775Z [container-agent] ConnectionRefusedError: [Errno 111] Connection refused
2024-03-02T03:17:04.775Z [container-agent] 
2024-03-02T03:17:04.775Z [container-agent] During handling of the above exception, another exception occurred:
2024-03-02T03:17:04.775Z [container-agent] 
2024-03-02T03:17:04.775Z [container-agent] Traceback (most recent call last):
2024-03-02T03:17:04.775Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1690, in _request_raw
2024-03-02T03:17:04.775Z [container-agent]     response = self.opener.open(request, timeout=self.timeout)
2024-03-02T03:17:04.775Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 519, in open
2024-03-02T03:17:04.775Z [container-agent]     response = self._open(req, data)
2024-03-02T03:17:04.775Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 536, in _open
2024-03-02T03:17:04.775Z [container-agent]     result = self._call_chain(self.handle_open, protocol, protocol +
2024-03-02T03:17:04.775Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 496, in _call_chain
2024-03-02T03:17:04.775Z [container-agent]     result = func(*args)
2024-03-02T03:17:04.775Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 311, in http_open
2024-03-02T03:17:04.775Z [container-agent]     return self.do_open(_UnixSocketConnection, req,  # type:ignore
2024-03-02T03:17:04.775Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 1351, in do_open
2024-03-02T03:17:04.775Z [container-agent]     raise URLError(err)
2024-03-02T03:17:04.775Z [container-agent] urllib.error.URLError: <urlopen error [Errno 111] Connection refused>
2024-03-02T03:17:04.775Z [container-agent] 
2024-03-02T03:17:04.775Z [container-agent] The above exception was the direct cause of the following exception:
2024-03-02T03:17:04.775Z [container-agent] 
2024-03-02T03:17:04.775Z [container-agent] Traceback (most recent call last):
2024-03-02T03:17:04.775Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/./src/charm.py", line 786, in <module>
2024-03-02T03:17:04.775Z [container-agent]     main(MySQLOperatorCharm)
2024-03-02T03:17:04.775Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 436, in main
2024-03-02T03:17:04.775Z [container-agent]     _emit_charm_event(charm, dispatcher.event_name)
2024-03-02T03:17:04.775Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 144, in _emit_charm_event
2024-03-02T03:17:04.775Z [container-agent]     event_to_emit.emit(*args, **kwargs)
2024-03-02T03:17:04.775Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 351, in emit
2024-03-02T03:17:04.775Z [container-agent]     framework._emit(event)
2024-03-02T03:17:04.775Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 853, in _emit
2024-03-02T03:17:04.775Z [container-agent]     self._reemit(event_path)
2024-03-02T03:17:04.775Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 942, in _reemit
2024-03-02T03:17:04.775Z [container-agent]     custom_handler(event)
2024-03-02T03:17:04.775Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/rotate_mysql_logs.py", line 55, in _rotate_mysql_logs
2024-03-02T03:17:04.775Z [container-agent]     self.charm._mysql.flush_mysql_logs(MySQLTextLogs.ERROR)
2024-03-02T03:17:04.775Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-03-02T03:17:04.775Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-03-02T03:17:04.775Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 665, in _run_mysqlsh_script
2024-03-02T03:17:04.775Z [container-agent]     process = self.container.exec(cmd, timeout=timeout)
2024-03-02T03:17:04.775Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/model.py", line 2688, in exec
2024-03-02T03:17:04.775Z [container-agent]     return self._pebble.exec(
2024-03-02T03:17:04.775Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 2514, in exec
2024-03-02T03:17:04.775Z [container-agent]     change = self.wait_change(ChangeID(change_id))
2024-03-02T03:17:04.775Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1893, in wait_change
2024-03-02T03:17:04.775Z [container-agent]     return self._wait_change_using_wait(change_id, timeout)
2024-03-02T03:17:04.775Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1914, in _wait_change_using_wait
2024-03-02T03:17:04.775Z [container-agent]     return self._wait_change(change_id, this_timeout)
2024-03-02T03:17:04.775Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1928, in _wait_change
2024-03-02T03:17:04.775Z [container-agent]     resp = self._request('GET', f'/v1/changes/{change_id}/wait', query)
2024-03-02T03:17:04.775Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1655, in _request
2024-03-02T03:17:04.775Z [container-agent]     response = self._request_raw(method, path, query, headers, data)
2024-03-02T03:17:04.775Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1707, in _request_raw
2024-03-02T03:17:04.775Z [container-agent]     raise ConnectionError(e.reason) from e
2024-03-02T03:17:04.775Z [container-agent] ops.pebble.ConnectionError: [Errno 111] Connection refused
2024-03-02T03:17:04.954Z [container-agent] 2024-03-02 03:17:04 INFO juju.util.exec exec.go:209 run result: exit status 1
2024-03-02T03:17:07.971Z [container-agent] 2024-03-02 03:17:07 INFO juju-log Setting up the logrotate configurations
2024-03-02T03:17:08.038Z [container-agent] 2024-03-02 03:17:08 INFO juju-log Adding pebble layer
2024-03-02T03:17:14.743Z [container-agent] 2024-03-02 03:17:14 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-03-02T03:17:14.765Z [container-agent] 2024-03-02 03:17:14 INFO juju-log Attempting reboot from complete outage.
2024-03-02T03:17:17.762Z [container-agent] 2024-03-02 03:17:17 INFO juju.worker.uniter.operation runhook.go:186 ran "mysql-pebble-ready" hook (via hook dispatching script: dispatch)
2024-03-02T03:20:59.925Z [container-agent] 2024-03-02 03:20:59 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T03:21:01.870Z [container-agent] 2024-03-02 03:21:01 WARNING juju-log No relation: certificates
2024-03-02T03:21:02.228Z [container-agent] 2024-03-02 03:21:02 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T03:25:01.507Z [container-agent] 2024-03-02 03:25:01 INFO juju-log Starting the log rotate manager
2024-03-02T03:25:01.522Z [container-agent] 2024-03-02 03:25:01 INFO juju-log Started log rotate manager process with PID 502687
2024-03-02T03:25:01.848Z [container-agent] 2024-03-02 03:25:01 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T03:25:03.976Z [container-agent] 2024-03-02 03:25:03 WARNING juju-log No relation: certificates
2024-03-02T03:25:04.315Z [container-agent] 2024-03-02 03:25:04 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T03:30:59.811Z [container-agent] 2024-03-02 03:30:59 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T03:31:02.345Z [container-agent] 2024-03-02 03:31:02 WARNING juju-log No relation: certificates
2024-03-02T03:31:02.721Z [container-agent] 2024-03-02 03:31:02 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T03:36:52.431Z [container-agent] 2024-03-02 03:36:52 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T03:36:55.592Z [container-agent] 2024-03-02 03:36:55 WARNING juju-log No relation: certificates
2024-03-02T03:36:55.963Z [container-agent] 2024-03-02 03:36:55 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T03:42:14.850Z [container-agent] 2024-03-02 03:42:14 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T03:42:18.292Z [container-agent] 2024-03-02 03:42:18 WARNING juju-log No relation: certificates
2024-03-02T03:42:18.654Z [container-agent] 2024-03-02 03:42:18 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T03:46:16.440Z [container-agent] 2024-03-02 03:46:16 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T03:46:20.484Z [container-agent] 2024-03-02 03:46:20 WARNING juju-log No relation: certificates
2024-03-02T03:46:20.832Z [container-agent] 2024-03-02 03:46:20 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T03:50:57.281Z [container-agent] 2024-03-02 03:50:57 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T03:51:01.582Z [container-agent] 2024-03-02 03:51:01 WARNING juju-log No relation: certificates
2024-03-02T03:51:01.955Z [container-agent] 2024-03-02 03:51:01 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T03:56:56.947Z [container-agent] 2024-03-02 03:56:56 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T03:57:01.655Z [container-agent] 2024-03-02 03:57:01 WARNING juju-log No relation: certificates
2024-03-02T03:57:02.076Z [container-agent] 2024-03-02 03:57:02 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T04:02:41.036Z [container-agent] 2024-03-02 04:02:41 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T04:02:46.059Z [container-agent] 2024-03-02 04:02:46 WARNING juju-log No relation: certificates
2024-03-02T04:02:46.420Z [container-agent] 2024-03-02 04:02:46 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T04:06:45.804Z [container-agent] 2024-03-02 04:06:45 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T04:06:51.357Z [container-agent] 2024-03-02 04:06:51 WARNING juju-log No relation: certificates
2024-03-02T04:06:51.695Z [container-agent] 2024-03-02 04:06:51 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T04:10:49.931Z [container-agent] 2024-03-02 04:10:49 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T04:10:56.157Z [container-agent] 2024-03-02 04:10:56 WARNING juju-log No relation: certificates
2024-03-02T04:10:56.513Z [container-agent] 2024-03-02 04:10:56 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T04:16:43.579Z [container-agent] 2024-03-02 04:16:43 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T04:16:50.444Z [container-agent] 2024-03-02 04:16:50 WARNING juju-log No relation: certificates
2024-03-02T04:16:50.784Z [container-agent] 2024-03-02 04:16:50 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T04:22:33.703Z [container-agent] 2024-03-02 04:22:33 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T04:22:40.985Z [container-agent] 2024-03-02 04:22:40 WARNING juju-log No relation: certificates
2024-03-02T04:22:41.403Z [container-agent] 2024-03-02 04:22:41 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T04:27:35.668Z [container-agent] 2024-03-02 04:27:35 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T04:27:43.131Z [container-agent] 2024-03-02 04:27:43 WARNING juju-log No relation: certificates
2024-03-02T04:27:43.485Z [container-agent] 2024-03-02 04:27:43 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T04:32:36.644Z [container-agent] 2024-03-02 04:32:36 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T04:32:44.176Z [container-agent] 2024-03-02 04:32:44 WARNING juju-log No relation: certificates
2024-03-02T04:32:44.513Z [container-agent] 2024-03-02 04:32:44 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T04:36:08.110Z [container-agent] 2024-03-02 04:36:08 ERROR juju-log Failed to flush ERROR LOGS logs.
2024-03-02T04:36:08.110Z [container-agent] Traceback (most recent call last):
2024-03-02T04:36:08.110Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 666, in _run_mysqlsh_script
2024-03-02T04:36:08.110Z [container-agent]     stdout, _ = process.wait_output()
2024-03-02T04:36:08.110Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1441, in wait_output
2024-03-02T04:36:08.110Z [container-agent]     raise ExecError[AnyStr](self._command, exit_code, out_value, err_value)
2024-03-02T04:36:08.110Z [container-agent] ops.pebble.ExecError: non-zero exit code 1 executing ['/usr/bin/mysqlsh', '--no-wizard', '--python', '--verbose=1', '-f', '/tmp/script.py', ';', 'rm', '/tmp/script.py'], stdout='', stderr='Cannot set LC_ALL to locale en_US.UTF-8: No such file or directory\nverbose: 2024-03-02T04:36:06Z: Loading startup files...\nverbose: 2024-03-02T04:36:06Z: Loading plugins...\nverbose: 2024-03-02T04:36:07Z: Connecting to MySQL at: serverconfig@horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local\nTraceback (most recent call last):\n  File "<string>", line 1, in <module>\nmysqlsh.DBError: MySQL Error (2003): Shell.connect: Can\'t connect to MySQL server on \'horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local:3306\' (111)\n'
2024-03-02T04:36:08.110Z [container-agent] 
2024-03-02T04:36:08.110Z [container-agent] During handling of the above exception, another exception occurred:
2024-03-02T04:36:08.110Z [container-agent] 
2024-03-02T04:36:08.110Z [container-agent] Traceback (most recent call last):
2024-03-02T04:36:08.110Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-03-02T04:36:08.110Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-03-02T04:36:08.110Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 669, in _run_mysqlsh_script
2024-03-02T04:36:08.110Z [container-agent]     raise MySQLClientError(e.stderr)
2024-03-02T04:36:08.110Z [container-agent] charms.mysql.v0.mysql.MySQLClientError: Cannot set LC_ALL to locale en_US.UTF-8: No such file or directory
2024-03-02T04:36:08.110Z [container-agent] verbose: 2024-03-02T04:36:06Z: Loading startup files...
2024-03-02T04:36:08.110Z [container-agent] verbose: 2024-03-02T04:36:06Z: Loading plugins...
2024-03-02T04:36:08.110Z [container-agent] verbose: 2024-03-02T04:36:07Z: Connecting to MySQL at: serverconfig@horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local
2024-03-02T04:36:08.110Z [container-agent] Traceback (most recent call last):
2024-03-02T04:36:08.110Z [container-agent]   File "<string>", line 1, in <module>
2024-03-02T04:36:08.110Z [container-agent] mysqlsh.DBError: MySQL Error (2003): Shell.connect: Can't connect to MySQL server on 'horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local:3306' (111)
2024-03-02T04:36:08.110Z [container-agent] 
2024-03-02T04:38:36.493Z [container-agent] 2024-03-02 04:38:36 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-03-02T04:38:36.518Z [container-agent] 2024-03-02 04:38:36 INFO juju-log Attempting reboot from complete outage.
2024-03-02T04:38:44.276Z [container-agent] 2024-03-02 04:38:44 WARNING juju-log No relation: certificates
2024-03-02T04:38:44.632Z [container-agent] 2024-03-02 04:38:44 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T04:41:04.797Z [container-agent] 2024-03-02 04:41:04 ERROR juju-log Uncaught exception while in charm code:
2024-03-02T04:41:04.797Z [container-agent] Traceback (most recent call last):
2024-03-02T04:41:04.797Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 2507, in exec
2024-03-02T04:41:04.797Z [container-agent]     control_ws = self._connect_websocket(task_id, 'control')
2024-03-02T04:41:04.797Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 2589, in _connect_websocket
2024-03-02T04:41:04.797Z [container-agent]     ws.connect(url, socket=sock)
2024-03-02T04:41:04.797Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_core.py", line 253, in connect
2024-03-02T04:41:04.797Z [container-agent]     self.handshake_response = handshake(self.sock, url, *addrs, **options)
2024-03-02T04:41:04.797Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_handshake.py", line 57, in handshake
2024-03-02T04:41:04.797Z [container-agent]     status, resp = _get_resp_headers(sock)
2024-03-02T04:41:04.797Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_handshake.py", line 143, in _get_resp_headers
2024-03-02T04:41:04.797Z [container-agent]     status, resp_headers, status_message = read_headers(sock)
2024-03-02T04:41:04.797Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_http.py", line 312, in read_headers
2024-03-02T04:41:04.797Z [container-agent]     line = recv_line(sock)
2024-03-02T04:41:04.797Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_socket.py", line 131, in recv_line
2024-03-02T04:41:04.797Z [container-agent]     c = recv(sock, 1)
2024-03-02T04:41:04.797Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_socket.py", line 122, in recv
2024-03-02T04:41:04.797Z [container-agent]     raise WebSocketConnectionClosedException(
2024-03-02T04:41:04.797Z [container-agent] websocket._exceptions.WebSocketConnectionClosedException: Connection to remote host was lost.
2024-03-02T04:41:04.797Z [container-agent] 
2024-03-02T04:41:04.797Z [container-agent] During handling of the above exception, another exception occurred:
2024-03-02T04:41:04.797Z [container-agent] 
2024-03-02T04:41:04.797Z [container-agent] Traceback (most recent call last):
2024-03-02T04:41:04.797Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 1348, in do_open
2024-03-02T04:41:04.797Z [container-agent]     h.request(req.get_method(), req.selector, req.data, headers,
2024-03-02T04:41:04.797Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1283, in request
2024-03-02T04:41:04.797Z [container-agent]     self._send_request(method, url, body, headers, encode_chunked)
2024-03-02T04:41:04.797Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1329, in _send_request
2024-03-02T04:41:04.797Z [container-agent]     self.endheaders(body, encode_chunked=encode_chunked)
2024-03-02T04:41:04.797Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1278, in endheaders
2024-03-02T04:41:04.797Z [container-agent]     self._send_output(message_body, encode_chunked=encode_chunked)
2024-03-02T04:41:04.797Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1038, in _send_output
2024-03-02T04:41:04.797Z [container-agent]     self.send(msg)
2024-03-02T04:41:04.797Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 976, in send
2024-03-02T04:41:04.797Z [container-agent]     self.connect()
2024-03-02T04:41:04.797Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 297, in connect
2024-03-02T04:41:04.797Z [container-agent]     self.sock.connect(self.socket_path)
2024-03-02T04:41:04.797Z [container-agent] ConnectionRefusedError: [Errno 111] Connection refused
2024-03-02T04:41:04.797Z [container-agent] 
2024-03-02T04:41:04.797Z [container-agent] During handling of the above exception, another exception occurred:
2024-03-02T04:41:04.797Z [container-agent] 
2024-03-02T04:41:04.797Z [container-agent] Traceback (most recent call last):
2024-03-02T04:41:04.797Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1690, in _request_raw
2024-03-02T04:41:04.797Z [container-agent]     response = self.opener.open(request, timeout=self.timeout)
2024-03-02T04:41:04.797Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 519, in open
2024-03-02T04:41:04.797Z [container-agent]     response = self._open(req, data)
2024-03-02T04:41:04.797Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 536, in _open
2024-03-02T04:41:04.797Z [container-agent]     result = self._call_chain(self.handle_open, protocol, protocol +
2024-03-02T04:41:04.797Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 496, in _call_chain
2024-03-02T04:41:04.797Z [container-agent]     result = func(*args)
2024-03-02T04:41:04.797Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 311, in http_open
2024-03-02T04:41:04.797Z [container-agent]     return self.do_open(_UnixSocketConnection, req,  # type:ignore
2024-03-02T04:41:04.797Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 1351, in do_open
2024-03-02T04:41:04.797Z [container-agent]     raise URLError(err)
2024-03-02T04:41:04.797Z [container-agent] urllib.error.URLError: <urlopen error [Errno 111] Connection refused>
2024-03-02T04:41:04.797Z [container-agent] 
2024-03-02T04:41:04.797Z [container-agent] The above exception was the direct cause of the following exception:
2024-03-02T04:41:04.797Z [container-agent] 
2024-03-02T04:41:04.797Z [container-agent] Traceback (most recent call last):
2024-03-02T04:41:04.797Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/./src/charm.py", line 786, in <module>
2024-03-02T04:41:04.797Z [container-agent]     main(MySQLOperatorCharm)
2024-03-02T04:41:04.797Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 436, in main
2024-03-02T04:41:04.797Z [container-agent]     _emit_charm_event(charm, dispatcher.event_name)
2024-03-02T04:41:04.797Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 144, in _emit_charm_event
2024-03-02T04:41:04.797Z [container-agent]     event_to_emit.emit(*args, **kwargs)
2024-03-02T04:41:04.797Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 351, in emit
2024-03-02T04:41:04.797Z [container-agent]     framework._emit(event)
2024-03-02T04:41:04.797Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 853, in _emit
2024-03-02T04:41:04.797Z [container-agent]     self._reemit(event_path)
2024-03-02T04:41:04.797Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 942, in _reemit
2024-03-02T04:41:04.797Z [container-agent]     custom_handler(event)
2024-03-02T04:41:04.797Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/rotate_mysql_logs.py", line 55, in _rotate_mysql_logs
2024-03-02T04:41:04.797Z [container-agent]     self.charm._mysql.flush_mysql_logs(MySQLTextLogs.ERROR)
2024-03-02T04:41:04.797Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-03-02T04:41:04.797Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-03-02T04:41:04.797Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 665, in _run_mysqlsh_script
2024-03-02T04:41:04.797Z [container-agent]     process = self.container.exec(cmd, timeout=timeout)
2024-03-02T04:41:04.797Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/model.py", line 2688, in exec
2024-03-02T04:41:04.797Z [container-agent]     return self._pebble.exec(
2024-03-02T04:41:04.797Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 2514, in exec
2024-03-02T04:41:04.797Z [container-agent]     change = self.wait_change(ChangeID(change_id))
2024-03-02T04:41:04.797Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1893, in wait_change
2024-03-02T04:41:04.797Z [container-agent]     return self._wait_change_using_wait(change_id, timeout)
2024-03-02T04:41:04.797Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1914, in _wait_change_using_wait
2024-03-02T04:41:04.797Z [container-agent]     return self._wait_change(change_id, this_timeout)
2024-03-02T04:41:04.797Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1928, in _wait_change
2024-03-02T04:41:04.797Z [container-agent]     resp = self._request('GET', f'/v1/changes/{change_id}/wait', query)
2024-03-02T04:41:04.797Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1655, in _request
2024-03-02T04:41:04.797Z [container-agent]     response = self._request_raw(method, path, query, headers, data)
2024-03-02T04:41:04.797Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1707, in _request_raw
2024-03-02T04:41:04.797Z [container-agent]     raise ConnectionError(e.reason) from e
2024-03-02T04:41:04.797Z [container-agent] ops.pebble.ConnectionError: [Errno 111] Connection refused
2024-03-02T04:41:04.941Z [container-agent] 2024-03-02 04:41:04 INFO juju.util.exec exec.go:209 run result: exit status 1
2024-03-02T04:41:06.811Z [container-agent] 2024-03-02 04:41:06 INFO juju-log Setting up the logrotate configurations
2024-03-02T04:41:06.884Z [container-agent] 2024-03-02 04:41:06 INFO juju-log Adding pebble layer
2024-03-02T04:41:13.680Z [container-agent] 2024-03-02 04:41:13 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-03-02T04:41:13.709Z [container-agent] 2024-03-02 04:41:13 INFO juju-log Attempting reboot from complete outage.
2024-03-02T04:41:16.744Z [container-agent] 2024-03-02 04:41:16 INFO juju.worker.uniter.operation runhook.go:186 ran "mysql-pebble-ready" hook (via hook dispatching script: dispatch)
2024-03-02T04:44:10.557Z [container-agent] 2024-03-02 04:44:10 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T04:44:12.414Z [container-agent] 2024-03-02 04:44:12 WARNING juju-log No relation: certificates
2024-03-02T04:44:12.770Z [container-agent] 2024-03-02 04:44:12 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T04:49:58.980Z [container-agent] 2024-03-02 04:49:58 INFO juju-log Starting the log rotate manager
2024-03-02T04:49:58.993Z [container-agent] 2024-03-02 04:49:58 INFO juju-log Started log rotate manager process with PID 510312
2024-03-02T04:49:59.274Z [container-agent] 2024-03-02 04:49:59 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T04:50:01.247Z [container-agent] 2024-03-02 04:50:01 WARNING juju-log No relation: certificates
2024-03-02T04:50:01.622Z [container-agent] 2024-03-02 04:50:01 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T04:54:12.343Z [container-agent] 2024-03-02 04:54:12 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T04:54:14.947Z [container-agent] 2024-03-02 04:54:14 WARNING juju-log No relation: certificates
2024-03-02T04:54:15.309Z [container-agent] 2024-03-02 04:54:15 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T04:58:16.678Z [container-agent] 2024-03-02 04:58:16 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T04:58:19.581Z [container-agent] 2024-03-02 04:58:19 WARNING juju-log No relation: certificates
2024-03-02T04:58:19.957Z [container-agent] 2024-03-02 04:58:19 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T05:02:42.484Z [container-agent] 2024-03-02 05:02:42 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T05:02:45.904Z [container-agent] 2024-03-02 05:02:45 WARNING juju-log No relation: certificates
2024-03-02T05:02:46.252Z [container-agent] 2024-03-02 05:02:46 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T05:07:45.980Z [container-agent] 2024-03-02 05:07:45 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T05:07:49.702Z [container-agent] 2024-03-02 05:07:49 WARNING juju-log No relation: certificates
2024-03-02T05:07:50.098Z [container-agent] 2024-03-02 05:07:50 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T05:12:50.195Z [container-agent] 2024-03-02 05:12:50 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T05:12:54.325Z [container-agent] 2024-03-02 05:12:54 WARNING juju-log No relation: certificates
2024-03-02T05:12:54.696Z [container-agent] 2024-03-02 05:12:54 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T05:17:30.117Z [container-agent] 2024-03-02 05:17:30 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T05:17:34.852Z [container-agent] 2024-03-02 05:17:34 WARNING juju-log No relation: certificates
2024-03-02T05:17:35.218Z [container-agent] 2024-03-02 05:17:35 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T05:22:36.707Z [container-agent] 2024-03-02 05:22:36 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T05:22:41.718Z [container-agent] 2024-03-02 05:22:41 WARNING juju-log No relation: certificates
2024-03-02T05:22:42.047Z [container-agent] 2024-03-02 05:22:42 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T05:28:33.082Z [container-agent] 2024-03-02 05:28:33 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T05:28:38.863Z [container-agent] 2024-03-02 05:28:38 WARNING juju-log No relation: certificates
2024-03-02T05:28:39.233Z [container-agent] 2024-03-02 05:28:39 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T05:32:53.961Z [container-agent] 2024-03-02 05:32:53 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T05:32:59.607Z [container-agent] 2024-03-02 05:32:59 WARNING juju-log No relation: certificates
2024-03-02T05:32:59.953Z [container-agent] 2024-03-02 05:32:59 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T05:37:41.831Z [container-agent] 2024-03-02 05:37:41 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T05:37:48.525Z [container-agent] 2024-03-02 05:37:48 WARNING juju-log No relation: certificates
2024-03-02T05:37:48.945Z [container-agent] 2024-03-02 05:37:48 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T05:42:20.520Z [container-agent] 2024-03-02 05:42:20 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T05:42:27.410Z [container-agent] 2024-03-02 05:42:27 WARNING juju-log No relation: certificates
2024-03-02T05:42:27.786Z [container-agent] 2024-03-02 05:42:27 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T05:46:54.006Z [container-agent] 2024-03-02 05:46:54 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T05:47:01.136Z [container-agent] 2024-03-02 05:47:01 WARNING juju-log No relation: certificates
2024-03-02T05:47:01.462Z [container-agent] 2024-03-02 05:47:01 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T05:51:22.151Z [container-agent] 2024-03-02 05:51:22 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T05:51:29.361Z [container-agent] 2024-03-02 05:51:29 WARNING juju-log No relation: certificates
2024-03-02T05:51:29.710Z [container-agent] 2024-03-02 05:51:29 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T05:55:49.878Z [container-agent] 2024-03-02 05:55:49 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T05:55:58.305Z [container-agent] 2024-03-02 05:55:58 WARNING juju-log No relation: certificates
2024-03-02T05:55:58.743Z [container-agent] 2024-03-02 05:55:58 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T05:57:07.686Z [container-agent] 2024-03-02 05:57:07 ERROR juju-log Failed to flush ERROR LOGS logs.
2024-03-02T05:57:07.686Z [container-agent] Traceback (most recent call last):
2024-03-02T05:57:07.686Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 666, in _run_mysqlsh_script
2024-03-02T05:57:07.686Z [container-agent]     stdout, _ = process.wait_output()
2024-03-02T05:57:07.686Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1441, in wait_output
2024-03-02T05:57:07.686Z [container-agent]     raise ExecError[AnyStr](self._command, exit_code, out_value, err_value)
2024-03-02T05:57:07.686Z [container-agent] ops.pebble.ExecError: non-zero exit code 1 executing ['/usr/bin/mysqlsh', '--no-wizard', '--python', '--verbose=1', '-f', '/tmp/script.py', ';', 'rm', '/tmp/script.py'], stdout='', stderr='Cannot set LC_ALL to locale en_US.UTF-8: No such file or directory\nverbose: 2024-03-02T05:57:06Z: Loading startup files...\nverbose: 2024-03-02T05:57:06Z: Loading plugins...\nverbose: 2024-03-02T05:57:06Z: Connecting to MySQL at: serverconfig@horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local\nTraceback (most recent call last):\n  File "<string>", line 1, in <module>\nmysqlsh.DBError: MySQL Error (2003): Shell.connect: Can\'t connect to MySQL server on \'horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local:3306\' (111)\n'
2024-03-02T05:57:07.686Z [container-agent] 
2024-03-02T05:57:07.686Z [container-agent] During handling of the above exception, another exception occurred:
2024-03-02T05:57:07.686Z [container-agent] 
2024-03-02T05:57:07.686Z [container-agent] Traceback (most recent call last):
2024-03-02T05:57:07.686Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-03-02T05:57:07.686Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-03-02T05:57:07.686Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 669, in _run_mysqlsh_script
2024-03-02T05:57:07.686Z [container-agent]     raise MySQLClientError(e.stderr)
2024-03-02T05:57:07.686Z [container-agent] charms.mysql.v0.mysql.MySQLClientError: Cannot set LC_ALL to locale en_US.UTF-8: No such file or directory
2024-03-02T05:57:07.686Z [container-agent] verbose: 2024-03-02T05:57:06Z: Loading startup files...
2024-03-02T05:57:07.686Z [container-agent] verbose: 2024-03-02T05:57:06Z: Loading plugins...
2024-03-02T05:57:07.686Z [container-agent] verbose: 2024-03-02T05:57:06Z: Connecting to MySQL at: serverconfig@horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local
2024-03-02T05:57:07.686Z [container-agent] Traceback (most recent call last):
2024-03-02T05:57:07.686Z [container-agent]   File "<string>", line 1, in <module>
2024-03-02T05:57:07.686Z [container-agent] mysqlsh.DBError: MySQL Error (2003): Shell.connect: Can't connect to MySQL server on 'horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local:3306' (111)
2024-03-02T05:57:07.686Z [container-agent] 
2024-03-02T06:00:26.929Z [container-agent] 2024-03-02 06:00:26 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-03-02T06:00:26.958Z [container-agent] 2024-03-02 06:00:26 INFO juju-log Attempting reboot from complete outage.
2024-03-02T06:00:34.837Z [container-agent] 2024-03-02 06:00:34 WARNING juju-log No relation: certificates
2024-03-02T06:00:35.189Z [container-agent] 2024-03-02 06:00:35 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T06:02:04.790Z [container-agent] 2024-03-02 06:02:04 ERROR juju-log Uncaught exception while in charm code:
2024-03-02T06:02:04.790Z [container-agent] Traceback (most recent call last):
2024-03-02T06:02:04.790Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/./src/charm.py", line 786, in <module>
2024-03-02T06:02:04.790Z [container-agent]     main(MySQLOperatorCharm)
2024-03-02T06:02:04.790Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 436, in main
2024-03-02T06:02:04.790Z [container-agent]     _emit_charm_event(charm, dispatcher.event_name)
2024-03-02T06:02:04.790Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 144, in _emit_charm_event
2024-03-02T06:02:04.790Z [container-agent]     event_to_emit.emit(*args, **kwargs)
2024-03-02T06:02:04.790Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 351, in emit
2024-03-02T06:02:04.790Z [container-agent]     framework._emit(event)
2024-03-02T06:02:04.790Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 853, in _emit
2024-03-02T06:02:04.790Z [container-agent]     self._reemit(event_path)
2024-03-02T06:02:04.791Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 942, in _reemit
2024-03-02T06:02:04.791Z [container-agent]     custom_handler(event)
2024-03-02T06:02:04.791Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/rotate_mysql_logs.py", line 55, in _rotate_mysql_logs
2024-03-02T06:02:04.791Z [container-agent]     self.charm._mysql.flush_mysql_logs(MySQLTextLogs.ERROR)
2024-03-02T06:02:04.791Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-03-02T06:02:04.791Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-03-02T06:02:04.791Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 665, in _run_mysqlsh_script
2024-03-02T06:02:04.791Z [container-agent]     process = self.container.exec(cmd, timeout=timeout)
2024-03-02T06:02:04.791Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/model.py", line 2688, in exec
2024-03-02T06:02:04.791Z [container-agent]     return self._pebble.exec(
2024-03-02T06:02:04.791Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 2501, in exec
2024-03-02T06:02:04.791Z [container-agent]     resp = self._request('POST', '/v1/exec', body=body)
2024-03-02T06:02:04.791Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1655, in _request
2024-03-02T06:02:04.791Z [container-agent]     response = self._request_raw(method, path, query, headers, data)
2024-03-02T06:02:04.791Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1690, in _request_raw
2024-03-02T06:02:04.791Z [container-agent]     response = self.opener.open(request, timeout=self.timeout)
2024-03-02T06:02:04.791Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 519, in open
2024-03-02T06:02:04.791Z [container-agent]     response = self._open(req, data)
2024-03-02T06:02:04.791Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 536, in _open
2024-03-02T06:02:04.791Z [container-agent]     result = self._call_chain(self.handle_open, protocol, protocol +
2024-03-02T06:02:04.791Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 496, in _call_chain
2024-03-02T06:02:04.791Z [container-agent]     result = func(*args)
2024-03-02T06:02:04.791Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 311, in http_open
2024-03-02T06:02:04.791Z [container-agent]     return self.do_open(_UnixSocketConnection, req,  # type:ignore
2024-03-02T06:02:04.791Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 1352, in do_open
2024-03-02T06:02:04.791Z [container-agent]     r = h.getresponse()
2024-03-02T06:02:04.791Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1375, in getresponse
2024-03-02T06:02:04.791Z [container-agent]     response.begin()
2024-03-02T06:02:04.791Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 318, in begin
2024-03-02T06:02:04.791Z [container-agent]     version, status, reason = self._read_status()
2024-03-02T06:02:04.791Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 287, in _read_status
2024-03-02T06:02:04.791Z [container-agent]     raise RemoteDisconnected("Remote end closed connection without"
2024-03-02T06:02:04.791Z [container-agent] http.client.RemoteDisconnected: Remote end closed connection without response
2024-03-02T06:02:04.918Z [container-agent] 2024-03-02 06:02:04 INFO juju.util.exec exec.go:209 run result: exit status 1
2024-03-02T06:02:09.991Z [container-agent] 2024-03-02 06:02:09 INFO juju-log Setting up the logrotate configurations
2024-03-02T06:02:10.055Z [container-agent] 2024-03-02 06:02:10 INFO juju-log Adding pebble layer
2024-03-02T06:02:16.769Z [container-agent] 2024-03-02 06:02:16 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-03-02T06:02:16.792Z [container-agent] 2024-03-02 06:02:16 INFO juju-log Attempting reboot from complete outage.
2024-03-02T06:02:19.734Z [container-agent] 2024-03-02 06:02:19 INFO juju.worker.uniter.operation runhook.go:186 ran "mysql-pebble-ready" hook (via hook dispatching script: dispatch)
2024-03-02T06:04:58.962Z [container-agent] 2024-03-02 06:04:58 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T06:05:00.943Z [container-agent] 2024-03-02 06:05:00 WARNING juju-log No relation: certificates
2024-03-02T06:05:01.314Z [container-agent] 2024-03-02 06:05:01 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T06:09:33.751Z [container-agent] 2024-03-02 06:09:33 INFO juju-log Starting the log rotate manager
2024-03-02T06:09:33.765Z [container-agent] 2024-03-02 06:09:33 INFO juju-log Started log rotate manager process with PID 518033
2024-03-02T06:09:34.070Z [container-agent] 2024-03-02 06:09:34 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T06:09:36.301Z [container-agent] 2024-03-02 06:09:36 WARNING juju-log No relation: certificates
2024-03-02T06:09:36.656Z [container-agent] 2024-03-02 06:09:36 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T06:14:24.182Z [container-agent] 2024-03-02 06:14:24 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T06:14:26.543Z [container-agent] 2024-03-02 06:14:26 WARNING juju-log No relation: certificates
2024-03-02T06:14:26.907Z [container-agent] 2024-03-02 06:14:26 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T06:19:29.212Z [container-agent] 2024-03-02 06:19:29 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T06:19:32.251Z [container-agent] 2024-03-02 06:19:32 WARNING juju-log No relation: certificates
2024-03-02T06:19:32.599Z [container-agent] 2024-03-02 06:19:32 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T06:23:48.950Z [container-agent] 2024-03-02 06:23:48 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T06:23:52.294Z [container-agent] 2024-03-02 06:23:52 WARNING juju-log No relation: certificates
2024-03-02T06:23:52.634Z [container-agent] 2024-03-02 06:23:52 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T06:29:31.293Z [container-agent] 2024-03-02 06:29:31 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T06:29:35.255Z [container-agent] 2024-03-02 06:29:35 WARNING juju-log No relation: certificates
2024-03-02T06:29:35.601Z [container-agent] 2024-03-02 06:29:35 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T06:34:30.857Z [container-agent] 2024-03-02 06:34:30 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T06:34:35.327Z [container-agent] 2024-03-02 06:34:35 WARNING juju-log No relation: certificates
2024-03-02T06:34:35.692Z [container-agent] 2024-03-02 06:34:35 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T06:39:39.178Z [container-agent] 2024-03-02 06:39:39 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T06:39:43.698Z [container-agent] 2024-03-02 06:39:43 WARNING juju-log No relation: certificates
2024-03-02T06:39:44.050Z [container-agent] 2024-03-02 06:39:44 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T06:44:31.428Z [container-agent] 2024-03-02 06:44:31 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T06:44:36.503Z [container-agent] 2024-03-02 06:44:36 WARNING juju-log No relation: certificates
2024-03-02T06:44:36.871Z [container-agent] 2024-03-02 06:44:36 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T06:49:17.803Z [container-agent] 2024-03-02 06:49:17 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T06:49:22.916Z [container-agent] 2024-03-02 06:49:22 WARNING juju-log No relation: certificates
2024-03-02T06:49:23.264Z [container-agent] 2024-03-02 06:49:23 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T06:55:15.763Z [container-agent] 2024-03-02 06:55:15 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T06:55:22.052Z [container-agent] 2024-03-02 06:55:22 WARNING juju-log No relation: certificates
2024-03-02T06:55:22.408Z [container-agent] 2024-03-02 06:55:22 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T06:59:26.094Z [container-agent] 2024-03-02 06:59:26 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T06:59:32.924Z [container-agent] 2024-03-02 06:59:32 WARNING juju-log No relation: certificates
2024-03-02T06:59:33.297Z [container-agent] 2024-03-02 06:59:33 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T07:04:32.387Z [container-agent] 2024-03-02 07:04:32 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T07:04:39.571Z [container-agent] 2024-03-02 07:04:39 WARNING juju-log No relation: certificates
2024-03-02T07:04:39.944Z [container-agent] 2024-03-02 07:04:39 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T07:09:01.010Z [container-agent] 2024-03-02 07:09:01 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T07:09:08.342Z [container-agent] 2024-03-02 07:09:08 WARNING juju-log No relation: certificates
2024-03-02T07:09:08.717Z [container-agent] 2024-03-02 07:09:08 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T07:13:44.474Z [container-agent] 2024-03-02 07:13:44 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T07:13:52.285Z [container-agent] 2024-03-02 07:13:52 WARNING juju-log No relation: certificates
2024-03-02T07:13:52.633Z [container-agent] 2024-03-02 07:13:52 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T07:18:50.478Z [container-agent] 2024-03-02 07:18:50 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T07:18:58.877Z [container-agent] 2024-03-02 07:18:58 WARNING juju-log No relation: certificates
2024-03-02T07:18:59.278Z [container-agent] 2024-03-02 07:18:59 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T07:22:58.247Z [container-agent] 2024-03-02 07:22:58 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T07:23:06.141Z [container-agent] 2024-03-02 07:23:06 WARNING juju-log No relation: certificates
2024-03-02T07:23:06.486Z [container-agent] 2024-03-02 07:23:06 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T07:28:35.765Z [container-agent] 2024-03-02 07:28:35 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-03-02T07:28:35.791Z [container-agent] 2024-03-02 07:28:35 INFO juju-log Attempting reboot from complete outage.
2024-03-02T07:28:43.983Z [container-agent] 2024-03-02 07:28:43 WARNING juju-log No relation: certificates
2024-03-02T07:28:44.361Z [container-agent] 2024-03-02 07:28:44 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T07:33:04.780Z [container-agent] 2024-03-02 07:33:04 ERROR juju-log Uncaught exception while in charm code:
2024-03-02T07:33:04.780Z [container-agent] Traceback (most recent call last):
2024-03-02T07:33:04.780Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/./src/charm.py", line 786, in <module>
2024-03-02T07:33:04.780Z [container-agent]     main(MySQLOperatorCharm)
2024-03-02T07:33:04.780Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 436, in main
2024-03-02T07:33:04.780Z [container-agent]     _emit_charm_event(charm, dispatcher.event_name)
2024-03-02T07:33:04.780Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 144, in _emit_charm_event
2024-03-02T07:33:04.780Z [container-agent]     event_to_emit.emit(*args, **kwargs)
2024-03-02T07:33:04.780Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 351, in emit
2024-03-02T07:33:04.780Z [container-agent]     framework._emit(event)
2024-03-02T07:33:04.780Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 853, in _emit
2024-03-02T07:33:04.780Z [container-agent]     self._reemit(event_path)
2024-03-02T07:33:04.780Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 942, in _reemit
2024-03-02T07:33:04.780Z [container-agent]     custom_handler(event)
2024-03-02T07:33:04.780Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/rotate_mysql_logs.py", line 55, in _rotate_mysql_logs
2024-03-02T07:33:04.780Z [container-agent]     self.charm._mysql.flush_mysql_logs(MySQLTextLogs.ERROR)
2024-03-02T07:33:04.780Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-03-02T07:33:04.780Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-03-02T07:33:04.780Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 665, in _run_mysqlsh_script
2024-03-02T07:33:04.780Z [container-agent]     process = self.container.exec(cmd, timeout=timeout)
2024-03-02T07:33:04.780Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/model.py", line 2688, in exec
2024-03-02T07:33:04.780Z [container-agent]     return self._pebble.exec(
2024-03-02T07:33:04.780Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 2501, in exec
2024-03-02T07:33:04.780Z [container-agent]     resp = self._request('POST', '/v1/exec', body=body)
2024-03-02T07:33:04.780Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1655, in _request
2024-03-02T07:33:04.780Z [container-agent]     response = self._request_raw(method, path, query, headers, data)
2024-03-02T07:33:04.780Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1690, in _request_raw
2024-03-02T07:33:04.780Z [container-agent]     response = self.opener.open(request, timeout=self.timeout)
2024-03-02T07:33:04.780Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 519, in open
2024-03-02T07:33:04.780Z [container-agent]     response = self._open(req, data)
2024-03-02T07:33:04.780Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 536, in _open
2024-03-02T07:33:04.780Z [container-agent]     result = self._call_chain(self.handle_open, protocol, protocol +
2024-03-02T07:33:04.780Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 496, in _call_chain
2024-03-02T07:33:04.780Z [container-agent]     result = func(*args)
2024-03-02T07:33:04.780Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 311, in http_open
2024-03-02T07:33:04.780Z [container-agent]     return self.do_open(_UnixSocketConnection, req,  # type:ignore
2024-03-02T07:33:04.780Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 1352, in do_open
2024-03-02T07:33:04.780Z [container-agent]     r = h.getresponse()
2024-03-02T07:33:04.780Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1375, in getresponse
2024-03-02T07:33:04.780Z [container-agent]     response.begin()
2024-03-02T07:33:04.780Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 318, in begin
2024-03-02T07:33:04.780Z [container-agent]     version, status, reason = self._read_status()
2024-03-02T07:33:04.780Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 287, in _read_status
2024-03-02T07:33:04.780Z [container-agent]     raise RemoteDisconnected("Remote end closed connection without"
2024-03-02T07:33:04.780Z [container-agent] http.client.RemoteDisconnected: Remote end closed connection without response
2024-03-02T07:33:04.897Z [container-agent] 2024-03-02 07:33:04 INFO juju.util.exec exec.go:209 run result: exit status 1
2024-03-02T07:33:08.244Z [container-agent] 2024-03-02 07:33:08 INFO juju-log Setting up the logrotate configurations
2024-03-02T07:33:08.310Z [container-agent] 2024-03-02 07:33:08 INFO juju-log Adding pebble layer
2024-03-02T07:33:15.057Z [container-agent] 2024-03-02 07:33:15 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-03-02T07:33:15.081Z [container-agent] 2024-03-02 07:33:15 INFO juju-log Attempting reboot from complete outage.
2024-03-02T07:33:18.153Z [container-agent] 2024-03-02 07:33:18 INFO juju.worker.uniter.operation runhook.go:186 ran "mysql-pebble-ready" hook (via hook dispatching script: dispatch)
2024-03-02T07:34:16.547Z [container-agent] 2024-03-02 07:34:16 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T07:34:18.608Z [container-agent] 2024-03-02 07:34:18 WARNING juju-log No relation: certificates
2024-03-02T07:34:18.966Z [container-agent] 2024-03-02 07:34:18 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T07:40:03.726Z [container-agent] 2024-03-02 07:40:03 INFO juju-log Starting the log rotate manager
2024-03-02T07:40:03.743Z [container-agent] 2024-03-02 07:40:03 INFO juju-log Started log rotate manager process with PID 526457
2024-03-02T07:40:04.064Z [container-agent] 2024-03-02 07:40:04 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T07:40:06.217Z [container-agent] 2024-03-02 07:40:06 WARNING juju-log No relation: certificates
2024-03-02T07:40:06.560Z [container-agent] 2024-03-02 07:40:06 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T07:44:08.113Z [container-agent] 2024-03-02 07:44:08 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T07:44:10.317Z [container-agent] 2024-03-02 07:44:10 WARNING juju-log No relation: certificates
2024-03-02T07:44:10.669Z [container-agent] 2024-03-02 07:44:10 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T07:49:37.400Z [container-agent] 2024-03-02 07:49:37 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T07:49:40.296Z [container-agent] 2024-03-02 07:49:40 WARNING juju-log No relation: certificates
2024-03-02T07:49:40.647Z [container-agent] 2024-03-02 07:49:40 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T07:54:24.949Z [container-agent] 2024-03-02 07:54:24 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T07:54:28.475Z [container-agent] 2024-03-02 07:54:28 WARNING juju-log No relation: certificates
2024-03-02T07:54:28.888Z [container-agent] 2024-03-02 07:54:28 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T07:59:09.389Z [container-agent] 2024-03-02 07:59:09 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T07:59:12.806Z [container-agent] 2024-03-02 07:59:12 WARNING juju-log No relation: certificates
2024-03-02T07:59:13.179Z [container-agent] 2024-03-02 07:59:13 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T08:03:54.646Z [container-agent] 2024-03-02 08:03:54 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T08:03:58.628Z [container-agent] 2024-03-02 08:03:58 WARNING juju-log No relation: certificates
2024-03-02T08:03:58.976Z [container-agent] 2024-03-02 08:03:58 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T08:09:09.557Z [container-agent] 2024-03-02 08:09:09 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T08:09:14.169Z [container-agent] 2024-03-02 08:09:14 WARNING juju-log No relation: certificates
2024-03-02T08:09:14.528Z [container-agent] 2024-03-02 08:09:14 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T08:13:54.594Z [container-agent] 2024-03-02 08:13:54 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T08:13:59.537Z [container-agent] 2024-03-02 08:13:59 WARNING juju-log No relation: certificates
2024-03-02T08:13:59.912Z [container-agent] 2024-03-02 08:13:59 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T08:17:58.766Z [container-agent] 2024-03-02 08:17:58 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T08:18:04.258Z [container-agent] 2024-03-02 08:18:04 WARNING juju-log No relation: certificates
2024-03-02T08:18:04.638Z [container-agent] 2024-03-02 08:18:04 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T08:22:31.107Z [container-agent] 2024-03-02 08:22:31 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T08:22:37.030Z [container-agent] 2024-03-02 08:22:37 WARNING juju-log No relation: certificates
2024-03-02T08:22:37.372Z [container-agent] 2024-03-02 08:22:37 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T08:28:22.331Z [container-agent] 2024-03-02 08:28:22 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T08:28:28.333Z [container-agent] 2024-03-02 08:28:28 WARNING juju-log No relation: certificates
2024-03-02T08:28:28.693Z [container-agent] 2024-03-02 08:28:28 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T08:33:37.245Z [container-agent] 2024-03-02 08:33:37 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T08:33:43.338Z [container-agent] 2024-03-02 08:33:43 WARNING juju-log No relation: certificates
2024-03-02T08:33:43.681Z [container-agent] 2024-03-02 08:33:43 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T08:38:12.790Z [container-agent] 2024-03-02 08:38:12 ERROR juju-log Failed to flush SLOW LOGS logs.
2024-03-02T08:38:12.790Z [container-agent] Traceback (most recent call last):
2024-03-02T08:38:12.790Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 666, in _run_mysqlsh_script
2024-03-02T08:38:12.790Z [container-agent]     stdout, _ = process.wait_output()
2024-03-02T08:38:12.790Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1441, in wait_output
2024-03-02T08:38:12.790Z [container-agent]     raise ExecError[AnyStr](self._command, exit_code, out_value, err_value)
2024-03-02T08:38:12.790Z [container-agent] ops.pebble.ExecError: non-zero exit code 1 executing ['/usr/bin/mysqlsh', '--no-wizard', '--python', '--verbose=1', '-f', '/tmp/script.py', ';', 'rm', '/tmp/script.py'], stdout='', stderr='Cannot set LC_ALL to locale en_US.UTF-8: No such file or directory\nverbose: 2024-03-02T08:38:11Z: Loading startup files...\nverbose: 2024-03-02T08:38:11Z: Loading plugins...\nverbose: 2024-03-02T08:38:11Z: Connecting to MySQL at: serverconfig@horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local\nTraceback (most recent call last):\n  File "<string>", line 1, in <module>\nmysqlsh.DBError: MySQL Error (2003): Shell.connect: Can\'t connect to MySQL server on \'horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local:3306\' (111)\n'
2024-03-02T08:38:12.790Z [container-agent] 
2024-03-02T08:38:12.790Z [container-agent] During handling of the above exception, another exception occurred:
2024-03-02T08:38:12.790Z [container-agent] 
2024-03-02T08:38:12.790Z [container-agent] Traceback (most recent call last):
2024-03-02T08:38:12.790Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-03-02T08:38:12.790Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-03-02T08:38:12.790Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 669, in _run_mysqlsh_script
2024-03-02T08:38:12.790Z [container-agent]     raise MySQLClientError(e.stderr)
2024-03-02T08:38:12.790Z [container-agent] charms.mysql.v0.mysql.MySQLClientError: Cannot set LC_ALL to locale en_US.UTF-8: No such file or directory
2024-03-02T08:38:12.790Z [container-agent] verbose: 2024-03-02T08:38:11Z: Loading startup files...
2024-03-02T08:38:12.790Z [container-agent] verbose: 2024-03-02T08:38:11Z: Loading plugins...
2024-03-02T08:38:12.790Z [container-agent] verbose: 2024-03-02T08:38:11Z: Connecting to MySQL at: serverconfig@horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local
2024-03-02T08:38:12.790Z [container-agent] Traceback (most recent call last):
2024-03-02T08:38:12.790Z [container-agent]   File "<string>", line 1, in <module>
2024-03-02T08:38:12.790Z [container-agent] mysqlsh.DBError: MySQL Error (2003): Shell.connect: Can't connect to MySQL server on 'horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local:3306' (111)
2024-03-02T08:38:12.790Z [container-agent] 
2024-03-02T08:39:32.763Z [container-agent] 2024-03-02 08:39:32 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-03-02T08:39:32.791Z [container-agent] 2024-03-02 08:39:32 INFO juju-log Attempting reboot from complete outage.
2024-03-02T08:39:39.815Z [container-agent] 2024-03-02 08:39:39 WARNING juju-log No relation: certificates
2024-03-02T08:39:40.180Z [container-agent] 2024-03-02 08:39:40 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T08:43:09.791Z [container-agent] 2024-03-02 08:43:09 ERROR juju-log Uncaught exception while in charm code:
2024-03-02T08:43:09.791Z [container-agent] Traceback (most recent call last):
2024-03-02T08:43:09.791Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 2507, in exec
2024-03-02T08:43:09.791Z [container-agent]     control_ws = self._connect_websocket(task_id, 'control')
2024-03-02T08:43:09.791Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 2589, in _connect_websocket
2024-03-02T08:43:09.791Z [container-agent]     ws.connect(url, socket=sock)
2024-03-02T08:43:09.791Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_core.py", line 253, in connect
2024-03-02T08:43:09.791Z [container-agent]     self.handshake_response = handshake(self.sock, url, *addrs, **options)
2024-03-02T08:43:09.791Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_handshake.py", line 57, in handshake
2024-03-02T08:43:09.791Z [container-agent]     status, resp = _get_resp_headers(sock)
2024-03-02T08:43:09.791Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_handshake.py", line 143, in _get_resp_headers
2024-03-02T08:43:09.791Z [container-agent]     status, resp_headers, status_message = read_headers(sock)
2024-03-02T08:43:09.791Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_http.py", line 312, in read_headers
2024-03-02T08:43:09.791Z [container-agent]     line = recv_line(sock)
2024-03-02T08:43:09.791Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_socket.py", line 131, in recv_line
2024-03-02T08:43:09.791Z [container-agent]     c = recv(sock, 1)
2024-03-02T08:43:09.791Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_socket.py", line 122, in recv
2024-03-02T08:43:09.791Z [container-agent]     raise WebSocketConnectionClosedException(
2024-03-02T08:43:09.791Z [container-agent] websocket._exceptions.WebSocketConnectionClosedException: Connection to remote host was lost.
2024-03-02T08:43:09.791Z [container-agent] 
2024-03-02T08:43:09.791Z [container-agent] During handling of the above exception, another exception occurred:
2024-03-02T08:43:09.791Z [container-agent] 
2024-03-02T08:43:09.791Z [container-agent] Traceback (most recent call last):
2024-03-02T08:43:09.791Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 1348, in do_open
2024-03-02T08:43:09.791Z [container-agent]     h.request(req.get_method(), req.selector, req.data, headers,
2024-03-02T08:43:09.791Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1283, in request
2024-03-02T08:43:09.791Z [container-agent]     self._send_request(method, url, body, headers, encode_chunked)
2024-03-02T08:43:09.791Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1329, in _send_request
2024-03-02T08:43:09.791Z [container-agent]     self.endheaders(body, encode_chunked=encode_chunked)
2024-03-02T08:43:09.791Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1278, in endheaders
2024-03-02T08:43:09.791Z [container-agent]     self._send_output(message_body, encode_chunked=encode_chunked)
2024-03-02T08:43:09.791Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1038, in _send_output
2024-03-02T08:43:09.791Z [container-agent]     self.send(msg)
2024-03-02T08:43:09.791Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 976, in send
2024-03-02T08:43:09.791Z [container-agent]     self.connect()
2024-03-02T08:43:09.791Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 297, in connect
2024-03-02T08:43:09.791Z [container-agent]     self.sock.connect(self.socket_path)
2024-03-02T08:43:09.791Z [container-agent] ConnectionRefusedError: [Errno 111] Connection refused
2024-03-02T08:43:09.791Z [container-agent] 
2024-03-02T08:43:09.791Z [container-agent] During handling of the above exception, another exception occurred:
2024-03-02T08:43:09.791Z [container-agent] 
2024-03-02T08:43:09.791Z [container-agent] Traceback (most recent call last):
2024-03-02T08:43:09.791Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1690, in _request_raw
2024-03-02T08:43:09.791Z [container-agent]     response = self.opener.open(request, timeout=self.timeout)
2024-03-02T08:43:09.791Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 519, in open
2024-03-02T08:43:09.791Z [container-agent]     response = self._open(req, data)
2024-03-02T08:43:09.791Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 536, in _open
2024-03-02T08:43:09.791Z [container-agent]     result = self._call_chain(self.handle_open, protocol, protocol +
2024-03-02T08:43:09.791Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 496, in _call_chain
2024-03-02T08:43:09.791Z [container-agent]     result = func(*args)
2024-03-02T08:43:09.791Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 311, in http_open
2024-03-02T08:43:09.791Z [container-agent]     return self.do_open(_UnixSocketConnection, req,  # type:ignore
2024-03-02T08:43:09.791Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 1351, in do_open
2024-03-02T08:43:09.791Z [container-agent]     raise URLError(err)
2024-03-02T08:43:09.791Z [container-agent] urllib.error.URLError: <urlopen error [Errno 111] Connection refused>
2024-03-02T08:43:09.791Z [container-agent] 
2024-03-02T08:43:09.791Z [container-agent] The above exception was the direct cause of the following exception:
2024-03-02T08:43:09.791Z [container-agent] 
2024-03-02T08:43:09.791Z [container-agent] Traceback (most recent call last):
2024-03-02T08:43:09.791Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/./src/charm.py", line 786, in <module>
2024-03-02T08:43:09.791Z [container-agent]     main(MySQLOperatorCharm)
2024-03-02T08:43:09.791Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 436, in main
2024-03-02T08:43:09.791Z [container-agent]     _emit_charm_event(charm, dispatcher.event_name)
2024-03-02T08:43:09.791Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 144, in _emit_charm_event
2024-03-02T08:43:09.791Z [container-agent]     event_to_emit.emit(*args, **kwargs)
2024-03-02T08:43:09.791Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 351, in emit
2024-03-02T08:43:09.791Z [container-agent]     framework._emit(event)
2024-03-02T08:43:09.791Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 853, in _emit
2024-03-02T08:43:09.791Z [container-agent]     self._reemit(event_path)
2024-03-02T08:43:09.791Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 942, in _reemit
2024-03-02T08:43:09.791Z [container-agent]     custom_handler(event)
2024-03-02T08:43:09.791Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/rotate_mysql_logs.py", line 57, in _rotate_mysql_logs
2024-03-02T08:43:09.791Z [container-agent]     self.charm._mysql.flush_mysql_logs(MySQLTextLogs.SLOW)
2024-03-02T08:43:09.791Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-03-02T08:43:09.791Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-03-02T08:43:09.791Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 665, in _run_mysqlsh_script
2024-03-02T08:43:09.791Z [container-agent]     process = self.container.exec(cmd, timeout=timeout)
2024-03-02T08:43:09.791Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/model.py", line 2688, in exec
2024-03-02T08:43:09.791Z [container-agent]     return self._pebble.exec(
2024-03-02T08:43:09.791Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 2514, in exec
2024-03-02T08:43:09.791Z [container-agent]     change = self.wait_change(ChangeID(change_id))
2024-03-02T08:43:09.791Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1893, in wait_change
2024-03-02T08:43:09.791Z [container-agent]     return self._wait_change_using_wait(change_id, timeout)
2024-03-02T08:43:09.791Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1914, in _wait_change_using_wait
2024-03-02T08:43:09.791Z [container-agent]     return self._wait_change(change_id, this_timeout)
2024-03-02T08:43:09.791Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1928, in _wait_change
2024-03-02T08:43:09.791Z [container-agent]     resp = self._request('GET', f'/v1/changes/{change_id}/wait', query)
2024-03-02T08:43:09.791Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1655, in _request
2024-03-02T08:43:09.791Z [container-agent]     response = self._request_raw(method, path, query, headers, data)
2024-03-02T08:43:09.791Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1707, in _request_raw
2024-03-02T08:43:09.791Z [container-agent]     raise ConnectionError(e.reason) from e
2024-03-02T08:43:09.791Z [container-agent] ops.pebble.ConnectionError: [Errno 111] Connection refused
2024-03-02T08:43:09.970Z [container-agent] 2024-03-02 08:43:09 INFO juju.util.exec exec.go:209 run result: exit status 1
2024-03-02T08:43:11.923Z [container-agent] 2024-03-02 08:43:11 INFO juju-log Setting up the logrotate configurations
2024-03-02T08:43:11.992Z [container-agent] 2024-03-02 08:43:11 INFO juju-log Adding pebble layer
2024-03-02T08:43:18.814Z [container-agent] 2024-03-02 08:43:18 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-03-02T08:43:18.840Z [container-agent] 2024-03-02 08:43:18 INFO juju-log Attempting reboot from complete outage.
2024-03-02T08:43:21.905Z [container-agent] 2024-03-02 08:43:21 INFO juju.worker.uniter.operation runhook.go:186 ran "mysql-pebble-ready" hook (via hook dispatching script: dispatch)
2024-03-02T08:44:11.174Z [container-agent] 2024-03-02 08:44:11 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T08:44:13.065Z [container-agent] 2024-03-02 08:44:13 WARNING juju-log No relation: certificates
2024-03-02T08:44:13.403Z [container-agent] 2024-03-02 08:44:13 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T08:50:01.962Z [container-agent] 2024-03-02 08:50:01 INFO juju-log Starting the log rotate manager
2024-03-02T08:50:01.978Z [container-agent] 2024-03-02 08:50:01 INFO juju-log Started log rotate manager process with PID 532908
2024-03-02T08:50:02.321Z [container-agent] 2024-03-02 08:50:02 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T08:50:04.505Z [container-agent] 2024-03-02 08:50:04 WARNING juju-log No relation: certificates
2024-03-02T08:50:04.882Z [container-agent] 2024-03-02 08:50:04 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T08:55:59.590Z [container-agent] 2024-03-02 08:55:59 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T08:56:02.060Z [container-agent] 2024-03-02 08:56:02 WARNING juju-log No relation: certificates
2024-03-02T08:56:02.481Z [container-agent] 2024-03-02 08:56:02 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T09:00:01.654Z [container-agent] 2024-03-02 09:00:01 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T09:00:04.458Z [container-agent] 2024-03-02 09:00:04 WARNING juju-log No relation: certificates
2024-03-02T09:00:04.838Z [container-agent] 2024-03-02 09:00:04 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T09:04:58.604Z [container-agent] 2024-03-02 09:04:58 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T09:05:01.776Z [container-agent] 2024-03-02 09:05:01 WARNING juju-log No relation: certificates
2024-03-02T09:05:02.120Z [container-agent] 2024-03-02 09:05:02 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T09:09:26.897Z [container-agent] 2024-03-02 09:09:26 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T09:09:30.994Z [container-agent] 2024-03-02 09:09:30 WARNING juju-log No relation: certificates
2024-03-02T09:09:31.346Z [container-agent] 2024-03-02 09:09:31 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T09:14:34.314Z [container-agent] 2024-03-02 09:14:34 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T09:14:38.648Z [container-agent] 2024-03-02 09:14:38 WARNING juju-log No relation: certificates
2024-03-02T09:14:39.013Z [container-agent] 2024-03-02 09:14:39 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T09:19:40.690Z [container-agent] 2024-03-02 09:19:40 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T09:19:45.670Z [container-agent] 2024-03-02 09:19:45 WARNING juju-log No relation: certificates
2024-03-02T09:19:46.019Z [container-agent] 2024-03-02 09:19:46 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T09:23:55.916Z [container-agent] 2024-03-02 09:23:55 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T09:24:00.909Z [container-agent] 2024-03-02 09:24:00 WARNING juju-log No relation: certificates
2024-03-02T09:24:01.238Z [container-agent] 2024-03-02 09:24:01 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T09:28:44.165Z [container-agent] 2024-03-02 09:28:44 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T09:28:49.875Z [container-agent] 2024-03-02 09:28:49 WARNING juju-log No relation: certificates
2024-03-02T09:28:50.229Z [container-agent] 2024-03-02 09:28:50 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T09:33:24.063Z [container-agent] 2024-03-02 09:33:24 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T09:33:30.228Z [container-agent] 2024-03-02 09:33:30 WARNING juju-log No relation: certificates
2024-03-02T09:33:30.599Z [container-agent] 2024-03-02 09:33:30 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T09:38:11.976Z [container-agent] 2024-03-02 09:38:11 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T09:38:18.493Z [container-agent] 2024-03-02 09:38:18 WARNING juju-log No relation: certificates
2024-03-02T09:38:18.852Z [container-agent] 2024-03-02 09:38:18 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T09:43:12.561Z [container-agent] 2024-03-02 09:43:12 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T09:43:19.782Z [container-agent] 2024-03-02 09:43:19 WARNING juju-log No relation: certificates
2024-03-02T09:43:20.142Z [container-agent] 2024-03-02 09:43:20 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T09:48:46.336Z [container-agent] 2024-03-02 09:48:46 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T09:48:53.748Z [container-agent] 2024-03-02 09:48:53 WARNING juju-log No relation: certificates
2024-03-02T09:48:54.103Z [container-agent] 2024-03-02 09:48:54 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T09:53:22.162Z [container-agent] 2024-03-02 09:53:22 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T09:53:30.040Z [container-agent] 2024-03-02 09:53:30 WARNING juju-log No relation: certificates
2024-03-02T09:53:30.413Z [container-agent] 2024-03-02 09:53:30 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T09:58:00.565Z [container-agent] 2024-03-02 09:58:00 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T09:58:07.948Z [container-agent] 2024-03-02 09:58:07 WARNING juju-log No relation: certificates
2024-03-02T09:58:08.282Z [container-agent] 2024-03-02 09:58:08 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T10:02:17.927Z [container-agent] 2024-03-02 10:02:17 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T10:02:26.605Z [container-agent] 2024-03-02 10:02:26 WARNING juju-log No relation: certificates
2024-03-02T10:02:26.952Z [container-agent] 2024-03-02 10:02:26 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T10:07:21.829Z [container-agent] 2024-03-02 10:07:21 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T10:07:30.241Z [container-agent] 2024-03-02 10:07:30 WARNING juju-log No relation: certificates
2024-03-02T10:07:30.588Z [container-agent] 2024-03-02 10:07:30 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T10:12:18.099Z [container-agent] 2024-03-02 10:12:18 ERROR juju-log Failed to get member state: mysqld daemon is down
2024-03-02T10:12:30.746Z [container-agent] 2024-03-02 10:12:30 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-03-02T10:12:30.781Z [container-agent] 2024-03-02 10:12:30 INFO juju-log Attempting reboot from complete outage.
2024-03-02T10:12:39.245Z [container-agent] 2024-03-02 10:12:39 WARNING juju-log No relation: certificates
2024-03-02T10:12:39.595Z [container-agent] 2024-03-02 10:12:39 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T10:16:34.720Z [container-agent] 2024-03-02 10:16:34 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T10:16:44.405Z [container-agent] 2024-03-02 10:16:44 WARNING juju-log No relation: certificates
2024-03-02T10:16:44.793Z [container-agent] 2024-03-02 10:16:44 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T10:17:20.597Z [container-agent] 2024-03-02 10:17:20 INFO juju-log Setting up the logrotate configurations
2024-03-02T10:17:20.671Z [container-agent] 2024-03-02 10:17:20 INFO juju-log Adding pebble layer
2024-03-02T10:17:27.439Z [container-agent] 2024-03-02 10:17:27 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-03-02T10:17:27.466Z [container-agent] 2024-03-02 10:17:27 INFO juju-log Attempting reboot from complete outage.
2024-03-02T10:17:30.469Z [container-agent] 2024-03-02 10:17:30 INFO juju.worker.uniter.operation runhook.go:186 ran "mysql-pebble-ready" hook (via hook dispatching script: dispatch)
2024-03-02T10:21:05.217Z [container-agent] 2024-03-02 10:21:05 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T10:21:07.445Z [container-agent] 2024-03-02 10:21:07 WARNING juju-log No relation: certificates
2024-03-02T10:21:07.819Z [container-agent] 2024-03-02 10:21:07 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T10:25:13.354Z [container-agent] 2024-03-02 10:25:13 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T10:25:16.089Z [container-agent] 2024-03-02 10:25:16 WARNING juju-log No relation: certificates
2024-03-02T10:25:16.490Z [container-agent] 2024-03-02 10:25:16 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T10:29:34.055Z [container-agent] 2024-03-02 10:29:34 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T10:29:37.290Z [container-agent] 2024-03-02 10:29:37 WARNING juju-log No relation: certificates
2024-03-02T10:29:37.659Z [container-agent] 2024-03-02 10:29:37 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T10:34:59.602Z [container-agent] 2024-03-02 10:34:59 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T10:35:02.993Z [container-agent] 2024-03-02 10:35:02 WARNING juju-log No relation: certificates
2024-03-02T10:35:03.350Z [container-agent] 2024-03-02 10:35:03 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T10:39:13.139Z [container-agent] 2024-03-02 10:39:13 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T10:39:17.333Z [container-agent] 2024-03-02 10:39:17 WARNING juju-log No relation: certificates
2024-03-02T10:39:17.680Z [container-agent] 2024-03-02 10:39:17 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T10:43:56.874Z [container-agent] 2024-03-02 10:43:56 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T10:44:01.066Z [container-agent] 2024-03-02 10:44:01 WARNING juju-log No relation: certificates
2024-03-02T10:44:01.426Z [container-agent] 2024-03-02 10:44:01 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T10:48:42.517Z [container-agent] 2024-03-02 10:48:42 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T10:48:47.540Z [container-agent] 2024-03-02 10:48:47 WARNING juju-log No relation: certificates
2024-03-02T10:48:47.901Z [container-agent] 2024-03-02 10:48:47 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T10:53:13.773Z [container-agent] 2024-03-02 10:53:13 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T10:53:18.660Z [container-agent] 2024-03-02 10:53:18 WARNING juju-log No relation: certificates
2024-03-02T10:53:19.000Z [container-agent] 2024-03-02 10:53:19 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T10:57:28.673Z [container-agent] 2024-03-02 10:57:28 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T10:57:34.536Z [container-agent] 2024-03-02 10:57:34 WARNING juju-log No relation: certificates
2024-03-02T10:57:34.919Z [container-agent] 2024-03-02 10:57:34 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T11:02:13.670Z [container-agent] 2024-03-02 11:02:13 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T11:02:19.975Z [container-agent] 2024-03-02 11:02:19 WARNING juju-log No relation: certificates
2024-03-02T11:02:20.338Z [container-agent] 2024-03-02 11:02:20 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T11:06:50.230Z [container-agent] 2024-03-02 11:06:50 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T11:06:56.274Z [container-agent] 2024-03-02 11:06:56 WARNING juju-log No relation: certificates
2024-03-02T11:06:56.649Z [container-agent] 2024-03-02 11:06:56 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T11:12:12.642Z [container-agent] 2024-03-02 11:12:12 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T11:12:19.845Z [container-agent] 2024-03-02 11:12:19 WARNING juju-log No relation: certificates
2024-03-02T11:12:20.201Z [container-agent] 2024-03-02 11:12:20 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T11:17:30.232Z [container-agent] 2024-03-02 11:17:30 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T11:17:37.725Z [container-agent] 2024-03-02 11:17:37 WARNING juju-log No relation: certificates
2024-03-02T11:17:38.092Z [container-agent] 2024-03-02 11:17:38 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T11:22:15.067Z [container-agent] 2024-03-02 11:22:15 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T11:22:22.383Z [container-agent] 2024-03-02 11:22:22 WARNING juju-log No relation: certificates
2024-03-02T11:22:22.733Z [container-agent] 2024-03-02 11:22:22 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T11:26:25.534Z [container-agent] 2024-03-02 11:26:25 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T11:26:34.064Z [container-agent] 2024-03-02 11:26:34 WARNING juju-log No relation: certificates
2024-03-02T11:26:34.437Z [container-agent] 2024-03-02 11:26:34 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T11:30:56.676Z [container-agent] 2024-03-02 11:30:56 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T11:31:05.343Z [container-agent] 2024-03-02 11:31:05 WARNING juju-log No relation: certificates
2024-03-02T11:31:05.734Z [container-agent] 2024-03-02 11:31:05 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T11:33:15.919Z [container-agent] 2024-03-02 11:33:15 ERROR juju-log Failed to flush SLOW LOGS logs.
2024-03-02T11:33:15.919Z [container-agent] Traceback (most recent call last):
2024-03-02T11:33:15.919Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 666, in _run_mysqlsh_script
2024-03-02T11:33:15.919Z [container-agent]     stdout, _ = process.wait_output()
2024-03-02T11:33:15.919Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1441, in wait_output
2024-03-02T11:33:15.919Z [container-agent]     raise ExecError[AnyStr](self._command, exit_code, out_value, err_value)
2024-03-02T11:33:15.919Z [container-agent] ops.pebble.ExecError: non-zero exit code 1 executing ['/usr/bin/mysqlsh', '--no-wizard', '--python', '--verbose=1', '-f', '/tmp/script.py', ';', 'rm', '/tmp/script.py'], stdout='', stderr='Cannot set LC_ALL to locale en_US.UTF-8: No such file or directory\nverbose: 2024-03-02T11:33:12Z: Loading startup files...\nverbose: 2024-03-02T11:33:12Z: Loading plugins...\nverbose: 2024-03-02T11:33:12Z: Connecting to MySQL at: serverconfig@horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local\nTraceback (most recent call last):\n  File "<string>", line 1, in <module>\nmysqlsh.DBError: MySQL Error (2013): Shell.connect: Lost connection to MySQL server at \'reading initial communication packet\', system error: 104\n'
2024-03-02T11:33:15.919Z [container-agent] 
2024-03-02T11:33:15.919Z [container-agent] During handling of the above exception, another exception occurred:
2024-03-02T11:33:15.919Z [container-agent] 
2024-03-02T11:33:15.919Z [container-agent] Traceback (most recent call last):
2024-03-02T11:33:15.919Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-03-02T11:33:15.919Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-03-02T11:33:15.919Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 669, in _run_mysqlsh_script
2024-03-02T11:33:15.919Z [container-agent]     raise MySQLClientError(e.stderr)
2024-03-02T11:33:15.919Z [container-agent] charms.mysql.v0.mysql.MySQLClientError: Cannot set LC_ALL to locale en_US.UTF-8: No such file or directory
2024-03-02T11:33:15.919Z [container-agent] verbose: 2024-03-02T11:33:12Z: Loading startup files...
2024-03-02T11:33:15.919Z [container-agent] verbose: 2024-03-02T11:33:12Z: Loading plugins...
2024-03-02T11:33:15.919Z [container-agent] verbose: 2024-03-02T11:33:12Z: Connecting to MySQL at: serverconfig@horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local
2024-03-02T11:33:15.919Z [container-agent] Traceback (most recent call last):
2024-03-02T11:33:15.919Z [container-agent]   File "<string>", line 1, in <module>
2024-03-02T11:33:15.919Z [container-agent] mysqlsh.DBError: MySQL Error (2013): Shell.connect: Lost connection to MySQL server at 'reading initial communication packet', system error: 104
2024-03-02T11:33:15.919Z [container-agent] 
2024-03-02T11:33:23.457Z [container-agent] 2024-03-02 11:33:23 INFO juju-log Setting up the logrotate configurations
2024-03-02T11:33:23.533Z [container-agent] 2024-03-02 11:33:23 INFO juju-log Adding pebble layer
2024-03-02T11:33:30.270Z [container-agent] 2024-03-02 11:33:30 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-03-02T11:33:30.306Z [container-agent] 2024-03-02 11:33:30 INFO juju-log Attempting reboot from complete outage.
2024-03-02T11:33:33.398Z [container-agent] 2024-03-02 11:33:33 INFO juju.worker.uniter.operation runhook.go:186 ran "mysql-pebble-ready" hook (via hook dispatching script: dispatch)
2024-03-02T11:35:07.625Z [container-agent] 2024-03-02 11:35:07 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T11:35:09.662Z [container-agent] 2024-03-02 11:35:09 WARNING juju-log No relation: certificates
2024-03-02T11:35:10.027Z [container-agent] 2024-03-02 11:35:10 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T11:39:14.874Z [container-agent] 2024-03-02 11:39:14 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T11:39:17.393Z [container-agent] 2024-03-02 11:39:17 WARNING juju-log No relation: certificates
2024-03-02T11:39:17.750Z [container-agent] 2024-03-02 11:39:17 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T11:43:16.057Z [container-agent] 2024-03-02 11:43:16 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T11:43:19.130Z [container-agent] 2024-03-02 11:43:19 WARNING juju-log No relation: certificates
2024-03-02T11:43:19.485Z [container-agent] 2024-03-02 11:43:19 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T11:48:38.688Z [container-agent] 2024-03-02 11:48:38 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T11:48:41.930Z [container-agent] 2024-03-02 11:48:41 WARNING juju-log No relation: certificates
2024-03-02T11:48:42.292Z [container-agent] 2024-03-02 11:48:42 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T11:52:46.691Z [container-agent] 2024-03-02 11:52:46 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T11:52:50.068Z [container-agent] 2024-03-02 11:52:50 WARNING juju-log No relation: certificates
2024-03-02T11:52:50.413Z [container-agent] 2024-03-02 11:52:50 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T11:57:34.371Z [container-agent] 2024-03-02 11:57:34 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T11:57:38.644Z [container-agent] 2024-03-02 11:57:38 WARNING juju-log No relation: certificates
2024-03-02T11:57:38.983Z [container-agent] 2024-03-02 11:57:38 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T12:01:55.815Z [container-agent] 2024-03-02 12:01:55 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T12:02:00.451Z [container-agent] 2024-03-02 12:02:00 WARNING juju-log No relation: certificates
2024-03-02T12:02:00.831Z [container-agent] 2024-03-02 12:02:00 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T12:06:51.101Z [container-agent] 2024-03-02 12:06:51 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T12:06:56.270Z [container-agent] 2024-03-02 12:06:56 WARNING juju-log No relation: certificates
2024-03-02T12:06:56.609Z [container-agent] 2024-03-02 12:06:56 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T12:11:31.966Z [container-agent] 2024-03-02 12:11:31 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T12:11:37.475Z [container-agent] 2024-03-02 12:11:37 WARNING juju-log No relation: certificates
2024-03-02T12:11:37.833Z [container-agent] 2024-03-02 12:11:37 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T12:17:10.823Z [container-agent] 2024-03-02 12:17:10 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T12:17:17.220Z [container-agent] 2024-03-02 12:17:17 WARNING juju-log No relation: certificates
2024-03-02T12:17:17.569Z [container-agent] 2024-03-02 12:17:17 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T12:21:36.596Z [container-agent] 2024-03-02 12:21:36 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T12:21:43.113Z [container-agent] 2024-03-02 12:21:43 WARNING juju-log No relation: certificates
2024-03-02T12:21:43.465Z [container-agent] 2024-03-02 12:21:43 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T12:26:52.080Z [container-agent] 2024-03-02 12:26:52 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T12:26:59.062Z [container-agent] 2024-03-02 12:26:59 WARNING juju-log No relation: certificates
2024-03-02T12:26:59.427Z [container-agent] 2024-03-02 12:26:59 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T12:32:12.869Z [container-agent] 2024-03-02 12:32:12 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T12:32:20.560Z [container-agent] 2024-03-02 12:32:20 WARNING juju-log No relation: certificates
2024-03-02T12:32:20.938Z [container-agent] 2024-03-02 12:32:20 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T12:36:15.005Z [container-agent] 2024-03-02 12:36:15 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T12:36:22.918Z [container-agent] 2024-03-02 12:36:22 WARNING juju-log No relation: certificates
2024-03-02T12:36:23.290Z [container-agent] 2024-03-02 12:36:23 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T12:40:17.265Z [container-agent] 2024-03-02 12:40:17 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T12:40:25.391Z [container-agent] 2024-03-02 12:40:25 WARNING juju-log No relation: certificates
2024-03-02T12:40:25.733Z [container-agent] 2024-03-02 12:40:25 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T12:44:45.961Z [container-agent] 2024-03-02 12:44:45 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T12:44:54.342Z [container-agent] 2024-03-02 12:44:54 WARNING juju-log No relation: certificates
2024-03-02T12:44:54.693Z [container-agent] 2024-03-02 12:44:54 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T12:49:03.697Z [container-agent] 2024-03-02 12:49:03 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T12:49:12.306Z [container-agent] 2024-03-02 12:49:12 WARNING juju-log No relation: certificates
2024-03-02T12:49:12.654Z [container-agent] 2024-03-02 12:49:12 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T12:54:43.127Z [container-agent] 2024-03-02 12:54:43 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-03-02T12:54:43.160Z [container-agent] 2024-03-02 12:54:43 INFO juju-log Attempting reboot from complete outage.
2024-03-02T12:54:51.826Z [container-agent] 2024-03-02 12:54:51 WARNING juju-log No relation: certificates
2024-03-02T12:54:52.206Z [container-agent] 2024-03-02 12:54:52 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T12:57:09.809Z [container-agent] 2024-03-02 12:57:09 ERROR juju-log Uncaught exception while in charm code:
2024-03-02T12:57:09.809Z [container-agent] Traceback (most recent call last):
2024-03-02T12:57:09.809Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/./src/charm.py", line 786, in <module>
2024-03-02T12:57:09.809Z [container-agent]     main(MySQLOperatorCharm)
2024-03-02T12:57:09.809Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 436, in main
2024-03-02T12:57:09.809Z [container-agent]     _emit_charm_event(charm, dispatcher.event_name)
2024-03-02T12:57:09.809Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 144, in _emit_charm_event
2024-03-02T12:57:09.809Z [container-agent]     event_to_emit.emit(*args, **kwargs)
2024-03-02T12:57:09.809Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 351, in emit
2024-03-02T12:57:09.809Z [container-agent]     framework._emit(event)
2024-03-02T12:57:09.809Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 853, in _emit
2024-03-02T12:57:09.809Z [container-agent]     self._reemit(event_path)
2024-03-02T12:57:09.809Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 942, in _reemit
2024-03-02T12:57:09.810Z [container-agent]     custom_handler(event)
2024-03-02T12:57:09.810Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/rotate_mysql_logs.py", line 57, in _rotate_mysql_logs
2024-03-02T12:57:09.810Z [container-agent]     self.charm._mysql.flush_mysql_logs(MySQLTextLogs.SLOW)
2024-03-02T12:57:09.810Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-03-02T12:57:09.810Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-03-02T12:57:09.810Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 665, in _run_mysqlsh_script
2024-03-02T12:57:09.810Z [container-agent]     process = self.container.exec(cmd, timeout=timeout)
2024-03-02T12:57:09.810Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/model.py", line 2688, in exec
2024-03-02T12:57:09.810Z [container-agent]     return self._pebble.exec(
2024-03-02T12:57:09.810Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 2501, in exec
2024-03-02T12:57:09.810Z [container-agent]     resp = self._request('POST', '/v1/exec', body=body)
2024-03-02T12:57:09.810Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1655, in _request
2024-03-02T12:57:09.810Z [container-agent]     response = self._request_raw(method, path, query, headers, data)
2024-03-02T12:57:09.810Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1690, in _request_raw
2024-03-02T12:57:09.810Z [container-agent]     response = self.opener.open(request, timeout=self.timeout)
2024-03-02T12:57:09.810Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 519, in open
2024-03-02T12:57:09.810Z [container-agent]     response = self._open(req, data)
2024-03-02T12:57:09.810Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 536, in _open
2024-03-02T12:57:09.810Z [container-agent]     result = self._call_chain(self.handle_open, protocol, protocol +
2024-03-02T12:57:09.810Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 496, in _call_chain
2024-03-02T12:57:09.810Z [container-agent]     result = func(*args)
2024-03-02T12:57:09.810Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 311, in http_open
2024-03-02T12:57:09.810Z [container-agent]     return self.do_open(_UnixSocketConnection, req,  # type:ignore
2024-03-02T12:57:09.810Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 1352, in do_open
2024-03-02T12:57:09.810Z [container-agent]     r = h.getresponse()
2024-03-02T12:57:09.810Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1375, in getresponse
2024-03-02T12:57:09.810Z [container-agent]     response.begin()
2024-03-02T12:57:09.810Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 318, in begin
2024-03-02T12:57:09.810Z [container-agent]     version, status, reason = self._read_status()
2024-03-02T12:57:09.810Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 287, in _read_status
2024-03-02T12:57:09.810Z [container-agent]     raise RemoteDisconnected("Remote end closed connection without"
2024-03-02T12:57:09.810Z [container-agent] http.client.RemoteDisconnected: Remote end closed connection without response
2024-03-02T12:57:09.938Z [container-agent] 2024-03-02 12:57:09 INFO juju.util.exec exec.go:209 run result: exit status 1
2024-03-02T12:57:11.988Z [container-agent] 2024-03-02 12:57:11 INFO juju-log Setting up the logrotate configurations
2024-03-02T12:57:12.058Z [container-agent] 2024-03-02 12:57:12 INFO juju-log Adding pebble layer
2024-03-02T12:57:18.841Z [container-agent] 2024-03-02 12:57:18 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-03-02T12:57:18.867Z [container-agent] 2024-03-02 12:57:18 INFO juju-log Attempting reboot from complete outage.
2024-03-02T12:57:21.824Z [container-agent] 2024-03-02 12:57:21 INFO juju.worker.uniter.operation runhook.go:186 ran "mysql-pebble-ready" hook (via hook dispatching script: dispatch)
2024-03-02T13:00:04.019Z [container-agent] 2024-03-02 13:00:04 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T13:00:06.066Z [container-agent] 2024-03-02 13:00:06 WARNING juju-log No relation: certificates
2024-03-02T13:00:06.434Z [container-agent] 2024-03-02 13:00:06 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T13:04:47.046Z [container-agent] 2024-03-02 13:04:47 INFO juju-log Starting the log rotate manager
2024-03-02T13:04:47.061Z [container-agent] 2024-03-02 13:04:47 INFO juju-log Started log rotate manager process with PID 557826
2024-03-02T13:04:47.346Z [container-agent] 2024-03-02 13:04:47 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T13:04:49.484Z [container-agent] 2024-03-02 13:04:49 WARNING juju-log No relation: certificates
2024-03-02T13:04:49.859Z [container-agent] 2024-03-02 13:04:49 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T13:10:29.666Z [container-agent] 2024-03-02 13:10:29 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T13:10:32.130Z [container-agent] 2024-03-02 13:10:32 WARNING juju-log No relation: certificates
2024-03-02T13:10:32.474Z [container-agent] 2024-03-02 13:10:32 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T13:16:10.210Z [container-agent] 2024-03-02 13:16:10 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T13:16:13.248Z [container-agent] 2024-03-02 13:16:13 WARNING juju-log No relation: certificates
2024-03-02T13:16:13.579Z [container-agent] 2024-03-02 13:16:13 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T13:22:07.611Z [container-agent] 2024-03-02 13:22:07 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T13:22:10.918Z [container-agent] 2024-03-02 13:22:10 WARNING juju-log No relation: certificates
2024-03-02T13:22:11.262Z [container-agent] 2024-03-02 13:22:11 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T13:27:47.514Z [container-agent] 2024-03-02 13:27:47 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T13:27:51.802Z [container-agent] 2024-03-02 13:27:51 WARNING juju-log No relation: certificates
2024-03-02T13:27:52.138Z [container-agent] 2024-03-02 13:27:52 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T13:32:26.987Z [container-agent] 2024-03-02 13:32:26 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T13:32:31.700Z [container-agent] 2024-03-02 13:32:31 WARNING juju-log No relation: certificates
2024-03-02T13:32:32.042Z [container-agent] 2024-03-02 13:32:32 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T13:38:23.709Z [container-agent] 2024-03-02 13:38:23 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T13:38:28.911Z [container-agent] 2024-03-02 13:38:28 WARNING juju-log No relation: certificates
2024-03-02T13:38:29.267Z [container-agent] 2024-03-02 13:38:29 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T13:42:36.026Z [container-agent] 2024-03-02 13:42:36 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T13:42:41.440Z [container-agent] 2024-03-02 13:42:41 WARNING juju-log No relation: certificates
2024-03-02T13:42:41.809Z [container-agent] 2024-03-02 13:42:41 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T13:48:34.289Z [container-agent] 2024-03-02 13:48:34 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T13:48:40.472Z [container-agent] 2024-03-02 13:48:40 WARNING juju-log No relation: certificates
2024-03-02T13:48:40.836Z [container-agent] 2024-03-02 13:48:40 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T13:53:44.244Z [container-agent] 2024-03-02 13:53:44 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T13:53:50.872Z [container-agent] 2024-03-02 13:53:50 WARNING juju-log No relation: certificates
2024-03-02T13:53:51.242Z [container-agent] 2024-03-02 13:53:51 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T13:58:16.701Z [container-agent] 2024-03-02 13:58:16 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T13:58:23.928Z [container-agent] 2024-03-02 13:58:23 WARNING juju-log No relation: certificates
2024-03-02T13:58:24.285Z [container-agent] 2024-03-02 13:58:24 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T14:02:20.658Z [container-agent] 2024-03-02 14:02:20 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T14:02:28.114Z [container-agent] 2024-03-02 14:02:28 WARNING juju-log No relation: certificates
2024-03-02T14:02:28.475Z [container-agent] 2024-03-02 14:02:28 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T14:08:19.629Z [container-agent] 2024-03-02 14:08:19 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T14:08:27.616Z [container-agent] 2024-03-02 14:08:27 WARNING juju-log No relation: certificates
2024-03-02T14:08:27.950Z [container-agent] 2024-03-02 14:08:27 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T14:12:24.571Z [container-agent] 2024-03-02 14:12:24 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T14:12:34.408Z [container-agent] 2024-03-02 14:12:34 ERROR juju-log Failed to get cluster status for cluster-7573647c1e26466beeeac32b95ab8f91
2024-03-02T14:12:34.488Z [container-agent] 2024-03-02 14:12:34 INFO juju-log Kubernetes pod label error created
2024-03-02T14:12:34.501Z [container-agent] 2024-03-02 14:12:34 WARNING juju-log No relation: certificates
2024-03-02T14:12:34.873Z [container-agent] 2024-03-02 14:12:34 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T14:17:14.911Z [container-agent] 2024-03-02 14:17:14 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-03-02T14:17:14.939Z [container-agent] 2024-03-02 14:17:14 INFO juju-log Attempting reboot from complete outage.
2024-03-02T14:17:22.824Z [container-agent] 2024-03-02 14:17:22 INFO juju-log Kubernetes pod label primary created
2024-03-02T14:17:22.839Z [container-agent] 2024-03-02 14:17:22 WARNING juju-log No relation: certificates
2024-03-02T14:17:23.199Z [container-agent] 2024-03-02 14:17:23 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T14:17:35.229Z [container-agent] 2024-03-02 14:17:35 INFO juju-log Setting up the logrotate configurations
2024-03-02T14:17:35.314Z [container-agent] 2024-03-02 14:17:35 INFO juju-log Adding pebble layer
2024-03-02T14:17:42.042Z [container-agent] 2024-03-02 14:17:42 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-03-02T14:17:42.063Z [container-agent] 2024-03-02 14:17:42 INFO juju-log Attempting reboot from complete outage.
2024-03-02T14:17:45.047Z [container-agent] 2024-03-02 14:17:45 INFO juju.worker.uniter.operation runhook.go:186 ran "mysql-pebble-ready" hook (via hook dispatching script: dispatch)
2024-03-02T14:21:36.495Z [container-agent] 2024-03-02 14:21:36 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T14:21:38.649Z [container-agent] 2024-03-02 14:21:38 WARNING juju-log No relation: certificates
2024-03-02T14:21:39.006Z [container-agent] 2024-03-02 14:21:39 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T14:27:16.101Z [container-agent] 2024-03-02 14:27:16 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T14:27:18.821Z [container-agent] 2024-03-02 14:27:18 WARNING juju-log No relation: certificates
2024-03-02T14:27:19.176Z [container-agent] 2024-03-02 14:27:19 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T14:32:16.810Z [container-agent] 2024-03-02 14:32:16 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T14:32:20.308Z [container-agent] 2024-03-02 14:32:20 WARNING juju-log No relation: certificates
2024-03-02T14:32:20.663Z [container-agent] 2024-03-02 14:32:20 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T14:37:51.661Z [container-agent] 2024-03-02 14:37:51 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T14:37:55.242Z [container-agent] 2024-03-02 14:37:55 WARNING juju-log No relation: certificates
2024-03-02T14:37:55.590Z [container-agent] 2024-03-02 14:37:55 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T14:42:08.904Z [container-agent] 2024-03-02 14:42:08 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T14:42:12.692Z [container-agent] 2024-03-02 14:42:12 WARNING juju-log No relation: certificates
2024-03-02T14:42:13.081Z [container-agent] 2024-03-02 14:42:13 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T14:46:21.722Z [container-agent] 2024-03-02 14:46:21 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T14:46:25.844Z [container-agent] 2024-03-02 14:46:25 WARNING juju-log No relation: certificates
2024-03-02T14:46:26.183Z [container-agent] 2024-03-02 14:46:26 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T14:51:21.839Z [container-agent] 2024-03-02 14:51:21 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T14:51:26.765Z [container-agent] 2024-03-02 14:51:26 WARNING juju-log No relation: certificates
2024-03-02T14:51:27.113Z [container-agent] 2024-03-02 14:51:27 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T14:57:10.161Z [container-agent] 2024-03-02 14:57:10 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T14:57:15.306Z [container-agent] 2024-03-02 14:57:15 WARNING juju-log No relation: certificates
2024-03-02T14:57:15.651Z [container-agent] 2024-03-02 14:57:15 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T15:02:48.714Z [container-agent] 2024-03-02 15:02:48 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T15:02:54.698Z [container-agent] 2024-03-02 15:02:54 WARNING juju-log No relation: certificates
2024-03-02T15:02:55.049Z [container-agent] 2024-03-02 15:02:55 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T15:07:53.070Z [container-agent] 2024-03-02 15:07:53 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T15:07:59.197Z [container-agent] 2024-03-02 15:07:59 WARNING juju-log No relation: certificates
2024-03-02T15:07:59.537Z [container-agent] 2024-03-02 15:07:59 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T15:12:00.271Z [container-agent] 2024-03-02 15:12:00 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T15:12:06.617Z [container-agent] 2024-03-02 15:12:06 WARNING juju-log No relation: certificates
2024-03-02T15:12:06.976Z [container-agent] 2024-03-02 15:12:06 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T15:17:21.750Z [container-agent] 2024-03-02 15:17:21 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T15:17:28.414Z [container-agent] 2024-03-02 15:17:28 WARNING juju-log No relation: certificates
2024-03-02T15:17:28.802Z [container-agent] 2024-03-02 15:17:28 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T15:22:28.122Z [container-agent] 2024-03-02 15:22:28 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-03-02T15:22:28.148Z [container-agent] 2024-03-02 15:22:28 INFO juju-log Attempting reboot from complete outage.
2024-03-02T15:22:35.222Z [container-agent] 2024-03-02 15:22:35 WARNING juju-log No relation: certificates
2024-03-02T15:22:35.568Z [container-agent] 2024-03-02 15:22:35 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T15:26:04.784Z [container-agent] 2024-03-02 15:26:04 ERROR juju-log Uncaught exception while in charm code:
2024-03-02T15:26:04.784Z [container-agent] Traceback (most recent call last):
2024-03-02T15:26:04.784Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 2507, in exec
2024-03-02T15:26:04.784Z [container-agent]     control_ws = self._connect_websocket(task_id, 'control')
2024-03-02T15:26:04.784Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 2589, in _connect_websocket
2024-03-02T15:26:04.784Z [container-agent]     ws.connect(url, socket=sock)
2024-03-02T15:26:04.784Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_core.py", line 253, in connect
2024-03-02T15:26:04.784Z [container-agent]     self.handshake_response = handshake(self.sock, url, *addrs, **options)
2024-03-02T15:26:04.784Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_handshake.py", line 57, in handshake
2024-03-02T15:26:04.784Z [container-agent]     status, resp = _get_resp_headers(sock)
2024-03-02T15:26:04.784Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_handshake.py", line 143, in _get_resp_headers
2024-03-02T15:26:04.784Z [container-agent]     status, resp_headers, status_message = read_headers(sock)
2024-03-02T15:26:04.784Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_http.py", line 312, in read_headers
2024-03-02T15:26:04.784Z [container-agent]     line = recv_line(sock)
2024-03-02T15:26:04.784Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_socket.py", line 131, in recv_line
2024-03-02T15:26:04.784Z [container-agent]     c = recv(sock, 1)
2024-03-02T15:26:04.784Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_socket.py", line 122, in recv
2024-03-02T15:26:04.784Z [container-agent]     raise WebSocketConnectionClosedException(
2024-03-02T15:26:04.784Z [container-agent] websocket._exceptions.WebSocketConnectionClosedException: Connection to remote host was lost.
2024-03-02T15:26:04.784Z [container-agent] 
2024-03-02T15:26:04.784Z [container-agent] During handling of the above exception, another exception occurred:
2024-03-02T15:26:04.784Z [container-agent] 
2024-03-02T15:26:04.784Z [container-agent] Traceback (most recent call last):
2024-03-02T15:26:04.784Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 1348, in do_open
2024-03-02T15:26:04.784Z [container-agent]     h.request(req.get_method(), req.selector, req.data, headers,
2024-03-02T15:26:04.784Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1283, in request
2024-03-02T15:26:04.784Z [container-agent]     self._send_request(method, url, body, headers, encode_chunked)
2024-03-02T15:26:04.784Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1329, in _send_request
2024-03-02T15:26:04.784Z [container-agent]     self.endheaders(body, encode_chunked=encode_chunked)
2024-03-02T15:26:04.784Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1278, in endheaders
2024-03-02T15:26:04.784Z [container-agent]     self._send_output(message_body, encode_chunked=encode_chunked)
2024-03-02T15:26:04.784Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1038, in _send_output
2024-03-02T15:26:04.784Z [container-agent]     self.send(msg)
2024-03-02T15:26:04.784Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 976, in send
2024-03-02T15:26:04.784Z [container-agent]     self.connect()
2024-03-02T15:26:04.784Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 297, in connect
2024-03-02T15:26:04.784Z [container-agent]     self.sock.connect(self.socket_path)
2024-03-02T15:26:04.784Z [container-agent] ConnectionRefusedError: [Errno 111] Connection refused
2024-03-02T15:26:04.784Z [container-agent] 
2024-03-02T15:26:04.784Z [container-agent] During handling of the above exception, another exception occurred:
2024-03-02T15:26:04.784Z [container-agent] 
2024-03-02T15:26:04.784Z [container-agent] Traceback (most recent call last):
2024-03-02T15:26:04.784Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1690, in _request_raw
2024-03-02T15:26:04.784Z [container-agent]     response = self.opener.open(request, timeout=self.timeout)
2024-03-02T15:26:04.784Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 519, in open
2024-03-02T15:26:04.784Z [container-agent]     response = self._open(req, data)
2024-03-02T15:26:04.784Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 536, in _open
2024-03-02T15:26:04.784Z [container-agent]     result = self._call_chain(self.handle_open, protocol, protocol +
2024-03-02T15:26:04.784Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 496, in _call_chain
2024-03-02T15:26:04.784Z [container-agent]     result = func(*args)
2024-03-02T15:26:04.784Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 311, in http_open
2024-03-02T15:26:04.784Z [container-agent]     return self.do_open(_UnixSocketConnection, req,  # type:ignore
2024-03-02T15:26:04.784Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 1351, in do_open
2024-03-02T15:26:04.784Z [container-agent]     raise URLError(err)
2024-03-02T15:26:04.784Z [container-agent] urllib.error.URLError: <urlopen error [Errno 111] Connection refused>
2024-03-02T15:26:04.784Z [container-agent] 
2024-03-02T15:26:04.784Z [container-agent] The above exception was the direct cause of the following exception:
2024-03-02T15:26:04.784Z [container-agent] 
2024-03-02T15:26:04.784Z [container-agent] Traceback (most recent call last):
2024-03-02T15:26:04.784Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/./src/charm.py", line 786, in <module>
2024-03-02T15:26:04.784Z [container-agent]     main(MySQLOperatorCharm)
2024-03-02T15:26:04.784Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 436, in main
2024-03-02T15:26:04.784Z [container-agent]     _emit_charm_event(charm, dispatcher.event_name)
2024-03-02T15:26:04.784Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 144, in _emit_charm_event
2024-03-02T15:26:04.784Z [container-agent]     event_to_emit.emit(*args, **kwargs)
2024-03-02T15:26:04.784Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 351, in emit
2024-03-02T15:26:04.784Z [container-agent]     framework._emit(event)
2024-03-02T15:26:04.784Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 853, in _emit
2024-03-02T15:26:04.784Z [container-agent]     self._reemit(event_path)
2024-03-02T15:26:04.784Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 942, in _reemit
2024-03-02T15:26:04.784Z [container-agent]     custom_handler(event)
2024-03-02T15:26:04.784Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/rotate_mysql_logs.py", line 55, in _rotate_mysql_logs
2024-03-02T15:26:04.784Z [container-agent]     self.charm._mysql.flush_mysql_logs(MySQLTextLogs.ERROR)
2024-03-02T15:26:04.784Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-03-02T15:26:04.784Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-03-02T15:26:04.784Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 665, in _run_mysqlsh_script
2024-03-02T15:26:04.784Z [container-agent]     process = self.container.exec(cmd, timeout=timeout)
2024-03-02T15:26:04.784Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/model.py", line 2688, in exec
2024-03-02T15:26:04.784Z [container-agent]     return self._pebble.exec(
2024-03-02T15:26:04.784Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 2514, in exec
2024-03-02T15:26:04.784Z [container-agent]     change = self.wait_change(ChangeID(change_id))
2024-03-02T15:26:04.784Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1893, in wait_change
2024-03-02T15:26:04.784Z [container-agent]     return self._wait_change_using_wait(change_id, timeout)
2024-03-02T15:26:04.784Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1914, in _wait_change_using_wait
2024-03-02T15:26:04.784Z [container-agent]     return self._wait_change(change_id, this_timeout)
2024-03-02T15:26:04.784Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1928, in _wait_change
2024-03-02T15:26:04.784Z [container-agent]     resp = self._request('GET', f'/v1/changes/{change_id}/wait', query)
2024-03-02T15:26:04.784Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1655, in _request
2024-03-02T15:26:04.784Z [container-agent]     response = self._request_raw(method, path, query, headers, data)
2024-03-02T15:26:04.784Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1707, in _request_raw
2024-03-02T15:26:04.784Z [container-agent]     raise ConnectionError(e.reason) from e
2024-03-02T15:26:04.784Z [container-agent] ops.pebble.ConnectionError: [Errno 111] Connection refused
2024-03-02T15:26:04.955Z [container-agent] 2024-03-02 15:26:04 INFO juju.util.exec exec.go:209 run result: exit status 1
2024-03-02T15:26:07.900Z [container-agent] 2024-03-02 15:26:07 INFO juju-log Setting up the logrotate configurations
2024-03-02T15:26:07.966Z [container-agent] 2024-03-02 15:26:07 INFO juju-log Adding pebble layer
2024-03-02T15:26:14.726Z [container-agent] 2024-03-02 15:26:14 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-03-02T15:26:14.752Z [container-agent] 2024-03-02 15:26:14 INFO juju-log Attempting reboot from complete outage.
2024-03-02T15:26:17.849Z [container-agent] 2024-03-02 15:26:17 INFO juju.worker.uniter.operation runhook.go:186 ran "mysql-pebble-ready" hook (via hook dispatching script: dispatch)
2024-03-02T15:26:51.944Z [container-agent] 2024-03-02 15:26:51 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T15:26:53.768Z [container-agent] 2024-03-02 15:26:53 WARNING juju-log No relation: certificates
2024-03-02T15:26:54.109Z [container-agent] 2024-03-02 15:26:54 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T15:31:45.612Z [container-agent] 2024-03-02 15:31:45 INFO juju-log Starting the log rotate manager
2024-03-02T15:31:45.626Z [container-agent] 2024-03-02 15:31:45 INFO juju-log Started log rotate manager process with PID 571971
2024-03-02T15:31:45.901Z [container-agent] 2024-03-02 15:31:45 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T15:31:47.949Z [container-agent] 2024-03-02 15:31:47 WARNING juju-log No relation: certificates
2024-03-02T15:31:48.298Z [container-agent] 2024-03-02 15:31:48 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T15:36:55.510Z [container-agent] 2024-03-02 15:36:55 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T15:36:57.984Z [container-agent] 2024-03-02 15:36:57 WARNING juju-log No relation: certificates
2024-03-02T15:36:58.325Z [container-agent] 2024-03-02 15:36:58 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T15:41:18.139Z [container-agent] 2024-03-02 15:41:18 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T15:41:21.258Z [container-agent] 2024-03-02 15:41:21 WARNING juju-log No relation: certificates
2024-03-02T15:41:21.593Z [container-agent] 2024-03-02 15:41:21 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T15:46:15.623Z [container-agent] 2024-03-02 15:46:15 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T15:46:18.954Z [container-agent] 2024-03-02 15:46:18 WARNING juju-log No relation: certificates
2024-03-02T15:46:19.314Z [container-agent] 2024-03-02 15:46:19 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T15:50:36.889Z [container-agent] 2024-03-02 15:50:36 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T15:50:40.663Z [container-agent] 2024-03-02 15:50:40 WARNING juju-log No relation: certificates
2024-03-02T15:50:41.010Z [container-agent] 2024-03-02 15:50:41 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T15:54:40.584Z [container-agent] 2024-03-02 15:54:40 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T15:54:44.678Z [container-agent] 2024-03-02 15:54:44 WARNING juju-log No relation: certificates
2024-03-02T15:54:45.069Z [container-agent] 2024-03-02 15:54:45 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T15:59:57.430Z [container-agent] 2024-03-02 15:59:57 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T16:00:01.785Z [container-agent] 2024-03-02 16:00:01 WARNING juju-log No relation: certificates
2024-03-02T16:00:02.112Z [container-agent] 2024-03-02 16:00:02 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T16:05:22.574Z [container-agent] 2024-03-02 16:05:22 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T16:05:27.053Z [container-agent] 2024-03-02 16:05:27 WARNING juju-log No relation: certificates
2024-03-02T16:05:27.392Z [container-agent] 2024-03-02 16:05:27 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T16:10:27.099Z [container-agent] 2024-03-02 16:10:27 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T16:10:32.690Z [container-agent] 2024-03-02 16:10:32 WARNING juju-log No relation: certificates
2024-03-02T16:10:33.042Z [container-agent] 2024-03-02 16:10:33 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T16:16:15.037Z [container-agent] 2024-03-02 16:16:15 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T16:16:21.330Z [container-agent] 2024-03-02 16:16:21 WARNING juju-log No relation: certificates
2024-03-02T16:16:21.679Z [container-agent] 2024-03-02 16:16:21 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T16:20:52.459Z [container-agent] 2024-03-02 16:20:52 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T16:20:58.718Z [container-agent] 2024-03-02 16:20:58 WARNING juju-log No relation: certificates
2024-03-02T16:20:59.081Z [container-agent] 2024-03-02 16:20:59 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T16:24:53.391Z [container-agent] 2024-03-02 16:24:53 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T16:24:59.666Z [container-agent] 2024-03-02 16:24:59 WARNING juju-log No relation: certificates
2024-03-02T16:25:00.043Z [container-agent] 2024-03-02 16:25:00 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T16:29:29.365Z [container-agent] 2024-03-02 16:29:29 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T16:29:36.935Z [container-agent] 2024-03-02 16:29:36 WARNING juju-log No relation: certificates
2024-03-02T16:29:37.296Z [container-agent] 2024-03-02 16:29:37 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T16:33:53.300Z [container-agent] 2024-03-02 16:33:53 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T16:34:00.539Z [container-agent] 2024-03-02 16:34:00 WARNING juju-log No relation: certificates
2024-03-02T16:34:00.923Z [container-agent] 2024-03-02 16:34:00 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T16:38:51.622Z [container-agent] 2024-03-02 16:38:51 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T16:38:59.144Z [container-agent] 2024-03-02 16:38:59 WARNING juju-log No relation: certificates
2024-03-02T16:38:59.480Z [container-agent] 2024-03-02 16:38:59 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T16:43:40.465Z [container-agent] 2024-03-02 16:43:40 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T16:43:48.862Z [container-agent] 2024-03-02 16:43:48 WARNING juju-log No relation: certificates
2024-03-02T16:43:49.207Z [container-agent] 2024-03-02 16:43:49 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T16:49:37.034Z [container-agent] 2024-03-02 16:49:37 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-03-02T16:49:37.059Z [container-agent] 2024-03-02 16:49:37 INFO juju-log Attempting reboot from complete outage.
2024-03-02T16:49:45.026Z [container-agent] 2024-03-02 16:49:45 WARNING juju-log No relation: certificates
2024-03-02T16:49:45.377Z [container-agent] 2024-03-02 16:49:45 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T16:50:04.753Z [container-agent] 2024-03-02 16:50:04 ERROR juju-log Uncaught exception while in charm code:
2024-03-02T16:50:04.753Z [container-agent] Traceback (most recent call last):
2024-03-02T16:50:04.753Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 2507, in exec
2024-03-02T16:50:04.753Z [container-agent]     control_ws = self._connect_websocket(task_id, 'control')
2024-03-02T16:50:04.753Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 2589, in _connect_websocket
2024-03-02T16:50:04.753Z [container-agent]     ws.connect(url, socket=sock)
2024-03-02T16:50:04.753Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_core.py", line 253, in connect
2024-03-02T16:50:04.753Z [container-agent]     self.handshake_response = handshake(self.sock, url, *addrs, **options)
2024-03-02T16:50:04.753Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_handshake.py", line 57, in handshake
2024-03-02T16:50:04.753Z [container-agent]     status, resp = _get_resp_headers(sock)
2024-03-02T16:50:04.753Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_handshake.py", line 143, in _get_resp_headers
2024-03-02T16:50:04.753Z [container-agent]     status, resp_headers, status_message = read_headers(sock)
2024-03-02T16:50:04.753Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_http.py", line 312, in read_headers
2024-03-02T16:50:04.753Z [container-agent]     line = recv_line(sock)
2024-03-02T16:50:04.753Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_socket.py", line 131, in recv_line
2024-03-02T16:50:04.753Z [container-agent]     c = recv(sock, 1)
2024-03-02T16:50:04.753Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_socket.py", line 122, in recv
2024-03-02T16:50:04.753Z [container-agent]     raise WebSocketConnectionClosedException(
2024-03-02T16:50:04.753Z [container-agent] websocket._exceptions.WebSocketConnectionClosedException: Connection to remote host was lost.
2024-03-02T16:50:04.753Z [container-agent] 
2024-03-02T16:50:04.753Z [container-agent] During handling of the above exception, another exception occurred:
2024-03-02T16:50:04.753Z [container-agent] 
2024-03-02T16:50:04.753Z [container-agent] Traceback (most recent call last):
2024-03-02T16:50:04.753Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 1348, in do_open
2024-03-02T16:50:04.753Z [container-agent]     h.request(req.get_method(), req.selector, req.data, headers,
2024-03-02T16:50:04.753Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1283, in request
2024-03-02T16:50:04.753Z [container-agent]     self._send_request(method, url, body, headers, encode_chunked)
2024-03-02T16:50:04.753Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1329, in _send_request
2024-03-02T16:50:04.753Z [container-agent]     self.endheaders(body, encode_chunked=encode_chunked)
2024-03-02T16:50:04.753Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1278, in endheaders
2024-03-02T16:50:04.753Z [container-agent]     self._send_output(message_body, encode_chunked=encode_chunked)
2024-03-02T16:50:04.753Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1038, in _send_output
2024-03-02T16:50:04.753Z [container-agent]     self.send(msg)
2024-03-02T16:50:04.753Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 976, in send
2024-03-02T16:50:04.753Z [container-agent]     self.connect()
2024-03-02T16:50:04.753Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 297, in connect
2024-03-02T16:50:04.753Z [container-agent]     self.sock.connect(self.socket_path)
2024-03-02T16:50:04.753Z [container-agent] ConnectionRefusedError: [Errno 111] Connection refused
2024-03-02T16:50:04.753Z [container-agent] 
2024-03-02T16:50:04.753Z [container-agent] During handling of the above exception, another exception occurred:
2024-03-02T16:50:04.753Z [container-agent] 
2024-03-02T16:50:04.753Z [container-agent] Traceback (most recent call last):
2024-03-02T16:50:04.753Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1690, in _request_raw
2024-03-02T16:50:04.753Z [container-agent]     response = self.opener.open(request, timeout=self.timeout)
2024-03-02T16:50:04.753Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 519, in open
2024-03-02T16:50:04.753Z [container-agent]     response = self._open(req, data)
2024-03-02T16:50:04.753Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 536, in _open
2024-03-02T16:50:04.753Z [container-agent]     result = self._call_chain(self.handle_open, protocol, protocol +
2024-03-02T16:50:04.753Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 496, in _call_chain
2024-03-02T16:50:04.753Z [container-agent]     result = func(*args)
2024-03-02T16:50:04.753Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 311, in http_open
2024-03-02T16:50:04.753Z [container-agent]     return self.do_open(_UnixSocketConnection, req,  # type:ignore
2024-03-02T16:50:04.753Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 1351, in do_open
2024-03-02T16:50:04.753Z [container-agent]     raise URLError(err)
2024-03-02T16:50:04.753Z [container-agent] urllib.error.URLError: <urlopen error [Errno 111] Connection refused>
2024-03-02T16:50:04.753Z [container-agent] 
2024-03-02T16:50:04.753Z [container-agent] The above exception was the direct cause of the following exception:
2024-03-02T16:50:04.753Z [container-agent] 
2024-03-02T16:50:04.753Z [container-agent] Traceback (most recent call last):
2024-03-02T16:50:04.753Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/./src/charm.py", line 786, in <module>
2024-03-02T16:50:04.753Z [container-agent]     main(MySQLOperatorCharm)
2024-03-02T16:50:04.753Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 436, in main
2024-03-02T16:50:04.753Z [container-agent]     _emit_charm_event(charm, dispatcher.event_name)
2024-03-02T16:50:04.753Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 144, in _emit_charm_event
2024-03-02T16:50:04.753Z [container-agent]     event_to_emit.emit(*args, **kwargs)
2024-03-02T16:50:04.753Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 351, in emit
2024-03-02T16:50:04.753Z [container-agent]     framework._emit(event)
2024-03-02T16:50:04.754Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 853, in _emit
2024-03-02T16:50:04.754Z [container-agent]     self._reemit(event_path)
2024-03-02T16:50:04.754Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 942, in _reemit
2024-03-02T16:50:04.754Z [container-agent]     custom_handler(event)
2024-03-02T16:50:04.754Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/rotate_mysql_logs.py", line 55, in _rotate_mysql_logs
2024-03-02T16:50:04.754Z [container-agent]     self.charm._mysql.flush_mysql_logs(MySQLTextLogs.ERROR)
2024-03-02T16:50:04.754Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-03-02T16:50:04.754Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-03-02T16:50:04.754Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 665, in _run_mysqlsh_script
2024-03-02T16:50:04.754Z [container-agent]     process = self.container.exec(cmd, timeout=timeout)
2024-03-02T16:50:04.754Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/model.py", line 2688, in exec
2024-03-02T16:50:04.754Z [container-agent]     return self._pebble.exec(
2024-03-02T16:50:04.754Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 2514, in exec
2024-03-02T16:50:04.754Z [container-agent]     change = self.wait_change(ChangeID(change_id))
2024-03-02T16:50:04.754Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1893, in wait_change
2024-03-02T16:50:04.754Z [container-agent]     return self._wait_change_using_wait(change_id, timeout)
2024-03-02T16:50:04.754Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1914, in _wait_change_using_wait
2024-03-02T16:50:04.754Z [container-agent]     return self._wait_change(change_id, this_timeout)
2024-03-02T16:50:04.754Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1928, in _wait_change
2024-03-02T16:50:04.754Z [container-agent]     resp = self._request('GET', f'/v1/changes/{change_id}/wait', query)
2024-03-02T16:50:04.754Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1655, in _request
2024-03-02T16:50:04.754Z [container-agent]     response = self._request_raw(method, path, query, headers, data)
2024-03-02T16:50:04.754Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1707, in _request_raw
2024-03-02T16:50:04.754Z [container-agent]     raise ConnectionError(e.reason) from e
2024-03-02T16:50:04.754Z [container-agent] ops.pebble.ConnectionError: [Errno 111] Connection refused
2024-03-02T16:50:04.933Z [container-agent] 2024-03-02 16:50:04 INFO juju.util.exec exec.go:209 run result: exit status 1
2024-03-02T16:50:06.978Z [container-agent] 2024-03-02 16:50:06 INFO juju-log Setting up the logrotate configurations
2024-03-02T16:50:07.050Z [container-agent] 2024-03-02 16:50:07 INFO juju-log Adding pebble layer
2024-03-02T16:50:13.834Z [container-agent] 2024-03-02 16:50:13 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-03-02T16:50:13.858Z [container-agent] 2024-03-02 16:50:13 INFO juju-log Attempting reboot from complete outage.
2024-03-02T16:50:16.912Z [container-agent] 2024-03-02 16:50:16 INFO juju.worker.uniter.operation runhook.go:186 ran "mysql-pebble-ready" hook (via hook dispatching script: dispatch)
2024-03-02T16:55:34.771Z [container-agent] 2024-03-02 16:55:34 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T16:55:36.706Z [container-agent] 2024-03-02 16:55:36 WARNING juju-log No relation: certificates
2024-03-02T16:55:37.051Z [container-agent] 2024-03-02 16:55:37 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T17:00:17.168Z [container-agent] 2024-03-02 17:00:17 INFO juju-log Starting the log rotate manager
2024-03-02T17:00:17.184Z [container-agent] 2024-03-02 17:00:17 INFO juju-log Started log rotate manager process with PID 580112
2024-03-02T17:00:17.471Z [container-agent] 2024-03-02 17:00:17 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T17:00:19.567Z [container-agent] 2024-03-02 17:00:19 WARNING juju-log No relation: certificates
2024-03-02T17:00:19.918Z [container-agent] 2024-03-02 17:00:19 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T17:04:26.095Z [container-agent] 2024-03-02 17:04:26 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T17:04:28.468Z [container-agent] 2024-03-02 17:04:28 WARNING juju-log No relation: certificates
2024-03-02T17:04:28.827Z [container-agent] 2024-03-02 17:04:28 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T17:08:53.142Z [container-agent] 2024-03-02 17:08:53 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T17:08:55.764Z [container-agent] 2024-03-02 17:08:55 WARNING juju-log No relation: certificates
2024-03-02T17:08:56.100Z [container-agent] 2024-03-02 17:08:56 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T17:14:44.921Z [container-agent] 2024-03-02 17:14:44 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T17:14:48.507Z [container-agent] 2024-03-02 17:14:48 WARNING juju-log No relation: certificates
2024-03-02T17:14:48.843Z [container-agent] 2024-03-02 17:14:48 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T17:20:23.927Z [container-agent] 2024-03-02 17:20:23 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T17:20:27.713Z [container-agent] 2024-03-02 17:20:27 WARNING juju-log No relation: certificates
2024-03-02T17:20:28.052Z [container-agent] 2024-03-02 17:20:28 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T17:26:21.737Z [container-agent] 2024-03-02 17:26:21 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T17:26:26.117Z [container-agent] 2024-03-02 17:26:26 WARNING juju-log No relation: certificates
2024-03-02T17:26:26.463Z [container-agent] 2024-03-02 17:26:26 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T17:30:55.643Z [container-agent] 2024-03-02 17:30:55 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T17:31:00.248Z [container-agent] 2024-03-02 17:31:00 WARNING juju-log No relation: certificates
2024-03-02T17:31:00.650Z [container-agent] 2024-03-02 17:31:00 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T17:36:40.738Z [container-agent] 2024-03-02 17:36:40 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T17:36:45.875Z [container-agent] 2024-03-02 17:36:45 WARNING juju-log No relation: certificates
2024-03-02T17:36:46.218Z [container-agent] 2024-03-02 17:36:46 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T17:40:50.829Z [container-agent] 2024-03-02 17:40:50 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T17:40:56.068Z [container-agent] 2024-03-02 17:40:56 WARNING juju-log No relation: certificates
2024-03-02T17:40:56.406Z [container-agent] 2024-03-02 17:40:56 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T17:45:11.502Z [container-agent] 2024-03-02 17:45:11 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T17:45:17.709Z [container-agent] 2024-03-02 17:45:17 WARNING juju-log No relation: certificates
2024-03-02T17:45:18.068Z [container-agent] 2024-03-02 17:45:18 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T17:50:19.174Z [container-agent] 2024-03-02 17:50:19 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T17:50:25.965Z [container-agent] 2024-03-02 17:50:25 WARNING juju-log No relation: certificates
2024-03-02T17:50:26.308Z [container-agent] 2024-03-02 17:50:26 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T17:55:18.906Z [container-agent] 2024-03-02 17:55:18 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T17:55:26.104Z [container-agent] 2024-03-02 17:55:26 WARNING juju-log No relation: certificates
2024-03-02T17:55:26.437Z [container-agent] 2024-03-02 17:55:26 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T17:59:34.501Z [container-agent] 2024-03-02 17:59:34 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T17:59:41.139Z [container-agent] 2024-03-02 17:59:41 WARNING juju-log No relation: certificates
2024-03-02T17:59:41.469Z [container-agent] 2024-03-02 17:59:41 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T18:04:40.840Z [container-agent] 2024-03-02 18:04:40 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T18:04:48.514Z [container-agent] 2024-03-02 18:04:48 WARNING juju-log No relation: certificates
2024-03-02T18:04:48.864Z [container-agent] 2024-03-02 18:04:48 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T18:09:14.445Z [container-agent] 2024-03-02 18:09:14 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T18:09:22.684Z [container-agent] 2024-03-02 18:09:22 WARNING juju-log No relation: certificates
2024-03-02T18:09:23.057Z [container-agent] 2024-03-02 18:09:23 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T18:13:52.052Z [container-agent] 2024-03-02 18:13:52 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-03-02T18:13:52.079Z [container-agent] 2024-03-02 18:13:52 INFO juju-log Attempting reboot from complete outage.
2024-03-02T18:13:59.824Z [container-agent] 2024-03-02 18:13:59 WARNING juju-log No relation: certificates
2024-03-02T18:14:00.176Z [container-agent] 2024-03-02 18:14:00 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T18:17:09.778Z [container-agent] 2024-03-02 18:17:09 ERROR juju-log Uncaught exception while in charm code:
2024-03-02T18:17:09.778Z [container-agent] Traceback (most recent call last):
2024-03-02T18:17:09.778Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/./src/charm.py", line 786, in <module>
2024-03-02T18:17:09.778Z [container-agent]     main(MySQLOperatorCharm)
2024-03-02T18:17:09.778Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 436, in main
2024-03-02T18:17:09.778Z [container-agent]     _emit_charm_event(charm, dispatcher.event_name)
2024-03-02T18:17:09.778Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 144, in _emit_charm_event
2024-03-02T18:17:09.778Z [container-agent]     event_to_emit.emit(*args, **kwargs)
2024-03-02T18:17:09.778Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 351, in emit
2024-03-02T18:17:09.778Z [container-agent]     framework._emit(event)
2024-03-02T18:17:09.778Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 853, in _emit
2024-03-02T18:17:09.778Z [container-agent]     self._reemit(event_path)
2024-03-02T18:17:09.778Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 942, in _reemit
2024-03-02T18:17:09.778Z [container-agent]     custom_handler(event)
2024-03-02T18:17:09.778Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/rotate_mysql_logs.py", line 57, in _rotate_mysql_logs
2024-03-02T18:17:09.778Z [container-agent]     self.charm._mysql.flush_mysql_logs(MySQLTextLogs.SLOW)
2024-03-02T18:17:09.778Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-03-02T18:17:09.778Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-03-02T18:17:09.778Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 665, in _run_mysqlsh_script
2024-03-02T18:17:09.778Z [container-agent]     process = self.container.exec(cmd, timeout=timeout)
2024-03-02T18:17:09.778Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/model.py", line 2688, in exec
2024-03-02T18:17:09.778Z [container-agent]     return self._pebble.exec(
2024-03-02T18:17:09.778Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 2501, in exec
2024-03-02T18:17:09.778Z [container-agent]     resp = self._request('POST', '/v1/exec', body=body)
2024-03-02T18:17:09.778Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1655, in _request
2024-03-02T18:17:09.778Z [container-agent]     response = self._request_raw(method, path, query, headers, data)
2024-03-02T18:17:09.778Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1690, in _request_raw
2024-03-02T18:17:09.778Z [container-agent]     response = self.opener.open(request, timeout=self.timeout)
2024-03-02T18:17:09.778Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 519, in open
2024-03-02T18:17:09.778Z [container-agent]     response = self._open(req, data)
2024-03-02T18:17:09.778Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 536, in _open
2024-03-02T18:17:09.778Z [container-agent]     result = self._call_chain(self.handle_open, protocol, protocol +
2024-03-02T18:17:09.778Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 496, in _call_chain
2024-03-02T18:17:09.778Z [container-agent]     result = func(*args)
2024-03-02T18:17:09.778Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 311, in http_open
2024-03-02T18:17:09.778Z [container-agent]     return self.do_open(_UnixSocketConnection, req,  # type:ignore
2024-03-02T18:17:09.778Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 1352, in do_open
2024-03-02T18:17:09.778Z [container-agent]     r = h.getresponse()
2024-03-02T18:17:09.778Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1375, in getresponse
2024-03-02T18:17:09.778Z [container-agent]     response.begin()
2024-03-02T18:17:09.778Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 318, in begin
2024-03-02T18:17:09.778Z [container-agent]     version, status, reason = self._read_status()
2024-03-02T18:17:09.778Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 287, in _read_status
2024-03-02T18:17:09.778Z [container-agent]     raise RemoteDisconnected("Remote end closed connection without"
2024-03-02T18:17:09.778Z [container-agent] http.client.RemoteDisconnected: Remote end closed connection without response
2024-03-02T18:17:09.944Z [container-agent] 2024-03-02 18:17:09 INFO juju.util.exec exec.go:209 run result: exit status 1
2024-03-02T18:17:15.346Z [container-agent] 2024-03-02 18:17:15 INFO juju-log Setting up the logrotate configurations
2024-03-02T18:17:15.424Z [container-agent] 2024-03-02 18:17:15 INFO juju-log Adding pebble layer
2024-03-02T18:17:22.164Z [container-agent] 2024-03-02 18:17:22 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-03-02T18:17:22.186Z [container-agent] 2024-03-02 18:17:22 INFO juju-log Attempting reboot from complete outage.
2024-03-02T18:17:25.158Z [container-agent] 2024-03-02 18:17:25 INFO juju.worker.uniter.operation runhook.go:186 ran "mysql-pebble-ready" hook (via hook dispatching script: dispatch)
2024-03-02T18:19:27.948Z [container-agent] 2024-03-02 18:19:27 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T18:19:29.814Z [container-agent] 2024-03-02 18:19:29 WARNING juju-log No relation: certificates
2024-03-02T18:19:30.165Z [container-agent] 2024-03-02 18:19:30 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T18:24:05.991Z [container-agent] 2024-03-02 18:24:05 INFO juju-log Starting the log rotate manager
2024-03-02T18:24:06.006Z [container-agent] 2024-03-02 18:24:06 INFO juju-log Started log rotate manager process with PID 587998
2024-03-02T18:24:06.354Z [container-agent] 2024-03-02 18:24:06 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T18:24:08.425Z [container-agent] 2024-03-02 18:24:08 WARNING juju-log No relation: certificates
2024-03-02T18:24:08.838Z [container-agent] 2024-03-02 18:24:08 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T18:30:00.780Z [container-agent] 2024-03-02 18:30:00 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T18:30:03.227Z [container-agent] 2024-03-02 18:30:03 WARNING juju-log No relation: certificates
2024-03-02T18:30:03.580Z [container-agent] 2024-03-02 18:30:03 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T18:34:28.272Z [container-agent] 2024-03-02 18:34:28 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T18:34:31.166Z [container-agent] 2024-03-02 18:34:31 WARNING juju-log No relation: certificates
2024-03-02T18:34:31.517Z [container-agent] 2024-03-02 18:34:31 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T18:39:30.099Z [container-agent] 2024-03-02 18:39:30 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T18:39:33.729Z [container-agent] 2024-03-02 18:39:33 WARNING juju-log No relation: certificates
2024-03-02T18:39:34.087Z [container-agent] 2024-03-02 18:39:34 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T18:44:58.914Z [container-agent] 2024-03-02 18:44:58 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T18:45:02.734Z [container-agent] 2024-03-02 18:45:02 WARNING juju-log No relation: certificates
2024-03-02T18:45:03.078Z [container-agent] 2024-03-02 18:45:03 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T18:50:01.292Z [container-agent] 2024-03-02 18:50:01 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T18:50:05.483Z [container-agent] 2024-03-02 18:50:05 WARNING juju-log No relation: certificates
2024-03-02T18:50:05.840Z [container-agent] 2024-03-02 18:50:05 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T18:54:02.636Z [container-agent] 2024-03-02 18:54:02 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T18:54:07.226Z [container-agent] 2024-03-02 18:54:07 WARNING juju-log No relation: certificates
2024-03-02T18:54:07.578Z [container-agent] 2024-03-02 18:54:07 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T18:58:43.276Z [container-agent] 2024-03-02 18:58:43 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T18:58:48.602Z [container-agent] 2024-03-02 18:58:48 WARNING juju-log No relation: certificates
2024-03-02T18:58:48.961Z [container-agent] 2024-03-02 18:58:48 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T19:03:33.337Z [container-agent] 2024-03-02 19:03:33 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T19:03:38.957Z [container-agent] 2024-03-02 19:03:38 WARNING juju-log No relation: certificates
2024-03-02T19:03:39.309Z [container-agent] 2024-03-02 19:03:39 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T19:08:11.174Z [container-agent] 2024-03-02 19:08:11 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T19:08:17.423Z [container-agent] 2024-03-02 19:08:17 WARNING juju-log No relation: certificates
2024-03-02T19:08:17.817Z [container-agent] 2024-03-02 19:08:17 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T19:14:11.903Z [container-agent] 2024-03-02 19:14:11 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T19:14:18.639Z [container-agent] 2024-03-02 19:14:18 WARNING juju-log No relation: certificates
2024-03-02T19:14:18.989Z [container-agent] 2024-03-02 19:14:18 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T19:18:25.582Z [container-agent] 2024-03-02 19:18:25 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T19:18:32.759Z [container-agent] 2024-03-02 19:18:32 WARNING juju-log No relation: certificates
2024-03-02T19:18:33.108Z [container-agent] 2024-03-02 19:18:33 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T19:22:38.666Z [container-agent] 2024-03-02 19:22:38 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T19:22:46.223Z [container-agent] 2024-03-02 19:22:46 WARNING juju-log No relation: certificates
2024-03-02T19:22:46.565Z [container-agent] 2024-03-02 19:22:46 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T19:28:13.911Z [container-agent] 2024-03-02 19:28:13 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T19:28:21.649Z [container-agent] 2024-03-02 19:28:21 WARNING juju-log No relation: certificates
2024-03-02T19:28:21.982Z [container-agent] 2024-03-02 19:28:21 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T19:33:34.595Z [container-agent] 2024-03-02 19:33:34 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T19:33:42.832Z [container-agent] 2024-03-02 19:33:42 WARNING juju-log No relation: certificates
2024-03-02T19:33:43.173Z [container-agent] 2024-03-02 19:33:43 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T19:38:32.348Z [container-agent] 2024-03-02 19:38:32 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-03-02T19:38:32.380Z [container-agent] 2024-03-02 19:38:32 INFO juju-log Attempting reboot from complete outage.
2024-03-02T19:38:40.510Z [container-agent] 2024-03-02 19:38:40 WARNING juju-log No relation: certificates
2024-03-02T19:38:40.863Z [container-agent] 2024-03-02 19:38:40 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T19:41:04.752Z [container-agent] 2024-03-02 19:41:04 ERROR juju-log Uncaught exception while in charm code:
2024-03-02T19:41:04.752Z [container-agent] Traceback (most recent call last):
2024-03-02T19:41:04.752Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/./src/charm.py", line 786, in <module>
2024-03-02T19:41:04.752Z [container-agent]     main(MySQLOperatorCharm)
2024-03-02T19:41:04.752Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 436, in main
2024-03-02T19:41:04.752Z [container-agent]     _emit_charm_event(charm, dispatcher.event_name)
2024-03-02T19:41:04.752Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 144, in _emit_charm_event
2024-03-02T19:41:04.752Z [container-agent]     event_to_emit.emit(*args, **kwargs)
2024-03-02T19:41:04.752Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 351, in emit
2024-03-02T19:41:04.752Z [container-agent]     framework._emit(event)
2024-03-02T19:41:04.752Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 853, in _emit
2024-03-02T19:41:04.752Z [container-agent]     self._reemit(event_path)
2024-03-02T19:41:04.752Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 942, in _reemit
2024-03-02T19:41:04.752Z [container-agent]     custom_handler(event)
2024-03-02T19:41:04.752Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/rotate_mysql_logs.py", line 55, in _rotate_mysql_logs
2024-03-02T19:41:04.752Z [container-agent]     self.charm._mysql.flush_mysql_logs(MySQLTextLogs.ERROR)
2024-03-02T19:41:04.752Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-03-02T19:41:04.752Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-03-02T19:41:04.752Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 665, in _run_mysqlsh_script
2024-03-02T19:41:04.752Z [container-agent]     process = self.container.exec(cmd, timeout=timeout)
2024-03-02T19:41:04.752Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/model.py", line 2688, in exec
2024-03-02T19:41:04.752Z [container-agent]     return self._pebble.exec(
2024-03-02T19:41:04.752Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 2501, in exec
2024-03-02T19:41:04.752Z [container-agent]     resp = self._request('POST', '/v1/exec', body=body)
2024-03-02T19:41:04.752Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1655, in _request
2024-03-02T19:41:04.752Z [container-agent]     response = self._request_raw(method, path, query, headers, data)
2024-03-02T19:41:04.752Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1690, in _request_raw
2024-03-02T19:41:04.752Z [container-agent]     response = self.opener.open(request, timeout=self.timeout)
2024-03-02T19:41:04.752Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 519, in open
2024-03-02T19:41:04.752Z [container-agent]     response = self._open(req, data)
2024-03-02T19:41:04.752Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 536, in _open
2024-03-02T19:41:04.752Z [container-agent]     result = self._call_chain(self.handle_open, protocol, protocol +
2024-03-02T19:41:04.752Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 496, in _call_chain
2024-03-02T19:41:04.752Z [container-agent]     result = func(*args)
2024-03-02T19:41:04.752Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 311, in http_open
2024-03-02T19:41:04.752Z [container-agent]     return self.do_open(_UnixSocketConnection, req,  # type:ignore
2024-03-02T19:41:04.752Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 1352, in do_open
2024-03-02T19:41:04.752Z [container-agent]     r = h.getresponse()
2024-03-02T19:41:04.752Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1375, in getresponse
2024-03-02T19:41:04.752Z [container-agent]     response.begin()
2024-03-02T19:41:04.752Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 318, in begin
2024-03-02T19:41:04.752Z [container-agent]     version, status, reason = self._read_status()
2024-03-02T19:41:04.752Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 287, in _read_status
2024-03-02T19:41:04.752Z [container-agent]     raise RemoteDisconnected("Remote end closed connection without"
2024-03-02T19:41:04.752Z [container-agent] http.client.RemoteDisconnected: Remote end closed connection without response
2024-03-02T19:41:04.888Z [container-agent] 2024-03-02 19:41:04 INFO juju.util.exec exec.go:209 run result: exit status 1
2024-03-02T19:41:08.468Z [container-agent] 2024-03-02 19:41:08 INFO juju-log Setting up the logrotate configurations
2024-03-02T19:41:08.545Z [container-agent] 2024-03-02 19:41:08 INFO juju-log Adding pebble layer
2024-03-02T19:41:15.295Z [container-agent] 2024-03-02 19:41:15 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-03-02T19:41:15.320Z [container-agent] 2024-03-02 19:41:15 INFO juju-log Attempting reboot from complete outage.
2024-03-02T19:41:18.285Z [container-agent] 2024-03-02 19:41:18 INFO juju.worker.uniter.operation runhook.go:186 ran "mysql-pebble-ready" hook (via hook dispatching script: dispatch)
2024-03-02T19:43:48.421Z [container-agent] 2024-03-02 19:43:48 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T19:43:50.454Z [container-agent] 2024-03-02 19:43:50 WARNING juju-log No relation: certificates
2024-03-02T19:43:50.805Z [container-agent] 2024-03-02 19:43:50 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T19:48:30.050Z [container-agent] 2024-03-02 19:48:30 INFO juju-log Starting the log rotate manager
2024-03-02T19:48:30.063Z [container-agent] 2024-03-02 19:48:30 INFO juju-log Started log rotate manager process with PID 595871
2024-03-02T19:48:30.337Z [container-agent] 2024-03-02 19:48:30 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T19:48:32.275Z [container-agent] 2024-03-02 19:48:32 WARNING juju-log No relation: certificates
2024-03-02T19:48:32.651Z [container-agent] 2024-03-02 19:48:32 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T19:53:06.031Z [container-agent] 2024-03-02 19:53:06 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T19:53:08.763Z [container-agent] 2024-03-02 19:53:08 WARNING juju-log No relation: certificates
2024-03-02T19:53:09.110Z [container-agent] 2024-03-02 19:53:09 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T19:58:41.450Z [container-agent] 2024-03-02 19:58:41 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T19:58:44.310Z [container-agent] 2024-03-02 19:58:44 WARNING juju-log No relation: certificates
2024-03-02T19:58:44.660Z [container-agent] 2024-03-02 19:58:44 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T20:04:38.310Z [container-agent] 2024-03-02 20:04:38 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T20:04:41.378Z [container-agent] 2024-03-02 20:04:41 WARNING juju-log No relation: certificates
2024-03-02T20:04:41.695Z [container-agent] 2024-03-02 20:04:41 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T20:10:08.055Z [container-agent] 2024-03-02 20:10:08 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T20:10:12.096Z [container-agent] 2024-03-02 20:10:12 WARNING juju-log No relation: certificates
2024-03-02T20:10:12.469Z [container-agent] 2024-03-02 20:10:12 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T20:15:50.521Z [container-agent] 2024-03-02 20:15:50 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T20:15:54.613Z [container-agent] 2024-03-02 20:15:54 WARNING juju-log No relation: certificates
2024-03-02T20:15:54.956Z [container-agent] 2024-03-02 20:15:54 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T20:20:16.535Z [container-agent] 2024-03-02 20:20:16 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T20:20:21.546Z [container-agent] 2024-03-02 20:20:21 WARNING juju-log No relation: certificates
2024-03-02T20:20:21.894Z [container-agent] 2024-03-02 20:20:21 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T20:25:10.183Z [container-agent] 2024-03-02 20:25:10 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T20:25:15.464Z [container-agent] 2024-03-02 20:25:15 WARNING juju-log No relation: certificates
2024-03-02T20:25:15.815Z [container-agent] 2024-03-02 20:25:15 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T20:30:28.856Z [container-agent] 2024-03-02 20:30:28 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T20:30:34.773Z [container-agent] 2024-03-02 20:30:34 WARNING juju-log No relation: certificates
2024-03-02T20:30:35.169Z [container-agent] 2024-03-02 20:30:35 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T20:35:16.774Z [container-agent] 2024-03-02 20:35:16 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T20:35:23.067Z [container-agent] 2024-03-02 20:35:23 WARNING juju-log No relation: certificates
2024-03-02T20:35:23.409Z [container-agent] 2024-03-02 20:35:23 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T20:40:14.484Z [container-agent] 2024-03-02 20:40:14 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T20:40:21.292Z [container-agent] 2024-03-02 20:40:21 WARNING juju-log No relation: certificates
2024-03-02T20:40:21.627Z [container-agent] 2024-03-02 20:40:21 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T20:44:46.872Z [container-agent] 2024-03-02 20:44:46 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T20:44:54.114Z [container-agent] 2024-03-02 20:44:54 WARNING juju-log No relation: certificates
2024-03-02T20:44:54.451Z [container-agent] 2024-03-02 20:44:54 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T20:48:47.609Z [container-agent] 2024-03-02 20:48:47 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T20:48:54.779Z [container-agent] 2024-03-02 20:48:54 WARNING juju-log No relation: certificates
2024-03-02T20:48:55.147Z [container-agent] 2024-03-02 20:48:55 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T20:53:57.669Z [container-agent] 2024-03-02 20:53:57 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T20:54:05.122Z [container-agent] 2024-03-02 20:54:05 WARNING juju-log No relation: certificates
2024-03-02T20:54:05.455Z [container-agent] 2024-03-02 20:54:05 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T20:59:17.060Z [container-agent] 2024-03-02 20:59:17 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T20:59:25.447Z [container-agent] 2024-03-02 20:59:25 WARNING juju-log No relation: certificates
2024-03-02T20:59:25.782Z [container-agent] 2024-03-02 20:59:25 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T21:04:16.300Z [container-agent] 2024-03-02 21:04:16 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-03-02T21:04:16.327Z [container-agent] 2024-03-02 21:04:16 INFO juju-log Attempting reboot from complete outage.
2024-03-02T21:04:24.337Z [container-agent] 2024-03-02 21:04:24 WARNING juju-log No relation: certificates
2024-03-02T21:04:24.683Z [container-agent] 2024-03-02 21:04:24 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T21:06:04.805Z [container-agent] 2024-03-02 21:06:04 ERROR juju-log Uncaught exception while in charm code:
2024-03-02T21:06:04.805Z [container-agent] Traceback (most recent call last):
2024-03-02T21:06:04.805Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 2507, in exec
2024-03-02T21:06:04.805Z [container-agent]     control_ws = self._connect_websocket(task_id, 'control')
2024-03-02T21:06:04.805Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 2589, in _connect_websocket
2024-03-02T21:06:04.805Z [container-agent]     ws.connect(url, socket=sock)
2024-03-02T21:06:04.805Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_core.py", line 253, in connect
2024-03-02T21:06:04.805Z [container-agent]     self.handshake_response = handshake(self.sock, url, *addrs, **options)
2024-03-02T21:06:04.805Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_handshake.py", line 57, in handshake
2024-03-02T21:06:04.805Z [container-agent]     status, resp = _get_resp_headers(sock)
2024-03-02T21:06:04.805Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_handshake.py", line 143, in _get_resp_headers
2024-03-02T21:06:04.805Z [container-agent]     status, resp_headers, status_message = read_headers(sock)
2024-03-02T21:06:04.805Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_http.py", line 312, in read_headers
2024-03-02T21:06:04.805Z [container-agent]     line = recv_line(sock)
2024-03-02T21:06:04.805Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_socket.py", line 131, in recv_line
2024-03-02T21:06:04.805Z [container-agent]     c = recv(sock, 1)
2024-03-02T21:06:04.805Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_socket.py", line 122, in recv
2024-03-02T21:06:04.805Z [container-agent]     raise WebSocketConnectionClosedException(
2024-03-02T21:06:04.805Z [container-agent] websocket._exceptions.WebSocketConnectionClosedException: Connection to remote host was lost.
2024-03-02T21:06:04.805Z [container-agent] 
2024-03-02T21:06:04.805Z [container-agent] During handling of the above exception, another exception occurred:
2024-03-02T21:06:04.805Z [container-agent] 
2024-03-02T21:06:04.805Z [container-agent] Traceback (most recent call last):
2024-03-02T21:06:04.805Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 1348, in do_open
2024-03-02T21:06:04.805Z [container-agent]     h.request(req.get_method(), req.selector, req.data, headers,
2024-03-02T21:06:04.805Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1283, in request
2024-03-02T21:06:04.805Z [container-agent]     self._send_request(method, url, body, headers, encode_chunked)
2024-03-02T21:06:04.805Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1329, in _send_request
2024-03-02T21:06:04.805Z [container-agent]     self.endheaders(body, encode_chunked=encode_chunked)
2024-03-02T21:06:04.805Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1278, in endheaders
2024-03-02T21:06:04.805Z [container-agent]     self._send_output(message_body, encode_chunked=encode_chunked)
2024-03-02T21:06:04.805Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1038, in _send_output
2024-03-02T21:06:04.805Z [container-agent]     self.send(msg)
2024-03-02T21:06:04.805Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 976, in send
2024-03-02T21:06:04.805Z [container-agent]     self.connect()
2024-03-02T21:06:04.805Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 297, in connect
2024-03-02T21:06:04.805Z [container-agent]     self.sock.connect(self.socket_path)
2024-03-02T21:06:04.805Z [container-agent] ConnectionRefusedError: [Errno 111] Connection refused
2024-03-02T21:06:04.805Z [container-agent] 
2024-03-02T21:06:04.805Z [container-agent] During handling of the above exception, another exception occurred:
2024-03-02T21:06:04.805Z [container-agent] 
2024-03-02T21:06:04.805Z [container-agent] Traceback (most recent call last):
2024-03-02T21:06:04.805Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1690, in _request_raw
2024-03-02T21:06:04.805Z [container-agent]     response = self.opener.open(request, timeout=self.timeout)
2024-03-02T21:06:04.805Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 519, in open
2024-03-02T21:06:04.805Z [container-agent]     response = self._open(req, data)
2024-03-02T21:06:04.805Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 536, in _open
2024-03-02T21:06:04.805Z [container-agent]     result = self._call_chain(self.handle_open, protocol, protocol +
2024-03-02T21:06:04.805Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 496, in _call_chain
2024-03-02T21:06:04.805Z [container-agent]     result = func(*args)
2024-03-02T21:06:04.805Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 311, in http_open
2024-03-02T21:06:04.805Z [container-agent]     return self.do_open(_UnixSocketConnection, req,  # type:ignore
2024-03-02T21:06:04.805Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 1351, in do_open
2024-03-02T21:06:04.805Z [container-agent]     raise URLError(err)
2024-03-02T21:06:04.805Z [container-agent] urllib.error.URLError: <urlopen error [Errno 111] Connection refused>
2024-03-02T21:06:04.805Z [container-agent] 
2024-03-02T21:06:04.805Z [container-agent] The above exception was the direct cause of the following exception:
2024-03-02T21:06:04.805Z [container-agent] 
2024-03-02T21:06:04.805Z [container-agent] Traceback (most recent call last):
2024-03-02T21:06:04.805Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/./src/charm.py", line 786, in <module>
2024-03-02T21:06:04.805Z [container-agent]     main(MySQLOperatorCharm)
2024-03-02T21:06:04.805Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 436, in main
2024-03-02T21:06:04.805Z [container-agent]     _emit_charm_event(charm, dispatcher.event_name)
2024-03-02T21:06:04.805Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 144, in _emit_charm_event
2024-03-02T21:06:04.805Z [container-agent]     event_to_emit.emit(*args, **kwargs)
2024-03-02T21:06:04.805Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 351, in emit
2024-03-02T21:06:04.805Z [container-agent]     framework._emit(event)
2024-03-02T21:06:04.805Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 853, in _emit
2024-03-02T21:06:04.805Z [container-agent]     self._reemit(event_path)
2024-03-02T21:06:04.805Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 942, in _reemit
2024-03-02T21:06:04.805Z [container-agent]     custom_handler(event)
2024-03-02T21:06:04.805Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/rotate_mysql_logs.py", line 55, in _rotate_mysql_logs
2024-03-02T21:06:04.805Z [container-agent]     self.charm._mysql.flush_mysql_logs(MySQLTextLogs.ERROR)
2024-03-02T21:06:04.805Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-03-02T21:06:04.805Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-03-02T21:06:04.805Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 665, in _run_mysqlsh_script
2024-03-02T21:06:04.805Z [container-agent]     process = self.container.exec(cmd, timeout=timeout)
2024-03-02T21:06:04.805Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/model.py", line 2688, in exec
2024-03-02T21:06:04.805Z [container-agent]     return self._pebble.exec(
2024-03-02T21:06:04.805Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 2514, in exec
2024-03-02T21:06:04.805Z [container-agent]     change = self.wait_change(ChangeID(change_id))
2024-03-02T21:06:04.805Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1893, in wait_change
2024-03-02T21:06:04.805Z [container-agent]     return self._wait_change_using_wait(change_id, timeout)
2024-03-02T21:06:04.806Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1914, in _wait_change_using_wait
2024-03-02T21:06:04.806Z [container-agent]     return self._wait_change(change_id, this_timeout)
2024-03-02T21:06:04.806Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1928, in _wait_change
2024-03-02T21:06:04.806Z [container-agent]     resp = self._request('GET', f'/v1/changes/{change_id}/wait', query)
2024-03-02T21:06:04.806Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1655, in _request
2024-03-02T21:06:04.806Z [container-agent]     response = self._request_raw(method, path, query, headers, data)
2024-03-02T21:06:04.806Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1707, in _request_raw
2024-03-02T21:06:04.806Z [container-agent]     raise ConnectionError(e.reason) from e
2024-03-02T21:06:04.806Z [container-agent] ops.pebble.ConnectionError: [Errno 111] Connection refused
2024-03-02T21:06:04.981Z [container-agent] 2024-03-02 21:06:04 INFO juju.util.exec exec.go:209 run result: exit status 1
2024-03-02T21:06:06.731Z [container-agent] 2024-03-02 21:06:06 INFO juju-log Setting up the logrotate configurations
2024-03-02T21:06:06.844Z [container-agent] 2024-03-02 21:06:06 INFO juju-log Adding pebble layer
2024-03-02T21:06:13.612Z [container-agent] 2024-03-02 21:06:13 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-03-02T21:06:13.634Z [container-agent] 2024-03-02 21:06:13 INFO juju-log Attempting reboot from complete outage.
2024-03-02T21:06:16.607Z [container-agent] 2024-03-02 21:06:16 INFO juju.worker.uniter.operation runhook.go:186 ran "mysql-pebble-ready" hook (via hook dispatching script: dispatch)
2024-03-02T21:09:58.470Z [container-agent] 2024-03-02 21:09:58 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T21:10:00.496Z [container-agent] 2024-03-02 21:10:00 WARNING juju-log No relation: certificates
2024-03-02T21:10:00.912Z [container-agent] 2024-03-02 21:10:00 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T21:14:21.039Z [container-agent] 2024-03-02 21:14:21 INFO juju-log Starting the log rotate manager
2024-03-02T21:14:21.053Z [container-agent] 2024-03-02 21:14:21 INFO juju-log Started log rotate manager process with PID 603721
2024-03-02T21:14:21.523Z [container-agent] 2024-03-02 21:14:21 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T21:14:23.519Z [container-agent] 2024-03-02 21:14:23 WARNING juju-log No relation: certificates
2024-03-02T21:14:23.895Z [container-agent] 2024-03-02 21:14:23 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T21:18:22.971Z [container-agent] 2024-03-02 21:18:22 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T21:18:25.254Z [container-agent] 2024-03-02 21:18:25 WARNING juju-log No relation: certificates
2024-03-02T21:18:25.632Z [container-agent] 2024-03-02 21:18:25 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T21:22:52.264Z [container-agent] 2024-03-02 21:22:52 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T21:22:54.895Z [container-agent] 2024-03-02 21:22:54 WARNING juju-log No relation: certificates
2024-03-02T21:22:55.230Z [container-agent] 2024-03-02 21:22:55 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T21:28:48.300Z [container-agent] 2024-03-02 21:28:48 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T21:28:51.842Z [container-agent] 2024-03-02 21:28:51 WARNING juju-log No relation: certificates
2024-03-02T21:28:52.182Z [container-agent] 2024-03-02 21:28:52 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T21:34:08.175Z [container-agent] 2024-03-02 21:34:08 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T21:34:11.957Z [container-agent] 2024-03-02 21:34:11 WARNING juju-log No relation: certificates
2024-03-02T21:34:12.300Z [container-agent] 2024-03-02 21:34:12 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T21:39:58.957Z [container-agent] 2024-03-02 21:39:58 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T21:40:03.129Z [container-agent] 2024-03-02 21:40:03 WARNING juju-log No relation: certificates
2024-03-02T21:40:03.495Z [container-agent] 2024-03-02 21:40:03 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T21:44:09.226Z [container-agent] 2024-03-02 21:44:09 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T21:44:14.072Z [container-agent] 2024-03-02 21:44:14 WARNING juju-log No relation: certificates
2024-03-02T21:44:14.442Z [container-agent] 2024-03-02 21:44:14 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T21:48:32.516Z [container-agent] 2024-03-02 21:48:32 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T21:48:37.537Z [container-agent] 2024-03-02 21:48:37 WARNING juju-log No relation: certificates
2024-03-02T21:48:37.894Z [container-agent] 2024-03-02 21:48:37 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T21:53:51.956Z [container-agent] 2024-03-02 21:53:51 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T21:53:56.983Z [container-agent] 2024-03-02 21:53:56 WARNING juju-log No relation: certificates
2024-03-02T21:53:57.317Z [container-agent] 2024-03-02 21:53:57 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T21:59:01.263Z [container-agent] 2024-03-02 21:59:01 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T21:59:06.690Z [container-agent] 2024-03-02 21:59:06 WARNING juju-log No relation: certificates
2024-03-02T21:59:07.012Z [container-agent] 2024-03-02 21:59:07 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T22:03:12.039Z [container-agent] 2024-03-02 22:03:12 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T22:03:18.530Z [container-agent] 2024-03-02 22:03:18 WARNING juju-log No relation: certificates
2024-03-02T22:03:18.891Z [container-agent] 2024-03-02 22:03:18 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T22:07:31.119Z [container-agent] 2024-03-02 22:07:31 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T22:07:38.206Z [container-agent] 2024-03-02 22:07:38 WARNING juju-log No relation: certificates
2024-03-02T22:07:38.554Z [container-agent] 2024-03-02 22:07:38 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T22:13:16.794Z [container-agent] 2024-03-02 22:13:16 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T22:13:24.179Z [container-agent] 2024-03-02 22:13:24 WARNING juju-log No relation: certificates
2024-03-02T22:13:24.533Z [container-agent] 2024-03-02 22:13:24 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T22:18:13.481Z [container-agent] 2024-03-02 22:18:13 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T22:18:21.671Z [container-agent] 2024-03-02 22:18:21 WARNING juju-log No relation: certificates
2024-03-02T22:18:22.045Z [container-agent] 2024-03-02 22:18:22 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T22:24:14.723Z [container-agent] 2024-03-02 22:24:14 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T22:24:21.931Z [container-agent] 2024-03-02 22:24:21 WARNING juju-log Failed to get cluster primary addresses
2024-03-02T22:24:21.931Z [container-agent] Traceback (most recent call last):
2024-03-02T22:24:21.931Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 666, in _run_mysqlsh_script
2024-03-02T22:24:21.931Z [container-agent]     stdout, _ = process.wait_output()
2024-03-02T22:24:21.931Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1441, in wait_output
2024-03-02T22:24:21.931Z [container-agent]     raise ExecError[AnyStr](self._command, exit_code, out_value, err_value)
2024-03-02T22:24:21.931Z [container-agent] ops.pebble.ExecError: non-zero exit code 1 executing ['/usr/bin/mysqlsh', '--no-wizard', '--python', '--verbose=1', '-f', '/tmp/script.py', ';', 'rm', '/tmp/script.py'], stdout='', stderr='Cannot set LC_ALL to locale en_US.UTF-8: No such file or directory\nverbose: 2024-03-02T22:24:20Z: Loading startup files...\nverbose: 2024-03-02T22:24:20Z: Loading plugins...\nverbose: 2024-03-02T22:24:20Z: Connecting to MySQL at: clusteradmin@horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local\nverbose: 2024-03-02T22:24:20Z: Shell.connect_to_primary: tid=22: CONNECTED: horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local\nverbose: 2024-03-02T22:24:20Z: Redirecting session from \'mysqlx://clusteradmin@horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local:33060\' to a PRIMARY of an InnoDB cluster or ReplicaSet...\nTraceback (most recent call last):\n  File "<string>", line 1, in <module>\nRuntimeError: Shell.connect_to_primary: No primaries found in an InnoDB cluster\n\n'
2024-03-02T22:24:21.931Z [container-agent] 
2024-03-02T22:24:21.931Z [container-agent] During handling of the above exception, another exception occurred:
2024-03-02T22:24:21.931Z [container-agent] 
2024-03-02T22:24:21.931Z [container-agent] Traceback (most recent call last):
2024-03-02T22:24:21.931Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 1767, in get_cluster_primary_address
2024-03-02T22:24:21.931Z [container-agent]     output = self._run_mysqlsh_script("\n".join(get_cluster_primary_commands))
2024-03-02T22:24:21.931Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 669, in _run_mysqlsh_script
2024-03-02T22:24:21.931Z [container-agent]     raise MySQLClientError(e.stderr)
2024-03-02T22:24:21.931Z [container-agent] charms.mysql.v0.mysql.MySQLClientError: Cannot set LC_ALL to locale en_US.UTF-8: No such file or directory
2024-03-02T22:24:21.931Z [container-agent] verbose: 2024-03-02T22:24:20Z: Loading startup files...
2024-03-02T22:24:21.931Z [container-agent] verbose: 2024-03-02T22:24:20Z: Loading plugins...
2024-03-02T22:24:21.931Z [container-agent] verbose: 2024-03-02T22:24:20Z: Connecting to MySQL at: clusteradmin@horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local
2024-03-02T22:24:21.931Z [container-agent] verbose: 2024-03-02T22:24:20Z: Shell.connect_to_primary: tid=22: CONNECTED: horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local
2024-03-02T22:24:21.931Z [container-agent] verbose: 2024-03-02T22:24:20Z: Redirecting session from 'mysqlx://clusteradmin@horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local:33060' to a PRIMARY of an InnoDB cluster or ReplicaSet...
2024-03-02T22:24:21.931Z [container-agent] Traceback (most recent call last):
2024-03-02T22:24:21.931Z [container-agent]   File "<string>", line 1, in <module>
2024-03-02T22:24:21.931Z [container-agent] RuntimeError: Shell.connect_to_primary: No primaries found in an InnoDB cluster
2024-03-02T22:24:21.931Z [container-agent] 
2024-03-02T22:24:21.931Z [container-agent] 
2024-03-02T22:24:24.540Z [container-agent] 2024-03-02 22:24:24 ERROR juju-log Failed to get cluster status for cluster-7573647c1e26466beeeac32b95ab8f91
2024-03-02T22:24:24.602Z [container-agent] 2024-03-02 22:24:24 INFO juju-log Kubernetes pod label error created
2024-03-02T22:24:24.617Z [container-agent] 2024-03-02 22:24:24 WARNING juju-log No relation: certificates
2024-03-02T22:24:24.972Z [container-agent] 2024-03-02 22:24:24 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T22:29:19.986Z [container-agent] 2024-03-02 22:29:19 INFO juju-log Setting up the logrotate configurations
2024-03-02T22:29:20.068Z [container-agent] 2024-03-02 22:29:20 INFO juju-log Adding pebble layer
2024-03-02T22:29:26.849Z [container-agent] 2024-03-02 22:29:26 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-03-02T22:29:26.881Z [container-agent] 2024-03-02 22:29:26 INFO juju-log Attempting reboot from complete outage.
2024-03-02T22:29:29.835Z [container-agent] 2024-03-02 22:29:29 INFO juju.worker.uniter.operation runhook.go:186 ran "mysql-pebble-ready" hook (via hook dispatching script: dispatch)
2024-03-02T22:29:52.934Z [container-agent] 2024-03-02 22:29:52 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T22:29:55.173Z [container-agent] 2024-03-02 22:29:55 INFO juju-log Kubernetes pod label primary created
2024-03-02T22:29:55.187Z [container-agent] 2024-03-02 22:29:55 WARNING juju-log No relation: certificates
2024-03-02T22:29:55.541Z [container-agent] 2024-03-02 22:29:55 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T22:35:18.501Z [container-agent] 2024-03-02 22:35:18 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T22:35:21.065Z [container-agent] 2024-03-02 22:35:21 WARNING juju-log No relation: certificates
2024-03-02T22:35:21.466Z [container-agent] 2024-03-02 22:35:21 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T22:40:46.001Z [container-agent] 2024-03-02 22:40:46 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T22:40:49.130Z [container-agent] 2024-03-02 22:40:49 WARNING juju-log No relation: certificates
2024-03-02T22:40:49.476Z [container-agent] 2024-03-02 22:40:49 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T22:45:57.001Z [container-agent] 2024-03-02 22:45:57 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T22:46:00.501Z [container-agent] 2024-03-02 22:46:00 WARNING juju-log No relation: certificates
2024-03-02T22:46:00.869Z [container-agent] 2024-03-02 22:46:00 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T22:50:51.615Z [container-agent] 2024-03-02 22:50:51 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T22:50:55.551Z [container-agent] 2024-03-02 22:50:55 WARNING juju-log No relation: certificates
2024-03-02T22:50:55.892Z [container-agent] 2024-03-02 22:50:55 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T22:56:15.596Z [container-agent] 2024-03-02 22:56:15 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T22:56:20.099Z [container-agent] 2024-03-02 22:56:20 WARNING juju-log No relation: certificates
2024-03-02T22:56:20.471Z [container-agent] 2024-03-02 22:56:20 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T23:01:09.192Z [container-agent] 2024-03-02 23:01:09 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T23:01:14.335Z [container-agent] 2024-03-02 23:01:14 WARNING juju-log No relation: certificates
2024-03-02T23:01:14.677Z [container-agent] 2024-03-02 23:01:14 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T23:06:38.441Z [container-agent] 2024-03-02 23:06:38 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T23:06:43.956Z [container-agent] 2024-03-02 23:06:43 WARNING juju-log No relation: certificates
2024-03-02T23:06:44.313Z [container-agent] 2024-03-02 23:06:44 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T23:11:51.815Z [container-agent] 2024-03-02 23:11:51 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T23:11:57.236Z [container-agent] 2024-03-02 23:11:57 WARNING juju-log No relation: certificates
2024-03-02T23:11:57.584Z [container-agent] 2024-03-02 23:11:57 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T23:17:21.127Z [container-agent] 2024-03-02 23:17:21 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T23:17:26.900Z [container-agent] 2024-03-02 23:17:26 WARNING juju-log No relation: certificates
2024-03-02T23:17:27.242Z [container-agent] 2024-03-02 23:17:27 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T23:22:51.064Z [container-agent] 2024-03-02 23:22:51 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T23:22:57.876Z [container-agent] 2024-03-02 23:22:57 WARNING juju-log No relation: certificates
2024-03-02T23:22:58.221Z [container-agent] 2024-03-02 23:22:58 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T23:27:31.671Z [container-agent] 2024-03-02 23:27:31 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T23:27:38.932Z [container-agent] 2024-03-02 23:27:38 WARNING juju-log No relation: certificates
2024-03-02T23:27:39.292Z [container-agent] 2024-03-02 23:27:39 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T23:31:45.772Z [container-agent] 2024-03-02 23:31:45 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T23:31:53.144Z [container-agent] 2024-03-02 23:31:53 WARNING juju-log No relation: certificates
2024-03-02T23:31:53.533Z [container-agent] 2024-03-02 23:31:53 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T23:36:14.704Z [container-agent] 2024-03-02 23:36:14 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T23:36:22.833Z [container-agent] 2024-03-02 23:36:22 WARNING juju-log No relation: certificates
2024-03-02T23:36:23.190Z [container-agent] 2024-03-02 23:36:23 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T23:38:10.416Z [container-agent] 2024-03-02 23:38:10 ERROR juju-log Failed to flush ERROR LOGS logs.
2024-03-02T23:38:10.416Z [container-agent] Traceback (most recent call last):
2024-03-02T23:38:10.416Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 666, in _run_mysqlsh_script
2024-03-02T23:38:10.416Z [container-agent]     stdout, _ = process.wait_output()
2024-03-02T23:38:10.416Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1441, in wait_output
2024-03-02T23:38:10.416Z [container-agent]     raise ExecError[AnyStr](self._command, exit_code, out_value, err_value)
2024-03-02T23:38:10.416Z [container-agent] ops.pebble.ExecError: non-zero exit code 1 executing ['/usr/bin/mysqlsh', '--no-wizard', '--python', '--verbose=1', '-f', '/tmp/script.py', ';', 'rm', '/tmp/script.py'], stdout='', stderr='Cannot set LC_ALL to locale en_US.UTF-8: No such file or directory\nverbose: 2024-03-02T23:38:06Z: Loading startup files...\nverbose: 2024-03-02T23:38:06Z: Loading plugins...\nverbose: 2024-03-02T23:38:06Z: Connecting to MySQL at: serverconfig@horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local\nTraceback (most recent call last):\n  File "<string>", line 1, in <module>\nmysqlsh.DBError: MySQL Error (2013): Shell.connect: Lost connection to MySQL server at \'reading initial communication packet\', system error: 104\n'
2024-03-02T23:38:10.416Z [container-agent] 
2024-03-02T23:38:10.416Z [container-agent] During handling of the above exception, another exception occurred:
2024-03-02T23:38:10.416Z [container-agent] 
2024-03-02T23:38:10.416Z [container-agent] Traceback (most recent call last):
2024-03-02T23:38:10.416Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-03-02T23:38:10.416Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-03-02T23:38:10.416Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 669, in _run_mysqlsh_script
2024-03-02T23:38:10.416Z [container-agent]     raise MySQLClientError(e.stderr)
2024-03-02T23:38:10.416Z [container-agent] charms.mysql.v0.mysql.MySQLClientError: Cannot set LC_ALL to locale en_US.UTF-8: No such file or directory
2024-03-02T23:38:10.416Z [container-agent] verbose: 2024-03-02T23:38:06Z: Loading startup files...
2024-03-02T23:38:10.416Z [container-agent] verbose: 2024-03-02T23:38:06Z: Loading plugins...
2024-03-02T23:38:10.416Z [container-agent] verbose: 2024-03-02T23:38:06Z: Connecting to MySQL at: serverconfig@horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local
2024-03-02T23:38:10.416Z [container-agent] Traceback (most recent call last):
2024-03-02T23:38:10.416Z [container-agent]   File "<string>", line 1, in <module>
2024-03-02T23:38:10.416Z [container-agent] mysqlsh.DBError: MySQL Error (2013): Shell.connect: Lost connection to MySQL server at 'reading initial communication packet', system error: 104
2024-03-02T23:38:10.416Z [container-agent] 
2024-03-02T23:38:12.680Z [container-agent] 2024-03-02 23:38:12 ERROR juju-log Failed to flush GENERAL LOGS logs.
2024-03-02T23:38:12.680Z [container-agent] Traceback (most recent call last):
2024-03-02T23:38:12.680Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 666, in _run_mysqlsh_script
2024-03-02T23:38:12.680Z [container-agent]     stdout, _ = process.wait_output()
2024-03-02T23:38:12.680Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1441, in wait_output
2024-03-02T23:38:12.680Z [container-agent]     raise ExecError[AnyStr](self._command, exit_code, out_value, err_value)
2024-03-02T23:38:12.680Z [container-agent] ops.pebble.ExecError: non-zero exit code 1 executing ['/usr/bin/mysqlsh', '--no-wizard', '--python', '--verbose=1', '-f', '/tmp/script.py', ';', 'rm', '/tmp/script.py'], stdout='', stderr='Cannot set LC_ALL to locale en_US.UTF-8: No such file or directory\nverbose: 2024-03-02T23:38:11Z: Loading startup files...\nverbose: 2024-03-02T23:38:11Z: Loading plugins...\nverbose: 2024-03-02T23:38:11Z: Connecting to MySQL at: serverconfig@horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local\nTraceback (most recent call last):\n  File "<string>", line 1, in <module>\nmysqlsh.DBError: MySQL Error (2003): Shell.connect: Can\'t connect to MySQL server on \'horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local:3306\' (111)\n'
2024-03-02T23:38:12.680Z [container-agent] 
2024-03-02T23:38:12.680Z [container-agent] During handling of the above exception, another exception occurred:
2024-03-02T23:38:12.680Z [container-agent] 
2024-03-02T23:38:12.680Z [container-agent] Traceback (most recent call last):
2024-03-02T23:38:12.680Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-03-02T23:38:12.680Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-03-02T23:38:12.680Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 669, in _run_mysqlsh_script
2024-03-02T23:38:12.680Z [container-agent]     raise MySQLClientError(e.stderr)
2024-03-02T23:38:12.680Z [container-agent] charms.mysql.v0.mysql.MySQLClientError: Cannot set LC_ALL to locale en_US.UTF-8: No such file or directory
2024-03-02T23:38:12.680Z [container-agent] verbose: 2024-03-02T23:38:11Z: Loading startup files...
2024-03-02T23:38:12.680Z [container-agent] verbose: 2024-03-02T23:38:11Z: Loading plugins...
2024-03-02T23:38:12.680Z [container-agent] verbose: 2024-03-02T23:38:11Z: Connecting to MySQL at: serverconfig@horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local
2024-03-02T23:38:12.680Z [container-agent] Traceback (most recent call last):
2024-03-02T23:38:12.680Z [container-agent]   File "<string>", line 1, in <module>
2024-03-02T23:38:12.680Z [container-agent] mysqlsh.DBError: MySQL Error (2003): Shell.connect: Can't connect to MySQL server on 'horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local:3306' (111)
2024-03-02T23:38:12.680Z [container-agent] 
2024-03-02T23:38:14.285Z [container-agent] 2024-03-02 23:38:14 ERROR juju-log Uncaught exception while in charm code:
2024-03-02T23:38:14.285Z [container-agent] Traceback (most recent call last):
2024-03-02T23:38:14.285Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/./src/charm.py", line 786, in <module>
2024-03-02T23:38:14.285Z [container-agent]     main(MySQLOperatorCharm)
2024-03-02T23:38:14.285Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 436, in main
2024-03-02T23:38:14.285Z [container-agent]     _emit_charm_event(charm, dispatcher.event_name)
2024-03-02T23:38:14.285Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 144, in _emit_charm_event
2024-03-02T23:38:14.285Z [container-agent]     event_to_emit.emit(*args, **kwargs)
2024-03-02T23:38:14.285Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 351, in emit
2024-03-02T23:38:14.285Z [container-agent]     framework._emit(event)
2024-03-02T23:38:14.285Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 853, in _emit
2024-03-02T23:38:14.285Z [container-agent]     self._reemit(event_path)
2024-03-02T23:38:14.285Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 942, in _reemit
2024-03-02T23:38:14.285Z [container-agent]     custom_handler(event)
2024-03-02T23:38:14.285Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/rotate_mysql_logs.py", line 57, in _rotate_mysql_logs
2024-03-02T23:38:14.285Z [container-agent]     self.charm._mysql.flush_mysql_logs(MySQLTextLogs.SLOW)
2024-03-02T23:38:14.285Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-03-02T23:38:14.285Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-03-02T23:38:14.285Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 665, in _run_mysqlsh_script
2024-03-02T23:38:14.285Z [container-agent]     process = self.container.exec(cmd, timeout=timeout)
2024-03-02T23:38:14.285Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/model.py", line 2688, in exec
2024-03-02T23:38:14.285Z [container-agent]     return self._pebble.exec(
2024-03-02T23:38:14.285Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 2508, in exec
2024-03-02T23:38:14.285Z [container-agent]     stdio_ws = self._connect_websocket(task_id, 'stdio')
2024-03-02T23:38:14.285Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 2586, in _connect_websocket
2024-03-02T23:38:14.285Z [container-agent]     sock.connect(self.socket_path)
2024-03-02T23:38:14.285Z [container-agent] FileNotFoundError: [Errno 2] No such file or directory
2024-03-02T23:38:14.437Z [container-agent] 2024-03-02 23:38:14 INFO juju.util.exec exec.go:209 run result: exit status 1
2024-03-02T23:38:17.602Z [container-agent] 2024-03-02 23:38:17 INFO juju-log Starting the log rotate manager
2024-03-02T23:38:17.616Z [container-agent] 2024-03-02 23:38:17 INFO juju-log Started log rotate manager process with PID 617868
2024-03-02T23:38:17.871Z [container-agent] 2024-03-02 23:38:17 INFO juju-log Setting up the logrotate configurations
2024-03-02T23:38:17.949Z [container-agent] 2024-03-02 23:38:17 INFO juju-log Adding pebble layer
2024-03-02T23:38:24.693Z [container-agent] 2024-03-02 23:38:24 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-03-02T23:38:24.723Z [container-agent] 2024-03-02 23:38:24 INFO juju-log Attempting reboot from complete outage.
2024-03-02T23:38:27.697Z [container-agent] 2024-03-02 23:38:27 INFO juju.worker.uniter.operation runhook.go:186 ran "mysql-pebble-ready" hook (via hook dispatching script: dispatch)
2024-03-02T23:40:26.126Z [container-agent] 2024-03-02 23:40:26 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T23:40:28.217Z [container-agent] 2024-03-02 23:40:28 WARNING juju-log No relation: certificates
2024-03-02T23:40:28.561Z [container-agent] 2024-03-02 23:40:28 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T23:46:22.304Z [container-agent] 2024-03-02 23:46:22 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T23:46:24.945Z [container-agent] 2024-03-02 23:46:24 WARNING juju-log No relation: certificates
2024-03-02T23:46:25.290Z [container-agent] 2024-03-02 23:46:25 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T23:50:34.459Z [container-agent] 2024-03-02 23:50:34 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T23:50:37.393Z [container-agent] 2024-03-02 23:50:37 WARNING juju-log No relation: certificates
2024-03-02T23:50:37.735Z [container-agent] 2024-03-02 23:50:37 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-02T23:54:52.130Z [container-agent] 2024-03-02 23:54:52 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-02T23:54:55.616Z [container-agent] 2024-03-02 23:54:55 WARNING juju-log No relation: certificates
2024-03-02T23:54:55.986Z [container-agent] 2024-03-02 23:54:55 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T00:00:10.795Z [container-agent] 2024-03-03 00:00:10 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T00:00:14.873Z [container-agent] 2024-03-03 00:00:14 WARNING juju-log No relation: certificates
2024-03-03T00:00:15.243Z [container-agent] 2024-03-03 00:00:15 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T00:05:30.638Z [container-agent] 2024-03-03 00:05:30 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T00:05:35.363Z [container-agent] 2024-03-03 00:05:35 WARNING juju-log No relation: certificates
2024-03-03T00:05:35.728Z [container-agent] 2024-03-03 00:05:35 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T00:09:31.267Z [container-agent] 2024-03-03 00:09:31 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T00:09:36.171Z [container-agent] 2024-03-03 00:09:36 WARNING juju-log No relation: certificates
2024-03-03T00:09:36.506Z [container-agent] 2024-03-03 00:09:36 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T00:14:27.025Z [container-agent] 2024-03-03 00:14:27 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T00:14:32.284Z [container-agent] 2024-03-03 00:14:32 WARNING juju-log No relation: certificates
2024-03-03T00:14:32.632Z [container-agent] 2024-03-03 00:14:32 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T00:20:13.776Z [container-agent] 2024-03-03 00:20:13 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T00:20:19.552Z [container-agent] 2024-03-03 00:20:19 WARNING juju-log No relation: certificates
2024-03-03T00:20:19.919Z [container-agent] 2024-03-03 00:20:19 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T00:25:46.077Z [container-agent] 2024-03-03 00:25:46 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T00:25:52.483Z [container-agent] 2024-03-03 00:25:52 WARNING juju-log No relation: certificates
2024-03-03T00:25:52.830Z [container-agent] 2024-03-03 00:25:52 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T00:30:41.731Z [container-agent] 2024-03-03 00:30:41 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T00:30:48.543Z [container-agent] 2024-03-03 00:30:48 WARNING juju-log No relation: certificates
2024-03-03T00:30:48.909Z [container-agent] 2024-03-03 00:30:48 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T00:34:45.096Z [container-agent] 2024-03-03 00:34:45 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T00:34:52.275Z [container-agent] 2024-03-03 00:34:52 WARNING juju-log No relation: certificates
2024-03-03T00:34:52.658Z [container-agent] 2024-03-03 00:34:52 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T00:39:13.376Z [container-agent] 2024-03-03 00:39:13 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T00:39:21.294Z [container-agent] 2024-03-03 00:39:21 WARNING juju-log No relation: certificates
2024-03-03T00:39:21.635Z [container-agent] 2024-03-03 00:39:21 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T00:44:34.299Z [container-agent] 2024-03-03 00:44:34 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T00:44:42.381Z [container-agent] 2024-03-03 00:44:42 WARNING juju-log No relation: certificates
2024-03-03T00:44:42.733Z [container-agent] 2024-03-03 00:44:42 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T00:50:22.110Z [container-agent] 2024-03-03 00:50:22 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T00:50:29.511Z [container-agent] 2024-03-03 00:50:29 WARNING juju-log Failed to get cluster primary addresses
2024-03-03T00:50:29.511Z [container-agent] Traceback (most recent call last):
2024-03-03T00:50:29.511Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 666, in _run_mysqlsh_script
2024-03-03T00:50:29.511Z [container-agent]     stdout, _ = process.wait_output()
2024-03-03T00:50:29.511Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1441, in wait_output
2024-03-03T00:50:29.511Z [container-agent]     raise ExecError[AnyStr](self._command, exit_code, out_value, err_value)
2024-03-03T00:50:29.511Z [container-agent] ops.pebble.ExecError: non-zero exit code 1 executing ['/usr/bin/mysqlsh', '--no-wizard', '--python', '--verbose=1', '-f', '/tmp/script.py', ';', 'rm', '/tmp/script.py'], stdout='', stderr='Cannot set LC_ALL to locale en_US.UTF-8: No such file or directory\nverbose: 2024-03-03T00:50:27Z: Loading startup files...\nverbose: 2024-03-03T00:50:27Z: Loading plugins...\nverbose: 2024-03-03T00:50:27Z: Connecting to MySQL at: clusteradmin@horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local\nverbose: 2024-03-03T00:50:28Z: Shell.connect_to_primary: tid=21: CONNECTED: horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local\nverbose: 2024-03-03T00:50:28Z: Redirecting session from \'mysqlx://clusteradmin@horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local:33060\' to a PRIMARY of an InnoDB cluster or ReplicaSet...\nTraceback (most recent call last):\n  File "<string>", line 1, in <module>\nRuntimeError: Shell.connect_to_primary: No primaries found in an InnoDB cluster\n\n'
2024-03-03T00:50:29.511Z [container-agent] 
2024-03-03T00:50:29.511Z [container-agent] During handling of the above exception, another exception occurred:
2024-03-03T00:50:29.511Z [container-agent] 
2024-03-03T00:50:29.511Z [container-agent] Traceback (most recent call last):
2024-03-03T00:50:29.511Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 1767, in get_cluster_primary_address
2024-03-03T00:50:29.511Z [container-agent]     output = self._run_mysqlsh_script("\n".join(get_cluster_primary_commands))
2024-03-03T00:50:29.511Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 669, in _run_mysqlsh_script
2024-03-03T00:50:29.511Z [container-agent]     raise MySQLClientError(e.stderr)
2024-03-03T00:50:29.511Z [container-agent] charms.mysql.v0.mysql.MySQLClientError: Cannot set LC_ALL to locale en_US.UTF-8: No such file or directory
2024-03-03T00:50:29.511Z [container-agent] verbose: 2024-03-03T00:50:27Z: Loading startup files...
2024-03-03T00:50:29.511Z [container-agent] verbose: 2024-03-03T00:50:27Z: Loading plugins...
2024-03-03T00:50:29.511Z [container-agent] verbose: 2024-03-03T00:50:27Z: Connecting to MySQL at: clusteradmin@horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local
2024-03-03T00:50:29.511Z [container-agent] verbose: 2024-03-03T00:50:28Z: Shell.connect_to_primary: tid=21: CONNECTED: horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local
2024-03-03T00:50:29.511Z [container-agent] verbose: 2024-03-03T00:50:28Z: Redirecting session from 'mysqlx://clusteradmin@horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local:33060' to a PRIMARY of an InnoDB cluster or ReplicaSet...
2024-03-03T00:50:29.511Z [container-agent] Traceback (most recent call last):
2024-03-03T00:50:29.511Z [container-agent]   File "<string>", line 1, in <module>
2024-03-03T00:50:29.511Z [container-agent] RuntimeError: Shell.connect_to_primary: No primaries found in an InnoDB cluster
2024-03-03T00:50:29.511Z [container-agent] 
2024-03-03T00:50:29.511Z [container-agent] 
2024-03-03T00:50:32.267Z [container-agent] 2024-03-03 00:50:32 ERROR juju-log Failed to get cluster status for cluster-7573647c1e26466beeeac32b95ab8f91
2024-03-03T00:50:32.354Z [container-agent] 2024-03-03 00:50:32 INFO juju-log Kubernetes pod label error created
2024-03-03T00:50:32.368Z [container-agent] 2024-03-03 00:50:32 WARNING juju-log No relation: certificates
2024-03-03T00:50:32.717Z [container-agent] 2024-03-03 00:50:32 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T00:54:31.372Z [container-agent] 2024-03-03 00:54:31 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-03-03T00:54:31.400Z [container-agent] 2024-03-03 00:54:31 INFO juju-log Attempting reboot from complete outage.
2024-03-03T00:54:39.649Z [container-agent] 2024-03-03 00:54:39 INFO juju-log Kubernetes pod label primary created
2024-03-03T00:54:39.666Z [container-agent] 2024-03-03 00:54:39 WARNING juju-log No relation: certificates
2024-03-03T00:54:40.022Z [container-agent] 2024-03-03 00:54:40 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T00:55:31.118Z [container-agent] 2024-03-03 00:55:31 INFO juju-log Setting up the logrotate configurations
2024-03-03T00:55:31.199Z [container-agent] 2024-03-03 00:55:31 INFO juju-log Adding pebble layer
2024-03-03T00:55:37.928Z [container-agent] 2024-03-03 00:55:37 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-03-03T00:55:37.952Z [container-agent] 2024-03-03 00:55:37 INFO juju-log Attempting reboot from complete outage.
2024-03-03T00:55:40.845Z [container-agent] 2024-03-03 00:55:40 INFO juju.worker.uniter.operation runhook.go:186 ran "mysql-pebble-ready" hook (via hook dispatching script: dispatch)
2024-03-03T00:59:43.169Z [container-agent] 2024-03-03 00:59:43 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T00:59:45.525Z [container-agent] 2024-03-03 00:59:45 WARNING juju-log No relation: certificates
2024-03-03T00:59:45.888Z [container-agent] 2024-03-03 00:59:45 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T01:04:44.805Z [container-agent] 2024-03-03 01:04:44 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T01:04:47.727Z [container-agent] 2024-03-03 01:04:47 WARNING juju-log No relation: certificates
2024-03-03T01:04:48.090Z [container-agent] 2024-03-03 01:04:48 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T01:10:37.265Z [container-agent] 2024-03-03 01:10:37 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T01:10:40.348Z [container-agent] 2024-03-03 01:10:40 WARNING juju-log No relation: certificates
2024-03-03T01:10:40.711Z [container-agent] 2024-03-03 01:10:40 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T01:15:42.829Z [container-agent] 2024-03-03 01:15:42 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T01:15:46.602Z [container-agent] 2024-03-03 01:15:46 WARNING juju-log No relation: certificates
2024-03-03T01:15:46.944Z [container-agent] 2024-03-03 01:15:46 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T01:21:22.005Z [container-agent] 2024-03-03 01:21:22 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T01:21:26.111Z [container-agent] 2024-03-03 01:21:26 WARNING juju-log No relation: certificates
2024-03-03T01:21:26.448Z [container-agent] 2024-03-03 01:21:26 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T01:25:32.102Z [container-agent] 2024-03-03 01:25:32 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T01:25:36.873Z [container-agent] 2024-03-03 01:25:36 WARNING juju-log No relation: certificates
2024-03-03T01:25:37.255Z [container-agent] 2024-03-03 01:25:37 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T01:30:09.618Z [container-agent] 2024-03-03 01:30:09 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T01:30:14.430Z [container-agent] 2024-03-03 01:30:14 WARNING juju-log No relation: certificates
2024-03-03T01:30:14.777Z [container-agent] 2024-03-03 01:30:14 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T01:34:53.655Z [container-agent] 2024-03-03 01:34:53 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T01:34:59.065Z [container-agent] 2024-03-03 01:34:59 WARNING juju-log No relation: certificates
2024-03-03T01:34:59.429Z [container-agent] 2024-03-03 01:34:59 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T01:40:01.773Z [container-agent] 2024-03-03 01:40:01 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T01:40:07.517Z [container-agent] 2024-03-03 01:40:07 WARNING juju-log No relation: certificates
2024-03-03T01:40:07.902Z [container-agent] 2024-03-03 01:40:07 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T01:45:38.272Z [container-agent] 2024-03-03 01:45:38 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T01:45:44.430Z [container-agent] 2024-03-03 01:45:44 WARNING juju-log No relation: certificates
2024-03-03T01:45:44.820Z [container-agent] 2024-03-03 01:45:44 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T01:49:55.757Z [container-agent] 2024-03-03 01:49:55 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T01:50:02.449Z [container-agent] 2024-03-03 01:50:02 WARNING juju-log No relation: certificates
2024-03-03T01:50:02.790Z [container-agent] 2024-03-03 01:50:02 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T01:54:57.176Z [container-agent] 2024-03-03 01:54:57 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T01:55:04.118Z [container-agent] 2024-03-03 01:55:04 WARNING juju-log No relation: certificates
2024-03-03T01:55:04.473Z [container-agent] 2024-03-03 01:55:04 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T01:59:25.375Z [container-agent] 2024-03-03 01:59:25 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T01:59:32.723Z [container-agent] 2024-03-03 01:59:32 WARNING juju-log No relation: certificates
2024-03-03T01:59:33.091Z [container-agent] 2024-03-03 01:59:33 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T02:04:02.029Z [container-agent] 2024-03-03 02:04:02 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-03-03T02:04:02.057Z [container-agent] 2024-03-03 02:04:02 INFO juju-log Attempting reboot from complete outage.
2024-03-03T02:04:08.834Z [container-agent] 2024-03-03 02:04:08 WARNING juju-log No relation: certificates
2024-03-03T02:04:09.186Z [container-agent] 2024-03-03 02:04:09 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T02:08:04.798Z [container-agent] 2024-03-03 02:08:04 ERROR juju-log Uncaught exception while in charm code:
2024-03-03T02:08:04.798Z [container-agent] Traceback (most recent call last):
2024-03-03T02:08:04.798Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/./src/charm.py", line 786, in <module>
2024-03-03T02:08:04.798Z [container-agent]     main(MySQLOperatorCharm)
2024-03-03T02:08:04.798Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 436, in main
2024-03-03T02:08:04.798Z [container-agent]     _emit_charm_event(charm, dispatcher.event_name)
2024-03-03T02:08:04.798Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 144, in _emit_charm_event
2024-03-03T02:08:04.798Z [container-agent]     event_to_emit.emit(*args, **kwargs)
2024-03-03T02:08:04.798Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 351, in emit
2024-03-03T02:08:04.798Z [container-agent]     framework._emit(event)
2024-03-03T02:08:04.798Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 853, in _emit
2024-03-03T02:08:04.798Z [container-agent]     self._reemit(event_path)
2024-03-03T02:08:04.798Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 942, in _reemit
2024-03-03T02:08:04.798Z [container-agent]     custom_handler(event)
2024-03-03T02:08:04.798Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/rotate_mysql_logs.py", line 55, in _rotate_mysql_logs
2024-03-03T02:08:04.798Z [container-agent]     self.charm._mysql.flush_mysql_logs(MySQLTextLogs.ERROR)
2024-03-03T02:08:04.798Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-03-03T02:08:04.798Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-03-03T02:08:04.798Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 666, in _run_mysqlsh_script
2024-03-03T02:08:04.798Z [container-agent]     stdout, _ = process.wait_output()
2024-03-03T02:08:04.798Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1436, in wait_output
2024-03-03T02:08:04.798Z [container-agent]     exit_code: int = self._wait()
2024-03-03T02:08:04.798Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1375, in _wait
2024-03-03T02:08:04.798Z [container-agent]     change = self._client.wait_change(self._change_id, timeout=timeout)
2024-03-03T02:08:04.798Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1893, in wait_change
2024-03-03T02:08:04.798Z [container-agent]     return self._wait_change_using_wait(change_id, timeout)
2024-03-03T02:08:04.798Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1914, in _wait_change_using_wait
2024-03-03T02:08:04.798Z [container-agent]     return self._wait_change(change_id, this_timeout)
2024-03-03T02:08:04.798Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1928, in _wait_change
2024-03-03T02:08:04.798Z [container-agent]     resp = self._request('GET', f'/v1/changes/{change_id}/wait', query)
2024-03-03T02:08:04.798Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1655, in _request
2024-03-03T02:08:04.798Z [container-agent]     response = self._request_raw(method, path, query, headers, data)
2024-03-03T02:08:04.798Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1690, in _request_raw
2024-03-03T02:08:04.798Z [container-agent]     response = self.opener.open(request, timeout=self.timeout)
2024-03-03T02:08:04.798Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 519, in open
2024-03-03T02:08:04.798Z [container-agent]     response = self._open(req, data)
2024-03-03T02:08:04.798Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 536, in _open
2024-03-03T02:08:04.798Z [container-agent]     result = self._call_chain(self.handle_open, protocol, protocol +
2024-03-03T02:08:04.798Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 496, in _call_chain
2024-03-03T02:08:04.798Z [container-agent]     result = func(*args)
2024-03-03T02:08:04.798Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 311, in http_open
2024-03-03T02:08:04.798Z [container-agent]     return self.do_open(_UnixSocketConnection, req,  # type:ignore
2024-03-03T02:08:04.798Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 1352, in do_open
2024-03-03T02:08:04.798Z [container-agent]     r = h.getresponse()
2024-03-03T02:08:04.798Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1375, in getresponse
2024-03-03T02:08:04.798Z [container-agent]     response.begin()
2024-03-03T02:08:04.798Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 318, in begin
2024-03-03T02:08:04.798Z [container-agent]     version, status, reason = self._read_status()
2024-03-03T02:08:04.798Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 287, in _read_status
2024-03-03T02:08:04.798Z [container-agent]     raise RemoteDisconnected("Remote end closed connection without"
2024-03-03T02:08:04.798Z [container-agent] http.client.RemoteDisconnected: Remote end closed connection without response
2024-03-03T02:08:04.924Z [container-agent] 2024-03-03 02:08:04 INFO juju.util.exec exec.go:209 run result: exit status 1
2024-03-03T02:08:08.927Z [container-agent] 2024-03-03 02:08:08 INFO juju-log Setting up the logrotate configurations
2024-03-03T02:08:09.048Z [container-agent] 2024-03-03 02:08:09 INFO juju-log Adding pebble layer
2024-03-03T02:08:15.856Z [container-agent] 2024-03-03 02:08:15 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-03-03T02:08:15.884Z [container-agent] 2024-03-03 02:08:15 INFO juju-log Attempting reboot from complete outage.
2024-03-03T02:08:18.936Z [container-agent] 2024-03-03 02:08:18 INFO juju.worker.uniter.operation runhook.go:186 ran "mysql-pebble-ready" hook (via hook dispatching script: dispatch)
2024-03-03T02:08:20.585Z [container-agent] 2024-03-03 02:08:20 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T02:08:22.699Z [container-agent] 2024-03-03 02:08:22 WARNING juju-log No relation: certificates
2024-03-03T02:08:23.085Z [container-agent] 2024-03-03 02:08:23 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T02:13:39.833Z [container-agent] 2024-03-03 02:13:39 INFO juju-log Starting the log rotate manager
2024-03-03T02:13:39.848Z [container-agent] 2024-03-03 02:13:39 INFO juju-log Started log rotate manager process with PID 632978
2024-03-03T02:13:40.122Z [container-agent] 2024-03-03 02:13:40 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T02:13:42.092Z [container-agent] 2024-03-03 02:13:42 WARNING juju-log No relation: certificates
2024-03-03T02:13:42.440Z [container-agent] 2024-03-03 02:13:42 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T02:18:56.755Z [container-agent] 2024-03-03 02:18:56 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T02:18:59.299Z [container-agent] 2024-03-03 02:18:59 WARNING juju-log No relation: certificates
2024-03-03T02:18:59.693Z [container-agent] 2024-03-03 02:18:59 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T02:24:49.942Z [container-agent] 2024-03-03 02:24:49 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T02:24:52.965Z [container-agent] 2024-03-03 02:24:52 WARNING juju-log No relation: certificates
2024-03-03T02:24:53.306Z [container-agent] 2024-03-03 02:24:53 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T02:30:01.456Z [container-agent] 2024-03-03 02:30:01 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T02:30:05.032Z [container-agent] 2024-03-03 02:30:05 WARNING juju-log No relation: certificates
2024-03-03T02:30:05.406Z [container-agent] 2024-03-03 02:30:05 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T02:35:31.349Z [container-agent] 2024-03-03 02:35:31 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T02:35:35.546Z [container-agent] 2024-03-03 02:35:35 WARNING juju-log No relation: certificates
2024-03-03T02:35:35.944Z [container-agent] 2024-03-03 02:35:35 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T02:39:54.709Z [container-agent] 2024-03-03 02:39:54 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T02:39:59.236Z [container-agent] 2024-03-03 02:39:59 WARNING juju-log No relation: certificates
2024-03-03T02:39:59.598Z [container-agent] 2024-03-03 02:39:59 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T02:45:27.417Z [container-agent] 2024-03-03 02:45:27 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T02:45:32.557Z [container-agent] 2024-03-03 02:45:32 WARNING juju-log No relation: certificates
2024-03-03T02:45:32.935Z [container-agent] 2024-03-03 02:45:32 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T02:50:10.066Z [container-agent] 2024-03-03 02:50:10 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T02:50:15.643Z [container-agent] 2024-03-03 02:50:15 WARNING juju-log No relation: certificates
2024-03-03T02:50:15.997Z [container-agent] 2024-03-03 02:50:15 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T02:54:54.915Z [container-agent] 2024-03-03 02:54:54 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T02:55:00.921Z [container-agent] 2024-03-03 02:55:00 WARNING juju-log No relation: certificates
2024-03-03T02:55:01.263Z [container-agent] 2024-03-03 02:55:01 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T02:58:58.935Z [container-agent] 2024-03-03 02:58:58 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T02:59:05.085Z [container-agent] 2024-03-03 02:59:05 WARNING juju-log No relation: certificates
2024-03-03T02:59:05.492Z [container-agent] 2024-03-03 02:59:05 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T03:03:00.952Z [container-agent] 2024-03-03 03:03:00 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T03:03:07.026Z [container-agent] 2024-03-03 03:03:07 WARNING juju-log No relation: certificates
2024-03-03T03:03:07.416Z [container-agent] 2024-03-03 03:03:07 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T03:08:12.901Z [container-agent] 2024-03-03 03:08:12 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T03:08:20.086Z [container-agent] 2024-03-03 03:08:20 WARNING juju-log No relation: certificates
2024-03-03T03:08:20.468Z [container-agent] 2024-03-03 03:08:20 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T03:13:13.299Z [container-agent] 2024-03-03 03:13:13 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T03:13:21.059Z [container-agent] 2024-03-03 03:13:21 WARNING juju-log No relation: certificates
2024-03-03T03:13:21.425Z [container-agent] 2024-03-03 03:13:21 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T03:17:27.248Z [container-agent] 2024-03-03 03:17:27 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T03:17:34.978Z [container-agent] 2024-03-03 03:17:34 WARNING juju-log No relation: certificates
2024-03-03T03:17:35.351Z [container-agent] 2024-03-03 03:17:35 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T03:22:14.807Z [container-agent] 2024-03-03 03:22:14 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T03:22:22.366Z [container-agent] 2024-03-03 03:22:22 WARNING juju-log No relation: certificates
2024-03-03T03:22:22.794Z [container-agent] 2024-03-03 03:22:22 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T03:28:15.103Z [container-agent] 2024-03-03 03:28:15 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T03:28:23.759Z [container-agent] 2024-03-03 03:28:23 WARNING juju-log No relation: certificates
2024-03-03T03:28:24.097Z [container-agent] 2024-03-03 03:28:24 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T03:30:07.780Z [container-agent] 2024-03-03 03:30:07 ERROR juju-log Failed to flush ERROR LOGS logs.
2024-03-03T03:30:07.780Z [container-agent] Traceback (most recent call last):
2024-03-03T03:30:07.780Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 666, in _run_mysqlsh_script
2024-03-03T03:30:07.780Z [container-agent]     stdout, _ = process.wait_output()
2024-03-03T03:30:07.780Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1441, in wait_output
2024-03-03T03:30:07.780Z [container-agent]     raise ExecError[AnyStr](self._command, exit_code, out_value, err_value)
2024-03-03T03:30:07.780Z [container-agent] ops.pebble.ExecError: non-zero exit code 1 executing ['/usr/bin/mysqlsh', '--no-wizard', '--python', '--verbose=1', '-f', '/tmp/script.py', ';', 'rm', '/tmp/script.py'], stdout='', stderr='Cannot set LC_ALL to locale en_US.UTF-8: No such file or directory\nverbose: 2024-03-03T03:30:06Z: Loading startup files...\nverbose: 2024-03-03T03:30:06Z: Loading plugins...\nverbose: 2024-03-03T03:30:06Z: Connecting to MySQL at: serverconfig@horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local\nTraceback (most recent call last):\n  File "<string>", line 1, in <module>\nmysqlsh.DBError: MySQL Error (2003): Shell.connect: Can\'t connect to MySQL server on \'horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local:3306\' (111)\n'
2024-03-03T03:30:07.780Z [container-agent] 
2024-03-03T03:30:07.780Z [container-agent] During handling of the above exception, another exception occurred:
2024-03-03T03:30:07.780Z [container-agent] 
2024-03-03T03:30:07.780Z [container-agent] Traceback (most recent call last):
2024-03-03T03:30:07.780Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-03-03T03:30:07.780Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-03-03T03:30:07.780Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 669, in _run_mysqlsh_script
2024-03-03T03:30:07.780Z [container-agent]     raise MySQLClientError(e.stderr)
2024-03-03T03:30:07.780Z [container-agent] charms.mysql.v0.mysql.MySQLClientError: Cannot set LC_ALL to locale en_US.UTF-8: No such file or directory
2024-03-03T03:30:07.780Z [container-agent] verbose: 2024-03-03T03:30:06Z: Loading startup files...
2024-03-03T03:30:07.780Z [container-agent] verbose: 2024-03-03T03:30:06Z: Loading plugins...
2024-03-03T03:30:07.780Z [container-agent] verbose: 2024-03-03T03:30:06Z: Connecting to MySQL at: serverconfig@horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local
2024-03-03T03:30:07.780Z [container-agent] Traceback (most recent call last):
2024-03-03T03:30:07.780Z [container-agent]   File "<string>", line 1, in <module>
2024-03-03T03:30:07.780Z [container-agent] mysqlsh.DBError: MySQL Error (2003): Shell.connect: Can't connect to MySQL server on 'horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local:3306' (111)
2024-03-03T03:30:07.780Z [container-agent] 
2024-03-03T03:33:19.941Z [container-agent] 2024-03-03 03:33:19 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-03-03T03:33:19.972Z [container-agent] 2024-03-03 03:33:19 INFO juju-log Attempting reboot from complete outage.
2024-03-03T03:33:28.279Z [container-agent] 2024-03-03 03:33:28 WARNING juju-log No relation: certificates
2024-03-03T03:33:28.643Z [container-agent] 2024-03-03 03:33:28 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T03:35:04.805Z [container-agent] 2024-03-03 03:35:04 ERROR juju-log Uncaught exception while in charm code:
2024-03-03T03:35:04.805Z [container-agent] Traceback (most recent call last):
2024-03-03T03:35:04.805Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/./src/charm.py", line 786, in <module>
2024-03-03T03:35:04.805Z [container-agent]     main(MySQLOperatorCharm)
2024-03-03T03:35:04.805Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 436, in main
2024-03-03T03:35:04.805Z [container-agent]     _emit_charm_event(charm, dispatcher.event_name)
2024-03-03T03:35:04.805Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 144, in _emit_charm_event
2024-03-03T03:35:04.805Z [container-agent]     event_to_emit.emit(*args, **kwargs)
2024-03-03T03:35:04.805Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 351, in emit
2024-03-03T03:35:04.805Z [container-agent]     framework._emit(event)
2024-03-03T03:35:04.805Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 853, in _emit
2024-03-03T03:35:04.805Z [container-agent]     self._reemit(event_path)
2024-03-03T03:35:04.805Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 942, in _reemit
2024-03-03T03:35:04.805Z [container-agent]     custom_handler(event)
2024-03-03T03:35:04.805Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/rotate_mysql_logs.py", line 55, in _rotate_mysql_logs
2024-03-03T03:35:04.805Z [container-agent]     self.charm._mysql.flush_mysql_logs(MySQLTextLogs.ERROR)
2024-03-03T03:35:04.805Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-03-03T03:35:04.805Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-03-03T03:35:04.805Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 665, in _run_mysqlsh_script
2024-03-03T03:35:04.805Z [container-agent]     process = self.container.exec(cmd, timeout=timeout)
2024-03-03T03:35:04.805Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/model.py", line 2688, in exec
2024-03-03T03:35:04.805Z [container-agent]     return self._pebble.exec(
2024-03-03T03:35:04.805Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 2501, in exec
2024-03-03T03:35:04.805Z [container-agent]     resp = self._request('POST', '/v1/exec', body=body)
2024-03-03T03:35:04.805Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1655, in _request
2024-03-03T03:35:04.805Z [container-agent]     response = self._request_raw(method, path, query, headers, data)
2024-03-03T03:35:04.805Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1690, in _request_raw
2024-03-03T03:35:04.805Z [container-agent]     response = self.opener.open(request, timeout=self.timeout)
2024-03-03T03:35:04.805Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 519, in open
2024-03-03T03:35:04.805Z [container-agent]     response = self._open(req, data)
2024-03-03T03:35:04.805Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 536, in _open
2024-03-03T03:35:04.805Z [container-agent]     result = self._call_chain(self.handle_open, protocol, protocol +
2024-03-03T03:35:04.805Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 496, in _call_chain
2024-03-03T03:35:04.805Z [container-agent]     result = func(*args)
2024-03-03T03:35:04.805Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 311, in http_open
2024-03-03T03:35:04.805Z [container-agent]     return self.do_open(_UnixSocketConnection, req,  # type:ignore
2024-03-03T03:35:04.805Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 1352, in do_open
2024-03-03T03:35:04.805Z [container-agent]     r = h.getresponse()
2024-03-03T03:35:04.805Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1375, in getresponse
2024-03-03T03:35:04.805Z [container-agent]     response.begin()
2024-03-03T03:35:04.805Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 318, in begin
2024-03-03T03:35:04.805Z [container-agent]     version, status, reason = self._read_status()
2024-03-03T03:35:04.805Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 287, in _read_status
2024-03-03T03:35:04.805Z [container-agent]     raise RemoteDisconnected("Remote end closed connection without"
2024-03-03T03:35:04.805Z [container-agent] http.client.RemoteDisconnected: Remote end closed connection without response
2024-03-03T03:35:04.961Z [container-agent] 2024-03-03 03:35:04 INFO juju.util.exec exec.go:209 run result: exit status 1
2024-03-03T03:35:07.648Z [container-agent] 2024-03-03 03:35:07 INFO juju-log Setting up the logrotate configurations
2024-03-03T03:35:07.721Z [container-agent] 2024-03-03 03:35:07 INFO juju-log Adding pebble layer
2024-03-03T03:35:14.443Z [container-agent] 2024-03-03 03:35:14 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-03-03T03:35:14.466Z [container-agent] 2024-03-03 03:35:14 INFO juju-log Attempting reboot from complete outage.
2024-03-03T03:35:17.364Z [container-agent] 2024-03-03 03:35:17 INFO juju.worker.uniter.operation runhook.go:186 ran "mysql-pebble-ready" hook (via hook dispatching script: dispatch)
2024-03-03T03:37:28.309Z [container-agent] 2024-03-03 03:37:28 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T03:37:30.458Z [container-agent] 2024-03-03 03:37:30 WARNING juju-log No relation: certificates
2024-03-03T03:37:30.837Z [container-agent] 2024-03-03 03:37:30 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T03:42:40.085Z [container-agent] 2024-03-03 03:42:40 INFO juju-log Starting the log rotate manager
2024-03-03T03:42:40.097Z [container-agent] 2024-03-03 03:42:40 INFO juju-log Started log rotate manager process with PID 641312
2024-03-03T03:42:40.514Z [container-agent] 2024-03-03 03:42:40 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T03:42:42.574Z [container-agent] 2024-03-03 03:42:42 WARNING juju-log No relation: certificates
2024-03-03T03:42:42.956Z [container-agent] 2024-03-03 03:42:42 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T03:48:37.810Z [container-agent] 2024-03-03 03:48:37 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T03:48:40.517Z [container-agent] 2024-03-03 03:48:40 WARNING juju-log No relation: certificates
2024-03-03T03:48:40.890Z [container-agent] 2024-03-03 03:48:40 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T03:52:46.192Z [container-agent] 2024-03-03 03:52:46 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T03:52:49.319Z [container-agent] 2024-03-03 03:52:49 WARNING juju-log No relation: certificates
2024-03-03T03:52:49.677Z [container-agent] 2024-03-03 03:52:49 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T03:56:54.372Z [container-agent] 2024-03-03 03:56:54 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T03:56:57.350Z [container-agent] 2024-03-03 03:56:57 WARNING juju-log No relation: certificates
2024-03-03T03:56:57.700Z [container-agent] 2024-03-03 03:56:57 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T04:02:22.514Z [container-agent] 2024-03-03 04:02:22 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T04:02:26.473Z [container-agent] 2024-03-03 04:02:26 WARNING juju-log No relation: certificates
2024-03-03T04:02:26.829Z [container-agent] 2024-03-03 04:02:26 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T04:06:23.233Z [container-agent] 2024-03-03 04:06:23 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T04:06:27.679Z [container-agent] 2024-03-03 04:06:27 WARNING juju-log No relation: certificates
2024-03-03T04:06:28.026Z [container-agent] 2024-03-03 04:06:28 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T04:10:47.281Z [container-agent] 2024-03-03 04:10:47 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T04:10:51.891Z [container-agent] 2024-03-03 04:10:51 WARNING juju-log No relation: certificates
2024-03-03T04:10:52.251Z [container-agent] 2024-03-03 04:10:52 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T04:15:35.297Z [container-agent] 2024-03-03 04:15:35 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T04:15:40.111Z [container-agent] 2024-03-03 04:15:40 WARNING juju-log No relation: certificates
2024-03-03T04:15:40.473Z [container-agent] 2024-03-03 04:15:40 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T04:20:01.882Z [container-agent] 2024-03-03 04:20:01 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T04:20:07.284Z [container-agent] 2024-03-03 04:20:07 WARNING juju-log No relation: certificates
2024-03-03T04:20:07.629Z [container-agent] 2024-03-03 04:20:07 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T04:25:23.504Z [container-agent] 2024-03-03 04:25:23 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T04:25:29.400Z [container-agent] 2024-03-03 04:25:29 WARNING juju-log No relation: certificates
2024-03-03T04:25:29.752Z [container-agent] 2024-03-03 04:25:29 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T04:30:35.669Z [container-agent] 2024-03-03 04:30:35 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T04:30:42.447Z [container-agent] 2024-03-03 04:30:42 WARNING juju-log No relation: certificates
2024-03-03T04:30:42.813Z [container-agent] 2024-03-03 04:30:42 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T04:35:02.381Z [container-agent] 2024-03-03 04:35:02 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T04:35:08.968Z [container-agent] 2024-03-03 04:35:08 WARNING juju-log No relation: certificates
2024-03-03T04:35:09.299Z [container-agent] 2024-03-03 04:35:09 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T04:39:13.708Z [container-agent] 2024-03-03 04:39:13 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T04:39:20.999Z [container-agent] 2024-03-03 04:39:20 WARNING juju-log No relation: certificates
2024-03-03T04:39:21.350Z [container-agent] 2024-03-03 04:39:21 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T04:43:36.198Z [container-agent] 2024-03-03 04:43:36 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T04:43:44.097Z [container-agent] 2024-03-03 04:43:44 WARNING juju-log No relation: certificates
2024-03-03T04:43:44.445Z [container-agent] 2024-03-03 04:43:44 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T04:49:18.878Z [container-agent] 2024-03-03 04:49:18 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T04:49:26.647Z [container-agent] 2024-03-03 04:49:26 WARNING juju-log No relation: certificates
2024-03-03T04:49:26.972Z [container-agent] 2024-03-03 04:49:26 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T04:54:24.200Z [container-agent] 2024-03-03 04:54:24 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T04:54:33.174Z [container-agent] 2024-03-03 04:54:33 WARNING juju-log No relation: certificates
2024-03-03T04:54:33.552Z [container-agent] 2024-03-03 04:54:33 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T05:00:15.896Z [container-agent] 2024-03-03 05:00:15 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T05:00:25.106Z [container-agent] 2024-03-03 05:00:25 WARNING juju-log No relation: certificates
2024-03-03T05:00:25.485Z [container-agent] 2024-03-03 05:00:25 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T05:04:29.729Z [container-agent] 2024-03-03 05:04:29 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T05:04:37.635Z [container-agent] 2024-03-03 05:04:37 WARNING juju-log Failed to get cluster primary addresses
2024-03-03T05:04:37.635Z [container-agent] Traceback (most recent call last):
2024-03-03T05:04:37.635Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 666, in _run_mysqlsh_script
2024-03-03T05:04:37.635Z [container-agent]     stdout, _ = process.wait_output()
2024-03-03T05:04:37.635Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1441, in wait_output
2024-03-03T05:04:37.635Z [container-agent]     raise ExecError[AnyStr](self._command, exit_code, out_value, err_value)
2024-03-03T05:04:37.635Z [container-agent] ops.pebble.ExecError: non-zero exit code 1 executing ['/usr/bin/mysqlsh', '--no-wizard', '--python', '--verbose=1', '-f', '/tmp/script.py', ';', 'rm', '/tmp/script.py'], stdout='', stderr='Cannot set LC_ALL to locale en_US.UTF-8: No such file or directory\nverbose: 2024-03-03T05:04:36Z: Loading startup files...\nverbose: 2024-03-03T05:04:36Z: Loading plugins...\nverbose: 2024-03-03T05:04:36Z: Connecting to MySQL at: clusteradmin@horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local\nverbose: 2024-03-03T05:04:36Z: Shell.connect_to_primary: tid=22: CONNECTED: horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local\nverbose: 2024-03-03T05:04:36Z: Redirecting session from \'mysqlx://clusteradmin@horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local:33060\' to a PRIMARY of an InnoDB cluster or ReplicaSet...\nTraceback (most recent call last):\n  File "<string>", line 1, in <module>\nRuntimeError: Shell.connect_to_primary: No primaries found in an InnoDB cluster\n\n'
2024-03-03T05:04:37.635Z [container-agent] 
2024-03-03T05:04:37.635Z [container-agent] During handling of the above exception, another exception occurred:
2024-03-03T05:04:37.635Z [container-agent] 
2024-03-03T05:04:37.635Z [container-agent] Traceback (most recent call last):
2024-03-03T05:04:37.635Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 1767, in get_cluster_primary_address
2024-03-03T05:04:37.635Z [container-agent]     output = self._run_mysqlsh_script("\n".join(get_cluster_primary_commands))
2024-03-03T05:04:37.635Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 669, in _run_mysqlsh_script
2024-03-03T05:04:37.635Z [container-agent]     raise MySQLClientError(e.stderr)
2024-03-03T05:04:37.635Z [container-agent] charms.mysql.v0.mysql.MySQLClientError: Cannot set LC_ALL to locale en_US.UTF-8: No such file or directory
2024-03-03T05:04:37.635Z [container-agent] verbose: 2024-03-03T05:04:36Z: Loading startup files...
2024-03-03T05:04:37.635Z [container-agent] verbose: 2024-03-03T05:04:36Z: Loading plugins...
2024-03-03T05:04:37.635Z [container-agent] verbose: 2024-03-03T05:04:36Z: Connecting to MySQL at: clusteradmin@horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local
2024-03-03T05:04:37.635Z [container-agent] verbose: 2024-03-03T05:04:36Z: Shell.connect_to_primary: tid=22: CONNECTED: horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local
2024-03-03T05:04:37.635Z [container-agent] verbose: 2024-03-03T05:04:36Z: Redirecting session from 'mysqlx://clusteradmin@horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local:33060' to a PRIMARY of an InnoDB cluster or ReplicaSet...
2024-03-03T05:04:37.635Z [container-agent] Traceback (most recent call last):
2024-03-03T05:04:37.635Z [container-agent]   File "<string>", line 1, in <module>
2024-03-03T05:04:37.635Z [container-agent] RuntimeError: Shell.connect_to_primary: No primaries found in an InnoDB cluster
2024-03-03T05:04:37.635Z [container-agent] 
2024-03-03T05:04:37.635Z [container-agent] 
2024-03-03T05:04:40.765Z [container-agent] 2024-03-03 05:04:40 ERROR juju-log Failed to get cluster status for cluster-7573647c1e26466beeeac32b95ab8f91
2024-03-03T05:04:41.010Z [container-agent] 2024-03-03 05:04:41 INFO juju-log Kubernetes pod label error created
2024-03-03T05:04:41.022Z [container-agent] 2024-03-03 05:04:41 WARNING juju-log No relation: certificates
2024-03-03T05:04:41.384Z [container-agent] 2024-03-03 05:04:41 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T05:08:42.516Z [container-agent] 2024-03-03 05:08:42 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-03-03T05:08:42.542Z [container-agent] 2024-03-03 05:08:42 INFO juju-log Attempting reboot from complete outage.
2024-03-03T05:08:51.456Z [container-agent] 2024-03-03 05:08:51 INFO juju-log Kubernetes pod label primary created
2024-03-03T05:08:51.471Z [container-agent] 2024-03-03 05:08:51 WARNING juju-log No relation: certificates
2024-03-03T05:08:51.838Z [container-agent] 2024-03-03 05:08:51 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T05:09:36.377Z [container-agent] 2024-03-03 05:09:36 INFO juju-log Setting up the logrotate configurations
2024-03-03T05:09:36.448Z [container-agent] 2024-03-03 05:09:36 INFO juju-log Adding pebble layer
2024-03-03T05:09:43.180Z [container-agent] 2024-03-03 05:09:43 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-03-03T05:09:43.206Z [container-agent] 2024-03-03 05:09:43 INFO juju-log Attempting reboot from complete outage.
2024-03-03T05:09:46.059Z [container-agent] 2024-03-03 05:09:46 INFO juju.worker.uniter.operation runhook.go:186 ran "mysql-pebble-ready" hook (via hook dispatching script: dispatch)
2024-03-03T05:12:50.530Z [container-agent] 2024-03-03 05:12:50 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T05:12:52.614Z [container-agent] 2024-03-03 05:12:52 WARNING juju-log No relation: certificates
2024-03-03T05:12:52.963Z [container-agent] 2024-03-03 05:12:52 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T05:18:36.608Z [container-agent] 2024-03-03 05:18:36 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T05:18:39.242Z [container-agent] 2024-03-03 05:18:39 WARNING juju-log No relation: certificates
2024-03-03T05:18:39.599Z [container-agent] 2024-03-03 05:18:39 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T05:23:08.626Z [container-agent] 2024-03-03 05:23:08 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T05:23:11.752Z [container-agent] 2024-03-03 05:23:11 WARNING juju-log No relation: certificates
2024-03-03T05:23:12.108Z [container-agent] 2024-03-03 05:23:12 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T05:28:00.274Z [container-agent] 2024-03-03 05:28:00 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T05:28:03.862Z [container-agent] 2024-03-03 05:28:03 WARNING juju-log No relation: certificates
2024-03-03T05:28:04.274Z [container-agent] 2024-03-03 05:28:04 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T05:32:21.616Z [container-agent] 2024-03-03 05:32:21 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T05:32:25.572Z [container-agent] 2024-03-03 05:32:25 WARNING juju-log No relation: certificates
2024-03-03T05:32:25.943Z [container-agent] 2024-03-03 05:32:25 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T05:36:37.504Z [container-agent] 2024-03-03 05:36:37 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T05:36:41.794Z [container-agent] 2024-03-03 05:36:41 WARNING juju-log No relation: certificates
2024-03-03T05:36:42.148Z [container-agent] 2024-03-03 05:36:42 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T05:42:12.109Z [container-agent] 2024-03-03 05:42:12 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T05:42:17.315Z [container-agent] 2024-03-03 05:42:17 WARNING juju-log No relation: certificates
2024-03-03T05:42:17.683Z [container-agent] 2024-03-03 05:42:17 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T05:46:32.611Z [container-agent] 2024-03-03 05:46:32 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T05:46:38.042Z [container-agent] 2024-03-03 05:46:38 WARNING juju-log No relation: certificates
2024-03-03T05:46:38.406Z [container-agent] 2024-03-03 05:46:38 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T05:52:01.272Z [container-agent] 2024-03-03 05:52:01 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T05:52:07.001Z [container-agent] 2024-03-03 05:52:07 WARNING juju-log No relation: certificates
2024-03-03T05:52:07.369Z [container-agent] 2024-03-03 05:52:07 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T05:56:20.336Z [container-agent] 2024-03-03 05:56:20 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T05:56:26.590Z [container-agent] 2024-03-03 05:56:26 WARNING juju-log No relation: certificates
2024-03-03T05:56:26.921Z [container-agent] 2024-03-03 05:56:26 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T06:00:25.640Z [container-agent] 2024-03-03 06:00:25 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T06:00:32.376Z [container-agent] 2024-03-03 06:00:32 WARNING juju-log No relation: certificates
2024-03-03T06:00:32.733Z [container-agent] 2024-03-03 06:00:32 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T06:06:12.379Z [container-agent] 2024-03-03 06:06:12 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T06:06:19.700Z [container-agent] 2024-03-03 06:06:19 WARNING juju-log No relation: certificates
2024-03-03T06:06:20.054Z [container-agent] 2024-03-03 06:06:20 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T06:10:40.709Z [container-agent] 2024-03-03 06:10:40 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T06:10:47.996Z [container-agent] 2024-03-03 06:10:47 WARNING juju-log No relation: certificates
2024-03-03T06:10:48.346Z [container-agent] 2024-03-03 06:10:48 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T06:15:48.651Z [container-agent] 2024-03-03 06:15:48 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T06:15:55.150Z [container-agent] 2024-03-03 06:15:55 WARNING juju-log Failed to get cluster primary addresses
2024-03-03T06:15:55.150Z [container-agent] Traceback (most recent call last):
2024-03-03T06:15:55.150Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 666, in _run_mysqlsh_script
2024-03-03T06:15:55.150Z [container-agent]     stdout, _ = process.wait_output()
2024-03-03T06:15:55.150Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1441, in wait_output
2024-03-03T06:15:55.150Z [container-agent]     raise ExecError[AnyStr](self._command, exit_code, out_value, err_value)
2024-03-03T06:15:55.150Z [container-agent] ops.pebble.ExecError: non-zero exit code 1 executing ['/usr/bin/mysqlsh', '--no-wizard', '--python', '--verbose=1', '-f', '/tmp/script.py', ';', 'rm', '/tmp/script.py'], stdout='', stderr='Cannot set LC_ALL to locale en_US.UTF-8: No such file or directory\nverbose: 2024-03-03T06:15:53Z: Loading startup files...\nverbose: 2024-03-03T06:15:53Z: Loading plugins...\nverbose: 2024-03-03T06:15:53Z: Connecting to MySQL at: clusteradmin@horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local\nverbose: 2024-03-03T06:15:54Z: Shell.connect_to_primary: tid=21: CONNECTED: horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local\nverbose: 2024-03-03T06:15:54Z: Redirecting session from \'mysqlx://clusteradmin@horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local:33060\' to a PRIMARY of an InnoDB cluster or ReplicaSet...\nTraceback (most recent call last):\n  File "<string>", line 1, in <module>\nRuntimeError: Shell.connect_to_primary: No primaries found in an InnoDB cluster\n\n'
2024-03-03T06:15:55.150Z [container-agent] 
2024-03-03T06:15:55.150Z [container-agent] During handling of the above exception, another exception occurred:
2024-03-03T06:15:55.150Z [container-agent] 
2024-03-03T06:15:55.150Z [container-agent] Traceback (most recent call last):
2024-03-03T06:15:55.150Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 1767, in get_cluster_primary_address
2024-03-03T06:15:55.150Z [container-agent]     output = self._run_mysqlsh_script("\n".join(get_cluster_primary_commands))
2024-03-03T06:15:55.150Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 669, in _run_mysqlsh_script
2024-03-03T06:15:55.150Z [container-agent]     raise MySQLClientError(e.stderr)
2024-03-03T06:15:55.150Z [container-agent] charms.mysql.v0.mysql.MySQLClientError: Cannot set LC_ALL to locale en_US.UTF-8: No such file or directory
2024-03-03T06:15:55.150Z [container-agent] verbose: 2024-03-03T06:15:53Z: Loading startup files...
2024-03-03T06:15:55.150Z [container-agent] verbose: 2024-03-03T06:15:53Z: Loading plugins...
2024-03-03T06:15:55.150Z [container-agent] verbose: 2024-03-03T06:15:53Z: Connecting to MySQL at: clusteradmin@horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local
2024-03-03T06:15:55.150Z [container-agent] verbose: 2024-03-03T06:15:54Z: Shell.connect_to_primary: tid=21: CONNECTED: horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local
2024-03-03T06:15:55.150Z [container-agent] verbose: 2024-03-03T06:15:54Z: Redirecting session from 'mysqlx://clusteradmin@horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local:33060' to a PRIMARY of an InnoDB cluster or ReplicaSet...
2024-03-03T06:15:55.150Z [container-agent] Traceback (most recent call last):
2024-03-03T06:15:55.150Z [container-agent]   File "<string>", line 1, in <module>
2024-03-03T06:15:55.150Z [container-agent] RuntimeError: Shell.connect_to_primary: No primaries found in an InnoDB cluster
2024-03-03T06:15:55.150Z [container-agent] 
2024-03-03T06:15:55.150Z [container-agent] 
2024-03-03T06:15:57.397Z [container-agent] 2024-03-03 06:15:57 ERROR juju-log Failed to get cluster status for cluster-7573647c1e26466beeeac32b95ab8f91
2024-03-03T06:15:57.464Z [container-agent] 2024-03-03 06:15:57 INFO juju-log Kubernetes pod label error created
2024-03-03T06:15:57.476Z [container-agent] 2024-03-03 06:15:57 WARNING juju-log No relation: certificates
2024-03-03T06:15:57.833Z [container-agent] 2024-03-03 06:15:57 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T06:20:54.076Z [container-agent] 2024-03-03 06:20:54 INFO juju-log Setting up the logrotate configurations
2024-03-03T06:20:54.150Z [container-agent] 2024-03-03 06:20:54 INFO juju-log Adding pebble layer
2024-03-03T06:21:00.865Z [container-agent] 2024-03-03 06:21:00 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-03-03T06:21:00.890Z [container-agent] 2024-03-03 06:21:00 INFO juju-log Attempting reboot from complete outage.
2024-03-03T06:21:03.597Z [container-agent] 2024-03-03 06:21:03 INFO juju.worker.uniter.operation runhook.go:186 ran "mysql-pebble-ready" hook (via hook dispatching script: dispatch)
2024-03-03T06:21:23.639Z [container-agent] 2024-03-03 06:21:23 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T06:21:25.604Z [container-agent] 2024-03-03 06:21:25 INFO juju-log Kubernetes pod label primary created
2024-03-03T06:21:25.618Z [container-agent] 2024-03-03 06:21:25 WARNING juju-log No relation: certificates
2024-03-03T06:21:25.967Z [container-agent] 2024-03-03 06:21:25 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T06:26:43.436Z [container-agent] 2024-03-03 06:26:43 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T06:26:45.995Z [container-agent] 2024-03-03 06:26:45 WARNING juju-log No relation: certificates
2024-03-03T06:26:46.349Z [container-agent] 2024-03-03 06:26:46 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T06:32:42.107Z [container-agent] 2024-03-03 06:32:42 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T06:32:44.958Z [container-agent] 2024-03-03 06:32:44 WARNING juju-log No relation: certificates
2024-03-03T06:32:45.302Z [container-agent] 2024-03-03 06:32:45 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T06:38:12.539Z [container-agent] 2024-03-03 06:38:12 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T06:38:16.361Z [container-agent] 2024-03-03 06:38:16 WARNING juju-log No relation: certificates
2024-03-03T06:38:16.720Z [container-agent] 2024-03-03 06:38:16 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T06:42:28.892Z [container-agent] 2024-03-03 06:42:28 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T06:42:32.838Z [container-agent] 2024-03-03 06:42:32 WARNING juju-log No relation: certificates
2024-03-03T06:42:33.206Z [container-agent] 2024-03-03 06:42:33 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T06:47:31.672Z [container-agent] 2024-03-03 06:47:31 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T06:47:36.075Z [container-agent] 2024-03-03 06:47:36 WARNING juju-log No relation: certificates
2024-03-03T06:47:36.429Z [container-agent] 2024-03-03 06:47:36 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T06:52:09.878Z [container-agent] 2024-03-03 06:52:09 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T06:52:14.834Z [container-agent] 2024-03-03 06:52:14 WARNING juju-log No relation: certificates
2024-03-03T06:52:15.194Z [container-agent] 2024-03-03 06:52:15 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T06:56:22.455Z [container-agent] 2024-03-03 06:56:22 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T06:56:27.822Z [container-agent] 2024-03-03 06:56:27 WARNING juju-log No relation: certificates
2024-03-03T06:56:28.197Z [container-agent] 2024-03-03 06:56:28 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T07:01:27.910Z [container-agent] 2024-03-03 07:01:27 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T07:01:33.709Z [container-agent] 2024-03-03 07:01:33 WARNING juju-log No relation: certificates
2024-03-03T07:01:34.081Z [container-agent] 2024-03-03 07:01:34 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T07:06:43.337Z [container-agent] 2024-03-03 07:06:43 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T07:06:49.496Z [container-agent] 2024-03-03 07:06:49 WARNING juju-log No relation: certificates
2024-03-03T07:06:49.883Z [container-agent] 2024-03-03 07:06:49 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T07:10:51.476Z [container-agent] 2024-03-03 07:10:51 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T07:10:57.438Z [container-agent] 2024-03-03 07:10:57 WARNING juju-log No relation: certificates
2024-03-03T07:10:57.784Z [container-agent] 2024-03-03 07:10:57 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T07:15:40.847Z [container-agent] 2024-03-03 07:15:40 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T07:15:48.080Z [container-agent] 2024-03-03 07:15:48 WARNING juju-log No relation: certificates
2024-03-03T07:15:48.451Z [container-agent] 2024-03-03 07:15:48 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T07:20:18.391Z [container-agent] 2024-03-03 07:20:18 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T07:20:25.684Z [container-agent] 2024-03-03 07:20:25 WARNING juju-log No relation: certificates
2024-03-03T07:20:26.041Z [container-agent] 2024-03-03 07:20:26 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T07:25:19.809Z [container-agent] 2024-03-03 07:25:19 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T07:25:27.559Z [container-agent] 2024-03-03 07:25:27 WARNING juju-log No relation: certificates
2024-03-03T07:25:27.915Z [container-agent] 2024-03-03 07:25:27 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T07:29:52.814Z [container-agent] 2024-03-03 07:29:52 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-03-03T07:29:52.844Z [container-agent] 2024-03-03 07:29:52 INFO juju-log Attempting reboot from complete outage.
2024-03-03T07:30:00.382Z [container-agent] 2024-03-03 07:30:00 WARNING juju-log No relation: certificates
2024-03-03T07:30:00.747Z [container-agent] 2024-03-03 07:30:00 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T07:31:09.796Z [container-agent] 2024-03-03 07:31:09 ERROR juju-log Uncaught exception while in charm code:
2024-03-03T07:31:09.796Z [container-agent] Traceback (most recent call last):
2024-03-03T07:31:09.796Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/./src/charm.py", line 786, in <module>
2024-03-03T07:31:09.796Z [container-agent]     main(MySQLOperatorCharm)
2024-03-03T07:31:09.796Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 436, in main
2024-03-03T07:31:09.796Z [container-agent]     _emit_charm_event(charm, dispatcher.event_name)
2024-03-03T07:31:09.796Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 144, in _emit_charm_event
2024-03-03T07:31:09.796Z [container-agent]     event_to_emit.emit(*args, **kwargs)
2024-03-03T07:31:09.796Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 351, in emit
2024-03-03T07:31:09.796Z [container-agent]     framework._emit(event)
2024-03-03T07:31:09.796Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 853, in _emit
2024-03-03T07:31:09.796Z [container-agent]     self._reemit(event_path)
2024-03-03T07:31:09.796Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 942, in _reemit
2024-03-03T07:31:09.796Z [container-agent]     custom_handler(event)
2024-03-03T07:31:09.796Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/rotate_mysql_logs.py", line 57, in _rotate_mysql_logs
2024-03-03T07:31:09.796Z [container-agent]     self.charm._mysql.flush_mysql_logs(MySQLTextLogs.SLOW)
2024-03-03T07:31:09.796Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-03-03T07:31:09.796Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-03-03T07:31:09.796Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 666, in _run_mysqlsh_script
2024-03-03T07:31:09.796Z [container-agent]     stdout, _ = process.wait_output()
2024-03-03T07:31:09.796Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1436, in wait_output
2024-03-03T07:31:09.796Z [container-agent]     exit_code: int = self._wait()
2024-03-03T07:31:09.796Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1375, in _wait
2024-03-03T07:31:09.796Z [container-agent]     change = self._client.wait_change(self._change_id, timeout=timeout)
2024-03-03T07:31:09.796Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1893, in wait_change
2024-03-03T07:31:09.796Z [container-agent]     return self._wait_change_using_wait(change_id, timeout)
2024-03-03T07:31:09.796Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1914, in _wait_change_using_wait
2024-03-03T07:31:09.796Z [container-agent]     return self._wait_change(change_id, this_timeout)
2024-03-03T07:31:09.796Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1928, in _wait_change
2024-03-03T07:31:09.796Z [container-agent]     resp = self._request('GET', f'/v1/changes/{change_id}/wait', query)
2024-03-03T07:31:09.796Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1655, in _request
2024-03-03T07:31:09.796Z [container-agent]     response = self._request_raw(method, path, query, headers, data)
2024-03-03T07:31:09.796Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1690, in _request_raw
2024-03-03T07:31:09.796Z [container-agent]     response = self.opener.open(request, timeout=self.timeout)
2024-03-03T07:31:09.796Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 519, in open
2024-03-03T07:31:09.796Z [container-agent]     response = self._open(req, data)
2024-03-03T07:31:09.796Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 536, in _open
2024-03-03T07:31:09.796Z [container-agent]     result = self._call_chain(self.handle_open, protocol, protocol +
2024-03-03T07:31:09.796Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 496, in _call_chain
2024-03-03T07:31:09.796Z [container-agent]     result = func(*args)
2024-03-03T07:31:09.796Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 311, in http_open
2024-03-03T07:31:09.796Z [container-agent]     return self.do_open(_UnixSocketConnection, req,  # type:ignore
2024-03-03T07:31:09.796Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 1352, in do_open
2024-03-03T07:31:09.796Z [container-agent]     r = h.getresponse()
2024-03-03T07:31:09.796Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1375, in getresponse
2024-03-03T07:31:09.796Z [container-agent]     response.begin()
2024-03-03T07:31:09.796Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 318, in begin
2024-03-03T07:31:09.796Z [container-agent]     version, status, reason = self._read_status()
2024-03-03T07:31:09.796Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 287, in _read_status
2024-03-03T07:31:09.796Z [container-agent]     raise RemoteDisconnected("Remote end closed connection without"
2024-03-03T07:31:09.796Z [container-agent] http.client.RemoteDisconnected: Remote end closed connection without response
2024-03-03T07:31:09.925Z [container-agent] 2024-03-03 07:31:09 INFO juju.util.exec exec.go:209 run result: exit status 1
2024-03-03T07:31:11.723Z [container-agent] 2024-03-03 07:31:11 INFO juju-log Setting up the logrotate configurations
2024-03-03T07:31:11.799Z [container-agent] 2024-03-03 07:31:11 INFO juju-log Adding pebble layer
2024-03-03T07:31:18.497Z [container-agent] 2024-03-03 07:31:18 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-03-03T07:31:18.516Z [container-agent] 2024-03-03 07:31:18 INFO juju-log Attempting reboot from complete outage.
2024-03-03T07:31:21.127Z [container-agent] 2024-03-03 07:31:21 INFO juju.worker.uniter.operation runhook.go:186 ran "mysql-pebble-ready" hook (via hook dispatching script: dispatch)
2024-03-03T07:35:28.977Z [container-agent] 2024-03-03 07:35:28 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T07:35:30.800Z [container-agent] 2024-03-03 07:35:30 WARNING juju-log No relation: certificates
2024-03-03T07:35:31.160Z [container-agent] 2024-03-03 07:35:31 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T07:40:10.938Z [container-agent] 2024-03-03 07:40:10 INFO juju-log Starting the log rotate manager
2024-03-03T07:40:10.951Z [container-agent] 2024-03-03 07:40:10 INFO juju-log Started log rotate manager process with PID 664362
2024-03-03T07:40:11.281Z [container-agent] 2024-03-03 07:40:11 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T07:40:13.332Z [container-agent] 2024-03-03 07:40:13 WARNING juju-log No relation: certificates
2024-03-03T07:40:13.722Z [container-agent] 2024-03-03 07:40:13 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T07:46:05.840Z [container-agent] 2024-03-03 07:46:05 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T07:46:08.376Z [container-agent] 2024-03-03 07:46:08 WARNING juju-log No relation: certificates
2024-03-03T07:46:08.723Z [container-agent] 2024-03-03 07:46:08 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T07:51:21.283Z [container-agent] 2024-03-03 07:51:21 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T07:51:24.111Z [container-agent] 2024-03-03 07:51:24 WARNING juju-log No relation: certificates
2024-03-03T07:51:24.446Z [container-agent] 2024-03-03 07:51:24 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T07:57:19.975Z [container-agent] 2024-03-03 07:57:19 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T07:57:23.349Z [container-agent] 2024-03-03 07:57:23 WARNING juju-log No relation: certificates
2024-03-03T07:57:23.705Z [container-agent] 2024-03-03 07:57:23 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T08:02:21.950Z [container-agent] 2024-03-03 08:02:21 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T08:02:25.879Z [container-agent] 2024-03-03 08:02:25 WARNING juju-log No relation: certificates
2024-03-03T08:02:26.223Z [container-agent] 2024-03-03 08:02:26 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T08:08:16.851Z [container-agent] 2024-03-03 08:08:16 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T08:08:21.273Z [container-agent] 2024-03-03 08:08:21 WARNING juju-log No relation: certificates
2024-03-03T08:08:21.614Z [container-agent] 2024-03-03 08:08:21 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T08:13:12.276Z [container-agent] 2024-03-03 08:13:12 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T08:13:17.337Z [container-agent] 2024-03-03 08:13:17 WARNING juju-log No relation: certificates
2024-03-03T08:13:17.677Z [container-agent] 2024-03-03 08:13:17 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T08:18:14.249Z [container-agent] 2024-03-03 08:18:14 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T08:18:19.419Z [container-agent] 2024-03-03 08:18:19 WARNING juju-log No relation: certificates
2024-03-03T08:18:19.776Z [container-agent] 2024-03-03 08:18:19 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T08:22:21.375Z [container-agent] 2024-03-03 08:22:21 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T08:22:26.665Z [container-agent] 2024-03-03 08:22:26 WARNING juju-log No relation: certificates
2024-03-03T08:22:27.010Z [container-agent] 2024-03-03 08:22:27 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T08:27:11.544Z [container-agent] 2024-03-03 08:27:11 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T08:27:18.121Z [container-agent] 2024-03-03 08:27:18 WARNING juju-log No relation: certificates
2024-03-03T08:27:18.473Z [container-agent] 2024-03-03 08:27:18 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T08:32:15.236Z [container-agent] 2024-03-03 08:32:15 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T08:32:22.280Z [container-agent] 2024-03-03 08:32:22 WARNING juju-log No relation: certificates
2024-03-03T08:32:22.623Z [container-agent] 2024-03-03 08:32:22 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T08:37:00.329Z [container-agent] 2024-03-03 08:37:00 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T08:37:06.935Z [container-agent] 2024-03-03 08:37:06 WARNING juju-log No relation: certificates
2024-03-03T08:37:07.347Z [container-agent] 2024-03-03 08:37:07 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T08:42:12.984Z [container-agent] 2024-03-03 08:42:12 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T08:42:20.823Z [container-agent] 2024-03-03 08:42:20 WARNING juju-log No relation: certificates
2024-03-03T08:42:21.169Z [container-agent] 2024-03-03 08:42:21 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T08:47:22.775Z [container-agent] 2024-03-03 08:47:22 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T08:47:30.645Z [container-agent] 2024-03-03 08:47:30 WARNING juju-log No relation: certificates
2024-03-03T08:47:31.006Z [container-agent] 2024-03-03 08:47:31 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T08:49:09.261Z [container-agent] 2024-03-03 08:49:09 ERROR juju-log Failed to flush GENERAL LOGS logs.
2024-03-03T08:49:09.261Z [container-agent] Traceback (most recent call last):
2024-03-03T08:49:09.261Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 666, in _run_mysqlsh_script
2024-03-03T08:49:09.261Z [container-agent]     stdout, _ = process.wait_output()
2024-03-03T08:49:09.261Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1441, in wait_output
2024-03-03T08:49:09.261Z [container-agent]     raise ExecError[AnyStr](self._command, exit_code, out_value, err_value)
2024-03-03T08:49:09.261Z [container-agent] ops.pebble.ExecError: non-zero exit code 1 executing ['/usr/bin/mysqlsh', '--no-wizard', '--python', '--verbose=1', '-f', '/tmp/script.py', ';', 'rm', '/tmp/script.py'], stdout='', stderr='Cannot set LC_ALL to locale en_US.UTF-8: No such file or directory\nverbose: 2024-03-03T08:49:08Z: Loading startup files...\nverbose: 2024-03-03T08:49:08Z: Loading plugins...\nverbose: 2024-03-03T08:49:08Z: Connecting to MySQL at: serverconfig@horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local\nTraceback (most recent call last):\n  File "<string>", line 1, in <module>\nmysqlsh.DBError: MySQL Error (2003): Shell.connect: Can\'t connect to MySQL server on \'horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local:3306\' (111)\n'
2024-03-03T08:49:09.261Z [container-agent] 
2024-03-03T08:49:09.261Z [container-agent] During handling of the above exception, another exception occurred:
2024-03-03T08:49:09.261Z [container-agent] 
2024-03-03T08:49:09.261Z [container-agent] Traceback (most recent call last):
2024-03-03T08:49:09.261Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-03-03T08:49:09.261Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-03-03T08:49:09.261Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 669, in _run_mysqlsh_script
2024-03-03T08:49:09.261Z [container-agent]     raise MySQLClientError(e.stderr)
2024-03-03T08:49:09.261Z [container-agent] charms.mysql.v0.mysql.MySQLClientError: Cannot set LC_ALL to locale en_US.UTF-8: No such file or directory
2024-03-03T08:49:09.261Z [container-agent] verbose: 2024-03-03T08:49:08Z: Loading startup files...
2024-03-03T08:49:09.261Z [container-agent] verbose: 2024-03-03T08:49:08Z: Loading plugins...
2024-03-03T08:49:09.261Z [container-agent] verbose: 2024-03-03T08:49:08Z: Connecting to MySQL at: serverconfig@horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local
2024-03-03T08:49:09.261Z [container-agent] Traceback (most recent call last):
2024-03-03T08:49:09.261Z [container-agent]   File "<string>", line 1, in <module>
2024-03-03T08:49:09.261Z [container-agent] mysqlsh.DBError: MySQL Error (2003): Shell.connect: Can't connect to MySQL server on 'horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local:3306' (111)
2024-03-03T08:49:09.261Z [container-agent] 
2024-03-03T08:52:14.978Z [container-agent] 2024-03-03 08:52:14 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-03-03T08:52:15.019Z [container-agent] 2024-03-03 08:52:15 INFO juju-log Attempting reboot from complete outage.
2024-03-03T08:52:23.034Z [container-agent] 2024-03-03 08:52:23 WARNING juju-log No relation: certificates
2024-03-03T08:52:23.387Z [container-agent] 2024-03-03 08:52:23 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T08:54:04.803Z [container-agent] 2024-03-03 08:54:04 ERROR juju-log Uncaught exception while in charm code:
2024-03-03T08:54:04.803Z [container-agent] Traceback (most recent call last):
2024-03-03T08:54:04.803Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/./src/charm.py", line 786, in <module>
2024-03-03T08:54:04.803Z [container-agent]     main(MySQLOperatorCharm)
2024-03-03T08:54:04.803Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 436, in main
2024-03-03T08:54:04.803Z [container-agent]     _emit_charm_event(charm, dispatcher.event_name)
2024-03-03T08:54:04.803Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 144, in _emit_charm_event
2024-03-03T08:54:04.803Z [container-agent]     event_to_emit.emit(*args, **kwargs)
2024-03-03T08:54:04.803Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 351, in emit
2024-03-03T08:54:04.803Z [container-agent]     framework._emit(event)
2024-03-03T08:54:04.803Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 853, in _emit
2024-03-03T08:54:04.803Z [container-agent]     self._reemit(event_path)
2024-03-03T08:54:04.803Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 942, in _reemit
2024-03-03T08:54:04.803Z [container-agent]     custom_handler(event)
2024-03-03T08:54:04.803Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/rotate_mysql_logs.py", line 55, in _rotate_mysql_logs
2024-03-03T08:54:04.803Z [container-agent]     self.charm._mysql.flush_mysql_logs(MySQLTextLogs.ERROR)
2024-03-03T08:54:04.803Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-03-03T08:54:04.803Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-03-03T08:54:04.803Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 666, in _run_mysqlsh_script
2024-03-03T08:54:04.803Z [container-agent]     stdout, _ = process.wait_output()
2024-03-03T08:54:04.803Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1436, in wait_output
2024-03-03T08:54:04.803Z [container-agent]     exit_code: int = self._wait()
2024-03-03T08:54:04.803Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1375, in _wait
2024-03-03T08:54:04.803Z [container-agent]     change = self._client.wait_change(self._change_id, timeout=timeout)
2024-03-03T08:54:04.803Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1893, in wait_change
2024-03-03T08:54:04.803Z [container-agent]     return self._wait_change_using_wait(change_id, timeout)
2024-03-03T08:54:04.803Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1914, in _wait_change_using_wait
2024-03-03T08:54:04.803Z [container-agent]     return self._wait_change(change_id, this_timeout)
2024-03-03T08:54:04.803Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1928, in _wait_change
2024-03-03T08:54:04.803Z [container-agent]     resp = self._request('GET', f'/v1/changes/{change_id}/wait', query)
2024-03-03T08:54:04.803Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1655, in _request
2024-03-03T08:54:04.803Z [container-agent]     response = self._request_raw(method, path, query, headers, data)
2024-03-03T08:54:04.803Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1690, in _request_raw
2024-03-03T08:54:04.803Z [container-agent]     response = self.opener.open(request, timeout=self.timeout)
2024-03-03T08:54:04.803Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 519, in open
2024-03-03T08:54:04.803Z [container-agent]     response = self._open(req, data)
2024-03-03T08:54:04.803Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 536, in _open
2024-03-03T08:54:04.803Z [container-agent]     result = self._call_chain(self.handle_open, protocol, protocol +
2024-03-03T08:54:04.803Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 496, in _call_chain
2024-03-03T08:54:04.803Z [container-agent]     result = func(*args)
2024-03-03T08:54:04.803Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 311, in http_open
2024-03-03T08:54:04.803Z [container-agent]     return self.do_open(_UnixSocketConnection, req,  # type:ignore
2024-03-03T08:54:04.803Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 1352, in do_open
2024-03-03T08:54:04.803Z [container-agent]     r = h.getresponse()
2024-03-03T08:54:04.803Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1375, in getresponse
2024-03-03T08:54:04.803Z [container-agent]     response.begin()
2024-03-03T08:54:04.803Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 318, in begin
2024-03-03T08:54:04.803Z [container-agent]     version, status, reason = self._read_status()
2024-03-03T08:54:04.803Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 287, in _read_status
2024-03-03T08:54:04.803Z [container-agent]     raise RemoteDisconnected("Remote end closed connection without"
2024-03-03T08:54:04.803Z [container-agent] http.client.RemoteDisconnected: Remote end closed connection without response
2024-03-03T08:54:04.961Z [container-agent] 2024-03-03 08:54:04 INFO juju.util.exec exec.go:209 run result: exit status 1
2024-03-03T08:54:09.391Z [container-agent] 2024-03-03 08:54:09 INFO juju-log Setting up the logrotate configurations
2024-03-03T08:54:09.475Z [container-agent] 2024-03-03 08:54:09 INFO juju-log Adding pebble layer
2024-03-03T08:54:16.275Z [container-agent] 2024-03-03 08:54:16 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-03-03T08:54:16.301Z [container-agent] 2024-03-03 08:54:16 INFO juju-log Attempting reboot from complete outage.
2024-03-03T08:54:19.330Z [container-agent] 2024-03-03 08:54:19 INFO juju.worker.uniter.operation runhook.go:186 ran "mysql-pebble-ready" hook (via hook dispatching script: dispatch)
2024-03-03T08:56:50.468Z [container-agent] 2024-03-03 08:56:50 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T08:56:52.318Z [container-agent] 2024-03-03 08:56:52 WARNING juju-log No relation: certificates
2024-03-03T08:56:52.713Z [container-agent] 2024-03-03 08:56:52 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T09:02:49.217Z [container-agent] 2024-03-03 09:02:49 INFO juju-log Starting the log rotate manager
2024-03-03T09:02:49.231Z [container-agent] 2024-03-03 09:02:49 INFO juju-log Started log rotate manager process with PID 671913
2024-03-03T09:02:49.715Z [container-agent] 2024-03-03 09:02:49 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T09:02:51.833Z [container-agent] 2024-03-03 09:02:51 WARNING juju-log No relation: certificates
2024-03-03T09:02:52.206Z [container-agent] 2024-03-03 09:02:52 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T09:08:10.091Z [container-agent] 2024-03-03 09:08:10 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T09:08:12.478Z [container-agent] 2024-03-03 09:08:12 WARNING juju-log No relation: certificates
2024-03-03T09:08:12.833Z [container-agent] 2024-03-03 09:08:12 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T09:13:37.859Z [container-agent] 2024-03-03 09:13:37 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T09:13:40.737Z [container-agent] 2024-03-03 09:13:40 WARNING juju-log No relation: certificates
2024-03-03T09:13:41.121Z [container-agent] 2024-03-03 09:13:41 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T09:18:14.827Z [container-agent] 2024-03-03 09:18:14 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T09:18:18.495Z [container-agent] 2024-03-03 09:18:18 WARNING juju-log No relation: certificates
2024-03-03T09:18:18.848Z [container-agent] 2024-03-03 09:18:18 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T09:22:25.015Z [container-agent] 2024-03-03 09:22:25 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T09:22:28.885Z [container-agent] 2024-03-03 09:22:28 WARNING juju-log No relation: certificates
2024-03-03T09:22:29.240Z [container-agent] 2024-03-03 09:22:29 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T09:27:52.752Z [container-agent] 2024-03-03 09:27:52 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T09:27:56.879Z [container-agent] 2024-03-03 09:27:56 WARNING juju-log No relation: certificates
2024-03-03T09:27:57.208Z [container-agent] 2024-03-03 09:27:57 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T09:33:27.356Z [container-agent] 2024-03-03 09:33:27 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T09:33:32.327Z [container-agent] 2024-03-03 09:33:32 WARNING juju-log No relation: certificates
2024-03-03T09:33:32.677Z [container-agent] 2024-03-03 09:33:32 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T09:38:28.288Z [container-agent] 2024-03-03 09:38:28 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T09:38:34.017Z [container-agent] 2024-03-03 09:38:34 WARNING juju-log No relation: certificates
2024-03-03T09:38:34.382Z [container-agent] 2024-03-03 09:38:34 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T09:43:23.309Z [container-agent] 2024-03-03 09:43:23 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T09:43:29.342Z [container-agent] 2024-03-03 09:43:29 WARNING juju-log No relation: certificates
2024-03-03T09:43:29.730Z [container-agent] 2024-03-03 09:43:29 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T09:49:11.240Z [container-agent] 2024-03-03 09:49:11 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T09:49:17.609Z [container-agent] 2024-03-03 09:49:17 WARNING juju-log No relation: certificates
2024-03-03T09:49:17.976Z [container-agent] 2024-03-03 09:49:17 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T09:54:16.148Z [container-agent] 2024-03-03 09:54:16 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T09:54:22.929Z [container-agent] 2024-03-03 09:54:22 WARNING juju-log No relation: certificates
2024-03-03T09:54:23.294Z [container-agent] 2024-03-03 09:54:23 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T09:58:41.528Z [container-agent] 2024-03-03 09:58:41 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T09:58:48.448Z [container-agent] 2024-03-03 09:58:48 WARNING juju-log No relation: certificates
2024-03-03T09:58:48.831Z [container-agent] 2024-03-03 09:58:48 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T10:03:48.015Z [container-agent] 2024-03-03 10:03:48 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T10:03:55.691Z [container-agent] 2024-03-03 10:03:55 WARNING juju-log No relation: certificates
2024-03-03T10:03:56.061Z [container-agent] 2024-03-03 10:03:56 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T10:08:14.127Z [container-agent] 2024-03-03 10:08:14 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T10:08:22.271Z [container-agent] 2024-03-03 10:08:22 WARNING juju-log No relation: certificates
2024-03-03T10:08:22.666Z [container-agent] 2024-03-03 10:08:22 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T10:10:12.821Z [container-agent] 2024-03-03 10:10:12 ERROR juju-log Failed to flush SLOW LOGS logs.
2024-03-03T10:10:12.821Z [container-agent] Traceback (most recent call last):
2024-03-03T10:10:12.821Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 666, in _run_mysqlsh_script
2024-03-03T10:10:12.821Z [container-agent]     stdout, _ = process.wait_output()
2024-03-03T10:10:12.821Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1441, in wait_output
2024-03-03T10:10:12.821Z [container-agent]     raise ExecError[AnyStr](self._command, exit_code, out_value, err_value)
2024-03-03T10:10:12.821Z [container-agent] ops.pebble.ExecError: non-zero exit code 1 executing ['/usr/bin/mysqlsh', '--no-wizard', '--python', '--verbose=1', '-f', '/tmp/script.py', ';', 'rm', '/tmp/script.py'], stdout='', stderr='Cannot set LC_ALL to locale en_US.UTF-8: No such file or directory\nverbose: 2024-03-03T10:10:11Z: Loading startup files...\nverbose: 2024-03-03T10:10:11Z: Loading plugins...\nverbose: 2024-03-03T10:10:11Z: Connecting to MySQL at: serverconfig@horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local\nTraceback (most recent call last):\n  File "<string>", line 1, in <module>\nmysqlsh.DBError: MySQL Error (2003): Shell.connect: Can\'t connect to MySQL server on \'horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local:3306\' (111)\n'
2024-03-03T10:10:12.821Z [container-agent] 
2024-03-03T10:10:12.821Z [container-agent] During handling of the above exception, another exception occurred:
2024-03-03T10:10:12.821Z [container-agent] 
2024-03-03T10:10:12.821Z [container-agent] Traceback (most recent call last):
2024-03-03T10:10:12.821Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-03-03T10:10:12.821Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-03-03T10:10:12.821Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 669, in _run_mysqlsh_script
2024-03-03T10:10:12.821Z [container-agent]     raise MySQLClientError(e.stderr)
2024-03-03T10:10:12.821Z [container-agent] charms.mysql.v0.mysql.MySQLClientError: Cannot set LC_ALL to locale en_US.UTF-8: No such file or directory
2024-03-03T10:10:12.821Z [container-agent] verbose: 2024-03-03T10:10:11Z: Loading startup files...
2024-03-03T10:10:12.821Z [container-agent] verbose: 2024-03-03T10:10:11Z: Loading plugins...
2024-03-03T10:10:12.821Z [container-agent] verbose: 2024-03-03T10:10:11Z: Connecting to MySQL at: serverconfig@horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local
2024-03-03T10:10:12.821Z [container-agent] Traceback (most recent call last):
2024-03-03T10:10:12.821Z [container-agent]   File "<string>", line 1, in <module>
2024-03-03T10:10:12.821Z [container-agent] mysqlsh.DBError: MySQL Error (2003): Shell.connect: Can't connect to MySQL server on 'horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local:3306' (111)
2024-03-03T10:10:12.821Z [container-agent] 
2024-03-03T10:12:50.671Z [container-agent] 2024-03-03 10:12:50 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-03-03T10:12:50.703Z [container-agent] 2024-03-03 10:12:50 INFO juju-log Attempting reboot from complete outage.
2024-03-03T10:12:58.417Z [container-agent] 2024-03-03 10:12:58 WARNING juju-log No relation: certificates
2024-03-03T10:12:58.770Z [container-agent] 2024-03-03 10:12:58 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T10:15:09.787Z [container-agent] 2024-03-03 10:15:09 ERROR juju-log Uncaught exception while in charm code:
2024-03-03T10:15:09.787Z [container-agent] Traceback (most recent call last):
2024-03-03T10:15:09.787Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/./src/charm.py", line 786, in <module>
2024-03-03T10:15:09.787Z [container-agent]     main(MySQLOperatorCharm)
2024-03-03T10:15:09.787Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 436, in main
2024-03-03T10:15:09.787Z [container-agent]     _emit_charm_event(charm, dispatcher.event_name)
2024-03-03T10:15:09.787Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 144, in _emit_charm_event
2024-03-03T10:15:09.787Z [container-agent]     event_to_emit.emit(*args, **kwargs)
2024-03-03T10:15:09.787Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 351, in emit
2024-03-03T10:15:09.787Z [container-agent]     framework._emit(event)
2024-03-03T10:15:09.787Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 853, in _emit
2024-03-03T10:15:09.787Z [container-agent]     self._reemit(event_path)
2024-03-03T10:15:09.787Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 942, in _reemit
2024-03-03T10:15:09.787Z [container-agent]     custom_handler(event)
2024-03-03T10:15:09.787Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/rotate_mysql_logs.py", line 57, in _rotate_mysql_logs
2024-03-03T10:15:09.787Z [container-agent]     self.charm._mysql.flush_mysql_logs(MySQLTextLogs.SLOW)
2024-03-03T10:15:09.787Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-03-03T10:15:09.787Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-03-03T10:15:09.787Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 665, in _run_mysqlsh_script
2024-03-03T10:15:09.787Z [container-agent]     process = self.container.exec(cmd, timeout=timeout)
2024-03-03T10:15:09.787Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/model.py", line 2688, in exec
2024-03-03T10:15:09.787Z [container-agent]     return self._pebble.exec(
2024-03-03T10:15:09.787Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 2501, in exec
2024-03-03T10:15:09.787Z [container-agent]     resp = self._request('POST', '/v1/exec', body=body)
2024-03-03T10:15:09.787Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1655, in _request
2024-03-03T10:15:09.787Z [container-agent]     response = self._request_raw(method, path, query, headers, data)
2024-03-03T10:15:09.787Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1690, in _request_raw
2024-03-03T10:15:09.787Z [container-agent]     response = self.opener.open(request, timeout=self.timeout)
2024-03-03T10:15:09.787Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 519, in open
2024-03-03T10:15:09.787Z [container-agent]     response = self._open(req, data)
2024-03-03T10:15:09.787Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 536, in _open
2024-03-03T10:15:09.787Z [container-agent]     result = self._call_chain(self.handle_open, protocol, protocol +
2024-03-03T10:15:09.787Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 496, in _call_chain
2024-03-03T10:15:09.787Z [container-agent]     result = func(*args)
2024-03-03T10:15:09.787Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 311, in http_open
2024-03-03T10:15:09.787Z [container-agent]     return self.do_open(_UnixSocketConnection, req,  # type:ignore
2024-03-03T10:15:09.787Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 1352, in do_open
2024-03-03T10:15:09.787Z [container-agent]     r = h.getresponse()
2024-03-03T10:15:09.787Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1375, in getresponse
2024-03-03T10:15:09.787Z [container-agent]     response.begin()
2024-03-03T10:15:09.787Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 318, in begin
2024-03-03T10:15:09.787Z [container-agent]     version, status, reason = self._read_status()
2024-03-03T10:15:09.787Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 287, in _read_status
2024-03-03T10:15:09.787Z [container-agent]     raise RemoteDisconnected("Remote end closed connection without"
2024-03-03T10:15:09.787Z [container-agent] http.client.RemoteDisconnected: Remote end closed connection without response
2024-03-03T10:15:09.955Z [container-agent] 2024-03-03 10:15:09 INFO juju.util.exec exec.go:209 run result: exit status 1
2024-03-03T10:15:12.699Z [container-agent] 2024-03-03 10:15:12 INFO juju-log Setting up the logrotate configurations
2024-03-03T10:15:12.785Z [container-agent] 2024-03-03 10:15:12 INFO juju-log Adding pebble layer
2024-03-03T10:15:19.514Z [container-agent] 2024-03-03 10:15:19 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-03-03T10:15:19.539Z [container-agent] 2024-03-03 10:15:19 INFO juju-log Attempting reboot from complete outage.
2024-03-03T10:15:22.491Z [container-agent] 2024-03-03 10:15:22 INFO juju.worker.uniter.operation runhook.go:186 ran "mysql-pebble-ready" hook (via hook dispatching script: dispatch)
2024-03-03T10:17:25.506Z [container-agent] 2024-03-03 10:17:25 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T10:17:27.417Z [container-agent] 2024-03-03 10:17:27 WARNING juju-log No relation: certificates
2024-03-03T10:17:27.764Z [container-agent] 2024-03-03 10:17:27 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T10:22:38.005Z [container-agent] 2024-03-03 10:22:38 INFO juju-log Starting the log rotate manager
2024-03-03T10:22:38.017Z [container-agent] 2024-03-03 10:22:38 INFO juju-log Started log rotate manager process with PID 679361
2024-03-03T10:22:38.304Z [container-agent] 2024-03-03 10:22:38 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T10:22:40.190Z [container-agent] 2024-03-03 10:22:40 WARNING juju-log No relation: certificates
2024-03-03T10:22:40.538Z [container-agent] 2024-03-03 10:22:40 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T10:27:22.678Z [container-agent] 2024-03-03 10:27:22 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T10:27:25.061Z [container-agent] 2024-03-03 10:27:25 WARNING juju-log No relation: certificates
2024-03-03T10:27:25.380Z [container-agent] 2024-03-03 10:27:25 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T10:31:46.461Z [container-agent] 2024-03-03 10:31:46 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T10:31:49.337Z [container-agent] 2024-03-03 10:31:49 WARNING juju-log No relation: certificates
2024-03-03T10:31:49.692Z [container-agent] 2024-03-03 10:31:49 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T10:37:07.341Z [container-agent] 2024-03-03 10:37:07 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T10:37:10.761Z [container-agent] 2024-03-03 10:37:10 WARNING juju-log No relation: certificates
2024-03-03T10:37:11.143Z [container-agent] 2024-03-03 10:37:11 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T10:42:08.042Z [container-agent] 2024-03-03 10:42:08 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T10:42:11.950Z [container-agent] 2024-03-03 10:42:11 WARNING juju-log No relation: certificates
2024-03-03T10:42:12.357Z [container-agent] 2024-03-03 10:42:12 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T10:46:09.456Z [container-agent] 2024-03-03 10:46:09 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T10:46:13.789Z [container-agent] 2024-03-03 10:46:13 WARNING juju-log No relation: certificates
2024-03-03T10:46:14.148Z [container-agent] 2024-03-03 10:46:14 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T10:51:46.540Z [container-agent] 2024-03-03 10:51:46 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T10:51:51.253Z [container-agent] 2024-03-03 10:51:51 WARNING juju-log No relation: certificates
2024-03-03T10:51:51.640Z [container-agent] 2024-03-03 10:51:51 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T10:56:46.950Z [container-agent] 2024-03-03 10:56:46 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T10:56:52.201Z [container-agent] 2024-03-03 10:56:52 WARNING juju-log No relation: certificates
2024-03-03T10:56:52.561Z [container-agent] 2024-03-03 10:56:52 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T11:02:39.927Z [container-agent] 2024-03-03 11:02:39 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T11:02:45.859Z [container-agent] 2024-03-03 11:02:45 WARNING juju-log No relation: certificates
2024-03-03T11:02:46.239Z [container-agent] 2024-03-03 11:02:46 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T11:08:32.226Z [container-agent] 2024-03-03 11:08:32 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T11:08:38.393Z [container-agent] 2024-03-03 11:08:38 WARNING juju-log No relation: certificates
2024-03-03T11:08:38.725Z [container-agent] 2024-03-03 11:08:38 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T11:14:18.364Z [container-agent] 2024-03-03 11:14:18 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T11:14:25.114Z [container-agent] 2024-03-03 11:14:25 WARNING juju-log No relation: certificates
2024-03-03T11:14:25.464Z [container-agent] 2024-03-03 11:14:25 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T11:19:34.978Z [container-agent] 2024-03-03 11:19:34 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T11:19:42.161Z [container-agent] 2024-03-03 11:19:42 WARNING juju-log No relation: certificates
2024-03-03T11:19:42.507Z [container-agent] 2024-03-03 11:19:42 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T11:23:55.354Z [container-agent] 2024-03-03 11:23:55 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T11:24:02.737Z [container-agent] 2024-03-03 11:24:02 WARNING juju-log No relation: certificates
2024-03-03T11:24:03.140Z [container-agent] 2024-03-03 11:24:03 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T11:28:52.470Z [container-agent] 2024-03-03 11:28:52 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T11:28:59.806Z [container-agent] 2024-03-03 11:28:59 WARNING juju-log No relation: certificates
2024-03-03T11:29:00.156Z [container-agent] 2024-03-03 11:29:00 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T11:34:48.913Z [container-agent] 2024-03-03 11:34:48 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-03-03T11:34:48.941Z [container-agent] 2024-03-03 11:34:48 INFO juju-log Attempting reboot from complete outage.
2024-03-03T11:34:56.755Z [container-agent] 2024-03-03 11:34:56 WARNING juju-log No relation: certificates
2024-03-03T11:34:57.116Z [container-agent] 2024-03-03 11:34:57 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T11:38:04.787Z [container-agent] 2024-03-03 11:38:04 ERROR juju-log Uncaught exception while in charm code:
2024-03-03T11:38:04.787Z [container-agent] Traceback (most recent call last):
2024-03-03T11:38:04.787Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/./src/charm.py", line 786, in <module>
2024-03-03T11:38:04.787Z [container-agent]     main(MySQLOperatorCharm)
2024-03-03T11:38:04.787Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 436, in main
2024-03-03T11:38:04.787Z [container-agent]     _emit_charm_event(charm, dispatcher.event_name)
2024-03-03T11:38:04.787Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 144, in _emit_charm_event
2024-03-03T11:38:04.787Z [container-agent]     event_to_emit.emit(*args, **kwargs)
2024-03-03T11:38:04.787Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 351, in emit
2024-03-03T11:38:04.787Z [container-agent]     framework._emit(event)
2024-03-03T11:38:04.787Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 853, in _emit
2024-03-03T11:38:04.787Z [container-agent]     self._reemit(event_path)
2024-03-03T11:38:04.787Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 942, in _reemit
2024-03-03T11:38:04.787Z [container-agent]     custom_handler(event)
2024-03-03T11:38:04.787Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/rotate_mysql_logs.py", line 55, in _rotate_mysql_logs
2024-03-03T11:38:04.787Z [container-agent]     self.charm._mysql.flush_mysql_logs(MySQLTextLogs.ERROR)
2024-03-03T11:38:04.787Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-03-03T11:38:04.787Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-03-03T11:38:04.787Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 665, in _run_mysqlsh_script
2024-03-03T11:38:04.787Z [container-agent]     process = self.container.exec(cmd, timeout=timeout)
2024-03-03T11:38:04.787Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/model.py", line 2688, in exec
2024-03-03T11:38:04.787Z [container-agent]     return self._pebble.exec(
2024-03-03T11:38:04.787Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 2501, in exec
2024-03-03T11:38:04.787Z [container-agent]     resp = self._request('POST', '/v1/exec', body=body)
2024-03-03T11:38:04.787Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1655, in _request
2024-03-03T11:38:04.787Z [container-agent]     response = self._request_raw(method, path, query, headers, data)
2024-03-03T11:38:04.787Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1690, in _request_raw
2024-03-03T11:38:04.787Z [container-agent]     response = self.opener.open(request, timeout=self.timeout)
2024-03-03T11:38:04.787Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 519, in open
2024-03-03T11:38:04.787Z [container-agent]     response = self._open(req, data)
2024-03-03T11:38:04.787Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 536, in _open
2024-03-03T11:38:04.787Z [container-agent]     result = self._call_chain(self.handle_open, protocol, protocol +
2024-03-03T11:38:04.787Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 496, in _call_chain
2024-03-03T11:38:04.787Z [container-agent]     result = func(*args)
2024-03-03T11:38:04.787Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 311, in http_open
2024-03-03T11:38:04.788Z [container-agent]     return self.do_open(_UnixSocketConnection, req,  # type:ignore
2024-03-03T11:38:04.788Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 1352, in do_open
2024-03-03T11:38:04.788Z [container-agent]     r = h.getresponse()
2024-03-03T11:38:04.788Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1375, in getresponse
2024-03-03T11:38:04.788Z [container-agent]     response.begin()
2024-03-03T11:38:04.788Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 318, in begin
2024-03-03T11:38:04.788Z [container-agent]     version, status, reason = self._read_status()
2024-03-03T11:38:04.788Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 287, in _read_status
2024-03-03T11:38:04.788Z [container-agent]     raise RemoteDisconnected("Remote end closed connection without"
2024-03-03T11:38:04.788Z [container-agent] http.client.RemoteDisconnected: Remote end closed connection without response
2024-03-03T11:38:04.988Z [container-agent] 2024-03-03 11:38:04 INFO juju.util.exec exec.go:209 run result: exit status 1
2024-03-03T11:38:07.015Z [container-agent] 2024-03-03 11:38:07 INFO juju-log Setting up the logrotate configurations
2024-03-03T11:38:07.095Z [container-agent] 2024-03-03 11:38:07 INFO juju-log Adding pebble layer
2024-03-03T11:38:13.851Z [container-agent] 2024-03-03 11:38:13 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-03-03T11:38:13.874Z [container-agent] 2024-03-03 11:38:13 INFO juju-log Attempting reboot from complete outage.
2024-03-03T11:38:16.944Z [container-agent] 2024-03-03 11:38:16 INFO juju.worker.uniter.operation runhook.go:186 ran "mysql-pebble-ready" hook (via hook dispatching script: dispatch)
2024-03-03T11:39:43.721Z [container-agent] 2024-03-03 11:39:43 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T11:39:45.717Z [container-agent] 2024-03-03 11:39:45 WARNING juju-log No relation: certificates
2024-03-03T11:39:46.071Z [container-agent] 2024-03-03 11:39:46 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T11:45:21.082Z [container-agent] 2024-03-03 11:45:21 INFO juju-log Starting the log rotate manager
2024-03-03T11:45:21.095Z [container-agent] 2024-03-03 11:45:21 INFO juju-log Started log rotate manager process with PID 686933
2024-03-03T11:45:21.462Z [container-agent] 2024-03-03 11:45:21 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T11:45:23.314Z [container-agent] 2024-03-03 11:45:23 WARNING juju-log No relation: certificates
2024-03-03T11:45:23.658Z [container-agent] 2024-03-03 11:45:23 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T11:49:40.567Z [container-agent] 2024-03-03 11:49:40 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T11:49:42.853Z [container-agent] 2024-03-03 11:49:42 WARNING juju-log No relation: certificates
2024-03-03T11:49:43.204Z [container-agent] 2024-03-03 11:49:43 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T11:54:13.531Z [container-agent] 2024-03-03 11:54:13 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T11:54:16.664Z [container-agent] 2024-03-03 11:54:16 WARNING juju-log No relation: certificates
2024-03-03T11:54:17.033Z [container-agent] 2024-03-03 11:54:17 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T11:59:30.797Z [container-agent] 2024-03-03 11:59:30 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T11:59:34.434Z [container-agent] 2024-03-03 11:59:34 WARNING juju-log No relation: certificates
2024-03-03T11:59:34.774Z [container-agent] 2024-03-03 11:59:34 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T12:05:19.763Z [container-agent] 2024-03-03 12:05:19 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T12:05:23.826Z [container-agent] 2024-03-03 12:05:23 WARNING juju-log No relation: certificates
2024-03-03T12:05:24.170Z [container-agent] 2024-03-03 12:05:24 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T12:10:47.084Z [container-agent] 2024-03-03 12:10:47 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T12:10:51.558Z [container-agent] 2024-03-03 12:10:51 WARNING juju-log No relation: certificates
2024-03-03T12:10:51.911Z [container-agent] 2024-03-03 12:10:51 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T12:15:46.094Z [container-agent] 2024-03-03 12:15:46 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T12:15:50.973Z [container-agent] 2024-03-03 12:15:50 WARNING juju-log No relation: certificates
2024-03-03T12:15:51.317Z [container-agent] 2024-03-03 12:15:51 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T12:20:26.883Z [container-agent] 2024-03-03 12:20:26 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T12:20:32.243Z [container-agent] 2024-03-03 12:20:32 WARNING juju-log No relation: certificates
2024-03-03T12:20:32.596Z [container-agent] 2024-03-03 12:20:32 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T12:26:19.149Z [container-agent] 2024-03-03 12:26:19 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T12:26:24.954Z [container-agent] 2024-03-03 12:26:24 WARNING juju-log No relation: certificates
2024-03-03T12:26:25.320Z [container-agent] 2024-03-03 12:26:25 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T12:31:20.657Z [container-agent] 2024-03-03 12:31:20 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T12:31:27.001Z [container-agent] 2024-03-03 12:31:27 WARNING juju-log No relation: certificates
2024-03-03T12:31:27.355Z [container-agent] 2024-03-03 12:31:27 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T12:36:22.828Z [container-agent] 2024-03-03 12:36:22 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T12:36:29.421Z [container-agent] 2024-03-03 12:36:29 WARNING juju-log No relation: certificates
2024-03-03T12:36:29.800Z [container-agent] 2024-03-03 12:36:29 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T12:41:58.345Z [container-agent] 2024-03-03 12:41:58 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T12:42:05.270Z [container-agent] 2024-03-03 12:42:05 WARNING juju-log No relation: certificates
2024-03-03T12:42:05.607Z [container-agent] 2024-03-03 12:42:05 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T12:47:33.692Z [container-agent] 2024-03-03 12:47:33 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T12:47:41.556Z [container-agent] 2024-03-03 12:47:41 WARNING juju-log No relation: certificates
2024-03-03T12:47:41.917Z [container-agent] 2024-03-03 12:47:41 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T12:51:40.611Z [container-agent] 2024-03-03 12:51:40 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T12:51:48.701Z [container-agent] 2024-03-03 12:51:48 WARNING juju-log No relation: certificates
2024-03-03T12:51:49.050Z [container-agent] 2024-03-03 12:51:49 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T12:57:14.879Z [container-agent] 2024-03-03 12:57:14 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T12:57:23.516Z [container-agent] 2024-03-03 12:57:23 WARNING juju-log No relation: certificates
2024-03-03T12:57:23.914Z [container-agent] 2024-03-03 12:57:23 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T13:02:15.647Z [container-agent] 2024-03-03 13:02:15 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-03-03T13:02:15.678Z [container-agent] 2024-03-03 13:02:15 INFO juju-log Attempting reboot from complete outage.
2024-03-03T13:02:24.039Z [container-agent] 2024-03-03 13:02:24 WARNING juju-log No relation: certificates
2024-03-03T13:02:24.404Z [container-agent] 2024-03-03 13:02:24 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T13:05:09.786Z [container-agent] 2024-03-03 13:05:09 ERROR juju-log Uncaught exception while in charm code:
2024-03-03T13:05:09.786Z [container-agent] Traceback (most recent call last):
2024-03-03T13:05:09.786Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 2507, in exec
2024-03-03T13:05:09.786Z [container-agent]     control_ws = self._connect_websocket(task_id, 'control')
2024-03-03T13:05:09.786Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 2589, in _connect_websocket
2024-03-03T13:05:09.786Z [container-agent]     ws.connect(url, socket=sock)
2024-03-03T13:05:09.786Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_core.py", line 253, in connect
2024-03-03T13:05:09.786Z [container-agent]     self.handshake_response = handshake(self.sock, url, *addrs, **options)
2024-03-03T13:05:09.786Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_handshake.py", line 57, in handshake
2024-03-03T13:05:09.786Z [container-agent]     status, resp = _get_resp_headers(sock)
2024-03-03T13:05:09.786Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_handshake.py", line 143, in _get_resp_headers
2024-03-03T13:05:09.786Z [container-agent]     status, resp_headers, status_message = read_headers(sock)
2024-03-03T13:05:09.786Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_http.py", line 312, in read_headers
2024-03-03T13:05:09.786Z [container-agent]     line = recv_line(sock)
2024-03-03T13:05:09.786Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_socket.py", line 131, in recv_line
2024-03-03T13:05:09.786Z [container-agent]     c = recv(sock, 1)
2024-03-03T13:05:09.786Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_socket.py", line 122, in recv
2024-03-03T13:05:09.786Z [container-agent]     raise WebSocketConnectionClosedException(
2024-03-03T13:05:09.786Z [container-agent] websocket._exceptions.WebSocketConnectionClosedException: Connection to remote host was lost.
2024-03-03T13:05:09.786Z [container-agent] 
2024-03-03T13:05:09.786Z [container-agent] During handling of the above exception, another exception occurred:
2024-03-03T13:05:09.786Z [container-agent] 
2024-03-03T13:05:09.786Z [container-agent] Traceback (most recent call last):
2024-03-03T13:05:09.786Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 1348, in do_open
2024-03-03T13:05:09.786Z [container-agent]     h.request(req.get_method(), req.selector, req.data, headers,
2024-03-03T13:05:09.786Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1283, in request
2024-03-03T13:05:09.786Z [container-agent]     self._send_request(method, url, body, headers, encode_chunked)
2024-03-03T13:05:09.786Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1329, in _send_request
2024-03-03T13:05:09.786Z [container-agent]     self.endheaders(body, encode_chunked=encode_chunked)
2024-03-03T13:05:09.786Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1278, in endheaders
2024-03-03T13:05:09.786Z [container-agent]     self._send_output(message_body, encode_chunked=encode_chunked)
2024-03-03T13:05:09.786Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1038, in _send_output
2024-03-03T13:05:09.786Z [container-agent]     self.send(msg)
2024-03-03T13:05:09.786Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 976, in send
2024-03-03T13:05:09.786Z [container-agent]     self.connect()
2024-03-03T13:05:09.786Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 297, in connect
2024-03-03T13:05:09.786Z [container-agent]     self.sock.connect(self.socket_path)
2024-03-03T13:05:09.786Z [container-agent] ConnectionRefusedError: [Errno 111] Connection refused
2024-03-03T13:05:09.786Z [container-agent] 
2024-03-03T13:05:09.786Z [container-agent] During handling of the above exception, another exception occurred:
2024-03-03T13:05:09.786Z [container-agent] 
2024-03-03T13:05:09.786Z [container-agent] Traceback (most recent call last):
2024-03-03T13:05:09.786Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1690, in _request_raw
2024-03-03T13:05:09.786Z [container-agent]     response = self.opener.open(request, timeout=self.timeout)
2024-03-03T13:05:09.786Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 519, in open
2024-03-03T13:05:09.786Z [container-agent]     response = self._open(req, data)
2024-03-03T13:05:09.786Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 536, in _open
2024-03-03T13:05:09.786Z [container-agent]     result = self._call_chain(self.handle_open, protocol, protocol +
2024-03-03T13:05:09.786Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 496, in _call_chain
2024-03-03T13:05:09.786Z [container-agent]     result = func(*args)
2024-03-03T13:05:09.786Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 311, in http_open
2024-03-03T13:05:09.786Z [container-agent]     return self.do_open(_UnixSocketConnection, req,  # type:ignore
2024-03-03T13:05:09.786Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 1351, in do_open
2024-03-03T13:05:09.786Z [container-agent]     raise URLError(err)
2024-03-03T13:05:09.786Z [container-agent] urllib.error.URLError: <urlopen error [Errno 111] Connection refused>
2024-03-03T13:05:09.786Z [container-agent] 
2024-03-03T13:05:09.786Z [container-agent] The above exception was the direct cause of the following exception:
2024-03-03T13:05:09.786Z [container-agent] 
2024-03-03T13:05:09.786Z [container-agent] Traceback (most recent call last):
2024-03-03T13:05:09.786Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/./src/charm.py", line 786, in <module>
2024-03-03T13:05:09.786Z [container-agent]     main(MySQLOperatorCharm)
2024-03-03T13:05:09.786Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 436, in main
2024-03-03T13:05:09.786Z [container-agent]     _emit_charm_event(charm, dispatcher.event_name)
2024-03-03T13:05:09.786Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 144, in _emit_charm_event
2024-03-03T13:05:09.786Z [container-agent]     event_to_emit.emit(*args, **kwargs)
2024-03-03T13:05:09.786Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 351, in emit
2024-03-03T13:05:09.786Z [container-agent]     framework._emit(event)
2024-03-03T13:05:09.786Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 853, in _emit
2024-03-03T13:05:09.786Z [container-agent]     self._reemit(event_path)
2024-03-03T13:05:09.786Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 942, in _reemit
2024-03-03T13:05:09.786Z [container-agent]     custom_handler(event)
2024-03-03T13:05:09.786Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/rotate_mysql_logs.py", line 57, in _rotate_mysql_logs
2024-03-03T13:05:09.786Z [container-agent]     self.charm._mysql.flush_mysql_logs(MySQLTextLogs.SLOW)
2024-03-03T13:05:09.787Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-03-03T13:05:09.787Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-03-03T13:05:09.787Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 665, in _run_mysqlsh_script
2024-03-03T13:05:09.787Z [container-agent]     process = self.container.exec(cmd, timeout=timeout)
2024-03-03T13:05:09.787Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/model.py", line 2688, in exec
2024-03-03T13:05:09.787Z [container-agent]     return self._pebble.exec(
2024-03-03T13:05:09.787Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 2514, in exec
2024-03-03T13:05:09.787Z [container-agent]     change = self.wait_change(ChangeID(change_id))
2024-03-03T13:05:09.787Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1893, in wait_change
2024-03-03T13:05:09.787Z [container-agent]     return self._wait_change_using_wait(change_id, timeout)
2024-03-03T13:05:09.787Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1914, in _wait_change_using_wait
2024-03-03T13:05:09.787Z [container-agent]     return self._wait_change(change_id, this_timeout)
2024-03-03T13:05:09.787Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1928, in _wait_change
2024-03-03T13:05:09.787Z [container-agent]     resp = self._request('GET', f'/v1/changes/{change_id}/wait', query)
2024-03-03T13:05:09.787Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1655, in _request
2024-03-03T13:05:09.787Z [container-agent]     response = self._request_raw(method, path, query, headers, data)
2024-03-03T13:05:09.787Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1707, in _request_raw
2024-03-03T13:05:09.787Z [container-agent]     raise ConnectionError(e.reason) from e
2024-03-03T13:05:09.787Z [container-agent] ops.pebble.ConnectionError: [Errno 111] Connection refused
2024-03-03T13:05:09.937Z [container-agent] 2024-03-03 13:05:09 INFO juju.util.exec exec.go:209 run result: exit status 1
2024-03-03T13:05:15.526Z [container-agent] 2024-03-03 13:05:15 INFO juju-log Setting up the logrotate configurations
2024-03-03T13:05:15.615Z [container-agent] 2024-03-03 13:05:15 INFO juju-log Adding pebble layer
2024-03-03T13:05:22.391Z [container-agent] 2024-03-03 13:05:22 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-03-03T13:05:22.416Z [container-agent] 2024-03-03 13:05:22 INFO juju-log Attempting reboot from complete outage.
2024-03-03T13:05:25.370Z [container-agent] 2024-03-03 13:05:25 INFO juju.worker.uniter.operation runhook.go:186 ran "mysql-pebble-ready" hook (via hook dispatching script: dispatch)
2024-03-03T13:07:57.090Z [container-agent] 2024-03-03 13:07:57 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T13:07:58.921Z [container-agent] 2024-03-03 13:07:58 WARNING juju-log No relation: certificates
2024-03-03T13:07:59.271Z [container-agent] 2024-03-03 13:07:59 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T13:13:29.641Z [container-agent] 2024-03-03 13:13:29 INFO juju-log Starting the log rotate manager
2024-03-03T13:13:29.652Z [container-agent] 2024-03-03 13:13:29 INFO juju-log Started log rotate manager process with PID 694948
2024-03-03T13:13:29.920Z [container-agent] 2024-03-03 13:13:29 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T13:13:32.009Z [container-agent] 2024-03-03 13:13:32 WARNING juju-log No relation: certificates
2024-03-03T13:13:32.386Z [container-agent] 2024-03-03 13:13:32 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T13:19:20.062Z [container-agent] 2024-03-03 13:19:20 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T13:19:22.574Z [container-agent] 2024-03-03 13:19:22 WARNING juju-log No relation: certificates
2024-03-03T13:19:22.899Z [container-agent] 2024-03-03 13:19:22 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T13:23:34.614Z [container-agent] 2024-03-03 13:23:34 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T13:23:37.395Z [container-agent] 2024-03-03 13:23:37 WARNING juju-log No relation: certificates
2024-03-03T13:23:37.745Z [container-agent] 2024-03-03 13:23:37 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T13:27:35.636Z [container-agent] 2024-03-03 13:27:35 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T13:27:39.125Z [container-agent] 2024-03-03 13:27:39 WARNING juju-log No relation: certificates
2024-03-03T13:27:39.465Z [container-agent] 2024-03-03 13:27:39 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T13:31:52.172Z [container-agent] 2024-03-03 13:31:52 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T13:31:55.995Z [container-agent] 2024-03-03 13:31:55 WARNING juju-log No relation: certificates
2024-03-03T13:31:56.349Z [container-agent] 2024-03-03 13:31:56 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T13:36:48.777Z [container-agent] 2024-03-03 13:36:48 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T13:36:53.053Z [container-agent] 2024-03-03 13:36:53 WARNING juju-log No relation: certificates
2024-03-03T13:36:53.401Z [container-agent] 2024-03-03 13:36:53 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T13:42:15.237Z [container-agent] 2024-03-03 13:42:15 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T13:42:20.264Z [container-agent] 2024-03-03 13:42:20 WARNING juju-log No relation: certificates
2024-03-03T13:42:20.634Z [container-agent] 2024-03-03 13:42:20 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T13:46:57.902Z [container-agent] 2024-03-03 13:46:57 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T13:47:02.988Z [container-agent] 2024-03-03 13:47:02 WARNING juju-log No relation: certificates
2024-03-03T13:47:03.323Z [container-agent] 2024-03-03 13:47:03 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T13:51:29.927Z [container-agent] 2024-03-03 13:51:29 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T13:51:35.660Z [container-agent] 2024-03-03 13:51:35 WARNING juju-log No relation: certificates
2024-03-03T13:51:36.013Z [container-agent] 2024-03-03 13:51:36 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T13:55:50.856Z [container-agent] 2024-03-03 13:55:50 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T13:55:56.934Z [container-agent] 2024-03-03 13:55:56 WARNING juju-log No relation: certificates
2024-03-03T13:55:57.293Z [container-agent] 2024-03-03 13:55:57 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T14:01:11.829Z [container-agent] 2024-03-03 14:01:11 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T14:01:18.539Z [container-agent] 2024-03-03 14:01:18 WARNING juju-log No relation: certificates
2024-03-03T14:01:18.890Z [container-agent] 2024-03-03 14:01:18 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T14:06:15.657Z [container-agent] 2024-03-03 14:06:15 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T14:06:22.838Z [container-agent] 2024-03-03 14:06:22 WARNING juju-log No relation: certificates
2024-03-03T14:06:23.185Z [container-agent] 2024-03-03 14:06:23 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T14:11:37.439Z [container-agent] 2024-03-03 14:11:37 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T14:11:44.912Z [container-agent] 2024-03-03 14:11:44 WARNING juju-log No relation: certificates
2024-03-03T14:11:45.315Z [container-agent] 2024-03-03 14:11:45 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T14:16:53.451Z [container-agent] 2024-03-03 14:16:53 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T14:17:01.335Z [container-agent] 2024-03-03 14:17:01 WARNING juju-log No relation: certificates
2024-03-03T14:17:01.685Z [container-agent] 2024-03-03 14:17:01 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T14:21:19.757Z [container-agent] 2024-03-03 14:21:19 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-03-03T14:21:19.789Z [container-agent] 2024-03-03 14:21:19 INFO juju-log Attempting reboot from complete outage.
2024-03-03T14:21:27.354Z [container-agent] 2024-03-03 14:21:27 WARNING juju-log No relation: certificates
2024-03-03T14:21:27.717Z [container-agent] 2024-03-03 14:21:27 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T14:23:04.794Z [container-agent] 2024-03-03 14:23:04 ERROR juju-log Uncaught exception while in charm code:
2024-03-03T14:23:04.794Z [container-agent] Traceback (most recent call last):
2024-03-03T14:23:04.794Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/./src/charm.py", line 786, in <module>
2024-03-03T14:23:04.794Z [container-agent]     main(MySQLOperatorCharm)
2024-03-03T14:23:04.794Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 436, in main
2024-03-03T14:23:04.794Z [container-agent]     _emit_charm_event(charm, dispatcher.event_name)
2024-03-03T14:23:04.794Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 144, in _emit_charm_event
2024-03-03T14:23:04.794Z [container-agent]     event_to_emit.emit(*args, **kwargs)
2024-03-03T14:23:04.794Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 351, in emit
2024-03-03T14:23:04.794Z [container-agent]     framework._emit(event)
2024-03-03T14:23:04.794Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 853, in _emit
2024-03-03T14:23:04.794Z [container-agent]     self._reemit(event_path)
2024-03-03T14:23:04.794Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 942, in _reemit
2024-03-03T14:23:04.794Z [container-agent]     custom_handler(event)
2024-03-03T14:23:04.794Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/rotate_mysql_logs.py", line 55, in _rotate_mysql_logs
2024-03-03T14:23:04.794Z [container-agent]     self.charm._mysql.flush_mysql_logs(MySQLTextLogs.ERROR)
2024-03-03T14:23:04.794Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-03-03T14:23:04.794Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-03-03T14:23:04.794Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 666, in _run_mysqlsh_script
2024-03-03T14:23:04.794Z [container-agent]     stdout, _ = process.wait_output()
2024-03-03T14:23:04.794Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1436, in wait_output
2024-03-03T14:23:04.794Z [container-agent]     exit_code: int = self._wait()
2024-03-03T14:23:04.794Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1375, in _wait
2024-03-03T14:23:04.794Z [container-agent]     change = self._client.wait_change(self._change_id, timeout=timeout)
2024-03-03T14:23:04.794Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1893, in wait_change
2024-03-03T14:23:04.794Z [container-agent]     return self._wait_change_using_wait(change_id, timeout)
2024-03-03T14:23:04.794Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1914, in _wait_change_using_wait
2024-03-03T14:23:04.794Z [container-agent]     return self._wait_change(change_id, this_timeout)
2024-03-03T14:23:04.794Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1928, in _wait_change
2024-03-03T14:23:04.794Z [container-agent]     resp = self._request('GET', f'/v1/changes/{change_id}/wait', query)
2024-03-03T14:23:04.794Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1655, in _request
2024-03-03T14:23:04.794Z [container-agent]     response = self._request_raw(method, path, query, headers, data)
2024-03-03T14:23:04.794Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1690, in _request_raw
2024-03-03T14:23:04.794Z [container-agent]     response = self.opener.open(request, timeout=self.timeout)
2024-03-03T14:23:04.794Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 519, in open
2024-03-03T14:23:04.794Z [container-agent]     response = self._open(req, data)
2024-03-03T14:23:04.794Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 536, in _open
2024-03-03T14:23:04.794Z [container-agent]     result = self._call_chain(self.handle_open, protocol, protocol +
2024-03-03T14:23:04.794Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 496, in _call_chain
2024-03-03T14:23:04.794Z [container-agent]     result = func(*args)
2024-03-03T14:23:04.794Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 311, in http_open
2024-03-03T14:23:04.794Z [container-agent]     return self.do_open(_UnixSocketConnection, req,  # type:ignore
2024-03-03T14:23:04.794Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 1352, in do_open
2024-03-03T14:23:04.794Z [container-agent]     r = h.getresponse()
2024-03-03T14:23:04.794Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1375, in getresponse
2024-03-03T14:23:04.794Z [container-agent]     response.begin()
2024-03-03T14:23:04.794Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 318, in begin
2024-03-03T14:23:04.794Z [container-agent]     version, status, reason = self._read_status()
2024-03-03T14:23:04.794Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 287, in _read_status
2024-03-03T14:23:04.794Z [container-agent]     raise RemoteDisconnected("Remote end closed connection without"
2024-03-03T14:23:04.794Z [container-agent] http.client.RemoteDisconnected: Remote end closed connection without response
2024-03-03T14:23:04.980Z [container-agent] 2024-03-03 14:23:04 INFO juju.util.exec exec.go:209 run result: exit status 1
2024-03-03T14:23:08.453Z [container-agent] 2024-03-03 14:23:08 INFO juju-log Setting up the logrotate configurations
2024-03-03T14:23:08.535Z [container-agent] 2024-03-03 14:23:08 INFO juju-log Adding pebble layer
2024-03-03T14:23:15.280Z [container-agent] 2024-03-03 14:23:15 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-03-03T14:23:15.302Z [container-agent] 2024-03-03 14:23:15 INFO juju-log Attempting reboot from complete outage.
2024-03-03T14:23:18.418Z [container-agent] 2024-03-03 14:23:18 INFO juju.worker.uniter.operation runhook.go:186 ran "mysql-pebble-ready" hook (via hook dispatching script: dispatch)
2024-03-03T14:26:12.453Z [container-agent] 2024-03-03 14:26:12 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T14:26:14.355Z [container-agent] 2024-03-03 14:26:14 WARNING juju-log No relation: certificates
2024-03-03T14:26:14.716Z [container-agent] 2024-03-03 14:26:14 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T14:31:14.612Z [container-agent] 2024-03-03 14:31:14 INFO juju-log Starting the log rotate manager
2024-03-03T14:31:14.624Z [container-agent] 2024-03-03 14:31:14 INFO juju-log Started log rotate manager process with PID 702143
2024-03-03T14:31:14.907Z [container-agent] 2024-03-03 14:31:14 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T14:31:16.975Z [container-agent] 2024-03-03 14:31:16 WARNING juju-log No relation: certificates
2024-03-03T14:31:17.354Z [container-agent] 2024-03-03 14:31:17 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T14:35:32.724Z [container-agent] 2024-03-03 14:35:32 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T14:35:35.256Z [container-agent] 2024-03-03 14:35:35 WARNING juju-log No relation: certificates
2024-03-03T14:35:35.602Z [container-agent] 2024-03-03 14:35:35 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T14:40:06.870Z [container-agent] 2024-03-03 14:40:06 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T14:40:09.629Z [container-agent] 2024-03-03 14:40:09 WARNING juju-log No relation: certificates
2024-03-03T14:40:09.987Z [container-agent] 2024-03-03 14:40:09 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T14:44:28.094Z [container-agent] 2024-03-03 14:44:28 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T14:44:31.364Z [container-agent] 2024-03-03 14:44:31 WARNING juju-log No relation: certificates
2024-03-03T14:44:31.705Z [container-agent] 2024-03-03 14:44:31 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T14:48:36.007Z [container-agent] 2024-03-03 14:48:36 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T14:48:39.511Z [container-agent] 2024-03-03 14:48:39 WARNING juju-log No relation: certificates
2024-03-03T14:48:39.847Z [container-agent] 2024-03-03 14:48:39 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T14:54:35.426Z [container-agent] 2024-03-03 14:54:35 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T14:54:39.392Z [container-agent] 2024-03-03 14:54:39 WARNING juju-log No relation: certificates
2024-03-03T14:54:39.723Z [container-agent] 2024-03-03 14:54:39 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T15:00:09.041Z [container-agent] 2024-03-03 15:00:09 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T15:00:13.986Z [container-agent] 2024-03-03 15:00:13 WARNING juju-log No relation: certificates
2024-03-03T15:00:14.344Z [container-agent] 2024-03-03 15:00:14 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T15:05:35.103Z [container-agent] 2024-03-03 15:05:35 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T15:05:40.204Z [container-agent] 2024-03-03 15:05:40 WARNING juju-log No relation: certificates
2024-03-03T15:05:40.542Z [container-agent] 2024-03-03 15:05:40 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T15:10:55.829Z [container-agent] 2024-03-03 15:10:55 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T15:11:01.575Z [container-agent] 2024-03-03 15:11:01 WARNING juju-log No relation: certificates
2024-03-03T15:11:01.955Z [container-agent] 2024-03-03 15:11:01 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T15:16:28.448Z [container-agent] 2024-03-03 15:16:28 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T15:16:34.892Z [container-agent] 2024-03-03 15:16:34 WARNING juju-log No relation: certificates
2024-03-03T15:16:35.240Z [container-agent] 2024-03-03 15:16:35 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T15:21:31.147Z [container-agent] 2024-03-03 15:21:31 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T15:21:37.879Z [container-agent] 2024-03-03 15:21:37 WARNING juju-log No relation: certificates
2024-03-03T15:21:38.250Z [container-agent] 2024-03-03 15:21:38 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T15:26:25.947Z [container-agent] 2024-03-03 15:26:25 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T15:26:32.972Z [container-agent] 2024-03-03 15:26:32 WARNING juju-log No relation: certificates
2024-03-03T15:26:33.390Z [container-agent] 2024-03-03 15:26:33 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T15:30:54.293Z [container-agent] 2024-03-03 15:30:54 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T15:31:01.565Z [container-agent] 2024-03-03 15:31:01 WARNING juju-log No relation: certificates
2024-03-03T15:31:01.909Z [container-agent] 2024-03-03 15:31:01 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T15:36:41.046Z [container-agent] 2024-03-03 15:36:41 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T15:36:49.259Z [container-agent] 2024-03-03 15:36:49 WARNING juju-log No relation: certificates
2024-03-03T15:36:49.619Z [container-agent] 2024-03-03 15:36:49 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T15:41:20.819Z [container-agent] 2024-03-03 15:41:20 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-03-03T15:41:20.851Z [container-agent] 2024-03-03 15:41:20 INFO juju-log Attempting reboot from complete outage.
2024-03-03T15:41:28.766Z [container-agent] 2024-03-03 15:41:28 WARNING juju-log No relation: certificates
2024-03-03T15:41:29.128Z [container-agent] 2024-03-03 15:41:29 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T15:44:09.764Z [container-agent] 2024-03-03 15:44:09 ERROR juju-log Uncaught exception while in charm code:
2024-03-03T15:44:09.764Z [container-agent] Traceback (most recent call last):
2024-03-03T15:44:09.764Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/./src/charm.py", line 786, in <module>
2024-03-03T15:44:09.764Z [container-agent]     main(MySQLOperatorCharm)
2024-03-03T15:44:09.764Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 436, in main
2024-03-03T15:44:09.764Z [container-agent]     _emit_charm_event(charm, dispatcher.event_name)
2024-03-03T15:44:09.764Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 144, in _emit_charm_event
2024-03-03T15:44:09.764Z [container-agent]     event_to_emit.emit(*args, **kwargs)
2024-03-03T15:44:09.764Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 351, in emit
2024-03-03T15:44:09.764Z [container-agent]     framework._emit(event)
2024-03-03T15:44:09.764Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 853, in _emit
2024-03-03T15:44:09.764Z [container-agent]     self._reemit(event_path)
2024-03-03T15:44:09.764Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 942, in _reemit
2024-03-03T15:44:09.764Z [container-agent]     custom_handler(event)
2024-03-03T15:44:09.764Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/rotate_mysql_logs.py", line 57, in _rotate_mysql_logs
2024-03-03T15:44:09.764Z [container-agent]     self.charm._mysql.flush_mysql_logs(MySQLTextLogs.SLOW)
2024-03-03T15:44:09.764Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-03-03T15:44:09.764Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-03-03T15:44:09.764Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 666, in _run_mysqlsh_script
2024-03-03T15:44:09.764Z [container-agent]     stdout, _ = process.wait_output()
2024-03-03T15:44:09.764Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1436, in wait_output
2024-03-03T15:44:09.764Z [container-agent]     exit_code: int = self._wait()
2024-03-03T15:44:09.764Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1375, in _wait
2024-03-03T15:44:09.764Z [container-agent]     change = self._client.wait_change(self._change_id, timeout=timeout)
2024-03-03T15:44:09.764Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1893, in wait_change
2024-03-03T15:44:09.764Z [container-agent]     return self._wait_change_using_wait(change_id, timeout)
2024-03-03T15:44:09.764Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1914, in _wait_change_using_wait
2024-03-03T15:44:09.764Z [container-agent]     return self._wait_change(change_id, this_timeout)
2024-03-03T15:44:09.764Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1928, in _wait_change
2024-03-03T15:44:09.764Z [container-agent]     resp = self._request('GET', f'/v1/changes/{change_id}/wait', query)
2024-03-03T15:44:09.764Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1655, in _request
2024-03-03T15:44:09.764Z [container-agent]     response = self._request_raw(method, path, query, headers, data)
2024-03-03T15:44:09.764Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1690, in _request_raw
2024-03-03T15:44:09.764Z [container-agent]     response = self.opener.open(request, timeout=self.timeout)
2024-03-03T15:44:09.764Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 519, in open
2024-03-03T15:44:09.764Z [container-agent]     response = self._open(req, data)
2024-03-03T15:44:09.764Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 536, in _open
2024-03-03T15:44:09.764Z [container-agent]     result = self._call_chain(self.handle_open, protocol, protocol +
2024-03-03T15:44:09.764Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 496, in _call_chain
2024-03-03T15:44:09.764Z [container-agent]     result = func(*args)
2024-03-03T15:44:09.764Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 311, in http_open
2024-03-03T15:44:09.764Z [container-agent]     return self.do_open(_UnixSocketConnection, req,  # type:ignore
2024-03-03T15:44:09.764Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 1352, in do_open
2024-03-03T15:44:09.764Z [container-agent]     r = h.getresponse()
2024-03-03T15:44:09.764Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1375, in getresponse
2024-03-03T15:44:09.764Z [container-agent]     response.begin()
2024-03-03T15:44:09.764Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 318, in begin
2024-03-03T15:44:09.764Z [container-agent]     version, status, reason = self._read_status()
2024-03-03T15:44:09.764Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 287, in _read_status
2024-03-03T15:44:09.764Z [container-agent]     raise RemoteDisconnected("Remote end closed connection without"
2024-03-03T15:44:09.764Z [container-agent] http.client.RemoteDisconnected: Remote end closed connection without response
2024-03-03T15:44:09.895Z [container-agent] 2024-03-03 15:44:09 INFO juju.util.exec exec.go:209 run result: exit status 1
2024-03-03T15:44:11.759Z [container-agent] 2024-03-03 15:44:11 INFO juju-log Setting up the logrotate configurations
2024-03-03T15:44:11.835Z [container-agent] 2024-03-03 15:44:11 INFO juju-log Adding pebble layer
2024-03-03T15:44:18.643Z [container-agent] 2024-03-03 15:44:18 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-03-03T15:44:18.665Z [container-agent] 2024-03-03 15:44:18 INFO juju-log Attempting reboot from complete outage.
2024-03-03T15:44:21.576Z [container-agent] 2024-03-03 15:44:21 INFO juju.worker.uniter.operation runhook.go:186 ran "mysql-pebble-ready" hook (via hook dispatching script: dispatch)
2024-03-03T15:46:27.764Z [container-agent] 2024-03-03 15:46:27 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T15:46:29.605Z [container-agent] 2024-03-03 15:46:29 WARNING juju-log No relation: certificates
2024-03-03T15:46:29.963Z [container-agent] 2024-03-03 15:46:29 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T15:51:20.050Z [container-agent] 2024-03-03 15:51:20 INFO juju-log Starting the log rotate manager
2024-03-03T15:51:20.062Z [container-agent] 2024-03-03 15:51:20 INFO juju-log Started log rotate manager process with PID 709642
2024-03-03T15:51:20.359Z [container-agent] 2024-03-03 15:51:20 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T15:51:22.287Z [container-agent] 2024-03-03 15:51:22 WARNING juju-log No relation: certificates
2024-03-03T15:51:22.694Z [container-agent] 2024-03-03 15:51:22 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T15:55:50.810Z [container-agent] 2024-03-03 15:55:50 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T15:55:53.150Z [container-agent] 2024-03-03 15:55:53 WARNING juju-log No relation: certificates
2024-03-03T15:55:53.481Z [container-agent] 2024-03-03 15:55:53 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T16:00:16.605Z [container-agent] 2024-03-03 16:00:16 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T16:00:19.641Z [container-agent] 2024-03-03 16:00:19 WARNING juju-log No relation: certificates
2024-03-03T16:00:20.001Z [container-agent] 2024-03-03 16:00:20 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T16:04:50.832Z [container-agent] 2024-03-03 16:04:50 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T16:04:54.015Z [container-agent] 2024-03-03 16:04:54 WARNING juju-log No relation: certificates
2024-03-03T16:04:54.367Z [container-agent] 2024-03-03 16:04:54 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T16:09:07.635Z [container-agent] 2024-03-03 16:09:07 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T16:09:11.148Z [container-agent] 2024-03-03 16:09:11 WARNING juju-log No relation: certificates
2024-03-03T16:09:11.499Z [container-agent] 2024-03-03 16:09:11 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T16:13:55.181Z [container-agent] 2024-03-03 16:13:55 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T16:13:59.146Z [container-agent] 2024-03-03 16:13:59 WARNING juju-log No relation: certificates
2024-03-03T16:13:59.524Z [container-agent] 2024-03-03 16:13:59 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T16:19:41.548Z [container-agent] 2024-03-03 16:19:41 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T16:19:46.209Z [container-agent] 2024-03-03 16:19:46 WARNING juju-log No relation: certificates
2024-03-03T16:19:46.571Z [container-agent] 2024-03-03 16:19:46 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T16:24:46.435Z [container-agent] 2024-03-03 16:24:46 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T16:24:51.461Z [container-agent] 2024-03-03 16:24:51 WARNING juju-log No relation: certificates
2024-03-03T16:24:51.802Z [container-agent] 2024-03-03 16:24:51 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T16:29:45.957Z [container-agent] 2024-03-03 16:29:45 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T16:29:51.640Z [container-agent] 2024-03-03 16:29:51 WARNING juju-log No relation: certificates
2024-03-03T16:29:51.993Z [container-agent] 2024-03-03 16:29:51 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T16:33:56.503Z [container-agent] 2024-03-03 16:33:56 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T16:34:02.370Z [container-agent] 2024-03-03 16:34:02 WARNING juju-log No relation: certificates
2024-03-03T16:34:02.734Z [container-agent] 2024-03-03 16:34:02 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T16:37:59.772Z [container-agent] 2024-03-03 16:37:59 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T16:38:06.486Z [container-agent] 2024-03-03 16:38:06 WARNING juju-log No relation: certificates
2024-03-03T16:38:06.811Z [container-agent] 2024-03-03 16:38:06 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T16:43:42.146Z [container-agent] 2024-03-03 16:43:42 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T16:43:48.682Z [container-agent] 2024-03-03 16:43:48 WARNING juju-log No relation: certificates
2024-03-03T16:43:49.061Z [container-agent] 2024-03-03 16:43:49 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T16:48:17.981Z [container-agent] 2024-03-03 16:48:17 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T16:48:25.378Z [container-agent] 2024-03-03 16:48:25 WARNING juju-log No relation: certificates
2024-03-03T16:48:25.729Z [container-agent] 2024-03-03 16:48:25 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T16:49:08.935Z [container-agent] 2024-03-03 16:49:08 ERROR juju-log Failed to flush GENERAL LOGS logs.
2024-03-03T16:49:08.935Z [container-agent] Traceback (most recent call last):
2024-03-03T16:49:08.935Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 666, in _run_mysqlsh_script
2024-03-03T16:49:08.935Z [container-agent]     stdout, _ = process.wait_output()
2024-03-03T16:49:08.935Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1441, in wait_output
2024-03-03T16:49:08.935Z [container-agent]     raise ExecError[AnyStr](self._command, exit_code, out_value, err_value)
2024-03-03T16:49:08.935Z [container-agent] ops.pebble.ExecError: non-zero exit code 1 executing ['/usr/bin/mysqlsh', '--no-wizard', '--python', '--verbose=1', '-f', '/tmp/script.py', ';', 'rm', '/tmp/script.py'], stdout='', stderr='Cannot set LC_ALL to locale en_US.UTF-8: No such file or directory\nverbose: 2024-03-03T16:49:07Z: Loading startup files...\nverbose: 2024-03-03T16:49:07Z: Loading plugins...\nverbose: 2024-03-03T16:49:08Z: Connecting to MySQL at: serverconfig@horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local\nTraceback (most recent call last):\n  File "<string>", line 1, in <module>\nmysqlsh.DBError: MySQL Error (2003): Shell.connect: Can\'t connect to MySQL server on \'horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local:3306\' (111)\n'
2024-03-03T16:49:08.935Z [container-agent] 
2024-03-03T16:49:08.935Z [container-agent] During handling of the above exception, another exception occurred:
2024-03-03T16:49:08.935Z [container-agent] 
2024-03-03T16:49:08.935Z [container-agent] Traceback (most recent call last):
2024-03-03T16:49:08.935Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-03-03T16:49:08.935Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-03-03T16:49:08.935Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 669, in _run_mysqlsh_script
2024-03-03T16:49:08.935Z [container-agent]     raise MySQLClientError(e.stderr)
2024-03-03T16:49:08.935Z [container-agent] charms.mysql.v0.mysql.MySQLClientError: Cannot set LC_ALL to locale en_US.UTF-8: No such file or directory
2024-03-03T16:49:08.935Z [container-agent] verbose: 2024-03-03T16:49:07Z: Loading startup files...
2024-03-03T16:49:08.935Z [container-agent] verbose: 2024-03-03T16:49:07Z: Loading plugins...
2024-03-03T16:49:08.935Z [container-agent] verbose: 2024-03-03T16:49:08Z: Connecting to MySQL at: serverconfig@horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local
2024-03-03T16:49:08.935Z [container-agent] Traceback (most recent call last):
2024-03-03T16:49:08.935Z [container-agent]   File "<string>", line 1, in <module>
2024-03-03T16:49:08.935Z [container-agent] mysqlsh.DBError: MySQL Error (2003): Shell.connect: Can't connect to MySQL server on 'horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local:3306' (111)
2024-03-03T16:49:08.935Z [container-agent] 
2024-03-03T16:53:19.451Z [container-agent] 2024-03-03 16:53:19 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-03-03T16:53:19.484Z [container-agent] 2024-03-03 16:53:19 INFO juju-log Attempting reboot from complete outage.
2024-03-03T16:53:26.662Z [container-agent] 2024-03-03 16:53:26 WARNING juju-log No relation: certificates
2024-03-03T16:53:27.036Z [container-agent] 2024-03-03 16:53:27 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T16:54:04.828Z [container-agent] 2024-03-03 16:54:04 ERROR juju-log Uncaught exception while in charm code:
2024-03-03T16:54:04.828Z [container-agent] Traceback (most recent call last):
2024-03-03T16:54:04.828Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/./src/charm.py", line 786, in <module>
2024-03-03T16:54:04.828Z [container-agent]     main(MySQLOperatorCharm)
2024-03-03T16:54:04.828Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 436, in main
2024-03-03T16:54:04.828Z [container-agent]     _emit_charm_event(charm, dispatcher.event_name)
2024-03-03T16:54:04.828Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 144, in _emit_charm_event
2024-03-03T16:54:04.828Z [container-agent]     event_to_emit.emit(*args, **kwargs)
2024-03-03T16:54:04.828Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 351, in emit
2024-03-03T16:54:04.828Z [container-agent]     framework._emit(event)
2024-03-03T16:54:04.828Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 853, in _emit
2024-03-03T16:54:04.828Z [container-agent]     self._reemit(event_path)
2024-03-03T16:54:04.828Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 942, in _reemit
2024-03-03T16:54:04.828Z [container-agent]     custom_handler(event)
2024-03-03T16:54:04.828Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/rotate_mysql_logs.py", line 55, in _rotate_mysql_logs
2024-03-03T16:54:04.828Z [container-agent]     self.charm._mysql.flush_mysql_logs(MySQLTextLogs.ERROR)
2024-03-03T16:54:04.828Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-03-03T16:54:04.828Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-03-03T16:54:04.828Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 665, in _run_mysqlsh_script
2024-03-03T16:54:04.828Z [container-agent]     process = self.container.exec(cmd, timeout=timeout)
2024-03-03T16:54:04.828Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/model.py", line 2688, in exec
2024-03-03T16:54:04.828Z [container-agent]     return self._pebble.exec(
2024-03-03T16:54:04.828Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 2501, in exec
2024-03-03T16:54:04.828Z [container-agent]     resp = self._request('POST', '/v1/exec', body=body)
2024-03-03T16:54:04.828Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1655, in _request
2024-03-03T16:54:04.828Z [container-agent]     response = self._request_raw(method, path, query, headers, data)
2024-03-03T16:54:04.828Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1690, in _request_raw
2024-03-03T16:54:04.828Z [container-agent]     response = self.opener.open(request, timeout=self.timeout)
2024-03-03T16:54:04.828Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 519, in open
2024-03-03T16:54:04.828Z [container-agent]     response = self._open(req, data)
2024-03-03T16:54:04.828Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 536, in _open
2024-03-03T16:54:04.828Z [container-agent]     result = self._call_chain(self.handle_open, protocol, protocol +
2024-03-03T16:54:04.828Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 496, in _call_chain
2024-03-03T16:54:04.828Z [container-agent]     result = func(*args)
2024-03-03T16:54:04.828Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 311, in http_open
2024-03-03T16:54:04.828Z [container-agent]     return self.do_open(_UnixSocketConnection, req,  # type:ignore
2024-03-03T16:54:04.828Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 1352, in do_open
2024-03-03T16:54:04.828Z [container-agent]     r = h.getresponse()
2024-03-03T16:54:04.829Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1375, in getresponse
2024-03-03T16:54:04.829Z [container-agent]     response.begin()
2024-03-03T16:54:04.829Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 318, in begin
2024-03-03T16:54:04.829Z [container-agent]     version, status, reason = self._read_status()
2024-03-03T16:54:04.829Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 287, in _read_status
2024-03-03T16:54:04.829Z [container-agent]     raise RemoteDisconnected("Remote end closed connection without"
2024-03-03T16:54:04.829Z [container-agent] http.client.RemoteDisconnected: Remote end closed connection without response
2024-03-03T16:54:04.949Z [container-agent] 2024-03-03 16:54:04 INFO juju.util.exec exec.go:209 run result: exit status 1
2024-03-03T16:54:09.721Z [container-agent] 2024-03-03 16:54:09 INFO juju-log Setting up the logrotate configurations
2024-03-03T16:54:09.795Z [container-agent] 2024-03-03 16:54:09 INFO juju-log Adding pebble layer
2024-03-03T16:54:16.575Z [container-agent] 2024-03-03 16:54:16 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-03-03T16:54:16.600Z [container-agent] 2024-03-03 16:54:16 INFO juju-log Attempting reboot from complete outage.
2024-03-03T16:54:19.584Z [container-agent] 2024-03-03 16:54:19 INFO juju.worker.uniter.operation runhook.go:186 ran "mysql-pebble-ready" hook (via hook dispatching script: dispatch)
2024-03-03T16:59:05.975Z [container-agent] 2024-03-03 16:59:05 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T16:59:07.976Z [container-agent] 2024-03-03 16:59:07 WARNING juju-log No relation: certificates
2024-03-03T16:59:08.336Z [container-agent] 2024-03-03 16:59:08 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T17:03:09.341Z [container-agent] 2024-03-03 17:03:09 INFO juju-log Starting the log rotate manager
2024-03-03T17:03:09.357Z [container-agent] 2024-03-03 17:03:09 INFO juju-log Started log rotate manager process with PID 716212
2024-03-03T17:03:09.671Z [container-agent] 2024-03-03 17:03:09 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T17:03:11.771Z [container-agent] 2024-03-03 17:03:11 WARNING juju-log No relation: certificates
2024-03-03T17:03:12.112Z [container-agent] 2024-03-03 17:03:12 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T17:07:36.033Z [container-agent] 2024-03-03 17:07:36 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T17:07:38.231Z [container-agent] 2024-03-03 17:07:38 WARNING juju-log No relation: certificates
2024-03-03T17:07:38.572Z [container-agent] 2024-03-03 17:07:38 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T17:12:55.006Z [container-agent] 2024-03-03 17:12:55 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T17:12:57.980Z [container-agent] 2024-03-03 17:12:57 WARNING juju-log No relation: certificates
2024-03-03T17:12:58.375Z [container-agent] 2024-03-03 17:12:58 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T17:18:31.015Z [container-agent] 2024-03-03 17:18:31 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T17:18:34.343Z [container-agent] 2024-03-03 17:18:34 WARNING juju-log No relation: certificates
2024-03-03T17:18:34.707Z [container-agent] 2024-03-03 17:18:34 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T17:23:11.178Z [container-agent] 2024-03-03 17:23:11 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T17:23:15.010Z [container-agent] 2024-03-03 17:23:15 WARNING juju-log No relation: certificates
2024-03-03T17:23:15.354Z [container-agent] 2024-03-03 17:23:15 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T17:27:55.889Z [container-agent] 2024-03-03 17:27:55 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T17:28:00.196Z [container-agent] 2024-03-03 17:28:00 WARNING juju-log No relation: certificates
2024-03-03T17:28:00.553Z [container-agent] 2024-03-03 17:28:00 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T17:32:46.509Z [container-agent] 2024-03-03 17:32:46 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T17:32:51.367Z [container-agent] 2024-03-03 17:32:51 WARNING juju-log No relation: certificates
2024-03-03T17:32:51.720Z [container-agent] 2024-03-03 17:32:51 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T17:37:44.501Z [container-agent] 2024-03-03 17:37:44 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T17:37:49.182Z [container-agent] 2024-03-03 17:37:49 WARNING juju-log No relation: certificates
2024-03-03T17:37:49.527Z [container-agent] 2024-03-03 17:37:49 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T17:42:10.242Z [container-agent] 2024-03-03 17:42:10 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T17:42:15.960Z [container-agent] 2024-03-03 17:42:15 WARNING juju-log No relation: certificates
2024-03-03T17:42:16.327Z [container-agent] 2024-03-03 17:42:16 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T17:47:49.737Z [container-agent] 2024-03-03 17:47:49 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T17:47:55.475Z [container-agent] 2024-03-03 17:47:55 WARNING juju-log No relation: certificates
2024-03-03T17:47:55.815Z [container-agent] 2024-03-03 17:47:55 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T17:51:52.225Z [container-agent] 2024-03-03 17:51:52 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T17:51:58.925Z [container-agent] 2024-03-03 17:51:58 WARNING juju-log No relation: certificates
2024-03-03T17:51:59.285Z [container-agent] 2024-03-03 17:51:59 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T17:56:31.675Z [container-agent] 2024-03-03 17:56:31 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T17:56:38.556Z [container-agent] 2024-03-03 17:56:38 WARNING juju-log No relation: certificates
2024-03-03T17:56:38.911Z [container-agent] 2024-03-03 17:56:38 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T18:01:12.589Z [container-agent] 2024-03-03 18:01:12 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T18:01:20.182Z [container-agent] 2024-03-03 18:01:20 WARNING juju-log No relation: certificates
2024-03-03T18:01:20.547Z [container-agent] 2024-03-03 18:01:20 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T18:07:13.454Z [container-agent] 2024-03-03 18:07:13 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T18:07:21.449Z [container-agent] 2024-03-03 18:07:21 WARNING juju-log No relation: certificates
2024-03-03T18:07:21.794Z [container-agent] 2024-03-03 18:07:21 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T18:12:18.818Z [container-agent] 2024-03-03 18:12:18 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T18:12:27.141Z [container-agent] 2024-03-03 18:12:27 WARNING juju-log No relation: certificates
2024-03-03T18:12:27.496Z [container-agent] 2024-03-03 18:12:27 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T18:16:26.788Z [container-agent] 2024-03-03 18:16:26 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T18:16:35.609Z [container-agent] 2024-03-03 18:16:35 WARNING juju-log No relation: certificates
2024-03-03T18:16:35.961Z [container-agent] 2024-03-03 18:16:35 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T18:20:53.138Z [container-agent] 2024-03-03 18:20:53 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T18:21:02.336Z [container-agent] 2024-03-03 18:21:02 WARNING juju-log No relation: certificates
2024-03-03T18:21:02.672Z [container-agent] 2024-03-03 18:21:02 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T18:25:31.262Z [container-agent] 2024-03-03 18:25:31 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-03-03T18:25:31.291Z [container-agent] 2024-03-03 18:25:31 INFO juju-log Attempting reboot from complete outage.
2024-03-03T18:25:40.236Z [container-agent] 2024-03-03 18:25:40 WARNING juju-log No relation: certificates
2024-03-03T18:25:40.583Z [container-agent] 2024-03-03 18:25:40 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T18:26:09.805Z [container-agent] 2024-03-03 18:26:09 ERROR juju-log Uncaught exception while in charm code:
2024-03-03T18:26:09.805Z [container-agent] Traceback (most recent call last):
2024-03-03T18:26:09.805Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/./src/charm.py", line 786, in <module>
2024-03-03T18:26:09.805Z [container-agent]     main(MySQLOperatorCharm)
2024-03-03T18:26:09.805Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 436, in main
2024-03-03T18:26:09.805Z [container-agent]     _emit_charm_event(charm, dispatcher.event_name)
2024-03-03T18:26:09.805Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 144, in _emit_charm_event
2024-03-03T18:26:09.805Z [container-agent]     event_to_emit.emit(*args, **kwargs)
2024-03-03T18:26:09.805Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 351, in emit
2024-03-03T18:26:09.805Z [container-agent]     framework._emit(event)
2024-03-03T18:26:09.805Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 853, in _emit
2024-03-03T18:26:09.805Z [container-agent]     self._reemit(event_path)
2024-03-03T18:26:09.805Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 942, in _reemit
2024-03-03T18:26:09.805Z [container-agent]     custom_handler(event)
2024-03-03T18:26:09.805Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/rotate_mysql_logs.py", line 56, in _rotate_mysql_logs
2024-03-03T18:26:09.805Z [container-agent]     self.charm._mysql.flush_mysql_logs(MySQLTextLogs.GENERAL)
2024-03-03T18:26:09.805Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-03-03T18:26:09.805Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-03-03T18:26:09.805Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 666, in _run_mysqlsh_script
2024-03-03T18:26:09.805Z [container-agent]     stdout, _ = process.wait_output()
2024-03-03T18:26:09.805Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1436, in wait_output
2024-03-03T18:26:09.805Z [container-agent]     exit_code: int = self._wait()
2024-03-03T18:26:09.805Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1375, in _wait
2024-03-03T18:26:09.805Z [container-agent]     change = self._client.wait_change(self._change_id, timeout=timeout)
2024-03-03T18:26:09.805Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1893, in wait_change
2024-03-03T18:26:09.805Z [container-agent]     return self._wait_change_using_wait(change_id, timeout)
2024-03-03T18:26:09.805Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1914, in _wait_change_using_wait
2024-03-03T18:26:09.805Z [container-agent]     return self._wait_change(change_id, this_timeout)
2024-03-03T18:26:09.806Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1928, in _wait_change
2024-03-03T18:26:09.806Z [container-agent]     resp = self._request('GET', f'/v1/changes/{change_id}/wait', query)
2024-03-03T18:26:09.806Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1655, in _request
2024-03-03T18:26:09.806Z [container-agent]     response = self._request_raw(method, path, query, headers, data)
2024-03-03T18:26:09.806Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1690, in _request_raw
2024-03-03T18:26:09.806Z [container-agent]     response = self.opener.open(request, timeout=self.timeout)
2024-03-03T18:26:09.806Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 519, in open
2024-03-03T18:26:09.806Z [container-agent]     response = self._open(req, data)
2024-03-03T18:26:09.806Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 536, in _open
2024-03-03T18:26:09.806Z [container-agent]     result = self._call_chain(self.handle_open, protocol, protocol +
2024-03-03T18:26:09.806Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 496, in _call_chain
2024-03-03T18:26:09.806Z [container-agent]     result = func(*args)
2024-03-03T18:26:09.806Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 311, in http_open
2024-03-03T18:26:09.806Z [container-agent]     return self.do_open(_UnixSocketConnection, req,  # type:ignore
2024-03-03T18:26:09.806Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 1352, in do_open
2024-03-03T18:26:09.806Z [container-agent]     r = h.getresponse()
2024-03-03T18:26:09.806Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1375, in getresponse
2024-03-03T18:26:09.806Z [container-agent]     response.begin()
2024-03-03T18:26:09.806Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 318, in begin
2024-03-03T18:26:09.806Z [container-agent]     version, status, reason = self._read_status()
2024-03-03T18:26:09.806Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 287, in _read_status
2024-03-03T18:26:09.806Z [container-agent]     raise RemoteDisconnected("Remote end closed connection without"
2024-03-03T18:26:09.806Z [container-agent] http.client.RemoteDisconnected: Remote end closed connection without response
2024-03-03T18:26:09.964Z [container-agent] 2024-03-03 18:26:09 INFO juju.util.exec exec.go:209 run result: exit status 1
2024-03-03T18:26:12.983Z [container-agent] 2024-03-03 18:26:12 INFO juju-log Setting up the logrotate configurations
2024-03-03T18:26:13.054Z [container-agent] 2024-03-03 18:26:13 INFO juju-log Adding pebble layer
2024-03-03T18:26:19.780Z [container-agent] 2024-03-03 18:26:19 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-03-03T18:26:19.806Z [container-agent] 2024-03-03 18:26:19 INFO juju-log Attempting reboot from complete outage.
2024-03-03T18:26:22.727Z [container-agent] 2024-03-03 18:26:22 INFO juju.worker.uniter.operation runhook.go:186 ran "mysql-pebble-ready" hook (via hook dispatching script: dispatch)
2024-03-03T18:30:10.787Z [container-agent] 2024-03-03 18:30:10 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T18:30:12.718Z [container-agent] 2024-03-03 18:30:12 WARNING juju-log No relation: certificates
2024-03-03T18:30:13.060Z [container-agent] 2024-03-03 18:30:13 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T18:35:58.470Z [container-agent] 2024-03-03 18:35:58 INFO juju-log Starting the log rotate manager
2024-03-03T18:35:58.486Z [container-agent] 2024-03-03 18:35:58 INFO juju-log Started log rotate manager process with PID 724755
2024-03-03T18:35:58.776Z [container-agent] 2024-03-03 18:35:58 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T18:36:00.709Z [container-agent] 2024-03-03 18:36:00 WARNING juju-log No relation: certificates
2024-03-03T18:36:01.088Z [container-agent] 2024-03-03 18:36:01 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T18:41:49.373Z [container-agent] 2024-03-03 18:41:49 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T18:41:51.935Z [container-agent] 2024-03-03 18:41:51 WARNING juju-log No relation: certificates
2024-03-03T18:41:52.331Z [container-agent] 2024-03-03 18:41:52 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T18:46:49.119Z [container-agent] 2024-03-03 18:46:49 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T18:46:52.192Z [container-agent] 2024-03-03 18:46:52 WARNING juju-log No relation: certificates
2024-03-03T18:46:52.572Z [container-agent] 2024-03-03 18:46:52 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T18:52:48.828Z [container-agent] 2024-03-03 18:52:48 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T18:52:52.392Z [container-agent] 2024-03-03 18:52:52 WARNING juju-log No relation: certificates
2024-03-03T18:52:52.750Z [container-agent] 2024-03-03 18:52:52 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T18:57:36.556Z [container-agent] 2024-03-03 18:57:36 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T18:57:40.534Z [container-agent] 2024-03-03 18:57:40 WARNING juju-log No relation: certificates
2024-03-03T18:57:40.882Z [container-agent] 2024-03-03 18:57:40 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T19:02:08.731Z [container-agent] 2024-03-03 19:02:08 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T19:02:13.360Z [container-agent] 2024-03-03 19:02:13 WARNING juju-log No relation: certificates
2024-03-03T19:02:13.712Z [container-agent] 2024-03-03 19:02:13 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T19:07:30.579Z [container-agent] 2024-03-03 19:07:30 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T19:07:35.304Z [container-agent] 2024-03-03 19:07:35 WARNING juju-log No relation: certificates
2024-03-03T19:07:35.664Z [container-agent] 2024-03-03 19:07:35 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T19:11:30.798Z [container-agent] 2024-03-03 19:11:30 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T19:11:36.393Z [container-agent] 2024-03-03 19:11:36 WARNING juju-log No relation: certificates
2024-03-03T19:11:36.732Z [container-agent] 2024-03-03 19:11:36 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T19:16:11.167Z [container-agent] 2024-03-03 19:16:11 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T19:16:16.889Z [container-agent] 2024-03-03 19:16:16 WARNING juju-log No relation: certificates
2024-03-03T19:16:17.244Z [container-agent] 2024-03-03 19:16:17 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T19:21:43.927Z [container-agent] 2024-03-03 19:21:43 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T19:21:50.174Z [container-agent] 2024-03-03 19:21:50 WARNING juju-log No relation: certificates
2024-03-03T19:21:50.508Z [container-agent] 2024-03-03 19:21:50 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T19:25:56.675Z [container-agent] 2024-03-03 19:25:56 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T19:26:03.105Z [container-agent] 2024-03-03 19:26:03 WARNING juju-log No relation: certificates
2024-03-03T19:26:03.467Z [container-agent] 2024-03-03 19:26:03 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T19:30:14.649Z [container-agent] 2024-03-03 19:30:14 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T19:30:21.743Z [container-agent] 2024-03-03 19:30:21 WARNING juju-log No relation: certificates
2024-03-03T19:30:22.096Z [container-agent] 2024-03-03 19:30:22 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T19:35:30.340Z [container-agent] 2024-03-03 19:35:30 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T19:35:37.977Z [container-agent] 2024-03-03 19:35:37 WARNING juju-log No relation: certificates
2024-03-03T19:35:38.350Z [container-agent] 2024-03-03 19:35:38 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T19:41:28.909Z [container-agent] 2024-03-03 19:41:28 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T19:41:36.993Z [container-agent] 2024-03-03 19:41:36 WARNING juju-log No relation: certificates
2024-03-03T19:41:37.338Z [container-agent] 2024-03-03 19:41:37 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T19:44:13.996Z [container-agent] 2024-03-03 19:44:13 ERROR juju-log Failed to flush SLOW LOGS logs.
2024-03-03T19:44:13.996Z [container-agent] Traceback (most recent call last):
2024-03-03T19:44:13.996Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 666, in _run_mysqlsh_script
2024-03-03T19:44:13.996Z [container-agent]     stdout, _ = process.wait_output()
2024-03-03T19:44:13.996Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1441, in wait_output
2024-03-03T19:44:13.996Z [container-agent]     raise ExecError[AnyStr](self._command, exit_code, out_value, err_value)
2024-03-03T19:44:13.996Z [container-agent] ops.pebble.ExecError: non-zero exit code 1 executing ['/usr/bin/mysqlsh', '--no-wizard', '--python', '--verbose=1', '-f', '/tmp/script.py', ';', 'rm', '/tmp/script.py'], stdout='', stderr='Cannot set LC_ALL to locale en_US.UTF-8: No such file or directory\nverbose: 2024-03-03T19:44:12Z: Loading startup files...\nverbose: 2024-03-03T19:44:12Z: Loading plugins...\nverbose: 2024-03-03T19:44:12Z: Connecting to MySQL at: serverconfig@horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local\nTraceback (most recent call last):\n  File "<string>", line 1, in <module>\nmysqlsh.DBError: MySQL Error (2003): Shell.connect: Can\'t connect to MySQL server on \'horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local:3306\' (111)\n'
2024-03-03T19:44:13.996Z [container-agent] 
2024-03-03T19:44:13.996Z [container-agent] During handling of the above exception, another exception occurred:
2024-03-03T19:44:13.996Z [container-agent] 
2024-03-03T19:44:13.996Z [container-agent] Traceback (most recent call last):
2024-03-03T19:44:13.996Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-03-03T19:44:13.996Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-03-03T19:44:13.996Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 669, in _run_mysqlsh_script
2024-03-03T19:44:13.996Z [container-agent]     raise MySQLClientError(e.stderr)
2024-03-03T19:44:13.996Z [container-agent] charms.mysql.v0.mysql.MySQLClientError: Cannot set LC_ALL to locale en_US.UTF-8: No such file or directory
2024-03-03T19:44:13.996Z [container-agent] verbose: 2024-03-03T19:44:12Z: Loading startup files...
2024-03-03T19:44:13.996Z [container-agent] verbose: 2024-03-03T19:44:12Z: Loading plugins...
2024-03-03T19:44:13.996Z [container-agent] verbose: 2024-03-03T19:44:12Z: Connecting to MySQL at: serverconfig@horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local
2024-03-03T19:44:13.996Z [container-agent] Traceback (most recent call last):
2024-03-03T19:44:13.996Z [container-agent]   File "<string>", line 1, in <module>
2024-03-03T19:44:13.996Z [container-agent] mysqlsh.DBError: MySQL Error (2003): Shell.connect: Can't connect to MySQL server on 'horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local:3306' (111)
2024-03-03T19:44:13.996Z [container-agent] 
2024-03-03T19:46:32.285Z [container-agent] 2024-03-03 19:46:32 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-03-03T19:46:32.315Z [container-agent] 2024-03-03 19:46:32 INFO juju-log Attempting reboot from complete outage.
2024-03-03T19:46:40.054Z [container-agent] 2024-03-03 19:46:40 WARNING juju-log No relation: certificates
2024-03-03T19:46:40.478Z [container-agent] 2024-03-03 19:46:40 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T19:49:09.819Z [container-agent] 2024-03-03 19:49:09 ERROR juju-log Uncaught exception while in charm code:
2024-03-03T19:49:09.819Z [container-agent] Traceback (most recent call last):
2024-03-03T19:49:09.819Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/./src/charm.py", line 786, in <module>
2024-03-03T19:49:09.819Z [container-agent]     main(MySQLOperatorCharm)
2024-03-03T19:49:09.819Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 436, in main
2024-03-03T19:49:09.819Z [container-agent]     _emit_charm_event(charm, dispatcher.event_name)
2024-03-03T19:49:09.819Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 144, in _emit_charm_event
2024-03-03T19:49:09.820Z [container-agent]     event_to_emit.emit(*args, **kwargs)
2024-03-03T19:49:09.820Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 351, in emit
2024-03-03T19:49:09.820Z [container-agent]     framework._emit(event)
2024-03-03T19:49:09.820Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 853, in _emit
2024-03-03T19:49:09.820Z [container-agent]     self._reemit(event_path)
2024-03-03T19:49:09.820Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 942, in _reemit
2024-03-03T19:49:09.820Z [container-agent]     custom_handler(event)
2024-03-03T19:49:09.820Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/rotate_mysql_logs.py", line 57, in _rotate_mysql_logs
2024-03-03T19:49:09.820Z [container-agent]     self.charm._mysql.flush_mysql_logs(MySQLTextLogs.SLOW)
2024-03-03T19:49:09.820Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-03-03T19:49:09.820Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-03-03T19:49:09.820Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 666, in _run_mysqlsh_script
2024-03-03T19:49:09.820Z [container-agent]     stdout, _ = process.wait_output()
2024-03-03T19:49:09.820Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1436, in wait_output
2024-03-03T19:49:09.820Z [container-agent]     exit_code: int = self._wait()
2024-03-03T19:49:09.820Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1375, in _wait
2024-03-03T19:49:09.820Z [container-agent]     change = self._client.wait_change(self._change_id, timeout=timeout)
2024-03-03T19:49:09.820Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1893, in wait_change
2024-03-03T19:49:09.820Z [container-agent]     return self._wait_change_using_wait(change_id, timeout)
2024-03-03T19:49:09.820Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1914, in _wait_change_using_wait
2024-03-03T19:49:09.820Z [container-agent]     return self._wait_change(change_id, this_timeout)
2024-03-03T19:49:09.820Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1928, in _wait_change
2024-03-03T19:49:09.820Z [container-agent]     resp = self._request('GET', f'/v1/changes/{change_id}/wait', query)
2024-03-03T19:49:09.820Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1655, in _request
2024-03-03T19:49:09.820Z [container-agent]     response = self._request_raw(method, path, query, headers, data)
2024-03-03T19:49:09.820Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1690, in _request_raw
2024-03-03T19:49:09.820Z [container-agent]     response = self.opener.open(request, timeout=self.timeout)
2024-03-03T19:49:09.820Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 519, in open
2024-03-03T19:49:09.820Z [container-agent]     response = self._open(req, data)
2024-03-03T19:49:09.820Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 536, in _open
2024-03-03T19:49:09.820Z [container-agent]     result = self._call_chain(self.handle_open, protocol, protocol +
2024-03-03T19:49:09.820Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 496, in _call_chain
2024-03-03T19:49:09.820Z [container-agent]     result = func(*args)
2024-03-03T19:49:09.820Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 311, in http_open
2024-03-03T19:49:09.820Z [container-agent]     return self.do_open(_UnixSocketConnection, req,  # type:ignore
2024-03-03T19:49:09.820Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 1352, in do_open
2024-03-03T19:49:09.820Z [container-agent]     r = h.getresponse()
2024-03-03T19:49:09.820Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1375, in getresponse
2024-03-03T19:49:09.820Z [container-agent]     response.begin()
2024-03-03T19:49:09.820Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 318, in begin
2024-03-03T19:49:09.820Z [container-agent]     version, status, reason = self._read_status()
2024-03-03T19:49:09.820Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 287, in _read_status
2024-03-03T19:49:09.820Z [container-agent]     raise RemoteDisconnected("Remote end closed connection without"
2024-03-03T19:49:09.820Z [container-agent] http.client.RemoteDisconnected: Remote end closed connection without response
2024-03-03T19:49:09.925Z [container-agent] 2024-03-03 19:49:09 INFO juju.util.exec exec.go:209 run result: exit status 1
2024-03-03T19:49:16.219Z [container-agent] 2024-03-03 19:49:16 INFO juju-log Setting up the logrotate configurations
2024-03-03T19:49:16.315Z [container-agent] 2024-03-03 19:49:16 INFO juju-log Adding pebble layer
2024-03-03T19:49:23.044Z [container-agent] 2024-03-03 19:49:23 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-03-03T19:49:23.065Z [container-agent] 2024-03-03 19:49:23 INFO juju-log Attempting reboot from complete outage.
2024-03-03T19:49:25.977Z [container-agent] 2024-03-03 19:49:25 INFO juju.worker.uniter.operation runhook.go:186 ran "mysql-pebble-ready" hook (via hook dispatching script: dispatch)
2024-03-03T19:51:07.709Z [container-agent] 2024-03-03 19:51:07 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T19:51:10.036Z [container-agent] 2024-03-03 19:51:10 WARNING juju-log No relation: certificates
2024-03-03T19:51:10.383Z [container-agent] 2024-03-03 19:51:10 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T19:56:45.705Z [container-agent] 2024-03-03 19:56:45 INFO juju-log Starting the log rotate manager
2024-03-03T19:56:45.718Z [container-agent] 2024-03-03 19:56:45 INFO juju-log Started log rotate manager process with PID 732262
2024-03-03T19:56:45.984Z [container-agent] 2024-03-03 19:56:45 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T19:56:48.028Z [container-agent] 2024-03-03 19:56:48 WARNING juju-log No relation: certificates
2024-03-03T19:56:48.390Z [container-agent] 2024-03-03 19:56:48 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T20:02:27.277Z [container-agent] 2024-03-03 20:02:27 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T20:02:29.946Z [container-agent] 2024-03-03 20:02:29 WARNING juju-log No relation: certificates
2024-03-03T20:02:30.308Z [container-agent] 2024-03-03 20:02:30 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T20:08:21.310Z [container-agent] 2024-03-03 20:08:21 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T20:08:24.341Z [container-agent] 2024-03-03 20:08:24 WARNING juju-log No relation: certificates
2024-03-03T20:08:24.683Z [container-agent] 2024-03-03 20:08:24 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T20:13:44.182Z [container-agent] 2024-03-03 20:13:44 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T20:13:47.792Z [container-agent] 2024-03-03 20:13:47 WARNING juju-log No relation: certificates
2024-03-03T20:13:48.148Z [container-agent] 2024-03-03 20:13:48 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T20:18:37.915Z [container-agent] 2024-03-03 20:18:37 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T20:18:42.066Z [container-agent] 2024-03-03 20:18:42 WARNING juju-log No relation: certificates
2024-03-03T20:18:42.402Z [container-agent] 2024-03-03 20:18:42 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T20:23:09.223Z [container-agent] 2024-03-03 20:23:09 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T20:23:13.765Z [container-agent] 2024-03-03 20:23:13 WARNING juju-log No relation: certificates
2024-03-03T20:23:14.124Z [container-agent] 2024-03-03 20:23:14 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T20:27:50.060Z [container-agent] 2024-03-03 20:27:50 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T20:27:54.836Z [container-agent] 2024-03-03 20:27:54 WARNING juju-log No relation: certificates
2024-03-03T20:27:55.179Z [container-agent] 2024-03-03 20:27:55 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T20:33:16.021Z [container-agent] 2024-03-03 20:33:16 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T20:33:21.267Z [container-agent] 2024-03-03 20:33:21 WARNING juju-log No relation: certificates
2024-03-03T20:33:21.609Z [container-agent] 2024-03-03 20:33:21 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T20:38:55.485Z [container-agent] 2024-03-03 20:38:55 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T20:39:01.439Z [container-agent] 2024-03-03 20:39:01 WARNING juju-log No relation: certificates
2024-03-03T20:39:01.810Z [container-agent] 2024-03-03 20:39:01 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T20:44:36.461Z [container-agent] 2024-03-03 20:44:36 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T20:44:42.148Z [container-agent] 2024-03-03 20:44:42 WARNING juju-log No relation: certificates
2024-03-03T20:44:42.489Z [container-agent] 2024-03-03 20:44:42 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T20:50:16.787Z [container-agent] 2024-03-03 20:50:16 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T20:50:23.843Z [container-agent] 2024-03-03 20:50:23 WARNING juju-log No relation: certificates
2024-03-03T20:50:24.199Z [container-agent] 2024-03-03 20:50:24 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T20:55:14.214Z [container-agent] 2024-03-03 20:55:14 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T20:55:21.621Z [container-agent] 2024-03-03 20:55:21 WARNING juju-log No relation: certificates
2024-03-03T20:55:21.974Z [container-agent] 2024-03-03 20:55:21 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T21:00:49.232Z [container-agent] 2024-03-03 21:00:49 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T21:00:57.131Z [container-agent] 2024-03-03 21:00:57 WARNING juju-log No relation: certificates
2024-03-03T21:00:57.542Z [container-agent] 2024-03-03 21:00:57 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T21:05:29.446Z [container-agent] 2024-03-03 21:05:29 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T21:05:37.695Z [container-agent] 2024-03-03 21:05:37 WARNING juju-log No relation: certificates
2024-03-03T21:05:38.054Z [container-agent] 2024-03-03 21:05:38 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T21:09:52.781Z [container-agent] 2024-03-03 21:09:52 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-03-03T21:09:52.817Z [container-agent] 2024-03-03 21:09:52 INFO juju-log Attempting reboot from complete outage.
2024-03-03T21:10:00.882Z [container-agent] 2024-03-03 21:10:00 WARNING juju-log No relation: certificates
2024-03-03T21:10:01.227Z [container-agent] 2024-03-03 21:10:01 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T21:14:04.743Z [container-agent] 2024-03-03 21:14:04 ERROR juju-log Uncaught exception while in charm code:
2024-03-03T21:14:04.743Z [container-agent] Traceback (most recent call last):
2024-03-03T21:14:04.743Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 2507, in exec
2024-03-03T21:14:04.743Z [container-agent]     control_ws = self._connect_websocket(task_id, 'control')
2024-03-03T21:14:04.743Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 2589, in _connect_websocket
2024-03-03T21:14:04.743Z [container-agent]     ws.connect(url, socket=sock)
2024-03-03T21:14:04.743Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_core.py", line 253, in connect
2024-03-03T21:14:04.743Z [container-agent]     self.handshake_response = handshake(self.sock, url, *addrs, **options)
2024-03-03T21:14:04.743Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_handshake.py", line 57, in handshake
2024-03-03T21:14:04.743Z [container-agent]     status, resp = _get_resp_headers(sock)
2024-03-03T21:14:04.743Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_handshake.py", line 143, in _get_resp_headers
2024-03-03T21:14:04.743Z [container-agent]     status, resp_headers, status_message = read_headers(sock)
2024-03-03T21:14:04.743Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_http.py", line 312, in read_headers
2024-03-03T21:14:04.743Z [container-agent]     line = recv_line(sock)
2024-03-03T21:14:04.743Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_socket.py", line 131, in recv_line
2024-03-03T21:14:04.743Z [container-agent]     c = recv(sock, 1)
2024-03-03T21:14:04.743Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_socket.py", line 122, in recv
2024-03-03T21:14:04.743Z [container-agent]     raise WebSocketConnectionClosedException(
2024-03-03T21:14:04.743Z [container-agent] websocket._exceptions.WebSocketConnectionClosedException: Connection to remote host was lost.
2024-03-03T21:14:04.743Z [container-agent] 
2024-03-03T21:14:04.743Z [container-agent] During handling of the above exception, another exception occurred:
2024-03-03T21:14:04.743Z [container-agent] 
2024-03-03T21:14:04.743Z [container-agent] Traceback (most recent call last):
2024-03-03T21:14:04.743Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 1348, in do_open
2024-03-03T21:14:04.743Z [container-agent]     h.request(req.get_method(), req.selector, req.data, headers,
2024-03-03T21:14:04.743Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1283, in request
2024-03-03T21:14:04.743Z [container-agent]     self._send_request(method, url, body, headers, encode_chunked)
2024-03-03T21:14:04.743Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1329, in _send_request
2024-03-03T21:14:04.743Z [container-agent]     self.endheaders(body, encode_chunked=encode_chunked)
2024-03-03T21:14:04.743Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1278, in endheaders
2024-03-03T21:14:04.743Z [container-agent]     self._send_output(message_body, encode_chunked=encode_chunked)
2024-03-03T21:14:04.743Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1038, in _send_output
2024-03-03T21:14:04.743Z [container-agent]     self.send(msg)
2024-03-03T21:14:04.743Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 976, in send
2024-03-03T21:14:04.743Z [container-agent]     self.connect()
2024-03-03T21:14:04.743Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 297, in connect
2024-03-03T21:14:04.743Z [container-agent]     self.sock.connect(self.socket_path)
2024-03-03T21:14:04.743Z [container-agent] ConnectionRefusedError: [Errno 111] Connection refused
2024-03-03T21:14:04.743Z [container-agent] 
2024-03-03T21:14:04.743Z [container-agent] During handling of the above exception, another exception occurred:
2024-03-03T21:14:04.743Z [container-agent] 
2024-03-03T21:14:04.743Z [container-agent] Traceback (most recent call last):
2024-03-03T21:14:04.743Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1690, in _request_raw
2024-03-03T21:14:04.743Z [container-agent]     response = self.opener.open(request, timeout=self.timeout)
2024-03-03T21:14:04.743Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 519, in open
2024-03-03T21:14:04.743Z [container-agent]     response = self._open(req, data)
2024-03-03T21:14:04.743Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 536, in _open
2024-03-03T21:14:04.743Z [container-agent]     result = self._call_chain(self.handle_open, protocol, protocol +
2024-03-03T21:14:04.743Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 496, in _call_chain
2024-03-03T21:14:04.743Z [container-agent]     result = func(*args)
2024-03-03T21:14:04.743Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 311, in http_open
2024-03-03T21:14:04.743Z [container-agent]     return self.do_open(_UnixSocketConnection, req,  # type:ignore
2024-03-03T21:14:04.743Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 1351, in do_open
2024-03-03T21:14:04.743Z [container-agent]     raise URLError(err)
2024-03-03T21:14:04.743Z [container-agent] urllib.error.URLError: <urlopen error [Errno 111] Connection refused>
2024-03-03T21:14:04.743Z [container-agent] 
2024-03-03T21:14:04.743Z [container-agent] The above exception was the direct cause of the following exception:
2024-03-03T21:14:04.743Z [container-agent] 
2024-03-03T21:14:04.743Z [container-agent] Traceback (most recent call last):
2024-03-03T21:14:04.743Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/./src/charm.py", line 786, in <module>
2024-03-03T21:14:04.743Z [container-agent]     main(MySQLOperatorCharm)
2024-03-03T21:14:04.743Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 436, in main
2024-03-03T21:14:04.743Z [container-agent]     _emit_charm_event(charm, dispatcher.event_name)
2024-03-03T21:14:04.743Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 144, in _emit_charm_event
2024-03-03T21:14:04.743Z [container-agent]     event_to_emit.emit(*args, **kwargs)
2024-03-03T21:14:04.743Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 351, in emit
2024-03-03T21:14:04.743Z [container-agent]     framework._emit(event)
2024-03-03T21:14:04.743Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 853, in _emit
2024-03-03T21:14:04.743Z [container-agent]     self._reemit(event_path)
2024-03-03T21:14:04.743Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 942, in _reemit
2024-03-03T21:14:04.743Z [container-agent]     custom_handler(event)
2024-03-03T21:14:04.743Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/rotate_mysql_logs.py", line 55, in _rotate_mysql_logs
2024-03-03T21:14:04.743Z [container-agent]     self.charm._mysql.flush_mysql_logs(MySQLTextLogs.ERROR)
2024-03-03T21:14:04.743Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-03-03T21:14:04.743Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-03-03T21:14:04.743Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 665, in _run_mysqlsh_script
2024-03-03T21:14:04.743Z [container-agent]     process = self.container.exec(cmd, timeout=timeout)
2024-03-03T21:14:04.743Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/model.py", line 2688, in exec
2024-03-03T21:14:04.743Z [container-agent]     return self._pebble.exec(
2024-03-03T21:14:04.743Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 2514, in exec
2024-03-03T21:14:04.743Z [container-agent]     change = self.wait_change(ChangeID(change_id))
2024-03-03T21:14:04.743Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1893, in wait_change
2024-03-03T21:14:04.743Z [container-agent]     return self._wait_change_using_wait(change_id, timeout)
2024-03-03T21:14:04.743Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1914, in _wait_change_using_wait
2024-03-03T21:14:04.743Z [container-agent]     return self._wait_change(change_id, this_timeout)
2024-03-03T21:14:04.743Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1928, in _wait_change
2024-03-03T21:14:04.743Z [container-agent]     resp = self._request('GET', f'/v1/changes/{change_id}/wait', query)
2024-03-03T21:14:04.743Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1655, in _request
2024-03-03T21:14:04.743Z [container-agent]     response = self._request_raw(method, path, query, headers, data)
2024-03-03T21:14:04.743Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1707, in _request_raw
2024-03-03T21:14:04.743Z [container-agent]     raise ConnectionError(e.reason) from e
2024-03-03T21:14:04.743Z [container-agent] ops.pebble.ConnectionError: [Errno 111] Connection refused
2024-03-03T21:14:04.874Z [container-agent] 2024-03-03 21:14:04 INFO juju.util.exec exec.go:209 run result: exit status 1
2024-03-03T21:14:09.029Z [container-agent] 2024-03-03 21:14:09 INFO juju-log Setting up the logrotate configurations
2024-03-03T21:14:09.095Z [container-agent] 2024-03-03 21:14:09 INFO juju-log Adding pebble layer
2024-03-03T21:14:15.886Z [container-agent] 2024-03-03 21:14:15 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-03-03T21:14:15.910Z [container-agent] 2024-03-03 21:14:15 INFO juju-log Attempting reboot from complete outage.
2024-03-03T21:14:18.880Z [container-agent] 2024-03-03 21:14:18 INFO juju.worker.uniter.operation runhook.go:186 ran "mysql-pebble-ready" hook (via hook dispatching script: dispatch)
2024-03-03T21:15:39.197Z [container-agent] 2024-03-03 21:15:39 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T21:15:41.086Z [container-agent] 2024-03-03 21:15:41 WARNING juju-log No relation: certificates
2024-03-03T21:15:41.437Z [container-agent] 2024-03-03 21:15:41 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T21:20:45.141Z [container-agent] 2024-03-03 21:20:45 INFO juju-log Starting the log rotate manager
2024-03-03T21:20:45.155Z [container-agent] 2024-03-03 21:20:45 INFO juju-log Started log rotate manager process with PID 740065
2024-03-03T21:20:45.444Z [container-agent] 2024-03-03 21:20:45 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T21:20:47.660Z [container-agent] 2024-03-03 21:20:47 WARNING juju-log No relation: certificates
2024-03-03T21:20:48.019Z [container-agent] 2024-03-03 21:20:48 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T21:25:45.938Z [container-agent] 2024-03-03 21:25:45 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T21:25:48.540Z [container-agent] 2024-03-03 21:25:48 WARNING juju-log No relation: certificates
2024-03-03T21:25:48.899Z [container-agent] 2024-03-03 21:25:48 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T21:29:57.402Z [container-agent] 2024-03-03 21:29:57 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T21:30:00.379Z [container-agent] 2024-03-03 21:30:00 WARNING juju-log No relation: certificates
2024-03-03T21:30:00.735Z [container-agent] 2024-03-03 21:30:00 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T21:35:17.959Z [container-agent] 2024-03-03 21:35:17 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T21:35:21.411Z [container-agent] 2024-03-03 21:35:21 WARNING juju-log No relation: certificates
2024-03-03T21:35:21.755Z [container-agent] 2024-03-03 21:35:21 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T21:40:30.385Z [container-agent] 2024-03-03 21:40:30 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T21:40:34.350Z [container-agent] 2024-03-03 21:40:34 WARNING juju-log No relation: certificates
2024-03-03T21:40:34.703Z [container-agent] 2024-03-03 21:40:34 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T21:45:38.454Z [container-agent] 2024-03-03 21:45:38 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T21:45:42.914Z [container-agent] 2024-03-03 21:45:42 WARNING juju-log No relation: certificates
2024-03-03T21:45:43.281Z [container-agent] 2024-03-03 21:45:43 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T21:49:42.109Z [container-agent] 2024-03-03 21:49:42 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T21:49:47.028Z [container-agent] 2024-03-03 21:49:47 WARNING juju-log No relation: certificates
2024-03-03T21:49:47.386Z [container-agent] 2024-03-03 21:49:47 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T21:55:41.322Z [container-agent] 2024-03-03 21:55:41 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T21:55:46.763Z [container-agent] 2024-03-03 21:55:46 WARNING juju-log No relation: certificates
2024-03-03T21:55:47.132Z [container-agent] 2024-03-03 21:55:47 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T22:00:29.057Z [container-agent] 2024-03-03 22:00:29 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T22:00:35.145Z [container-agent] 2024-03-03 22:00:35 WARNING juju-log No relation: certificates
2024-03-03T22:00:35.523Z [container-agent] 2024-03-03 22:00:35 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T22:05:52.768Z [container-agent] 2024-03-03 22:05:52 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T22:05:58.767Z [container-agent] 2024-03-03 22:05:58 WARNING juju-log No relation: certificates
2024-03-03T22:05:59.114Z [container-agent] 2024-03-03 22:05:59 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T22:11:02.010Z [container-agent] 2024-03-03 22:11:02 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T22:11:08.584Z [container-agent] 2024-03-03 22:11:08 WARNING juju-log No relation: certificates
2024-03-03T22:11:08.983Z [container-agent] 2024-03-03 22:11:08 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T22:15:27.922Z [container-agent] 2024-03-03 22:15:27 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T22:15:35.057Z [container-agent] 2024-03-03 22:15:35 WARNING juju-log No relation: certificates
2024-03-03T22:15:35.430Z [container-agent] 2024-03-03 22:15:35 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T22:21:16.655Z [container-agent] 2024-03-03 22:21:16 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T22:21:23.674Z [container-agent] 2024-03-03 22:21:23 WARNING juju-log No relation: certificates
2024-03-03T22:21:24.035Z [container-agent] 2024-03-03 22:21:24 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T22:25:51.507Z [container-agent] 2024-03-03 22:25:51 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T22:25:58.868Z [container-agent] 2024-03-03 22:25:58 WARNING juju-log No relation: certificates
2024-03-03T22:25:59.211Z [container-agent] 2024-03-03 22:25:59 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T22:26:07.795Z [container-agent] 2024-03-03 22:26:07 ERROR juju-log Failed to flush ERROR LOGS logs.
2024-03-03T22:26:07.795Z [container-agent] Traceback (most recent call last):
2024-03-03T22:26:07.795Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 666, in _run_mysqlsh_script
2024-03-03T22:26:07.795Z [container-agent]     stdout, _ = process.wait_output()
2024-03-03T22:26:07.795Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1441, in wait_output
2024-03-03T22:26:07.795Z [container-agent]     raise ExecError[AnyStr](self._command, exit_code, out_value, err_value)
2024-03-03T22:26:07.795Z [container-agent] ops.pebble.ExecError: non-zero exit code 1 executing ['/usr/bin/mysqlsh', '--no-wizard', '--python', '--verbose=1', '-f', '/tmp/script.py', ';', 'rm', '/tmp/script.py'], stdout='', stderr='Cannot set LC_ALL to locale en_US.UTF-8: No such file or directory\nverbose: 2024-03-03T22:26:06Z: Loading startup files...\nverbose: 2024-03-03T22:26:06Z: Loading plugins...\nverbose: 2024-03-03T22:26:06Z: Connecting to MySQL at: serverconfig@horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local\nTraceback (most recent call last):\n  File "<string>", line 1, in <module>\nmysqlsh.DBError: MySQL Error (2003): Shell.connect: Can\'t connect to MySQL server on \'horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local:3306\' (111)\n'
2024-03-03T22:26:07.795Z [container-agent] 
2024-03-03T22:26:07.795Z [container-agent] During handling of the above exception, another exception occurred:
2024-03-03T22:26:07.795Z [container-agent] 
2024-03-03T22:26:07.795Z [container-agent] Traceback (most recent call last):
2024-03-03T22:26:07.795Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-03-03T22:26:07.795Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-03-03T22:26:07.795Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 669, in _run_mysqlsh_script
2024-03-03T22:26:07.795Z [container-agent]     raise MySQLClientError(e.stderr)
2024-03-03T22:26:07.795Z [container-agent] charms.mysql.v0.mysql.MySQLClientError: Cannot set LC_ALL to locale en_US.UTF-8: No such file or directory
2024-03-03T22:26:07.795Z [container-agent] verbose: 2024-03-03T22:26:06Z: Loading startup files...
2024-03-03T22:26:07.795Z [container-agent] verbose: 2024-03-03T22:26:06Z: Loading plugins...
2024-03-03T22:26:07.795Z [container-agent] verbose: 2024-03-03T22:26:06Z: Connecting to MySQL at: serverconfig@horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local
2024-03-03T22:26:07.795Z [container-agent] Traceback (most recent call last):
2024-03-03T22:26:07.795Z [container-agent]   File "<string>", line 1, in <module>
2024-03-03T22:26:07.795Z [container-agent] mysqlsh.DBError: MySQL Error (2003): Shell.connect: Can't connect to MySQL server on 'horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local:3306' (111)
2024-03-03T22:26:07.795Z [container-agent] 
2024-03-03T22:31:04.833Z [container-agent] 2024-03-03 22:31:04 ERROR juju-log Uncaught exception while in charm code:
2024-03-03T22:31:04.833Z [container-agent] Traceback (most recent call last):
2024-03-03T22:31:04.833Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 2507, in exec
2024-03-03T22:31:04.833Z [container-agent]     control_ws = self._connect_websocket(task_id, 'control')
2024-03-03T22:31:04.833Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 2589, in _connect_websocket
2024-03-03T22:31:04.833Z [container-agent]     ws.connect(url, socket=sock)
2024-03-03T22:31:04.833Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_core.py", line 253, in connect
2024-03-03T22:31:04.833Z [container-agent]     self.handshake_response = handshake(self.sock, url, *addrs, **options)
2024-03-03T22:31:04.833Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_handshake.py", line 57, in handshake
2024-03-03T22:31:04.833Z [container-agent]     status, resp = _get_resp_headers(sock)
2024-03-03T22:31:04.833Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_handshake.py", line 143, in _get_resp_headers
2024-03-03T22:31:04.833Z [container-agent]     status, resp_headers, status_message = read_headers(sock)
2024-03-03T22:31:04.833Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_http.py", line 312, in read_headers
2024-03-03T22:31:04.833Z [container-agent]     line = recv_line(sock)
2024-03-03T22:31:04.833Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_socket.py", line 131, in recv_line
2024-03-03T22:31:04.833Z [container-agent]     c = recv(sock, 1)
2024-03-03T22:31:04.833Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_socket.py", line 122, in recv
2024-03-03T22:31:04.833Z [container-agent]     raise WebSocketConnectionClosedException(
2024-03-03T22:31:04.833Z [container-agent] websocket._exceptions.WebSocketConnectionClosedException: Connection to remote host was lost.
2024-03-03T22:31:04.833Z [container-agent] 
2024-03-03T22:31:04.833Z [container-agent] During handling of the above exception, another exception occurred:
2024-03-03T22:31:04.833Z [container-agent] 
2024-03-03T22:31:04.833Z [container-agent] Traceback (most recent call last):
2024-03-03T22:31:04.833Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 1348, in do_open
2024-03-03T22:31:04.833Z [container-agent]     h.request(req.get_method(), req.selector, req.data, headers,
2024-03-03T22:31:04.833Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1283, in request
2024-03-03T22:31:04.833Z [container-agent]     self._send_request(method, url, body, headers, encode_chunked)
2024-03-03T22:31:04.833Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1329, in _send_request
2024-03-03T22:31:04.833Z [container-agent]     self.endheaders(body, encode_chunked=encode_chunked)
2024-03-03T22:31:04.833Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1278, in endheaders
2024-03-03T22:31:04.833Z [container-agent]     self._send_output(message_body, encode_chunked=encode_chunked)
2024-03-03T22:31:04.833Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1038, in _send_output
2024-03-03T22:31:04.833Z [container-agent]     self.send(msg)
2024-03-03T22:31:04.833Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 976, in send
2024-03-03T22:31:04.833Z [container-agent]     self.connect()
2024-03-03T22:31:04.833Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 297, in connect
2024-03-03T22:31:04.833Z [container-agent]     self.sock.connect(self.socket_path)
2024-03-03T22:31:04.833Z [container-agent] ConnectionRefusedError: [Errno 111] Connection refused
2024-03-03T22:31:04.833Z [container-agent] 
2024-03-03T22:31:04.833Z [container-agent] During handling of the above exception, another exception occurred:
2024-03-03T22:31:04.833Z [container-agent] 
2024-03-03T22:31:04.833Z [container-agent] Traceback (most recent call last):
2024-03-03T22:31:04.833Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1690, in _request_raw
2024-03-03T22:31:04.833Z [container-agent]     response = self.opener.open(request, timeout=self.timeout)
2024-03-03T22:31:04.833Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 519, in open
2024-03-03T22:31:04.833Z [container-agent]     response = self._open(req, data)
2024-03-03T22:31:04.833Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 536, in _open
2024-03-03T22:31:04.833Z [container-agent]     result = self._call_chain(self.handle_open, protocol, protocol +
2024-03-03T22:31:04.833Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 496, in _call_chain
2024-03-03T22:31:04.833Z [container-agent]     result = func(*args)
2024-03-03T22:31:04.833Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 311, in http_open
2024-03-03T22:31:04.833Z [container-agent]     return self.do_open(_UnixSocketConnection, req,  # type:ignore
2024-03-03T22:31:04.833Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 1351, in do_open
2024-03-03T22:31:04.833Z [container-agent]     raise URLError(err)
2024-03-03T22:31:04.833Z [container-agent] urllib.error.URLError: <urlopen error [Errno 111] Connection refused>
2024-03-03T22:31:04.833Z [container-agent] 
2024-03-03T22:31:04.833Z [container-agent] The above exception was the direct cause of the following exception:
2024-03-03T22:31:04.833Z [container-agent] 
2024-03-03T22:31:04.833Z [container-agent] Traceback (most recent call last):
2024-03-03T22:31:04.833Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/./src/charm.py", line 786, in <module>
2024-03-03T22:31:04.833Z [container-agent]     main(MySQLOperatorCharm)
2024-03-03T22:31:04.833Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 436, in main
2024-03-03T22:31:04.833Z [container-agent]     _emit_charm_event(charm, dispatcher.event_name)
2024-03-03T22:31:04.833Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 144, in _emit_charm_event
2024-03-03T22:31:04.833Z [container-agent]     event_to_emit.emit(*args, **kwargs)
2024-03-03T22:31:04.833Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 351, in emit
2024-03-03T22:31:04.833Z [container-agent]     framework._emit(event)
2024-03-03T22:31:04.833Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 853, in _emit
2024-03-03T22:31:04.833Z [container-agent]     self._reemit(event_path)
2024-03-03T22:31:04.833Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 942, in _reemit
2024-03-03T22:31:04.833Z [container-agent]     custom_handler(event)
2024-03-03T22:31:04.833Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/rotate_mysql_logs.py", line 55, in _rotate_mysql_logs
2024-03-03T22:31:04.833Z [container-agent]     self.charm._mysql.flush_mysql_logs(MySQLTextLogs.ERROR)
2024-03-03T22:31:04.833Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-03-03T22:31:04.833Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-03-03T22:31:04.833Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 665, in _run_mysqlsh_script
2024-03-03T22:31:04.833Z [container-agent]     process = self.container.exec(cmd, timeout=timeout)
2024-03-03T22:31:04.833Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/model.py", line 2688, in exec
2024-03-03T22:31:04.833Z [container-agent]     return self._pebble.exec(
2024-03-03T22:31:04.833Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 2514, in exec
2024-03-03T22:31:04.833Z [container-agent]     change = self.wait_change(ChangeID(change_id))
2024-03-03T22:31:04.833Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1893, in wait_change
2024-03-03T22:31:04.833Z [container-agent]     return self._wait_change_using_wait(change_id, timeout)
2024-03-03T22:31:04.833Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1914, in _wait_change_using_wait
2024-03-03T22:31:04.833Z [container-agent]     return self._wait_change(change_id, this_timeout)
2024-03-03T22:31:04.833Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1928, in _wait_change
2024-03-03T22:31:04.833Z [container-agent]     resp = self._request('GET', f'/v1/changes/{change_id}/wait', query)
2024-03-03T22:31:04.833Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1655, in _request
2024-03-03T22:31:04.833Z [container-agent]     response = self._request_raw(method, path, query, headers, data)
2024-03-03T22:31:04.833Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1707, in _request_raw
2024-03-03T22:31:04.833Z [container-agent]     raise ConnectionError(e.reason) from e
2024-03-03T22:31:04.833Z [container-agent] ops.pebble.ConnectionError: [Errno 111] Connection refused
2024-03-03T22:31:05.009Z [container-agent] 2024-03-03 22:31:05 INFO juju.util.exec exec.go:209 run result: exit status 1
2024-03-03T22:31:06.677Z [container-agent] 2024-03-03 22:31:06 INFO juju-log Starting the log rotate manager
2024-03-03T22:31:06.690Z [container-agent] 2024-03-03 22:31:06 INFO juju-log Started log rotate manager process with PID 746894
2024-03-03T22:31:06.864Z [container-agent] 2024-03-03 22:31:06 INFO juju-log Setting up the logrotate configurations
2024-03-03T22:31:06.944Z [container-agent] 2024-03-03 22:31:06 INFO juju-log Adding pebble layer
2024-03-03T22:31:13.749Z [container-agent] 2024-03-03 22:31:13 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-03-03T22:31:13.778Z [container-agent] 2024-03-03 22:31:13 INFO juju-log Attempting reboot from complete outage.
2024-03-03T22:31:16.740Z [container-agent] 2024-03-03 22:31:16 INFO juju.worker.uniter.operation runhook.go:186 ran "mysql-pebble-ready" hook (via hook dispatching script: dispatch)
2024-03-03T22:31:18.288Z [container-agent] 2024-03-03 22:31:18 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T22:31:20.344Z [container-agent] 2024-03-03 22:31:20 WARNING juju-log No relation: certificates
2024-03-03T22:31:20.707Z [container-agent] 2024-03-03 22:31:20 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T22:36:45.382Z [container-agent] 2024-03-03 22:36:45 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T22:36:47.735Z [container-agent] 2024-03-03 22:36:47 WARNING juju-log No relation: certificates
2024-03-03T22:36:48.102Z [container-agent] 2024-03-03 22:36:48 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T22:41:53.450Z [container-agent] 2024-03-03 22:41:53 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T22:41:56.505Z [container-agent] 2024-03-03 22:41:56 WARNING juju-log No relation: certificates
2024-03-03T22:41:56.858Z [container-agent] 2024-03-03 22:41:56 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T22:47:46.727Z [container-agent] 2024-03-03 22:47:46 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T22:47:50.378Z [container-agent] 2024-03-03 22:47:50 WARNING juju-log No relation: certificates
2024-03-03T22:47:50.717Z [container-agent] 2024-03-03 22:47:50 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T22:52:54.394Z [container-agent] 2024-03-03 22:52:54 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T22:52:58.477Z [container-agent] 2024-03-03 22:52:58 WARNING juju-log No relation: certificates
2024-03-03T22:52:58.873Z [container-agent] 2024-03-03 22:52:58 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T22:57:15.741Z [container-agent] 2024-03-03 22:57:15 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T22:57:20.402Z [container-agent] 2024-03-03 22:57:20 WARNING juju-log No relation: certificates
2024-03-03T22:57:20.773Z [container-agent] 2024-03-03 22:57:20 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T23:02:34.485Z [container-agent] 2024-03-03 23:02:34 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T23:02:39.346Z [container-agent] 2024-03-03 23:02:39 WARNING juju-log No relation: certificates
2024-03-03T23:02:39.684Z [container-agent] 2024-03-03 23:02:39 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T23:06:47.644Z [container-agent] 2024-03-03 23:06:47 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T23:06:52.789Z [container-agent] 2024-03-03 23:06:52 WARNING juju-log No relation: certificates
2024-03-03T23:06:53.186Z [container-agent] 2024-03-03 23:06:53 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T23:12:21.559Z [container-agent] 2024-03-03 23:12:21 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T23:12:26.495Z [container-agent] 2024-03-03 23:12:26 WARNING juju-log No relation: certificates
2024-03-03T23:12:26.848Z [container-agent] 2024-03-03 23:12:26 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T23:17:15.857Z [container-agent] 2024-03-03 23:17:15 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T23:17:22.118Z [container-agent] 2024-03-03 23:17:22 WARNING juju-log No relation: certificates
2024-03-03T23:17:22.467Z [container-agent] 2024-03-03 23:17:22 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T23:22:40.664Z [container-agent] 2024-03-03 23:22:40 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T23:22:47.106Z [container-agent] 2024-03-03 23:22:47 WARNING juju-log No relation: certificates
2024-03-03T23:22:47.445Z [container-agent] 2024-03-03 23:22:47 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T23:27:55.626Z [container-agent] 2024-03-03 23:27:55 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T23:28:02.723Z [container-agent] 2024-03-03 23:28:02 WARNING juju-log No relation: certificates
2024-03-03T23:28:03.073Z [container-agent] 2024-03-03 23:28:03 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T23:33:18.783Z [container-agent] 2024-03-03 23:33:18 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T23:33:26.556Z [container-agent] 2024-03-03 23:33:26 WARNING juju-log No relation: certificates
2024-03-03T23:33:26.924Z [container-agent] 2024-03-03 23:33:26 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T23:38:44.202Z [container-agent] 2024-03-03 23:38:44 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T23:38:52.087Z [container-agent] 2024-03-03 23:38:52 WARNING juju-log No relation: certificates
2024-03-03T23:38:52.418Z [container-agent] 2024-03-03 23:38:52 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T23:41:07.800Z [container-agent] 2024-03-03 23:41:07 ERROR juju-log Failed to flush ERROR LOGS logs.
2024-03-03T23:41:07.800Z [container-agent] Traceback (most recent call last):
2024-03-03T23:41:07.800Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 666, in _run_mysqlsh_script
2024-03-03T23:41:07.800Z [container-agent]     stdout, _ = process.wait_output()
2024-03-03T23:41:07.800Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1441, in wait_output
2024-03-03T23:41:07.800Z [container-agent]     raise ExecError[AnyStr](self._command, exit_code, out_value, err_value)
2024-03-03T23:41:07.800Z [container-agent] ops.pebble.ExecError: non-zero exit code 1 executing ['/usr/bin/mysqlsh', '--no-wizard', '--python', '--verbose=1', '-f', '/tmp/script.py', ';', 'rm', '/tmp/script.py'], stdout='', stderr='Cannot set LC_ALL to locale en_US.UTF-8: No such file or directory\nverbose: 2024-03-03T23:41:06Z: Loading startup files...\nverbose: 2024-03-03T23:41:06Z: Loading plugins...\nverbose: 2024-03-03T23:41:06Z: Connecting to MySQL at: serverconfig@horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local\nTraceback (most recent call last):\n  File "<string>", line 1, in <module>\nmysqlsh.DBError: MySQL Error (2003): Shell.connect: Can\'t connect to MySQL server on \'horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local:3306\' (111)\n'
2024-03-03T23:41:07.800Z [container-agent] 
2024-03-03T23:41:07.800Z [container-agent] During handling of the above exception, another exception occurred:
2024-03-03T23:41:07.800Z [container-agent] 
2024-03-03T23:41:07.800Z [container-agent] Traceback (most recent call last):
2024-03-03T23:41:07.800Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-03-03T23:41:07.800Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-03-03T23:41:07.800Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 669, in _run_mysqlsh_script
2024-03-03T23:41:07.800Z [container-agent]     raise MySQLClientError(e.stderr)
2024-03-03T23:41:07.800Z [container-agent] charms.mysql.v0.mysql.MySQLClientError: Cannot set LC_ALL to locale en_US.UTF-8: No such file or directory
2024-03-03T23:41:07.800Z [container-agent] verbose: 2024-03-03T23:41:06Z: Loading startup files...
2024-03-03T23:41:07.800Z [container-agent] verbose: 2024-03-03T23:41:06Z: Loading plugins...
2024-03-03T23:41:07.800Z [container-agent] verbose: 2024-03-03T23:41:06Z: Connecting to MySQL at: serverconfig@horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local
2024-03-03T23:41:07.800Z [container-agent] Traceback (most recent call last):
2024-03-03T23:41:07.800Z [container-agent]   File "<string>", line 1, in <module>
2024-03-03T23:41:07.800Z [container-agent] mysqlsh.DBError: MySQL Error (2003): Shell.connect: Can't connect to MySQL server on 'horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local:3306' (111)
2024-03-03T23:41:07.800Z [container-agent] 
2024-03-03T23:42:45.917Z [container-agent] 2024-03-03 23:42:45 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-03-03T23:42:45.947Z [container-agent] 2024-03-03 23:42:45 INFO juju-log Attempting reboot from complete outage.
2024-03-03T23:42:53.665Z [container-agent] 2024-03-03 23:42:53 WARNING juju-log No relation: certificates
2024-03-03T23:42:54.043Z [container-agent] 2024-03-03 23:42:54 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T23:46:04.795Z [container-agent] 2024-03-03 23:46:04 ERROR juju-log Uncaught exception while in charm code:
2024-03-03T23:46:04.795Z [container-agent] Traceback (most recent call last):
2024-03-03T23:46:04.795Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 2507, in exec
2024-03-03T23:46:04.795Z [container-agent]     control_ws = self._connect_websocket(task_id, 'control')
2024-03-03T23:46:04.795Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 2589, in _connect_websocket
2024-03-03T23:46:04.795Z [container-agent]     ws.connect(url, socket=sock)
2024-03-03T23:46:04.795Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_core.py", line 253, in connect
2024-03-03T23:46:04.795Z [container-agent]     self.handshake_response = handshake(self.sock, url, *addrs, **options)
2024-03-03T23:46:04.795Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_handshake.py", line 57, in handshake
2024-03-03T23:46:04.795Z [container-agent]     status, resp = _get_resp_headers(sock)
2024-03-03T23:46:04.795Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_handshake.py", line 143, in _get_resp_headers
2024-03-03T23:46:04.795Z [container-agent]     status, resp_headers, status_message = read_headers(sock)
2024-03-03T23:46:04.795Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_http.py", line 312, in read_headers
2024-03-03T23:46:04.795Z [container-agent]     line = recv_line(sock)
2024-03-03T23:46:04.795Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_socket.py", line 131, in recv_line
2024-03-03T23:46:04.795Z [container-agent]     c = recv(sock, 1)
2024-03-03T23:46:04.795Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_socket.py", line 122, in recv
2024-03-03T23:46:04.795Z [container-agent]     raise WebSocketConnectionClosedException(
2024-03-03T23:46:04.795Z [container-agent] websocket._exceptions.WebSocketConnectionClosedException: Connection to remote host was lost.
2024-03-03T23:46:04.795Z [container-agent] 
2024-03-03T23:46:04.795Z [container-agent] During handling of the above exception, another exception occurred:
2024-03-03T23:46:04.795Z [container-agent] 
2024-03-03T23:46:04.795Z [container-agent] Traceback (most recent call last):
2024-03-03T23:46:04.795Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 1348, in do_open
2024-03-03T23:46:04.795Z [container-agent]     h.request(req.get_method(), req.selector, req.data, headers,
2024-03-03T23:46:04.795Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1283, in request
2024-03-03T23:46:04.795Z [container-agent]     self._send_request(method, url, body, headers, encode_chunked)
2024-03-03T23:46:04.795Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1329, in _send_request
2024-03-03T23:46:04.795Z [container-agent]     self.endheaders(body, encode_chunked=encode_chunked)
2024-03-03T23:46:04.795Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1278, in endheaders
2024-03-03T23:46:04.795Z [container-agent]     self._send_output(message_body, encode_chunked=encode_chunked)
2024-03-03T23:46:04.795Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1038, in _send_output
2024-03-03T23:46:04.795Z [container-agent]     self.send(msg)
2024-03-03T23:46:04.795Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 976, in send
2024-03-03T23:46:04.795Z [container-agent]     self.connect()
2024-03-03T23:46:04.795Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 297, in connect
2024-03-03T23:46:04.795Z [container-agent]     self.sock.connect(self.socket_path)
2024-03-03T23:46:04.795Z [container-agent] ConnectionRefusedError: [Errno 111] Connection refused
2024-03-03T23:46:04.795Z [container-agent] 
2024-03-03T23:46:04.795Z [container-agent] During handling of the above exception, another exception occurred:
2024-03-03T23:46:04.795Z [container-agent] 
2024-03-03T23:46:04.795Z [container-agent] Traceback (most recent call last):
2024-03-03T23:46:04.795Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1690, in _request_raw
2024-03-03T23:46:04.795Z [container-agent]     response = self.opener.open(request, timeout=self.timeout)
2024-03-03T23:46:04.795Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 519, in open
2024-03-03T23:46:04.795Z [container-agent]     response = self._open(req, data)
2024-03-03T23:46:04.795Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 536, in _open
2024-03-03T23:46:04.795Z [container-agent]     result = self._call_chain(self.handle_open, protocol, protocol +
2024-03-03T23:46:04.795Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 496, in _call_chain
2024-03-03T23:46:04.795Z [container-agent]     result = func(*args)
2024-03-03T23:46:04.795Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 311, in http_open
2024-03-03T23:46:04.795Z [container-agent]     return self.do_open(_UnixSocketConnection, req,  # type:ignore
2024-03-03T23:46:04.795Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 1351, in do_open
2024-03-03T23:46:04.795Z [container-agent]     raise URLError(err)
2024-03-03T23:46:04.795Z [container-agent] urllib.error.URLError: <urlopen error [Errno 111] Connection refused>
2024-03-03T23:46:04.795Z [container-agent] 
2024-03-03T23:46:04.795Z [container-agent] The above exception was the direct cause of the following exception:
2024-03-03T23:46:04.795Z [container-agent] 
2024-03-03T23:46:04.795Z [container-agent] Traceback (most recent call last):
2024-03-03T23:46:04.795Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/./src/charm.py", line 786, in <module>
2024-03-03T23:46:04.795Z [container-agent]     main(MySQLOperatorCharm)
2024-03-03T23:46:04.795Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 436, in main
2024-03-03T23:46:04.795Z [container-agent]     _emit_charm_event(charm, dispatcher.event_name)
2024-03-03T23:46:04.795Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 144, in _emit_charm_event
2024-03-03T23:46:04.795Z [container-agent]     event_to_emit.emit(*args, **kwargs)
2024-03-03T23:46:04.795Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 351, in emit
2024-03-03T23:46:04.795Z [container-agent]     framework._emit(event)
2024-03-03T23:46:04.795Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 853, in _emit
2024-03-03T23:46:04.795Z [container-agent]     self._reemit(event_path)
2024-03-03T23:46:04.795Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 942, in _reemit
2024-03-03T23:46:04.795Z [container-agent]     custom_handler(event)
2024-03-03T23:46:04.796Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/rotate_mysql_logs.py", line 55, in _rotate_mysql_logs
2024-03-03T23:46:04.796Z [container-agent]     self.charm._mysql.flush_mysql_logs(MySQLTextLogs.ERROR)
2024-03-03T23:46:04.796Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-03-03T23:46:04.796Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-03-03T23:46:04.796Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 665, in _run_mysqlsh_script
2024-03-03T23:46:04.796Z [container-agent]     process = self.container.exec(cmd, timeout=timeout)
2024-03-03T23:46:04.796Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/model.py", line 2688, in exec
2024-03-03T23:46:04.796Z [container-agent]     return self._pebble.exec(
2024-03-03T23:46:04.796Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 2514, in exec
2024-03-03T23:46:04.796Z [container-agent]     change = self.wait_change(ChangeID(change_id))
2024-03-03T23:46:04.796Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1893, in wait_change
2024-03-03T23:46:04.796Z [container-agent]     return self._wait_change_using_wait(change_id, timeout)
2024-03-03T23:46:04.796Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1914, in _wait_change_using_wait
2024-03-03T23:46:04.796Z [container-agent]     return self._wait_change(change_id, this_timeout)
2024-03-03T23:46:04.796Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1928, in _wait_change
2024-03-03T23:46:04.796Z [container-agent]     resp = self._request('GET', f'/v1/changes/{change_id}/wait', query)
2024-03-03T23:46:04.796Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1655, in _request
2024-03-03T23:46:04.796Z [container-agent]     response = self._request_raw(method, path, query, headers, data)
2024-03-03T23:46:04.796Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1707, in _request_raw
2024-03-03T23:46:04.796Z [container-agent]     raise ConnectionError(e.reason) from e
2024-03-03T23:46:04.796Z [container-agent] ops.pebble.ConnectionError: [Errno 111] Connection refused
2024-03-03T23:46:04.947Z [container-agent] 2024-03-03 23:46:04 INFO juju.util.exec exec.go:209 run result: exit status 1
2024-03-03T23:46:09.747Z [container-agent] 2024-03-03 23:46:09 INFO juju-log Setting up the logrotate configurations
2024-03-03T23:46:09.809Z [container-agent] 2024-03-03 23:46:09 INFO juju-log Adding pebble layer
2024-03-03T23:46:16.624Z [container-agent] 2024-03-03 23:46:16 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-03-03T23:46:16.654Z [container-agent] 2024-03-03 23:46:16 INFO juju-log Attempting reboot from complete outage.
2024-03-03T23:46:19.552Z [container-agent] 2024-03-03 23:46:19 INFO juju.worker.uniter.operation runhook.go:186 ran "mysql-pebble-ready" hook (via hook dispatching script: dispatch)
2024-03-03T23:46:50.778Z [container-agent] 2024-03-03 23:46:50 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T23:46:52.749Z [container-agent] 2024-03-03 23:46:52 WARNING juju-log No relation: certificates
2024-03-03T23:46:53.086Z [container-agent] 2024-03-03 23:46:53 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T23:51:33.782Z [container-agent] 2024-03-03 23:51:33 INFO juju-log Starting the log rotate manager
2024-03-03T23:51:33.796Z [container-agent] 2024-03-03 23:51:33 INFO juju-log Started log rotate manager process with PID 754711
2024-03-03T23:51:34.154Z [container-agent] 2024-03-03 23:51:34 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T23:51:36.183Z [container-agent] 2024-03-03 23:51:36 WARNING juju-log No relation: certificates
2024-03-03T23:51:36.565Z [container-agent] 2024-03-03 23:51:36 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-03T23:57:21.477Z [container-agent] 2024-03-03 23:57:21 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-03T23:57:23.812Z [container-agent] 2024-03-03 23:57:23 WARNING juju-log No relation: certificates
2024-03-03T23:57:24.156Z [container-agent] 2024-03-03 23:57:24 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T00:03:07.345Z [container-agent] 2024-03-04 00:03:07 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T00:03:10.334Z [container-agent] 2024-03-04 00:03:10 WARNING juju-log No relation: certificates
2024-03-04T00:03:10.664Z [container-agent] 2024-03-04 00:03:10 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T00:07:55.009Z [container-agent] 2024-03-04 00:07:55 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T00:07:58.481Z [container-agent] 2024-03-04 00:07:58 WARNING juju-log No relation: certificates
2024-03-04T00:07:58.850Z [container-agent] 2024-03-04 00:07:58 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T00:12:52.008Z [container-agent] 2024-03-04 00:12:52 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T00:12:55.687Z [container-agent] 2024-03-04 00:12:55 WARNING juju-log No relation: certificates
2024-03-04T00:12:56.026Z [container-agent] 2024-03-04 00:12:56 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T00:16:54.035Z [container-agent] 2024-03-04 00:16:54 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T00:16:58.105Z [container-agent] 2024-03-04 00:16:58 WARNING juju-log No relation: certificates
2024-03-04T00:16:58.469Z [container-agent] 2024-03-04 00:16:58 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T00:21:01.289Z [container-agent] 2024-03-04 00:21:01 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T00:21:05.733Z [container-agent] 2024-03-04 00:21:05 WARNING juju-log No relation: certificates
2024-03-04T00:21:06.061Z [container-agent] 2024-03-04 00:21:06 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T00:26:40.843Z [container-agent] 2024-03-04 00:26:40 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T00:26:46.204Z [container-agent] 2024-03-04 00:26:46 WARNING juju-log No relation: certificates
2024-03-04T00:26:46.623Z [container-agent] 2024-03-04 00:26:46 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T00:31:19.955Z [container-agent] 2024-03-04 00:31:19 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T00:31:25.873Z [container-agent] 2024-03-04 00:31:25 WARNING juju-log No relation: certificates
2024-03-04T00:31:26.215Z [container-agent] 2024-03-04 00:31:26 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T00:36:23.329Z [container-agent] 2024-03-04 00:36:23 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T00:36:29.451Z [container-agent] 2024-03-04 00:36:29 WARNING juju-log No relation: certificates
2024-03-04T00:36:29.798Z [container-agent] 2024-03-04 00:36:29 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T00:41:31.739Z [container-agent] 2024-03-04 00:41:31 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T00:41:38.484Z [container-agent] 2024-03-04 00:41:38 WARNING juju-log No relation: certificates
2024-03-04T00:41:38.853Z [container-agent] 2024-03-04 00:41:38 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T00:46:33.990Z [container-agent] 2024-03-04 00:46:33 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T00:46:40.904Z [container-agent] 2024-03-04 00:46:40 WARNING juju-log No relation: certificates
2024-03-04T00:46:41.245Z [container-agent] 2024-03-04 00:46:41 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T00:50:50.681Z [container-agent] 2024-03-04 00:50:50 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T00:50:57.989Z [container-agent] 2024-03-04 00:50:57 WARNING juju-log No relation: certificates
2024-03-04T00:50:58.348Z [container-agent] 2024-03-04 00:50:58 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T00:55:43.037Z [container-agent] 2024-03-04 00:55:43 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T00:55:50.995Z [container-agent] 2024-03-04 00:55:50 WARNING juju-log No relation: certificates
2024-03-04T00:55:51.400Z [container-agent] 2024-03-04 00:55:51 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T01:00:38.418Z [container-agent] 2024-03-04 01:00:38 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T01:00:46.342Z [container-agent] 2024-03-04 01:00:46 WARNING juju-log No relation: certificates
2024-03-04T01:00:46.709Z [container-agent] 2024-03-04 01:00:46 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T01:04:10.651Z [container-agent] 2024-03-04 01:04:10 ERROR juju-log Failed to flush ERROR LOGS logs.
2024-03-04T01:04:10.651Z [container-agent] Traceback (most recent call last):
2024-03-04T01:04:10.651Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 666, in _run_mysqlsh_script
2024-03-04T01:04:10.651Z [container-agent]     stdout, _ = process.wait_output()
2024-03-04T01:04:10.651Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1441, in wait_output
2024-03-04T01:04:10.651Z [container-agent]     raise ExecError[AnyStr](self._command, exit_code, out_value, err_value)
2024-03-04T01:04:10.651Z [container-agent] ops.pebble.ExecError: non-zero exit code 1 executing ['/usr/bin/mysqlsh', '--no-wizard', '--python', '--verbose=1', '-f', '/tmp/script.py', ';', 'rm', '/tmp/script.py'], stdout='', stderr='Cannot set LC_ALL to locale en_US.UTF-8: No such file or directory\nverbose: 2024-03-04T01:04:06Z: Loading startup files...\nverbose: 2024-03-04T01:04:06Z: Loading plugins...\nverbose: 2024-03-04T01:04:06Z: Connecting to MySQL at: serverconfig@horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local\nTraceback (most recent call last):\n  File "<string>", line 1, in <module>\nmysqlsh.DBError: MySQL Error (2013): Shell.connect: Lost connection to MySQL server at \'reading initial communication packet\', system error: 104\n'
2024-03-04T01:04:10.651Z [container-agent] 
2024-03-04T01:04:10.651Z [container-agent] During handling of the above exception, another exception occurred:
2024-03-04T01:04:10.651Z [container-agent] 
2024-03-04T01:04:10.651Z [container-agent] Traceback (most recent call last):
2024-03-04T01:04:10.651Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-03-04T01:04:10.651Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-03-04T01:04:10.651Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 669, in _run_mysqlsh_script
2024-03-04T01:04:10.651Z [container-agent]     raise MySQLClientError(e.stderr)
2024-03-04T01:04:10.651Z [container-agent] charms.mysql.v0.mysql.MySQLClientError: Cannot set LC_ALL to locale en_US.UTF-8: No such file or directory
2024-03-04T01:04:10.651Z [container-agent] verbose: 2024-03-04T01:04:06Z: Loading startup files...
2024-03-04T01:04:10.651Z [container-agent] verbose: 2024-03-04T01:04:06Z: Loading plugins...
2024-03-04T01:04:10.651Z [container-agent] verbose: 2024-03-04T01:04:06Z: Connecting to MySQL at: serverconfig@horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local
2024-03-04T01:04:10.651Z [container-agent] Traceback (most recent call last):
2024-03-04T01:04:10.651Z [container-agent]   File "<string>", line 1, in <module>
2024-03-04T01:04:10.651Z [container-agent] mysqlsh.DBError: MySQL Error (2013): Shell.connect: Lost connection to MySQL server at 'reading initial communication packet', system error: 104
2024-03-04T01:04:10.651Z [container-agent] 
2024-03-04T01:04:13.922Z [container-agent] 2024-03-04 01:04:13 ERROR juju-log Uncaught exception while in charm code:
2024-03-04T01:04:13.923Z [container-agent] Traceback (most recent call last):
2024-03-04T01:04:13.923Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/./src/charm.py", line 786, in <module>
2024-03-04T01:04:13.923Z [container-agent]     main(MySQLOperatorCharm)
2024-03-04T01:04:13.923Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 436, in main
2024-03-04T01:04:13.923Z [container-agent]     _emit_charm_event(charm, dispatcher.event_name)
2024-03-04T01:04:13.923Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 144, in _emit_charm_event
2024-03-04T01:04:13.923Z [container-agent]     event_to_emit.emit(*args, **kwargs)
2024-03-04T01:04:13.923Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 351, in emit
2024-03-04T01:04:13.923Z [container-agent]     framework._emit(event)
2024-03-04T01:04:13.923Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 853, in _emit
2024-03-04T01:04:13.923Z [container-agent]     self._reemit(event_path)
2024-03-04T01:04:13.923Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 942, in _reemit
2024-03-04T01:04:13.923Z [container-agent]     custom_handler(event)
2024-03-04T01:04:13.923Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/rotate_mysql_logs.py", line 56, in _rotate_mysql_logs
2024-03-04T01:04:13.923Z [container-agent]     self.charm._mysql.flush_mysql_logs(MySQLTextLogs.GENERAL)
2024-03-04T01:04:13.923Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-03-04T01:04:13.923Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-03-04T01:04:13.923Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 666, in _run_mysqlsh_script
2024-03-04T01:04:13.923Z [container-agent]     stdout, _ = process.wait_output()
2024-03-04T01:04:13.923Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1436, in wait_output
2024-03-04T01:04:13.923Z [container-agent]     exit_code: int = self._wait()
2024-03-04T01:04:13.923Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1375, in _wait
2024-03-04T01:04:13.923Z [container-agent]     change = self._client.wait_change(self._change_id, timeout=timeout)
2024-03-04T01:04:13.923Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1893, in wait_change
2024-03-04T01:04:13.923Z [container-agent]     return self._wait_change_using_wait(change_id, timeout)
2024-03-04T01:04:13.923Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1914, in _wait_change_using_wait
2024-03-04T01:04:13.923Z [container-agent]     return self._wait_change(change_id, this_timeout)
2024-03-04T01:04:13.923Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1928, in _wait_change
2024-03-04T01:04:13.923Z [container-agent]     resp = self._request('GET', f'/v1/changes/{change_id}/wait', query)
2024-03-04T01:04:13.923Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1655, in _request
2024-03-04T01:04:13.923Z [container-agent]     response = self._request_raw(method, path, query, headers, data)
2024-03-04T01:04:13.923Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1690, in _request_raw
2024-03-04T01:04:13.923Z [container-agent]     response = self.opener.open(request, timeout=self.timeout)
2024-03-04T01:04:13.923Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 519, in open
2024-03-04T01:04:13.923Z [container-agent]     response = self._open(req, data)
2024-03-04T01:04:13.923Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 536, in _open
2024-03-04T01:04:13.923Z [container-agent]     result = self._call_chain(self.handle_open, protocol, protocol +
2024-03-04T01:04:13.923Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 496, in _call_chain
2024-03-04T01:04:13.923Z [container-agent]     result = func(*args)
2024-03-04T01:04:13.923Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 311, in http_open
2024-03-04T01:04:13.923Z [container-agent]     return self.do_open(_UnixSocketConnection, req,  # type:ignore
2024-03-04T01:04:13.923Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 1352, in do_open
2024-03-04T01:04:13.923Z [container-agent]     r = h.getresponse()
2024-03-04T01:04:13.923Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1375, in getresponse
2024-03-04T01:04:13.923Z [container-agent]     response.begin()
2024-03-04T01:04:13.923Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 318, in begin
2024-03-04T01:04:13.923Z [container-agent]     version, status, reason = self._read_status()
2024-03-04T01:04:13.923Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 287, in _read_status
2024-03-04T01:04:13.923Z [container-agent]     raise RemoteDisconnected("Remote end closed connection without"
2024-03-04T01:04:13.923Z [container-agent] http.client.RemoteDisconnected: Remote end closed connection without response
2024-03-04T01:04:14.109Z [container-agent] 2024-03-04 01:04:14 INFO juju.util.exec exec.go:209 run result: exit status 1
2024-03-04T01:04:17.346Z [container-agent] 2024-03-04 01:04:17 INFO juju-log Starting the log rotate manager
2024-03-04T01:04:17.359Z [container-agent] 2024-03-04 01:04:17 INFO juju-log Started log rotate manager process with PID 761818
2024-03-04T01:04:17.709Z [container-agent] 2024-03-04 01:04:17 INFO juju-log Setting up the logrotate configurations
2024-03-04T01:04:17.799Z [container-agent] 2024-03-04 01:04:17 INFO juju-log Adding pebble layer
2024-03-04T01:04:24.525Z [container-agent] 2024-03-04 01:04:24 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-03-04T01:04:24.554Z [container-agent] 2024-03-04 01:04:24 INFO juju-log Attempting reboot from complete outage.
2024-03-04T01:04:27.515Z [container-agent] 2024-03-04 01:04:27 INFO juju.worker.uniter.operation runhook.go:186 ran "mysql-pebble-ready" hook (via hook dispatching script: dispatch)
2024-03-04T01:06:31.206Z [container-agent] 2024-03-04 01:06:31 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T01:06:33.461Z [container-agent] 2024-03-04 01:06:33 WARNING juju-log No relation: certificates
2024-03-04T01:06:34.310Z [container-agent] 2024-03-04 01:06:34 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T01:11:39.214Z [container-agent] 2024-03-04 01:11:39 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T01:11:41.892Z [container-agent] 2024-03-04 01:11:41 WARNING juju-log No relation: certificates
2024-03-04T01:11:42.244Z [container-agent] 2024-03-04 01:11:42 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T01:17:15.923Z [container-agent] 2024-03-04 01:17:15 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T01:17:19.704Z [container-agent] 2024-03-04 01:17:19 WARNING juju-log No relation: certificates
2024-03-04T01:17:20.075Z [container-agent] 2024-03-04 01:17:20 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T01:23:12.242Z [container-agent] 2024-03-04 01:23:12 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T01:23:15.736Z [container-agent] 2024-03-04 01:23:15 WARNING juju-log No relation: certificates
2024-03-04T01:23:16.106Z [container-agent] 2024-03-04 01:23:16 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T01:28:46.126Z [container-agent] 2024-03-04 01:28:46 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T01:28:50.432Z [container-agent] 2024-03-04 01:28:50 WARNING juju-log No relation: certificates
2024-03-04T01:28:50.798Z [container-agent] 2024-03-04 01:28:50 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T01:33:09.893Z [container-agent] 2024-03-04 01:33:09 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T01:33:14.649Z [container-agent] 2024-03-04 01:33:14 WARNING juju-log No relation: certificates
2024-03-04T01:33:15.015Z [container-agent] 2024-03-04 01:33:15 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T01:38:21.768Z [container-agent] 2024-03-04 01:38:21 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T01:38:26.216Z [container-agent] 2024-03-04 01:38:26 WARNING juju-log No relation: certificates
2024-03-04T01:38:26.564Z [container-agent] 2024-03-04 01:38:26 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T01:42:49.658Z [container-agent] 2024-03-04 01:42:49 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T01:42:55.269Z [container-agent] 2024-03-04 01:42:55 WARNING juju-log No relation: certificates
2024-03-04T01:42:55.628Z [container-agent] 2024-03-04 01:42:55 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T01:47:44.783Z [container-agent] 2024-03-04 01:47:44 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T01:47:50.890Z [container-agent] 2024-03-04 01:47:50 WARNING juju-log No relation: certificates
2024-03-04T01:47:51.242Z [container-agent] 2024-03-04 01:47:51 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T01:52:35.776Z [container-agent] 2024-03-04 01:52:35 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T01:52:41.608Z [container-agent] 2024-03-04 01:52:41 WARNING juju-log No relation: certificates
2024-03-04T01:52:41.958Z [container-agent] 2024-03-04 01:52:41 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T01:58:26.956Z [container-agent] 2024-03-04 01:58:26 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T01:58:33.858Z [container-agent] 2024-03-04 01:58:33 WARNING juju-log No relation: certificates
2024-03-04T01:58:34.239Z [container-agent] 2024-03-04 01:58:34 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T02:03:15.170Z [container-agent] 2024-03-04 02:03:15 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T02:03:22.555Z [container-agent] 2024-03-04 02:03:22 WARNING juju-log No relation: certificates
2024-03-04T02:03:22.902Z [container-agent] 2024-03-04 02:03:22 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T02:08:48.750Z [container-agent] 2024-03-04 02:08:48 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T02:08:56.421Z [container-agent] 2024-03-04 02:08:56 WARNING juju-log No relation: certificates
2024-03-04T02:08:56.772Z [container-agent] 2024-03-04 02:08:56 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T02:14:04.756Z [container-agent] 2024-03-04 02:14:04 ERROR juju-log Uncaught exception while in charm code:
2024-03-04T02:14:04.756Z [container-agent] Traceback (most recent call last):
2024-03-04T02:14:04.756Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 2507, in exec
2024-03-04T02:14:04.756Z [container-agent]     control_ws = self._connect_websocket(task_id, 'control')
2024-03-04T02:14:04.756Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 2589, in _connect_websocket
2024-03-04T02:14:04.756Z [container-agent]     ws.connect(url, socket=sock)
2024-03-04T02:14:04.756Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_core.py", line 253, in connect
2024-03-04T02:14:04.756Z [container-agent]     self.handshake_response = handshake(self.sock, url, *addrs, **options)
2024-03-04T02:14:04.756Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_handshake.py", line 57, in handshake
2024-03-04T02:14:04.756Z [container-agent]     status, resp = _get_resp_headers(sock)
2024-03-04T02:14:04.756Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_handshake.py", line 143, in _get_resp_headers
2024-03-04T02:14:04.756Z [container-agent]     status, resp_headers, status_message = read_headers(sock)
2024-03-04T02:14:04.756Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_http.py", line 312, in read_headers
2024-03-04T02:14:04.756Z [container-agent]     line = recv_line(sock)
2024-03-04T02:14:04.756Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_socket.py", line 131, in recv_line
2024-03-04T02:14:04.756Z [container-agent]     c = recv(sock, 1)
2024-03-04T02:14:04.756Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_socket.py", line 122, in recv
2024-03-04T02:14:04.756Z [container-agent]     raise WebSocketConnectionClosedException(
2024-03-04T02:14:04.756Z [container-agent] websocket._exceptions.WebSocketConnectionClosedException: Connection to remote host was lost.
2024-03-04T02:14:04.756Z [container-agent] 
2024-03-04T02:14:04.756Z [container-agent] During handling of the above exception, another exception occurred:
2024-03-04T02:14:04.756Z [container-agent] 
2024-03-04T02:14:04.756Z [container-agent] Traceback (most recent call last):
2024-03-04T02:14:04.756Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 1348, in do_open
2024-03-04T02:14:04.756Z [container-agent]     h.request(req.get_method(), req.selector, req.data, headers,
2024-03-04T02:14:04.756Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1283, in request
2024-03-04T02:14:04.756Z [container-agent]     self._send_request(method, url, body, headers, encode_chunked)
2024-03-04T02:14:04.756Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1329, in _send_request
2024-03-04T02:14:04.756Z [container-agent]     self.endheaders(body, encode_chunked=encode_chunked)
2024-03-04T02:14:04.756Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1278, in endheaders
2024-03-04T02:14:04.756Z [container-agent]     self._send_output(message_body, encode_chunked=encode_chunked)
2024-03-04T02:14:04.756Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1038, in _send_output
2024-03-04T02:14:04.756Z [container-agent]     self.send(msg)
2024-03-04T02:14:04.756Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 976, in send
2024-03-04T02:14:04.756Z [container-agent]     self.connect()
2024-03-04T02:14:04.756Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 297, in connect
2024-03-04T02:14:04.756Z [container-agent]     self.sock.connect(self.socket_path)
2024-03-04T02:14:04.756Z [container-agent] ConnectionRefusedError: [Errno 111] Connection refused
2024-03-04T02:14:04.756Z [container-agent] 
2024-03-04T02:14:04.756Z [container-agent] During handling of the above exception, another exception occurred:
2024-03-04T02:14:04.756Z [container-agent] 
2024-03-04T02:14:04.756Z [container-agent] Traceback (most recent call last):
2024-03-04T02:14:04.756Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1690, in _request_raw
2024-03-04T02:14:04.756Z [container-agent]     response = self.opener.open(request, timeout=self.timeout)
2024-03-04T02:14:04.756Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 519, in open
2024-03-04T02:14:04.756Z [container-agent]     response = self._open(req, data)
2024-03-04T02:14:04.756Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 536, in _open
2024-03-04T02:14:04.756Z [container-agent]     result = self._call_chain(self.handle_open, protocol, protocol +
2024-03-04T02:14:04.756Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 496, in _call_chain
2024-03-04T02:14:04.756Z [container-agent]     result = func(*args)
2024-03-04T02:14:04.756Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 311, in http_open
2024-03-04T02:14:04.756Z [container-agent]     return self.do_open(_UnixSocketConnection, req,  # type:ignore
2024-03-04T02:14:04.756Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 1351, in do_open
2024-03-04T02:14:04.756Z [container-agent]     raise URLError(err)
2024-03-04T02:14:04.756Z [container-agent] urllib.error.URLError: <urlopen error [Errno 111] Connection refused>
2024-03-04T02:14:04.756Z [container-agent] 
2024-03-04T02:14:04.756Z [container-agent] The above exception was the direct cause of the following exception:
2024-03-04T02:14:04.756Z [container-agent] 
2024-03-04T02:14:04.756Z [container-agent] Traceback (most recent call last):
2024-03-04T02:14:04.756Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/./src/charm.py", line 786, in <module>
2024-03-04T02:14:04.756Z [container-agent]     main(MySQLOperatorCharm)
2024-03-04T02:14:04.756Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 436, in main
2024-03-04T02:14:04.756Z [container-agent]     _emit_charm_event(charm, dispatcher.event_name)
2024-03-04T02:14:04.756Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 144, in _emit_charm_event
2024-03-04T02:14:04.756Z [container-agent]     event_to_emit.emit(*args, **kwargs)
2024-03-04T02:14:04.756Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 351, in emit
2024-03-04T02:14:04.756Z [container-agent]     framework._emit(event)
2024-03-04T02:14:04.756Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 853, in _emit
2024-03-04T02:14:04.756Z [container-agent]     self._reemit(event_path)
2024-03-04T02:14:04.756Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 942, in _reemit
2024-03-04T02:14:04.756Z [container-agent]     custom_handler(event)
2024-03-04T02:14:04.756Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/rotate_mysql_logs.py", line 55, in _rotate_mysql_logs
2024-03-04T02:14:04.756Z [container-agent]     self.charm._mysql.flush_mysql_logs(MySQLTextLogs.ERROR)
2024-03-04T02:14:04.756Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-03-04T02:14:04.756Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-03-04T02:14:04.756Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 665, in _run_mysqlsh_script
2024-03-04T02:14:04.756Z [container-agent]     process = self.container.exec(cmd, timeout=timeout)
2024-03-04T02:14:04.756Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/model.py", line 2688, in exec
2024-03-04T02:14:04.756Z [container-agent]     return self._pebble.exec(
2024-03-04T02:14:04.756Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 2514, in exec
2024-03-04T02:14:04.756Z [container-agent]     change = self.wait_change(ChangeID(change_id))
2024-03-04T02:14:04.756Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1893, in wait_change
2024-03-04T02:14:04.756Z [container-agent]     return self._wait_change_using_wait(change_id, timeout)
2024-03-04T02:14:04.756Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1914, in _wait_change_using_wait
2024-03-04T02:14:04.756Z [container-agent]     return self._wait_change(change_id, this_timeout)
2024-03-04T02:14:04.756Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1928, in _wait_change
2024-03-04T02:14:04.756Z [container-agent]     resp = self._request('GET', f'/v1/changes/{change_id}/wait', query)
2024-03-04T02:14:04.756Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1655, in _request
2024-03-04T02:14:04.756Z [container-agent]     response = self._request_raw(method, path, query, headers, data)
2024-03-04T02:14:04.756Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1707, in _request_raw
2024-03-04T02:14:04.756Z [container-agent]     raise ConnectionError(e.reason) from e
2024-03-04T02:14:04.756Z [container-agent] ops.pebble.ConnectionError: [Errno 111] Connection refused
2024-03-04T02:14:04.948Z [container-agent] 2024-03-04 02:14:04 INFO juju.util.exec exec.go:209 run result: exit status 1
2024-03-04T02:14:06.692Z [container-agent] 2024-03-04 02:14:06 INFO juju-log Starting the log rotate manager
2024-03-04T02:14:06.706Z [container-agent] 2024-03-04 02:14:06 INFO juju-log Started log rotate manager process with PID 768673
2024-03-04T02:14:06.893Z [container-agent] 2024-03-04 02:14:06 INFO juju-log Setting up the logrotate configurations
2024-03-04T02:14:06.966Z [container-agent] 2024-03-04 02:14:06 INFO juju-log Adding pebble layer
2024-03-04T02:14:13.721Z [container-agent] 2024-03-04 02:14:13 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-03-04T02:14:13.747Z [container-agent] 2024-03-04 02:14:13 INFO juju-log Attempting reboot from complete outage.
2024-03-04T02:14:16.584Z [container-agent] 2024-03-04 02:14:16 INFO juju.worker.uniter.operation runhook.go:186 ran "mysql-pebble-ready" hook (via hook dispatching script: dispatch)
2024-03-04T02:14:34.882Z [container-agent] 2024-03-04 02:14:34 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T02:14:36.682Z [container-agent] 2024-03-04 02:14:36 WARNING juju-log No relation: certificates
2024-03-04T02:14:37.051Z [container-agent] 2024-03-04 02:14:37 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T02:20:11.289Z [container-agent] 2024-03-04 02:20:11 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T02:20:13.995Z [container-agent] 2024-03-04 02:20:13 WARNING juju-log No relation: certificates
2024-03-04T02:20:14.349Z [container-agent] 2024-03-04 02:20:14 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T02:25:52.262Z [container-agent] 2024-03-04 02:25:52 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T02:25:55.099Z [container-agent] 2024-03-04 02:25:55 WARNING juju-log No relation: certificates
2024-03-04T02:25:55.478Z [container-agent] 2024-03-04 02:25:55 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T02:30:15.105Z [container-agent] 2024-03-04 02:30:15 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T02:30:18.688Z [container-agent] 2024-03-04 02:30:18 WARNING juju-log No relation: certificates
2024-03-04T02:30:19.033Z [container-agent] 2024-03-04 02:30:19 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T02:35:27.427Z [container-agent] 2024-03-04 02:35:27 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T02:35:31.605Z [container-agent] 2024-03-04 02:35:31 WARNING juju-log No relation: certificates
2024-03-04T02:35:31.957Z [container-agent] 2024-03-04 02:35:31 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T02:41:08.790Z [container-agent] 2024-03-04 02:41:08 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T02:41:13.359Z [container-agent] 2024-03-04 02:41:13 WARNING juju-log No relation: certificates
2024-03-04T02:41:13.704Z [container-agent] 2024-03-04 02:41:13 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T02:46:54.919Z [container-agent] 2024-03-04 02:46:54 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T02:46:59.996Z [container-agent] 2024-03-04 02:46:59 WARNING juju-log No relation: certificates
2024-03-04T02:47:00.351Z [container-agent] 2024-03-04 02:47:00 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T02:52:50.827Z [container-agent] 2024-03-04 02:52:50 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T02:52:56.288Z [container-agent] 2024-03-04 02:52:56 WARNING juju-log No relation: certificates
2024-03-04T02:52:56.627Z [container-agent] 2024-03-04 02:52:56 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T02:58:40.280Z [container-agent] 2024-03-04 02:58:40 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T02:58:46.333Z [container-agent] 2024-03-04 02:58:46 WARNING juju-log No relation: certificates
2024-03-04T02:58:46.680Z [container-agent] 2024-03-04 02:58:46 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T03:03:11.503Z [container-agent] 2024-03-04 03:03:11 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T03:03:18.179Z [container-agent] 2024-03-04 03:03:18 WARNING juju-log No relation: certificates
2024-03-04T03:03:18.534Z [container-agent] 2024-03-04 03:03:18 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T03:08:58.575Z [container-agent] 2024-03-04 03:08:58 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T03:09:05.166Z [container-agent] 2024-03-04 03:09:05 WARNING juju-log No relation: certificates
2024-03-04T03:09:05.509Z [container-agent] 2024-03-04 03:09:05 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T03:14:14.154Z [container-agent] 2024-03-04 03:14:14 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T03:14:21.688Z [container-agent] 2024-03-04 03:14:21 WARNING juju-log No relation: certificates
2024-03-04T03:14:22.033Z [container-agent] 2024-03-04 03:14:22 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T03:19:20.595Z [container-agent] 2024-03-04 03:19:20 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T03:19:28.065Z [container-agent] 2024-03-04 03:19:28 WARNING juju-log No relation: certificates
2024-03-04T03:19:28.422Z [container-agent] 2024-03-04 03:19:28 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T03:23:26.642Z [container-agent] 2024-03-04 03:23:26 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T03:23:34.997Z [container-agent] 2024-03-04 03:23:34 WARNING juju-log No relation: certificates
2024-03-04T03:23:35.390Z [container-agent] 2024-03-04 03:23:35 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T03:28:35.575Z [container-agent] 2024-03-04 03:28:35 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T03:28:43.992Z [container-agent] 2024-03-04 03:28:43 WARNING juju-log No relation: certificates
2024-03-04T03:28:44.326Z [container-agent] 2024-03-04 03:28:44 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T03:34:26.706Z [container-agent] 2024-03-04 03:34:26 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-03-04T03:34:26.733Z [container-agent] 2024-03-04 03:34:26 INFO juju-log Attempting reboot from complete outage.
2024-03-04T03:34:35.116Z [container-agent] 2024-03-04 03:34:35 WARNING juju-log No relation: certificates
2024-03-04T03:34:35.481Z [container-agent] 2024-03-04 03:34:35 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T03:35:09.820Z [container-agent] 2024-03-04 03:35:09 ERROR juju-log Uncaught exception while in charm code:
2024-03-04T03:35:09.820Z [container-agent] Traceback (most recent call last):
2024-03-04T03:35:09.820Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/./src/charm.py", line 786, in <module>
2024-03-04T03:35:09.820Z [container-agent]     main(MySQLOperatorCharm)
2024-03-04T03:35:09.820Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 436, in main
2024-03-04T03:35:09.820Z [container-agent]     _emit_charm_event(charm, dispatcher.event_name)
2024-03-04T03:35:09.820Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 144, in _emit_charm_event
2024-03-04T03:35:09.820Z [container-agent]     event_to_emit.emit(*args, **kwargs)
2024-03-04T03:35:09.820Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 351, in emit
2024-03-04T03:35:09.820Z [container-agent]     framework._emit(event)
2024-03-04T03:35:09.820Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 853, in _emit
2024-03-04T03:35:09.820Z [container-agent]     self._reemit(event_path)
2024-03-04T03:35:09.820Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 942, in _reemit
2024-03-04T03:35:09.820Z [container-agent]     custom_handler(event)
2024-03-04T03:35:09.820Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/rotate_mysql_logs.py", line 56, in _rotate_mysql_logs
2024-03-04T03:35:09.820Z [container-agent]     self.charm._mysql.flush_mysql_logs(MySQLTextLogs.GENERAL)
2024-03-04T03:35:09.820Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-03-04T03:35:09.820Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-03-04T03:35:09.820Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 666, in _run_mysqlsh_script
2024-03-04T03:35:09.820Z [container-agent]     stdout, _ = process.wait_output()
2024-03-04T03:35:09.820Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1436, in wait_output
2024-03-04T03:35:09.820Z [container-agent]     exit_code: int = self._wait()
2024-03-04T03:35:09.820Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1375, in _wait
2024-03-04T03:35:09.820Z [container-agent]     change = self._client.wait_change(self._change_id, timeout=timeout)
2024-03-04T03:35:09.820Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1893, in wait_change
2024-03-04T03:35:09.820Z [container-agent]     return self._wait_change_using_wait(change_id, timeout)
2024-03-04T03:35:09.820Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1914, in _wait_change_using_wait
2024-03-04T03:35:09.820Z [container-agent]     return self._wait_change(change_id, this_timeout)
2024-03-04T03:35:09.820Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1928, in _wait_change
2024-03-04T03:35:09.820Z [container-agent]     resp = self._request('GET', f'/v1/changes/{change_id}/wait', query)
2024-03-04T03:35:09.820Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1655, in _request
2024-03-04T03:35:09.820Z [container-agent]     response = self._request_raw(method, path, query, headers, data)
2024-03-04T03:35:09.820Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1690, in _request_raw
2024-03-04T03:35:09.820Z [container-agent]     response = self.opener.open(request, timeout=self.timeout)
2024-03-04T03:35:09.820Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 519, in open
2024-03-04T03:35:09.820Z [container-agent]     response = self._open(req, data)
2024-03-04T03:35:09.820Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 536, in _open
2024-03-04T03:35:09.820Z [container-agent]     result = self._call_chain(self.handle_open, protocol, protocol +
2024-03-04T03:35:09.820Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 496, in _call_chain
2024-03-04T03:35:09.820Z [container-agent]     result = func(*args)
2024-03-04T03:35:09.820Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 311, in http_open
2024-03-04T03:35:09.820Z [container-agent]     return self.do_open(_UnixSocketConnection, req,  # type:ignore
2024-03-04T03:35:09.820Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 1352, in do_open
2024-03-04T03:35:09.820Z [container-agent]     r = h.getresponse()
2024-03-04T03:35:09.820Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1375, in getresponse
2024-03-04T03:35:09.820Z [container-agent]     response.begin()
2024-03-04T03:35:09.820Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 318, in begin
2024-03-04T03:35:09.820Z [container-agent]     version, status, reason = self._read_status()
2024-03-04T03:35:09.820Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 287, in _read_status
2024-03-04T03:35:09.820Z [container-agent]     raise RemoteDisconnected("Remote end closed connection without"
2024-03-04T03:35:09.820Z [container-agent] http.client.RemoteDisconnected: Remote end closed connection without response
2024-03-04T03:35:10.045Z [container-agent] 2024-03-04 03:35:10 INFO juju.util.exec exec.go:209 run result: exit status 1
2024-03-04T03:35:14.776Z [container-agent] 2024-03-04 03:35:14 INFO juju-log Setting up the logrotate configurations
2024-03-04T03:35:14.864Z [container-agent] 2024-03-04 03:35:14 INFO juju-log Adding pebble layer
2024-03-04T03:35:21.614Z [container-agent] 2024-03-04 03:35:21 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-03-04T03:35:21.639Z [container-agent] 2024-03-04 03:35:21 INFO juju-log Attempting reboot from complete outage.
2024-03-04T03:35:24.679Z [container-agent] 2024-03-04 03:35:24 INFO juju.worker.uniter.operation runhook.go:186 ran "mysql-pebble-ready" hook (via hook dispatching script: dispatch)
2024-03-04T03:39:17.862Z [container-agent] 2024-03-04 03:39:17 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T03:39:19.872Z [container-agent] 2024-03-04 03:39:19 WARNING juju-log No relation: certificates
2024-03-04T03:39:20.232Z [container-agent] 2024-03-04 03:39:20 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T03:44:02.058Z [container-agent] 2024-03-04 03:44:02 INFO juju-log Starting the log rotate manager
2024-03-04T03:44:02.076Z [container-agent] 2024-03-04 03:44:02 INFO juju-log Started log rotate manager process with PID 776997
2024-03-04T03:44:02.435Z [container-agent] 2024-03-04 03:44:02 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T03:44:04.518Z [container-agent] 2024-03-04 03:44:04 WARNING juju-log No relation: certificates
2024-03-04T03:44:04.888Z [container-agent] 2024-03-04 03:44:04 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T03:48:48.690Z [container-agent] 2024-03-04 03:48:48 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T03:48:51.150Z [container-agent] 2024-03-04 03:48:51 WARNING juju-log No relation: certificates
2024-03-04T03:48:51.490Z [container-agent] 2024-03-04 03:48:51 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T03:54:06.300Z [container-agent] 2024-03-04 03:54:06 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T03:54:09.266Z [container-agent] 2024-03-04 03:54:09 WARNING juju-log No relation: certificates
2024-03-04T03:54:09.590Z [container-agent] 2024-03-04 03:54:09 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T03:58:52.656Z [container-agent] 2024-03-04 03:58:52 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T03:58:55.694Z [container-agent] 2024-03-04 03:58:55 WARNING juju-log No relation: certificates
2024-03-04T03:58:56.016Z [container-agent] 2024-03-04 03:58:56 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T04:04:15.146Z [container-agent] 2024-03-04 04:04:15 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T04:04:19.295Z [container-agent] 2024-03-04 04:04:19 WARNING juju-log No relation: certificates
2024-03-04T04:04:19.672Z [container-agent] 2024-03-04 04:04:19 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T04:08:52.685Z [container-agent] 2024-03-04 04:08:52 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T04:08:56.747Z [container-agent] 2024-03-04 04:08:56 WARNING juju-log No relation: certificates
2024-03-04T04:08:57.091Z [container-agent] 2024-03-04 04:08:57 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T04:13:39.754Z [container-agent] 2024-03-04 04:13:39 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T04:13:44.547Z [container-agent] 2024-03-04 04:13:44 WARNING juju-log No relation: certificates
2024-03-04T04:13:44.931Z [container-agent] 2024-03-04 04:13:44 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T04:18:34.771Z [container-agent] 2024-03-04 04:18:34 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T04:18:39.910Z [container-agent] 2024-03-04 04:18:39 WARNING juju-log No relation: certificates
2024-03-04T04:18:40.271Z [container-agent] 2024-03-04 04:18:40 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T04:24:10.984Z [container-agent] 2024-03-04 04:24:10 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T04:24:16.635Z [container-agent] 2024-03-04 04:24:16 WARNING juju-log No relation: certificates
2024-03-04T04:24:17.009Z [container-agent] 2024-03-04 04:24:17 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T04:28:15.106Z [container-agent] 2024-03-04 04:28:15 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T04:28:20.825Z [container-agent] 2024-03-04 04:28:20 WARNING juju-log No relation: certificates
2024-03-04T04:28:21.167Z [container-agent] 2024-03-04 04:28:21 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T04:32:30.513Z [container-agent] 2024-03-04 04:32:30 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T04:32:36.931Z [container-agent] 2024-03-04 04:32:36 WARNING juju-log No relation: certificates
2024-03-04T04:32:37.271Z [container-agent] 2024-03-04 04:32:37 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T04:36:36.424Z [container-agent] 2024-03-04 04:36:36 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T04:36:43.002Z [container-agent] 2024-03-04 04:36:43 WARNING juju-log No relation: certificates
2024-03-04T04:36:43.368Z [container-agent] 2024-03-04 04:36:43 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T04:42:12.813Z [container-agent] 2024-03-04 04:42:12 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T04:42:20.448Z [container-agent] 2024-03-04 04:42:20 WARNING juju-log No relation: certificates
2024-03-04T04:42:20.822Z [container-agent] 2024-03-04 04:42:20 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T04:48:14.510Z [container-agent] 2024-03-04 04:48:14 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T04:48:22.550Z [container-agent] 2024-03-04 04:48:22 WARNING juju-log No relation: certificates
2024-03-04T04:48:22.899Z [container-agent] 2024-03-04 04:48:22 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T04:52:07.893Z [container-agent] 2024-03-04 04:52:07 ERROR juju-log Failed to flush ERROR LOGS logs.
2024-03-04T04:52:07.893Z [container-agent] Traceback (most recent call last):
2024-03-04T04:52:07.893Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 666, in _run_mysqlsh_script
2024-03-04T04:52:07.893Z [container-agent]     stdout, _ = process.wait_output()
2024-03-04T04:52:07.893Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1441, in wait_output
2024-03-04T04:52:07.893Z [container-agent]     raise ExecError[AnyStr](self._command, exit_code, out_value, err_value)
2024-03-04T04:52:07.893Z [container-agent] ops.pebble.ExecError: non-zero exit code 1 executing ['/usr/bin/mysqlsh', '--no-wizard', '--python', '--verbose=1', '-f', '/tmp/script.py', ';', 'rm', '/tmp/script.py'], stdout='', stderr='Cannot set LC_ALL to locale en_US.UTF-8: No such file or directory\nverbose: 2024-03-04T04:52:06Z: Loading startup files...\nverbose: 2024-03-04T04:52:06Z: Loading plugins...\nverbose: 2024-03-04T04:52:06Z: Connecting to MySQL at: serverconfig@horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local\nTraceback (most recent call last):\n  File "<string>", line 1, in <module>\nmysqlsh.DBError: MySQL Error (2003): Shell.connect: Can\'t connect to MySQL server on \'horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local:3306\' (111)\n'
2024-03-04T04:52:07.893Z [container-agent] 
2024-03-04T04:52:07.893Z [container-agent] During handling of the above exception, another exception occurred:
2024-03-04T04:52:07.893Z [container-agent] 
2024-03-04T04:52:07.893Z [container-agent] Traceback (most recent call last):
2024-03-04T04:52:07.893Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-03-04T04:52:07.893Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-03-04T04:52:07.893Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 669, in _run_mysqlsh_script
2024-03-04T04:52:07.893Z [container-agent]     raise MySQLClientError(e.stderr)
2024-03-04T04:52:07.893Z [container-agent] charms.mysql.v0.mysql.MySQLClientError: Cannot set LC_ALL to locale en_US.UTF-8: No such file or directory
2024-03-04T04:52:07.893Z [container-agent] verbose: 2024-03-04T04:52:06Z: Loading startup files...
2024-03-04T04:52:07.893Z [container-agent] verbose: 2024-03-04T04:52:06Z: Loading plugins...
2024-03-04T04:52:07.893Z [container-agent] verbose: 2024-03-04T04:52:06Z: Connecting to MySQL at: serverconfig@horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local
2024-03-04T04:52:07.893Z [container-agent] Traceback (most recent call last):
2024-03-04T04:52:07.893Z [container-agent]   File "<string>", line 1, in <module>
2024-03-04T04:52:07.893Z [container-agent] mysqlsh.DBError: MySQL Error (2003): Shell.connect: Can't connect to MySQL server on 'horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local:3306' (111)
2024-03-04T04:52:07.893Z [container-agent] 
2024-03-04T04:52:29.949Z [container-agent] 2024-03-04 04:52:29 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-03-04T04:52:29.980Z [container-agent] 2024-03-04 04:52:29 INFO juju-log Attempting reboot from complete outage.
2024-03-04T04:52:37.618Z [container-agent] 2024-03-04 04:52:37 WARNING juju-log No relation: certificates
2024-03-04T04:52:37.964Z [container-agent] 2024-03-04 04:52:37 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T04:56:54.254Z [container-agent] 2024-03-04 04:56:54 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T04:57:02.934Z [container-agent] 2024-03-04 04:57:02 WARNING juju-log No relation: certificates
2024-03-04T04:57:03.322Z [container-agent] 2024-03-04 04:57:03 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T04:57:04.782Z [container-agent] 2024-03-04 04:57:04 ERROR juju-log Uncaught exception while in charm code:
2024-03-04T04:57:04.782Z [container-agent] Traceback (most recent call last):
2024-03-04T04:57:04.782Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/./src/charm.py", line 786, in <module>
2024-03-04T04:57:04.782Z [container-agent]     main(MySQLOperatorCharm)
2024-03-04T04:57:04.782Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 436, in main
2024-03-04T04:57:04.782Z [container-agent]     _emit_charm_event(charm, dispatcher.event_name)
2024-03-04T04:57:04.782Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 144, in _emit_charm_event
2024-03-04T04:57:04.782Z [container-agent]     event_to_emit.emit(*args, **kwargs)
2024-03-04T04:57:04.782Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 351, in emit
2024-03-04T04:57:04.782Z [container-agent]     framework._emit(event)
2024-03-04T04:57:04.782Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 853, in _emit
2024-03-04T04:57:04.782Z [container-agent]     self._reemit(event_path)
2024-03-04T04:57:04.782Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 942, in _reemit
2024-03-04T04:57:04.782Z [container-agent]     custom_handler(event)
2024-03-04T04:57:04.782Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/rotate_mysql_logs.py", line 50, in _rotate_mysql_logs
2024-03-04T04:57:04.782Z [container-agent]     self.charm._mysql._execute_commands(["logrotate", "-f", LOG_ROTATE_CONFIG_FILE])
2024-03-04T04:57:04.782Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 622, in _execute_commands
2024-03-04T04:57:04.782Z [container-agent]     process = self.container.exec(
2024-03-04T04:57:04.782Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/model.py", line 2688, in exec
2024-03-04T04:57:04.782Z [container-agent]     return self._pebble.exec(
2024-03-04T04:57:04.782Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 2501, in exec
2024-03-04T04:57:04.782Z [container-agent]     resp = self._request('POST', '/v1/exec', body=body)
2024-03-04T04:57:04.782Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1655, in _request
2024-03-04T04:57:04.782Z [container-agent]     response = self._request_raw(method, path, query, headers, data)
2024-03-04T04:57:04.782Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1690, in _request_raw
2024-03-04T04:57:04.782Z [container-agent]     response = self.opener.open(request, timeout=self.timeout)
2024-03-04T04:57:04.782Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 519, in open
2024-03-04T04:57:04.782Z [container-agent]     response = self._open(req, data)
2024-03-04T04:57:04.782Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 536, in _open
2024-03-04T04:57:04.782Z [container-agent]     result = self._call_chain(self.handle_open, protocol, protocol +
2024-03-04T04:57:04.782Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 496, in _call_chain
2024-03-04T04:57:04.782Z [container-agent]     result = func(*args)
2024-03-04T04:57:04.782Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 311, in http_open
2024-03-04T04:57:04.782Z [container-agent]     return self.do_open(_UnixSocketConnection, req,  # type:ignore
2024-03-04T04:57:04.782Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 1352, in do_open
2024-03-04T04:57:04.782Z [container-agent]     r = h.getresponse()
2024-03-04T04:57:04.782Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1375, in getresponse
2024-03-04T04:57:04.782Z [container-agent]     response.begin()
2024-03-04T04:57:04.782Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 318, in begin
2024-03-04T04:57:04.782Z [container-agent]     version, status, reason = self._read_status()
2024-03-04T04:57:04.782Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 287, in _read_status
2024-03-04T04:57:04.782Z [container-agent]     raise RemoteDisconnected("Remote end closed connection without"
2024-03-04T04:57:04.782Z [container-agent] http.client.RemoteDisconnected: Remote end closed connection without response
2024-03-04T04:57:04.908Z [container-agent] 2024-03-04 04:57:04 INFO juju.util.exec exec.go:209 run result: exit status 1
2024-03-04T04:57:07.247Z [container-agent] 2024-03-04 04:57:07 INFO juju-log Starting the log rotate manager
2024-03-04T04:57:07.260Z [container-agent] 2024-03-04 04:57:07 INFO juju-log Started log rotate manager process with PID 784368
2024-03-04T04:57:07.435Z [container-agent] 2024-03-04 04:57:07 INFO juju-log Setting up the logrotate configurations
2024-03-04T04:57:07.497Z [container-agent] 2024-03-04 04:57:07 INFO juju-log Adding pebble layer
2024-03-04T04:57:14.236Z [container-agent] 2024-03-04 04:57:14 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-03-04T04:57:14.266Z [container-agent] 2024-03-04 04:57:14 INFO juju-log Attempting reboot from complete outage.
2024-03-04T04:57:17.375Z [container-agent] 2024-03-04 04:57:17 INFO juju.worker.uniter.operation runhook.go:186 ran "mysql-pebble-ready" hook (via hook dispatching script: dispatch)
2024-03-04T05:02:38.465Z [container-agent] 2024-03-04 05:02:38 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T05:02:40.799Z [container-agent] 2024-03-04 05:02:40 WARNING juju-log No relation: certificates
2024-03-04T05:02:41.149Z [container-agent] 2024-03-04 05:02:41 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T05:08:10.313Z [container-agent] 2024-03-04 05:08:10 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T05:08:13.073Z [container-agent] 2024-03-04 05:08:13 WARNING juju-log No relation: certificates
2024-03-04T05:08:13.448Z [container-agent] 2024-03-04 05:08:13 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T05:12:18.250Z [container-agent] 2024-03-04 05:12:18 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T05:12:21.551Z [container-agent] 2024-03-04 05:12:21 WARNING juju-log No relation: certificates
2024-03-04T05:12:21.891Z [container-agent] 2024-03-04 05:12:21 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T05:16:27.121Z [container-agent] 2024-03-04 05:16:27 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T05:16:30.885Z [container-agent] 2024-03-04 05:16:30 WARNING juju-log No relation: certificates
2024-03-04T05:16:31.244Z [container-agent] 2024-03-04 05:16:31 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T05:20:52.706Z [container-agent] 2024-03-04 05:20:52 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T05:20:56.399Z [container-agent] 2024-03-04 05:20:56 WARNING juju-log No relation: certificates
2024-03-04T05:20:56.734Z [container-agent] 2024-03-04 05:20:56 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T05:25:58.467Z [container-agent] 2024-03-04 05:25:58 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T05:26:02.853Z [container-agent] 2024-03-04 05:26:02 WARNING juju-log No relation: certificates
2024-03-04T05:26:03.200Z [container-agent] 2024-03-04 05:26:03 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T05:31:55.129Z [container-agent] 2024-03-04 05:31:55 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T05:31:59.916Z [container-agent] 2024-03-04 05:31:59 WARNING juju-log No relation: certificates
2024-03-04T05:32:00.285Z [container-agent] 2024-03-04 05:32:00 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T05:37:09.841Z [container-agent] 2024-03-04 05:37:09 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T05:37:15.379Z [container-agent] 2024-03-04 05:37:15 WARNING juju-log No relation: certificates
2024-03-04T05:37:15.760Z [container-agent] 2024-03-04 05:37:15 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T05:41:35.005Z [container-agent] 2024-03-04 05:41:35 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T05:41:40.924Z [container-agent] 2024-03-04 05:41:40 WARNING juju-log No relation: certificates
2024-03-04T05:41:41.269Z [container-agent] 2024-03-04 05:41:41 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T05:46:52.981Z [container-agent] 2024-03-04 05:46:52 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T05:46:59.579Z [container-agent] 2024-03-04 05:46:59 WARNING juju-log No relation: certificates
2024-03-04T05:46:59.948Z [container-agent] 2024-03-04 05:46:59 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T05:51:51.947Z [container-agent] 2024-03-04 05:51:51 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T05:51:57.776Z [container-agent] 2024-03-04 05:51:57 WARNING juju-log No relation: certificates
2024-03-04T05:51:58.156Z [container-agent] 2024-03-04 05:51:58 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T05:56:50.083Z [container-agent] 2024-03-04 05:56:50 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T05:56:57.245Z [container-agent] 2024-03-04 05:56:57 WARNING juju-log No relation: certificates
2024-03-04T05:56:57.598Z [container-agent] 2024-03-04 05:56:57 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T06:02:14.360Z [container-agent] 2024-03-04 06:02:14 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T06:02:22.185Z [container-agent] 2024-03-04 06:02:22 WARNING juju-log No relation: certificates
2024-03-04T06:02:22.552Z [container-agent] 2024-03-04 06:02:22 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T06:07:15.326Z [container-agent] 2024-03-04 06:07:15 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T06:07:23.414Z [container-agent] 2024-03-04 06:07:23 WARNING juju-log No relation: certificates
2024-03-04T06:07:23.764Z [container-agent] 2024-03-04 06:07:23 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T06:11:16.704Z [container-agent] 2024-03-04 06:11:16 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-03-04T06:11:16.730Z [container-agent] 2024-03-04 06:11:16 INFO juju-log Attempting reboot from complete outage.
2024-03-04T06:11:24.766Z [container-agent] 2024-03-04 06:11:24 WARNING juju-log No relation: certificates
2024-03-04T06:11:25.111Z [container-agent] 2024-03-04 06:11:25 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T06:14:09.793Z [container-agent] 2024-03-04 06:14:09 ERROR juju-log Uncaught exception while in charm code:
2024-03-04T06:14:09.793Z [container-agent] Traceback (most recent call last):
2024-03-04T06:14:09.793Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/./src/charm.py", line 786, in <module>
2024-03-04T06:14:09.793Z [container-agent]     main(MySQLOperatorCharm)
2024-03-04T06:14:09.793Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 436, in main
2024-03-04T06:14:09.793Z [container-agent]     _emit_charm_event(charm, dispatcher.event_name)
2024-03-04T06:14:09.793Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 144, in _emit_charm_event
2024-03-04T06:14:09.793Z [container-agent]     event_to_emit.emit(*args, **kwargs)
2024-03-04T06:14:09.793Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 351, in emit
2024-03-04T06:14:09.793Z [container-agent]     framework._emit(event)
2024-03-04T06:14:09.793Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 853, in _emit
2024-03-04T06:14:09.793Z [container-agent]     self._reemit(event_path)
2024-03-04T06:14:09.793Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 942, in _reemit
2024-03-04T06:14:09.793Z [container-agent]     custom_handler(event)
2024-03-04T06:14:09.793Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/rotate_mysql_logs.py", line 56, in _rotate_mysql_logs
2024-03-04T06:14:09.793Z [container-agent]     self.charm._mysql.flush_mysql_logs(MySQLTextLogs.GENERAL)
2024-03-04T06:14:09.793Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-03-04T06:14:09.793Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-03-04T06:14:09.793Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 666, in _run_mysqlsh_script
2024-03-04T06:14:09.793Z [container-agent]     stdout, _ = process.wait_output()
2024-03-04T06:14:09.793Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1436, in wait_output
2024-03-04T06:14:09.793Z [container-agent]     exit_code: int = self._wait()
2024-03-04T06:14:09.793Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1375, in _wait
2024-03-04T06:14:09.793Z [container-agent]     change = self._client.wait_change(self._change_id, timeout=timeout)
2024-03-04T06:14:09.793Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1893, in wait_change
2024-03-04T06:14:09.793Z [container-agent]     return self._wait_change_using_wait(change_id, timeout)
2024-03-04T06:14:09.793Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1914, in _wait_change_using_wait
2024-03-04T06:14:09.793Z [container-agent]     return self._wait_change(change_id, this_timeout)
2024-03-04T06:14:09.793Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1928, in _wait_change
2024-03-04T06:14:09.793Z [container-agent]     resp = self._request('GET', f'/v1/changes/{change_id}/wait', query)
2024-03-04T06:14:09.793Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1655, in _request
2024-03-04T06:14:09.793Z [container-agent]     response = self._request_raw(method, path, query, headers, data)
2024-03-04T06:14:09.793Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1690, in _request_raw
2024-03-04T06:14:09.793Z [container-agent]     response = self.opener.open(request, timeout=self.timeout)
2024-03-04T06:14:09.793Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 519, in open
2024-03-04T06:14:09.793Z [container-agent]     response = self._open(req, data)
2024-03-04T06:14:09.793Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 536, in _open
2024-03-04T06:14:09.793Z [container-agent]     result = self._call_chain(self.handle_open, protocol, protocol +
2024-03-04T06:14:09.793Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 496, in _call_chain
2024-03-04T06:14:09.793Z [container-agent]     result = func(*args)
2024-03-04T06:14:09.793Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 311, in http_open
2024-03-04T06:14:09.793Z [container-agent]     return self.do_open(_UnixSocketConnection, req,  # type:ignore
2024-03-04T06:14:09.793Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 1352, in do_open
2024-03-04T06:14:09.793Z [container-agent]     r = h.getresponse()
2024-03-04T06:14:09.793Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1375, in getresponse
2024-03-04T06:14:09.793Z [container-agent]     response.begin()
2024-03-04T06:14:09.793Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 318, in begin
2024-03-04T06:14:09.793Z [container-agent]     version, status, reason = self._read_status()
2024-03-04T06:14:09.793Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 287, in _read_status
2024-03-04T06:14:09.793Z [container-agent]     raise RemoteDisconnected("Remote end closed connection without"
2024-03-04T06:14:09.793Z [container-agent] http.client.RemoteDisconnected: Remote end closed connection without response
2024-03-04T06:14:09.942Z [container-agent] 2024-03-04 06:14:09 INFO juju.util.exec exec.go:209 run result: exit status 1
2024-03-04T06:14:11.571Z [container-agent] 2024-03-04 06:14:11 INFO juju-log Setting up the logrotate configurations
2024-03-04T06:14:11.635Z [container-agent] 2024-03-04 06:14:11 INFO juju-log Adding pebble layer
2024-03-04T06:14:18.422Z [container-agent] 2024-03-04 06:14:18 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-03-04T06:14:18.450Z [container-agent] 2024-03-04 06:14:18 INFO juju-log Attempting reboot from complete outage.
2024-03-04T06:14:21.512Z [container-agent] 2024-03-04 06:14:21 INFO juju.worker.uniter.operation runhook.go:186 ran "mysql-pebble-ready" hook (via hook dispatching script: dispatch)
2024-03-04T06:15:51.612Z [container-agent] 2024-03-04 06:15:51 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T06:15:53.469Z [container-agent] 2024-03-04 06:15:53 WARNING juju-log No relation: certificates
2024-03-04T06:15:53.816Z [container-agent] 2024-03-04 06:15:53 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T06:21:19.651Z [container-agent] 2024-03-04 06:21:19 INFO juju-log Starting the log rotate manager
2024-03-04T06:21:19.666Z [container-agent] 2024-03-04 06:21:19 INFO juju-log Started log rotate manager process with PID 792250
2024-03-04T06:21:19.960Z [container-agent] 2024-03-04 06:21:19 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T06:21:21.862Z [container-agent] 2024-03-04 06:21:21 WARNING juju-log No relation: certificates
2024-03-04T06:21:22.250Z [container-agent] 2024-03-04 06:21:22 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T06:26:40.989Z [container-agent] 2024-03-04 06:26:40 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T06:26:43.277Z [container-agent] 2024-03-04 06:26:43 WARNING juju-log No relation: certificates
2024-03-04T06:26:43.657Z [container-agent] 2024-03-04 06:26:43 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T06:32:23.171Z [container-agent] 2024-03-04 06:32:23 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T06:32:25.960Z [container-agent] 2024-03-04 06:32:25 WARNING juju-log No relation: certificates
2024-03-04T06:32:26.329Z [container-agent] 2024-03-04 06:32:26 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T06:37:41.941Z [container-agent] 2024-03-04 06:37:41 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T06:37:45.496Z [container-agent] 2024-03-04 06:37:45 WARNING juju-log No relation: certificates
2024-03-04T06:37:45.892Z [container-agent] 2024-03-04 06:37:45 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T06:42:07.619Z [container-agent] 2024-03-04 06:42:07 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T06:42:11.586Z [container-agent] 2024-03-04 06:42:11 WARNING juju-log No relation: certificates
2024-03-04T06:42:11.926Z [container-agent] 2024-03-04 06:42:11 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T06:46:26.744Z [container-agent] 2024-03-04 06:46:26 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T06:46:31.087Z [container-agent] 2024-03-04 06:46:31 WARNING juju-log No relation: certificates
2024-03-04T06:46:31.439Z [container-agent] 2024-03-04 06:46:31 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T06:52:09.615Z [container-agent] 2024-03-04 06:52:09 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T06:52:14.569Z [container-agent] 2024-03-04 06:52:14 WARNING juju-log No relation: certificates
2024-03-04T06:52:14.912Z [container-agent] 2024-03-04 06:52:14 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T06:57:11.101Z [container-agent] 2024-03-04 06:57:11 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T06:57:16.487Z [container-agent] 2024-03-04 06:57:16 WARNING juju-log No relation: certificates
2024-03-04T06:57:16.832Z [container-agent] 2024-03-04 06:57:16 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T07:01:25.105Z [container-agent] 2024-03-04 07:01:25 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T07:01:30.970Z [container-agent] 2024-03-04 07:01:30 WARNING juju-log No relation: certificates
2024-03-04T07:01:31.344Z [container-agent] 2024-03-04 07:01:31 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T07:06:52.705Z [container-agent] 2024-03-04 07:06:52 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T07:06:58.194Z [container-agent] 2024-03-04 07:06:58 WARNING juju-log No relation: certificates
2024-03-04T07:06:58.561Z [container-agent] 2024-03-04 07:06:58 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T07:12:50.550Z [container-agent] 2024-03-04 07:12:50 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T07:12:57.311Z [container-agent] 2024-03-04 07:12:57 WARNING juju-log No relation: certificates
2024-03-04T07:12:57.671Z [container-agent] 2024-03-04 07:12:57 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T07:17:45.232Z [container-agent] 2024-03-04 07:17:45 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T07:17:52.458Z [container-agent] 2024-03-04 07:17:52 WARNING juju-log No relation: certificates
2024-03-04T07:17:52.803Z [container-agent] 2024-03-04 07:17:52 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T07:23:38.309Z [container-agent] 2024-03-04 07:23:38 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T07:23:46.187Z [container-agent] 2024-03-04 07:23:46 WARNING juju-log No relation: certificates
2024-03-04T07:23:46.527Z [container-agent] 2024-03-04 07:23:46 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T07:28:34.397Z [container-agent] 2024-03-04 07:28:34 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T07:28:42.540Z [container-agent] 2024-03-04 07:28:42 WARNING juju-log No relation: certificates
2024-03-04T07:28:42.886Z [container-agent] 2024-03-04 07:28:42 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T07:34:23.624Z [container-agent] 2024-03-04 07:34:23 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-03-04T07:34:23.652Z [container-agent] 2024-03-04 07:34:23 INFO juju-log Attempting reboot from complete outage.
2024-03-04T07:34:31.586Z [container-agent] 2024-03-04 07:34:31 WARNING juju-log No relation: certificates
2024-03-04T07:34:31.947Z [container-agent] 2024-03-04 07:34:31 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T07:37:09.801Z [container-agent] 2024-03-04 07:37:09 ERROR juju-log Uncaught exception while in charm code:
2024-03-04T07:37:09.801Z [container-agent] Traceback (most recent call last):
2024-03-04T07:37:09.801Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/./src/charm.py", line 786, in <module>
2024-03-04T07:37:09.801Z [container-agent]     main(MySQLOperatorCharm)
2024-03-04T07:37:09.801Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 436, in main
2024-03-04T07:37:09.801Z [container-agent]     _emit_charm_event(charm, dispatcher.event_name)
2024-03-04T07:37:09.801Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 144, in _emit_charm_event
2024-03-04T07:37:09.801Z [container-agent]     event_to_emit.emit(*args, **kwargs)
2024-03-04T07:37:09.801Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 351, in emit
2024-03-04T07:37:09.801Z [container-agent]     framework._emit(event)
2024-03-04T07:37:09.801Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 853, in _emit
2024-03-04T07:37:09.801Z [container-agent]     self._reemit(event_path)
2024-03-04T07:37:09.801Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 942, in _reemit
2024-03-04T07:37:09.801Z [container-agent]     custom_handler(event)
2024-03-04T07:37:09.801Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/rotate_mysql_logs.py", line 57, in _rotate_mysql_logs
2024-03-04T07:37:09.801Z [container-agent]     self.charm._mysql.flush_mysql_logs(MySQLTextLogs.SLOW)
2024-03-04T07:37:09.801Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-03-04T07:37:09.801Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-03-04T07:37:09.801Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 665, in _run_mysqlsh_script
2024-03-04T07:37:09.801Z [container-agent]     process = self.container.exec(cmd, timeout=timeout)
2024-03-04T07:37:09.801Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/model.py", line 2688, in exec
2024-03-04T07:37:09.801Z [container-agent]     return self._pebble.exec(
2024-03-04T07:37:09.801Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 2501, in exec
2024-03-04T07:37:09.801Z [container-agent]     resp = self._request('POST', '/v1/exec', body=body)
2024-03-04T07:37:09.801Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1655, in _request
2024-03-04T07:37:09.801Z [container-agent]     response = self._request_raw(method, path, query, headers, data)
2024-03-04T07:37:09.801Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1690, in _request_raw
2024-03-04T07:37:09.801Z [container-agent]     response = self.opener.open(request, timeout=self.timeout)
2024-03-04T07:37:09.801Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 519, in open
2024-03-04T07:37:09.801Z [container-agent]     response = self._open(req, data)
2024-03-04T07:37:09.801Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 536, in _open
2024-03-04T07:37:09.801Z [container-agent]     result = self._call_chain(self.handle_open, protocol, protocol +
2024-03-04T07:37:09.801Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 496, in _call_chain
2024-03-04T07:37:09.801Z [container-agent]     result = func(*args)
2024-03-04T07:37:09.801Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 311, in http_open
2024-03-04T07:37:09.801Z [container-agent]     return self.do_open(_UnixSocketConnection, req,  # type:ignore
2024-03-04T07:37:09.801Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 1352, in do_open
2024-03-04T07:37:09.801Z [container-agent]     r = h.getresponse()
2024-03-04T07:37:09.801Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1375, in getresponse
2024-03-04T07:37:09.801Z [container-agent]     response.begin()
2024-03-04T07:37:09.801Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 318, in begin
2024-03-04T07:37:09.801Z [container-agent]     version, status, reason = self._read_status()
2024-03-04T07:37:09.801Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 287, in _read_status
2024-03-04T07:37:09.801Z [container-agent]     raise RemoteDisconnected("Remote end closed connection without"
2024-03-04T07:37:09.801Z [container-agent] http.client.RemoteDisconnected: Remote end closed connection without response
2024-03-04T07:37:09.919Z [container-agent] 2024-03-04 07:37:09 INFO juju.util.exec exec.go:209 run result: exit status 1
2024-03-04T07:37:14.699Z [container-agent] 2024-03-04 07:37:14 INFO juju-log Setting up the logrotate configurations
2024-03-04T07:37:14.798Z [container-agent] 2024-03-04 07:37:14 INFO juju-log Adding pebble layer
2024-03-04T07:37:21.457Z [container-agent] 2024-03-04 07:37:21 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-03-04T07:37:21.493Z [container-agent] 2024-03-04 07:37:21 INFO juju-log Attempting reboot from complete outage.
2024-03-04T07:37:24.259Z [container-agent] 2024-03-04 07:37:24 INFO juju.worker.uniter.operation runhook.go:186 ran "mysql-pebble-ready" hook (via hook dispatching script: dispatch)
2024-03-04T07:40:13.808Z [container-agent] 2024-03-04 07:40:13 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T07:40:15.770Z [container-agent] 2024-03-04 07:40:15 WARNING juju-log No relation: certificates
2024-03-04T07:40:16.119Z [container-agent] 2024-03-04 07:40:16 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T07:45:13.143Z [container-agent] 2024-03-04 07:45:13 INFO juju-log Starting the log rotate manager
2024-03-04T07:45:13.156Z [container-agent] 2024-03-04 07:45:13 INFO juju-log Started log rotate manager process with PID 799919
2024-03-04T07:45:13.432Z [container-agent] 2024-03-04 07:45:13 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T07:45:15.445Z [container-agent] 2024-03-04 07:45:15 WARNING juju-log No relation: certificates
2024-03-04T07:45:15.840Z [container-agent] 2024-03-04 07:45:15 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T07:49:52.184Z [container-agent] 2024-03-04 07:49:52 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T07:49:54.557Z [container-agent] 2024-03-04 07:49:54 WARNING juju-log No relation: certificates
2024-03-04T07:49:54.895Z [container-agent] 2024-03-04 07:49:54 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T07:55:43.339Z [container-agent] 2024-03-04 07:55:43 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T07:55:46.280Z [container-agent] 2024-03-04 07:55:46 WARNING juju-log No relation: certificates
2024-03-04T07:55:46.655Z [container-agent] 2024-03-04 07:55:46 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T08:00:29.991Z [container-agent] 2024-03-04 08:00:29 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T08:00:33.515Z [container-agent] 2024-03-04 08:00:33 WARNING juju-log No relation: certificates
2024-03-04T08:00:33.887Z [container-agent] 2024-03-04 08:00:33 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T08:06:29.016Z [container-agent] 2024-03-04 08:06:29 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T08:06:33.230Z [container-agent] 2024-03-04 08:06:33 WARNING juju-log No relation: certificates
2024-03-04T08:06:33.581Z [container-agent] 2024-03-04 08:06:33 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T08:10:30.368Z [container-agent] 2024-03-04 08:10:30 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T08:10:34.652Z [container-agent] 2024-03-04 08:10:34 WARNING juju-log No relation: certificates
2024-03-04T08:10:35.003Z [container-agent] 2024-03-04 08:10:35 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T08:16:18.484Z [container-agent] 2024-03-04 08:16:18 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T08:16:23.496Z [container-agent] 2024-03-04 08:16:23 WARNING juju-log No relation: certificates
2024-03-04T08:16:23.855Z [container-agent] 2024-03-04 08:16:23 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T08:20:56.108Z [container-agent] 2024-03-04 08:20:56 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T08:21:01.430Z [container-agent] 2024-03-04 08:21:01 WARNING juju-log No relation: certificates
2024-03-04T08:21:01.820Z [container-agent] 2024-03-04 08:21:01 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T08:25:00.896Z [container-agent] 2024-03-04 08:25:00 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T08:25:06.592Z [container-agent] 2024-03-04 08:25:06 WARNING juju-log No relation: certificates
2024-03-04T08:25:06.973Z [container-agent] 2024-03-04 08:25:06 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T08:29:11.507Z [container-agent] 2024-03-04 08:29:11 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T08:29:18.132Z [container-agent] 2024-03-04 08:29:18 WARNING juju-log No relation: certificates
2024-03-04T08:29:18.482Z [container-agent] 2024-03-04 08:29:18 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T08:33:28.595Z [container-agent] 2024-03-04 08:33:28 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T08:33:34.793Z [container-agent] 2024-03-04 08:33:34 WARNING juju-log No relation: certificates
2024-03-04T08:33:35.144Z [container-agent] 2024-03-04 08:33:35 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T08:37:56.524Z [container-agent] 2024-03-04 08:37:56 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T08:38:03.317Z [container-agent] 2024-03-04 08:38:03 WARNING juju-log No relation: certificates
2024-03-04T08:38:03.663Z [container-agent] 2024-03-04 08:38:03 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T08:42:43.331Z [container-agent] 2024-03-04 08:42:43 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T08:42:50.623Z [container-agent] 2024-03-04 08:42:50 WARNING juju-log No relation: certificates
2024-03-04T08:42:50.996Z [container-agent] 2024-03-04 08:42:50 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T08:46:56.890Z [container-agent] 2024-03-04 08:46:56 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T08:47:04.512Z [container-agent] 2024-03-04 08:47:04 WARNING juju-log No relation: certificates
2024-03-04T08:47:04.868Z [container-agent] 2024-03-04 08:47:04 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T08:51:48.005Z [container-agent] 2024-03-04 08:51:48 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-03-04T08:51:48.037Z [container-agent] 2024-03-04 08:51:48 INFO juju-log Attempting reboot from complete outage.
2024-03-04T08:51:55.837Z [container-agent] 2024-03-04 08:51:55 WARNING juju-log No relation: certificates
2024-03-04T08:51:56.173Z [container-agent] 2024-03-04 08:51:56 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T08:53:04.798Z [container-agent] 2024-03-04 08:53:04 ERROR juju-log Uncaught exception while in charm code:
2024-03-04T08:53:04.798Z [container-agent] Traceback (most recent call last):
2024-03-04T08:53:04.798Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/./src/charm.py", line 786, in <module>
2024-03-04T08:53:04.798Z [container-agent]     main(MySQLOperatorCharm)
2024-03-04T08:53:04.798Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 436, in main
2024-03-04T08:53:04.798Z [container-agent]     _emit_charm_event(charm, dispatcher.event_name)
2024-03-04T08:53:04.798Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 144, in _emit_charm_event
2024-03-04T08:53:04.798Z [container-agent]     event_to_emit.emit(*args, **kwargs)
2024-03-04T08:53:04.798Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 351, in emit
2024-03-04T08:53:04.798Z [container-agent]     framework._emit(event)
2024-03-04T08:53:04.798Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 853, in _emit
2024-03-04T08:53:04.798Z [container-agent]     self._reemit(event_path)
2024-03-04T08:53:04.798Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 942, in _reemit
2024-03-04T08:53:04.798Z [container-agent]     custom_handler(event)
2024-03-04T08:53:04.798Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/rotate_mysql_logs.py", line 55, in _rotate_mysql_logs
2024-03-04T08:53:04.798Z [container-agent]     self.charm._mysql.flush_mysql_logs(MySQLTextLogs.ERROR)
2024-03-04T08:53:04.798Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-03-04T08:53:04.798Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-03-04T08:53:04.798Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 666, in _run_mysqlsh_script
2024-03-04T08:53:04.798Z [container-agent]     stdout, _ = process.wait_output()
2024-03-04T08:53:04.798Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1436, in wait_output
2024-03-04T08:53:04.798Z [container-agent]     exit_code: int = self._wait()
2024-03-04T08:53:04.798Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1375, in _wait
2024-03-04T08:53:04.798Z [container-agent]     change = self._client.wait_change(self._change_id, timeout=timeout)
2024-03-04T08:53:04.798Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1893, in wait_change
2024-03-04T08:53:04.798Z [container-agent]     return self._wait_change_using_wait(change_id, timeout)
2024-03-04T08:53:04.798Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1914, in _wait_change_using_wait
2024-03-04T08:53:04.798Z [container-agent]     return self._wait_change(change_id, this_timeout)
2024-03-04T08:53:04.798Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1928, in _wait_change
2024-03-04T08:53:04.798Z [container-agent]     resp = self._request('GET', f'/v1/changes/{change_id}/wait', query)
2024-03-04T08:53:04.798Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1655, in _request
2024-03-04T08:53:04.798Z [container-agent]     response = self._request_raw(method, path, query, headers, data)
2024-03-04T08:53:04.798Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1690, in _request_raw
2024-03-04T08:53:04.798Z [container-agent]     response = self.opener.open(request, timeout=self.timeout)
2024-03-04T08:53:04.798Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 519, in open
2024-03-04T08:53:04.798Z [container-agent]     response = self._open(req, data)
2024-03-04T08:53:04.798Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 536, in _open
2024-03-04T08:53:04.798Z [container-agent]     result = self._call_chain(self.handle_open, protocol, protocol +
2024-03-04T08:53:04.798Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 496, in _call_chain
2024-03-04T08:53:04.798Z [container-agent]     result = func(*args)
2024-03-04T08:53:04.798Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 311, in http_open
2024-03-04T08:53:04.798Z [container-agent]     return self.do_open(_UnixSocketConnection, req,  # type:ignore
2024-03-04T08:53:04.798Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 1352, in do_open
2024-03-04T08:53:04.798Z [container-agent]     r = h.getresponse()
2024-03-04T08:53:04.798Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1375, in getresponse
2024-03-04T08:53:04.798Z [container-agent]     response.begin()
2024-03-04T08:53:04.798Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 318, in begin
2024-03-04T08:53:04.798Z [container-agent]     version, status, reason = self._read_status()
2024-03-04T08:53:04.798Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 287, in _read_status
2024-03-04T08:53:04.798Z [container-agent]     raise RemoteDisconnected("Remote end closed connection without"
2024-03-04T08:53:04.798Z [container-agent] http.client.RemoteDisconnected: Remote end closed connection without response
2024-03-04T08:53:04.946Z [container-agent] 2024-03-04 08:53:04 INFO juju.util.exec exec.go:209 run result: exit status 1
2024-03-04T08:53:07.222Z [container-agent] 2024-03-04 08:53:07 INFO juju-log Setting up the logrotate configurations
2024-03-04T08:53:07.290Z [container-agent] 2024-03-04 08:53:07 INFO juju-log Adding pebble layer
2024-03-04T08:53:14.031Z [container-agent] 2024-03-04 08:53:14 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-03-04T08:53:14.058Z [container-agent] 2024-03-04 08:53:14 INFO juju-log Attempting reboot from complete outage.
2024-03-04T08:53:17.043Z [container-agent] 2024-03-04 08:53:17 INFO juju.worker.uniter.operation runhook.go:186 ran "mysql-pebble-ready" hook (via hook dispatching script: dispatch)
2024-03-04T08:56:35.008Z [container-agent] 2024-03-04 08:56:35 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T08:56:36.873Z [container-agent] 2024-03-04 08:56:36 WARNING juju-log No relation: certificates
2024-03-04T08:56:37.213Z [container-agent] 2024-03-04 08:56:37 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T09:01:58.695Z [container-agent] 2024-03-04 09:01:58 INFO juju-log Starting the log rotate manager
2024-03-04T09:01:58.708Z [container-agent] 2024-03-04 09:01:58 INFO juju-log Started log rotate manager process with PID 807048
2024-03-04T09:01:58.999Z [container-agent] 2024-03-04 09:01:58 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T09:02:00.911Z [container-agent] 2024-03-04 09:02:00 WARNING juju-log No relation: certificates
2024-03-04T09:02:01.267Z [container-agent] 2024-03-04 09:02:01 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T09:07:37.806Z [container-agent] 2024-03-04 09:07:37 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T09:07:40.449Z [container-agent] 2024-03-04 09:07:40 WARNING juju-log No relation: certificates
2024-03-04T09:07:40.820Z [container-agent] 2024-03-04 09:07:40 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T09:13:00.914Z [container-agent] 2024-03-04 09:13:00 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T09:13:04.055Z [container-agent] 2024-03-04 09:13:04 WARNING juju-log No relation: certificates
2024-03-04T09:13:04.411Z [container-agent] 2024-03-04 09:13:04 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T09:18:11.770Z [container-agent] 2024-03-04 09:18:11 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T09:18:15.452Z [container-agent] 2024-03-04 09:18:15 WARNING juju-log No relation: certificates
2024-03-04T09:18:15.791Z [container-agent] 2024-03-04 09:18:15 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T09:23:08.074Z [container-agent] 2024-03-04 09:23:08 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T09:23:11.750Z [container-agent] 2024-03-04 09:23:11 WARNING juju-log No relation: certificates
2024-03-04T09:23:12.118Z [container-agent] 2024-03-04 09:23:12 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T09:28:36.227Z [container-agent] 2024-03-04 09:28:36 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T09:28:40.667Z [container-agent] 2024-03-04 09:28:40 WARNING juju-log No relation: certificates
2024-03-04T09:28:41.008Z [container-agent] 2024-03-04 09:28:41 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T09:34:22.126Z [container-agent] 2024-03-04 09:34:22 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T09:34:27.194Z [container-agent] 2024-03-04 09:34:27 WARNING juju-log No relation: certificates
2024-03-04T09:34:27.547Z [container-agent] 2024-03-04 09:34:27 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T09:38:49.751Z [container-agent] 2024-03-04 09:38:49 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T09:38:55.437Z [container-agent] 2024-03-04 09:38:55 WARNING juju-log No relation: certificates
2024-03-04T09:38:55.778Z [container-agent] 2024-03-04 09:38:55 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T09:44:00.265Z [container-agent] 2024-03-04 09:44:00 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T09:44:06.110Z [container-agent] 2024-03-04 09:44:06 WARNING juju-log No relation: certificates
2024-03-04T09:44:06.463Z [container-agent] 2024-03-04 09:44:06 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T09:49:44.730Z [container-agent] 2024-03-04 09:49:44 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T09:49:51.167Z [container-agent] 2024-03-04 09:49:51 WARNING juju-log No relation: certificates
2024-03-04T09:49:51.524Z [container-agent] 2024-03-04 09:49:51 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T09:54:22.384Z [container-agent] 2024-03-04 09:54:22 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T09:54:28.888Z [container-agent] 2024-03-04 09:54:28 WARNING juju-log No relation: certificates
2024-03-04T09:54:29.232Z [container-agent] 2024-03-04 09:54:29 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T09:59:09.552Z [container-agent] 2024-03-04 09:59:09 ERROR juju-log Failed to flush GENERAL LOGS logs.
2024-03-04T09:59:09.552Z [container-agent] Traceback (most recent call last):
2024-03-04T09:59:09.552Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 666, in _run_mysqlsh_script
2024-03-04T09:59:09.552Z [container-agent]     stdout, _ = process.wait_output()
2024-03-04T09:59:09.552Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1441, in wait_output
2024-03-04T09:59:09.552Z [container-agent]     raise ExecError[AnyStr](self._command, exit_code, out_value, err_value)
2024-03-04T09:59:09.552Z [container-agent] ops.pebble.ExecError: non-zero exit code 1 executing ['/usr/bin/mysqlsh', '--no-wizard', '--python', '--verbose=1', '-f', '/tmp/script.py', ';', 'rm', '/tmp/script.py'], stdout='', stderr='Cannot set LC_ALL to locale en_US.UTF-8: No such file or directory\nverbose: 2024-03-04T09:59:08Z: Loading startup files...\nverbose: 2024-03-04T09:59:08Z: Loading plugins...\nverbose: 2024-03-04T09:59:08Z: Connecting to MySQL at: serverconfig@horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local\nTraceback (most recent call last):\n  File "<string>", line 1, in <module>\nmysqlsh.DBError: MySQL Error (2003): Shell.connect: Can\'t connect to MySQL server on \'horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local:3306\' (111)\n'
2024-03-04T09:59:09.552Z [container-agent] 
2024-03-04T09:59:09.552Z [container-agent] During handling of the above exception, another exception occurred:
2024-03-04T09:59:09.552Z [container-agent] 
2024-03-04T09:59:09.552Z [container-agent] Traceback (most recent call last):
2024-03-04T09:59:09.552Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-03-04T09:59:09.552Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-03-04T09:59:09.552Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 669, in _run_mysqlsh_script
2024-03-04T09:59:09.552Z [container-agent]     raise MySQLClientError(e.stderr)
2024-03-04T09:59:09.552Z [container-agent] charms.mysql.v0.mysql.MySQLClientError: Cannot set LC_ALL to locale en_US.UTF-8: No such file or directory
2024-03-04T09:59:09.552Z [container-agent] verbose: 2024-03-04T09:59:08Z: Loading startup files...
2024-03-04T09:59:09.552Z [container-agent] verbose: 2024-03-04T09:59:08Z: Loading plugins...
2024-03-04T09:59:09.552Z [container-agent] verbose: 2024-03-04T09:59:08Z: Connecting to MySQL at: serverconfig@horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local
2024-03-04T09:59:09.552Z [container-agent] Traceback (most recent call last):
2024-03-04T09:59:09.552Z [container-agent]   File "<string>", line 1, in <module>
2024-03-04T09:59:09.552Z [container-agent] mysqlsh.DBError: MySQL Error (2003): Shell.connect: Can't connect to MySQL server on 'horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local:3306' (111)
2024-03-04T09:59:09.552Z [container-agent] 
2024-03-04T09:59:58.937Z [container-agent] 2024-03-04 09:59:58 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-03-04T09:59:58.965Z [container-agent] 2024-03-04 09:59:58 INFO juju-log Attempting reboot from complete outage.
2024-03-04T10:00:05.925Z [container-agent] 2024-03-04 10:00:05 WARNING juju-log No relation: certificates
2024-03-04T10:00:06.348Z [container-agent] 2024-03-04 10:00:06 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T10:04:04.816Z [container-agent] 2024-03-04 10:04:04 ERROR juju-log Uncaught exception while in charm code:
2024-03-04T10:04:04.816Z [container-agent] Traceback (most recent call last):
2024-03-04T10:04:04.816Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/./src/charm.py", line 786, in <module>
2024-03-04T10:04:04.816Z [container-agent]     main(MySQLOperatorCharm)
2024-03-04T10:04:04.816Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 436, in main
2024-03-04T10:04:04.816Z [container-agent]     _emit_charm_event(charm, dispatcher.event_name)
2024-03-04T10:04:04.816Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 144, in _emit_charm_event
2024-03-04T10:04:04.816Z [container-agent]     event_to_emit.emit(*args, **kwargs)
2024-03-04T10:04:04.816Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 351, in emit
2024-03-04T10:04:04.816Z [container-agent]     framework._emit(event)
2024-03-04T10:04:04.816Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 853, in _emit
2024-03-04T10:04:04.816Z [container-agent]     self._reemit(event_path)
2024-03-04T10:04:04.816Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 942, in _reemit
2024-03-04T10:04:04.816Z [container-agent]     custom_handler(event)
2024-03-04T10:04:04.816Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/rotate_mysql_logs.py", line 55, in _rotate_mysql_logs
2024-03-04T10:04:04.816Z [container-agent]     self.charm._mysql.flush_mysql_logs(MySQLTextLogs.ERROR)
2024-03-04T10:04:04.816Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-03-04T10:04:04.816Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-03-04T10:04:04.816Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 666, in _run_mysqlsh_script
2024-03-04T10:04:04.816Z [container-agent]     stdout, _ = process.wait_output()
2024-03-04T10:04:04.816Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1436, in wait_output
2024-03-04T10:04:04.816Z [container-agent]     exit_code: int = self._wait()
2024-03-04T10:04:04.816Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1375, in _wait
2024-03-04T10:04:04.816Z [container-agent]     change = self._client.wait_change(self._change_id, timeout=timeout)
2024-03-04T10:04:04.816Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1893, in wait_change
2024-03-04T10:04:04.816Z [container-agent]     return self._wait_change_using_wait(change_id, timeout)
2024-03-04T10:04:04.816Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1914, in _wait_change_using_wait
2024-03-04T10:04:04.816Z [container-agent]     return self._wait_change(change_id, this_timeout)
2024-03-04T10:04:04.816Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1928, in _wait_change
2024-03-04T10:04:04.816Z [container-agent]     resp = self._request('GET', f'/v1/changes/{change_id}/wait', query)
2024-03-04T10:04:04.816Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1655, in _request
2024-03-04T10:04:04.816Z [container-agent]     response = self._request_raw(method, path, query, headers, data)
2024-03-04T10:04:04.816Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1690, in _request_raw
2024-03-04T10:04:04.816Z [container-agent]     response = self.opener.open(request, timeout=self.timeout)
2024-03-04T10:04:04.816Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 519, in open
2024-03-04T10:04:04.816Z [container-agent]     response = self._open(req, data)
2024-03-04T10:04:04.816Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 536, in _open
2024-03-04T10:04:04.816Z [container-agent]     result = self._call_chain(self.handle_open, protocol, protocol +
2024-03-04T10:04:04.816Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 496, in _call_chain
2024-03-04T10:04:04.816Z [container-agent]     result = func(*args)
2024-03-04T10:04:04.816Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 311, in http_open
2024-03-04T10:04:04.816Z [container-agent]     return self.do_open(_UnixSocketConnection, req,  # type:ignore
2024-03-04T10:04:04.816Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 1352, in do_open
2024-03-04T10:04:04.816Z [container-agent]     r = h.getresponse()
2024-03-04T10:04:04.816Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1375, in getresponse
2024-03-04T10:04:04.816Z [container-agent]     response.begin()
2024-03-04T10:04:04.816Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 318, in begin
2024-03-04T10:04:04.816Z [container-agent]     version, status, reason = self._read_status()
2024-03-04T10:04:04.816Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 287, in _read_status
2024-03-04T10:04:04.816Z [container-agent]     raise RemoteDisconnected("Remote end closed connection without"
2024-03-04T10:04:04.816Z [container-agent] http.client.RemoteDisconnected: Remote end closed connection without response
2024-03-04T10:04:04.993Z [container-agent] 2024-03-04 10:04:04 INFO juju.util.exec exec.go:209 run result: exit status 1
2024-03-04T10:04:09.988Z [container-agent] 2024-03-04 10:04:09 INFO juju-log Setting up the logrotate configurations
2024-03-04T10:04:10.058Z [container-agent] 2024-03-04 10:04:10 INFO juju-log Adding pebble layer
2024-03-04T10:04:16.872Z [container-agent] 2024-03-04 10:04:16 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-03-04T10:04:16.896Z [container-agent] 2024-03-04 10:04:16 INFO juju-log Attempting reboot from complete outage.
2024-03-04T10:04:19.869Z [container-agent] 2024-03-04 10:04:19 INFO juju.worker.uniter.operation runhook.go:186 ran "mysql-pebble-ready" hook (via hook dispatching script: dispatch)
2024-03-04T10:05:36.708Z [container-agent] 2024-03-04 10:05:36 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T10:05:38.547Z [container-agent] 2024-03-04 10:05:38 WARNING juju-log No relation: certificates
2024-03-04T10:05:38.895Z [container-agent] 2024-03-04 10:05:38 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T10:10:09.741Z [container-agent] 2024-03-04 10:10:09 INFO juju-log Starting the log rotate manager
2024-03-04T10:10:09.756Z [container-agent] 2024-03-04 10:10:09 INFO juju-log Started log rotate manager process with PID 813408
2024-03-04T10:10:10.195Z [container-agent] 2024-03-04 10:10:10 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T10:10:12.367Z [container-agent] 2024-03-04 10:10:12 WARNING juju-log No relation: certificates
2024-03-04T10:10:12.817Z [container-agent] 2024-03-04 10:10:12 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T10:14:55.469Z [container-agent] 2024-03-04 10:14:55 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T10:14:57.669Z [container-agent] 2024-03-04 10:14:57 WARNING juju-log No relation: certificates
2024-03-04T10:14:58.033Z [container-agent] 2024-03-04 10:14:58 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T10:20:06.493Z [container-agent] 2024-03-04 10:20:06 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T10:20:09.497Z [container-agent] 2024-03-04 10:20:09 WARNING juju-log No relation: certificates
2024-03-04T10:20:09.851Z [container-agent] 2024-03-04 10:20:09 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T10:24:27.885Z [container-agent] 2024-03-04 10:24:27 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T10:24:31.414Z [container-agent] 2024-03-04 10:24:31 WARNING juju-log No relation: certificates
2024-03-04T10:24:31.766Z [container-agent] 2024-03-04 10:24:31 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T10:29:52.500Z [container-agent] 2024-03-04 10:29:52 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T10:29:55.946Z [container-agent] 2024-03-04 10:29:55 WARNING juju-log No relation: certificates
2024-03-04T10:29:56.271Z [container-agent] 2024-03-04 10:29:56 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T10:34:02.028Z [container-agent] 2024-03-04 10:34:02 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T10:34:06.039Z [container-agent] 2024-03-04 10:34:06 WARNING juju-log No relation: certificates
2024-03-04T10:34:06.378Z [container-agent] 2024-03-04 10:34:06 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T10:39:09.978Z [container-agent] 2024-03-04 10:39:09 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T10:39:14.561Z [container-agent] 2024-03-04 10:39:14 WARNING juju-log No relation: certificates
2024-03-04T10:39:15.003Z [container-agent] 2024-03-04 10:39:15 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T10:44:16.435Z [container-agent] 2024-03-04 10:44:16 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T10:44:21.572Z [container-agent] 2024-03-04 10:44:21 WARNING juju-log No relation: certificates
2024-03-04T10:44:21.951Z [container-agent] 2024-03-04 10:44:21 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T10:50:10.629Z [container-agent] 2024-03-04 10:50:10 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T10:50:16.366Z [container-agent] 2024-03-04 10:50:16 WARNING juju-log No relation: certificates
2024-03-04T10:50:16.733Z [container-agent] 2024-03-04 10:50:16 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T10:55:56.240Z [container-agent] 2024-03-04 10:55:56 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T10:56:01.833Z [container-agent] 2024-03-04 10:56:01 WARNING juju-log No relation: certificates
2024-03-04T10:56:02.192Z [container-agent] 2024-03-04 10:56:02 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T11:01:43.383Z [container-agent] 2024-03-04 11:01:43 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T11:01:50.082Z [container-agent] 2024-03-04 11:01:50 WARNING juju-log No relation: certificates
2024-03-04T11:01:50.415Z [container-agent] 2024-03-04 11:01:50 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T11:07:14.948Z [container-agent] 2024-03-04 11:07:14 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-03-04T11:07:14.982Z [container-agent] 2024-03-04 11:07:14 INFO juju-log Attempting reboot from complete outage.
2024-03-04T11:07:22.225Z [container-agent] 2024-03-04 11:07:22 WARNING juju-log No relation: certificates
2024-03-04T11:07:22.584Z [container-agent] 2024-03-04 11:07:22 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T11:12:04.814Z [container-agent] 2024-03-04 11:12:04 ERROR juju-log Uncaught exception while in charm code:
2024-03-04T11:12:04.814Z [container-agent] Traceback (most recent call last):
2024-03-04T11:12:04.814Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/./src/charm.py", line 786, in <module>
2024-03-04T11:12:04.814Z [container-agent]     main(MySQLOperatorCharm)
2024-03-04T11:12:04.814Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 436, in main
2024-03-04T11:12:04.814Z [container-agent]     _emit_charm_event(charm, dispatcher.event_name)
2024-03-04T11:12:04.814Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 144, in _emit_charm_event
2024-03-04T11:12:04.814Z [container-agent]     event_to_emit.emit(*args, **kwargs)
2024-03-04T11:12:04.814Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 351, in emit
2024-03-04T11:12:04.814Z [container-agent]     framework._emit(event)
2024-03-04T11:12:04.814Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 853, in _emit
2024-03-04T11:12:04.814Z [container-agent]     self._reemit(event_path)
2024-03-04T11:12:04.814Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 942, in _reemit
2024-03-04T11:12:04.814Z [container-agent]     custom_handler(event)
2024-03-04T11:12:04.814Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/rotate_mysql_logs.py", line 55, in _rotate_mysql_logs
2024-03-04T11:12:04.814Z [container-agent]     self.charm._mysql.flush_mysql_logs(MySQLTextLogs.ERROR)
2024-03-04T11:12:04.814Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-03-04T11:12:04.814Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-03-04T11:12:04.814Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 666, in _run_mysqlsh_script
2024-03-04T11:12:04.814Z [container-agent]     stdout, _ = process.wait_output()
2024-03-04T11:12:04.814Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1436, in wait_output
2024-03-04T11:12:04.814Z [container-agent]     exit_code: int = self._wait()
2024-03-04T11:12:04.814Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1375, in _wait
2024-03-04T11:12:04.814Z [container-agent]     change = self._client.wait_change(self._change_id, timeout=timeout)
2024-03-04T11:12:04.814Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1893, in wait_change
2024-03-04T11:12:04.814Z [container-agent]     return self._wait_change_using_wait(change_id, timeout)
2024-03-04T11:12:04.814Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1914, in _wait_change_using_wait
2024-03-04T11:12:04.814Z [container-agent]     return self._wait_change(change_id, this_timeout)
2024-03-04T11:12:04.814Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1928, in _wait_change
2024-03-04T11:12:04.814Z [container-agent]     resp = self._request('GET', f'/v1/changes/{change_id}/wait', query)
2024-03-04T11:12:04.814Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1655, in _request
2024-03-04T11:12:04.814Z [container-agent]     response = self._request_raw(method, path, query, headers, data)
2024-03-04T11:12:04.814Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1690, in _request_raw
2024-03-04T11:12:04.814Z [container-agent]     response = self.opener.open(request, timeout=self.timeout)
2024-03-04T11:12:04.814Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 519, in open
2024-03-04T11:12:04.814Z [container-agent]     response = self._open(req, data)
2024-03-04T11:12:04.814Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 536, in _open
2024-03-04T11:12:04.814Z [container-agent]     result = self._call_chain(self.handle_open, protocol, protocol +
2024-03-04T11:12:04.814Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 496, in _call_chain
2024-03-04T11:12:04.814Z [container-agent]     result = func(*args)
2024-03-04T11:12:04.814Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 311, in http_open
2024-03-04T11:12:04.814Z [container-agent]     return self.do_open(_UnixSocketConnection, req,  # type:ignore
2024-03-04T11:12:04.814Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 1352, in do_open
2024-03-04T11:12:04.814Z [container-agent]     r = h.getresponse()
2024-03-04T11:12:04.814Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1375, in getresponse
2024-03-04T11:12:04.814Z [container-agent]     response.begin()
2024-03-04T11:12:04.814Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 318, in begin
2024-03-04T11:12:04.814Z [container-agent]     version, status, reason = self._read_status()
2024-03-04T11:12:04.814Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 287, in _read_status
2024-03-04T11:12:04.814Z [container-agent]     raise RemoteDisconnected("Remote end closed connection without"
2024-03-04T11:12:04.814Z [container-agent] http.client.RemoteDisconnected: Remote end closed connection without response
2024-03-04T11:12:04.939Z [container-agent] 2024-03-04 11:12:04 INFO juju.util.exec exec.go:209 run result: exit status 1
2024-03-04T11:12:07.335Z [container-agent] 2024-03-04 11:12:07 INFO juju-log Setting up the logrotate configurations
2024-03-04T11:12:07.455Z [container-agent] 2024-03-04 11:12:07 INFO juju-log Adding pebble layer
2024-03-04T11:12:14.272Z [container-agent] 2024-03-04 11:12:14 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-03-04T11:12:14.300Z [container-agent] 2024-03-04 11:12:14 INFO juju-log Attempting reboot from complete outage.
2024-03-04T11:12:17.291Z [container-agent] 2024-03-04 11:12:17 INFO juju.worker.uniter.operation runhook.go:186 ran "mysql-pebble-ready" hook (via hook dispatching script: dispatch)
2024-03-04T11:12:43.525Z [container-agent] 2024-03-04 11:12:43 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T11:12:45.614Z [container-agent] 2024-03-04 11:12:45 WARNING juju-log No relation: certificates
2024-03-04T11:12:45.963Z [container-agent] 2024-03-04 11:12:45 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T11:17:57.670Z [container-agent] 2024-03-04 11:17:57 INFO juju-log Starting the log rotate manager
2024-03-04T11:17:57.684Z [container-agent] 2024-03-04 11:17:57 INFO juju-log Started log rotate manager process with PID 819738
2024-03-04T11:17:58.009Z [container-agent] 2024-03-04 11:17:58 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T11:17:59.960Z [container-agent] 2024-03-04 11:17:59 WARNING juju-log No relation: certificates
2024-03-04T11:18:00.366Z [container-agent] 2024-03-04 11:18:00 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T11:23:47.689Z [container-agent] 2024-03-04 11:23:47 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T11:23:50.225Z [container-agent] 2024-03-04 11:23:50 WARNING juju-log No relation: certificates
2024-03-04T11:23:50.601Z [container-agent] 2024-03-04 11:23:50 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T11:28:50.961Z [container-agent] 2024-03-04 11:28:50 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T11:28:53.823Z [container-agent] 2024-03-04 11:28:53 WARNING juju-log No relation: certificates
2024-03-04T11:28:54.188Z [container-agent] 2024-03-04 11:28:54 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T11:34:49.871Z [container-agent] 2024-03-04 11:34:49 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T11:34:53.205Z [container-agent] 2024-03-04 11:34:53 WARNING juju-log No relation: certificates
2024-03-04T11:34:53.549Z [container-agent] 2024-03-04 11:34:53 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T11:38:53.625Z [container-agent] 2024-03-04 11:38:53 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T11:38:57.413Z [container-agent] 2024-03-04 11:38:57 WARNING juju-log No relation: certificates
2024-03-04T11:38:57.777Z [container-agent] 2024-03-04 11:38:57 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T11:43:36.344Z [container-agent] 2024-03-04 11:43:36 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T11:43:40.455Z [container-agent] 2024-03-04 11:43:40 WARNING juju-log No relation: certificates
2024-03-04T11:43:40.851Z [container-agent] 2024-03-04 11:43:40 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T11:49:32.406Z [container-agent] 2024-03-04 11:49:32 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T11:49:37.351Z [container-agent] 2024-03-04 11:49:37 WARNING juju-log No relation: certificates
2024-03-04T11:49:37.709Z [container-agent] 2024-03-04 11:49:37 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T11:54:00.699Z [container-agent] 2024-03-04 11:54:00 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T11:54:05.859Z [container-agent] 2024-03-04 11:54:05 WARNING juju-log No relation: certificates
2024-03-04T11:54:06.191Z [container-agent] 2024-03-04 11:54:06 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T11:59:44.946Z [container-agent] 2024-03-04 11:59:44 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T11:59:50.945Z [container-agent] 2024-03-04 11:59:50 WARNING juju-log No relation: certificates
2024-03-04T11:59:51.292Z [container-agent] 2024-03-04 11:59:51 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T12:05:00.534Z [container-agent] 2024-03-04 12:05:00 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T12:05:06.999Z [container-agent] 2024-03-04 12:05:06 WARNING juju-log No relation: certificates
2024-03-04T12:05:07.338Z [container-agent] 2024-03-04 12:05:07 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T12:09:19.006Z [container-agent] 2024-03-04 12:09:19 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T12:09:25.946Z [container-agent] 2024-03-04 12:09:25 WARNING juju-log No relation: certificates
2024-03-04T12:09:26.300Z [container-agent] 2024-03-04 12:09:26 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T12:14:45.669Z [container-agent] 2024-03-04 12:14:45 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T12:14:52.861Z [container-agent] 2024-03-04 12:14:52 WARNING juju-log No relation: certificates
2024-03-04T12:14:53.231Z [container-agent] 2024-03-04 12:14:53 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T12:19:26.367Z [container-agent] 2024-03-04 12:19:26 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T12:19:33.818Z [container-agent] 2024-03-04 12:19:33 WARNING juju-log No relation: certificates
2024-03-04T12:19:34.160Z [container-agent] 2024-03-04 12:19:34 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T12:24:03.212Z [container-agent] 2024-03-04 12:24:03 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T12:24:10.888Z [container-agent] 2024-03-04 12:24:10 WARNING juju-log No relation: certificates
2024-03-04T12:24:11.262Z [container-agent] 2024-03-04 12:24:11 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T12:28:57.793Z [container-agent] 2024-03-04 12:28:57 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T12:29:05.984Z [container-agent] 2024-03-04 12:29:05 WARNING juju-log No relation: certificates
2024-03-04T12:29:06.337Z [container-agent] 2024-03-04 12:29:06 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T12:34:15.360Z [container-agent] 2024-03-04 12:34:15 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-03-04T12:34:15.389Z [container-agent] 2024-03-04 12:34:15 INFO juju-log Attempting reboot from complete outage.
2024-03-04T12:34:23.693Z [container-agent] 2024-03-04 12:34:23 WARNING juju-log No relation: certificates
2024-03-04T12:34:24.068Z [container-agent] 2024-03-04 12:34:24 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T12:35:09.774Z [container-agent] 2024-03-04 12:35:09 ERROR juju-log Uncaught exception while in charm code:
2024-03-04T12:35:09.774Z [container-agent] Traceback (most recent call last):
2024-03-04T12:35:09.774Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/./src/charm.py", line 786, in <module>
2024-03-04T12:35:09.774Z [container-agent]     main(MySQLOperatorCharm)
2024-03-04T12:35:09.774Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 436, in main
2024-03-04T12:35:09.774Z [container-agent]     _emit_charm_event(charm, dispatcher.event_name)
2024-03-04T12:35:09.774Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 144, in _emit_charm_event
2024-03-04T12:35:09.774Z [container-agent]     event_to_emit.emit(*args, **kwargs)
2024-03-04T12:35:09.774Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 351, in emit
2024-03-04T12:35:09.774Z [container-agent]     framework._emit(event)
2024-03-04T12:35:09.774Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 853, in _emit
2024-03-04T12:35:09.774Z [container-agent]     self._reemit(event_path)
2024-03-04T12:35:09.774Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 942, in _reemit
2024-03-04T12:35:09.774Z [container-agent]     custom_handler(event)
2024-03-04T12:35:09.774Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/rotate_mysql_logs.py", line 56, in _rotate_mysql_logs
2024-03-04T12:35:09.774Z [container-agent]     self.charm._mysql.flush_mysql_logs(MySQLTextLogs.GENERAL)
2024-03-04T12:35:09.774Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-03-04T12:35:09.774Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-03-04T12:35:09.774Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 666, in _run_mysqlsh_script
2024-03-04T12:35:09.774Z [container-agent]     stdout, _ = process.wait_output()
2024-03-04T12:35:09.774Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1436, in wait_output
2024-03-04T12:35:09.774Z [container-agent]     exit_code: int = self._wait()
2024-03-04T12:35:09.774Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1375, in _wait
2024-03-04T12:35:09.774Z [container-agent]     change = self._client.wait_change(self._change_id, timeout=timeout)
2024-03-04T12:35:09.774Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1893, in wait_change
2024-03-04T12:35:09.774Z [container-agent]     return self._wait_change_using_wait(change_id, timeout)
2024-03-04T12:35:09.774Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1914, in _wait_change_using_wait
2024-03-04T12:35:09.774Z [container-agent]     return self._wait_change(change_id, this_timeout)
2024-03-04T12:35:09.774Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1928, in _wait_change
2024-03-04T12:35:09.774Z [container-agent]     resp = self._request('GET', f'/v1/changes/{change_id}/wait', query)
2024-03-04T12:35:09.774Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1655, in _request
2024-03-04T12:35:09.774Z [container-agent]     response = self._request_raw(method, path, query, headers, data)
2024-03-04T12:35:09.774Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1690, in _request_raw
2024-03-04T12:35:09.774Z [container-agent]     response = self.opener.open(request, timeout=self.timeout)
2024-03-04T12:35:09.774Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 519, in open
2024-03-04T12:35:09.774Z [container-agent]     response = self._open(req, data)
2024-03-04T12:35:09.774Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 536, in _open
2024-03-04T12:35:09.774Z [container-agent]     result = self._call_chain(self.handle_open, protocol, protocol +
2024-03-04T12:35:09.774Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 496, in _call_chain
2024-03-04T12:35:09.774Z [container-agent]     result = func(*args)
2024-03-04T12:35:09.774Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 311, in http_open
2024-03-04T12:35:09.774Z [container-agent]     return self.do_open(_UnixSocketConnection, req,  # type:ignore
2024-03-04T12:35:09.774Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 1352, in do_open
2024-03-04T12:35:09.774Z [container-agent]     r = h.getresponse()
2024-03-04T12:35:09.774Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1375, in getresponse
2024-03-04T12:35:09.774Z [container-agent]     response.begin()
2024-03-04T12:35:09.774Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 318, in begin
2024-03-04T12:35:09.774Z [container-agent]     version, status, reason = self._read_status()
2024-03-04T12:35:09.774Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 287, in _read_status
2024-03-04T12:35:09.774Z [container-agent]     raise RemoteDisconnected("Remote end closed connection without"
2024-03-04T12:35:09.774Z [container-agent] http.client.RemoteDisconnected: Remote end closed connection without response
2024-03-04T12:35:09.916Z [container-agent] 2024-03-04 12:35:09 INFO juju.util.exec exec.go:209 run result: exit status 1
2024-03-04T12:35:11.688Z [container-agent] 2024-03-04 12:35:11 INFO juju-log Setting up the logrotate configurations
2024-03-04T12:35:11.757Z [container-agent] 2024-03-04 12:35:11 INFO juju-log Adding pebble layer
2024-03-04T12:35:18.558Z [container-agent] 2024-03-04 12:35:18 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-03-04T12:35:18.583Z [container-agent] 2024-03-04 12:35:18 INFO juju-log Attempting reboot from complete outage.
2024-03-04T12:35:21.540Z [container-agent] 2024-03-04 12:35:21 INFO juju.worker.uniter.operation runhook.go:186 ran "mysql-pebble-ready" hook (via hook dispatching script: dispatch)
2024-03-04T12:39:22.995Z [container-agent] 2024-03-04 12:39:22 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T12:39:24.834Z [container-agent] 2024-03-04 12:39:24 WARNING juju-log No relation: certificates
2024-03-04T12:39:25.181Z [container-agent] 2024-03-04 12:39:25 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T12:43:42.217Z [container-agent] 2024-03-04 12:43:42 INFO juju-log Starting the log rotate manager
2024-03-04T12:43:42.230Z [container-agent] 2024-03-04 12:43:42 INFO juju-log Started log rotate manager process with PID 827665
2024-03-04T12:43:42.537Z [container-agent] 2024-03-04 12:43:42 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T12:43:44.517Z [container-agent] 2024-03-04 12:43:44 WARNING juju-log No relation: certificates
2024-03-04T12:43:44.879Z [container-agent] 2024-03-04 12:43:44 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T12:49:16.205Z [container-agent] 2024-03-04 12:49:16 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T12:49:18.850Z [container-agent] 2024-03-04 12:49:18 WARNING juju-log No relation: certificates
2024-03-04T12:49:19.216Z [container-agent] 2024-03-04 12:49:19 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T12:54:51.936Z [container-agent] 2024-03-04 12:54:51 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T12:54:54.892Z [container-agent] 2024-03-04 12:54:54 WARNING juju-log No relation: certificates
2024-03-04T12:54:55.260Z [container-agent] 2024-03-04 12:54:55 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T12:58:59.026Z [container-agent] 2024-03-04 12:58:59 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T12:59:02.785Z [container-agent] 2024-03-04 12:59:02 WARNING juju-log No relation: certificates
2024-03-04T12:59:03.161Z [container-agent] 2024-03-04 12:59:03 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T13:03:12.982Z [container-agent] 2024-03-04 13:03:12 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T13:03:17.013Z [container-agent] 2024-03-04 13:03:17 WARNING juju-log No relation: certificates
2024-03-04T13:03:17.372Z [container-agent] 2024-03-04 13:03:17 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T13:08:43.215Z [container-agent] 2024-03-04 13:08:43 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T13:08:47.669Z [container-agent] 2024-03-04 13:08:47 WARNING juju-log No relation: certificates
2024-03-04T13:08:48.053Z [container-agent] 2024-03-04 13:08:48 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T13:12:55.929Z [container-agent] 2024-03-04 13:12:55 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T13:13:00.598Z [container-agent] 2024-03-04 13:13:00 WARNING juju-log No relation: certificates
2024-03-04T13:13:00.969Z [container-agent] 2024-03-04 13:13:00 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T13:18:48.826Z [container-agent] 2024-03-04 13:18:48 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T13:18:54.033Z [container-agent] 2024-03-04 13:18:54 WARNING juju-log No relation: certificates
2024-03-04T13:18:54.383Z [container-agent] 2024-03-04 13:18:54 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T13:24:29.530Z [container-agent] 2024-03-04 13:24:29 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T13:24:35.433Z [container-agent] 2024-03-04 13:24:35 WARNING juju-log No relation: certificates
2024-03-04T13:24:35.804Z [container-agent] 2024-03-04 13:24:35 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T13:30:03.665Z [container-agent] 2024-03-04 13:30:03 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T13:30:09.739Z [container-agent] 2024-03-04 13:30:09 WARNING juju-log No relation: certificates
2024-03-04T13:30:10.135Z [container-agent] 2024-03-04 13:30:10 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T13:34:35.252Z [container-agent] 2024-03-04 13:34:35 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T13:34:42.019Z [container-agent] 2024-03-04 13:34:42 WARNING juju-log No relation: certificates
2024-03-04T13:34:42.365Z [container-agent] 2024-03-04 13:34:42 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T13:39:51.173Z [container-agent] 2024-03-04 13:39:51 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T13:39:58.255Z [container-agent] 2024-03-04 13:39:58 WARNING juju-log No relation: certificates
2024-03-04T13:39:58.602Z [container-agent] 2024-03-04 13:39:58 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T13:44:52.092Z [container-agent] 2024-03-04 13:44:52 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T13:44:59.913Z [container-agent] 2024-03-04 13:44:59 WARNING juju-log No relation: certificates
2024-03-04T13:45:00.334Z [container-agent] 2024-03-04 13:45:00 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T13:48:10.574Z [container-agent] 2024-03-04 13:48:10 ERROR juju-log Failed to flush ERROR LOGS logs.
2024-03-04T13:48:10.574Z [container-agent] Traceback (most recent call last):
2024-03-04T13:48:10.574Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 666, in _run_mysqlsh_script
2024-03-04T13:48:10.574Z [container-agent]     stdout, _ = process.wait_output()
2024-03-04T13:48:10.574Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1441, in wait_output
2024-03-04T13:48:10.574Z [container-agent]     raise ExecError[AnyStr](self._command, exit_code, out_value, err_value)
2024-03-04T13:48:10.574Z [container-agent] ops.pebble.ExecError: non-zero exit code 1 executing ['/usr/bin/mysqlsh', '--no-wizard', '--python', '--verbose=1', '-f', '/tmp/script.py', ';', 'rm', '/tmp/script.py'], stdout='', stderr='Cannot set LC_ALL to locale en_US.UTF-8: No such file or directory\nverbose: 2024-03-04T13:48:06Z: Loading startup files...\nverbose: 2024-03-04T13:48:06Z: Loading plugins...\nverbose: 2024-03-04T13:48:06Z: Connecting to MySQL at: serverconfig@horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local\nTraceback (most recent call last):\n  File "<string>", line 1, in <module>\nmysqlsh.DBError: MySQL Error (2013): Shell.connect: Lost connection to MySQL server at \'reading initial communication packet\', system error: 104\n'
2024-03-04T13:48:10.574Z [container-agent] 
2024-03-04T13:48:10.574Z [container-agent] During handling of the above exception, another exception occurred:
2024-03-04T13:48:10.574Z [container-agent] 
2024-03-04T13:48:10.574Z [container-agent] Traceback (most recent call last):
2024-03-04T13:48:10.574Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-03-04T13:48:10.574Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-03-04T13:48:10.574Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 669, in _run_mysqlsh_script
2024-03-04T13:48:10.574Z [container-agent]     raise MySQLClientError(e.stderr)
2024-03-04T13:48:10.574Z [container-agent] charms.mysql.v0.mysql.MySQLClientError: Cannot set LC_ALL to locale en_US.UTF-8: No such file or directory
2024-03-04T13:48:10.574Z [container-agent] verbose: 2024-03-04T13:48:06Z: Loading startup files...
2024-03-04T13:48:10.574Z [container-agent] verbose: 2024-03-04T13:48:06Z: Loading plugins...
2024-03-04T13:48:10.574Z [container-agent] verbose: 2024-03-04T13:48:06Z: Connecting to MySQL at: serverconfig@horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local
2024-03-04T13:48:10.574Z [container-agent] Traceback (most recent call last):
2024-03-04T13:48:10.574Z [container-agent]   File "<string>", line 1, in <module>
2024-03-04T13:48:10.574Z [container-agent] mysqlsh.DBError: MySQL Error (2013): Shell.connect: Lost connection to MySQL server at 'reading initial communication packet', system error: 104
2024-03-04T13:48:10.574Z [container-agent] 
2024-03-04T13:48:13.398Z [container-agent] 2024-03-04 13:48:13 ERROR juju-log Failed to flush GENERAL LOGS logs.
2024-03-04T13:48:13.398Z [container-agent] Traceback (most recent call last):
2024-03-04T13:48:13.398Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 666, in _run_mysqlsh_script
2024-03-04T13:48:13.398Z [container-agent]     stdout, _ = process.wait_output()
2024-03-04T13:48:13.398Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1441, in wait_output
2024-03-04T13:48:13.398Z [container-agent]     raise ExecError[AnyStr](self._command, exit_code, out_value, err_value)
2024-03-04T13:48:13.398Z [container-agent] ops.pebble.ExecError: non-zero exit code 1 executing ['/usr/bin/mysqlsh', '--no-wizard', '--python', '--verbose=1', '-f', '/tmp/script.py', ';', 'rm', '/tmp/script.py'], stdout='', stderr='Cannot set LC_ALL to locale en_US.UTF-8: No such file or directory\nverbose: 2024-03-04T13:48:11Z: Loading startup files...\nverbose: 2024-03-04T13:48:11Z: Loading plugins...\nverbose: 2024-03-04T13:48:11Z: Connecting to MySQL at: serverconfig@horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local\nTraceback (most recent call last):\n  File "<string>", line 1, in <module>\nmysqlsh.DBError: MySQL Error (2003): Shell.connect: Can\'t connect to MySQL server on \'horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local:3306\' (111)\n'
2024-03-04T13:48:13.398Z [container-agent] 
2024-03-04T13:48:13.398Z [container-agent] During handling of the above exception, another exception occurred:
2024-03-04T13:48:13.398Z [container-agent] 
2024-03-04T13:48:13.398Z [container-agent] Traceback (most recent call last):
2024-03-04T13:48:13.398Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-03-04T13:48:13.398Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-03-04T13:48:13.398Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 669, in _run_mysqlsh_script
2024-03-04T13:48:13.398Z [container-agent]     raise MySQLClientError(e.stderr)
2024-03-04T13:48:13.398Z [container-agent] charms.mysql.v0.mysql.MySQLClientError: Cannot set LC_ALL to locale en_US.UTF-8: No such file or directory
2024-03-04T13:48:13.398Z [container-agent] verbose: 2024-03-04T13:48:11Z: Loading startup files...
2024-03-04T13:48:13.398Z [container-agent] verbose: 2024-03-04T13:48:11Z: Loading plugins...
2024-03-04T13:48:13.398Z [container-agent] verbose: 2024-03-04T13:48:11Z: Connecting to MySQL at: serverconfig@horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local
2024-03-04T13:48:13.398Z [container-agent] Traceback (most recent call last):
2024-03-04T13:48:13.398Z [container-agent]   File "<string>", line 1, in <module>
2024-03-04T13:48:13.398Z [container-agent] mysqlsh.DBError: MySQL Error (2003): Shell.connect: Can't connect to MySQL server on 'horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local:3306' (111)
2024-03-04T13:48:13.398Z [container-agent] 
2024-03-04T13:48:13.413Z [container-agent] 2024-03-04 13:48:13 ERROR juju-log Uncaught exception while in charm code:
2024-03-04T13:48:13.413Z [container-agent] Traceback (most recent call last):
2024-03-04T13:48:13.413Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/./src/charm.py", line 786, in <module>
2024-03-04T13:48:13.413Z [container-agent]     main(MySQLOperatorCharm)
2024-03-04T13:48:13.413Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 436, in main
2024-03-04T13:48:13.413Z [container-agent]     _emit_charm_event(charm, dispatcher.event_name)
2024-03-04T13:48:13.413Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 144, in _emit_charm_event
2024-03-04T13:48:13.413Z [container-agent]     event_to_emit.emit(*args, **kwargs)
2024-03-04T13:48:13.413Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 351, in emit
2024-03-04T13:48:13.413Z [container-agent]     framework._emit(event)
2024-03-04T13:48:13.413Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 853, in _emit
2024-03-04T13:48:13.413Z [container-agent]     self._reemit(event_path)
2024-03-04T13:48:13.413Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 942, in _reemit
2024-03-04T13:48:13.413Z [container-agent]     custom_handler(event)
2024-03-04T13:48:13.413Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/rotate_mysql_logs.py", line 57, in _rotate_mysql_logs
2024-03-04T13:48:13.413Z [container-agent]     self.charm._mysql.flush_mysql_logs(MySQLTextLogs.SLOW)
2024-03-04T13:48:13.413Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-03-04T13:48:13.413Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-03-04T13:48:13.413Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 649, in _run_mysqlsh_script
2024-03-04T13:48:13.413Z [container-agent]     self.container.push(path=MYSQLSH_SCRIPT_FILE, source=script)
2024-03-04T13:48:13.413Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/model.py", line 2272, in push
2024-03-04T13:48:13.413Z [container-agent]     self._pebble.push(str(path), source, encoding=encoding,
2024-03-04T13:48:13.413Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 2125, in push
2024-03-04T13:48:13.413Z [container-agent]     response = self._request_raw('POST', '/v1/files', None, headers, data)
2024-03-04T13:48:13.413Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1704, in _request_raw
2024-03-04T13:48:13.413Z [container-agent]     raise ConnectionError(
2024-03-04T13:48:13.413Z [container-agent] ops.pebble.ConnectionError: Could not connect to Pebble: socket not found at '/charm/containers/mysql/pebble.socket' (container restarted?)
2024-03-04T13:48:13.557Z [container-agent] 2024-03-04 13:48:13 INFO juju.util.exec exec.go:209 run result: exit status 1
2024-03-04T13:48:19.232Z [container-agent] 2024-03-04 13:48:19 INFO juju-log Starting the log rotate manager
2024-03-04T13:48:19.245Z [container-agent] 2024-03-04 13:48:19 INFO juju-log Started log rotate manager process with PID 833945
2024-03-04T13:48:19.460Z [container-agent] 2024-03-04 13:48:19 INFO juju-log Setting up the logrotate configurations
2024-03-04T13:48:19.538Z [container-agent] 2024-03-04 13:48:19 INFO juju-log Adding pebble layer
2024-03-04T13:48:26.255Z [container-agent] 2024-03-04 13:48:26 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-03-04T13:48:26.285Z [container-agent] 2024-03-04 13:48:26 INFO juju-log Attempting reboot from complete outage.
2024-03-04T13:48:29.428Z [container-agent] 2024-03-04 13:48:29 INFO juju.worker.uniter.operation runhook.go:186 ran "mysql-pebble-ready" hook (via hook dispatching script: dispatch)
2024-03-04T13:48:54.882Z [container-agent] 2024-03-04 13:48:54 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T13:48:56.706Z [container-agent] 2024-03-04 13:48:56 WARNING juju-log No relation: certificates
2024-03-04T13:48:57.062Z [container-agent] 2024-03-04 13:48:57 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T13:54:20.645Z [container-agent] 2024-03-04 13:54:20 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T13:54:22.966Z [container-agent] 2024-03-04 13:54:22 WARNING juju-log No relation: certificates
2024-03-04T13:54:23.329Z [container-agent] 2024-03-04 13:54:23 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T13:58:30.468Z [container-agent] 2024-03-04 13:58:30 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T13:58:33.528Z [container-agent] 2024-03-04 13:58:33 WARNING juju-log No relation: certificates
2024-03-04T13:58:33.879Z [container-agent] 2024-03-04 13:58:33 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T14:04:15.356Z [container-agent] 2024-03-04 14:04:15 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T14:04:18.657Z [container-agent] 2024-03-04 14:04:18 WARNING juju-log No relation: certificates
2024-03-04T14:04:18.999Z [container-agent] 2024-03-04 14:04:18 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T14:08:49.980Z [container-agent] 2024-03-04 14:08:49 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T14:08:53.704Z [container-agent] 2024-03-04 14:08:53 WARNING juju-log No relation: certificates
2024-03-04T14:08:54.051Z [container-agent] 2024-03-04 14:08:54 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T14:13:18.810Z [container-agent] 2024-03-04 14:13:18 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T14:13:23.225Z [container-agent] 2024-03-04 14:13:23 WARNING juju-log No relation: certificates
2024-03-04T14:13:23.569Z [container-agent] 2024-03-04 14:13:23 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T14:18:28.600Z [container-agent] 2024-03-04 14:18:28 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T14:18:33.356Z [container-agent] 2024-03-04 14:18:33 WARNING juju-log No relation: certificates
2024-03-04T14:18:33.735Z [container-agent] 2024-03-04 14:18:33 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T14:23:24.066Z [container-agent] 2024-03-04 14:23:24 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T14:23:28.957Z [container-agent] 2024-03-04 14:23:28 WARNING juju-log No relation: certificates
2024-03-04T14:23:29.329Z [container-agent] 2024-03-04 14:23:29 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T14:27:58.673Z [container-agent] 2024-03-04 14:27:58 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T14:28:03.841Z [container-agent] 2024-03-04 14:28:03 WARNING juju-log No relation: certificates
2024-03-04T14:28:04.186Z [container-agent] 2024-03-04 14:28:04 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T14:32:36.340Z [container-agent] 2024-03-04 14:32:36 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T14:32:42.425Z [container-agent] 2024-03-04 14:32:42 WARNING juju-log No relation: certificates
2024-03-04T14:32:42.772Z [container-agent] 2024-03-04 14:32:42 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T14:38:12.333Z [container-agent] 2024-03-04 14:38:12 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T14:38:19.420Z [container-agent] 2024-03-04 14:38:19 WARNING juju-log No relation: certificates
2024-03-04T14:38:19.820Z [container-agent] 2024-03-04 14:38:19 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T14:42:17.057Z [container-agent] 2024-03-04 14:42:17 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T14:42:23.753Z [container-agent] 2024-03-04 14:42:23 WARNING juju-log No relation: certificates
2024-03-04T14:42:24.101Z [container-agent] 2024-03-04 14:42:24 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T14:46:48.250Z [container-agent] 2024-03-04 14:46:48 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T14:46:55.758Z [container-agent] 2024-03-04 14:46:55 WARNING juju-log No relation: certificates
2024-03-04T14:46:56.120Z [container-agent] 2024-03-04 14:46:56 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T14:51:59.318Z [container-agent] 2024-03-04 14:51:59 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T14:52:07.054Z [container-agent] 2024-03-04 14:52:07 WARNING juju-log No relation: certificates
2024-03-04T14:52:07.398Z [container-agent] 2024-03-04 14:52:07 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T14:56:39.495Z [container-agent] 2024-03-04 14:56:39 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-03-04T14:56:39.530Z [container-agent] 2024-03-04 14:56:39 INFO juju-log Attempting reboot from complete outage.
2024-03-04T14:56:47.402Z [container-agent] 2024-03-04 14:56:47 WARNING juju-log No relation: certificates
2024-03-04T14:56:47.753Z [container-agent] 2024-03-04 14:56:47 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T14:59:04.827Z [container-agent] 2024-03-04 14:59:04 ERROR juju-log Uncaught exception while in charm code:
2024-03-04T14:59:04.827Z [container-agent] Traceback (most recent call last):
2024-03-04T14:59:04.827Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/./src/charm.py", line 786, in <module>
2024-03-04T14:59:04.827Z [container-agent]     main(MySQLOperatorCharm)
2024-03-04T14:59:04.827Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 436, in main
2024-03-04T14:59:04.827Z [container-agent]     _emit_charm_event(charm, dispatcher.event_name)
2024-03-04T14:59:04.827Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 144, in _emit_charm_event
2024-03-04T14:59:04.827Z [container-agent]     event_to_emit.emit(*args, **kwargs)
2024-03-04T14:59:04.827Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 351, in emit
2024-03-04T14:59:04.827Z [container-agent]     framework._emit(event)
2024-03-04T14:59:04.827Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 853, in _emit
2024-03-04T14:59:04.827Z [container-agent]     self._reemit(event_path)
2024-03-04T14:59:04.827Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 942, in _reemit
2024-03-04T14:59:04.827Z [container-agent]     custom_handler(event)
2024-03-04T14:59:04.827Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/rotate_mysql_logs.py", line 55, in _rotate_mysql_logs
2024-03-04T14:59:04.827Z [container-agent]     self.charm._mysql.flush_mysql_logs(MySQLTextLogs.ERROR)
2024-03-04T14:59:04.827Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-03-04T14:59:04.827Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-03-04T14:59:04.827Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 666, in _run_mysqlsh_script
2024-03-04T14:59:04.827Z [container-agent]     stdout, _ = process.wait_output()
2024-03-04T14:59:04.827Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1436, in wait_output
2024-03-04T14:59:04.827Z [container-agent]     exit_code: int = self._wait()
2024-03-04T14:59:04.827Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1375, in _wait
2024-03-04T14:59:04.827Z [container-agent]     change = self._client.wait_change(self._change_id, timeout=timeout)
2024-03-04T14:59:04.827Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1893, in wait_change
2024-03-04T14:59:04.827Z [container-agent]     return self._wait_change_using_wait(change_id, timeout)
2024-03-04T14:59:04.827Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1914, in _wait_change_using_wait
2024-03-04T14:59:04.827Z [container-agent]     return self._wait_change(change_id, this_timeout)
2024-03-04T14:59:04.827Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1928, in _wait_change
2024-03-04T14:59:04.827Z [container-agent]     resp = self._request('GET', f'/v1/changes/{change_id}/wait', query)
2024-03-04T14:59:04.827Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1655, in _request
2024-03-04T14:59:04.827Z [container-agent]     response = self._request_raw(method, path, query, headers, data)
2024-03-04T14:59:04.827Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1690, in _request_raw
2024-03-04T14:59:04.827Z [container-agent]     response = self.opener.open(request, timeout=self.timeout)
2024-03-04T14:59:04.827Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 519, in open
2024-03-04T14:59:04.827Z [container-agent]     response = self._open(req, data)
2024-03-04T14:59:04.827Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 536, in _open
2024-03-04T14:59:04.827Z [container-agent]     result = self._call_chain(self.handle_open, protocol, protocol +
2024-03-04T14:59:04.828Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 496, in _call_chain
2024-03-04T14:59:04.828Z [container-agent]     result = func(*args)
2024-03-04T14:59:04.828Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 311, in http_open
2024-03-04T14:59:04.828Z [container-agent]     return self.do_open(_UnixSocketConnection, req,  # type:ignore
2024-03-04T14:59:04.828Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 1352, in do_open
2024-03-04T14:59:04.828Z [container-agent]     r = h.getresponse()
2024-03-04T14:59:04.828Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1375, in getresponse
2024-03-04T14:59:04.828Z [container-agent]     response.begin()
2024-03-04T14:59:04.828Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 318, in begin
2024-03-04T14:59:04.828Z [container-agent]     version, status, reason = self._read_status()
2024-03-04T14:59:04.828Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 287, in _read_status
2024-03-04T14:59:04.828Z [container-agent]     raise RemoteDisconnected("Remote end closed connection without"
2024-03-04T14:59:04.828Z [container-agent] http.client.RemoteDisconnected: Remote end closed connection without response
2024-03-04T14:59:04.981Z [container-agent] 2024-03-04 14:59:04 INFO juju.util.exec exec.go:209 run result: exit status 1
2024-03-04T14:59:07.382Z [container-agent] 2024-03-04 14:59:07 INFO juju-log Setting up the logrotate configurations
2024-03-04T14:59:07.464Z [container-agent] 2024-03-04 14:59:07 INFO juju-log Adding pebble layer
2024-03-04T14:59:14.224Z [container-agent] 2024-03-04 14:59:14 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-03-04T14:59:14.250Z [container-agent] 2024-03-04 14:59:14 INFO juju-log Attempting reboot from complete outage.
2024-03-04T14:59:17.368Z [container-agent] 2024-03-04 14:59:17 INFO juju.worker.uniter.operation runhook.go:186 ran "mysql-pebble-ready" hook (via hook dispatching script: dispatch)
2024-03-04T15:02:27.934Z [container-agent] 2024-03-04 15:02:27 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T15:02:29.830Z [container-agent] 2024-03-04 15:02:29 WARNING juju-log No relation: certificates
2024-03-04T15:02:30.212Z [container-agent] 2024-03-04 15:02:30 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T15:07:57.433Z [container-agent] 2024-03-04 15:07:57 INFO juju-log Starting the log rotate manager
2024-03-04T15:07:57.444Z [container-agent] 2024-03-04 15:07:57 INFO juju-log Started log rotate manager process with PID 841446
2024-03-04T15:07:57.719Z [container-agent] 2024-03-04 15:07:57 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T15:07:59.857Z [container-agent] 2024-03-04 15:07:59 WARNING juju-log No relation: certificates
2024-03-04T15:08:00.209Z [container-agent] 2024-03-04 15:08:00 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T15:12:44.909Z [container-agent] 2024-03-04 15:12:44 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T15:12:47.560Z [container-agent] 2024-03-04 15:12:47 WARNING juju-log No relation: certificates
2024-03-04T15:12:47.896Z [container-agent] 2024-03-04 15:12:47 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T15:17:23.040Z [container-agent] 2024-03-04 15:17:23 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T15:17:25.765Z [container-agent] 2024-03-04 15:17:25 WARNING juju-log No relation: certificates
2024-03-04T15:17:26.114Z [container-agent] 2024-03-04 15:17:26 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T15:22:29.881Z [container-agent] 2024-03-04 15:22:29 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T15:22:33.466Z [container-agent] 2024-03-04 15:22:33 WARNING juju-log No relation: certificates
2024-03-04T15:22:33.815Z [container-agent] 2024-03-04 15:22:33 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T15:26:41.285Z [container-agent] 2024-03-04 15:26:41 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T15:26:45.420Z [container-agent] 2024-03-04 15:26:45 WARNING juju-log No relation: certificates
2024-03-04T15:26:45.790Z [container-agent] 2024-03-04 15:26:45 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T15:31:52.301Z [container-agent] 2024-03-04 15:31:52 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T15:31:56.110Z [container-agent] 2024-03-04 15:31:56 WARNING juju-log No relation: certificates
2024-03-04T15:31:56.447Z [container-agent] 2024-03-04 15:31:56 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T15:36:18.112Z [container-agent] 2024-03-04 15:36:18 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T15:36:22.931Z [container-agent] 2024-03-04 15:36:22 WARNING juju-log No relation: certificates
2024-03-04T15:36:23.295Z [container-agent] 2024-03-04 15:36:23 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T15:41:23.241Z [container-agent] 2024-03-04 15:41:23 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T15:41:28.590Z [container-agent] 2024-03-04 15:41:28 WARNING juju-log No relation: certificates
2024-03-04T15:41:28.941Z [container-agent] 2024-03-04 15:41:28 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T15:46:42.851Z [container-agent] 2024-03-04 15:46:42 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T15:46:48.586Z [container-agent] 2024-03-04 15:46:48 WARNING juju-log No relation: certificates
2024-03-04T15:46:48.938Z [container-agent] 2024-03-04 15:46:48 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T15:52:10.741Z [container-agent] 2024-03-04 15:52:10 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T15:52:16.583Z [container-agent] 2024-03-04 15:52:16 WARNING juju-log No relation: certificates
2024-03-04T15:52:16.958Z [container-agent] 2024-03-04 15:52:16 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T15:57:38.318Z [container-agent] 2024-03-04 15:57:38 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T15:57:44.971Z [container-agent] 2024-03-04 15:57:44 WARNING juju-log No relation: certificates
2024-03-04T15:57:45.338Z [container-agent] 2024-03-04 15:57:45 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T16:03:35.437Z [container-agent] 2024-03-04 16:03:35 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T16:03:41.883Z [container-agent] 2024-03-04 16:03:41 WARNING juju-log No relation: certificates
2024-03-04T16:03:42.238Z [container-agent] 2024-03-04 16:03:42 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T16:08:28.732Z [container-agent] 2024-03-04 16:08:28 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T16:08:36.403Z [container-agent] 2024-03-04 16:08:36 WARNING juju-log No relation: certificates
2024-03-04T16:08:36.794Z [container-agent] 2024-03-04 16:08:36 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T16:13:14.272Z [container-agent] 2024-03-04 16:13:14 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T16:13:22.482Z [container-agent] 2024-03-04 16:13:22 WARNING juju-log No relation: certificates
2024-03-04T16:13:22.839Z [container-agent] 2024-03-04 16:13:22 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T16:17:25.638Z [container-agent] 2024-03-04 16:17:25 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T16:17:33.683Z [container-agent] 2024-03-04 16:17:33 WARNING juju-log No relation: certificates
2024-03-04T16:17:34.032Z [container-agent] 2024-03-04 16:17:34 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T16:22:18.131Z [container-agent] 2024-03-04 16:22:18 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-03-04T16:22:18.168Z [container-agent] 2024-03-04 16:22:18 INFO juju-log Attempting reboot from complete outage.
2024-03-04T16:22:26.224Z [container-agent] 2024-03-04 16:22:26 WARNING juju-log No relation: certificates
2024-03-04T16:22:26.563Z [container-agent] 2024-03-04 16:22:26 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T16:25:09.821Z [container-agent] 2024-03-04 16:25:09 ERROR juju-log Uncaught exception while in charm code:
2024-03-04T16:25:09.821Z [container-agent] Traceback (most recent call last):
2024-03-04T16:25:09.821Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/./src/charm.py", line 786, in <module>
2024-03-04T16:25:09.821Z [container-agent]     main(MySQLOperatorCharm)
2024-03-04T16:25:09.821Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 436, in main
2024-03-04T16:25:09.821Z [container-agent]     _emit_charm_event(charm, dispatcher.event_name)
2024-03-04T16:25:09.821Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 144, in _emit_charm_event
2024-03-04T16:25:09.821Z [container-agent]     event_to_emit.emit(*args, **kwargs)
2024-03-04T16:25:09.821Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 351, in emit
2024-03-04T16:25:09.821Z [container-agent]     framework._emit(event)
2024-03-04T16:25:09.821Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 853, in _emit
2024-03-04T16:25:09.821Z [container-agent]     self._reemit(event_path)
2024-03-04T16:25:09.821Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 942, in _reemit
2024-03-04T16:25:09.821Z [container-agent]     custom_handler(event)
2024-03-04T16:25:09.821Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/rotate_mysql_logs.py", line 57, in _rotate_mysql_logs
2024-03-04T16:25:09.821Z [container-agent]     self.charm._mysql.flush_mysql_logs(MySQLTextLogs.SLOW)
2024-03-04T16:25:09.821Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-03-04T16:25:09.821Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-03-04T16:25:09.821Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 665, in _run_mysqlsh_script
2024-03-04T16:25:09.821Z [container-agent]     process = self.container.exec(cmd, timeout=timeout)
2024-03-04T16:25:09.821Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/model.py", line 2688, in exec
2024-03-04T16:25:09.821Z [container-agent]     return self._pebble.exec(
2024-03-04T16:25:09.821Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 2501, in exec
2024-03-04T16:25:09.821Z [container-agent]     resp = self._request('POST', '/v1/exec', body=body)
2024-03-04T16:25:09.821Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1655, in _request
2024-03-04T16:25:09.821Z [container-agent]     response = self._request_raw(method, path, query, headers, data)
2024-03-04T16:25:09.821Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1690, in _request_raw
2024-03-04T16:25:09.821Z [container-agent]     response = self.opener.open(request, timeout=self.timeout)
2024-03-04T16:25:09.821Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 519, in open
2024-03-04T16:25:09.821Z [container-agent]     response = self._open(req, data)
2024-03-04T16:25:09.821Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 536, in _open
2024-03-04T16:25:09.821Z [container-agent]     result = self._call_chain(self.handle_open, protocol, protocol +
2024-03-04T16:25:09.821Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 496, in _call_chain
2024-03-04T16:25:09.821Z [container-agent]     result = func(*args)
2024-03-04T16:25:09.821Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 311, in http_open
2024-03-04T16:25:09.821Z [container-agent]     return self.do_open(_UnixSocketConnection, req,  # type:ignore
2024-03-04T16:25:09.821Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 1352, in do_open
2024-03-04T16:25:09.821Z [container-agent]     r = h.getresponse()
2024-03-04T16:25:09.821Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1375, in getresponse
2024-03-04T16:25:09.821Z [container-agent]     response.begin()
2024-03-04T16:25:09.821Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 318, in begin
2024-03-04T16:25:09.821Z [container-agent]     version, status, reason = self._read_status()
2024-03-04T16:25:09.821Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 287, in _read_status
2024-03-04T16:25:09.821Z [container-agent]     raise RemoteDisconnected("Remote end closed connection without"
2024-03-04T16:25:09.821Z [container-agent] http.client.RemoteDisconnected: Remote end closed connection without response
2024-03-04T16:25:09.988Z [container-agent] 2024-03-04 16:25:09 INFO juju.util.exec exec.go:209 run result: exit status 1
2024-03-04T16:25:15.616Z [container-agent] 2024-03-04 16:25:15 INFO juju-log Setting up the logrotate configurations
2024-03-04T16:25:15.695Z [container-agent] 2024-03-04 16:25:15 INFO juju-log Adding pebble layer
2024-03-04T16:25:22.489Z [container-agent] 2024-03-04 16:25:22 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-03-04T16:25:22.518Z [container-agent] 2024-03-04 16:25:22 INFO juju-log Attempting reboot from complete outage.
2024-03-04T16:25:25.450Z [container-agent] 2024-03-04 16:25:25 INFO juju.worker.uniter.operation runhook.go:186 ran "mysql-pebble-ready" hook (via hook dispatching script: dispatch)
2024-03-04T16:26:58.614Z [container-agent] 2024-03-04 16:26:58 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T16:27:00.509Z [container-agent] 2024-03-04 16:27:00 WARNING juju-log No relation: certificates
2024-03-04T16:27:00.860Z [container-agent] 2024-03-04 16:27:00 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T16:30:58.738Z [container-agent] 2024-03-04 16:30:58 INFO juju-log Starting the log rotate manager
2024-03-04T16:30:58.752Z [container-agent] 2024-03-04 16:30:58 INFO juju-log Started log rotate manager process with PID 849414
2024-03-04T16:30:59.038Z [container-agent] 2024-03-04 16:30:59 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T16:31:01.021Z [container-agent] 2024-03-04 16:31:01 WARNING juju-log No relation: certificates
2024-03-04T16:31:01.352Z [container-agent] 2024-03-04 16:31:01 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T16:35:12.512Z [container-agent] 2024-03-04 16:35:12 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T16:35:15.079Z [container-agent] 2024-03-04 16:35:15 WARNING juju-log No relation: certificates
2024-03-04T16:35:15.421Z [container-agent] 2024-03-04 16:35:15 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T16:39:35.305Z [container-agent] 2024-03-04 16:39:35 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T16:39:38.228Z [container-agent] 2024-03-04 16:39:38 WARNING juju-log No relation: certificates
2024-03-04T16:39:38.575Z [container-agent] 2024-03-04 16:39:38 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T16:45:20.229Z [container-agent] 2024-03-04 16:45:20 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T16:45:23.686Z [container-agent] 2024-03-04 16:45:23 WARNING juju-log No relation: certificates
2024-03-04T16:45:24.034Z [container-agent] 2024-03-04 16:45:24 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T16:49:27.503Z [container-agent] 2024-03-04 16:49:27 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T16:49:31.472Z [container-agent] 2024-03-04 16:49:31 WARNING juju-log No relation: certificates
2024-03-04T16:49:31.839Z [container-agent] 2024-03-04 16:49:31 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T16:54:46.521Z [container-agent] 2024-03-04 16:54:46 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T16:54:51.009Z [container-agent] 2024-03-04 16:54:51 WARNING juju-log No relation: certificates
2024-03-04T16:54:51.377Z [container-agent] 2024-03-04 16:54:51 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T17:00:44.571Z [container-agent] 2024-03-04 17:00:44 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T17:00:49.536Z [container-agent] 2024-03-04 17:00:49 WARNING juju-log No relation: certificates
2024-03-04T17:00:49.888Z [container-agent] 2024-03-04 17:00:49 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T17:05:09.458Z [container-agent] 2024-03-04 17:05:09 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T17:05:14.687Z [container-agent] 2024-03-04 17:05:14 WARNING juju-log No relation: certificates
2024-03-04T17:05:15.034Z [container-agent] 2024-03-04 17:05:15 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T17:09:14.451Z [container-agent] 2024-03-04 17:09:14 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T17:09:20.334Z [container-agent] 2024-03-04 17:09:20 WARNING juju-log No relation: certificates
2024-03-04T17:09:20.684Z [container-agent] 2024-03-04 17:09:20 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T17:14:57.279Z [container-agent] 2024-03-04 17:14:57 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T17:15:03.403Z [container-agent] 2024-03-04 17:15:03 WARNING juju-log No relation: certificates
2024-03-04T17:15:03.755Z [container-agent] 2024-03-04 17:15:03 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T17:20:24.061Z [container-agent] 2024-03-04 17:20:24 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T17:20:30.581Z [container-agent] 2024-03-04 17:20:30 WARNING juju-log No relation: certificates
2024-03-04T17:20:30.919Z [container-agent] 2024-03-04 17:20:30 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T17:25:57.468Z [container-agent] 2024-03-04 17:25:57 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T17:26:04.195Z [container-agent] 2024-03-04 17:26:04 WARNING juju-log No relation: certificates
2024-03-04T17:26:04.524Z [container-agent] 2024-03-04 17:26:04 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T17:31:39.378Z [container-agent] 2024-03-04 17:31:39 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T17:31:46.920Z [container-agent] 2024-03-04 17:31:46 WARNING juju-log No relation: certificates
2024-03-04T17:31:47.268Z [container-agent] 2024-03-04 17:31:47 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T17:37:14.528Z [container-agent] 2024-03-04 17:37:14 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T17:37:22.655Z [container-agent] 2024-03-04 17:37:22 WARNING juju-log No relation: certificates
2024-03-04T17:37:23.012Z [container-agent] 2024-03-04 17:37:23 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T17:42:45.163Z [container-agent] 2024-03-04 17:42:45 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-03-04T17:42:45.195Z [container-agent] 2024-03-04 17:42:45 INFO juju-log Attempting reboot from complete outage.
2024-03-04T17:42:53.169Z [container-agent] 2024-03-04 17:42:53 WARNING juju-log No relation: certificates
2024-03-04T17:42:53.517Z [container-agent] 2024-03-04 17:42:53 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T17:44:09.742Z [container-agent] 2024-03-04 17:44:09 ERROR juju-log Uncaught exception while in charm code:
2024-03-04T17:44:09.742Z [container-agent] Traceback (most recent call last):
2024-03-04T17:44:09.742Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/./src/charm.py", line 786, in <module>
2024-03-04T17:44:09.742Z [container-agent]     main(MySQLOperatorCharm)
2024-03-04T17:44:09.742Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 436, in main
2024-03-04T17:44:09.742Z [container-agent]     _emit_charm_event(charm, dispatcher.event_name)
2024-03-04T17:44:09.742Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 144, in _emit_charm_event
2024-03-04T17:44:09.742Z [container-agent]     event_to_emit.emit(*args, **kwargs)
2024-03-04T17:44:09.742Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 351, in emit
2024-03-04T17:44:09.742Z [container-agent]     framework._emit(event)
2024-03-04T17:44:09.742Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 853, in _emit
2024-03-04T17:44:09.742Z [container-agent]     self._reemit(event_path)
2024-03-04T17:44:09.742Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 942, in _reemit
2024-03-04T17:44:09.742Z [container-agent]     custom_handler(event)
2024-03-04T17:44:09.742Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/rotate_mysql_logs.py", line 57, in _rotate_mysql_logs
2024-03-04T17:44:09.742Z [container-agent]     self.charm._mysql.flush_mysql_logs(MySQLTextLogs.SLOW)
2024-03-04T17:44:09.742Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-03-04T17:44:09.742Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-03-04T17:44:09.742Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 665, in _run_mysqlsh_script
2024-03-04T17:44:09.742Z [container-agent]     process = self.container.exec(cmd, timeout=timeout)
2024-03-04T17:44:09.742Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/model.py", line 2688, in exec
2024-03-04T17:44:09.742Z [container-agent]     return self._pebble.exec(
2024-03-04T17:44:09.742Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 2501, in exec
2024-03-04T17:44:09.742Z [container-agent]     resp = self._request('POST', '/v1/exec', body=body)
2024-03-04T17:44:09.742Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1655, in _request
2024-03-04T17:44:09.742Z [container-agent]     response = self._request_raw(method, path, query, headers, data)
2024-03-04T17:44:09.742Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1690, in _request_raw
2024-03-04T17:44:09.742Z [container-agent]     response = self.opener.open(request, timeout=self.timeout)
2024-03-04T17:44:09.742Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 519, in open
2024-03-04T17:44:09.742Z [container-agent]     response = self._open(req, data)
2024-03-04T17:44:09.742Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 536, in _open
2024-03-04T17:44:09.742Z [container-agent]     result = self._call_chain(self.handle_open, protocol, protocol +
2024-03-04T17:44:09.742Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 496, in _call_chain
2024-03-04T17:44:09.742Z [container-agent]     result = func(*args)
2024-03-04T17:44:09.742Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 311, in http_open
2024-03-04T17:44:09.742Z [container-agent]     return self.do_open(_UnixSocketConnection, req,  # type:ignore
2024-03-04T17:44:09.742Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 1352, in do_open
2024-03-04T17:44:09.742Z [container-agent]     r = h.getresponse()
2024-03-04T17:44:09.742Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1375, in getresponse
2024-03-04T17:44:09.742Z [container-agent]     response.begin()
2024-03-04T17:44:09.742Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 318, in begin
2024-03-04T17:44:09.742Z [container-agent]     version, status, reason = self._read_status()
2024-03-04T17:44:09.742Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 287, in _read_status
2024-03-04T17:44:09.742Z [container-agent]     raise RemoteDisconnected("Remote end closed connection without"
2024-03-04T17:44:09.742Z [container-agent] http.client.RemoteDisconnected: Remote end closed connection without response
2024-03-04T17:44:09.908Z [container-agent] 2024-03-04 17:44:09 INFO juju.util.exec exec.go:209 run result: exit status 1
2024-03-04T17:44:13.057Z [container-agent] 2024-03-04 17:44:13 INFO juju-log Setting up the logrotate configurations
2024-03-04T17:44:13.133Z [container-agent] 2024-03-04 17:44:13 INFO juju-log Adding pebble layer
2024-03-04T17:44:19.919Z [container-agent] 2024-03-04 17:44:19 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-03-04T17:44:19.945Z [container-agent] 2024-03-04 17:44:19 INFO juju-log Attempting reboot from complete outage.
2024-03-04T17:44:22.713Z [container-agent] 2024-03-04 17:44:22 INFO juju.worker.uniter.operation runhook.go:186 ran "mysql-pebble-ready" hook (via hook dispatching script: dispatch)
2024-03-04T17:46:46.782Z [container-agent] 2024-03-04 17:46:46 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T17:46:48.766Z [container-agent] 2024-03-04 17:46:48 WARNING juju-log No relation: certificates
2024-03-04T17:46:49.140Z [container-agent] 2024-03-04 17:46:49 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T17:51:36.374Z [container-agent] 2024-03-04 17:51:36 INFO juju-log Starting the log rotate manager
2024-03-04T17:51:36.389Z [container-agent] 2024-03-04 17:51:36 INFO juju-log Started log rotate manager process with PID 856992
2024-03-04T17:51:36.703Z [container-agent] 2024-03-04 17:51:36 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T17:51:38.565Z [container-agent] 2024-03-04 17:51:38 WARNING juju-log No relation: certificates
2024-03-04T17:51:38.959Z [container-agent] 2024-03-04 17:51:38 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T17:56:29.006Z [container-agent] 2024-03-04 17:56:29 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T17:56:31.594Z [container-agent] 2024-03-04 17:56:31 WARNING juju-log No relation: certificates
2024-03-04T17:56:32.000Z [container-agent] 2024-03-04 17:56:31 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T18:02:06.972Z [container-agent] 2024-03-04 18:02:06 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T18:02:10.205Z [container-agent] 2024-03-04 18:02:10 WARNING juju-log No relation: certificates
2024-03-04T18:02:10.630Z [container-agent] 2024-03-04 18:02:10 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T18:06:37.855Z [container-agent] 2024-03-04 18:06:37 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T18:06:41.431Z [container-agent] 2024-03-04 18:06:41 WARNING juju-log No relation: certificates
2024-03-04T18:06:41.788Z [container-agent] 2024-03-04 18:06:41 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T18:12:15.172Z [container-agent] 2024-03-04 18:12:15 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T18:12:19.369Z [container-agent] 2024-03-04 18:12:19 WARNING juju-log No relation: certificates
2024-03-04T18:12:19.710Z [container-agent] 2024-03-04 18:12:19 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T18:17:58.525Z [container-agent] 2024-03-04 18:17:58 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T18:18:02.863Z [container-agent] 2024-03-04 18:18:02 WARNING juju-log No relation: certificates
2024-03-04T18:18:03.221Z [container-agent] 2024-03-04 18:18:03 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T18:22:43.245Z [container-agent] 2024-03-04 18:22:43 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T18:22:48.473Z [container-agent] 2024-03-04 18:22:48 WARNING juju-log No relation: certificates
2024-03-04T18:22:48.820Z [container-agent] 2024-03-04 18:22:48 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T18:27:53.247Z [container-agent] 2024-03-04 18:27:53 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T18:27:58.591Z [container-agent] 2024-03-04 18:27:58 WARNING juju-log No relation: certificates
2024-03-04T18:27:58.934Z [container-agent] 2024-03-04 18:27:58 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T18:32:32.740Z [container-agent] 2024-03-04 18:32:32 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T18:32:38.673Z [container-agent] 2024-03-04 18:32:38 WARNING juju-log No relation: certificates
2024-03-04T18:32:39.030Z [container-agent] 2024-03-04 18:32:39 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T18:37:51.552Z [container-agent] 2024-03-04 18:37:51 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T18:37:57.259Z [container-agent] 2024-03-04 18:37:57 WARNING juju-log No relation: certificates
2024-03-04T18:37:57.605Z [container-agent] 2024-03-04 18:37:57 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T18:43:49.872Z [container-agent] 2024-03-04 18:43:49 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T18:43:56.693Z [container-agent] 2024-03-04 18:43:56 WARNING juju-log No relation: certificates
2024-03-04T18:43:57.054Z [container-agent] 2024-03-04 18:43:57 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T18:48:18.821Z [container-agent] 2024-03-04 18:48:18 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T18:48:26.029Z [container-agent] 2024-03-04 18:48:26 WARNING juju-log No relation: certificates
2024-03-04T18:48:26.400Z [container-agent] 2024-03-04 18:48:26 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T18:54:00.380Z [container-agent] 2024-03-04 18:54:00 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T18:54:07.930Z [container-agent] 2024-03-04 18:54:07 WARNING juju-log No relation: certificates
2024-03-04T18:54:08.286Z [container-agent] 2024-03-04 18:54:08 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T18:59:26.534Z [container-agent] 2024-03-04 18:59:26 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T18:59:34.405Z [container-agent] 2024-03-04 18:59:34 WARNING juju-log No relation: certificates
2024-03-04T18:59:34.770Z [container-agent] 2024-03-04 18:59:34 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T19:03:45.413Z [container-agent] 2024-03-04 19:03:45 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-03-04T19:03:45.441Z [container-agent] 2024-03-04 19:03:45 INFO juju-log Attempting reboot from complete outage.
2024-03-04T19:03:53.425Z [container-agent] 2024-03-04 19:03:53 WARNING juju-log No relation: certificates
2024-03-04T19:03:53.806Z [container-agent] 2024-03-04 19:03:53 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T19:06:09.782Z [container-agent] 2024-03-04 19:06:09 ERROR juju-log Uncaught exception while in charm code:
2024-03-04T19:06:09.782Z [container-agent] Traceback (most recent call last):
2024-03-04T19:06:09.782Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/./src/charm.py", line 786, in <module>
2024-03-04T19:06:09.782Z [container-agent]     main(MySQLOperatorCharm)
2024-03-04T19:06:09.782Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 436, in main
2024-03-04T19:06:09.782Z [container-agent]     _emit_charm_event(charm, dispatcher.event_name)
2024-03-04T19:06:09.782Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 144, in _emit_charm_event
2024-03-04T19:06:09.782Z [container-agent]     event_to_emit.emit(*args, **kwargs)
2024-03-04T19:06:09.782Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 351, in emit
2024-03-04T19:06:09.782Z [container-agent]     framework._emit(event)
2024-03-04T19:06:09.782Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 853, in _emit
2024-03-04T19:06:09.782Z [container-agent]     self._reemit(event_path)
2024-03-04T19:06:09.782Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 942, in _reemit
2024-03-04T19:06:09.782Z [container-agent]     custom_handler(event)
2024-03-04T19:06:09.782Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/rotate_mysql_logs.py", line 57, in _rotate_mysql_logs
2024-03-04T19:06:09.782Z [container-agent]     self.charm._mysql.flush_mysql_logs(MySQLTextLogs.SLOW)
2024-03-04T19:06:09.782Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-03-04T19:06:09.782Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-03-04T19:06:09.782Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 665, in _run_mysqlsh_script
2024-03-04T19:06:09.782Z [container-agent]     process = self.container.exec(cmd, timeout=timeout)
2024-03-04T19:06:09.782Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/model.py", line 2688, in exec
2024-03-04T19:06:09.782Z [container-agent]     return self._pebble.exec(
2024-03-04T19:06:09.782Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 2501, in exec
2024-03-04T19:06:09.782Z [container-agent]     resp = self._request('POST', '/v1/exec', body=body)
2024-03-04T19:06:09.782Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1655, in _request
2024-03-04T19:06:09.782Z [container-agent]     response = self._request_raw(method, path, query, headers, data)
2024-03-04T19:06:09.782Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1690, in _request_raw
2024-03-04T19:06:09.782Z [container-agent]     response = self.opener.open(request, timeout=self.timeout)
2024-03-04T19:06:09.782Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 519, in open
2024-03-04T19:06:09.782Z [container-agent]     response = self._open(req, data)
2024-03-04T19:06:09.782Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 536, in _open
2024-03-04T19:06:09.782Z [container-agent]     result = self._call_chain(self.handle_open, protocol, protocol +
2024-03-04T19:06:09.782Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 496, in _call_chain
2024-03-04T19:06:09.782Z [container-agent]     result = func(*args)
2024-03-04T19:06:09.782Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 311, in http_open
2024-03-04T19:06:09.782Z [container-agent]     return self.do_open(_UnixSocketConnection, req,  # type:ignore
2024-03-04T19:06:09.782Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 1352, in do_open
2024-03-04T19:06:09.782Z [container-agent]     r = h.getresponse()
2024-03-04T19:06:09.782Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1375, in getresponse
2024-03-04T19:06:09.782Z [container-agent]     response.begin()
2024-03-04T19:06:09.782Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 318, in begin
2024-03-04T19:06:09.782Z [container-agent]     version, status, reason = self._read_status()
2024-03-04T19:06:09.782Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 287, in _read_status
2024-03-04T19:06:09.782Z [container-agent]     raise RemoteDisconnected("Remote end closed connection without"
2024-03-04T19:06:09.782Z [container-agent] http.client.RemoteDisconnected: Remote end closed connection without response
2024-03-04T19:06:09.899Z [container-agent] 2024-03-04 19:06:09 INFO juju.util.exec exec.go:209 run result: exit status 1
2024-03-04T19:06:11.908Z [container-agent] 2024-03-04 19:06:11 INFO juju-log Setting up the logrotate configurations
2024-03-04T19:06:11.976Z [container-agent] 2024-03-04 19:06:11 INFO juju-log Adding pebble layer
2024-03-04T19:06:18.743Z [container-agent] 2024-03-04 19:06:18 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-03-04T19:06:18.767Z [container-agent] 2024-03-04 19:06:18 INFO juju-log Attempting reboot from complete outage.
2024-03-04T19:06:21.762Z [container-agent] 2024-03-04 19:06:21 INFO juju.worker.uniter.operation runhook.go:186 ran "mysql-pebble-ready" hook (via hook dispatching script: dispatch)
2024-03-04T19:08:21.116Z [container-agent] 2024-03-04 19:08:21 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T19:08:22.979Z [container-agent] 2024-03-04 19:08:22 WARNING juju-log No relation: certificates
2024-03-04T19:08:23.332Z [container-agent] 2024-03-04 19:08:23 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T19:13:02.594Z [container-agent] 2024-03-04 19:13:02 INFO juju-log Starting the log rotate manager
2024-03-04T19:13:02.607Z [container-agent] 2024-03-04 19:13:02 INFO juju-log Started log rotate manager process with PID 864514
2024-03-04T19:13:02.956Z [container-agent] 2024-03-04 19:13:02 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T19:13:05.036Z [container-agent] 2024-03-04 19:13:05 WARNING juju-log No relation: certificates
2024-03-04T19:13:05.365Z [container-agent] 2024-03-04 19:13:05 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T19:18:20.559Z [container-agent] 2024-03-04 19:18:20 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T19:18:22.878Z [container-agent] 2024-03-04 19:18:22 WARNING juju-log No relation: certificates
2024-03-04T19:18:23.222Z [container-agent] 2024-03-04 19:18:23 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T19:23:45.168Z [container-agent] 2024-03-04 19:23:45 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T19:23:48.115Z [container-agent] 2024-03-04 19:23:48 WARNING juju-log No relation: certificates
2024-03-04T19:23:48.459Z [container-agent] 2024-03-04 19:23:48 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T19:29:19.291Z [container-agent] 2024-03-04 19:29:19 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T19:29:22.830Z [container-agent] 2024-03-04 19:29:22 WARNING juju-log No relation: certificates
2024-03-04T19:29:23.176Z [container-agent] 2024-03-04 19:29:23 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T19:33:25.082Z [container-agent] 2024-03-04 19:33:25 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T19:33:29.237Z [container-agent] 2024-03-04 19:33:29 WARNING juju-log No relation: certificates
2024-03-04T19:33:29.613Z [container-agent] 2024-03-04 19:33:29 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T19:37:47.325Z [container-agent] 2024-03-04 19:37:47 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T19:37:51.713Z [container-agent] 2024-03-04 19:37:51 WARNING juju-log No relation: certificates
2024-03-04T19:37:52.091Z [container-agent] 2024-03-04 19:37:52 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T19:42:10.999Z [container-agent] 2024-03-04 19:42:10 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T19:42:15.604Z [container-agent] 2024-03-04 19:42:15 WARNING juju-log No relation: certificates
2024-03-04T19:42:15.945Z [container-agent] 2024-03-04 19:42:15 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T19:46:42.994Z [container-agent] 2024-03-04 19:46:42 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T19:46:48.203Z [container-agent] 2024-03-04 19:46:48 WARNING juju-log No relation: certificates
2024-03-04T19:46:48.583Z [container-agent] 2024-03-04 19:46:48 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T19:52:10.529Z [container-agent] 2024-03-04 19:52:10 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T19:52:16.252Z [container-agent] 2024-03-04 19:52:16 WARNING juju-log No relation: certificates
2024-03-04T19:52:16.628Z [container-agent] 2024-03-04 19:52:16 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T19:56:17.313Z [container-agent] 2024-03-04 19:56:17 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T19:56:23.006Z [container-agent] 2024-03-04 19:56:23 WARNING juju-log No relation: certificates
2024-03-04T19:56:23.353Z [container-agent] 2024-03-04 19:56:23 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T20:00:36.229Z [container-agent] 2024-03-04 20:00:36 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T20:00:41.923Z [container-agent] 2024-03-04 20:00:41 WARNING juju-log No relation: certificates
2024-03-04T20:00:42.306Z [container-agent] 2024-03-04 20:00:42 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T20:05:40.290Z [container-agent] 2024-03-04 20:05:40 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T20:05:47.084Z [container-agent] 2024-03-04 20:05:47 WARNING juju-log No relation: certificates
2024-03-04T20:05:47.467Z [container-agent] 2024-03-04 20:05:47 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T20:10:45.777Z [container-agent] 2024-03-04 20:10:45 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T20:10:53.066Z [container-agent] 2024-03-04 20:10:53 WARNING juju-log No relation: certificates
2024-03-04T20:10:53.407Z [container-agent] 2024-03-04 20:10:53 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T20:16:20.428Z [container-agent] 2024-03-04 20:16:20 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T20:16:28.035Z [container-agent] 2024-03-04 20:16:28 WARNING juju-log No relation: certificates
2024-03-04T20:16:28.378Z [container-agent] 2024-03-04 20:16:28 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T20:22:14.295Z [container-agent] 2024-03-04 20:22:14 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T20:22:22.723Z [container-agent] 2024-03-04 20:22:22 WARNING juju-log No relation: certificates
2024-03-04T20:22:23.065Z [container-agent] 2024-03-04 20:22:23 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T20:25:07.844Z [container-agent] 2024-03-04 20:25:07 ERROR juju-log Failed to flush ERROR LOGS logs.
2024-03-04T20:25:07.844Z [container-agent] Traceback (most recent call last):
2024-03-04T20:25:07.844Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 666, in _run_mysqlsh_script
2024-03-04T20:25:07.844Z [container-agent]     stdout, _ = process.wait_output()
2024-03-04T20:25:07.844Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1441, in wait_output
2024-03-04T20:25:07.844Z [container-agent]     raise ExecError[AnyStr](self._command, exit_code, out_value, err_value)
2024-03-04T20:25:07.844Z [container-agent] ops.pebble.ExecError: non-zero exit code 1 executing ['/usr/bin/mysqlsh', '--no-wizard', '--python', '--verbose=1', '-f', '/tmp/script.py', ';', 'rm', '/tmp/script.py'], stdout='', stderr='Cannot set LC_ALL to locale en_US.UTF-8: No such file or directory\nverbose: 2024-03-04T20:25:06Z: Loading startup files...\nverbose: 2024-03-04T20:25:06Z: Loading plugins...\nverbose: 2024-03-04T20:25:06Z: Connecting to MySQL at: serverconfig@horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local\nTraceback (most recent call last):\n  File "<string>", line 1, in <module>\nmysqlsh.DBError: MySQL Error (2003): Shell.connect: Can\'t connect to MySQL server on \'horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local:3306\' (111)\n'
2024-03-04T20:25:07.844Z [container-agent] 
2024-03-04T20:25:07.844Z [container-agent] During handling of the above exception, another exception occurred:
2024-03-04T20:25:07.844Z [container-agent] 
2024-03-04T20:25:07.844Z [container-agent] Traceback (most recent call last):
2024-03-04T20:25:07.844Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-03-04T20:25:07.844Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-03-04T20:25:07.844Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 669, in _run_mysqlsh_script
2024-03-04T20:25:07.844Z [container-agent]     raise MySQLClientError(e.stderr)
2024-03-04T20:25:07.844Z [container-agent] charms.mysql.v0.mysql.MySQLClientError: Cannot set LC_ALL to locale en_US.UTF-8: No such file or directory
2024-03-04T20:25:07.844Z [container-agent] verbose: 2024-03-04T20:25:06Z: Loading startup files...
2024-03-04T20:25:07.844Z [container-agent] verbose: 2024-03-04T20:25:06Z: Loading plugins...
2024-03-04T20:25:07.844Z [container-agent] verbose: 2024-03-04T20:25:06Z: Connecting to MySQL at: serverconfig@horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local
2024-03-04T20:25:07.844Z [container-agent] Traceback (most recent call last):
2024-03-04T20:25:07.844Z [container-agent]   File "<string>", line 1, in <module>
2024-03-04T20:25:07.844Z [container-agent] mysqlsh.DBError: MySQL Error (2003): Shell.connect: Can't connect to MySQL server on 'horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local:3306' (111)
2024-03-04T20:25:07.844Z [container-agent] 
2024-03-04T20:26:39.930Z [container-agent] 2024-03-04 20:26:39 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-03-04T20:26:39.961Z [container-agent] 2024-03-04 20:26:39 INFO juju-log Attempting reboot from complete outage.
2024-03-04T20:26:47.722Z [container-agent] 2024-03-04 20:26:47 WARNING juju-log No relation: certificates
2024-03-04T20:26:48.081Z [container-agent] 2024-03-04 20:26:48 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T20:30:04.798Z [container-agent] 2024-03-04 20:30:04 ERROR juju-log Uncaught exception while in charm code:
2024-03-04T20:30:04.798Z [container-agent] Traceback (most recent call last):
2024-03-04T20:30:04.798Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/./src/charm.py", line 786, in <module>
2024-03-04T20:30:04.798Z [container-agent]     main(MySQLOperatorCharm)
2024-03-04T20:30:04.798Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 436, in main
2024-03-04T20:30:04.798Z [container-agent]     _emit_charm_event(charm, dispatcher.event_name)
2024-03-04T20:30:04.798Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 144, in _emit_charm_event
2024-03-04T20:30:04.798Z [container-agent]     event_to_emit.emit(*args, **kwargs)
2024-03-04T20:30:04.798Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 351, in emit
2024-03-04T20:30:04.798Z [container-agent]     framework._emit(event)
2024-03-04T20:30:04.798Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 853, in _emit
2024-03-04T20:30:04.798Z [container-agent]     self._reemit(event_path)
2024-03-04T20:30:04.798Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 942, in _reemit
2024-03-04T20:30:04.798Z [container-agent]     custom_handler(event)
2024-03-04T20:30:04.798Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/rotate_mysql_logs.py", line 55, in _rotate_mysql_logs
2024-03-04T20:30:04.798Z [container-agent]     self.charm._mysql.flush_mysql_logs(MySQLTextLogs.ERROR)
2024-03-04T20:30:04.798Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-03-04T20:30:04.798Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-03-04T20:30:04.798Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 665, in _run_mysqlsh_script
2024-03-04T20:30:04.798Z [container-agent]     process = self.container.exec(cmd, timeout=timeout)
2024-03-04T20:30:04.798Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/model.py", line 2688, in exec
2024-03-04T20:30:04.798Z [container-agent]     return self._pebble.exec(
2024-03-04T20:30:04.798Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 2501, in exec
2024-03-04T20:30:04.798Z [container-agent]     resp = self._request('POST', '/v1/exec', body=body)
2024-03-04T20:30:04.798Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1655, in _request
2024-03-04T20:30:04.798Z [container-agent]     response = self._request_raw(method, path, query, headers, data)
2024-03-04T20:30:04.798Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1690, in _request_raw
2024-03-04T20:30:04.798Z [container-agent]     response = self.opener.open(request, timeout=self.timeout)
2024-03-04T20:30:04.798Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 519, in open
2024-03-04T20:30:04.798Z [container-agent]     response = self._open(req, data)
2024-03-04T20:30:04.798Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 536, in _open
2024-03-04T20:30:04.798Z [container-agent]     result = self._call_chain(self.handle_open, protocol, protocol +
2024-03-04T20:30:04.798Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 496, in _call_chain
2024-03-04T20:30:04.798Z [container-agent]     result = func(*args)
2024-03-04T20:30:04.798Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 311, in http_open
2024-03-04T20:30:04.798Z [container-agent]     return self.do_open(_UnixSocketConnection, req,  # type:ignore
2024-03-04T20:30:04.798Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 1352, in do_open
2024-03-04T20:30:04.798Z [container-agent]     r = h.getresponse()
2024-03-04T20:30:04.798Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1375, in getresponse
2024-03-04T20:30:04.798Z [container-agent]     response.begin()
2024-03-04T20:30:04.798Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 318, in begin
2024-03-04T20:30:04.798Z [container-agent]     version, status, reason = self._read_status()
2024-03-04T20:30:04.798Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 287, in _read_status
2024-03-04T20:30:04.798Z [container-agent]     raise RemoteDisconnected("Remote end closed connection without"
2024-03-04T20:30:04.798Z [container-agent] http.client.RemoteDisconnected: Remote end closed connection without response
2024-03-04T20:30:04.988Z [container-agent] 2024-03-04 20:30:04 INFO juju.util.exec exec.go:209 run result: exit status 1
2024-03-04T20:30:09.524Z [container-agent] 2024-03-04 20:30:09 INFO juju-log Setting up the logrotate configurations
2024-03-04T20:30:09.612Z [container-agent] 2024-03-04 20:30:09 INFO juju-log Adding pebble layer
2024-03-04T20:30:16.348Z [container-agent] 2024-03-04 20:30:16 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-03-04T20:30:16.377Z [container-agent] 2024-03-04 20:30:16 INFO juju-log Attempting reboot from complete outage.
2024-03-04T20:30:19.381Z [container-agent] 2024-03-04 20:30:19 INFO juju.worker.uniter.operation runhook.go:186 ran "mysql-pebble-ready" hook (via hook dispatching script: dispatch)
2024-03-04T20:31:54.910Z [container-agent] 2024-03-04 20:31:54 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T20:31:56.796Z [container-agent] 2024-03-04 20:31:56 WARNING juju-log No relation: certificates
2024-03-04T20:31:57.145Z [container-agent] 2024-03-04 20:31:57 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T20:36:06.474Z [container-agent] 2024-03-04 20:36:06 INFO juju-log Starting the log rotate manager
2024-03-04T20:36:06.490Z [container-agent] 2024-03-04 20:36:06 INFO juju-log Started log rotate manager process with PID 872385
2024-03-04T20:36:06.854Z [container-agent] 2024-03-04 20:36:06 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T20:36:08.886Z [container-agent] 2024-03-04 20:36:08 WARNING juju-log No relation: certificates
2024-03-04T20:36:09.249Z [container-agent] 2024-03-04 20:36:09 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T20:41:30.462Z [container-agent] 2024-03-04 20:41:30 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T20:41:32.969Z [container-agent] 2024-03-04 20:41:32 WARNING juju-log No relation: certificates
2024-03-04T20:41:33.363Z [container-agent] 2024-03-04 20:41:33 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T20:47:23.414Z [container-agent] 2024-03-04 20:47:23 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T20:47:26.257Z [container-agent] 2024-03-04 20:47:26 WARNING juju-log No relation: certificates
2024-03-04T20:47:26.601Z [container-agent] 2024-03-04 20:47:26 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T20:51:50.903Z [container-agent] 2024-03-04 20:51:50 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T20:51:54.074Z [container-agent] 2024-03-04 20:51:54 WARNING juju-log No relation: certificates
2024-03-04T20:51:54.421Z [container-agent] 2024-03-04 20:51:54 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T20:56:07.832Z [container-agent] 2024-03-04 20:56:07 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T20:56:11.638Z [container-agent] 2024-03-04 20:56:11 WARNING juju-log No relation: certificates
2024-03-04T20:56:12.004Z [container-agent] 2024-03-04 20:56:12 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T21:01:49.324Z [container-agent] 2024-03-04 21:01:49 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T21:01:53.637Z [container-agent] 2024-03-04 21:01:53 WARNING juju-log No relation: certificates
2024-03-04T21:01:54.000Z [container-agent] 2024-03-04 21:01:54 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T21:06:19.671Z [container-agent] 2024-03-04 21:06:19 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T21:06:24.181Z [container-agent] 2024-03-04 21:06:24 WARNING juju-log No relation: certificates
2024-03-04T21:06:24.513Z [container-agent] 2024-03-04 21:06:24 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T21:10:40.112Z [container-agent] 2024-03-04 21:10:40 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T21:10:45.402Z [container-agent] 2024-03-04 21:10:45 WARNING juju-log No relation: certificates
2024-03-04T21:10:45.769Z [container-agent] 2024-03-04 21:10:45 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T21:14:56.149Z [container-agent] 2024-03-04 21:14:56 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T21:15:01.497Z [container-agent] 2024-03-04 21:15:01 WARNING juju-log No relation: certificates
2024-03-04T21:15:01.861Z [container-agent] 2024-03-04 21:15:01 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T21:20:32.752Z [container-agent] 2024-03-04 21:20:32 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T21:20:38.807Z [container-agent] 2024-03-04 21:20:38 WARNING juju-log No relation: certificates
2024-03-04T21:20:39.148Z [container-agent] 2024-03-04 21:20:39 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T21:25:29.155Z [container-agent] 2024-03-04 21:25:29 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T21:25:36.002Z [container-agent] 2024-03-04 21:25:36 WARNING juju-log No relation: certificates
2024-03-04T21:25:36.351Z [container-agent] 2024-03-04 21:25:36 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T21:30:42.602Z [container-agent] 2024-03-04 21:30:42 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T21:30:49.581Z [container-agent] 2024-03-04 21:30:49 WARNING juju-log No relation: certificates
2024-03-04T21:30:49.960Z [container-agent] 2024-03-04 21:30:49 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T21:35:51.699Z [container-agent] 2024-03-04 21:35:51 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T21:35:58.973Z [container-agent] 2024-03-04 21:35:58 WARNING juju-log No relation: certificates
2024-03-04T21:35:59.321Z [container-agent] 2024-03-04 21:35:59 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T21:40:25.572Z [container-agent] 2024-03-04 21:40:25 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T21:40:32.918Z [container-agent] 2024-03-04 21:40:32 WARNING juju-log No relation: certificates
2024-03-04T21:40:33.272Z [container-agent] 2024-03-04 21:40:33 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T21:45:14.838Z [container-agent] 2024-03-04 21:45:14 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T21:45:23.255Z [container-agent] 2024-03-04 21:45:23 WARNING juju-log No relation: certificates
2024-03-04T21:45:23.627Z [container-agent] 2024-03-04 21:45:23 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T21:50:47.939Z [container-agent] 2024-03-04 21:50:47 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-03-04T21:50:47.990Z [container-agent] 2024-03-04 21:50:47 INFO juju-log Attempting reboot from complete outage.
2024-03-04T21:50:55.699Z [container-agent] 2024-03-04 21:50:55 WARNING juju-log No relation: certificates
2024-03-04T21:50:56.105Z [container-agent] 2024-03-04 21:50:56 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T21:51:04.767Z [container-agent] 2024-03-04 21:51:04 ERROR juju-log Uncaught exception while in charm code:
2024-03-04T21:51:04.767Z [container-agent] Traceback (most recent call last):
2024-03-04T21:51:04.767Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 2507, in exec
2024-03-04T21:51:04.767Z [container-agent]     control_ws = self._connect_websocket(task_id, 'control')
2024-03-04T21:51:04.767Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 2589, in _connect_websocket
2024-03-04T21:51:04.767Z [container-agent]     ws.connect(url, socket=sock)
2024-03-04T21:51:04.767Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_core.py", line 253, in connect
2024-03-04T21:51:04.767Z [container-agent]     self.handshake_response = handshake(self.sock, url, *addrs, **options)
2024-03-04T21:51:04.767Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_handshake.py", line 57, in handshake
2024-03-04T21:51:04.767Z [container-agent]     status, resp = _get_resp_headers(sock)
2024-03-04T21:51:04.767Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_handshake.py", line 143, in _get_resp_headers
2024-03-04T21:51:04.767Z [container-agent]     status, resp_headers, status_message = read_headers(sock)
2024-03-04T21:51:04.767Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_http.py", line 312, in read_headers
2024-03-04T21:51:04.767Z [container-agent]     line = recv_line(sock)
2024-03-04T21:51:04.767Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_socket.py", line 131, in recv_line
2024-03-04T21:51:04.767Z [container-agent]     c = recv(sock, 1)
2024-03-04T21:51:04.767Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_socket.py", line 122, in recv
2024-03-04T21:51:04.767Z [container-agent]     raise WebSocketConnectionClosedException(
2024-03-04T21:51:04.767Z [container-agent] websocket._exceptions.WebSocketConnectionClosedException: Connection to remote host was lost.
2024-03-04T21:51:04.767Z [container-agent] 
2024-03-04T21:51:04.767Z [container-agent] During handling of the above exception, another exception occurred:
2024-03-04T21:51:04.767Z [container-agent] 
2024-03-04T21:51:04.767Z [container-agent] Traceback (most recent call last):
2024-03-04T21:51:04.767Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 1348, in do_open
2024-03-04T21:51:04.767Z [container-agent]     h.request(req.get_method(), req.selector, req.data, headers,
2024-03-04T21:51:04.767Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1283, in request
2024-03-04T21:51:04.767Z [container-agent]     self._send_request(method, url, body, headers, encode_chunked)
2024-03-04T21:51:04.767Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1329, in _send_request
2024-03-04T21:51:04.767Z [container-agent]     self.endheaders(body, encode_chunked=encode_chunked)
2024-03-04T21:51:04.767Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1278, in endheaders
2024-03-04T21:51:04.767Z [container-agent]     self._send_output(message_body, encode_chunked=encode_chunked)
2024-03-04T21:51:04.767Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1038, in _send_output
2024-03-04T21:51:04.767Z [container-agent]     self.send(msg)
2024-03-04T21:51:04.767Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 976, in send
2024-03-04T21:51:04.767Z [container-agent]     self.connect()
2024-03-04T21:51:04.767Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 297, in connect
2024-03-04T21:51:04.767Z [container-agent]     self.sock.connect(self.socket_path)
2024-03-04T21:51:04.767Z [container-agent] ConnectionRefusedError: [Errno 111] Connection refused
2024-03-04T21:51:04.767Z [container-agent] 
2024-03-04T21:51:04.767Z [container-agent] During handling of the above exception, another exception occurred:
2024-03-04T21:51:04.767Z [container-agent] 
2024-03-04T21:51:04.767Z [container-agent] Traceback (most recent call last):
2024-03-04T21:51:04.767Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1690, in _request_raw
2024-03-04T21:51:04.767Z [container-agent]     response = self.opener.open(request, timeout=self.timeout)
2024-03-04T21:51:04.767Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 519, in open
2024-03-04T21:51:04.767Z [container-agent]     response = self._open(req, data)
2024-03-04T21:51:04.767Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 536, in _open
2024-03-04T21:51:04.767Z [container-agent]     result = self._call_chain(self.handle_open, protocol, protocol +
2024-03-04T21:51:04.767Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 496, in _call_chain
2024-03-04T21:51:04.767Z [container-agent]     result = func(*args)
2024-03-04T21:51:04.767Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 311, in http_open
2024-03-04T21:51:04.767Z [container-agent]     return self.do_open(_UnixSocketConnection, req,  # type:ignore
2024-03-04T21:51:04.767Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 1351, in do_open
2024-03-04T21:51:04.767Z [container-agent]     raise URLError(err)
2024-03-04T21:51:04.767Z [container-agent] urllib.error.URLError: <urlopen error [Errno 111] Connection refused>
2024-03-04T21:51:04.767Z [container-agent] 
2024-03-04T21:51:04.767Z [container-agent] The above exception was the direct cause of the following exception:
2024-03-04T21:51:04.767Z [container-agent] 
2024-03-04T21:51:04.767Z [container-agent] Traceback (most recent call last):
2024-03-04T21:51:04.767Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/./src/charm.py", line 786, in <module>
2024-03-04T21:51:04.767Z [container-agent]     main(MySQLOperatorCharm)
2024-03-04T21:51:04.767Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 436, in main
2024-03-04T21:51:04.767Z [container-agent]     _emit_charm_event(charm, dispatcher.event_name)
2024-03-04T21:51:04.767Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 144, in _emit_charm_event
2024-03-04T21:51:04.767Z [container-agent]     event_to_emit.emit(*args, **kwargs)
2024-03-04T21:51:04.767Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 351, in emit
2024-03-04T21:51:04.767Z [container-agent]     framework._emit(event)
2024-03-04T21:51:04.767Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 853, in _emit
2024-03-04T21:51:04.767Z [container-agent]     self._reemit(event_path)
2024-03-04T21:51:04.767Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 942, in _reemit
2024-03-04T21:51:04.767Z [container-agent]     custom_handler(event)
2024-03-04T21:51:04.767Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/rotate_mysql_logs.py", line 55, in _rotate_mysql_logs
2024-03-04T21:51:04.767Z [container-agent]     self.charm._mysql.flush_mysql_logs(MySQLTextLogs.ERROR)
2024-03-04T21:51:04.767Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-03-04T21:51:04.767Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-03-04T21:51:04.767Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 665, in _run_mysqlsh_script
2024-03-04T21:51:04.767Z [container-agent]     process = self.container.exec(cmd, timeout=timeout)
2024-03-04T21:51:04.767Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/model.py", line 2688, in exec
2024-03-04T21:51:04.767Z [container-agent]     return self._pebble.exec(
2024-03-04T21:51:04.767Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 2514, in exec
2024-03-04T21:51:04.767Z [container-agent]     change = self.wait_change(ChangeID(change_id))
2024-03-04T21:51:04.767Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1893, in wait_change
2024-03-04T21:51:04.767Z [container-agent]     return self._wait_change_using_wait(change_id, timeout)
2024-03-04T21:51:04.767Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1914, in _wait_change_using_wait
2024-03-04T21:51:04.767Z [container-agent]     return self._wait_change(change_id, this_timeout)
2024-03-04T21:51:04.767Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1928, in _wait_change
2024-03-04T21:51:04.767Z [container-agent]     resp = self._request('GET', f'/v1/changes/{change_id}/wait', query)
2024-03-04T21:51:04.767Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1655, in _request
2024-03-04T21:51:04.767Z [container-agent]     response = self._request_raw(method, path, query, headers, data)
2024-03-04T21:51:04.767Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1707, in _request_raw
2024-03-04T21:51:04.767Z [container-agent]     raise ConnectionError(e.reason) from e
2024-03-04T21:51:04.767Z [container-agent] ops.pebble.ConnectionError: [Errno 111] Connection refused
2024-03-04T21:51:04.906Z [container-agent] 2024-03-04 21:51:04 INFO juju.util.exec exec.go:209 run result: exit status 1
2024-03-04T21:51:07.460Z [container-agent] 2024-03-04 21:51:07 INFO juju-log Setting up the logrotate configurations
2024-03-04T21:51:07.526Z [container-agent] 2024-03-04 21:51:07 INFO juju-log Adding pebble layer
2024-03-04T21:51:14.283Z [container-agent] 2024-03-04 21:51:14 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-03-04T21:51:14.306Z [container-agent] 2024-03-04 21:51:14 INFO juju-log Attempting reboot from complete outage.
2024-03-04T21:51:17.311Z [container-agent] 2024-03-04 21:51:17 INFO juju.worker.uniter.operation runhook.go:186 ran "mysql-pebble-ready" hook (via hook dispatching script: dispatch)
2024-03-04T21:56:45.725Z [container-agent] 2024-03-04 21:56:45 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T21:56:47.600Z [container-agent] 2024-03-04 21:56:47 WARNING juju-log No relation: certificates
2024-03-04T21:56:47.962Z [container-agent] 2024-03-04 21:56:47 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T22:01:58.710Z [container-agent] 2024-03-04 22:01:58 INFO juju-log Starting the log rotate manager
2024-03-04T22:01:58.723Z [container-agent] 2024-03-04 22:01:58 INFO juju-log Started log rotate manager process with PID 880101
2024-03-04T22:01:59.006Z [container-agent] 2024-03-04 22:01:59 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T22:02:00.978Z [container-agent] 2024-03-04 22:02:00 WARNING juju-log No relation: certificates
2024-03-04T22:02:01.350Z [container-agent] 2024-03-04 22:02:01 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T22:06:28.686Z [container-agent] 2024-03-04 22:06:28 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T22:06:31.467Z [container-agent] 2024-03-04 22:06:31 WARNING juju-log No relation: certificates
2024-03-04T22:06:31.873Z [container-agent] 2024-03-04 22:06:31 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T22:10:53.029Z [container-agent] 2024-03-04 22:10:53 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T22:10:55.811Z [container-agent] 2024-03-04 22:10:55 WARNING juju-log No relation: certificates
2024-03-04T22:10:56.174Z [container-agent] 2024-03-04 22:10:56 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T22:15:43.484Z [container-agent] 2024-03-04 22:15:43 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T22:15:47.025Z [container-agent] 2024-03-04 22:15:47 WARNING juju-log No relation: certificates
2024-03-04T22:15:47.390Z [container-agent] 2024-03-04 22:15:47 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T22:19:56.061Z [container-agent] 2024-03-04 22:19:56 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T22:19:59.932Z [container-agent] 2024-03-04 22:19:59 WARNING juju-log No relation: certificates
2024-03-04T22:20:00.329Z [container-agent] 2024-03-04 22:20:00 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T22:25:19.932Z [container-agent] 2024-03-04 22:25:19 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T22:25:24.016Z [container-agent] 2024-03-04 22:25:24 WARNING juju-log No relation: certificates
2024-03-04T22:25:24.360Z [container-agent] 2024-03-04 22:25:24 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T22:29:27.287Z [container-agent] 2024-03-04 22:29:27 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T22:29:32.026Z [container-agent] 2024-03-04 22:29:32 WARNING juju-log No relation: certificates
2024-03-04T22:29:32.376Z [container-agent] 2024-03-04 22:29:32 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T22:34:46.350Z [container-agent] 2024-03-04 22:34:46 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T22:34:51.558Z [container-agent] 2024-03-04 22:34:51 WARNING juju-log No relation: certificates
2024-03-04T22:34:51.897Z [container-agent] 2024-03-04 22:34:51 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T22:39:13.531Z [container-agent] 2024-03-04 22:39:13 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T22:39:19.309Z [container-agent] 2024-03-04 22:39:19 WARNING juju-log No relation: certificates
2024-03-04T22:39:19.672Z [container-agent] 2024-03-04 22:39:19 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T22:43:53.446Z [container-agent] 2024-03-04 22:43:53 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T22:43:59.446Z [container-agent] 2024-03-04 22:43:59 WARNING juju-log No relation: certificates
2024-03-04T22:43:59.794Z [container-agent] 2024-03-04 22:43:59 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T22:49:11.798Z [container-agent] 2024-03-04 22:49:11 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T22:49:18.335Z [container-agent] 2024-03-04 22:49:18 WARNING juju-log No relation: certificates
2024-03-04T22:49:18.684Z [container-agent] 2024-03-04 22:49:18 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T22:53:22.005Z [container-agent] 2024-03-04 22:53:22 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T22:53:28.477Z [container-agent] 2024-03-04 22:53:28 WARNING juju-log No relation: certificates
2024-03-04T22:53:28.831Z [container-agent] 2024-03-04 22:53:28 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T22:59:18.543Z [container-agent] 2024-03-04 22:59:18 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T22:59:25.895Z [container-agent] 2024-03-04 22:59:25 WARNING juju-log No relation: certificates
2024-03-04T22:59:26.268Z [container-agent] 2024-03-04 22:59:26 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T23:03:07.653Z [container-agent] 2024-03-04 23:03:07 ERROR juju-log Failed to flush ERROR LOGS logs.
2024-03-04T23:03:07.653Z [container-agent] Traceback (most recent call last):
2024-03-04T23:03:07.653Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 666, in _run_mysqlsh_script
2024-03-04T23:03:07.653Z [container-agent]     stdout, _ = process.wait_output()
2024-03-04T23:03:07.653Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1441, in wait_output
2024-03-04T23:03:07.653Z [container-agent]     raise ExecError[AnyStr](self._command, exit_code, out_value, err_value)
2024-03-04T23:03:07.653Z [container-agent] ops.pebble.ExecError: non-zero exit code 1 executing ['/usr/bin/mysqlsh', '--no-wizard', '--python', '--verbose=1', '-f', '/tmp/script.py', ';', 'rm', '/tmp/script.py'], stdout='', stderr='Cannot set LC_ALL to locale en_US.UTF-8: No such file or directory\nverbose: 2024-03-04T23:03:06Z: Loading startup files...\nverbose: 2024-03-04T23:03:06Z: Loading plugins...\nverbose: 2024-03-04T23:03:06Z: Connecting to MySQL at: serverconfig@horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local\nTraceback (most recent call last):\n  File "<string>", line 1, in <module>\nmysqlsh.DBError: MySQL Error (2003): Shell.connect: Can\'t connect to MySQL server on \'horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local:3306\' (111)\n'
2024-03-04T23:03:07.653Z [container-agent] 
2024-03-04T23:03:07.653Z [container-agent] During handling of the above exception, another exception occurred:
2024-03-04T23:03:07.653Z [container-agent] 
2024-03-04T23:03:07.653Z [container-agent] Traceback (most recent call last):
2024-03-04T23:03:07.653Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-03-04T23:03:07.653Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-03-04T23:03:07.653Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 669, in _run_mysqlsh_script
2024-03-04T23:03:07.653Z [container-agent]     raise MySQLClientError(e.stderr)
2024-03-04T23:03:07.653Z [container-agent] charms.mysql.v0.mysql.MySQLClientError: Cannot set LC_ALL to locale en_US.UTF-8: No such file or directory
2024-03-04T23:03:07.653Z [container-agent] verbose: 2024-03-04T23:03:06Z: Loading startup files...
2024-03-04T23:03:07.653Z [container-agent] verbose: 2024-03-04T23:03:06Z: Loading plugins...
2024-03-04T23:03:07.653Z [container-agent] verbose: 2024-03-04T23:03:06Z: Connecting to MySQL at: serverconfig@horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local
2024-03-04T23:03:07.653Z [container-agent] Traceback (most recent call last):
2024-03-04T23:03:07.653Z [container-agent]   File "<string>", line 1, in <module>
2024-03-04T23:03:07.653Z [container-agent] mysqlsh.DBError: MySQL Error (2003): Shell.connect: Can't connect to MySQL server on 'horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local:3306' (111)
2024-03-04T23:03:07.653Z [container-agent] 
2024-03-04T23:03:49.659Z [container-agent] 2024-03-04 23:03:49 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-03-04T23:03:49.689Z [container-agent] 2024-03-04 23:03:49 INFO juju-log Attempting reboot from complete outage.
2024-03-04T23:03:56.967Z [container-agent] 2024-03-04 23:03:56 WARNING juju-log No relation: certificates
2024-03-04T23:03:57.329Z [container-agent] 2024-03-04 23:03:57 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T23:08:04.793Z [container-agent] 2024-03-04 23:08:04 ERROR juju-log Uncaught exception while in charm code:
2024-03-04T23:08:04.793Z [container-agent] Traceback (most recent call last):
2024-03-04T23:08:04.793Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/./src/charm.py", line 786, in <module>
2024-03-04T23:08:04.793Z [container-agent]     main(MySQLOperatorCharm)
2024-03-04T23:08:04.793Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 436, in main
2024-03-04T23:08:04.793Z [container-agent]     _emit_charm_event(charm, dispatcher.event_name)
2024-03-04T23:08:04.793Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 144, in _emit_charm_event
2024-03-04T23:08:04.793Z [container-agent]     event_to_emit.emit(*args, **kwargs)
2024-03-04T23:08:04.793Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 351, in emit
2024-03-04T23:08:04.793Z [container-agent]     framework._emit(event)
2024-03-04T23:08:04.793Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 853, in _emit
2024-03-04T23:08:04.793Z [container-agent]     self._reemit(event_path)
2024-03-04T23:08:04.793Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 942, in _reemit
2024-03-04T23:08:04.793Z [container-agent]     custom_handler(event)
2024-03-04T23:08:04.793Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/rotate_mysql_logs.py", line 55, in _rotate_mysql_logs
2024-03-04T23:08:04.793Z [container-agent]     self.charm._mysql.flush_mysql_logs(MySQLTextLogs.ERROR)
2024-03-04T23:08:04.793Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-03-04T23:08:04.793Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-03-04T23:08:04.793Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 666, in _run_mysqlsh_script
2024-03-04T23:08:04.793Z [container-agent]     stdout, _ = process.wait_output()
2024-03-04T23:08:04.793Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1436, in wait_output
2024-03-04T23:08:04.793Z [container-agent]     exit_code: int = self._wait()
2024-03-04T23:08:04.793Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1375, in _wait
2024-03-04T23:08:04.793Z [container-agent]     change = self._client.wait_change(self._change_id, timeout=timeout)
2024-03-04T23:08:04.793Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1893, in wait_change
2024-03-04T23:08:04.793Z [container-agent]     return self._wait_change_using_wait(change_id, timeout)
2024-03-04T23:08:04.793Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1914, in _wait_change_using_wait
2024-03-04T23:08:04.793Z [container-agent]     return self._wait_change(change_id, this_timeout)
2024-03-04T23:08:04.793Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1928, in _wait_change
2024-03-04T23:08:04.793Z [container-agent]     resp = self._request('GET', f'/v1/changes/{change_id}/wait', query)
2024-03-04T23:08:04.793Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1655, in _request
2024-03-04T23:08:04.793Z [container-agent]     response = self._request_raw(method, path, query, headers, data)
2024-03-04T23:08:04.793Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1690, in _request_raw
2024-03-04T23:08:04.793Z [container-agent]     response = self.opener.open(request, timeout=self.timeout)
2024-03-04T23:08:04.793Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 519, in open
2024-03-04T23:08:04.793Z [container-agent]     response = self._open(req, data)
2024-03-04T23:08:04.793Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 536, in _open
2024-03-04T23:08:04.793Z [container-agent]     result = self._call_chain(self.handle_open, protocol, protocol +
2024-03-04T23:08:04.793Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 496, in _call_chain
2024-03-04T23:08:04.793Z [container-agent]     result = func(*args)
2024-03-04T23:08:04.793Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 311, in http_open
2024-03-04T23:08:04.793Z [container-agent]     return self.do_open(_UnixSocketConnection, req,  # type:ignore
2024-03-04T23:08:04.793Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 1352, in do_open
2024-03-04T23:08:04.793Z [container-agent]     r = h.getresponse()
2024-03-04T23:08:04.793Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1375, in getresponse
2024-03-04T23:08:04.793Z [container-agent]     response.begin()
2024-03-04T23:08:04.793Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 318, in begin
2024-03-04T23:08:04.793Z [container-agent]     version, status, reason = self._read_status()
2024-03-04T23:08:04.793Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 287, in _read_status
2024-03-04T23:08:04.793Z [container-agent]     raise RemoteDisconnected("Remote end closed connection without"
2024-03-04T23:08:04.793Z [container-agent] http.client.RemoteDisconnected: Remote end closed connection without response
2024-03-04T23:08:04.987Z [container-agent] 2024-03-04 23:08:04 INFO juju.util.exec exec.go:209 run result: exit status 1
2024-03-04T23:08:10.136Z [container-agent] 2024-03-04 23:08:10 INFO juju-log Setting up the logrotate configurations
2024-03-04T23:08:10.234Z [container-agent] 2024-03-04 23:08:10 INFO juju-log Adding pebble layer
2024-03-04T23:08:17.011Z [container-agent] 2024-03-04 23:08:17 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-03-04T23:08:17.045Z [container-agent] 2024-03-04 23:08:17 INFO juju-log Attempting reboot from complete outage.
2024-03-04T23:08:19.905Z [container-agent] 2024-03-04 23:08:19 INFO juju.worker.uniter.operation runhook.go:186 ran "mysql-pebble-ready" hook (via hook dispatching script: dispatch)
2024-03-04T23:09:11.079Z [container-agent] 2024-03-04 23:09:11 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T23:09:13.056Z [container-agent] 2024-03-04 23:09:13 WARNING juju-log No relation: certificates
2024-03-04T23:09:13.430Z [container-agent] 2024-03-04 23:09:13 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T23:14:05.322Z [container-agent] 2024-03-04 23:14:05 INFO juju-log Starting the log rotate manager
2024-03-04T23:14:05.338Z [container-agent] 2024-03-04 23:14:05 INFO juju-log Started log rotate manager process with PID 886972
2024-03-04T23:14:05.813Z [container-agent] 2024-03-04 23:14:05 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T23:14:07.909Z [container-agent] 2024-03-04 23:14:07 WARNING juju-log No relation: certificates
2024-03-04T23:14:08.251Z [container-agent] 2024-03-04 23:14:08 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T23:18:43.205Z [container-agent] 2024-03-04 23:18:43 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T23:18:45.769Z [container-agent] 2024-03-04 23:18:45 WARNING juju-log No relation: certificates
2024-03-04T23:18:46.117Z [container-agent] 2024-03-04 23:18:46 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T23:23:54.272Z [container-agent] 2024-03-04 23:23:54 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T23:23:57.193Z [container-agent] 2024-03-04 23:23:57 WARNING juju-log No relation: certificates
2024-03-04T23:23:57.563Z [container-agent] 2024-03-04 23:23:57 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T23:29:41.396Z [container-agent] 2024-03-04 23:29:41 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T23:29:44.858Z [container-agent] 2024-03-04 23:29:44 WARNING juju-log No relation: certificates
2024-03-04T23:29:45.213Z [container-agent] 2024-03-04 23:29:45 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T23:33:59.937Z [container-agent] 2024-03-04 23:33:59 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T23:34:03.831Z [container-agent] 2024-03-04 23:34:03 WARNING juju-log No relation: certificates
2024-03-04T23:34:04.193Z [container-agent] 2024-03-04 23:34:04 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T23:38:21.298Z [container-agent] 2024-03-04 23:38:21 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T23:38:25.578Z [container-agent] 2024-03-04 23:38:25 WARNING juju-log No relation: certificates
2024-03-04T23:38:25.917Z [container-agent] 2024-03-04 23:38:25 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T23:43:37.103Z [container-agent] 2024-03-04 23:43:37 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T23:43:41.625Z [container-agent] 2024-03-04 23:43:41 WARNING juju-log No relation: certificates
2024-03-04T23:43:41.970Z [container-agent] 2024-03-04 23:43:41 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T23:48:13.314Z [container-agent] 2024-03-04 23:48:13 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T23:48:18.393Z [container-agent] 2024-03-04 23:48:18 WARNING juju-log No relation: certificates
2024-03-04T23:48:18.748Z [container-agent] 2024-03-04 23:48:18 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T23:52:15.108Z [container-agent] 2024-03-04 23:52:15 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T23:52:20.788Z [container-agent] 2024-03-04 23:52:20 WARNING juju-log No relation: certificates
2024-03-04T23:52:21.131Z [container-agent] 2024-03-04 23:52:21 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-04T23:58:11.197Z [container-agent] 2024-03-04 23:58:11 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-04T23:58:17.647Z [container-agent] 2024-03-04 23:58:17 WARNING juju-log No relation: certificates
2024-03-04T23:58:18.003Z [container-agent] 2024-03-04 23:58:18 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-05T00:03:48.337Z [container-agent] 2024-03-05 00:03:48 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-05T00:03:54.801Z [container-agent] 2024-03-05 00:03:54 WARNING juju-log No relation: certificates
2024-03-05T00:03:55.154Z [container-agent] 2024-03-05 00:03:55 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-05T00:08:12.738Z [container-agent] 2024-03-05 00:08:12 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-05T00:08:20.026Z [container-agent] 2024-03-05 00:08:20 WARNING juju-log No relation: certificates
2024-03-05T00:08:20.396Z [container-agent] 2024-03-05 00:08:20 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-05T00:14:01.951Z [container-agent] 2024-03-05 00:14:01 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-05T00:14:08.633Z [container-agent] 2024-03-05 00:14:08 WARNING juju-log No relation: certificates
2024-03-05T00:14:08.992Z [container-agent] 2024-03-05 00:14:08 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-05T00:19:27.951Z [container-agent] 2024-03-05 00:19:27 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-05T00:19:36.139Z [container-agent] 2024-03-05 00:19:36 WARNING juju-log No relation: certificates
2024-03-05T00:19:36.481Z [container-agent] 2024-03-05 00:19:36 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-05T00:24:00.784Z [container-agent] 2024-03-05 00:24:00 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-05T00:24:08.803Z [container-agent] 2024-03-05 00:24:08 WARNING juju-log No relation: certificates
2024-03-05T00:24:09.163Z [container-agent] 2024-03-05 00:24:09 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-05T00:29:37.280Z [container-agent] 2024-03-05 00:29:37 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-05T00:29:45.632Z [container-agent] 2024-03-05 00:29:45 WARNING juju-log No relation: certificates
2024-03-05T00:29:45.988Z [container-agent] 2024-03-05 00:29:45 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-05T00:31:07.986Z [container-agent] 2024-03-05 00:31:07 ERROR juju-log Failed to flush ERROR LOGS logs.
2024-03-05T00:31:07.986Z [container-agent] Traceback (most recent call last):
2024-03-05T00:31:07.986Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 666, in _run_mysqlsh_script
2024-03-05T00:31:07.986Z [container-agent]     stdout, _ = process.wait_output()
2024-03-05T00:31:07.986Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1441, in wait_output
2024-03-05T00:31:07.986Z [container-agent]     raise ExecError[AnyStr](self._command, exit_code, out_value, err_value)
2024-03-05T00:31:07.986Z [container-agent] ops.pebble.ExecError: non-zero exit code 1 executing ['/usr/bin/mysqlsh', '--no-wizard', '--python', '--verbose=1', '-f', '/tmp/script.py', ';', 'rm', '/tmp/script.py'], stdout='', stderr='Cannot set LC_ALL to locale en_US.UTF-8: No such file or directory\nverbose: 2024-03-05T00:31:06Z: Loading startup files...\nverbose: 2024-03-05T00:31:06Z: Loading plugins...\nverbose: 2024-03-05T00:31:06Z: Connecting to MySQL at: serverconfig@horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local\nTraceback (most recent call last):\n  File "<string>", line 1, in <module>\nmysqlsh.DBError: MySQL Error (2003): Shell.connect: Can\'t connect to MySQL server on \'horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local:3306\' (111)\n'
2024-03-05T00:31:07.986Z [container-agent] 
2024-03-05T00:31:07.986Z [container-agent] During handling of the above exception, another exception occurred:
2024-03-05T00:31:07.986Z [container-agent] 
2024-03-05T00:31:07.986Z [container-agent] Traceback (most recent call last):
2024-03-05T00:31:07.986Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-03-05T00:31:07.986Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-03-05T00:31:07.986Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 669, in _run_mysqlsh_script
2024-03-05T00:31:07.986Z [container-agent]     raise MySQLClientError(e.stderr)
2024-03-05T00:31:07.986Z [container-agent] charms.mysql.v0.mysql.MySQLClientError: Cannot set LC_ALL to locale en_US.UTF-8: No such file or directory
2024-03-05T00:31:07.986Z [container-agent] verbose: 2024-03-05T00:31:06Z: Loading startup files...
2024-03-05T00:31:07.986Z [container-agent] verbose: 2024-03-05T00:31:06Z: Loading plugins...
2024-03-05T00:31:07.986Z [container-agent] verbose: 2024-03-05T00:31:06Z: Connecting to MySQL at: serverconfig@horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local
2024-03-05T00:31:07.986Z [container-agent] Traceback (most recent call last):
2024-03-05T00:31:07.986Z [container-agent]   File "<string>", line 1, in <module>
2024-03-05T00:31:07.986Z [container-agent] mysqlsh.DBError: MySQL Error (2003): Shell.connect: Can't connect to MySQL server on 'horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local:3306' (111)
2024-03-05T00:31:07.986Z [container-agent] 
2024-03-05T00:34:15.789Z [container-agent] 2024-03-05 00:34:15 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-03-05T00:34:15.827Z [container-agent] 2024-03-05 00:34:15 INFO juju-log Attempting reboot from complete outage.
2024-03-05T00:34:24.327Z [container-agent] 2024-03-05 00:34:24 WARNING juju-log No relation: certificates
2024-03-05T00:34:24.678Z [container-agent] 2024-03-05 00:34:24 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-05T00:36:04.784Z [container-agent] 2024-03-05 00:36:04 ERROR juju-log Uncaught exception while in charm code:
2024-03-05T00:36:04.784Z [container-agent] Traceback (most recent call last):
2024-03-05T00:36:04.784Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 2507, in exec
2024-03-05T00:36:04.784Z [container-agent]     control_ws = self._connect_websocket(task_id, 'control')
2024-03-05T00:36:04.784Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 2589, in _connect_websocket
2024-03-05T00:36:04.784Z [container-agent]     ws.connect(url, socket=sock)
2024-03-05T00:36:04.784Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_core.py", line 253, in connect
2024-03-05T00:36:04.784Z [container-agent]     self.handshake_response = handshake(self.sock, url, *addrs, **options)
2024-03-05T00:36:04.784Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_handshake.py", line 57, in handshake
2024-03-05T00:36:04.784Z [container-agent]     status, resp = _get_resp_headers(sock)
2024-03-05T00:36:04.784Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_handshake.py", line 143, in _get_resp_headers
2024-03-05T00:36:04.784Z [container-agent]     status, resp_headers, status_message = read_headers(sock)
2024-03-05T00:36:04.784Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_http.py", line 312, in read_headers
2024-03-05T00:36:04.784Z [container-agent]     line = recv_line(sock)
2024-03-05T00:36:04.784Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_socket.py", line 131, in recv_line
2024-03-05T00:36:04.784Z [container-agent]     c = recv(sock, 1)
2024-03-05T00:36:04.784Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_socket.py", line 122, in recv
2024-03-05T00:36:04.784Z [container-agent]     raise WebSocketConnectionClosedException(
2024-03-05T00:36:04.784Z [container-agent] websocket._exceptions.WebSocketConnectionClosedException: Connection to remote host was lost.
2024-03-05T00:36:04.784Z [container-agent] 
2024-03-05T00:36:04.784Z [container-agent] During handling of the above exception, another exception occurred:
2024-03-05T00:36:04.784Z [container-agent] 
2024-03-05T00:36:04.784Z [container-agent] Traceback (most recent call last):
2024-03-05T00:36:04.784Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 1348, in do_open
2024-03-05T00:36:04.784Z [container-agent]     h.request(req.get_method(), req.selector, req.data, headers,
2024-03-05T00:36:04.784Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1283, in request
2024-03-05T00:36:04.784Z [container-agent]     self._send_request(method, url, body, headers, encode_chunked)
2024-03-05T00:36:04.784Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1329, in _send_request
2024-03-05T00:36:04.784Z [container-agent]     self.endheaders(body, encode_chunked=encode_chunked)
2024-03-05T00:36:04.784Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1278, in endheaders
2024-03-05T00:36:04.784Z [container-agent]     self._send_output(message_body, encode_chunked=encode_chunked)
2024-03-05T00:36:04.784Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1038, in _send_output
2024-03-05T00:36:04.784Z [container-agent]     self.send(msg)
2024-03-05T00:36:04.784Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 976, in send
2024-03-05T00:36:04.784Z [container-agent]     self.connect()
2024-03-05T00:36:04.784Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 297, in connect
2024-03-05T00:36:04.784Z [container-agent]     self.sock.connect(self.socket_path)
2024-03-05T00:36:04.784Z [container-agent] ConnectionRefusedError: [Errno 111] Connection refused
2024-03-05T00:36:04.784Z [container-agent] 
2024-03-05T00:36:04.784Z [container-agent] During handling of the above exception, another exception occurred:
2024-03-05T00:36:04.784Z [container-agent] 
2024-03-05T00:36:04.784Z [container-agent] Traceback (most recent call last):
2024-03-05T00:36:04.784Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1690, in _request_raw
2024-03-05T00:36:04.784Z [container-agent]     response = self.opener.open(request, timeout=self.timeout)
2024-03-05T00:36:04.784Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 519, in open
2024-03-05T00:36:04.784Z [container-agent]     response = self._open(req, data)
2024-03-05T00:36:04.784Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 536, in _open
2024-03-05T00:36:04.784Z [container-agent]     result = self._call_chain(self.handle_open, protocol, protocol +
2024-03-05T00:36:04.784Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 496, in _call_chain
2024-03-05T00:36:04.784Z [container-agent]     result = func(*args)
2024-03-05T00:36:04.784Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 311, in http_open
2024-03-05T00:36:04.784Z [container-agent]     return self.do_open(_UnixSocketConnection, req,  # type:ignore
2024-03-05T00:36:04.785Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 1351, in do_open
2024-03-05T00:36:04.785Z [container-agent]     raise URLError(err)
2024-03-05T00:36:04.785Z [container-agent] urllib.error.URLError: <urlopen error [Errno 111] Connection refused>
2024-03-05T00:36:04.785Z [container-agent] 
2024-03-05T00:36:04.785Z [container-agent] The above exception was the direct cause of the following exception:
2024-03-05T00:36:04.785Z [container-agent] 
2024-03-05T00:36:04.785Z [container-agent] Traceback (most recent call last):
2024-03-05T00:36:04.785Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/./src/charm.py", line 786, in <module>
2024-03-05T00:36:04.785Z [container-agent]     main(MySQLOperatorCharm)
2024-03-05T00:36:04.785Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 436, in main
2024-03-05T00:36:04.785Z [container-agent]     _emit_charm_event(charm, dispatcher.event_name)
2024-03-05T00:36:04.785Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 144, in _emit_charm_event
2024-03-05T00:36:04.785Z [container-agent]     event_to_emit.emit(*args, **kwargs)
2024-03-05T00:36:04.785Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 351, in emit
2024-03-05T00:36:04.785Z [container-agent]     framework._emit(event)
2024-03-05T00:36:04.785Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 853, in _emit
2024-03-05T00:36:04.785Z [container-agent]     self._reemit(event_path)
2024-03-05T00:36:04.785Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 942, in _reemit
2024-03-05T00:36:04.785Z [container-agent]     custom_handler(event)
2024-03-05T00:36:04.785Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/rotate_mysql_logs.py", line 55, in _rotate_mysql_logs
2024-03-05T00:36:04.785Z [container-agent]     self.charm._mysql.flush_mysql_logs(MySQLTextLogs.ERROR)
2024-03-05T00:36:04.785Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-03-05T00:36:04.785Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-03-05T00:36:04.785Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 665, in _run_mysqlsh_script
2024-03-05T00:36:04.785Z [container-agent]     process = self.container.exec(cmd, timeout=timeout)
2024-03-05T00:36:04.785Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/model.py", line 2688, in exec
2024-03-05T00:36:04.785Z [container-agent]     return self._pebble.exec(
2024-03-05T00:36:04.785Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 2514, in exec
2024-03-05T00:36:04.785Z [container-agent]     change = self.wait_change(ChangeID(change_id))
2024-03-05T00:36:04.785Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1893, in wait_change
2024-03-05T00:36:04.785Z [container-agent]     return self._wait_change_using_wait(change_id, timeout)
2024-03-05T00:36:04.785Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1914, in _wait_change_using_wait
2024-03-05T00:36:04.785Z [container-agent]     return self._wait_change(change_id, this_timeout)
2024-03-05T00:36:04.785Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1928, in _wait_change
2024-03-05T00:36:04.785Z [container-agent]     resp = self._request('GET', f'/v1/changes/{change_id}/wait', query)
2024-03-05T00:36:04.785Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1655, in _request
2024-03-05T00:36:04.785Z [container-agent]     response = self._request_raw(method, path, query, headers, data)
2024-03-05T00:36:04.785Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1707, in _request_raw
2024-03-05T00:36:04.785Z [container-agent]     raise ConnectionError(e.reason) from e
2024-03-05T00:36:04.785Z [container-agent] ops.pebble.ConnectionError: [Errno 111] Connection refused
2024-03-05T00:36:04.911Z [container-agent] 2024-03-05 00:36:04 INFO juju.util.exec exec.go:209 run result: exit status 1
2024-03-05T00:36:08.085Z [container-agent] 2024-03-05 00:36:08 INFO juju-log Setting up the logrotate configurations
2024-03-05T00:36:08.174Z [container-agent] 2024-03-05 00:36:08 INFO juju-log Adding pebble layer
2024-03-05T00:36:14.943Z [container-agent] 2024-03-05 00:36:14 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-03-05T00:36:14.972Z [container-agent] 2024-03-05 00:36:14 INFO juju-log Attempting reboot from complete outage.
2024-03-05T00:36:17.859Z [container-agent] 2024-03-05 00:36:17 INFO juju.worker.uniter.operation runhook.go:186 ran "mysql-pebble-ready" hook (via hook dispatching script: dispatch)
2024-03-05T00:38:52.346Z [container-agent] 2024-03-05 00:38:52 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-05T00:38:54.129Z [container-agent] 2024-03-05 00:38:54 WARNING juju-log No relation: certificates
2024-03-05T00:38:54.465Z [container-agent] 2024-03-05 00:38:54 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-05T00:44:28.888Z [container-agent] 2024-03-05 00:44:28 INFO juju-log Starting the log rotate manager
2024-03-05T00:44:28.902Z [container-agent] 2024-03-05 00:44:28 INFO juju-log Started log rotate manager process with PID 895406
2024-03-05T00:44:29.276Z [container-agent] 2024-03-05 00:44:29 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-05T00:44:31.274Z [container-agent] 2024-03-05 00:44:31 WARNING juju-log No relation: certificates
2024-03-05T00:44:31.668Z [container-agent] 2024-03-05 00:44:31 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-05T00:49:30.446Z [container-agent] 2024-03-05 00:49:30 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-05T00:49:33.094Z [container-agent] 2024-03-05 00:49:33 WARNING juju-log No relation: certificates
2024-03-05T00:49:33.444Z [container-agent] 2024-03-05 00:49:33 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-05T00:54:47.718Z [container-agent] 2024-03-05 00:54:47 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-05T00:54:50.728Z [container-agent] 2024-03-05 00:54:50 WARNING juju-log No relation: certificates
2024-03-05T00:54:51.113Z [container-agent] 2024-03-05 00:54:51 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-05T00:59:57.343Z [container-agent] 2024-03-05 00:59:57 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-05T01:00:00.768Z [container-agent] 2024-03-05 01:00:00 WARNING juju-log No relation: certificates
2024-03-05T01:00:01.138Z [container-agent] 2024-03-05 01:00:01 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-05T01:04:24.398Z [container-agent] 2024-03-05 01:04:24 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-05T01:04:27.926Z [container-agent] 2024-03-05 01:04:27 WARNING juju-log No relation: certificates
2024-03-05T01:04:28.286Z [container-agent] 2024-03-05 01:04:28 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-05T01:09:49.042Z [container-agent] 2024-03-05 01:09:49 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-05T01:09:53.336Z [container-agent] 2024-03-05 01:09:53 WARNING juju-log No relation: certificates
2024-03-05T01:09:53.697Z [container-agent] 2024-03-05 01:09:53 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-05T01:15:38.963Z [container-agent] 2024-03-05 01:15:38 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-05T01:15:43.406Z [container-agent] 2024-03-05 01:15:43 WARNING juju-log No relation: certificates
2024-03-05T01:15:43.777Z [container-agent] 2024-03-05 01:15:43 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-05T01:21:28.904Z [container-agent] 2024-03-05 01:21:28 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-05T01:21:34.537Z [container-agent] 2024-03-05 01:21:34 WARNING juju-log No relation: certificates
2024-03-05T01:21:34.919Z [container-agent] 2024-03-05 01:21:34 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-05T01:25:51.912Z [container-agent] 2024-03-05 01:25:51 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-05T01:25:56.940Z [container-agent] 2024-03-05 01:25:56 WARNING juju-log No relation: certificates
2024-03-05T01:25:57.294Z [container-agent] 2024-03-05 01:25:57 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-05T01:31:11.061Z [container-agent] 2024-03-05 01:31:11 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-05T01:31:17.679Z [container-agent] 2024-03-05 01:31:17 WARNING juju-log No relation: certificates
2024-03-05T01:31:18.030Z [container-agent] 2024-03-05 01:31:18 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-05T01:36:21.257Z [container-agent] 2024-03-05 01:36:21 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-05T01:36:27.915Z [container-agent] 2024-03-05 01:36:27 WARNING juju-log No relation: certificates
2024-03-05T01:36:28.258Z [container-agent] 2024-03-05 01:36:28 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-05T01:41:49.070Z [container-agent] 2024-03-05 01:41:49 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-05T01:41:56.113Z [container-agent] 2024-03-05 01:41:56 WARNING juju-log No relation: certificates
2024-03-05T01:41:56.468Z [container-agent] 2024-03-05 01:41:56 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-05T01:45:55.253Z [container-agent] 2024-03-05 01:45:55 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-05T01:46:02.831Z [container-agent] 2024-03-05 01:46:02 WARNING juju-log No relation: certificates
2024-03-05T01:46:03.174Z [container-agent] 2024-03-05 01:46:03 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-05T01:51:38.276Z [container-agent] 2024-03-05 01:51:38 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-05T01:51:46.189Z [container-agent] 2024-03-05 01:51:46 WARNING juju-log No relation: certificates
2024-03-05T01:51:46.532Z [container-agent] 2024-03-05 01:51:46 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-05T01:55:39.008Z [container-agent] 2024-03-05 01:55:39 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-05T01:55:47.217Z [container-agent] 2024-03-05 01:55:47 WARNING juju-log No relation: certificates
2024-03-05T01:55:47.591Z [container-agent] 2024-03-05 01:55:47 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-05T02:00:23.074Z [container-agent] 2024-03-05 02:00:23 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-05T02:00:32.110Z [container-agent] 2024-03-05 02:00:32 WARNING juju-log No relation: certificates
2024-03-05T02:00:32.467Z [container-agent] 2024-03-05 02:00:32 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-05T02:06:16.163Z [container-agent] 2024-03-05 02:06:16 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-03-05T02:06:16.195Z [container-agent] 2024-03-05 02:06:16 INFO juju-log Attempting reboot from complete outage.
2024-03-05T02:06:24.331Z [container-agent] 2024-03-05 02:06:24 WARNING juju-log No relation: certificates
2024-03-05T02:06:24.705Z [container-agent] 2024-03-05 02:06:24 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-05T02:10:09.819Z [container-agent] 2024-03-05 02:10:09 ERROR juju-log Uncaught exception while in charm code:
2024-03-05T02:10:09.819Z [container-agent] Traceback (most recent call last):
2024-03-05T02:10:09.819Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/./src/charm.py", line 786, in <module>
2024-03-05T02:10:09.819Z [container-agent]     main(MySQLOperatorCharm)
2024-03-05T02:10:09.819Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 436, in main
2024-03-05T02:10:09.819Z [container-agent]     _emit_charm_event(charm, dispatcher.event_name)
2024-03-05T02:10:09.819Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 144, in _emit_charm_event
2024-03-05T02:10:09.819Z [container-agent]     event_to_emit.emit(*args, **kwargs)
2024-03-05T02:10:09.819Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 351, in emit
2024-03-05T02:10:09.819Z [container-agent]     framework._emit(event)
2024-03-05T02:10:09.819Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 853, in _emit
2024-03-05T02:10:09.819Z [container-agent]     self._reemit(event_path)
2024-03-05T02:10:09.819Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 942, in _reemit
2024-03-05T02:10:09.819Z [container-agent]     custom_handler(event)
2024-03-05T02:10:09.819Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/rotate_mysql_logs.py", line 56, in _rotate_mysql_logs
2024-03-05T02:10:09.819Z [container-agent]     self.charm._mysql.flush_mysql_logs(MySQLTextLogs.GENERAL)
2024-03-05T02:10:09.819Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-03-05T02:10:09.819Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-03-05T02:10:09.819Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 666, in _run_mysqlsh_script
2024-03-05T02:10:09.819Z [container-agent]     stdout, _ = process.wait_output()
2024-03-05T02:10:09.819Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1436, in wait_output
2024-03-05T02:10:09.819Z [container-agent]     exit_code: int = self._wait()
2024-03-05T02:10:09.819Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1375, in _wait
2024-03-05T02:10:09.819Z [container-agent]     change = self._client.wait_change(self._change_id, timeout=timeout)
2024-03-05T02:10:09.819Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1893, in wait_change
2024-03-05T02:10:09.819Z [container-agent]     return self._wait_change_using_wait(change_id, timeout)
2024-03-05T02:10:09.819Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1914, in _wait_change_using_wait
2024-03-05T02:10:09.819Z [container-agent]     return self._wait_change(change_id, this_timeout)
2024-03-05T02:10:09.819Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1928, in _wait_change
2024-03-05T02:10:09.820Z [container-agent]     resp = self._request('GET', f'/v1/changes/{change_id}/wait', query)
2024-03-05T02:10:09.820Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1655, in _request
2024-03-05T02:10:09.820Z [container-agent]     response = self._request_raw(method, path, query, headers, data)
2024-03-05T02:10:09.820Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1690, in _request_raw
2024-03-05T02:10:09.820Z [container-agent]     response = self.opener.open(request, timeout=self.timeout)
2024-03-05T02:10:09.820Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 519, in open
2024-03-05T02:10:09.820Z [container-agent]     response = self._open(req, data)
2024-03-05T02:10:09.820Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 536, in _open
2024-03-05T02:10:09.820Z [container-agent]     result = self._call_chain(self.handle_open, protocol, protocol +
2024-03-05T02:10:09.820Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 496, in _call_chain
2024-03-05T02:10:09.820Z [container-agent]     result = func(*args)
2024-03-05T02:10:09.820Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 311, in http_open
2024-03-05T02:10:09.820Z [container-agent]     return self.do_open(_UnixSocketConnection, req,  # type:ignore
2024-03-05T02:10:09.820Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 1352, in do_open
2024-03-05T02:10:09.820Z [container-agent]     r = h.getresponse()
2024-03-05T02:10:09.820Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1375, in getresponse
2024-03-05T02:10:09.820Z [container-agent]     response.begin()
2024-03-05T02:10:09.820Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 318, in begin
2024-03-05T02:10:09.820Z [container-agent]     version, status, reason = self._read_status()
2024-03-05T02:10:09.820Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 287, in _read_status
2024-03-05T02:10:09.820Z [container-agent]     raise RemoteDisconnected("Remote end closed connection without"
2024-03-05T02:10:09.820Z [container-agent] http.client.RemoteDisconnected: Remote end closed connection without response
2024-03-05T02:10:09.997Z [container-agent] 2024-03-05 02:10:09 INFO juju.util.exec exec.go:209 run result: exit status 1
2024-03-05T02:10:11.975Z [container-agent] 2024-03-05 02:10:11 INFO juju-log Setting up the logrotate configurations
2024-03-05T02:10:12.051Z [container-agent] 2024-03-05 02:10:12 INFO juju-log Adding pebble layer
2024-03-05T02:10:18.771Z [container-agent] 2024-03-05 02:10:18 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-03-05T02:10:18.799Z [container-agent] 2024-03-05 02:10:18 INFO juju-log Attempting reboot from complete outage.
2024-03-05T02:10:21.783Z [container-agent] 2024-03-05 02:10:21 INFO juju.worker.uniter.operation runhook.go:186 ran "mysql-pebble-ready" hook (via hook dispatching script: dispatch)
2024-03-05T02:10:29.122Z [container-agent] 2024-03-05 02:10:29 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-05T02:10:31.043Z [container-agent] 2024-03-05 02:10:31 WARNING juju-log No relation: certificates
2024-03-05T02:10:31.408Z [container-agent] 2024-03-05 02:10:31 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-05T02:16:02.805Z [container-agent] 2024-03-05 02:16:02 INFO juju-log Starting the log rotate manager
2024-03-05T02:16:02.821Z [container-agent] 2024-03-05 02:16:02 INFO juju-log Started log rotate manager process with PID 903949
2024-03-05T02:16:03.188Z [container-agent] 2024-03-05 02:16:03 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-05T02:16:05.477Z [container-agent] 2024-03-05 02:16:05 WARNING juju-log No relation: certificates
2024-03-05T02:16:05.843Z [container-agent] 2024-03-05 02:16:05 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-05T02:20:10.794Z [container-agent] 2024-03-05 02:20:10 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-05T02:20:13.297Z [container-agent] 2024-03-05 02:20:13 WARNING juju-log No relation: certificates
2024-03-05T02:20:13.660Z [container-agent] 2024-03-05 02:20:13 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-05T02:25:43.394Z [container-agent] 2024-03-05 02:25:43 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-05T02:25:46.461Z [container-agent] 2024-03-05 02:25:46 WARNING juju-log No relation: certificates
2024-03-05T02:25:46.829Z [container-agent] 2024-03-05 02:25:46 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-05T02:30:40.399Z [container-agent] 2024-03-05 02:30:40 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-05T02:30:43.770Z [container-agent] 2024-03-05 02:30:43 WARNING juju-log No relation: certificates
2024-03-05T02:30:44.154Z [container-agent] 2024-03-05 02:30:44 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-05T02:36:27.466Z [container-agent] 2024-03-05 02:36:27 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-05T02:36:31.356Z [container-agent] 2024-03-05 02:36:31 WARNING juju-log No relation: certificates
2024-03-05T02:36:31.687Z [container-agent] 2024-03-05 02:36:31 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-05T02:41:44.857Z [container-agent] 2024-03-05 02:41:44 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-05T02:41:49.281Z [container-agent] 2024-03-05 02:41:49 WARNING juju-log No relation: certificates
2024-03-05T02:41:49.649Z [container-agent] 2024-03-05 02:41:49 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-05T02:46:35.718Z [container-agent] 2024-03-05 02:46:35 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-05T02:46:40.526Z [container-agent] 2024-03-05 02:46:40 WARNING juju-log No relation: certificates
2024-03-05T02:46:40.873Z [container-agent] 2024-03-05 02:46:40 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-05T02:50:42.111Z [container-agent] 2024-03-05 02:50:42 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-05T02:50:47.316Z [container-agent] 2024-03-05 02:50:47 WARNING juju-log No relation: certificates
2024-03-05T02:50:47.689Z [container-agent] 2024-03-05 02:50:47 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-05T02:54:57.995Z [container-agent] 2024-03-05 02:54:57 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-05T02:55:03.222Z [container-agent] 2024-03-05 02:55:03 WARNING juju-log No relation: certificates
2024-03-05T02:55:03.560Z [container-agent] 2024-03-05 02:55:03 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-05T02:59:31.850Z [container-agent] 2024-03-05 02:59:31 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-05T02:59:37.783Z [container-agent] 2024-03-05 02:59:37 WARNING juju-log No relation: certificates
2024-03-05T02:59:38.147Z [container-agent] 2024-03-05 02:59:38 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-05T03:03:59.090Z [container-agent] 2024-03-05 03:03:59 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-05T03:04:05.345Z [container-agent] 2024-03-05 03:04:05 WARNING juju-log No relation: certificates
2024-03-05T03:04:05.715Z [container-agent] 2024-03-05 03:04:05 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-05T03:09:12.887Z [container-agent] 2024-03-05 03:09:12 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-05T03:09:19.964Z [container-agent] 2024-03-05 03:09:19 WARNING juju-log No relation: certificates
2024-03-05T03:09:20.327Z [container-agent] 2024-03-05 03:09:20 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-05T03:14:13.445Z [container-agent] 2024-03-05 03:14:13 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-05T03:14:21.010Z [container-agent] 2024-03-05 03:14:21 WARNING juju-log No relation: certificates
2024-03-05T03:14:21.377Z [container-agent] 2024-03-05 03:14:21 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-05T03:19:28.437Z [container-agent] 2024-03-05 03:19:28 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-05T03:19:36.563Z [container-agent] 2024-03-05 03:19:36 WARNING juju-log No relation: certificates
2024-03-05T03:19:36.912Z [container-agent] 2024-03-05 03:19:36 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-05T03:24:57.757Z [container-agent] 2024-03-05 03:24:57 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-05T03:25:05.740Z [container-agent] 2024-03-05 03:25:05 WARNING juju-log No relation: certificates
2024-03-05T03:25:06.091Z [container-agent] 2024-03-05 03:25:06 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-05T03:30:20.258Z [container-agent] 2024-03-05 03:30:20 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-05T03:30:29.073Z [container-agent] 2024-03-05 03:30:29 WARNING juju-log No relation: certificates
2024-03-05T03:30:29.421Z [container-agent] 2024-03-05 03:30:29 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-05T03:35:21.348Z [container-agent] 2024-03-05 03:35:21 ERROR juju-log Failed to get member state: mysqld daemon is down
2024-03-05T03:35:31.420Z [container-agent] 2024-03-05 03:35:31 ERROR juju-log Failed to get member state: mysqld daemon is down
2024-03-05T03:35:41.496Z [container-agent] 2024-03-05 03:35:41 ERROR juju-log Failed to get member state: mysqld daemon is down
2024-03-05T03:35:41.503Z [container-agent] 2024-03-05 03:35:41 ERROR juju-log Error getting member state. Avoiding potential cluster crash recovery
2024-03-05T03:35:42.055Z [container-agent] 2024-03-05 03:35:42 ERROR juju-log Failed to get cluster status for cluster-7573647c1e26466beeeac32b95ab8f91
2024-03-05T03:35:42.138Z [container-agent] 2024-03-05 03:35:42 INFO juju-log Kubernetes pod label error created
2024-03-05T03:35:42.153Z [container-agent] 2024-03-05 03:35:42 WARNING juju-log No relation: certificates
2024-03-05T03:35:42.497Z [container-agent] 2024-03-05 03:35:42 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-05T03:35:44.091Z [container-agent] 2024-03-05 03:35:44 INFO juju-log Setting up the logrotate configurations
2024-03-05T03:35:44.193Z [container-agent] 2024-03-05 03:35:44 INFO juju-log Adding pebble layer
2024-03-05T03:35:50.965Z [container-agent] 2024-03-05 03:35:50 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-03-05T03:35:50.993Z [container-agent] 2024-03-05 03:35:50 INFO juju-log Attempting reboot from complete outage.
2024-03-05T03:35:53.973Z [container-agent] 2024-03-05 03:35:53 INFO juju.worker.uniter.operation runhook.go:186 ran "mysql-pebble-ready" hook (via hook dispatching script: dispatch)
2024-03-05T03:40:43.062Z [container-agent] 2024-03-05 03:40:43 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-05T03:40:45.647Z [container-agent] 2024-03-05 03:40:45 INFO juju-log Kubernetes pod label primary created
2024-03-05T03:40:45.662Z [container-agent] 2024-03-05 03:40:45 WARNING juju-log No relation: certificates
2024-03-05T03:40:46.049Z [container-agent] 2024-03-05 03:40:46 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-05T03:46:08.747Z [container-agent] 2024-03-05 03:46:08 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-05T03:46:11.651Z [container-agent] 2024-03-05 03:46:11 WARNING juju-log No relation: certificates
2024-03-05T03:46:12.045Z [container-agent] 2024-03-05 03:46:12 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-05T03:51:47.169Z [container-agent] 2024-03-05 03:51:47 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-05T03:51:50.865Z [container-agent] 2024-03-05 03:51:50 WARNING juju-log No relation: certificates
2024-03-05T03:51:51.230Z [container-agent] 2024-03-05 03:51:51 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-05T03:57:08.013Z [container-agent] 2024-03-05 03:57:08 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-05T03:57:11.985Z [container-agent] 2024-03-05 03:57:11 WARNING juju-log No relation: certificates
2024-03-05T03:57:12.336Z [container-agent] 2024-03-05 03:57:12 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-05T04:02:54.811Z [container-agent] 2024-03-05 04:02:54 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-05T04:02:58.958Z [container-agent] 2024-03-05 04:02:58 WARNING juju-log No relation: certificates
2024-03-05T04:02:59.307Z [container-agent] 2024-03-05 04:02:59 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-05T04:07:56.628Z [container-agent] 2024-03-05 04:07:56 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-05T04:08:01.490Z [container-agent] 2024-03-05 04:08:01 WARNING juju-log No relation: certificates
2024-03-05T04:08:01.871Z [container-agent] 2024-03-05 04:08:01 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-05T04:13:41.278Z [container-agent] 2024-03-05 04:13:41 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-05T04:13:46.497Z [container-agent] 2024-03-05 04:13:46 WARNING juju-log No relation: certificates
2024-03-05T04:13:46.852Z [container-agent] 2024-03-05 04:13:46 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-05T04:18:15.620Z [container-agent] 2024-03-05 04:18:15 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-05T04:18:21.606Z [container-agent] 2024-03-05 04:18:21 WARNING juju-log No relation: certificates
2024-03-05T04:18:21.973Z [container-agent] 2024-03-05 04:18:21 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-05T04:24:11.328Z [container-agent] 2024-03-05 04:24:11 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-05T04:24:17.113Z [container-agent] 2024-03-05 04:24:17 WARNING juju-log No relation: certificates
2024-03-05T04:24:17.444Z [container-agent] 2024-03-05 04:24:17 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-05T04:28:47.875Z [container-agent] 2024-03-05 04:28:47 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-05T04:28:54.574Z [container-agent] 2024-03-05 04:28:54 WARNING juju-log No relation: certificates
2024-03-05T04:28:54.916Z [container-agent] 2024-03-05 04:28:54 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-05T04:34:08.866Z [container-agent] 2024-03-05 04:34:08 ERROR juju-log Failed to flush GENERAL LOGS logs.
2024-03-05T04:34:08.867Z [container-agent] Traceback (most recent call last):
2024-03-05T04:34:08.867Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 666, in _run_mysqlsh_script
2024-03-05T04:34:08.867Z [container-agent]     stdout, _ = process.wait_output()
2024-03-05T04:34:08.867Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1441, in wait_output
2024-03-05T04:34:08.867Z [container-agent]     raise ExecError[AnyStr](self._command, exit_code, out_value, err_value)
2024-03-05T04:34:08.867Z [container-agent] ops.pebble.ExecError: non-zero exit code 1 executing ['/usr/bin/mysqlsh', '--no-wizard', '--python', '--verbose=1', '-f', '/tmp/script.py', ';', 'rm', '/tmp/script.py'], stdout='', stderr='Cannot set LC_ALL to locale en_US.UTF-8: No such file or directory\nverbose: 2024-03-05T04:34:07Z: Loading startup files...\nverbose: 2024-03-05T04:34:07Z: Loading plugins...\nverbose: 2024-03-05T04:34:07Z: Connecting to MySQL at: serverconfig@horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local\nTraceback (most recent call last):\n  File "<string>", line 1, in <module>\nmysqlsh.DBError: MySQL Error (2003): Shell.connect: Can\'t connect to MySQL server on \'horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local:3306\' (111)\n'
2024-03-05T04:34:08.867Z [container-agent] 
2024-03-05T04:34:08.867Z [container-agent] During handling of the above exception, another exception occurred:
2024-03-05T04:34:08.867Z [container-agent] 
2024-03-05T04:34:08.867Z [container-agent] Traceback (most recent call last):
2024-03-05T04:34:08.867Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-03-05T04:34:08.867Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-03-05T04:34:08.867Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 669, in _run_mysqlsh_script
2024-03-05T04:34:08.867Z [container-agent]     raise MySQLClientError(e.stderr)
2024-03-05T04:34:08.867Z [container-agent] charms.mysql.v0.mysql.MySQLClientError: Cannot set LC_ALL to locale en_US.UTF-8: No such file or directory
2024-03-05T04:34:08.867Z [container-agent] verbose: 2024-03-05T04:34:07Z: Loading startup files...
2024-03-05T04:34:08.867Z [container-agent] verbose: 2024-03-05T04:34:07Z: Loading plugins...
2024-03-05T04:34:08.867Z [container-agent] verbose: 2024-03-05T04:34:07Z: Connecting to MySQL at: serverconfig@horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local
2024-03-05T04:34:08.867Z [container-agent] Traceback (most recent call last):
2024-03-05T04:34:08.867Z [container-agent]   File "<string>", line 1, in <module>
2024-03-05T04:34:08.867Z [container-agent] mysqlsh.DBError: MySQL Error (2003): Shell.connect: Can't connect to MySQL server on 'horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local:3306' (111)
2024-03-05T04:34:08.867Z [container-agent] 
2024-03-05T04:34:20.076Z [container-agent] 2024-03-05 04:34:20 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-03-05T04:34:20.106Z [container-agent] 2024-03-05 04:34:20 INFO juju-log Attempting reboot from complete outage.
2024-03-05T04:34:27.072Z [container-agent] 2024-03-05 04:34:27 WARNING juju-log No relation: certificates
2024-03-05T04:34:27.417Z [container-agent] 2024-03-05 04:34:27 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-05T04:38:39.411Z [container-agent] 2024-03-05 04:38:39 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-05T04:38:47.345Z [container-agent] 2024-03-05 04:38:47 WARNING juju-log No relation: certificates
2024-03-05T04:38:47.728Z [container-agent] 2024-03-05 04:38:47 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-05T04:39:04.783Z [container-agent] 2024-03-05 04:39:04 ERROR juju-log Uncaught exception while in charm code:
2024-03-05T04:39:04.783Z [container-agent] Traceback (most recent call last):
2024-03-05T04:39:04.783Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/./src/charm.py", line 786, in <module>
2024-03-05T04:39:04.783Z [container-agent]     main(MySQLOperatorCharm)
2024-03-05T04:39:04.783Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 436, in main
2024-03-05T04:39:04.783Z [container-agent]     _emit_charm_event(charm, dispatcher.event_name)
2024-03-05T04:39:04.783Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 144, in _emit_charm_event
2024-03-05T04:39:04.783Z [container-agent]     event_to_emit.emit(*args, **kwargs)
2024-03-05T04:39:04.783Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 351, in emit
2024-03-05T04:39:04.783Z [container-agent]     framework._emit(event)
2024-03-05T04:39:04.783Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 853, in _emit
2024-03-05T04:39:04.783Z [container-agent]     self._reemit(event_path)
2024-03-05T04:39:04.783Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 942, in _reemit
2024-03-05T04:39:04.783Z [container-agent]     custom_handler(event)
2024-03-05T04:39:04.783Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/rotate_mysql_logs.py", line 55, in _rotate_mysql_logs
2024-03-05T04:39:04.783Z [container-agent]     self.charm._mysql.flush_mysql_logs(MySQLTextLogs.ERROR)
2024-03-05T04:39:04.783Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-03-05T04:39:04.783Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-03-05T04:39:04.783Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 666, in _run_mysqlsh_script
2024-03-05T04:39:04.783Z [container-agent]     stdout, _ = process.wait_output()
2024-03-05T04:39:04.783Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1436, in wait_output
2024-03-05T04:39:04.783Z [container-agent]     exit_code: int = self._wait()
2024-03-05T04:39:04.783Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1375, in _wait
2024-03-05T04:39:04.783Z [container-agent]     change = self._client.wait_change(self._change_id, timeout=timeout)
2024-03-05T04:39:04.783Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1893, in wait_change
2024-03-05T04:39:04.783Z [container-agent]     return self._wait_change_using_wait(change_id, timeout)
2024-03-05T04:39:04.783Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1914, in _wait_change_using_wait
2024-03-05T04:39:04.783Z [container-agent]     return self._wait_change(change_id, this_timeout)
2024-03-05T04:39:04.783Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1928, in _wait_change
2024-03-05T04:39:04.783Z [container-agent]     resp = self._request('GET', f'/v1/changes/{change_id}/wait', query)
2024-03-05T04:39:04.783Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1655, in _request
2024-03-05T04:39:04.783Z [container-agent]     response = self._request_raw(method, path, query, headers, data)
2024-03-05T04:39:04.783Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1690, in _request_raw
2024-03-05T04:39:04.783Z [container-agent]     response = self.opener.open(request, timeout=self.timeout)
2024-03-05T04:39:04.783Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 519, in open
2024-03-05T04:39:04.783Z [container-agent]     response = self._open(req, data)
2024-03-05T04:39:04.783Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 536, in _open
2024-03-05T04:39:04.783Z [container-agent]     result = self._call_chain(self.handle_open, protocol, protocol +
2024-03-05T04:39:04.783Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 496, in _call_chain
2024-03-05T04:39:04.783Z [container-agent]     result = func(*args)
2024-03-05T04:39:04.783Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 311, in http_open
2024-03-05T04:39:04.783Z [container-agent]     return self.do_open(_UnixSocketConnection, req,  # type:ignore
2024-03-05T04:39:04.783Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 1352, in do_open
2024-03-05T04:39:04.783Z [container-agent]     r = h.getresponse()
2024-03-05T04:39:04.783Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1375, in getresponse
2024-03-05T04:39:04.783Z [container-agent]     response.begin()
2024-03-05T04:39:04.783Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 318, in begin
2024-03-05T04:39:04.783Z [container-agent]     version, status, reason = self._read_status()
2024-03-05T04:39:04.783Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 287, in _read_status
2024-03-05T04:39:04.783Z [container-agent]     raise RemoteDisconnected("Remote end closed connection without"
2024-03-05T04:39:04.783Z [container-agent] http.client.RemoteDisconnected: Remote end closed connection without response
2024-03-05T04:39:04.944Z [container-agent] 2024-03-05 04:39:04 INFO juju.util.exec exec.go:209 run result: exit status 1
2024-03-05T04:39:06.665Z [container-agent] 2024-03-05 04:39:06 INFO juju-log Starting the log rotate manager
2024-03-05T04:39:06.679Z [container-agent] 2024-03-05 04:39:06 INFO juju-log Started log rotate manager process with PID 918145
2024-03-05T04:39:06.851Z [container-agent] 2024-03-05 04:39:06 INFO juju-log Setting up the logrotate configurations
2024-03-05T04:39:06.914Z [container-agent] 2024-03-05 04:39:06 INFO juju-log Adding pebble layer
2024-03-05T04:39:13.666Z [container-agent] 2024-03-05 04:39:13 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-03-05T04:39:13.693Z [container-agent] 2024-03-05 04:39:13 INFO juju-log Attempting reboot from complete outage.
2024-03-05T04:39:16.671Z [container-agent] 2024-03-05 04:39:16 INFO juju.worker.uniter.operation runhook.go:186 ran "mysql-pebble-ready" hook (via hook dispatching script: dispatch)
2024-03-05T04:44:05.444Z [container-agent] 2024-03-05 04:44:05 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-05T04:44:07.775Z [container-agent] 2024-03-05 04:44:07 WARNING juju-log No relation: certificates
2024-03-05T04:44:08.121Z [container-agent] 2024-03-05 04:44:08 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-05T04:48:22.337Z [container-agent] 2024-03-05 04:48:22 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-05T04:48:24.938Z [container-agent] 2024-03-05 04:48:24 WARNING juju-log No relation: certificates
2024-03-05T04:48:25.291Z [container-agent] 2024-03-05 04:48:25 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-05T04:53:52.665Z [container-agent] 2024-03-05 04:53:52 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-05T04:53:55.544Z [container-agent] 2024-03-05 04:53:55 WARNING juju-log No relation: certificates
2024-03-05T04:53:55.890Z [container-agent] 2024-03-05 04:53:55 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-05T04:58:59.125Z [container-agent] 2024-03-05 04:58:59 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-05T04:59:02.748Z [container-agent] 2024-03-05 04:59:02 WARNING juju-log No relation: certificates
2024-03-05T04:59:03.081Z [container-agent] 2024-03-05 04:59:03 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-05T05:04:28.855Z [container-agent] 2024-03-05 05:04:28 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-05T05:04:33.228Z [container-agent] 2024-03-05 05:04:33 WARNING juju-log No relation: certificates
2024-03-05T05:04:33.627Z [container-agent] 2024-03-05 05:04:33 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-05T05:09:41.765Z [container-agent] 2024-03-05 05:09:41 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-05T05:09:46.487Z [container-agent] 2024-03-05 05:09:46 WARNING juju-log No relation: certificates
2024-03-05T05:09:46.831Z [container-agent] 2024-03-05 05:09:46 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-05T05:14:21.782Z [container-agent] 2024-03-05 05:14:21 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-05T05:14:26.384Z [container-agent] 2024-03-05 05:14:26 WARNING juju-log No relation: certificates
2024-03-05T05:14:26.762Z [container-agent] 2024-03-05 05:14:26 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-05T05:19:12.564Z [container-agent] 2024-03-05 05:19:12 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-05T05:19:18.189Z [container-agent] 2024-03-05 05:19:18 WARNING juju-log No relation: certificates
2024-03-05T05:19:18.552Z [container-agent] 2024-03-05 05:19:18 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-05T05:24:01.102Z [container-agent] 2024-03-05 05:24:01 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-05T05:24:06.677Z [container-agent] 2024-03-05 05:24:06 WARNING juju-log No relation: certificates
2024-03-05T05:24:07.009Z [container-agent] 2024-03-05 05:24:07 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-05T05:29:35.801Z [container-agent] 2024-03-05 05:29:35 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-05T05:29:41.410Z [container-agent] 2024-03-05 05:29:41 WARNING juju-log No relation: certificates
2024-03-05T05:29:41.776Z [container-agent] 2024-03-05 05:29:41 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-05T05:35:23.342Z [container-agent] 2024-03-05 05:35:23 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-05T05:35:29.695Z [container-agent] 2024-03-05 05:35:29 WARNING juju-log No relation: certificates
2024-03-05T05:35:30.052Z [container-agent] 2024-03-05 05:35:30 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-05T05:41:14.562Z [container-agent] 2024-03-05 05:41:14 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-05T05:41:21.504Z [container-agent] 2024-03-05 05:41:21 WARNING juju-log No relation: certificates
2024-03-05T05:41:21.846Z [container-agent] 2024-03-05 05:41:21 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-05T05:47:02.255Z [container-agent] 2024-03-05 05:47:02 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-05T05:47:06.389Z [container-agent] 2024-03-05 05:47:06 WARNING juju-log Failed to get node count
2024-03-05T05:47:06.389Z [container-agent] Traceback (most recent call last):
2024-03-05T05:47:06.389Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 666, in _run_mysqlsh_script
2024-03-05T05:47:06.389Z [container-agent]     stdout, _ = process.wait_output()
2024-03-05T05:47:06.389Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1441, in wait_output
2024-03-05T05:47:06.389Z [container-agent]     raise ExecError[AnyStr](self._command, exit_code, out_value, err_value)
2024-03-05T05:47:06.389Z [container-agent] ops.pebble.ExecError: non-zero exit code 1 executing ['/usr/bin/mysqlsh', '--no-wizard', '--python', '--verbose=1', '-f', '/tmp/script.py', ';', 'rm', '/tmp/script.py'], stdout='', stderr='Cannot set LC_ALL to locale en_US.UTF-8: No such file or directory\nverbose: 2024-03-05T05:47:03Z: Loading startup files...\nverbose: 2024-03-05T05:47:03Z: Loading plugins...\nverbose: 2024-03-05T05:47:03Z: Connecting to MySQL at: clusteradmin@horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local\nTraceback (most recent call last):\n  File "<string>", line 1, in <module>\nmysqlsh.DBError: MySQL Error (2013): Shell.connect: Lost connection to MySQL server at \'reading initial communication packet\', system error: 104\n'
2024-03-05T05:47:06.389Z [container-agent] 
2024-03-05T05:47:06.389Z [container-agent] During handling of the above exception, another exception occurred:
2024-03-05T05:47:06.389Z [container-agent] 
2024-03-05T05:47:06.389Z [container-agent] Traceback (most recent call last):
2024-03-05T05:47:06.389Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 1508, in get_cluster_node_count
2024-03-05T05:47:06.389Z [container-agent]     output = self._run_mysqlsh_script("\n".join(size_commands))
2024-03-05T05:47:06.389Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 669, in _run_mysqlsh_script
2024-03-05T05:47:06.389Z [container-agent]     raise MySQLClientError(e.stderr)
2024-03-05T05:47:06.389Z [container-agent] charms.mysql.v0.mysql.MySQLClientError: Cannot set LC_ALL to locale en_US.UTF-8: No such file or directory
2024-03-05T05:47:06.389Z [container-agent] verbose: 2024-03-05T05:47:03Z: Loading startup files...
2024-03-05T05:47:06.389Z [container-agent] verbose: 2024-03-05T05:47:03Z: Loading plugins...
2024-03-05T05:47:06.389Z [container-agent] verbose: 2024-03-05T05:47:03Z: Connecting to MySQL at: clusteradmin@horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local
2024-03-05T05:47:06.389Z [container-agent] Traceback (most recent call last):
2024-03-05T05:47:06.389Z [container-agent]   File "<string>", line 1, in <module>
2024-03-05T05:47:06.389Z [container-agent] mysqlsh.DBError: MySQL Error (2013): Shell.connect: Lost connection to MySQL server at 'reading initial communication packet', system error: 104
2024-03-05T05:47:06.389Z [container-agent] 
2024-03-05T05:47:08.626Z [container-agent] 2024-03-05 05:47:08 WARNING juju-log Failed to get cluster primary addresses
2024-03-05T05:47:08.626Z [container-agent] Traceback (most recent call last):
2024-03-05T05:47:08.626Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 666, in _run_mysqlsh_script
2024-03-05T05:47:08.626Z [container-agent]     stdout, _ = process.wait_output()
2024-03-05T05:47:08.626Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1441, in wait_output
2024-03-05T05:47:08.626Z [container-agent]     raise ExecError[AnyStr](self._command, exit_code, out_value, err_value)
2024-03-05T05:47:08.626Z [container-agent] ops.pebble.ExecError: non-zero exit code 1 executing ['/usr/bin/mysqlsh', '--no-wizard', '--python', '--verbose=1', '-f', '/tmp/script.py', ';', 'rm', '/tmp/script.py'], stdout='', stderr='Cannot set LC_ALL to locale en_US.UTF-8: No such file or directory\nverbose: 2024-03-05T05:47:07Z: Loading startup files...\nverbose: 2024-03-05T05:47:07Z: Loading plugins...\nverbose: 2024-03-05T05:47:07Z: Connecting to MySQL at: clusteradmin@horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local\nTraceback (most recent call last):\n  File "<string>", line 1, in <module>\nmysqlsh.DBError: MySQL Error (2003): Shell.connect_to_primary: Can\'t connect to MySQL server on \'horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local:3306\' (111)\n'
2024-03-05T05:47:08.626Z [container-agent] 
2024-03-05T05:47:08.626Z [container-agent] During handling of the above exception, another exception occurred:
2024-03-05T05:47:08.626Z [container-agent] 
2024-03-05T05:47:08.626Z [container-agent] Traceback (most recent call last):
2024-03-05T05:47:08.626Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 1767, in get_cluster_primary_address
2024-03-05T05:47:08.626Z [container-agent]     output = self._run_mysqlsh_script("\n".join(get_cluster_primary_commands))
2024-03-05T05:47:08.626Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 669, in _run_mysqlsh_script
2024-03-05T05:47:08.626Z [container-agent]     raise MySQLClientError(e.stderr)
2024-03-05T05:47:08.627Z [container-agent] charms.mysql.v0.mysql.MySQLClientError: Cannot set LC_ALL to locale en_US.UTF-8: No such file or directory
2024-03-05T05:47:08.627Z [container-agent] verbose: 2024-03-05T05:47:07Z: Loading startup files...
2024-03-05T05:47:08.627Z [container-agent] verbose: 2024-03-05T05:47:07Z: Loading plugins...
2024-03-05T05:47:08.627Z [container-agent] verbose: 2024-03-05T05:47:07Z: Connecting to MySQL at: clusteradmin@horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local
2024-03-05T05:47:08.627Z [container-agent] Traceback (most recent call last):
2024-03-05T05:47:08.627Z [container-agent]   File "<string>", line 1, in <module>
2024-03-05T05:47:08.627Z [container-agent] mysqlsh.DBError: MySQL Error (2003): Shell.connect_to_primary: Can't connect to MySQL server on 'horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local:3306' (111)
2024-03-05T05:47:08.627Z [container-agent] 
2024-03-05T05:47:10.022Z [container-agent] 2024-03-05 05:47:10 ERROR juju-log Uncaught exception while in charm code:
2024-03-05T05:47:10.022Z [container-agent] Traceback (most recent call last):
2024-03-05T05:47:10.022Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/./src/charm.py", line 786, in <module>
2024-03-05T05:47:10.022Z [container-agent]     main(MySQLOperatorCharm)
2024-03-05T05:47:10.022Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 436, in main
2024-03-05T05:47:10.022Z [container-agent]     _emit_charm_event(charm, dispatcher.event_name)
2024-03-05T05:47:10.022Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 144, in _emit_charm_event
2024-03-05T05:47:10.022Z [container-agent]     event_to_emit.emit(*args, **kwargs)
2024-03-05T05:47:10.022Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 351, in emit
2024-03-05T05:47:10.022Z [container-agent]     framework._emit(event)
2024-03-05T05:47:10.022Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 853, in _emit
2024-03-05T05:47:10.022Z [container-agent]     self._reemit(event_path)
2024-03-05T05:47:10.022Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 942, in _reemit
2024-03-05T05:47:10.022Z [container-agent]     custom_handler(event)
2024-03-05T05:47:10.022Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/relations/mysql_provider.py", line 281, in _on_update_status
2024-03-05T05:47:10.022Z [container-agent]     self._update_pod_endpoint()
2024-03-05T05:47:10.022Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/relations/mysql_provider.py", line 121, in _update_pod_endpoint
2024-03-05T05:47:10.022Z [container-agent]     cluster_status = self.charm._mysql.get_cluster_status()
2024-03-05T05:47:10.022Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 1488, in get_cluster_status
2024-03-05T05:47:10.022Z [container-agent]     output = self._run_mysqlsh_script("\n".join(status_commands), timeout=30)
2024-03-05T05:47:10.022Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 665, in _run_mysqlsh_script
2024-03-05T05:47:10.022Z [container-agent]     process = self.container.exec(cmd, timeout=timeout)
2024-03-05T05:47:10.022Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/model.py", line 2688, in exec
2024-03-05T05:47:10.022Z [container-agent]     return self._pebble.exec(
2024-03-05T05:47:10.022Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 2508, in exec
2024-03-05T05:47:10.022Z [container-agent]     stdio_ws = self._connect_websocket(task_id, 'stdio')
2024-03-05T05:47:10.022Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 2586, in _connect_websocket
2024-03-05T05:47:10.022Z [container-agent]     sock.connect(self.socket_path)
2024-03-05T05:47:10.022Z [container-agent] FileNotFoundError: [Errno 2] No such file or directory
2024-03-05T05:47:10.362Z [container-agent] 2024-03-05 05:47:10 ERROR juju.worker.uniter.operation runhook.go:180 hook "update-status" (via hook dispatching script: dispatch) failed: exit status 1
2024-03-05T05:47:11.833Z [container-agent] 2024-03-05 05:47:11 ERROR juju-log Failed to rotate mysql logs
2024-03-05T05:47:11.833Z [container-agent] Traceback (most recent call last):
2024-03-05T05:47:11.833Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 628, in _execute_commands
2024-03-05T05:47:11.833Z [container-agent]     stdout, stderr = process.wait_output()
2024-03-05T05:47:11.833Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1441, in wait_output
2024-03-05T05:47:11.833Z [container-agent]     raise ExecError[AnyStr](self._command, exit_code, out_value, err_value)
2024-03-05T05:47:11.833Z [container-agent] ops.pebble.ExecError: non-zero exit code 1 executing ['logrotate', '-f', '/etc/logrotate.d/flush_mysql_logs'], stdout='', stderr='error: cannot stat /etc/logrotate.d/flush_mysql_logs: No such file or directory\n'
2024-03-05T05:47:11.833Z [container-agent] 
2024-03-05T05:47:11.833Z [container-agent] During handling of the above exception, another exception occurred:
2024-03-05T05:47:11.833Z [container-agent] 
2024-03-05T05:47:11.833Z [container-agent] Traceback (most recent call last):
2024-03-05T05:47:11.833Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/rotate_mysql_logs.py", line 50, in _rotate_mysql_logs
2024-03-05T05:47:11.833Z [container-agent]     self.charm._mysql._execute_commands(["logrotate", "-f", LOG_ROTATE_CONFIG_FILE])
2024-03-05T05:47:11.833Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 632, in _execute_commands
2024-03-05T05:47:11.833Z [container-agent]     raise MySQLExecError(e.stderr)
2024-03-05T05:47:11.833Z [container-agent] charms.mysql.v0.mysql.MySQLExecError: error: cannot stat /etc/logrotate.d/flush_mysql_logs: No such file or directory
2024-03-05T05:47:11.833Z [container-agent] 
2024-03-05T05:47:11.977Z [container-agent] 2024-03-05 05:47:11 INFO juju.worker.uniter resolver.go:161 awaiting error resolution for "update-status" hook
2024-03-05T05:47:12.002Z [container-agent] 2024-03-05 05:47:12 INFO juju.worker.uniter resolver.go:161 awaiting error resolution for "update-status" hook
2024-03-05T05:47:12.971Z [container-agent] 2024-03-05 05:47:12 INFO juju.worker.uniter resolver.go:161 awaiting error resolution for "update-status" hook
2024-03-05T05:47:13.973Z [container-agent] 2024-03-05 05:47:13 INFO juju.worker.uniter resolver.go:161 awaiting error resolution for "update-status" hook
2024-03-05T05:47:17.003Z [container-agent] 2024-03-05 05:47:17 INFO juju.worker.uniter resolver.go:161 awaiting error resolution for "update-status" hook
2024-03-05T05:47:19.070Z [container-agent] 2024-03-05 05:47:19 ERROR juju-log Failed to get cluster status for cluster-7573647c1e26466beeeac32b95ab8f91
2024-03-05T05:47:19.170Z [container-agent] 2024-03-05 05:47:19 INFO juju-log Kubernetes pod label error created
2024-03-05T05:47:19.184Z [container-agent] 2024-03-05 05:47:19 WARNING juju-log No relation: certificates
2024-03-05T05:47:19.575Z [container-agent] 2024-03-05 05:47:19 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-05T05:47:21.242Z [container-agent] 2024-03-05 05:47:21 INFO juju-log Setting up the logrotate configurations
2024-03-05T05:47:21.325Z [container-agent] 2024-03-05 05:47:21 INFO juju-log Adding pebble layer
2024-03-05T05:47:28.027Z [container-agent] 2024-03-05 05:47:28 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-03-05T05:47:28.059Z [container-agent] 2024-03-05 05:47:28 INFO juju-log Attempting reboot from complete outage.
2024-03-05T05:47:31.091Z [container-agent] 2024-03-05 05:47:31 INFO juju.worker.uniter.operation runhook.go:186 ran "mysql-pebble-ready" hook (via hook dispatching script: dispatch)
2024-03-05T05:52:05.718Z [container-agent] 2024-03-05 05:52:05 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-05T05:52:08.242Z [container-agent] 2024-03-05 05:52:08 INFO juju-log Kubernetes pod label primary created
2024-03-05T05:52:08.257Z [container-agent] 2024-03-05 05:52:08 WARNING juju-log No relation: certificates
2024-03-05T05:52:08.630Z [container-agent] 2024-03-05 05:52:08 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-05T05:57:19.295Z [container-agent] 2024-03-05 05:57:19 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-05T05:57:22.151Z [container-agent] 2024-03-05 05:57:22 WARNING juju-log No relation: certificates
2024-03-05T05:57:22.550Z [container-agent] 2024-03-05 05:57:22 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-05T06:02:35.133Z [container-agent] 2024-03-05 06:02:35 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-05T06:02:38.376Z [container-agent] 2024-03-05 06:02:38 WARNING juju-log No relation: certificates
2024-03-05T06:02:38.729Z [container-agent] 2024-03-05 06:02:38 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-05T06:06:50.764Z [container-agent] 2024-03-05 06:06:50 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-05T06:06:54.412Z [container-agent] 2024-03-05 06:06:54 WARNING juju-log No relation: certificates
2024-03-05T06:06:54.765Z [container-agent] 2024-03-05 06:06:54 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-05T06:11:08.433Z [container-agent] 2024-03-05 06:11:08 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-05T06:11:12.336Z [container-agent] 2024-03-05 06:11:12 WARNING juju-log No relation: certificates
2024-03-05T06:11:12.696Z [container-agent] 2024-03-05 06:11:12 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-05T06:16:59.807Z [container-agent] 2024-03-05 06:16:59 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-05T06:17:04.290Z [container-agent] 2024-03-05 06:17:04 WARNING juju-log No relation: certificates
2024-03-05T06:17:04.666Z [container-agent] 2024-03-05 06:17:04 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-05T06:21:09.866Z [container-agent] 2024-03-05 06:21:09 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-05T06:21:14.859Z [container-agent] 2024-03-05 06:21:14 WARNING juju-log No relation: certificates
2024-03-05T06:21:15.253Z [container-agent] 2024-03-05 06:21:15 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-05T06:25:52.808Z [container-agent] 2024-03-05 06:25:52 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-05T06:25:57.769Z [container-agent] 2024-03-05 06:25:57 WARNING juju-log No relation: certificates
2024-03-05T06:25:58.094Z [container-agent] 2024-03-05 06:25:58 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-05T06:31:40.377Z [container-agent] 2024-03-05 06:31:40 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-05T06:31:46.598Z [container-agent] 2024-03-05 06:31:46 WARNING juju-log No relation: certificates
2024-03-05T06:31:46.956Z [container-agent] 2024-03-05 06:31:46 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-05T06:36:37.448Z [container-agent] 2024-03-05 06:36:37 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-05T06:36:44.128Z [container-agent] 2024-03-05 06:36:44 WARNING juju-log No relation: certificates
2024-03-05T06:36:44.482Z [container-agent] 2024-03-05 06:36:44 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-05T06:42:12.810Z [container-agent] 2024-03-05 06:42:12 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-05T06:42:19.831Z [container-agent] 2024-03-05 06:42:19 WARNING juju-log No relation: certificates
2024-03-05T06:42:20.176Z [container-agent] 2024-03-05 06:42:20 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-05T06:48:14.244Z [container-agent] 2024-03-05 06:48:14 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-05T06:48:21.881Z [container-agent] 2024-03-05 06:48:21 WARNING juju-log No relation: certificates
2024-03-05T06:48:22.234Z [container-agent] 2024-03-05 06:48:22 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-05T06:53:13.879Z [container-agent] 2024-03-05 06:53:13 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-05T06:53:21.936Z [container-agent] 2024-03-05 06:53:21 WARNING juju-log No relation: certificates
2024-03-05T06:53:22.301Z [container-agent] 2024-03-05 06:53:22 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-05T06:57:47.531Z [container-agent] 2024-03-05 06:57:47 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-03-05T06:57:47.563Z [container-agent] 2024-03-05 06:57:47 INFO juju-log Attempting reboot from complete outage.
2024-03-05T06:57:55.463Z [container-agent] 2024-03-05 06:57:55 WARNING juju-log No relation: certificates
2024-03-05T06:57:55.819Z [container-agent] 2024-03-05 06:57:55 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-05T07:02:04.804Z [container-agent] 2024-03-05 07:02:04 ERROR juju-log Uncaught exception while in charm code:
2024-03-05T07:02:04.804Z [container-agent] Traceback (most recent call last):
2024-03-05T07:02:04.804Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/./src/charm.py", line 786, in <module>
2024-03-05T07:02:04.804Z [container-agent]     main(MySQLOperatorCharm)
2024-03-05T07:02:04.804Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 436, in main
2024-03-05T07:02:04.804Z [container-agent]     _emit_charm_event(charm, dispatcher.event_name)
2024-03-05T07:02:04.804Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 144, in _emit_charm_event
2024-03-05T07:02:04.804Z [container-agent]     event_to_emit.emit(*args, **kwargs)
2024-03-05T07:02:04.804Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 351, in emit
2024-03-05T07:02:04.804Z [container-agent]     framework._emit(event)
2024-03-05T07:02:04.804Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 853, in _emit
2024-03-05T07:02:04.804Z [container-agent]     self._reemit(event_path)
2024-03-05T07:02:04.804Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 942, in _reemit
2024-03-05T07:02:04.804Z [container-agent]     custom_handler(event)
2024-03-05T07:02:04.804Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/rotate_mysql_logs.py", line 55, in _rotate_mysql_logs
2024-03-05T07:02:04.804Z [container-agent]     self.charm._mysql.flush_mysql_logs(MySQLTextLogs.ERROR)
2024-03-05T07:02:04.804Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-03-05T07:02:04.804Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-03-05T07:02:04.804Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 666, in _run_mysqlsh_script
2024-03-05T07:02:04.804Z [container-agent]     stdout, _ = process.wait_output()
2024-03-05T07:02:04.804Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1436, in wait_output
2024-03-05T07:02:04.804Z [container-agent]     exit_code: int = self._wait()
2024-03-05T07:02:04.804Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1375, in _wait
2024-03-05T07:02:04.804Z [container-agent]     change = self._client.wait_change(self._change_id, timeout=timeout)
2024-03-05T07:02:04.804Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1893, in wait_change
2024-03-05T07:02:04.804Z [container-agent]     return self._wait_change_using_wait(change_id, timeout)
2024-03-05T07:02:04.804Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1914, in _wait_change_using_wait
2024-03-05T07:02:04.804Z [container-agent]     return self._wait_change(change_id, this_timeout)
2024-03-05T07:02:04.804Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1928, in _wait_change
2024-03-05T07:02:04.804Z [container-agent]     resp = self._request('GET', f'/v1/changes/{change_id}/wait', query)
2024-03-05T07:02:04.804Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1655, in _request
2024-03-05T07:02:04.804Z [container-agent]     response = self._request_raw(method, path, query, headers, data)
2024-03-05T07:02:04.804Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1690, in _request_raw
2024-03-05T07:02:04.804Z [container-agent]     response = self.opener.open(request, timeout=self.timeout)
2024-03-05T07:02:04.804Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 519, in open
2024-03-05T07:02:04.804Z [container-agent]     response = self._open(req, data)
2024-03-05T07:02:04.804Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 536, in _open
2024-03-05T07:02:04.804Z [container-agent]     result = self._call_chain(self.handle_open, protocol, protocol +
2024-03-05T07:02:04.804Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 496, in _call_chain
2024-03-05T07:02:04.804Z [container-agent]     result = func(*args)
2024-03-05T07:02:04.804Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 311, in http_open
2024-03-05T07:02:04.804Z [container-agent]     return self.do_open(_UnixSocketConnection, req,  # type:ignore
2024-03-05T07:02:04.804Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 1352, in do_open
2024-03-05T07:02:04.804Z [container-agent]     r = h.getresponse()
2024-03-05T07:02:04.804Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1375, in getresponse
2024-03-05T07:02:04.804Z [container-agent]     response.begin()
2024-03-05T07:02:04.804Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 318, in begin
2024-03-05T07:02:04.804Z [container-agent]     version, status, reason = self._read_status()
2024-03-05T07:02:04.804Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 287, in _read_status
2024-03-05T07:02:04.804Z [container-agent]     raise RemoteDisconnected("Remote end closed connection without"
2024-03-05T07:02:04.804Z [container-agent] http.client.RemoteDisconnected: Remote end closed connection without response
2024-03-05T07:02:04.954Z [container-agent] 2024-03-05 07:02:04 INFO juju.util.exec exec.go:209 run result: exit status 1
2024-03-05T07:02:09.316Z [container-agent] 2024-03-05 07:02:09 INFO juju-log Setting up the logrotate configurations
2024-03-05T07:02:09.462Z [container-agent] 2024-03-05 07:02:09 INFO juju-log Adding pebble layer
2024-03-05T07:02:16.320Z [container-agent] 2024-03-05 07:02:16 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-03-05T07:02:16.344Z [container-agent] 2024-03-05 07:02:16 INFO juju-log Attempting reboot from complete outage.
2024-03-05T07:02:19.334Z [container-agent] 2024-03-05 07:02:19 INFO juju.worker.uniter.operation runhook.go:186 ran "mysql-pebble-ready" hook (via hook dispatching script: dispatch)
2024-03-05T07:02:20.950Z [container-agent] 2024-03-05 07:02:20 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-05T07:02:23.151Z [container-agent] 2024-03-05 07:02:23 WARNING juju-log No relation: certificates
2024-03-05T07:02:23.517Z [container-agent] 2024-03-05 07:02:23 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-05T07:06:46.330Z [container-agent] 2024-03-05 07:06:46 INFO juju-log Starting the log rotate manager
2024-03-05T07:06:46.345Z [container-agent] 2024-03-05 07:06:46 INFO juju-log Started log rotate manager process with PID 932465
2024-03-05T07:06:46.652Z [container-agent] 2024-03-05 07:06:46 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-05T07:06:48.676Z [container-agent] 2024-03-05 07:06:48 WARNING juju-log No relation: certificates
2024-03-05T07:06:49.048Z [container-agent] 2024-03-05 07:06:49 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-05T07:11:50.669Z [container-agent] 2024-03-05 07:11:50 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-05T07:11:53.008Z [container-agent] 2024-03-05 07:11:53 WARNING juju-log No relation: certificates
2024-03-05T07:11:53.363Z [container-agent] 2024-03-05 07:11:53 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-05T07:17:16.811Z [container-agent] 2024-03-05 07:17:16 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-05T07:17:20.019Z [container-agent] 2024-03-05 07:17:20 WARNING juju-log No relation: certificates
2024-03-05T07:17:20.360Z [container-agent] 2024-03-05 07:17:20 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-05T07:21:20.265Z [container-agent] 2024-03-05 07:21:20 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-05T07:21:23.551Z [container-agent] 2024-03-05 07:21:23 WARNING juju-log No relation: certificates
2024-03-05T07:21:23.893Z [container-agent] 2024-03-05 07:21:23 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-05T07:26:49.401Z [container-agent] 2024-03-05 07:26:49 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-05T07:26:53.200Z [container-agent] 2024-03-05 07:26:53 WARNING juju-log No relation: certificates
2024-03-05T07:26:53.531Z [container-agent] 2024-03-05 07:26:53 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-05T07:31:20.156Z [container-agent] 2024-03-05 07:31:20 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-05T07:31:24.577Z [container-agent] 2024-03-05 07:31:24 WARNING juju-log No relation: certificates
2024-03-05T07:31:24.927Z [container-agent] 2024-03-05 07:31:24 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-05T07:36:37.897Z [container-agent] 2024-03-05 07:36:37 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-05T07:36:42.468Z [container-agent] 2024-03-05 07:36:42 WARNING juju-log No relation: certificates
2024-03-05T07:36:42.811Z [container-agent] 2024-03-05 07:36:42 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-05T07:41:16.954Z [container-agent] 2024-03-05 07:41:16 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-05T07:41:22.175Z [container-agent] 2024-03-05 07:41:22 WARNING juju-log No relation: certificates
2024-03-05T07:41:22.519Z [container-agent] 2024-03-05 07:41:22 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-05T07:47:13.226Z [container-agent] 2024-03-05 07:47:13 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-05T07:47:19.144Z [container-agent] 2024-03-05 07:47:19 WARNING juju-log No relation: certificates
2024-03-05T07:47:19.509Z [container-agent] 2024-03-05 07:47:19 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-05T07:52:36.012Z [container-agent] 2024-03-05 07:52:36 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-05T07:52:42.314Z [container-agent] 2024-03-05 07:52:42 WARNING juju-log No relation: certificates
2024-03-05T07:52:42.716Z [container-agent] 2024-03-05 07:52:42 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-05T07:58:15.234Z [container-agent] 2024-03-05 07:58:15 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-05T07:58:22.016Z [container-agent] 2024-03-05 07:58:22 WARNING juju-log No relation: certificates
2024-03-05T07:58:22.419Z [container-agent] 2024-03-05 07:58:22 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-05T08:02:49.490Z [container-agent] 2024-03-05 08:02:49 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-05T08:02:56.192Z [container-agent] 2024-03-05 08:02:56 WARNING juju-log No relation: certificates
2024-03-05T08:02:56.530Z [container-agent] 2024-03-05 08:02:56 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-05T08:07:14.102Z [container-agent] 2024-03-05 08:07:14 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-05T08:07:22.034Z [container-agent] 2024-03-05 08:07:22 WARNING juju-log No relation: certificates
2024-03-05T08:07:22.397Z [container-agent] 2024-03-05 08:07:22 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-05T08:12:07.994Z [container-agent] 2024-03-05 08:12:07 ERROR juju-log Failed to flush ERROR LOGS logs.
2024-03-05T08:12:07.994Z [container-agent] Traceback (most recent call last):
2024-03-05T08:12:07.994Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 666, in _run_mysqlsh_script
2024-03-05T08:12:07.994Z [container-agent]     stdout, _ = process.wait_output()
2024-03-05T08:12:07.994Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1441, in wait_output
2024-03-05T08:12:07.994Z [container-agent]     raise ExecError[AnyStr](self._command, exit_code, out_value, err_value)
2024-03-05T08:12:07.994Z [container-agent] ops.pebble.ExecError: non-zero exit code 1 executing ['/usr/bin/mysqlsh', '--no-wizard', '--python', '--verbose=1', '-f', '/tmp/script.py', ';', 'rm', '/tmp/script.py'], stdout='', stderr='Cannot set LC_ALL to locale en_US.UTF-8: No such file or directory\nverbose: 2024-03-05T08:12:06Z: Loading startup files...\nverbose: 2024-03-05T08:12:06Z: Loading plugins...\nverbose: 2024-03-05T08:12:07Z: Connecting to MySQL at: serverconfig@horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local\nTraceback (most recent call last):\n  File "<string>", line 1, in <module>\nmysqlsh.DBError: MySQL Error (2003): Shell.connect: Can\'t connect to MySQL server on \'horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local:3306\' (111)\n'
2024-03-05T08:12:07.994Z [container-agent] 
2024-03-05T08:12:07.994Z [container-agent] During handling of the above exception, another exception occurred:
2024-03-05T08:12:07.994Z [container-agent] 
2024-03-05T08:12:07.994Z [container-agent] Traceback (most recent call last):
2024-03-05T08:12:07.994Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-03-05T08:12:07.994Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-03-05T08:12:07.994Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 669, in _run_mysqlsh_script
2024-03-05T08:12:07.994Z [container-agent]     raise MySQLClientError(e.stderr)
2024-03-05T08:12:07.994Z [container-agent] charms.mysql.v0.mysql.MySQLClientError: Cannot set LC_ALL to locale en_US.UTF-8: No such file or directory
2024-03-05T08:12:07.994Z [container-agent] verbose: 2024-03-05T08:12:06Z: Loading startup files...
2024-03-05T08:12:07.994Z [container-agent] verbose: 2024-03-05T08:12:06Z: Loading plugins...
2024-03-05T08:12:07.994Z [container-agent] verbose: 2024-03-05T08:12:07Z: Connecting to MySQL at: serverconfig@horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local
2024-03-05T08:12:07.994Z [container-agent] Traceback (most recent call last):
2024-03-05T08:12:07.994Z [container-agent]   File "<string>", line 1, in <module>
2024-03-05T08:12:07.994Z [container-agent] mysqlsh.DBError: MySQL Error (2003): Shell.connect: Can't connect to MySQL server on 'horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local:3306' (111)
2024-03-05T08:12:07.994Z [container-agent] 
2024-03-05T08:12:46.299Z [container-agent] 2024-03-05 08:12:46 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-03-05T08:12:46.335Z [container-agent] 2024-03-05 08:12:46 INFO juju-log Attempting reboot from complete outage.
2024-03-05T08:12:54.092Z [container-agent] 2024-03-05 08:12:54 WARNING juju-log No relation: certificates
2024-03-05T08:12:54.454Z [container-agent] 2024-03-05 08:12:54 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-05T08:17:04.797Z [container-agent] 2024-03-05 08:17:04 ERROR juju-log Uncaught exception while in charm code:
2024-03-05T08:17:04.797Z [container-agent] Traceback (most recent call last):
2024-03-05T08:17:04.797Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/./src/charm.py", line 786, in <module>
2024-03-05T08:17:04.797Z [container-agent]     main(MySQLOperatorCharm)
2024-03-05T08:17:04.797Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 436, in main
2024-03-05T08:17:04.797Z [container-agent]     _emit_charm_event(charm, dispatcher.event_name)
2024-03-05T08:17:04.797Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 144, in _emit_charm_event
2024-03-05T08:17:04.797Z [container-agent]     event_to_emit.emit(*args, **kwargs)
2024-03-05T08:17:04.797Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 351, in emit
2024-03-05T08:17:04.797Z [container-agent]     framework._emit(event)
2024-03-05T08:17:04.797Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 853, in _emit
2024-03-05T08:17:04.797Z [container-agent]     self._reemit(event_path)
2024-03-05T08:17:04.797Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 942, in _reemit
2024-03-05T08:17:04.797Z [container-agent]     custom_handler(event)
2024-03-05T08:17:04.797Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/rotate_mysql_logs.py", line 55, in _rotate_mysql_logs
2024-03-05T08:17:04.797Z [container-agent]     self.charm._mysql.flush_mysql_logs(MySQLTextLogs.ERROR)
2024-03-05T08:17:04.797Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-03-05T08:17:04.797Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-03-05T08:17:04.797Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 666, in _run_mysqlsh_script
2024-03-05T08:17:04.797Z [container-agent]     stdout, _ = process.wait_output()
2024-03-05T08:17:04.797Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1436, in wait_output
2024-03-05T08:17:04.797Z [container-agent]     exit_code: int = self._wait()
2024-03-05T08:17:04.797Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1375, in _wait
2024-03-05T08:17:04.797Z [container-agent]     change = self._client.wait_change(self._change_id, timeout=timeout)
2024-03-05T08:17:04.797Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1893, in wait_change
2024-03-05T08:17:04.797Z [container-agent]     return self._wait_change_using_wait(change_id, timeout)
2024-03-05T08:17:04.797Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1914, in _wait_change_using_wait
2024-03-05T08:17:04.797Z [container-agent]     return self._wait_change(change_id, this_timeout)
2024-03-05T08:17:04.797Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1928, in _wait_change
2024-03-05T08:17:04.797Z [container-agent]     resp = self._request('GET', f'/v1/changes/{change_id}/wait', query)
2024-03-05T08:17:04.797Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1655, in _request
2024-03-05T08:17:04.797Z [container-agent]     response = self._request_raw(method, path, query, headers, data)
2024-03-05T08:17:04.797Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1690, in _request_raw
2024-03-05T08:17:04.797Z [container-agent]     response = self.opener.open(request, timeout=self.timeout)
2024-03-05T08:17:04.797Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 519, in open
2024-03-05T08:17:04.797Z [container-agent]     response = self._open(req, data)
2024-03-05T08:17:04.797Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 536, in _open
2024-03-05T08:17:04.797Z [container-agent]     result = self._call_chain(self.handle_open, protocol, protocol +
2024-03-05T08:17:04.797Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 496, in _call_chain
2024-03-05T08:17:04.797Z [container-agent]     result = func(*args)
2024-03-05T08:17:04.797Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 311, in http_open
2024-03-05T08:17:04.797Z [container-agent]     return self.do_open(_UnixSocketConnection, req,  # type:ignore
2024-03-05T08:17:04.797Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 1352, in do_open
2024-03-05T08:17:04.797Z [container-agent]     r = h.getresponse()
2024-03-05T08:17:04.797Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1375, in getresponse
2024-03-05T08:17:04.797Z [container-agent]     response.begin()
2024-03-05T08:17:04.797Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 318, in begin
2024-03-05T08:17:04.797Z [container-agent]     version, status, reason = self._read_status()
2024-03-05T08:17:04.797Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 287, in _read_status
2024-03-05T08:17:04.797Z [container-agent]     raise RemoteDisconnected("Remote end closed connection without"
2024-03-05T08:17:04.797Z [container-agent] http.client.RemoteDisconnected: Remote end closed connection without response
2024-03-05T08:17:04.950Z [container-agent] 2024-03-05 08:17:04 INFO juju.util.exec exec.go:209 run result: exit status 1
2024-03-05T08:17:07.349Z [container-agent] 2024-03-05 08:17:07 INFO juju-log Setting up the logrotate configurations
2024-03-05T08:17:07.418Z [container-agent] 2024-03-05 08:17:07 INFO juju-log Adding pebble layer
2024-03-05T08:17:14.223Z [container-agent] 2024-03-05 08:17:14 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-03-05T08:17:14.246Z [container-agent] 2024-03-05 08:17:14 INFO juju-log Attempting reboot from complete outage.
2024-03-05T08:17:17.264Z [container-agent] 2024-03-05 08:17:17 INFO juju.worker.uniter.operation runhook.go:186 ran "mysql-pebble-ready" hook (via hook dispatching script: dispatch)
2024-03-05T08:18:10.477Z [container-agent] 2024-03-05 08:18:10 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-05T08:18:12.352Z [container-agent] 2024-03-05 08:18:12 WARNING juju-log No relation: certificates
2024-03-05T08:18:12.699Z [container-agent] 2024-03-05 08:18:12 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-05T08:23:14.642Z [container-agent] 2024-03-05 08:23:14 INFO juju-log Starting the log rotate manager
2024-03-05T08:23:14.657Z [container-agent] 2024-03-05 08:23:14 INFO juju-log Started log rotate manager process with PID 939651
2024-03-05T08:23:14.940Z [container-agent] 2024-03-05 08:23:14 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-05T08:23:17.111Z [container-agent] 2024-03-05 08:23:17 WARNING juju-log No relation: certificates
2024-03-05T08:23:17.467Z [container-agent] 2024-03-05 08:23:17 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-05T08:28:05.664Z [container-agent] 2024-03-05 08:28:05 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-05T08:28:08.137Z [container-agent] 2024-03-05 08:28:08 WARNING juju-log No relation: certificates
2024-03-05T08:28:08.480Z [container-agent] 2024-03-05 08:28:08 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-05T08:32:48.310Z [container-agent] 2024-03-05 08:32:48 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-05T08:32:50.990Z [container-agent] 2024-03-05 08:32:50 WARNING juju-log No relation: certificates
2024-03-05T08:32:51.346Z [container-agent] 2024-03-05 08:32:51 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-05T08:38:16.770Z [container-agent] 2024-03-05 08:38:16 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-05T08:38:20.197Z [container-agent] 2024-03-05 08:38:20 WARNING juju-log No relation: certificates
2024-03-05T08:38:20.540Z [container-agent] 2024-03-05 08:38:20 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-05T08:43:07.949Z [container-agent] 2024-03-05 08:43:07 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-05T08:43:11.850Z [container-agent] 2024-03-05 08:43:11 WARNING juju-log No relation: certificates
2024-03-05T08:43:12.192Z [container-agent] 2024-03-05 08:43:12 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-05T08:47:54.049Z [container-agent] 2024-03-05 08:47:54 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-05T08:47:58.022Z [container-agent] 2024-03-05 08:47:58 WARNING juju-log No relation: certificates
2024-03-05T08:47:58.385Z [container-agent] 2024-03-05 08:47:58 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-05T08:52:32.209Z [container-agent] 2024-03-05 08:52:32 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-05T08:52:36.751Z [container-agent] 2024-03-05 08:52:36 WARNING juju-log No relation: certificates
2024-03-05T08:52:37.096Z [container-agent] 2024-03-05 08:52:37 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-05T08:57:00.307Z [container-agent] 2024-03-05 08:57:00 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-05T08:57:05.271Z [container-agent] 2024-03-05 08:57:05 WARNING juju-log No relation: certificates
2024-03-05T08:57:05.691Z [container-agent] 2024-03-05 08:57:05 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-05T09:01:18.158Z [container-agent] 2024-03-05 09:01:18 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-05T09:01:23.904Z [container-agent] 2024-03-05 09:01:23 WARNING juju-log No relation: certificates
2024-03-05T09:01:24.269Z [container-agent] 2024-03-05 09:01:24 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-05T09:07:17.367Z [container-agent] 2024-03-05 09:07:17 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-05T09:07:23.487Z [container-agent] 2024-03-05 09:07:23 WARNING juju-log No relation: certificates
2024-03-05T09:07:23.864Z [container-agent] 2024-03-05 09:07:23 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-05T09:12:15.705Z [container-agent] 2024-03-05 09:12:15 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-05T09:12:22.552Z [container-agent] 2024-03-05 09:12:22 WARNING juju-log No relation: certificates
2024-03-05T09:12:22.943Z [container-agent] 2024-03-05 09:12:22 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-05T09:17:00.601Z [container-agent] 2024-03-05 09:17:00 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-05T09:17:07.400Z [container-agent] 2024-03-05 09:17:07 WARNING juju-log No relation: certificates
2024-03-05T09:17:07.746Z [container-agent] 2024-03-05 09:17:07 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-05T09:22:27.983Z [container-agent] 2024-03-05 09:22:27 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-05T09:22:35.651Z [container-agent] 2024-03-05 09:22:35 WARNING juju-log No relation: certificates
2024-03-05T09:22:36.001Z [container-agent] 2024-03-05 09:22:36 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-05T09:27:19.502Z [container-agent] 2024-03-05 09:27:19 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-05T09:27:27.491Z [container-agent] 2024-03-05 09:27:27 WARNING juju-log No relation: certificates
2024-03-05T09:27:27.862Z [container-agent] 2024-03-05 09:27:27 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-05T09:31:12.862Z [container-agent] 2024-03-05 09:31:12 ERROR juju-log Failed to flush SLOW LOGS logs.
2024-03-05T09:31:12.862Z [container-agent] Traceback (most recent call last):
2024-03-05T09:31:12.862Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 666, in _run_mysqlsh_script
2024-03-05T09:31:12.862Z [container-agent]     stdout, _ = process.wait_output()
2024-03-05T09:31:12.862Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1441, in wait_output
2024-03-05T09:31:12.862Z [container-agent]     raise ExecError[AnyStr](self._command, exit_code, out_value, err_value)
2024-03-05T09:31:12.862Z [container-agent] ops.pebble.ExecError: non-zero exit code 1 executing ['/usr/bin/mysqlsh', '--no-wizard', '--python', '--verbose=1', '-f', '/tmp/script.py', ';', 'rm', '/tmp/script.py'], stdout='', stderr='Cannot set LC_ALL to locale en_US.UTF-8: No such file or directory\nverbose: 2024-03-05T09:31:11Z: Loading startup files...\nverbose: 2024-03-05T09:31:11Z: Loading plugins...\nverbose: 2024-03-05T09:31:11Z: Connecting to MySQL at: serverconfig@horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local\nTraceback (most recent call last):\n  File "<string>", line 1, in <module>\nmysqlsh.DBError: MySQL Error (2003): Shell.connect: Can\'t connect to MySQL server on \'horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local:3306\' (111)\n'
2024-03-05T09:31:12.862Z [container-agent] 
2024-03-05T09:31:12.862Z [container-agent] During handling of the above exception, another exception occurred:
2024-03-05T09:31:12.862Z [container-agent] 
2024-03-05T09:31:12.862Z [container-agent] Traceback (most recent call last):
2024-03-05T09:31:12.862Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-03-05T09:31:12.862Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-03-05T09:31:12.862Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 669, in _run_mysqlsh_script
2024-03-05T09:31:12.862Z [container-agent]     raise MySQLClientError(e.stderr)
2024-03-05T09:31:12.862Z [container-agent] charms.mysql.v0.mysql.MySQLClientError: Cannot set LC_ALL to locale en_US.UTF-8: No such file or directory
2024-03-05T09:31:12.862Z [container-agent] verbose: 2024-03-05T09:31:11Z: Loading startup files...
2024-03-05T09:31:12.862Z [container-agent] verbose: 2024-03-05T09:31:11Z: Loading plugins...
2024-03-05T09:31:12.862Z [container-agent] verbose: 2024-03-05T09:31:11Z: Connecting to MySQL at: serverconfig@horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local
2024-03-05T09:31:12.862Z [container-agent] Traceback (most recent call last):
2024-03-05T09:31:12.862Z [container-agent]   File "<string>", line 1, in <module>
2024-03-05T09:31:12.862Z [container-agent] mysqlsh.DBError: MySQL Error (2003): Shell.connect: Can't connect to MySQL server on 'horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local:3306' (111)
2024-03-05T09:31:12.862Z [container-agent] 
2024-03-05T09:32:40.524Z [container-agent] 2024-03-05 09:32:40 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-03-05T09:32:40.552Z [container-agent] 2024-03-05 09:32:40 INFO juju-log Attempting reboot from complete outage.
2024-03-05T09:32:48.371Z [container-agent] 2024-03-05 09:32:48 WARNING juju-log No relation: certificates
2024-03-05T09:32:48.727Z [container-agent] 2024-03-05 09:32:48 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-05T09:36:09.821Z [container-agent] 2024-03-05 09:36:09 ERROR juju-log Uncaught exception while in charm code:
2024-03-05T09:36:09.821Z [container-agent] Traceback (most recent call last):
2024-03-05T09:36:09.821Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 2507, in exec
2024-03-05T09:36:09.821Z [container-agent]     control_ws = self._connect_websocket(task_id, 'control')
2024-03-05T09:36:09.821Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 2589, in _connect_websocket
2024-03-05T09:36:09.821Z [container-agent]     ws.connect(url, socket=sock)
2024-03-05T09:36:09.821Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_core.py", line 253, in connect
2024-03-05T09:36:09.821Z [container-agent]     self.handshake_response = handshake(self.sock, url, *addrs, **options)
2024-03-05T09:36:09.821Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_handshake.py", line 57, in handshake
2024-03-05T09:36:09.821Z [container-agent]     status, resp = _get_resp_headers(sock)
2024-03-05T09:36:09.821Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_handshake.py", line 143, in _get_resp_headers
2024-03-05T09:36:09.821Z [container-agent]     status, resp_headers, status_message = read_headers(sock)
2024-03-05T09:36:09.821Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_http.py", line 312, in read_headers
2024-03-05T09:36:09.821Z [container-agent]     line = recv_line(sock)
2024-03-05T09:36:09.821Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_socket.py", line 131, in recv_line
2024-03-05T09:36:09.821Z [container-agent]     c = recv(sock, 1)
2024-03-05T09:36:09.821Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_socket.py", line 122, in recv
2024-03-05T09:36:09.821Z [container-agent]     raise WebSocketConnectionClosedException(
2024-03-05T09:36:09.821Z [container-agent] websocket._exceptions.WebSocketConnectionClosedException: Connection to remote host was lost.
2024-03-05T09:36:09.821Z [container-agent] 
2024-03-05T09:36:09.821Z [container-agent] During handling of the above exception, another exception occurred:
2024-03-05T09:36:09.821Z [container-agent] 
2024-03-05T09:36:09.821Z [container-agent] Traceback (most recent call last):
2024-03-05T09:36:09.821Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 1348, in do_open
2024-03-05T09:36:09.821Z [container-agent]     h.request(req.get_method(), req.selector, req.data, headers,
2024-03-05T09:36:09.821Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1283, in request
2024-03-05T09:36:09.821Z [container-agent]     self._send_request(method, url, body, headers, encode_chunked)
2024-03-05T09:36:09.821Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1329, in _send_request
2024-03-05T09:36:09.821Z [container-agent]     self.endheaders(body, encode_chunked=encode_chunked)
2024-03-05T09:36:09.821Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1278, in endheaders
2024-03-05T09:36:09.821Z [container-agent]     self._send_output(message_body, encode_chunked=encode_chunked)
2024-03-05T09:36:09.821Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1038, in _send_output
2024-03-05T09:36:09.821Z [container-agent]     self.send(msg)
2024-03-05T09:36:09.821Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 976, in send
2024-03-05T09:36:09.821Z [container-agent]     self.connect()
2024-03-05T09:36:09.821Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 297, in connect
2024-03-05T09:36:09.821Z [container-agent]     self.sock.connect(self.socket_path)
2024-03-05T09:36:09.821Z [container-agent] ConnectionRefusedError: [Errno 111] Connection refused
2024-03-05T09:36:09.821Z [container-agent] 
2024-03-05T09:36:09.821Z [container-agent] During handling of the above exception, another exception occurred:
2024-03-05T09:36:09.821Z [container-agent] 
2024-03-05T09:36:09.821Z [container-agent] Traceback (most recent call last):
2024-03-05T09:36:09.821Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1690, in _request_raw
2024-03-05T09:36:09.821Z [container-agent]     response = self.opener.open(request, timeout=self.timeout)
2024-03-05T09:36:09.821Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 519, in open
2024-03-05T09:36:09.821Z [container-agent]     response = self._open(req, data)
2024-03-05T09:36:09.821Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 536, in _open
2024-03-05T09:36:09.821Z [container-agent]     result = self._call_chain(self.handle_open, protocol, protocol +
2024-03-05T09:36:09.821Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 496, in _call_chain
2024-03-05T09:36:09.821Z [container-agent]     result = func(*args)
2024-03-05T09:36:09.821Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 311, in http_open
2024-03-05T09:36:09.821Z [container-agent]     return self.do_open(_UnixSocketConnection, req,  # type:ignore
2024-03-05T09:36:09.821Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 1351, in do_open
2024-03-05T09:36:09.821Z [container-agent]     raise URLError(err)
2024-03-05T09:36:09.821Z [container-agent] urllib.error.URLError: <urlopen error [Errno 111] Connection refused>
2024-03-05T09:36:09.821Z [container-agent] 
2024-03-05T09:36:09.821Z [container-agent] The above exception was the direct cause of the following exception:
2024-03-05T09:36:09.821Z [container-agent] 
2024-03-05T09:36:09.821Z [container-agent] Traceback (most recent call last):
2024-03-05T09:36:09.821Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/./src/charm.py", line 786, in <module>
2024-03-05T09:36:09.821Z [container-agent]     main(MySQLOperatorCharm)
2024-03-05T09:36:09.821Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 436, in main
2024-03-05T09:36:09.821Z [container-agent]     _emit_charm_event(charm, dispatcher.event_name)
2024-03-05T09:36:09.821Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 144, in _emit_charm_event
2024-03-05T09:36:09.821Z [container-agent]     event_to_emit.emit(*args, **kwargs)
2024-03-05T09:36:09.821Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 351, in emit
2024-03-05T09:36:09.821Z [container-agent]     framework._emit(event)
2024-03-05T09:36:09.821Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 853, in _emit
2024-03-05T09:36:09.821Z [container-agent]     self._reemit(event_path)
2024-03-05T09:36:09.821Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 942, in _reemit
2024-03-05T09:36:09.821Z [container-agent]     custom_handler(event)
2024-03-05T09:36:09.821Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/rotate_mysql_logs.py", line 57, in _rotate_mysql_logs
2024-03-05T09:36:09.821Z [container-agent]     self.charm._mysql.flush_mysql_logs(MySQLTextLogs.SLOW)
2024-03-05T09:36:09.821Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-03-05T09:36:09.821Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-03-05T09:36:09.821Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 665, in _run_mysqlsh_script
2024-03-05T09:36:09.821Z [container-agent]     process = self.container.exec(cmd, timeout=timeout)
2024-03-05T09:36:09.821Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/model.py", line 2688, in exec
2024-03-05T09:36:09.821Z [container-agent]     return self._pebble.exec(
2024-03-05T09:36:09.821Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 2514, in exec
2024-03-05T09:36:09.821Z [container-agent]     change = self.wait_change(ChangeID(change_id))
2024-03-05T09:36:09.821Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1893, in wait_change
2024-03-05T09:36:09.821Z [container-agent]     return self._wait_change_using_wait(change_id, timeout)
2024-03-05T09:36:09.821Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1914, in _wait_change_using_wait
2024-03-05T09:36:09.821Z [container-agent]     return self._wait_change(change_id, this_timeout)
2024-03-05T09:36:09.821Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1928, in _wait_change
2024-03-05T09:36:09.821Z [container-agent]     resp = self._request('GET', f'/v1/changes/{change_id}/wait', query)
2024-03-05T09:36:09.821Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1655, in _request
2024-03-05T09:36:09.821Z [container-agent]     response = self._request_raw(method, path, query, headers, data)
2024-03-05T09:36:09.821Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1707, in _request_raw
2024-03-05T09:36:09.821Z [container-agent]     raise ConnectionError(e.reason) from e
2024-03-05T09:36:09.821Z [container-agent] ops.pebble.ConnectionError: [Errno 111] Connection refused
2024-03-05T09:36:09.968Z [container-agent] 2024-03-05 09:36:09 INFO juju.util.exec exec.go:209 run result: exit status 1
2024-03-05T09:36:15.596Z [container-agent] 2024-03-05 09:36:15 INFO juju-log Setting up the logrotate configurations
2024-03-05T09:36:15.685Z [container-agent] 2024-03-05 09:36:15 INFO juju-log Adding pebble layer
2024-03-05T09:36:22.412Z [container-agent] 2024-03-05 09:36:22 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-03-05T09:36:22.436Z [container-agent] 2024-03-05 09:36:22 INFO juju-log Attempting reboot from complete outage.
2024-03-05T09:36:25.281Z [container-agent] 2024-03-05 09:36:25 INFO juju.worker.uniter.operation runhook.go:186 ran "mysql-pebble-ready" hook (via hook dispatching script: dispatch)
2024-03-05T09:38:01.311Z [container-agent] 2024-03-05 09:38:01 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-05T09:38:03.264Z [container-agent] 2024-03-05 09:38:03 WARNING juju-log No relation: certificates
2024-03-05T09:38:03.740Z [container-agent] 2024-03-05 09:38:03 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-05T09:42:51.781Z [container-agent] 2024-03-05 09:42:51 INFO juju-log Starting the log rotate manager
2024-03-05T09:42:51.794Z [container-agent] 2024-03-05 09:42:51 INFO juju-log Started log rotate manager process with PID 947071
2024-03-05T09:42:52.088Z [container-agent] 2024-03-05 09:42:52 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-05T09:42:54.054Z [container-agent] 2024-03-05 09:42:54 WARNING juju-log No relation: certificates
2024-03-05T09:42:54.389Z [container-agent] 2024-03-05 09:42:54 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-05T09:47:16.074Z [container-agent] 2024-03-05 09:47:16 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-05T09:47:18.549Z [container-agent] 2024-03-05 09:47:18 WARNING juju-log No relation: certificates
2024-03-05T09:47:18.888Z [container-agent] 2024-03-05 09:47:18 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-05T09:51:21.012Z [container-agent] 2024-03-05 09:51:21 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-05T09:51:23.804Z [container-agent] 2024-03-05 09:51:23 WARNING juju-log No relation: certificates
2024-03-05T09:51:24.156Z [container-agent] 2024-03-05 09:51:24 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-05T09:55:48.807Z [container-agent] 2024-03-05 09:55:48 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-05T09:55:51.858Z [container-agent] 2024-03-05 09:55:51 WARNING juju-log No relation: certificates
2024-03-05T09:55:52.233Z [container-agent] 2024-03-05 09:55:52 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-05T10:01:26.370Z [container-agent] 2024-03-05 10:01:26 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-05T10:01:29.943Z [container-agent] 2024-03-05 10:01:29 WARNING juju-log No relation: certificates
2024-03-05T10:01:30.286Z [container-agent] 2024-03-05 10:01:30 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-05T10:05:32.580Z [container-agent] 2024-03-05 10:05:32 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-05T10:05:36.647Z [container-agent] 2024-03-05 10:05:36 WARNING juju-log No relation: certificates
2024-03-05T10:05:37.045Z [container-agent] 2024-03-05 10:05:37 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-05T10:10:54.449Z [container-agent] 2024-03-05 10:10:54 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-05T10:10:58.954Z [container-agent] 2024-03-05 10:10:58 WARNING juju-log No relation: certificates
2024-03-05T10:10:59.308Z [container-agent] 2024-03-05 10:10:59 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-05T10:15:18.254Z [container-agent] 2024-03-05 10:15:18 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-05T10:15:23.610Z [container-agent] 2024-03-05 10:15:23 WARNING juju-log No relation: certificates
2024-03-05T10:15:23.972Z [container-agent] 2024-03-05 10:15:23 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-05T10:20:18.464Z [container-agent] 2024-03-05 10:20:18 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-05T10:20:24.021Z [container-agent] 2024-03-05 10:20:24 WARNING juju-log No relation: certificates
2024-03-05T10:20:24.355Z [container-agent] 2024-03-05 10:20:24 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-05T10:25:38.353Z [container-agent] 2024-03-05 10:25:38 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-05T10:25:44.474Z [container-agent] 2024-03-05 10:25:44 WARNING juju-log No relation: certificates
2024-03-05T10:25:44.822Z [container-agent] 2024-03-05 10:25:44 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-05T10:29:59.049Z [container-agent] 2024-03-05 10:29:59 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-05T10:30:05.216Z [container-agent] 2024-03-05 10:30:05 WARNING juju-log No relation: certificates
2024-03-05T10:30:05.572Z [container-agent] 2024-03-05 10:30:05 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-05T10:35:20.324Z [container-agent] 2024-03-05 10:35:20 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-05T10:35:27.287Z [container-agent] 2024-03-05 10:35:27 WARNING juju-log No relation: certificates
2024-03-05T10:35:27.648Z [container-agent] 2024-03-05 10:35:27 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-05T10:40:24.252Z [container-agent] 2024-03-05 10:40:24 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-05T10:40:31.041Z [container-agent] 2024-03-05 10:40:31 WARNING juju-log No relation: certificates
2024-03-05T10:40:31.412Z [container-agent] 2024-03-05 10:40:31 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-05T10:46:13.906Z [container-agent] 2024-03-05 10:46:13 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-05T10:46:21.742Z [container-agent] 2024-03-05 10:46:21 WARNING juju-log No relation: certificates
2024-03-05T10:46:22.092Z [container-agent] 2024-03-05 10:46:22 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-05T10:51:23.102Z [container-agent] 2024-03-05 10:51:23 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-03-05T10:51:23.126Z [container-agent] 2024-03-05 10:51:23 INFO juju-log Attempting reboot from complete outage.
2024-03-05T10:51:30.410Z [container-agent] 2024-03-05 10:51:30 WARNING juju-log No relation: certificates
2024-03-05T10:51:30.782Z [container-agent] 2024-03-05 10:51:30 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-05T10:52:04.790Z [container-agent] 2024-03-05 10:52:04 ERROR juju-log Uncaught exception while in charm code:
2024-03-05T10:52:04.790Z [container-agent] Traceback (most recent call last):
2024-03-05T10:52:04.790Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 2507, in exec
2024-03-05T10:52:04.790Z [container-agent]     control_ws = self._connect_websocket(task_id, 'control')
2024-03-05T10:52:04.790Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 2589, in _connect_websocket
2024-03-05T10:52:04.790Z [container-agent]     ws.connect(url, socket=sock)
2024-03-05T10:52:04.790Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_core.py", line 253, in connect
2024-03-05T10:52:04.790Z [container-agent]     self.handshake_response = handshake(self.sock, url, *addrs, **options)
2024-03-05T10:52:04.790Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_handshake.py", line 57, in handshake
2024-03-05T10:52:04.790Z [container-agent]     status, resp = _get_resp_headers(sock)
2024-03-05T10:52:04.790Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_handshake.py", line 143, in _get_resp_headers
2024-03-05T10:52:04.790Z [container-agent]     status, resp_headers, status_message = read_headers(sock)
2024-03-05T10:52:04.790Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_http.py", line 312, in read_headers
2024-03-05T10:52:04.790Z [container-agent]     line = recv_line(sock)
2024-03-05T10:52:04.790Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_socket.py", line 131, in recv_line
2024-03-05T10:52:04.790Z [container-agent]     c = recv(sock, 1)
2024-03-05T10:52:04.790Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/websocket/_socket.py", line 122, in recv
2024-03-05T10:52:04.790Z [container-agent]     raise WebSocketConnectionClosedException(
2024-03-05T10:52:04.790Z [container-agent] websocket._exceptions.WebSocketConnectionClosedException: Connection to remote host was lost.
2024-03-05T10:52:04.790Z [container-agent] 
2024-03-05T10:52:04.790Z [container-agent] During handling of the above exception, another exception occurred:
2024-03-05T10:52:04.790Z [container-agent] 
2024-03-05T10:52:04.790Z [container-agent] Traceback (most recent call last):
2024-03-05T10:52:04.790Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 1348, in do_open
2024-03-05T10:52:04.790Z [container-agent]     h.request(req.get_method(), req.selector, req.data, headers,
2024-03-05T10:52:04.790Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1283, in request
2024-03-05T10:52:04.790Z [container-agent]     self._send_request(method, url, body, headers, encode_chunked)
2024-03-05T10:52:04.790Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1329, in _send_request
2024-03-05T10:52:04.790Z [container-agent]     self.endheaders(body, encode_chunked=encode_chunked)
2024-03-05T10:52:04.790Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1278, in endheaders
2024-03-05T10:52:04.790Z [container-agent]     self._send_output(message_body, encode_chunked=encode_chunked)
2024-03-05T10:52:04.790Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1038, in _send_output
2024-03-05T10:52:04.790Z [container-agent]     self.send(msg)
2024-03-05T10:52:04.790Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 976, in send
2024-03-05T10:52:04.790Z [container-agent]     self.connect()
2024-03-05T10:52:04.790Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 297, in connect
2024-03-05T10:52:04.790Z [container-agent]     self.sock.connect(self.socket_path)
2024-03-05T10:52:04.790Z [container-agent] ConnectionRefusedError: [Errno 111] Connection refused
2024-03-05T10:52:04.790Z [container-agent] 
2024-03-05T10:52:04.790Z [container-agent] During handling of the above exception, another exception occurred:
2024-03-05T10:52:04.790Z [container-agent] 
2024-03-05T10:52:04.790Z [container-agent] Traceback (most recent call last):
2024-03-05T10:52:04.790Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1690, in _request_raw
2024-03-05T10:52:04.790Z [container-agent]     response = self.opener.open(request, timeout=self.timeout)
2024-03-05T10:52:04.790Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 519, in open
2024-03-05T10:52:04.790Z [container-agent]     response = self._open(req, data)
2024-03-05T10:52:04.790Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 536, in _open
2024-03-05T10:52:04.790Z [container-agent]     result = self._call_chain(self.handle_open, protocol, protocol +
2024-03-05T10:52:04.790Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 496, in _call_chain
2024-03-05T10:52:04.790Z [container-agent]     result = func(*args)
2024-03-05T10:52:04.790Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 311, in http_open
2024-03-05T10:52:04.790Z [container-agent]     return self.do_open(_UnixSocketConnection, req,  # type:ignore
2024-03-05T10:52:04.791Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 1351, in do_open
2024-03-05T10:52:04.791Z [container-agent]     raise URLError(err)
2024-03-05T10:52:04.791Z [container-agent] urllib.error.URLError: <urlopen error [Errno 111] Connection refused>
2024-03-05T10:52:04.791Z [container-agent] 
2024-03-05T10:52:04.791Z [container-agent] The above exception was the direct cause of the following exception:
2024-03-05T10:52:04.791Z [container-agent] 
2024-03-05T10:52:04.791Z [container-agent] Traceback (most recent call last):
2024-03-05T10:52:04.791Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/./src/charm.py", line 786, in <module>
2024-03-05T10:52:04.791Z [container-agent]     main(MySQLOperatorCharm)
2024-03-05T10:52:04.791Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 436, in main
2024-03-05T10:52:04.791Z [container-agent]     _emit_charm_event(charm, dispatcher.event_name)
2024-03-05T10:52:04.791Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 144, in _emit_charm_event
2024-03-05T10:52:04.791Z [container-agent]     event_to_emit.emit(*args, **kwargs)
2024-03-05T10:52:04.791Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 351, in emit
2024-03-05T10:52:04.791Z [container-agent]     framework._emit(event)
2024-03-05T10:52:04.791Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 853, in _emit
2024-03-05T10:52:04.791Z [container-agent]     self._reemit(event_path)
2024-03-05T10:52:04.791Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 942, in _reemit
2024-03-05T10:52:04.791Z [container-agent]     custom_handler(event)
2024-03-05T10:52:04.791Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/rotate_mysql_logs.py", line 55, in _rotate_mysql_logs
2024-03-05T10:52:04.791Z [container-agent]     self.charm._mysql.flush_mysql_logs(MySQLTextLogs.ERROR)
2024-03-05T10:52:04.791Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-03-05T10:52:04.791Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-03-05T10:52:04.791Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 665, in _run_mysqlsh_script
2024-03-05T10:52:04.791Z [container-agent]     process = self.container.exec(cmd, timeout=timeout)
2024-03-05T10:52:04.791Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/model.py", line 2688, in exec
2024-03-05T10:52:04.791Z [container-agent]     return self._pebble.exec(
2024-03-05T10:52:04.791Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 2514, in exec
2024-03-05T10:52:04.791Z [container-agent]     change = self.wait_change(ChangeID(change_id))
2024-03-05T10:52:04.791Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1893, in wait_change
2024-03-05T10:52:04.791Z [container-agent]     return self._wait_change_using_wait(change_id, timeout)
2024-03-05T10:52:04.791Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1914, in _wait_change_using_wait
2024-03-05T10:52:04.791Z [container-agent]     return self._wait_change(change_id, this_timeout)
2024-03-05T10:52:04.791Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1928, in _wait_change
2024-03-05T10:52:04.791Z [container-agent]     resp = self._request('GET', f'/v1/changes/{change_id}/wait', query)
2024-03-05T10:52:04.791Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1655, in _request
2024-03-05T10:52:04.791Z [container-agent]     response = self._request_raw(method, path, query, headers, data)
2024-03-05T10:52:04.791Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1707, in _request_raw
2024-03-05T10:52:04.791Z [container-agent]     raise ConnectionError(e.reason) from e
2024-03-05T10:52:04.791Z [container-agent] ops.pebble.ConnectionError: [Errno 111] Connection refused
2024-03-05T10:52:04.970Z [container-agent] 2024-03-05 10:52:04 INFO juju.util.exec exec.go:209 run result: exit status 1
2024-03-05T10:52:08.372Z [container-agent] 2024-03-05 10:52:08 INFO juju-log Setting up the logrotate configurations
2024-03-05T10:52:08.456Z [container-agent] 2024-03-05 10:52:08 INFO juju-log Adding pebble layer
2024-03-05T10:52:15.203Z [container-agent] 2024-03-05 10:52:15 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-03-05T10:52:15.227Z [container-agent] 2024-03-05 10:52:15 INFO juju-log Attempting reboot from complete outage.
2024-03-05T10:52:18.308Z [container-agent] 2024-03-05 10:52:18 INFO juju.worker.uniter.operation runhook.go:186 ran "mysql-pebble-ready" hook (via hook dispatching script: dispatch)
2024-03-05T10:56:41.443Z [container-agent] 2024-03-05 10:56:41 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-05T10:56:43.485Z [container-agent] 2024-03-05 10:56:43 WARNING juju-log No relation: certificates
2024-03-05T10:56:43.835Z [container-agent] 2024-03-05 10:56:43 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-05T11:00:45.667Z [container-agent] 2024-03-05 11:00:45 INFO juju-log Starting the log rotate manager
2024-03-05T11:00:45.682Z [container-agent] 2024-03-05 11:00:45 INFO juju-log Started log rotate manager process with PID 954363
2024-03-05T11:00:45.956Z [container-agent] 2024-03-05 11:00:45 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-05T11:00:48.185Z [container-agent] 2024-03-05 11:00:48 WARNING juju-log No relation: certificates
2024-03-05T11:00:48.568Z [container-agent] 2024-03-05 11:00:48 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-05T11:06:22.031Z [container-agent] 2024-03-05 11:06:22 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-05T11:06:24.503Z [container-agent] 2024-03-05 11:06:24 WARNING juju-log No relation: certificates
2024-03-05T11:06:24.840Z [container-agent] 2024-03-05 11:06:24 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-05T11:10:43.436Z [container-agent] 2024-03-05 11:10:43 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-05T11:10:46.439Z [container-agent] 2024-03-05 11:10:46 WARNING juju-log No relation: certificates
2024-03-05T11:10:46.789Z [container-agent] 2024-03-05 11:10:46 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-05T11:15:45.597Z [container-agent] 2024-03-05 11:15:45 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-05T11:15:49.460Z [container-agent] 2024-03-05 11:15:49 WARNING juju-log No relation: certificates
2024-03-05T11:15:49.835Z [container-agent] 2024-03-05 11:15:49 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-05T11:20:26.353Z [container-agent] 2024-03-05 11:20:26 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-05T11:20:30.251Z [container-agent] 2024-03-05 11:20:30 WARNING juju-log No relation: certificates
2024-03-05T11:20:30.599Z [container-agent] 2024-03-05 11:20:30 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-05T11:24:49.343Z [container-agent] 2024-03-05 11:24:49 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-05T11:24:53.465Z [container-agent] 2024-03-05 11:24:53 WARNING juju-log No relation: certificates
2024-03-05T11:24:53.808Z [container-agent] 2024-03-05 11:24:53 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-05T11:29:54.273Z [container-agent] 2024-03-05 11:29:54 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-05T11:29:59.027Z [container-agent] 2024-03-05 11:29:59 WARNING juju-log No relation: certificates
2024-03-05T11:29:59.386Z [container-agent] 2024-03-05 11:29:59 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-05T11:35:23.456Z [container-agent] 2024-03-05 11:35:23 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-05T11:35:27.907Z [container-agent] 2024-03-05 11:35:27 WARNING juju-log No relation: certificates
2024-03-05T11:35:28.266Z [container-agent] 2024-03-05 11:35:28 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-05T11:40:11.361Z [container-agent] 2024-03-05 11:40:11 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-05T11:40:16.758Z [container-agent] 2024-03-05 11:40:16 WARNING juju-log No relation: certificates
2024-03-05T11:40:17.111Z [container-agent] 2024-03-05 11:40:17 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-05T11:44:58.451Z [container-agent] 2024-03-05 11:44:58 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-05T11:45:04.248Z [container-agent] 2024-03-05 11:45:04 WARNING juju-log No relation: certificates
2024-03-05T11:45:04.584Z [container-agent] 2024-03-05 11:45:04 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-05T11:50:42.575Z [container-agent] 2024-03-05 11:50:42 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-05T11:50:49.233Z [container-agent] 2024-03-05 11:50:49 WARNING juju-log No relation: certificates
2024-03-05T11:50:49.569Z [container-agent] 2024-03-05 11:50:49 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-05T11:56:09.156Z [container-agent] 2024-03-05 11:56:09 ERROR juju-log Failed to flush GENERAL LOGS logs.
2024-03-05T11:56:09.156Z [container-agent] Traceback (most recent call last):
2024-03-05T11:56:09.156Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 666, in _run_mysqlsh_script
2024-03-05T11:56:09.156Z [container-agent]     stdout, _ = process.wait_output()
2024-03-05T11:56:09.156Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1441, in wait_output
2024-03-05T11:56:09.156Z [container-agent]     raise ExecError[AnyStr](self._command, exit_code, out_value, err_value)
2024-03-05T11:56:09.156Z [container-agent] ops.pebble.ExecError: non-zero exit code 1 executing ['/usr/bin/mysqlsh', '--no-wizard', '--python', '--verbose=1', '-f', '/tmp/script.py', ';', 'rm', '/tmp/script.py'], stdout='', stderr='Cannot set LC_ALL to locale en_US.UTF-8: No such file or directory\nverbose: 2024-03-05T11:56:08Z: Loading startup files...\nverbose: 2024-03-05T11:56:08Z: Loading plugins...\nverbose: 2024-03-05T11:56:08Z: Connecting to MySQL at: serverconfig@horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local\nTraceback (most recent call last):\n  File "<string>", line 1, in <module>\nmysqlsh.DBError: MySQL Error (2003): Shell.connect: Can\'t connect to MySQL server on \'horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local:3306\' (111)\n'
2024-03-05T11:56:09.156Z [container-agent] 
2024-03-05T11:56:09.156Z [container-agent] During handling of the above exception, another exception occurred:
2024-03-05T11:56:09.156Z [container-agent] 
2024-03-05T11:56:09.156Z [container-agent] Traceback (most recent call last):
2024-03-05T11:56:09.156Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-03-05T11:56:09.156Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-03-05T11:56:09.156Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 669, in _run_mysqlsh_script
2024-03-05T11:56:09.156Z [container-agent]     raise MySQLClientError(e.stderr)
2024-03-05T11:56:09.156Z [container-agent] charms.mysql.v0.mysql.MySQLClientError: Cannot set LC_ALL to locale en_US.UTF-8: No such file or directory
2024-03-05T11:56:09.156Z [container-agent] verbose: 2024-03-05T11:56:08Z: Loading startup files...
2024-03-05T11:56:09.156Z [container-agent] verbose: 2024-03-05T11:56:08Z: Loading plugins...
2024-03-05T11:56:09.156Z [container-agent] verbose: 2024-03-05T11:56:08Z: Connecting to MySQL at: serverconfig@horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local
2024-03-05T11:56:09.156Z [container-agent] Traceback (most recent call last):
2024-03-05T11:56:09.156Z [container-agent]   File "<string>", line 1, in <module>
2024-03-05T11:56:09.156Z [container-agent] mysqlsh.DBError: MySQL Error (2003): Shell.connect: Can't connect to MySQL server on 'horizon-mysql-0.horizon-mysql-endpoints.openstack.svc.cluster.local:3306' (111)
2024-03-05T11:56:09.156Z [container-agent] 
2024-03-05T11:56:14.322Z [container-agent] 2024-03-05 11:56:14 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-03-05T11:56:14.352Z [container-agent] 2024-03-05 11:56:14 INFO juju-log Attempting reboot from complete outage.
2024-03-05T11:56:21.579Z [container-agent] 2024-03-05 11:56:21 WARNING juju-log No relation: certificates
2024-03-05T11:56:21.939Z [container-agent] 2024-03-05 11:56:21 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-05T12:01:04.752Z [container-agent] 2024-03-05 12:01:04 ERROR juju-log Uncaught exception while in charm code:
2024-03-05T12:01:04.752Z [container-agent] Traceback (most recent call last):
2024-03-05T12:01:04.752Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/./src/charm.py", line 786, in <module>
2024-03-05T12:01:04.752Z [container-agent]     main(MySQLOperatorCharm)
2024-03-05T12:01:04.752Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 436, in main
2024-03-05T12:01:04.752Z [container-agent]     _emit_charm_event(charm, dispatcher.event_name)
2024-03-05T12:01:04.752Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/main.py", line 144, in _emit_charm_event
2024-03-05T12:01:04.752Z [container-agent]     event_to_emit.emit(*args, **kwargs)
2024-03-05T12:01:04.752Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 351, in emit
2024-03-05T12:01:04.752Z [container-agent]     framework._emit(event)
2024-03-05T12:01:04.752Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 853, in _emit
2024-03-05T12:01:04.752Z [container-agent]     self._reemit(event_path)
2024-03-05T12:01:04.752Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/framework.py", line 942, in _reemit
2024-03-05T12:01:04.752Z [container-agent]     custom_handler(event)
2024-03-05T12:01:04.752Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/rotate_mysql_logs.py", line 55, in _rotate_mysql_logs
2024-03-05T12:01:04.752Z [container-agent]     self.charm._mysql.flush_mysql_logs(MySQLTextLogs.ERROR)
2024-03-05T12:01:04.752Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/lib/charms/mysql/v0/mysql.py", line 2585, in flush_mysql_logs
2024-03-05T12:01:04.752Z [container-agent]     self._run_mysqlsh_script("\n".join(flush_logs_commands))
2024-03-05T12:01:04.752Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/src/mysql_k8s_helpers.py", line 666, in _run_mysqlsh_script
2024-03-05T12:01:04.752Z [container-agent]     stdout, _ = process.wait_output()
2024-03-05T12:01:04.752Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1436, in wait_output
2024-03-05T12:01:04.752Z [container-agent]     exit_code: int = self._wait()
2024-03-05T12:01:04.752Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1375, in _wait
2024-03-05T12:01:04.752Z [container-agent]     change = self._client.wait_change(self._change_id, timeout=timeout)
2024-03-05T12:01:04.752Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1893, in wait_change
2024-03-05T12:01:04.752Z [container-agent]     return self._wait_change_using_wait(change_id, timeout)
2024-03-05T12:01:04.752Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1914, in _wait_change_using_wait
2024-03-05T12:01:04.752Z [container-agent]     return self._wait_change(change_id, this_timeout)
2024-03-05T12:01:04.752Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1928, in _wait_change
2024-03-05T12:01:04.752Z [container-agent]     resp = self._request('GET', f'/v1/changes/{change_id}/wait', query)
2024-03-05T12:01:04.752Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1655, in _request
2024-03-05T12:01:04.752Z [container-agent]     response = self._request_raw(method, path, query, headers, data)
2024-03-05T12:01:04.752Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 1690, in _request_raw
2024-03-05T12:01:04.752Z [container-agent]     response = self.opener.open(request, timeout=self.timeout)
2024-03-05T12:01:04.752Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 519, in open
2024-03-05T12:01:04.752Z [container-agent]     response = self._open(req, data)
2024-03-05T12:01:04.752Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 536, in _open
2024-03-05T12:01:04.752Z [container-agent]     result = self._call_chain(self.handle_open, protocol, protocol +
2024-03-05T12:01:04.752Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 496, in _call_chain
2024-03-05T12:01:04.752Z [container-agent]     result = func(*args)
2024-03-05T12:01:04.752Z [container-agent]   File "/var/lib/juju/agents/unit-horizon-mysql-0/charm/venv/ops/pebble.py", line 311, in http_open
2024-03-05T12:01:04.752Z [container-agent]     return self.do_open(_UnixSocketConnection, req,  # type:ignore
2024-03-05T12:01:04.752Z [container-agent]   File "/usr/lib/python3.10/urllib/request.py", line 1352, in do_open
2024-03-05T12:01:04.752Z [container-agent]     r = h.getresponse()
2024-03-05T12:01:04.752Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 1375, in getresponse
2024-03-05T12:01:04.752Z [container-agent]     response.begin()
2024-03-05T12:01:04.752Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 318, in begin
2024-03-05T12:01:04.752Z [container-agent]     version, status, reason = self._read_status()
2024-03-05T12:01:04.752Z [container-agent]   File "/usr/lib/python3.10/http/client.py", line 287, in _read_status
2024-03-05T12:01:04.752Z [container-agent]     raise RemoteDisconnected("Remote end closed connection without"
2024-03-05T12:01:04.752Z [container-agent] http.client.RemoteDisconnected: Remote end closed connection without response
2024-03-05T12:01:04.886Z [container-agent] 2024-03-05 12:01:04 INFO juju.util.exec exec.go:209 run result: exit status 1
2024-03-05T12:01:06.805Z [container-agent] 2024-03-05 12:01:06 INFO juju-log Setting up the logrotate configurations
2024-03-05T12:01:06.945Z [container-agent] 2024-03-05 12:01:06 INFO juju-log Adding pebble layer
2024-03-05T12:01:13.709Z [container-agent] 2024-03-05 12:01:13 INFO juju-log Unit workload member-state is offline with member-role unknown
2024-03-05T12:01:13.733Z [container-agent] 2024-03-05 12:01:13 INFO juju-log Attempting reboot from complete outage.
2024-03-05T12:01:16.831Z [container-agent] 2024-03-05 12:01:16 INFO juju.worker.uniter.operation runhook.go:186 ran "mysql-pebble-ready" hook (via hook dispatching script: dispatch)
2024-03-05T12:01:43.035Z [container-agent] 2024-03-05 12:01:43 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-05T12:01:45.113Z [container-agent] 2024-03-05 12:01:45 WARNING juju-log No relation: certificates
2024-03-05T12:01:45.487Z [container-agent] 2024-03-05 12:01:45 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)
2024-03-05T12:07:03.301Z [container-agent] 2024-03-05 12:07:03 INFO juju-log Starting the log rotate manager
2024-03-05T12:07:03.317Z [container-agent] 2024-03-05 12:07:03 INFO juju-log Started log rotate manager process with PID 960595
2024-03-05T12:07:03.909Z [container-agent] 2024-03-05 12:07:03 INFO juju-log Unit workload member-state is online with member-role primary
2024-03-05T12:07:06.001Z [container-agent] 2024-03-05 12:07:06 WARNING juju-log No relation: certificates
2024-03-05T12:07:06.378Z [container-agent] 2024-03-05 12:07:06 INFO juju.worker.uniter.operation runhook.go:186 ran "update-status" hook (via hook dispatching script: dispatch)