Xenial - Can't connect to local MySQL server through socket /var/run/mysqld/mysqld.sock

2016-07-04 22:45:58 INFO shared-db-relation-changed _mysql_exceptions.OperationalError: (2002, "Can't connect to local MySQL server through socket '/var/run/mysqld/mysqld.sock' (2)")

2016-07-04 22:45:58 ERROR juju.worker.uniter.operation runhook.go:107 hook "shared-db-relation-changed" failed: exit status 1


# bundle (xenial-mitaka target)
http://bazaar.launchpad.net/~ost-maintainers/openstack-charm-testing/trunk/view/head:/bundles/sparse/next.yaml#L164


# deployer output
00:14:14.484 2016-07-04 22:47:22 [INFO] deployer.import:  Adding relation neutron-openvswitch <-> rabbitmq-server
00:14:14.834 2016-07-04 22:47:22 [DEBUG] deployer.import: Waiting for relation convergence 60s
00:15:15.004 2016-07-04 22:48:23 [ERROR] deployer.env: The following units had errors:
00:15:15.004    unit: mysql/0: machine: 10 agent-state: error details: hook failed: "shared-db-relation-changed"
00:15:15.013 2016-07-04 22:48:23 [INFO] deployer.cli: Deployment stopped. run time: 869.73


# pxc unit log
2016-07-04 22:45:58 INFO shared-db-relation-changed Traceback (most recent call last):
2016-07-04 22:45:58 INFO shared-db-relation-changed   File "/var/lib/juju/agents/unit-mysql-0/charm/hooks/shared-db-relation-changed", line 691, in <module>
2016-07-04 22:45:58 INFO shared-db-relation-changed     main()
2016-07-04 22:45:58 INFO shared-db-relation-changed   File "/var/lib/juju/agents/unit-mysql-0/charm/hooks/shared-db-relation-changed", line 684, in main
2016-07-04 22:45:58 INFO shared-db-relation-changed     hooks.execute(sys.argv)
2016-07-04 22:45:58 INFO shared-db-relation-changed   File "/var/lib/juju/agents/unit-mysql-0/charm/hooks/charmhelpers/core/hookenv.py", line 717, in execute
2016-07-04 22:45:58 INFO shared-db-relation-changed     self._hooks[hook_name]()
2016-07-04 22:45:58 INFO shared-db-relation-changed   File "/var/lib/juju/agents/unit-mysql-0/charm/hooks/shared-db-relation-changed", line 507, in shared_db_changed
2016-07-04 22:45:58 INFO shared-db-relation-changed     db_helper)
2016-07-04 22:45:58 INFO shared-db-relation-changed   File "/var/lib/juju/agents/unit-mysql-0/charm/hooks/shared-db-relation-changed", line 449, in configure_db_for_hosts
2016-07-04 22:45:58 INFO shared-db-relation-changed     password = db_helper.configure_db(host, database, username)
2016-07-04 22:45:58 INFO shared-db-relation-changed   File "/var/lib/juju/agents/unit-mysql-0/charm/hooks/charmhelpers/contrib/database/mysql.py", line 299, in configure_db
2016-07-04 22:45:58 INFO shared-db-relation-changed     self.connect(password=self.get_mysql_root_password())
2016-07-04 22:45:58 INFO shared-db-relation-changed   File "/var/lib/juju/agents/unit-mysql-0/charm/hooks/charmhelpers/contrib/database/mysql.py", line 62, in connect
2016-07-04 22:45:58 INFO shared-db-relation-changed     passwd=password)
2016-07-04 22:45:58 INFO shared-db-relation-changed   File "/usr/lib/python2.7/dist-packages/MySQLdb/__init__.py", line 81, in Connect
2016-07-04 22:45:58 INFO shared-db-relation-changed     return Connection(*args, **kwargs)
2016-07-04 22:45:58 INFO shared-db-relation-changed   File "/usr/lib/python2.7/dist-packages/MySQLdb/connections.py", line 204, in __init__
2016-07-04 22:45:58 INFO shared-db-relation-changed     super(Connection, self).__init__(*args, **kwargs2)
2016-07-04 22:45:58 INFO shared-db-relation-changed _mysql_exceptions.OperationalError: (2002, "Can't connect to local MySQL server through socket '/var/run/mysqld/mysqld.sock' (2)")
2016-07-04 22:45:58 INFO juju.worker.uniter.context context.go:579 handling reboot
2016-07-04 22:45:58 ERROR juju.worker.uniter.operation runhook.go:107 hook "shared-db-relation-changed" failed: exit status 1
2016-07-04 22:45:58 DEBUG juju.worker.uniter modes.go:31 [AGENT-STATUS] failed: run relation-changed (25; glance/0) hook


# juju stat
[Services]            
NAME                  STATUS  EXPOSED CHARM                                  
ceilometer            waiting false   local:xenial/ceilometer-44             
ceilometer-agent              false   local:xenial/ceilometer-agent-20       
ceph                  active  false   local:xenial/ceph-105                  
cinder                waiting false   local:xenial/cinder-136                
cinder-ceph                   false   local:xenial/cinder-ceph-2             
glance                waiting false   local:xenial/glance-150                
heat                  waiting false   local:xenial/heat-12                   
keystone              waiting false   local:xenial/keystone-0                
mongodb               unknown false   local:xenial/mongodb-30                
mysql                 error   false   local:xenial/percona-cluster-45        
neutron-api           waiting false   local:xenial/neutron-api-0             
neutron-gateway       active  false   local:xenial/neutron-gateway-64        
neutron-openvswitch           false   local:xenial/neutron-openvswitch-0     
nova-cloud-controller waiting false   local:xenial/nova-cloud-controller-501 
nova-compute          waiting false   local:xenial/nova-compute-133          
openstack-dashboard   waiting false   local:xenial/openstack-dashboard-32    
rabbitmq-server       active  false   local:xenial/rabbitmq-server-150       
swift-proxy           active  false   local:xenial/swift-proxy-147           
swift-storage-z1      active  false   local:xenial/swift-storage-90          
swift-storage-z2      active  false   local:xenial/swift-storage-91          
swift-storage-z3      active  false   local:xenial/swift-storage-92          

[Units]                 
ID                      WORKLOAD-STATE AGENT-STATE VERSION MACHINE PORTS                                   PUBLIC-ADDRESS MESSAGE                                                                 
ceilometer/0            waiting        executing   1.25.5  1       8777/tcp                                172.17.119.153 (update-status) Incomplete relations: messaging, identity               
ceph/0                  active         executing   1.25.5  2                                               172.17.119.154 Unit is ready and clustered                                             
ceph/1                  active         executing   1.25.5  3                                               172.17.119.155 Unit is ready and clustered                                             
ceph/2                  active         executing   1.25.5  4                                               172.17.119.156 Unit is ready and clustered                                             
cinder/0                waiting        idle        1.25.5  5       8776/tcp                                172.17.119.157 Incomplete relations: messaging, identity, database                     
  cinder-ceph/0         blocked        executing   1.25.5                                                  172.17.119.157 (config-changed) Missing relations: ceph                                
glance/0                waiting        executing   1.25.5  6       9292/tcp                                172.17.119.158 Incomplete relations: messaging, identity, database                     
heat/0                  waiting        idle        1.25.5  7       8000/tcp,8004/tcp                       172.17.119.159 Incomplete relations: identity, database                                
keystone/0              waiting        executing   1.25.5  8       5000/tcp                                172.17.119.160 Incomplete relations: database                                          
mongodb/0               unknown        idle        1.25.5  9       27017/tcp,27019/tcp,27021/tcp,28017/tcp 172.17.119.161                                                                         
mysql/0                 error          idle        1.25.5  10                                              172.17.119.162 hook failed: "shared-db-relation-changed" for glance:shared-db          
neutron-api/0           waiting        idle        1.25.5  11      9696/tcp                                172.17.119.163 Incomplete relations: identity, database                                
neutron-gateway/0       active         idle        1.25.5  12                                              172.17.119.164 Unit is ready                                                           
nova-cloud-controller/0 waiting        executing   1.25.5  13      8774/tcp                                172.17.119.168 Incomplete relations: neutron-api, image, database, identity, messaging 
nova-compute/0          waiting        executing   1.25.5  14                                              172.17.119.169 Incomplete relations: messaging, image, storage-backend, database       
  ceilometer-agent/2    unknown        allocating  1.25.5                                                  172.17.119.169 Waiting for agent initialization to finish                              
  neutron-openvswitch/2 unknown        allocating  1.25.5                                                  172.17.119.169 Waiting for agent initialization to finish                              
nova-compute/1          waiting        executing   1.25.5  15                                              172.17.119.165 Incomplete relations: messaging, database                               
  ceilometer-agent/0    unknown        allocating  1.25.5                                                  172.17.119.165 Waiting for agent initialization to finish                              
  neutron-openvswitch/0 unknown        allocating  1.25.5                                                  172.17.119.165 Waiting for agent initialization to finish                              
nova-compute/2          waiting        executing   1.25.5  16                                              172.17.119.166 Incomplete relations: messaging, image, storage-backend, database       
  ceilometer-agent/1    unknown        allocating  1.25.5                                                  172.17.119.166 Waiting for agent initialization to finish                              
  neutron-openvswitch/1 unknown        allocating  1.25.5                                                  172.17.119.166 Waiting for agent initialization to finish                              
openstack-dashboard/0   waiting        idle        1.25.5  17      80/tcp,443/tcp                          172.17.119.167 Incomplete relations: identity                                          
rabbitmq-server/0       active         executing   1.25.5  18      5672/tcp                                172.17.119.172 Unit is ready                                                           
swift-proxy/0           active         idle        1.25.5  19      8080/tcp                                172.17.119.173 Unit is ready                                                           
swift-storage-z1/0      active         idle        1.25.5  20                                              172.17.119.170 Unit is ready                                                           
swift-storage-z2/0      active         idle        1.25.5  21                                              172.17.119.171 Unit is ready                                                           
swift-storage-z3/0      active         idle        1.25.5  22                                              172.17.119.174 Unit is ready                                                           

[Machines] 
ID         STATE   VERSION DNS            INS-ID                               SERIES HARDWARE                                                                 
0          started 1.25.5  172.17.119.152 70215aae-97e9-46a9-818a-690d2cc2b104 xenial arch=amd64 cpu-cores=1 mem=1536M root-disk=10240M availability-zone=nova 
1          started 1.25.5  172.17.119.153 c42eddfa-a7bb-4f13-a040-8b1dc03710a7 xenial arch=amd64 cpu-cores=1 mem=1536M root-disk=10240M availability-zone=nova 
2          started 1.25.5  172.17.119.154 f15cd101-e475-494c-90b1-3bdcc7d9013a xenial arch=amd64 cpu-cores=1 mem=1536M root-disk=10240M availability-zone=nova 
3          started 1.25.5  172.17.119.155 a1dd9c02-17a6-4dd9-89c9-7ccfe2a99128 xenial arch=amd64 cpu-cores=1 mem=1536M root-disk=10240M availability-zone=nova 
4          started 1.25.5  172.17.119.156 26acd435-eab0-4f49-aeb2-fec99e51cf50 xenial arch=amd64 cpu-cores=1 mem=1536M root-disk=10240M availability-zone=nova 
5          started 1.25.5  172.17.119.157 cd513d5d-b573-46ae-ba01-9a105c3a105e xenial arch=amd64 cpu-cores=1 mem=1536M root-disk=10240M availability-zone=nova 
6          started 1.25.5  172.17.119.158 faef3c1f-9b5c-4c51-bf33-40db68317735 xenial arch=amd64 cpu-cores=1 mem=1536M root-disk=10240M availability-zone=nova 
7          started 1.25.5  172.17.119.159 c92ca3ca-1d1e-4d6f-a2b3-039999dce967 xenial arch=amd64 cpu-cores=1 mem=1536M root-disk=10240M availability-zone=nova 
8          started 1.25.5  172.17.119.160 4dd6ea0f-7c6c-47ab-89fc-6b75569f0076 xenial arch=amd64 cpu-cores=1 mem=1536M root-disk=10240M availability-zone=nova 
9          started 1.25.5  172.17.119.161 c7d48040-01d5-4377-8224-4883131a0ce0 xenial arch=amd64 cpu-cores=1 mem=1536M root-disk=10240M availability-zone=nova 
10         started 1.25.5  172.17.119.162 f5392f84-f4ed-403f-b913-2bcf6cc2e012 xenial arch=amd64 cpu-cores=2 mem=4096M root-disk=40960M availability-zone=nova 
11         started 1.25.5  172.17.119.163 8c5095cb-f8a7-4590-87c9-50cf408a3b21 xenial arch=amd64 cpu-cores=1 mem=1536M root-disk=10240M availability-zone=nova 
12         started 1.25.5  172.17.119.164 bc093fc2-1a4a-49ff-9ec1-0a164308faeb xenial arch=amd64 cpu-cores=1 mem=1536M root-disk=10240M availability-zone=nova 
13         started 1.25.5  172.17.119.168 9b7835c1-7e44-4a23-982b-b0951c8f279c xenial arch=amd64 cpu-cores=1 mem=1536M root-disk=10240M availability-zone=nova 
14         started 1.25.5  172.17.119.169 6960c1b7-7bbd-479a-a862-11be87b7eac2 xenial arch=amd64 cpu-cores=2 mem=4096M root-disk=40960M availability-zone=nova 
15         started 1.25.5  172.17.119.165 1f9620d1-6767-4a43-be47-65c729226fb7 xenial arch=amd64 cpu-cores=2 mem=4096M root-disk=40960M availability-zone=nova 
16         started 1.25.5  172.17.119.166 d0a0a66b-7ba5-40aa-a21a-d1fdc37d9316 xenial arch=amd64 cpu-cores=2 mem=4096M root-disk=40960M availability-zone=nova 
17         started 1.25.5  172.17.119.167 7defeaee-544a-4e3f-99c1-4c9d868b3388 xenial arch=amd64 cpu-cores=1 mem=1536M root-disk=10240M availability-zone=nova 
18         started 1.25.5  172.17.119.172 6b74b0a2-fce2-4838-8841-518979efedad xenial arch=amd64 cpu-cores=1 mem=1536M root-disk=10240M availability-zone=nova 
19         started 1.25.5  172.17.119.173 15bbe4f1-1793-473f-a48a-bcb5ca6393de xenial arch=amd64 cpu-cores=1 mem=1536M root-disk=10240M availability-zone=nova 
20         started 1.25.5  172.17.119.170 888d323c-2729-4542-ba7f-65ff24c1fdee xenial arch=amd64 cpu-cores=1 mem=1536M root-disk=10240M availability-zone=nova 
21         started 1.25.5  172.17.119.171 4394586b-64da-4c5f-9295-c69415294e26 xenial arch=amd64 cpu-cores=1 mem=1536M root-disk=10240M availability-zone=nova 
22         started 1.25.5  172.17.119.174 43210747-ce7f-45e6-8349-6ab22997aff6 xenial arch=amd64 cpu-cores=1 mem=1536M root-disk=10240M availability-zone=nova