After restoring the databases, zookeeper went down and other services failed to connect to zookeeper. later found that the ownership of the /var/lib/zookeeper set to ntp.. changing ownership brings back all the service up and running
root@contrail-node7:/opt/contrail/utils# ls -altr /var/lib/ | grep zookeeper drwxr-xr-x 3 ntp ntp 4096 Feb 2 22:29 zookeeper root@contrail-node7:/opt/contrail/utils# ls -altr /var/lib/zookeeper total 12 lrwxrwxrwx 1 root root 24 Jun 10 2013 myid -> /etc/zookeeper/conf/myid drwxr-xr-x 3 ntp ntp 4096 Feb 2 22:29 . drwxr-xr-x 2 ntp ntp 4096 Mar 5 04:42 version-2 drwxr-xr-x 61 root root 4096 Mar 15 14:40 .. root@contrail-node7:/opt/contrail/utils# ls -altr /var/lib/ | grep cass drwxr-xr-x 5 cassandra cassandra 4096 Mar 15 11:45 cassandra root@contrail-node7:/opt/contrail/utils# ls -altr /var/lib/ | grep my drwxr-xr-x 10 mysql mysql 4096 Mar 15 11:57 mysql root@contrail-node7:/opt/contrail/utils# root@contrail-node7:/opt/contrail/utils# root@contrail-node7:/opt/contrail/utils# root@contrail-node7:/opt/contrail/utils# root@contrail-node7:/opt/contrail/utils# fab -R database -- "chown -R zookeeper:zookeeper /var/lib/zookeeper/" [root@10.102.44.41] Executing task '<remainder>' 2016-03-15 16:07:16:463172: [root@10.102.44.41] run: chown -R zookeeper:zookeeper /var/lib/zookeeper/ 2016-03-15 16:07:16:463317: [root@10.102.44.42] Executing task '<remainder>' 2016-03-15 16:07:16:707813: [root@10.102.44.42] run: chown -R zookeeper:zookeeper /var/lib/zookeeper/ 2016-03-15 16:07:16:707939: [root@10.102.44.43] Executing task '<remainder>' 2016-03-15 16:07:16:951495: [root@10.102.44.43] run: chown -R zookeeper:zookeeper /var/lib/zookeeper/ 2016-03-15 16:07:16:951745: 2016-03-15 16:07:17:206137: Done. 2016-03-15 16:07:17:206190: Disconnecting from 10.102.44.43... done. 2016-03-15 16:07:17:319913: Disconnecting from 10.102.44.41... done. 2016-03-15 16:07:17:351463: Disconnecting from 10.102.44.42... done. 2016-03-15 16:07:17:465336: root@contrail-node7:/opt/contrail/utils# root@contrail-node7:/opt/contrail/utils# root@contrail-node7:/opt/contrail/utils# root@contrail-node7:/opt/contrail/utils# ls -altr /var/lib/ | grep zookeeper drwxr-xr-x 3 zookeeper zookeeper 4096 Feb 2 22:29 zookeeper root@contrail-node7:/opt/contrail/utils# root@contrail-node7:/opt/contrail/utils# root@contrail-node7:/opt/contrail/utils# root@contrail-node7:/opt/contrail/utils# ps -ef | grep Zoo root 12815 30324 0 16:07 pts/3 00:00:00 grep --color=auto Zoo root@contrail-node7:/opt/contrail/utils# ps -ef | grep zoo zookeep+ 12888 1 0 16:07 ? 00:00:00 /usr/bin/java -cp /etc/zookeeper/conf:/usr/share/java/jline.jar:/usr/share/java/log4j-1.2.jar:/usr/share/java/xercesImpl.jar:/usr/share/java/xmlParserAPIs.jar:/usr/share/java/netty.jar:/usr/share/java/slf4j-api.jar:/usr/share/java/slf4j-log4j12.jar:/usr/share/java/zookeeper.jar -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.local.only=false -Dzookeeper.log.dir=/var/log/zookeeper -Dzookeeper.root.logger=INFO,CONSOLE,ROLLINGFILE org.apache.zookeeper.server.quorum.QuorumPeerMain /etc/zookeeper/conf/zoo.cfg root 12907 30324 0 16:07 pts/3 00:00:00 grep --color=auto zoo root@contrail-node7:/opt/contrail/utils# contrail-status == Contrail vRouter == supervisor-vrouter: active
After restoring the databases, zookeeper went down and other services failed to connect to zookeeper.
later found that the ownership of the /var/lib/zookeeper set to ntp..
changing ownership brings back all the service up and running
root@contrail- node7:/ opt/contrail/ utils# ls -altr /var/lib/ | grep zookeeper node7:/ opt/contrail/ utils# ls -altr /var/lib/zookeeper conf/myid node7:/ opt/contrail/ utils# ls -altr /var/lib/ | grep cass node7:/ opt/contrail/ utils# ls -altr /var/lib/ | grep my node7:/ opt/contrail/ utils# node7:/ opt/contrail/ utils# node7:/ opt/contrail/ utils# node7:/ opt/contrail/ utils# node7:/ opt/contrail/ utils# fab -R database -- "chown -R zookeeper:zookeeper /var/lib/ zookeeper/ " node7:/ opt/contrail/ utils# node7:/ opt/contrail/ utils# node7:/ opt/contrail/ utils# node7:/ opt/contrail/ utils# ls -altr /var/lib/ | grep zookeeper node7:/ opt/contrail/ utils# node7:/ opt/contrail/ utils# node7:/ opt/contrail/ utils# node7:/ opt/contrail/ utils# ps -ef | grep Zoo node7:/ opt/contrail/ utils# ps -ef | grep zoo conf:/usr/ share/java/ jline.jar: /usr/share/ java/log4j- 1.2.jar: /usr/share/ java/xercesImpl .jar:/usr/ share/java/ xmlParserAPIs. jar:/usr/ share/java/ netty.jar: /usr/share/ java/slf4j- api.jar: /usr/share/ java/slf4j- log4j12. jar:/usr/ share/java/ zookeeper. jar -Dcom.sun. management. jmxremote -Dcom.sun. management. jmxremote. local.only= false -Dzookeeper. log.dir= /var/log/ zookeeper -Dzookeeper. root.logger= INFO,CONSOLE, ROLLINGFILE org.apache. zookeeper. server. quorum. QuorumPeerMain /etc/zookeeper/ conf/zoo. cfg node7:/ opt/contrail/ utils# contrail-status
drwxr-xr-x 3 ntp ntp 4096 Feb 2 22:29 zookeeper
root@contrail-
total 12
lrwxrwxrwx 1 root root 24 Jun 10 2013 myid -> /etc/zookeeper/
drwxr-xr-x 3 ntp ntp 4096 Feb 2 22:29 .
drwxr-xr-x 2 ntp ntp 4096 Mar 5 04:42 version-2
drwxr-xr-x 61 root root 4096 Mar 15 14:40 ..
root@contrail-
drwxr-xr-x 5 cassandra cassandra 4096 Mar 15 11:45 cassandra
root@contrail-
drwxr-xr-x 10 mysql mysql 4096 Mar 15 11:57 mysql
root@contrail-
root@contrail-
root@contrail-
root@contrail-
root@contrail-
[root@10.102.44.41] Executing task '<remainder>'
2016-03-15 16:07:16:463172: [root@10.102.44.41] run: chown -R zookeeper:zookeeper /var/lib/zookeeper/
2016-03-15 16:07:16:463317: [root@10.102.44.42] Executing task '<remainder>'
2016-03-15 16:07:16:707813: [root@10.102.44.42] run: chown -R zookeeper:zookeeper /var/lib/zookeeper/
2016-03-15 16:07:16:707939: [root@10.102.44.43] Executing task '<remainder>'
2016-03-15 16:07:16:951495: [root@10.102.44.43] run: chown -R zookeeper:zookeeper /var/lib/zookeeper/
2016-03-15 16:07:16:951745:
2016-03-15 16:07:17:206137: Done.
2016-03-15 16:07:17:206190: Disconnecting from 10.102.44.43... done.
2016-03-15 16:07:17:319913: Disconnecting from 10.102.44.41... done.
2016-03-15 16:07:17:351463: Disconnecting from 10.102.44.42... done.
2016-03-15 16:07:17:465336: root@contrail-
root@contrail-
root@contrail-
root@contrail-
drwxr-xr-x 3 zookeeper zookeeper 4096 Feb 2 22:29 zookeeper
root@contrail-
root@contrail-
root@contrail-
root@contrail-
root 12815 30324 0 16:07 pts/3 00:00:00 grep --color=auto Zoo
root@contrail-
zookeep+ 12888 1 0 16:07 ? 00:00:00 /usr/bin/java -cp /etc/zookeeper/
root 12907 30324 0 16:07 pts/3 00:00:00 grep --color=auto zoo
root@contrail-
== Contrail vRouter ==
supervisor-vrouter: active