After restoring the databases, zookeeper went down and other services failed to connect to zookeeper.
later found that the ownership of the /var/lib/zookeeper set to ntp..
changing ownership brings back all the service up and running
also the nova instances directory..
root@contrail-node7:/opt/contrail/utils# ls -altr /var/lib/ | grep zookeeper
drwxr-xr-x 3 ntp ntp 4096 Feb 2 22:29 zookeeper
root@contrail-node7:/opt/contrail/utils# ls -altr /var/lib/zookeeper
total 12
lrwxrwxrwx 1 root root 24 Jun 10 2013 myid -> /etc/zookeeper/conf/myid
drwxr-xr-x 3 ntp ntp 4096 Feb 2 22:29 .
drwxr-xr-x 2 ntp ntp 4096 Mar 5 04:42 version-2
drwxr-xr-x 61 root root 4096 Mar 15 14:40 ..
root@contrail-node7:/opt/contrail/utils# ls -altr /var/lib/ | grep cass
drwxr-xr-x 5 cassandra cassandra 4096 Mar 15 11:45 cassandra
root@contrail-node7:/opt/contrail/utils# ls -altr /var/lib/ | grep my
drwxr-xr-x 10 mysql mysql 4096 Mar 15 11:57 mysql
root@contrail-node7:/opt/contrail/utils#
root@contrail-node7:/opt/contrail/utils#
root@contrail-node7:/opt/contrail/utils#
root@contrail-node7:/opt/contrail/utils#
root@contrail-node7:/opt/contrail/utils# fab -R database -- "chown -R zookeeper:zookeeper /var/lib/zookeeper/"
[root@10.102.44.41] Executing task '<remainder>'
2016-03-15 16:07:16:463172: [root@10.102.44.41] run: chown -R zookeeper:zookeeper /var/lib/zookeeper/
2016-03-15 16:07:16:463317: [root@10.102.44.42] Executing task '<remainder>'
2016-03-15 16:07:16:707813: [root@10.102.44.42] run: chown -R zookeeper:zookeeper /var/lib/zookeeper/
2016-03-15 16:07:16:707939: [root@10.102.44.43] Executing task '<remainder>'
2016-03-15 16:07:16:951495: [root@10.102.44.43] run: chown -R zookeeper:zookeeper /var/lib/zookeeper/
2016-03-15 16:07:16:951745:
2016-03-15 16:07:17:206137: Done.
2016-03-15 16:07:17:206190: Disconnecting from 10.102.44.43... done.
2016-03-15 16:07:17:319913: Disconnecting from 10.102.44.41... done.
2016-03-15 16:07:17:351463: Disconnecting from 10.102.44.42... done.
2016-03-15 16:07:17:465336: root@contrail-node7:/opt/contrail/utils#
root@contrail-node7:/opt/contrail/utils#
root@contrail-node7:/opt/contrail/utils#
root@contrail-node7:/opt/contrail/utils# ls -altr /var/lib/ | grep zookeeper
drwxr-xr-x 3 zookeeper zookeeper 4096 Feb 2 22:29 zookeeper
root@contrail-node7:/opt/contrail/utils#
root@contrail-node7:/opt/contrail/utils#
root@contrail-node7:/opt/contrail/utils#
root@contrail-node7:/opt/contrail/utils# ps -ef | grep Zoo
root 12815 30324 0 16:07 pts/3 00:00:00 grep --color=auto Zoo
root@contrail-node7:/opt/contrail/utils# ps -ef | grep zoo
zookeep+ 12888 1 0 16:07 ? 00:00:00 /usr/bin/java -cp /etc/zookeeper/conf:/usr/share/java/jline.jar:/usr/share/java/log4j-1.2.jar:/usr/share/java/xercesImpl.jar:/usr/share/java/xmlParserAPIs.jar:/usr/share/java/netty.jar:/usr/share/java/slf4j-api.jar:/usr/share/java/slf4j-log4j12.jar:/usr/share/java/zookeeper.jar -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.local.only=false -Dzookeeper.log.dir=/var/log/zookeeper -Dzookeeper.root.logger=INFO,CONSOLE,ROLLINGFILE org.apache.zookeeper.server.quorum.QuorumPeerMain /etc/zookeeper/conf/zoo.cfg
root 12907 30324 0 16:07 pts/3 00:00:00 grep --color=auto zoo
root@contrail-node7:/opt/contrail/utils# contrail-status
== Contrail vRouter ==
supervisor-vrouter: active
Fix has done already in R2.21.x
I will add changes to remaining branches