Version : R5.0 build-33 ( Openshift-3.7 )
This issue not seen when non-HA and seen only when HA on all of the 3 nodes.
[root@b3s2 ~]#
[root@b3s2 ~]# oc get pods -o wide
NAME READY STATUS RESTARTS AGE IP NODE
config-zookeeper-6ph79 1/1 Running 0 1h 10.84.23.1 b3s1
config-zookeeper-ncnrr 1/1 Running 0 1h 10.84.23.4 b3s4
config-zookeeper-x2p8l 1/1 Running 0 1h 10.84.23.2 b3s2
contrail-agent-fb92z 2/2 Running 6 1h 10.84.23.6 b3s6
contrail-agent-mjsk4 2/2 Running 6 1h 10.84.23.5 b3s5
contrail-analytics-7lgw2 5/7 CrashLoopBackOff 44 1h 10.84.23.1 b3s1
contrail-analytics-f4kwj 5/7 CrashLoopBackOff 45 1h 10.84.23.4 b3s4
contrail-analytics-tkb5q 5/7 CrashLoopBackOff 45 1h 10.84.23.2 b3s2
contrail-analytics-zookeeper-4km64 1/1 Running 0 1h 10.84.23.4 b3s4
contrail-analytics-zookeeper-5p8ml 1/1 Running 0 1h 10.84.23.2 b3s2
contrail-analytics-zookeeper-nc6lr 1/1 Running 0 1h 10.84.23.1 b3s1
contrail-analyticsdb-6zhd6 1/1 Running 0 1h 10.84.23.1 b3s1
contrail-analyticsdb-vxhpn 1/1 Running 0 1h 10.84.23.2 b3s2
contrail-analyticsdb-xxg9p 1/1 Running 0 1h 10.84.23.4 b3s4
contrail-configdb-5x9xg 1/1 Running 0 1h 10.84.23.2 b3s2
contrail-configdb-jhqr2 1/1 Running 0 1h 10.84.23.4 b3s4
contrail-configdb-mwx8x 1/1 Running 0 1h 10.84.23.1 b3s1
contrail-controller-config-4ph7x 5/5 Running 12 1h 10.84.23.2 b3s2
contrail-controller-config-gkqfk 5/5 Running 10 1h 10.84.23.4 b3s4
contrail-controller-config-jgl9f 5/5 Running 8 1h 10.84.23.1 b3s1
contrail-controller-control-clfv2 4/4 Running 1 1h 10.84.23.1 b3s1
contrail-controller-control-cwd5g 4/4 Running 1 1h 10.84.23.4 b3s4
contrail-controller-control-lxz6j 4/4 Running 1 1h 10.84.23.2 b3s2
contrail-controller-webui-4h7nk 2/2 Running 0 1h 10.84.23.1 b3s1
contrail-controller-webui-dcv2l 2/2 Running 0 1h 10.84.23.4 b3s4
contrail-controller-webui-tchg4 2/2 Running 0 1h 10.84.23.2 b3s2
contrail-kube-manager-4lxgg 1/1 Running 0 1h 10.84.23.1 b3s1
contrail-kube-manager-99vqk 1/1 Running 0 1h 10.84.23.2 b3s2
contrail-kube-manager-flsdx 1/1 Running 0 1h 10.84.23.4 b3s4
kafka-f2rnl 0/1 CrashLoopBackOff 17 1h 10.84.23.2 b3s2
kafka-kp4h8 0/1 CrashLoopBackOff 17 1h 10.84.23.4 b3s4
kafka-nsxlt 0/1 CrashLoopBackOff 17 1h 10.84.23.1 b3s1
rabbitmq-8v5zp 1/1 Running 0 1h 10.84.23.4 b3s4
rabbitmq-p957c 1/1 Running 0 1h 10.84.23.1 b3s1
rabbitmq-pdmws 1/1 Running 0 1h 10.84.23.2 b3s2
redis-mjswt 1/1 Running 0 1h 10.84.23.2 b3s2
redis-nfdf6 1/1 Running 0 1h 10.84.23.1 b3s1
redis-vfb7z 1/1 Running 0 1h 10.84.23.4 b3s4
[root@b3s2 ~]#
[root@b3s2 ~]# oc describe pod contrail-analytics-7lgw2
Name: contrail-analytics-7lgw2
Namespace: kube-system
Node: b3s1/10.84.23.1
Start Time: Mon, 23 Apr 2018 14:09:39 -0700
Labels: app=contrail-analytics
controller-revision-hash=2566708624
pod-template-generation=1
Annotations: kubernetes.io/created-by={"kind":"SerializedReference","apiVersion":"v1","reference":{"kind":"DaemonSet","namespace":"kube-system","name":"contrail-analytics","uid":"a18508ab-473a-11e8-98ff-00259093d0...
openshift.io/scc=privileged
Status: Running
IP: 10.84.23.1
Created By: DaemonSet/contrail-analytics
Controlled By: DaemonSet/contrail-analytics
Containers:
contrail-analytics-api:
Container ID: docker://97d6b3c49fb3cf2a49fdadac53c036095c130f52821b892846f221d760de317d
Image: ci-repo.englab.juniper.net:5000/contrail-analytics-api:ocata-5.0-33
Image ID: docker-pullable://ci-repo.englab.juniper.net:5000/contrail-analytics-api@sha256:e1fefdb437665d935b9cf1bd6fe969389f7e8e1a1c22bf65e9a242f9974a243d
Port: <none>
State: Running
Started: Mon, 23 Apr 2018 14:19:49 -0700
Ready: True
Restart Count: 0
Environment Variables from:
env ConfigMap Optional: false
contrail-analytics-zookeeper ConfigMap Optional: false
Environment: <none>
Mounts:
/var/log/contrail from analytics-logs (rw)
/var/run/secrets/kubernetes.io/serviceaccount from default-token-qjhnt (ro)
contrail-analytics-collector:
Container ID: docker://2850b9a153b84d29e94fa7e38a19660d3f5c90ae51791c7d569c68375bd2e721
Image: ci-repo.englab.juniper.net:5000/contrail-analytics-collector:ocata-5.0-33
Image ID: docker-pullable://ci-repo.englab.juniper.net:5000/contrail-analytics-collector@sha256:8597706dc2668a01a5f9eec4b96f171a1a604c529bf8bfd16852b62f090167de
Port: <none>
State: Running
Started: Mon, 23 Apr 2018 14:21:40 -0700
Ready: True
Restart Count: 0
Environment Variables from:
env ConfigMap Optional: false
contrail-analytics-zookeeper ConfigMap Optional: false
Environment: <none>
Mounts:
/var/log/contrail from analytics-logs (rw)
/var/run/secrets/kubernetes.io/serviceaccount from default-token-qjhnt (ro)
contrail-analytics-alarm-gen:
Container ID: docker://2be9b97ffe31561ef9abac3a3a535b079c7254b0ff7ba92aec875b48569dc21f
Image: ci-repo.englab.juniper.net:5000/contrail-analytics-alarm-gen:ocata-5.0-33
Image ID: docker-pullable://ci-repo.englab.juniper.net:5000/contrail-analytics-alarm-gen@sha256:64caeb1b68edd4186d6618e653161166b6f0482c1c89c29b0766b29cf4701d9b
Port: <none>
State: Running
Started: Mon, 23 Apr 2018 15:01:44 -0700
Last State: Terminated
Reason: Error
Exit Code: 2
Started: Mon, 23 Apr 2018 14:53:30 -0700
Finished: Mon, 23 Apr 2018 14:58:46 -0700
Ready: True
Restart Count: 6
Environment Variables from:
env ConfigMap Optional: false
contrail-analytics-zookeeper ConfigMap Optional: false
Environment: <none>
Mounts:
/var/log/contrail from analytics-logs (rw)
/var/run/secrets/kubernetes.io/serviceaccount from default-token-qjhnt (ro)
contrail-analytics-query-engine:
Container ID: docker://a79f968949c0cc634ddeab6b0714367981d739758949856353d78aa981ec5719
Image: ci-repo.englab.juniper.net:5000/contrail-analytics-query-engine:ocata-5.0-33
Image ID: docker-pullable://ci-repo.englab.juniper.net:5000/contrail-analytics-query-engine@sha256:95f1390b86c6ff5f8d492e613b61d031a715f5769d5ae8fe10ac7dd9899f616d
Port: <none>
State: Running
Started: Mon, 23 Apr 2018 14:24:04 -0700
Ready: True
Restart Count: 0
Environment Variables from:
env ConfigMap Optional: false
contrail-analytics-zookeeper ConfigMap Optional: false
Environment: <none>
Mounts:
/var/log/contrail from analytics-logs (rw)
/var/run/secrets/kubernetes.io/serviceaccount from default-token-qjhnt (ro)
contrail-analytics-snmp-collector:
Container ID: docker://b735a38edbe2b65b2d6f58506cec6040fa7266e5f378071febebd51f308c9575
Image: ci-repo.englab.juniper.net:5000/contrail-analytics-snmp-collector:ocata-5.0-33
Image ID: docker-pullable://ci-repo.englab.juniper.net:5000/contrail-analytics-snmp-collector@sha256:6854a096a7fa1a1f1cff053083ae65c4d3f33a435ca15e22f57b1c62a9662a78
Port: <none>
State: Waiting
Reason: CrashLoopBackOff
Last State: Terminated
Reason: Error
Exit Code: 1
Started: Mon, 23 Apr 2018 15:04:57 -0700
Finished: Mon, 23 Apr 2018 15:05:20 -0700
Ready: False
Restart Count: 11
Environment Variables from:
env ConfigMap Optional: false
contrail-analytics-zookeeper ConfigMap Optional: false
Environment: <none>
Mounts:
/var/log/contrail from analytics-logs (rw)
/var/run/secrets/kubernetes.io/serviceaccount from default-token-qjhnt (ro)
contrail-analytics-topology:
Container ID: docker://6215043a315e6146ed46bc4a40999c709fe159f4d1f4074950db389a8a89ff5b
Image: ci-repo.englab.juniper.net:5000/contrail-analytics-topology:ocata-5.0-33
Image ID: docker-pullable://ci-repo.englab.juniper.net:5000/contrail-analytics-topology@sha256:921dfec9a994f4dfe2ec7e81eb7eed709b9ca310c376571df6f54321b88fd110
Port: <none>
State: Running
Started: Mon, 23 Apr 2018 15:06:04 -0700
Last State: Terminated
Reason: Error
Exit Code: 1
Started: Mon, 23 Apr 2018 15:00:32 -0700
Finished: Mon, 23 Apr 2018 15:00:55 -0700
Ready: True
Restart Count: 11
Environment Variables from:
env ConfigMap Optional: false
contrail-analytics-zookeeper ConfigMap Optional: false
Environment: <none>
Mounts:
/var/log/contrail from analytics-logs (rw)
/var/run/secrets/kubernetes.io/serviceaccount from default-token-qjhnt (ro)
contrail-analytics-nodemgr:
Container ID: docker://08b34232c509aa05e6805c0792dec962289e151ea47a123e3275719db1565d1f
Image: ci-repo.englab.juniper.net:5000/contrail-nodemgr:ocata-5.0-33
Image ID: docker-pullable://ci-repo.englab.juniper.net:5000/contrail-nodemgr@sha256:9f182bece168c42283b7683f90d48e109b8cb5221ae09ec7c7f169a9046c6fae
Port: <none>
State: Running
Started: Mon, 23 Apr 2018 14:39:27 -0700
Last State: Terminated
Reason: Error
Exit Code: 1
Started: Mon, 23 Apr 2018 14:32:00 -0700
Finished: Mon, 23 Apr 2018 14:38:55 -0700
Ready: True
Restart Count: 3
Environment Variables from:
env ConfigMap Optional: false
nodemgr-config ConfigMap Optional: false
contrail-analytics-zookeeper ConfigMap Optional: false
Environment:
NODE_TYPE: analytics
Mounts:
/mnt from docker-unix-socket (rw)
/var/log/contrail from analytics-logs (rw)
/var/run/secrets/kubernetes.io/serviceaccount from default-token-qjhnt (ro)
Conditions:
Type Status
Initialized True
Ready False
PodScheduled True
Volumes:
analytics-logs:
Type: HostPath (bare host directory volume)
Path: /var/log/contrail/analytics
docker-unix-socket:
Type: HostPath (bare host directory volume)
Path: /var/run
default-token-qjhnt:
Type: Secret (a volume populated by a Secret)
SecretName: default-token-qjhnt
Optional: false
QoS Class: BestEffort
Node-Selectors: <none>
Tolerations: node-role.kubernetes.io/master:NoSchedule
node.alpha.kubernetes.io/notReady:NoExecute
node.alpha.kubernetes.io/unreachable:NoExecute
Events:
FirstSeen LastSeen Count From SubObjectPath Type Reason Message
--------- -------- ----- ---- ------------- -------- ------ -------
56m 56m 1 kubelet, b3s1 Normal SuccessfulMountVolume MountVolume.SetUp succeeded for volume "analytics-logs"
56m 56m 1 kubelet, b3s1 Normal SuccessfulMountVolume MountVolume.SetUp succeeded for volume "docker-unix-socket"
56m 56m 1 kubelet, b3s1 Normal SuccessfulMountVolume MountVolume.SetUp succeeded for volume "default-token-qjhnt"
54m 54m 1 kubelet, b3s1 spec.containers{contrail-analytics-api} Normal Pulling pulling image "ci-repo.englab.juniper.net:5000/contrail-analytics-api:ocata-5.0-33"
46m 46m 1 kubelet, b3s1 spec.containers{contrail-analytics-api} Normal Pulled Successfully pulled image "ci-repo.englab.juniper.net:5000/contrail-analytics-api:ocata-5.0-33"
46m 46m 1 kubelet, b3s1 spec.containers{contrail-analytics-api} Normal Created Created container
45m 45m 1 kubelet, b3s1 spec.containers{contrail-analytics-api} Normal Started Started container
45m 45m 1 kubelet, b3s1 spec.containers{contrail-analytics-collector} Normal Pulling pulling image "ci-repo.englab.juniper.net:5000/contrail-analytics-collector:ocata-5.0-33"
45m 45m 1 kubelet, b3s1 spec.containers{contrail-analytics-collector} Normal Pulled Successfully pulled image "ci-repo.englab.juniper.net:5000/contrail-analytics-collector:ocata-5.0-33"
44m 44m 1 kubelet, b3s1 spec.containers{contrail-analytics-collector} Normal Created Created container
44m 44m 1 kubelet, b3s1 spec.containers{contrail-analytics-collector} Normal Started Started container
44m 44m 1 kubelet, b3s1 spec.containers{contrail-analytics-alarm-gen} Normal Pulling pulling image "ci-repo.englab.juniper.net:5000/contrail-analytics-alarm-gen:ocata-5.0-33"
44m 44m 1 kubelet, b3s1 spec.containers{contrail-analytics-alarm-gen} Normal Pulled Successfully pulled image "ci-repo.englab.juniper.net:5000/contrail-analytics-alarm-gen:ocata-5.0-33"
43m 43m 1 kubelet, b3s1 spec.containers{contrail-analytics-alarm-gen} Normal Created Created container
43m 43m 1 kubelet, b3s1 spec.containers{contrail-analytics-alarm-gen} Normal Started Started container
43m 43m 1 kubelet, b3s1 spec.containers{contrail-analytics-query-engine} Normal Pulling pulling image "ci-repo.englab.juniper.net:5000/contrail-analytics-query-engine:ocata-5.0-33"
42m 42m 1 kubelet, b3s1 spec.containers{contrail-analytics-query-engine} Normal Pulled Successfully pulled image "ci-repo.englab.juniper.net:5000/contrail-analytics-query-engine:ocata-5.0-33"
42m 42m 1 kubelet, b3s1 spec.containers{contrail-analytics-query-engine} Normal Created Created container
42m 42m 1 kubelet, b3s1 spec.containers{contrail-analytics-query-engine} Normal Started Started container
42m 42m 1 kubelet, b3s1 spec.containers{contrail-analytics-snmp-collector} Normal Pulling pulling image "ci-repo.englab.juniper.net:5000/contrail-analytics-snmp-collector:ocata-5.0-33"
41m 41m 1 kubelet, b3s1 spec.containers{contrail-analytics-snmp-collector} Normal Pulled Successfully pulled image "ci-repo.englab.juniper.net:5000/contrail-analytics-snmp-collector:ocata-5.0-33"
41m 41m 1 kubelet, b3s1 spec.containers{contrail-analytics-snmp-collector} Normal Created Created container
41m 41m 1 kubelet, b3s1 spec.containers{contrail-analytics-snmp-collector} Normal Started Started container
41m 41m 1 kubelet, b3s1 spec.containers{contrail-analytics-topology} Normal Pulling pulling image "ci-repo.englab.juniper.net:5000/contrail-analytics-topology:ocata-5.0-33"
41m 41m 1 kubelet, b3s1 spec.containers{contrail-analytics-topology} Normal Pulled Successfully pulled image "ci-repo.englab.juniper.net:5000/contrail-analytics-topology:ocata-5.0-33"
40m 40m 1 kubelet, b3s1 spec.containers{contrail-analytics-topology} Normal Created Created container
40m 40m 1 kubelet, b3s1 spec.containers{contrail-analytics-nodemgr} Normal Pulled Container image "ci-repo.englab.juniper.net:5000/contrail-nodemgr:ocata-5.0-33" already present on machine
40m 36m 3 kubelet, b3s1 spec.containers{contrail-analytics-topology} Normal Started Started container
35m 11m 100 kubelet, b3s1 spec.containers{contrail-analytics-snmp-collector} Warning BackOff Back-off restarting failed container
31m 6m 20 kubelet, b3s1 spec.containers{contrail-analytics-alarm-gen} Warning BackOff Back-off restarting failed container
40m 1m 11 kubelet, b3s1 spec.containers{contrail-analytics-snmp-collector} Normal Pulled Container image "ci-repo.englab.juniper.net:5000/contrail-analytics-snmp-collector:ocata-5.0-33" already present on machine
[root@b3s2 ~]#
[root@b3s2 ~]#
[root@b3s2 ~]#
[root@b3s2 ~]# oc logs contrail-analytics-7lgw2 -c contrail-analytics-nodemgr -n kube-system
INFO: Provisioning cmdline: python /opt/contrail/utils/provision_analytics_node.py --oper add --host_name b3s1 --host_ip 10.84.23.1 --api_server_ip {for each node from the list: 10.84.23.1,10.84.23.2,10.84.23.4} --api_server_port 8082
INFO: Provisioning attempt 1 of 10 (pause 3)
INFO: Provisioning was succeeded
04/23/2018 09:39:33 PM [contrail-analytics-nodemgr]: SANDESH: CONNECT TO COLLECTOR: True
04/23/2018 09:39:33 PM [contrail-analytics-nodemgr]: SANDESH: Logging: LEVEL: [SYS_INFO] -> [SYS_NOTICE]
[root@b3s2 ~]#
[root@b3s2 ~]#
[root@b3s2 ~]#
[root@b3s2 ~]#
[root@b3s2 ~]#
[root@b3s2 ~]#
[root@b3s2 ~]#
[root@b3s2 ~]# oc logs contrail-analytics-7lgw2 -c contrail-analytics-collector -n kube-system
2018-04-23 21:21:46,117:1(0x7f1bd26dc8c0):ZOO_INFO@log_env@726: Client environment:zookeeper.version=zookeeper C client 3.4.8
2018-04-23 21:21:46,117:1(0x7f1bd26dc8c0):ZOO_INFO@log_env@730: Client environment:host.name=b3s1
2018-04-23 21:21:46,117:1(0x7f1bd26dc8c0):ZOO_INFO@log_env@737: Client environment:os.name=Linux
2018-04-23 21:21:46,117:1(0x7f1bd26dc8c0):ZOO_INFO@log_env@738: Client environment:os.arch=3.10.0-693.21.1.el7.x86_64
2018-04-23 21:21:46,117:1(0x7f1bd26dc8c0):ZOO_INFO@log_env@739: Client environment:os.version=#1 SMP Wed Mar 7 19:03:37 UTC 2018
2018-04-23 21:21:46,150:1(0x7f1bd26dc8c0):ZOO_INFO@log_env@747: Client environment:user.name=(null)
2018-04-23 21:21:46,150:1(0x7f1bd26dc8c0):ZOO_INFO@log_env@755: Client environment:user.home=/root
2018-04-23 21:21:46,150:1(0x7f1bd26dc8c0):ZOO_INFO@log_env@767: Client environment:user.dir=/
2018-04-23 21:21:46,150:1(0x7f1bd26dc8c0):ZOO_INFO@zookeeper_init@800: Initiating client connection, host=10.84.23.1:2182,10.84.23.2:2182,10.84.23.4:2182 sessionTimeout=10000 watcher=(nil) sessionId=0 sessionPasswd=<null> context=(nil) flags=0
2018-04-23 21:21:46,150:1(0x7f1bd26dc8c0):ZOO_DEBUG@start_threads@221: starting threads...
2018-04-23 21:21:46,150:1(0x7f1b67bfe700):ZOO_DEBUG@do_completion@459: started completion thread
2018-04-23 21:21:46,150:1(0x7f1b74bf2700):ZOO_DEBUG@do_io@367: started IO thread
2018-04-23 21:21:46,150:1(0x7f1b74bf2700):ZOO_INFO@check_events@1728: initiated connection to server [10.84.23.4:2182]
2018-04-23 21:21:46,151:1(0x7f1b74bf2700):ZOO_ERROR@handle_socket_error_msg@1746: Socket [10.84.23.4:2182] zk retcode=-4, errno=112(Host is down): failed while receiving a server response
2018-04-23 21:21:46,151:1(0x7f1b74bf2700):ZOO_INFO@check_events@1728: initiated connection to server [10.84.23.1:2182]
2018-04-23 21:21:46,152:1(0x7f1b74bf2700):ZOO_ERROR@handle_socket_error_msg@1746: Socket [10.84.23.1:2182] zk retcode=-4, errno=112(Host is down): failed while receiving a server response
2018-04-23 21:21:46,152:1(0x7f1b74bf2700):ZOO_INFO@check_events@1728: initiated connection to server [10.84.23.2:2182]
2018-04-23 21:21:46,153:1(0x7f1b74bf2700):ZOO_ERROR@handle_socket_error_msg@1746: Socket [10.84.23.2:2182] zk retcode=-4, errno=112(Host is down): failed while receiving a server response
2018-04-23 21:21:49,490:1(0x7f1b74bf2700):ZOO_INFO@check_events@1728: initiated connection to server [10.84.23.4:2182]
2018-04-23 21:21:49,491:1(0x7f1b74bf2700):ZOO_ERROR@handle_socket_error_msg@1746: Socket [10.84.23.4:2182] zk retcode=-4, errno=112(Host is down): failed while receiving a server response
2018-04-23 21:21:49,491:1(0x7f1b74bf2700):ZOO_INFO@check_events@1728: initiated connection to server [10.84.23.1:2182]
2018-04-23 21:21:49,492:1(0x7f1b74bf2700):ZOO_ERROR@handle_socket_error_msg@1746: Socket [10.84.23.1:2182] zk retcode=-4, errno=112(Host is down): failed while receiving a server response
2018-04-23 21:21:49,492:1(0x7f1b74bf2700):ZOO_INFO@check_events@1728: initiated connection to server [10.84.23.2:2182]
2018-04-23 21:21:49,493:1(0x7f1b74bf2700):ZOO_ERROR@handle_socket_error_msg@1746: Socket [10.84.23.2:2182] zk retcode=-4, errno=112(Host is down): failed while receiving a server response
2018-04-23 21:21:52,829:1(0x7f1b74bf2700):ZOO_INFO@check_events@1728: initiated connection to server [10.84.23.4:2182]
2018-04-23 21:21:52,830:1(0x7f1b74bf2700):ZOO_ERROR@handle_socket_error_msg@1746: Socket [10.84.23.4:2182] zk retcode=-4, errno=112(Host is down): failed while receiving a server response
2018-04-23 21:21:52,830:1(0x7f1b74bf2700):ZOO_INFO@check_events@1728: initiated connection to server [10.84.23.1:2182]
2018-04-23 21:21:52,831:1(0x7f1b74bf2700):ZOO_ERROR@handle_socket_error_msg@1746: Socket [10.84.23.1:2182] zk retcode=-4, errno=112(Host is down): failed while receiving a server response
2018-04-23 21:21:52,831:1(0x7f1b74bf2700):ZOO_INFO@check_events@1728: initiated connection to server [10.84.23.2:2182]
2018-04-23 21:21:52,832:1(0x7f1b74bf2700):ZOO_ERROR@handle_socket_error_msg@1746: Socket [10.84.23.2:2182] zk retcode=-4, errno=112(Host is down): failed while receiving a server response
2018-04-23 21:21:56,169:1(0x7f1b74bf2700):ZOO_INFO@check_events@1728: initiated connection to server [10.84.23.4:2182]
2018-04-23 21:21:56,169:1(0x7f1b74bf2700):ZOO_ERROR@handle_socket_error_msg@1746: Socket [10.84.23.4:2182] zk retcode=-4, errno=112(Host is down): failed while receiving a server response
2018-04-23 21:21:56,170:1(0x7f1b74bf2700):ZOO_INFO@check_events@1728: initiated connection to server [10.84.23.1:2182]
2018-04-23 21:21:56,170:1(0x7f1b74bf2700):ZOO_ERROR@handle_socket_error_msg@1746: Socket [10.84.23.1:2182] zk retcode=-4, errno=112(Host is down): failed while receiving a server response
2018-04-23 21:21:56,170:1(0x7f1b74bf2700):ZOO_INFO@check_events@1728: initiated connection to server [10.84.23.2:2182]
2018-04-23 21:21:56,171:1(0x7f1b74bf2700):ZOO_ERROR@handle_socket_error_msg@1746: Socket [10.84.23.2:2182] zk retcode=-4, errno=112(Host is down): failed while receiving a server response
2018-04-23 21:21:59,508:1(0x7f1b74bf2700):ZOO_INFO@check_events@1728: initiated connection to server [10.84.23.4:2182]
2018-04-23 21:21:59,509:1(0x7f1b74bf2700):ZOO_ERROR@handle_socket_error_msg@1746: Socket [10.84.23.4:2182] zk retcode=-4, errno=112(Host is down): failed while receiving a server response
2018-04-23 21:21:59,509:1(0x7f1b74bf2700):ZOO_INFO@check_events@1728: initiated connection to server [10.84.23.1:2182]
2018-04-23 21:21:59,510:1(0x7f1b74bf2700):ZOO_ERROR@handle_socket_error_msg@1746: Socket [10.84.23.1:2182] zk retcode=-4, errno=112(Host is down): failed while receiving a server response
2018-04-23 21:21:59,510:1(0x7f1b74bf2700):ZOO_INFO@check_events@1728: initiated connection to server [10.84.23.2:2182]
2018-04-23 21:21:59,511:1(0x7f1b74bf2700):ZOO_ERROR@handle_socket_error_msg@1746: Socket [10.84.23.2:2182] zk retcode=-4, errno=112(Host is down): failed while receiving a server response
2018-04-23 21:22:02,846:1(0x7f1b74bf2700):ZOO_INFO@check_events@1728: initiated connection to server [10.84.23.4:2182]
2018-04-23 21:22:02,846:1(0x7f1b74bf2700):ZOO_ERROR@handle_socket_error_msg@1746: Socket [10.84.23.4:2182] zk retcode=-4, errno=112(Host is down): failed while receiving a server response
2018-04-23 21:22:02,847:1(0x7f1b74bf2700):ZOO_INFO@check_events@1728: initiated connection to server [10.84.23.1:2182]
2018-04-23 21:22:02,847:1(0x7f1b74bf2700):ZOO_ERROR@handle_socket_error_msg@1746: Socket [10.84.23.1:2182] zk retcode=-4, errno=112(Host is down): failed while receiving a server response
2018-04-23 21:22:02,847:1(0x7f1b74bf2700):ZOO_INFO@check_events@1728: initiated connection to server [10.84.23.2:2182]
2018-04-23 21:22:02,848:1(0x7f1b74bf2700):ZOO_ERROR@handle_socket_error_msg@1746: Socket [10.84.23.2:2182] zk retcode=-4, errno=112(Host is down): failed while receiving a server response
2018-04-23 21:22:06,185:1(0x7f1b74bf2700):ZOO_INFO@check_events@1728: initiated connection to server [10.84.23.4:2182]
2018-04-23 21:22:06,185:1(0x7f1b74bf2700):ZOO_ERROR@handle_socket_error_msg@1746: Socket [10.84.23.4:2182] zk retcode=-4, errno=112(Host is down): failed while receiving a server response
2018-04-23 21:22:06,185:1(0x7f1b74bf2700):ZOO_INFO@check_events@1728: initiated connection to server [10.84.23.1:2182]
2018-04-23 21:22:06,186:1(0x7f1b74bf2700):ZOO_ERROR@handle_socket_error_msg@1746: Socket [10.84.23.1:2182] zk retcode=-4, errno=112(Host is down): failed while receiving a server response
2018-04-23 21:22:06,186:1(0x7f1b74bf2700):ZOO_INFO@check_events@1728: initiated connection to server [10.84.23.2:2182]
2018-04-23 21:22:06,187:1(0x7f1b74bf2700):ZOO_ERROR@handle_socket_error_msg@1746: Socket [10.84.23.2:2182] zk retcode=-4, errno=112(Host is down): failed while receiving a server response
2018-04-23 21:22:09,522:1(0x7f1b74bf2700):ZOO_INFO@check_events@1728: initiated connection to server [10.84.23.4:2182]
2018-04-23 21:22:09,522:1(0x7f1b74bf2700):ZOO_ERROR@handle_socket_error_msg@1746: Socket [10.84.23.4:2182] zk retcode=-4, errno=112(Host is down): failed while receiving a server response
2018-04-23 21:22:09,523:1(0x7f1b74bf2700):ZOO_INFO@check_events@1728: initiated connection to server [10.84.23.1:2182]
2018-04-23 21:22:09,523:1(0x7f1b74bf2700):ZOO_ERROR@handle_socket_error_msg@1746: Socket [10.84.23.1:2182] zk retcode=-4, errno=112(Host is down): failed while receiving a server response
2018-04-23 21:22:09,524:1(0x7f1b74bf2700):ZOO_INFO@check_events@1728: initiated connection to server [10.84.23.2:2182]
2018-04-23 21:22:09,524:1(0x7f1b74bf2700):ZOO_ERROR@handle_socket_error_msg@1746: Socket [10.84.23.2:2182] zk retcode=-4, errno=112(Host is down): failed while receiving a server response
2018-04-23 21:22:12,860:1(0x7f1b74bf2700):ZOO_INFO@check_events@1728: initiated connection to server [10.84.23.4:2182]
2018-04-23 21:22:12,861:1(0x7f1b74bf2700):ZOO_ERROR@handle_socket_error_msg@1746: Socket [10.84.23.4:2182] zk retcode=-4, errno=112(Host is down): failed while receiving a server response
2018-04-23 21:22:12,861:1(0x7f1b74bf2700):ZOO_INFO@check_events@1728: initiated connection to server [10.84.23.1:2182]
2018-04-23 21:22:12,862:1(0x7f1b74bf2700):ZOO_ERROR@handle_socket_error_msg@1746: Socket [10.84.23.1:2182] zk retcode=-4, errno=112(Host is down): failed while receiving a server response
2018-04-23 21:22:12,862:1(0x7f1b74bf2700):ZOO_INFO@check_events@1728: initiated connection to server [10.84.23.2:2182]
2018-04-23 21:22:12,863:1(0x7f1b74bf2700):ZOO_ERROR@handle_socket_error_msg@1746: Socket [10.84.23.2:2182] zk retcode=-4, errno=112(Host is down): failed while receiving a server response
2018-04-23 21:22:16,197:1(0x7f1b74bf2700):ZOO_INFO@check_events@1728: initiated connection to server [10.84.23.4:2182]
2018-04-23 21:22:16,198:1(0x7f1b74bf2700):ZOO_ERROR@handle_socket_error_msg@1746: Socket [10.84.23.4:2182] zk retcode=-4, errno=112(Host is down): failed while receiving a server response
2018-04-23 21:22:16,198:1(0x7f1b74bf2700):ZOO_INFO@check_events@1728: initiated connection to server [10.84.23.1:2182]
2018-04-23 21:22:16,199:1(0x7f1b74bf2700):ZOO_ERROR@handle_socket_error_msg@1746: Socket [10.84.23.1:2182] zk retcode=-4, errno=112(Host is down): failed while receiving a server response
2018-04-23 21:22:16,199:1(0x7f1b74bf2700):ZOO_INFO@check_events@1728: initiated connection to server [10.84.23.2:2182]
2018-04-23 21:22:16,200:1(0x7f1b74bf2700):ZOO_ERROR@handle_socket_error_msg@1746: Socket [10.84.23.2:2182] zk retcode=-4, errno=112(Host is down): failed while receiving a server response
2018-04-23 21:22:19,538:1(0x7f1b74bf2700):ZOO_INFO@check_events@1728: initiated connection to server [10.84.23.4:2182]
2018-04-23 21:22:19,539:1(0x7f1b74bf2700):ZOO_ERROR@handle_socket_error_msg@1746: Socket [10.84.23.4:2182] zk retcode=-4, errno=112(Host is down): failed while receiving a server response
2018-04-23 21:22:19,540:1(0x7f1b74bf2700):ZOO_INFO@check_events@1728: initiated connection to server [10.84.23.1:2182]
2018-04-23 21:22:19,541:1(0x7f1b74bf2700):ZOO_ERROR@handle_socket_error_msg@1746: Socket [10.84.23.1:2182] zk retcode=-4, errno=112(Host is down): failed while receiving a server response
2018-04-23 21:22:19,541:1(0x7f1b74bf2700):ZOO_INFO@check_events@1728: initiated connection to server [10.84.23.2:2182]
2018-04-23 21:22:19,541:1(0x7f1b74bf2700):ZOO_ERROR@handle_socket_error_msg@1746: Socket [10.84.23.2:2182] zk retcode=-4, errno=112(Host is down): failed while receiving a server response
2018-04-23 21:22:22,878:1(0x7f1b74bf2700):ZOO_INFO@check_events@1728: initiated connection to server [10.84.23.4:2182]
2018-04-23 21:22:22,879:1(0x7f1b74bf2700):ZOO_ERROR@handle_socket_error_msg@1746: Socket [10.84.23.4:2182] zk retcode=-4, errno=112(Host is down): failed while receiving a server response
2018-04-23 21:22:22,879:1(0x7f1b74bf2700):ZOO_INFO@check_events@1728: initiated connection to server [10.84.23.1:2182]
2018-04-23 21:22:22,879:1(0x7f1b74bf2700):ZOO_ERROR@handle_socket_error_msg@1746: Socket [10.84.23.1:2182] zk retcode=-4, errno=112(Host is down): failed while receiving a server response
2018-04-23 21:22:22,879:1(0x7f1b74bf2700):ZOO_INFO@check_events@1728: initiated connection to server [10.84.23.2:2182]
2018-04-23 21:22:22,880:1(0x7f1b74bf2700):ZOO_ERROR@handle_socket_error_msg@1746: Socket [10.84.23.2:2182] zk retcode=-4, errno=112(Host is down): failed while receiving a server response
2018-04-23 21:22:26,217:1(0x7f1b74bf2700):ZOO_INFO@check_events@1728: initiated connection to server [10.84.23.4:2182]
commit 24d77d7a312a2b7 bb483812ae320ac e6595f80a9
Author: VinayRao <email address hidden>
Date: Tue Apr 24 21:05:23 2018 +0000
1765573-Kafka not coming up on HA