[root@overcloud-controller-0 cluster]# traceroute overcloud-controller-2 traceroute to overcloud-controller-2 (172.17.0.14), 30 hops max, 60 byte packets 1 overcloud-controller-2.localdomain (172.17.0.14) 0.787 ms !X 0.681 ms !X 0.635 ms !X [root@overcloud-controller-0 cluster]# /usr/bin/python /sbin/pcs --debug cluster auth overcloud-controller-0 overcloud-controller-1 overcloud-controller-2 -u hacluster -p eiiAvTbJGvbPVokW --force Running: /usr/bin/ruby -I/usr/lib/pcsd/ /usr/lib/pcsd/pcsd-cli.rb auth --Debug Input Start-- {"username": "hacluster", "local": false, "nodes": ["overcloud-controller-2", "overcloud-controller-1", "overcloud-controller-0"], "password": "eiiAvTbJGvbPVokW", "force": true} --Debug Input End-- Return Value: 0 --Debug Output Start-- { "status": "ok", "data": { "auth_responses": { "overcloud-controller-2": { "status": "noresponse" }, "overcloud-controller-1": { "status": "noresponse" }, "overcloud-controller-0": { "status": "ok", "token": "8bc0f73a-394d-4dab-ba04-de4018df4a5f" } }, "sync_successful": true, "sync_nodes_err": [ ], "sync_responses": { } }, "log": [ "I, [2017-03-10T14:20:08.265162 #2270] INFO -- : PCSD Debugging enabled\n", "D, [2017-03-10T14:20:08.265251 #2270] DEBUG -- : Did not detect RHEL 6\n", "I, [2017-03-10T14:20:08.265296 #2270] INFO -- : Running: /usr/sbin/corosync-cmapctl totem.cluster_name\n", "I, [2017-03-10T14:20:08.265329 #2270] INFO -- : CIB USER: hacluster, groups: \n", "D, [2017-03-10T14:20:08.268641 #2270] DEBUG -- : []\n", "D, [2017-03-10T14:20:08.268695 #2270] DEBUG -- : [\"Failed to initialize the cmap API. Error CS_ERR_LIBRARY\\n\"]\n", "D, [2017-03-10T14:20:08.268754 #2270] DEBUG -- : Duration: 0.003288473s\n", "I, [2017-03-10T14:20:08.268822 #2270] INFO -- : Return Value: 1\n", "W, [2017-03-10T14:20:08.268882 #2270] WARN -- : Cannot read config 'corosync.conf' from '/etc/corosync/corosync.conf': No such file\n", "W, [2017-03-10T14:20:08.268947 #2270] WARN -- : Cannot read config 'corosync.conf' from '/etc/corosync/corosync.conf': No such file or directory - /etc/corosync/corosync.conf\n", "I, [2017-03-10T14:20:08.272840 #2270] INFO -- : No response from: overcloud-controller-2 request: /auth, exception: No route to host - connect(2)\n", "I, [2017-03-10T14:20:08.272969 #2270] INFO -- : No response from: overcloud-controller-1 request: /auth, exception: No route to host - connect(2)\n", "I, [2017-03-10T14:20:08.429989 #2270] INFO -- : Running: /usr/sbin/pcs status nodes corosync\n", "I, [2017-03-10T14:20:08.430061 #2270] INFO -- : CIB USER: hacluster, groups: \n", "D, [2017-03-10T14:20:09.100646 #2270] DEBUG -- : []\n", "D, [2017-03-10T14:20:09.100763 #2270] DEBUG -- : [\"Error: Unable to read /etc/corosync/corosync.conf: No such file or directory\\n\"]\n", "D, [2017-03-10T14:20:09.100818 #2270] DEBUG -- : Duration: 0.670553772s\n", "I, [2017-03-10T14:20:09.100913 #2270] INFO -- : Return Value: 1\n", "I, [2017-03-10T14:20:09.101561 #2270] INFO -- : Saved config 'tokens' version 665 327998fb875b65c762f66026216a4fa530011074 to '/var/lib/pcsd/tokens'\n", "I, [2017-03-10T14:20:09.106627 #2270] INFO -- : No response from: overcloud-controller-1 request: /auth, exception: No route to host - connect(2)\n", "I, [2017-03-10T14:20:09.106757 #2270] INFO -- : No response from: overcloud-controller-2 request: /auth, exception: No route to host - connect(2)\n" ] } --Debug Output End-- Error: Unable to communicate with overcloud-controller-2 Error: Unable to communicate with overcloud-controller-1 overcloud-controller-0: Authorized [root@overcloud-controller-0 cluster]# iptables -nL Chain INPUT (policy ACCEPT) target prot opt source destination ACCEPT all -- 0.0.0.0/0 0.0.0.0/0 state RELATED,ESTABLISHED ACCEPT icmp -- 0.0.0.0/0 0.0.0.0/0 ACCEPT all -- 0.0.0.0/0 0.0.0.0/0 ACCEPT tcp -- 0.0.0.0/0 0.0.0.0/0 state NEW tcp dpt:22 REJECT all -- 0.0.0.0/0 0.0.0.0/0 reject-with icmp-host-prohibited Chain FORWARD (policy ACCEPT) target prot opt source destination REJECT all -- 0.0.0.0/0 0.0.0.0/0 reject-with icmp-host-prohibited Chain OUTPUT (policy ACCEPT) target prot opt source destination [root@overcloud-controller-0 cluster]# /usr/bin/python /sbin/pcs --debug cluster auth overcloud-controller-0 overcloud-controller-1 overcloud-controller-2 -u hacluster -p eiiAvTbJGvbPVokW --force Running: /usr/bin/ruby -I/usr/lib/pcsd/ /usr/lib/pcsd/pcsd-cli.rb auth --Debug Input Start-- {"username": "hacluster", "local": false, "nodes": ["overcloud-controller-2", "overcloud-controller-1", "overcloud-controller-0"], "password": "eiiAvTbJGvbPVokW", "force": true} --Debug Input End-- Return Value: 0 --Debug Output Start-- { "status": "ok", "data": { "auth_responses": { "overcloud-controller-2": { "status": "noresponse" }, "overcloud-controller-0": { "status": "ok", "token": "d065a17a-38a7-4c8b-85c8-361be725fe0d" }, "overcloud-controller-1": { "status": "ok", "token": "c65b910d-f45c-43ba-acd2-95a7158dcf5b" } }, "sync_successful": true, "sync_nodes_err": [ ], "sync_responses": { } }, "log": [ "I, [2017-03-10T14:21:54.395322 #2803] INFO -- : PCSD Debugging enabled\n", "D, [2017-03-10T14:21:54.395412 #2803] DEBUG -- : Did not detect RHEL 6\n", "I, [2017-03-10T14:21:54.395458 #2803] INFO -- : Running: /usr/sbin/corosync-cmapctl totem.cluster_name\n", "I, [2017-03-10T14:21:54.395515 #2803] INFO -- : CIB USER: hacluster, groups: \n", "D, [2017-03-10T14:21:54.398678 #2803] DEBUG -- : []\n", "D, [2017-03-10T14:21:54.398761 #2803] DEBUG -- : [\"Failed to initialize the cmap API. Error CS_ERR_LIBRARY\\n\"]\n", "D, [2017-03-10T14:21:54.398839 #2803] DEBUG -- : Duration: 0.003136466s\n", "I, [2017-03-10T14:21:54.398931 #2803] INFO -- : Return Value: 1\n", "W, [2017-03-10T14:21:54.399014 #2803] WARN -- : Cannot read config 'corosync.conf' from '/etc/corosync/corosync.conf': No such file\n", "W, [2017-03-10T14:21:54.399102 #2803] WARN -- : Cannot read config 'corosync.conf' from '/etc/corosync/corosync.conf': No such file or directory - /etc/corosync/corosync.conf\n", "I, [2017-03-10T14:21:54.404138 #2803] INFO -- : No response from: overcloud-controller-2 request: /auth, exception: No route to host - connect(2)\n", "I, [2017-03-10T14:21:54.577395 #2803] INFO -- : Running: /usr/sbin/pcs status nodes corosync\n", "I, [2017-03-10T14:21:54.577537 #2803] INFO -- : CIB USER: hacluster, groups: \n", "D, [2017-03-10T14:21:55.259455 #2803] DEBUG -- : []\n", "D, [2017-03-10T14:21:55.259631 #2803] DEBUG -- : [\"Error: Unable to read /etc/corosync/corosync.conf: No such file or directory\\n\"]\n", "D, [2017-03-10T14:21:55.259714 #2803] DEBUG -- : Duration: 0.681941214s\n", "I, [2017-03-10T14:21:55.259829 #2803] INFO -- : Return Value: 1\n", "I, [2017-03-10T14:21:55.260646 #2803] INFO -- : Saved config 'tokens' version 683 e4e0e327c605a3bcfa73ad618062230b212e068d to '/var/lib/pcsd/tokens'\n", "I, [2017-03-10T14:21:55.263668 #2803] INFO -- : No response from: overcloud-controller-2 request: /auth, exception: No route to host - connect(2)\n" ] } --Debug Output End--