Comment 1 for bug 1672216

[root@overcloud-controller-0 cluster]# traceroute overcloud-controller-2
traceroute to overcloud-controller-2 (172.17.0.14), 30 hops max, 60 byte packets
 1 overcloud-controller-2.localdomain (172.17.0.14) 0.787 ms !X 0.681 ms !X 0.635 ms !X
[root@overcloud-controller-0 cluster]# /usr/bin/python /sbin/pcs --debug cluster auth overcloud-controller-0 overcloud-controller-1 overcloud-controller-2 -u hacluster -p eiiAvTbJGvbPVokW --force
Running: /usr/bin/ruby -I/usr/lib/pcsd/ /usr/lib/pcsd/pcsd-cli.rb auth
--Debug Input Start--
{"username": "hacluster", "local": false, "nodes": ["overcloud-controller-2", "overcloud-controller-1", "overcloud-controller-0"], "password": "eiiAvTbJGvbPVokW", "force": true}
--Debug Input End--
Return Value: 0
--Debug Output Start--
{
  "status": "ok",
  "data": {
    "auth_responses": {
      "overcloud-controller-2": {
        "status": "noresponse"
      },
      "overcloud-controller-1": {
        "status": "noresponse"
      },
      "overcloud-controller-0": {
        "status": "ok",
        "token": "8bc0f73a-394d-4dab-ba04-de4018df4a5f"
      }
    },
    "sync_successful": true,
    "sync_nodes_err": [

    ],
    "sync_responses": {
    }
  },
  "log": [
    "I, [2017-03-10T14:20:08.265162 #2270] INFO -- : PCSD Debugging enabled\n",
    "D, [2017-03-10T14:20:08.265251 #2270] DEBUG -- : Did not detect RHEL 6\n",
    "I, [2017-03-10T14:20:08.265296 #2270] INFO -- : Running: /usr/sbin/corosync-cmapctl totem.cluster_name\n",
    "I, [2017-03-10T14:20:08.265329 #2270] INFO -- : CIB USER: hacluster, groups: \n",
    "D, [2017-03-10T14:20:08.268641 #2270] DEBUG -- : []\n",
    "D, [2017-03-10T14:20:08.268695 #2270] DEBUG -- : [\"Failed to initialize the cmap API. Error CS_ERR_LIBRARY\\n\"]\n",
    "D, [2017-03-10T14:20:08.268754 #2270] DEBUG -- : Duration: 0.003288473s\n",
    "I, [2017-03-10T14:20:08.268822 #2270] INFO -- : Return Value: 1\n",
    "W, [2017-03-10T14:20:08.268882 #2270] WARN -- : Cannot read config 'corosync.conf' from '/etc/corosync/corosync.conf': No such file\n",
    "W, [2017-03-10T14:20:08.268947 #2270] WARN -- : Cannot read config 'corosync.conf' from '/etc/corosync/corosync.conf': No such file or directory - /etc/corosync/corosync.conf\n",
    "I, [2017-03-10T14:20:08.272840 #2270] INFO -- : No response from: overcloud-controller-2 request: /auth, exception: No route to host - connect(2)\n",
    "I, [2017-03-10T14:20:08.272969 #2270] INFO -- : No response from: overcloud-controller-1 request: /auth, exception: No route to host - connect(2)\n",
    "I, [2017-03-10T14:20:08.429989 #2270] INFO -- : Running: /usr/sbin/pcs status nodes corosync\n",
    "I, [2017-03-10T14:20:08.430061 #2270] INFO -- : CIB USER: hacluster, groups: \n",
    "D, [2017-03-10T14:20:09.100646 #2270] DEBUG -- : []\n",
    "D, [2017-03-10T14:20:09.100763 #2270] DEBUG -- : [\"Error: Unable to read /etc/corosync/corosync.conf: No such file or directory\\n\"]\n",
    "D, [2017-03-10T14:20:09.100818 #2270] DEBUG -- : Duration: 0.670553772s\n",
    "I, [2017-03-10T14:20:09.100913 #2270] INFO -- : Return Value: 1\n",
    "I, [2017-03-10T14:20:09.101561 #2270] INFO -- : Saved config 'tokens' version 665 327998fb875b65c762f66026216a4fa530011074 to '/var/lib/pcsd/tokens'\n",
    "I, [2017-03-10T14:20:09.106627 #2270] INFO -- : No response from: overcloud-controller-1 request: /auth, exception: No route to host - connect(2)\n",
    "I, [2017-03-10T14:20:09.106757 #2270] INFO -- : No response from: overcloud-controller-2 request: /auth, exception: No route to host - connect(2)\n"
  ]
}
--Debug Output End--

Error: Unable to communicate with overcloud-controller-2
Error: Unable to communicate with overcloud-controller-1
overcloud-controller-0: Authorized
[root@overcloud-controller-0 cluster]# iptables -nL
Chain INPUT (policy ACCEPT)
target prot opt source destination
ACCEPT all -- 0.0.0.0/0 0.0.0.0/0 state RELATED,ESTABLISHED
ACCEPT icmp -- 0.0.0.0/0 0.0.0.0/0
ACCEPT all -- 0.0.0.0/0 0.0.0.0/0
ACCEPT tcp -- 0.0.0.0/0 0.0.0.0/0 state NEW tcp dpt:22
REJECT all -- 0.0.0.0/0 0.0.0.0/0 reject-with icmp-host-prohibited

Chain FORWARD (policy ACCEPT)
target prot opt source destination
REJECT all -- 0.0.0.0/0 0.0.0.0/0 reject-with icmp-host-prohibited

Chain OUTPUT (policy ACCEPT)
target prot opt source destination
[root@overcloud-controller-0 cluster]# /usr/bin/python /sbin/pcs --debug cluster auth overcloud-controller-0 overcloud-controller-1 overcloud-controller-2 -u hacluster -p eiiAvTbJGvbPVokW --force
Running: /usr/bin/ruby -I/usr/lib/pcsd/ /usr/lib/pcsd/pcsd-cli.rb auth
--Debug Input Start--
{"username": "hacluster", "local": false, "nodes": ["overcloud-controller-2", "overcloud-controller-1", "overcloud-controller-0"], "password": "eiiAvTbJGvbPVokW", "force": true}
--Debug Input End--
Return Value: 0
--Debug Output Start--
{
  "status": "ok",
  "data": {
    "auth_responses": {
      "overcloud-controller-2": {
        "status": "noresponse"
      },
      "overcloud-controller-0": {
        "status": "ok",
        "token": "d065a17a-38a7-4c8b-85c8-361be725fe0d"
      },
      "overcloud-controller-1": {
        "status": "ok",
        "token": "c65b910d-f45c-43ba-acd2-95a7158dcf5b"
      }
    },
    "sync_successful": true,
    "sync_nodes_err": [

    ],
    "sync_responses": {
    }
  },
  "log": [
    "I, [2017-03-10T14:21:54.395322 #2803] INFO -- : PCSD Debugging enabled\n",
    "D, [2017-03-10T14:21:54.395412 #2803] DEBUG -- : Did not detect RHEL 6\n",
    "I, [2017-03-10T14:21:54.395458 #2803] INFO -- : Running: /usr/sbin/corosync-cmapctl totem.cluster_name\n",
    "I, [2017-03-10T14:21:54.395515 #2803] INFO -- : CIB USER: hacluster, groups: \n",
    "D, [2017-03-10T14:21:54.398678 #2803] DEBUG -- : []\n",
    "D, [2017-03-10T14:21:54.398761 #2803] DEBUG -- : [\"Failed to initialize the cmap API. Error CS_ERR_LIBRARY\\n\"]\n",
    "D, [2017-03-10T14:21:54.398839 #2803] DEBUG -- : Duration: 0.003136466s\n",
    "I, [2017-03-10T14:21:54.398931 #2803] INFO -- : Return Value: 1\n",
    "W, [2017-03-10T14:21:54.399014 #2803] WARN -- : Cannot read config 'corosync.conf' from '/etc/corosync/corosync.conf': No such file\n",
    "W, [2017-03-10T14:21:54.399102 #2803] WARN -- : Cannot read config 'corosync.conf' from '/etc/corosync/corosync.conf': No such file or directory - /etc/corosync/corosync.conf\n",
    "I, [2017-03-10T14:21:54.404138 #2803] INFO -- : No response from: overcloud-controller-2 request: /auth, exception: No route to host - connect(2)\n",
    "I, [2017-03-10T14:21:54.577395 #2803] INFO -- : Running: /usr/sbin/pcs status nodes corosync\n",
    "I, [2017-03-10T14:21:54.577537 #2803] INFO -- : CIB USER: hacluster, groups: \n",
    "D, [2017-03-10T14:21:55.259455 #2803] DEBUG -- : []\n",
    "D, [2017-03-10T14:21:55.259631 #2803] DEBUG -- : [\"Error: Unable to read /etc/corosync/corosync.conf: No such file or directory\\n\"]\n",
    "D, [2017-03-10T14:21:55.259714 #2803] DEBUG -- : Duration: 0.681941214s\n",
    "I, [2017-03-10T14:21:55.259829 #2803] INFO -- : Return Value: 1\n",
    "I, [2017-03-10T14:21:55.260646 #2803] INFO -- : Saved config 'tokens' version 683 e4e0e327c605a3bcfa73ad618062230b212e068d to '/var/lib/pcsd/tokens'\n",
    "I, [2017-03-10T14:21:55.263668 #2803] INFO -- : No response from: overcloud-controller-2 request: /auth, exception: No route to host - connect(2)\n"
  ]
}
--Debug Output End--