the tcpkeepalive mechanism doesn't seem to be working.. looks like till the send-buffer gets full, till then the connection doesn't close and switch.. we have to look into why the tcp keepalive mechanism is not working..
root@vse2100-4:/var/log/contrail# ss -e | grep 5.5.5.13 tcp SYN-SENT 0 1 5.5.5.14:47609 5.5.5.13:11211 timer:(on,304ms,0) uid:111 ino:27471385 sk:ffff88052705c600 <-> tcp ESTAB 0 0 5.5.5.14:35000 5.5.5.13:9160 ino:26239287 sk:ffff88050f418e00 <-> tcp ESTAB 0 0 5.5.5.14:afs3-fileserver 5.5.5.13:53622 ino:720801 sk:ffff88050f496900 <-> tcp ESTAB 0 0 5.5.5.14:9081 5.5.5.13:53323 uid:118 ino:711727 sk:ffff880b4f210e00 <-> tcp ESTAB 0 0 5.5.5.14:9081 5.5.5.13:35284 uid:118 ino:26901943 sk:ffff880b4f214d00 <-> tcp SYN-SENT 0 1 5.5.5.14:34160 5.5.5.13:mysql timer:(on,560ms,0) uid:111 ino:27484000 sk:ffff8805301fb100 <-> tcp ESTAB 0 0 5.5.5.14:9081 5.5.5.13:53694 uid:118 ino:27162139 sk:ffff8805897db100 <-> tcp SYN-SENT 0 1 5.5.5.14:34167 5.5.5.13:mysql timer:(on,892ms,0) uid:111 ino:27484010 sk:ffff8805301faa00 <-> tcp SYN-SENT 0 1 5.5.5.14:42928 5.5.5.13:afs3-fileserver timer:(on,19sec,6) ino:27449446 sk:ffff88051dd23800 <-> tcp ESTAB 0 0 5.5.5.14:40185 5.5.5.13:9160 uid:118 ino:27165580 sk:ffff8805de259c00 <-> tcp ESTAB 0 0 5.5.5.14:afs3-fileserver 5.5.5.13:34976 ino:26751407 sk:ffff880c1c71e900 <-> tcp SYN-SENT 0 1 5.5.5.14:34097 5.5.5.13:mysql timer:(on,1.052ms,1) uid:111 ino:27484482 sk:ffff880b442f5b00 <-> tcp SYN-SENT 0 1 5.5.5.14:50098 5.5.5.13:9100 timer:(on,224ms,0) uid:111 ino:27483990 sk:ffff88058627e900 <-> tcp ESTAB 0 65892 5.5.5.14:55247 5.5.5.13:8086 timer:(on,58sec,14) uid:118 ino:27226323 sk:ffff880530028700 <-> tcp ESTAB 0 0 5.5.5.14:35009 5.5.5.13:9160 ino:26240195 sk:ffff880b44362300 <-> tcp ESTAB 0 8993 5.5.5.14:40041 5.5.5.13:9160 timer:(on,1min,14) uid:118 ino:27170293 sk:ffff880b440d1500 <-> tcp SYN-SENT 0 1 5.5.5.14:35784 5.5.5.13:9697 timer:(on,812ms,0) uid:111 ino:27471393 sk:ffff88052705a300 <->
the tcpkeepalive mechanism doesn't seem to be working.. looks like till the send-buffer gets full, till then the connection doesn't close and switch.. we have to look into why the tcp keepalive mechanism is not working..
root@vse2100- 4:/var/ log/contrail# ss -e | grep 5.5.5.13 afs3-fileserver 5.5.5.13:53622 ino:720801 sk:ffff88050f496900 <-> afs3-fileserver timer:(on,19sec,6) ino:27449446 sk:ffff88051dd23800 <-> afs3-fileserver 5.5.5.13:34976 ino:26751407 sk:ffff880c1c71e900 <-> on,1.052ms, 1) uid:111 ino:27484482 sk:ffff880b442f5b00 <->
tcp SYN-SENT 0 1 5.5.5.14:47609 5.5.5.13:11211 timer:(on,304ms,0) uid:111 ino:27471385 sk:ffff88052705c600 <->
tcp ESTAB 0 0 5.5.5.14:35000 5.5.5.13:9160 ino:26239287 sk:ffff88050f418e00 <->
tcp ESTAB 0 0 5.5.5.14:
tcp ESTAB 0 0 5.5.5.14:9081 5.5.5.13:53323 uid:118 ino:711727 sk:ffff880b4f210e00 <->
tcp ESTAB 0 0 5.5.5.14:9081 5.5.5.13:35284 uid:118 ino:26901943 sk:ffff880b4f214d00 <->
tcp SYN-SENT 0 1 5.5.5.14:34160 5.5.5.13:mysql timer:(on,560ms,0) uid:111 ino:27484000 sk:ffff8805301fb100 <->
tcp ESTAB 0 0 5.5.5.14:9081 5.5.5.13:53694 uid:118 ino:27162139 sk:ffff8805897db100 <->
tcp SYN-SENT 0 1 5.5.5.14:34167 5.5.5.13:mysql timer:(on,892ms,0) uid:111 ino:27484010 sk:ffff8805301faa00 <->
tcp SYN-SENT 0 1 5.5.5.14:42928 5.5.5.13:
tcp ESTAB 0 0 5.5.5.14:40185 5.5.5.13:9160 uid:118 ino:27165580 sk:ffff8805de259c00 <->
tcp ESTAB 0 0 5.5.5.14:
tcp SYN-SENT 0 1 5.5.5.14:34097 5.5.5.13:mysql timer:(
tcp SYN-SENT 0 1 5.5.5.14:50098 5.5.5.13:9100 timer:(on,224ms,0) uid:111 ino:27483990 sk:ffff88058627e900 <->
tcp ESTAB 0 65892 5.5.5.14:55247 5.5.5.13:8086 timer:(on,58sec,14) uid:118 ino:27226323 sk:ffff880530028700 <->
tcp ESTAB 0 0 5.5.5.14:35009 5.5.5.13:9160 ino:26240195 sk:ffff880b44362300 <->
tcp ESTAB 0 8993 5.5.5.14:40041 5.5.5.13:9160 timer:(on,1min,14) uid:118 ino:27170293 sk:ffff880b440d1500 <->
tcp SYN-SENT 0 1 5.5.5.14:35784 5.5.5.13:9697 timer:(on,812ms,0) uid:111 ino:27471393 sk:ffff88052705a300 <->