Logs saved at :http://10.204.216.50/Docs/bugs/<bug-id>
Build:3.02 35 kilo
Had the setup as below
Setup was running around 140 vms with traffic.
The cassandra got exited in all the 3 database nodes with the below error
org.apache.thrift.transport.TTransportException: java.net.SocketException: Too many open files in system
Logs
====
WARN [Thread-13] 2016-05-14 21:41:20,331 CustomTThreadPoolServer.java:122 - Transport error occurred during acceptance of message.
org.apache.thrift.transport.TTransportException: java.net.SocketException: Too many open files in system
at org.apache.cassandra.thrift.TCustomServerSocket.acceptImpl(TCustomServerSocket.java:108) ~[apache-cassandra-2.1.13.jar:2.1.13]
at org.apache.cassandra.thrift.TCustomServerSocket.acceptImpl(TCustomServerSocket.java:36) ~[apache-cassandra-2.1.13.jar:2.1.13]
at org.apache.thrift.transport.TServerTransport.accept(TServerTransport.java:60) ~[libthrift-0.9.2.jar:0.9.2]
at org.apache.cassandra.thrift.CustomTThreadPoolServer.serve(CustomTThreadPoolServer.java:110) ~[apache-cassandra-2.1.13.jar:2.1.13]
at org.apache.cassandra.thrift.ThriftServer$ThriftServerThread.run(ThriftServer.java:137) [apache-cassandra-2.1.13.jar:2.1.13]
Caused by: java.net.SocketException: Too many open files in system
at java.net.PlainSocketImpl.socketAccept(Native Method) ~[na:1.7.0_95]
at java.net.AbstractPlainSocketImpl.accept(AbstractPlainSocketImpl.java:398) ~[na:1.7.0_95]
at java.net.ServerSocket.implAccept(ServerSocket.java:530) ~[na:1.7.0_95]
at java.net.ServerSocket.accept(ServerSocket.java:498) ~[na:1.7.0_95]
at org.apache.cassandra.thrift.TCustomServerSocket.acceptImpl(TCustomServerSocket.java:102) ~[apache-cassandra-2.1.13.jar:2.1.13]
... 4 common frames omitted
root@nodei27:~#
root@nodei27:~#
root@nodei27:~# uptime
12:31:58 up 5 days, 2:33, 1 user, load average: 3.08, 2.11, 1.87
setup
======
host1 = 'root@10.204.217.139'
host2 = 'root@10.204.217.140'
host3 = 'root@10.204.217.147'
host4 = 'root@10.204.217.144'
host5 = 'root@10.204.217.147'
host6 = 'root@10.204.217.148'
host7 = 'root@10.204.217.149'
host8 = 'root@10.204.217.150'
host9 = 'root@10.204.217.210'
host10 = 'root@10.204.217.217'
host11 = 'root@10.204.217.218'
host12 = 'root@10.204.217.220'
host13 = 'root@10.204.217.247'
host14 = 'root@10.204.217.248'
host15 = 'root@10.204.217.249'
host16 = 'root@10.204.217.118'
host17 = 'root@10.204.217.119'
host18 = 'root@10.204.217.120'
host19 = 'root@10.204.217.121'
host20 = 'root@10.204.217.122'
host21 = 'root@10.204.217.123'
host22 = 'root@10.204.217.124'
host23 = 'root@10.204.217.131'
ext_routers = [('blr-mx2', '10.204.216.245')]
router_asn = 64512
public_vn_rtgt = 30001
#public_vn_subnet = "10.204.219.72/29"
host_build = 'vjoshi@10.204.216.56'
env.roledefs = {
'all': [host1, host2, host3, host4, host5, host6,host7, host8, host9, host10, host11, host12, host13, host14, host15,host16,host17,host18,host19,host20,host21,host22,host23],
'cfgm': [host1, host2, host3],
'openstack': [host4, host5, host6],
'webui': [host1, host2, host3],
'control': [host1, host2, host3],
'compute': [host7, host8, host9, host10, host11, host12, host13, host14, host15,host16,host17,host18,host19,host20,host21,host22,host23],
'collector': [host1, host2, host3],
'database': [host1, host2, host3],
'build': [host_build],
}
env.hostnames = {
'all': ['nodei27', 'nodei28', 'nodei35', 'nodei32', 'nodei35', 'nodei36', 'nodei37', 'nodei38', 'nodel4', 'nodel7', 'nodel8', 'nodel9', 'nodel10', 'nodel11', 'nodel12','nodei6','nodei7','nodei8','nodei9','nodei10','nodei11','nodei12','nodei19']
}
Review in progress for https:/
Submitter: Megh Bhatt (<email address hidden>)