1. nova-comp open too many files about 1112. (but the output of "ulimit -n" just is 1024)
2. nova-comp open too many files inode 3641 /lib/modules/2.6.32-279.el6.x86_64/kernel/net/sunrpc/auth_gss/rpcsec_gss_krb5.ko
nova-comp 20916 olympics 28u REG 0,9 0 3641 anon_inode nova-comp 20916 olympics 29u REG 0,9 0 3641 anon_inode ...
nova-comp 20975 olympics 22u REG 0,9 0 3641 anon_inode nova-comp 20975 olympics 23u REG 0,9 0 3641 anon_inode ...
--------------------------- [olympics@Openstack_Grizzly_ControlNode ~]$ ps -ef | grep nova olympics 10374 10357 0 04:54 pts/1 00:00:00 grep nova olympics 11292 11272 0 Dec04 pts/7 00:00:01 python /home/olympics/source/nova/bin/nova-api olympics 11300 11292 0 Dec04 pts/7 00:00:01 python /home/olympics/source/nova/bin/nova-api olympics 11303 11292 0 Dec04 pts/7 00:00:50 python /home/olympics/source/nova/bin/nova-api olympics 11304 11292 0 Dec04 pts/7 00:00:01 python /home/olympics/source/nova/bin/nova-api olympics 11479 11373 0 Dec04 pts/9 01:21:03 python /home/olympics/source/nova/bin/nova-network olympics 11650 11485 0 Dec04 pts/10 00:39:28 python /home/olympics/source/nova/bin/nova-scheduler nobody 13152 1 0 Dec04 ? 00:02:29 /usr/sbin/dnsmasq --strict-order --bind-interfaces --conf-file= --domain=novalocal --pid-file=/home/olympics/source/data/nova/networks/nova-br100.pid --listen-address=10.0.1.1 --except-interface=lo --dhcp-range=set:'private',10.0.1.2,static,120s --dhcp-lease-max=256 --dhcp-hostsfile=/home/olympics/source/data/nova/networks/nova-br100.conf --dhcp-script=/home/olympics/source/nova/bin/nova-dhcpbridge --leasefile-ro root 13153 13152 0 Dec04 ? 00:00:00 /usr/sbin/dnsmasq --strict-order --bind-interfaces --conf-file= --domain=novalocal --pid-file=/home/olympics/source/data/nova/networks/nova-br100.pid --listen-address=10.0.1.1 --except-interface=lo --dhcp-range=set:'private',10.0.1.2,static,120s --dhcp-lease-max=256 --dhcp-hostsfile=/home/olympics/source/data/nova/networks/nova-br100.conf --dhcp-script=/home/olympics/source/nova/bin/nova-dhcpbridge --leasefile-ro olympics 20916 1 0 Dec05 ? 01:25:35 /usr/bin/python /usr/bin/nova-compute olympics 20975 1 0 Dec05 ? 01:24:55 /usr/bin/python /usr/bin/nova-compute --config-file=/etc/nova/nova2.conf [olympics@Openstack_Grizzly_ControlNode ~]$ [olympics@Openstack_Grizzly_ControlNode ~]$ ulimit -n 1024 [olympics@Openstack_Grizzly_ControlNode ~]$ lsof -p 20916 | wc -l 1112 [olympics@Openstack_Grizzly_ControlNode ~]$ lsof -p 20975 | wc -l 1112 [olympics@Openstack_Grizzly_ControlNode ~]$ lsof -p 20975 | grep 3641 | wc -l 1009 [olympics@Openstack_Grizzly_ControlNode ~]$ lsof -p 20916 | grep 3641 | wc -l 1009 [olympics@Openstack_Grizzly_ControlNode ~]$ [olympics@Openstack_Grizzly_ControlNode ~]$ ulimit -a core file size (blocks, -c) 0 data seg size (kbytes, -d) unlimited scheduling priority (-e) 0 file size (blocks, -f) unlimited pending signals (-i) 14874 max locked memory (kbytes, -l) 64 max memory size (kbytes, -m) unlimited open files (-n) 1024 pipe size (512 bytes, -p) 8 POSIX message queues (bytes, -q) 819200 real-time priority (-r) 0 stack size (kbytes, -s) 10240 cpu time (seconds, -t) unlimited max user processes (-u) 1024 virtual memory (kbytes, -v) unlimited file locks (-x) unlimited [olympics@Openstack_Grizzly_ControlNode ~]$ [olympics@Openstack_Grizzly_ControlNode ~]$ ls -li /lib/modules/2.6.32-279.el6.x86_64/kernel/net/sunrpc/auth_gss/rpcsec_gss_krb5.ko 3641 -rwxr--r--. 1 root root 49576 Jun 14 2012 /lib/modules/2.6.32-279.el6.x86_64/kernel/net/sunrpc/auth_gss/rpcsec_gss_krb5.ko [olympics@Openstack_Grizzly_ControlNode ~]$
nova-compute.log: 2012-12-13 22:31:21 20916 ERROR nova.virt.powervm-next.common [-] Error while trying to connect: Error reading SSH protocol banner[Errno 24] Too many open files 2012-12-13 22:31:21 20916 TRACE nova.virt.powervm-next.common Traceback (most recent call last): 2012-12-13 22:31:21 20916 TRACE nova.virt.powervm-next.common File "/home/olympics/source/nova/nova/virt/powervm-next/common.py", line 60, in ssh_connect 2012-12-13 22:31:21 20916 TRACE nova.virt.powervm-next.common port=port) 2012-12-13 22:31:21 20916 TRACE nova.virt.powervm-next.common File "/usr/lib/python2.6/site-packages/paramiko/client.py", line 295, in connect 2012-12-13 22:31:21 20916 TRACE nova.virt.powervm-next.common t.start_client() 2012-12-13 22:31:21 20916 TRACE nova.virt.powervm-next.common File "/usr/lib/python2.6/site-packages/paramiko/transport.py", line 451, in start_client 2012-12-13 22:31:21 20916 TRACE nova.virt.powervm-next.common raise e 2012-12-13 22:31:21 20916 TRACE nova.virt.powervm-next.common SSHException: Error reading SSH protocol banner[Errno 24] Too many open files 2012-12-13 22:31:21 20916 TRACE nova.virt.powervm-next.common
1. nova-comp open too many files about 1112. (but the output of "ulimit -n" just is 1024)
2. nova-comp open too many files inode 3641 /lib/modules/ 2.6.32- 279.el6. x86_64/ kernel/ net/sunrpc/ auth_gss/ rpcsec_ gss_krb5. ko
nova-comp 20916 olympics 28u REG 0,9 0 3641 anon_inode
nova-comp 20916 olympics 29u REG 0,9 0 3641 anon_inode
...
nova-comp 20975 olympics 22u REG 0,9 0 3641 anon_inode
nova-comp 20975 olympics 23u REG 0,9 0 3641 anon_inode
...
------- ------- ------- ------ Openstack_ Grizzly_ ControlNode ~]$ ps -ef | grep nova source/ nova/bin/ nova-api source/ nova/bin/ nova-api source/ nova/bin/ nova-api source/ nova/bin/ nova-api source/ nova/bin/ nova-network source/ nova/bin/ nova-scheduler /home/olympics/ source/ data/nova/ networks/ nova-br100. pid --listen- address= 10.0.1. 1 --except- interface= lo --dhcp- range=set: 'private' ,10.0.1. 2,static, 120s --dhcp- lease-max= 256 --dhcp- hostsfile= /home/olympics/ source/ data/nova/ networks/ nova-br100. conf --dhcp- script= /home/olympics/ source/ nova/bin/ nova-dhcpbridge --leasefile-ro /home/olympics/ source/ data/nova/ networks/ nova-br100. pid --listen- address= 10.0.1. 1 --except- interface= lo --dhcp- range=set: 'private' ,10.0.1. 2,static, 120s --dhcp- lease-max= 256 --dhcp- hostsfile= /home/olympics/ source/ data/nova/ networks/ nova-br100. conf --dhcp- script= /home/olympics/ source/ nova/bin/ nova-dhcpbridge --leasefile-ro nova-compute nova-compute --config- file=/etc/ nova/nova2. conf Openstack_ Grizzly_ ControlNode ~]$ Openstack_ Grizzly_ ControlNode ~]$ ulimit -n Openstack_ Grizzly_ ControlNode ~]$ lsof -p 20916 | wc -l Openstack_ Grizzly_ ControlNode ~]$ lsof -p 20975 | wc -l Openstack_ Grizzly_ ControlNode ~]$ lsof -p 20975 | grep 3641 | wc -l Openstack_ Grizzly_ ControlNode ~]$ lsof -p 20916 | grep 3641 | wc -l Openstack_ Grizzly_ ControlNode ~]$ Openstack_ Grizzly_ ControlNode ~]$ ulimit -a Openstack_ Grizzly_ ControlNode ~]$ Openstack_ Grizzly_ ControlNode ~]$ ls -li /lib/modules/ 2.6.32- 279.el6. x86_64/ kernel/ net/sunrpc/ auth_gss/ rpcsec_ gss_krb5. ko 2.6.32- 279.el6. x86_64/ kernel/ net/sunrpc/ auth_gss/ rpcsec_ gss_krb5. ko Openstack_ Grizzly_ ControlNode ~]$
[olympics@
olympics 10374 10357 0 04:54 pts/1 00:00:00 grep nova
olympics 11292 11272 0 Dec04 pts/7 00:00:01 python /home/olympics/
olympics 11300 11292 0 Dec04 pts/7 00:00:01 python /home/olympics/
olympics 11303 11292 0 Dec04 pts/7 00:00:50 python /home/olympics/
olympics 11304 11292 0 Dec04 pts/7 00:00:01 python /home/olympics/
olympics 11479 11373 0 Dec04 pts/9 01:21:03 python /home/olympics/
olympics 11650 11485 0 Dec04 pts/10 00:39:28 python /home/olympics/
nobody 13152 1 0 Dec04 ? 00:02:29 /usr/sbin/dnsmasq --strict-order --bind-interfaces --conf-file= --domain=novalocal --pid-file=
root 13153 13152 0 Dec04 ? 00:00:00 /usr/sbin/dnsmasq --strict-order --bind-interfaces --conf-file= --domain=novalocal --pid-file=
olympics 20916 1 0 Dec05 ? 01:25:35 /usr/bin/python /usr/bin/
olympics 20975 1 0 Dec05 ? 01:24:55 /usr/bin/python /usr/bin/
[olympics@
[olympics@
1024
[olympics@
1112
[olympics@
1112
[olympics@
1009
[olympics@
1009
[olympics@
[olympics@
core file size (blocks, -c) 0
data seg size (kbytes, -d) unlimited
scheduling priority (-e) 0
file size (blocks, -f) unlimited
pending signals (-i) 14874
max locked memory (kbytes, -l) 64
max memory size (kbytes, -m) unlimited
open files (-n) 1024
pipe size (512 bytes, -p) 8
POSIX message queues (bytes, -q) 819200
real-time priority (-r) 0
stack size (kbytes, -s) 10240
cpu time (seconds, -t) unlimited
max user processes (-u) 1024
virtual memory (kbytes, -v) unlimited
file locks (-x) unlimited
[olympics@
[olympics@
3641 -rwxr--r--. 1 root root 49576 Jun 14 2012 /lib/modules/
[olympics@
nova-compute.log: powervm- next.common [-] Error while trying to connect: Error reading SSH protocol banner[Errno 24] Too many open files powervm- next.common Traceback (most recent call last): powervm- next.common File "/home/ olympics/ source/ nova/nova/ virt/powervm- next/common. py", line 60, in ssh_connect powervm- next.common port=port) powervm- next.common File "/usr/lib/ python2. 6/site- packages/ paramiko/ client. py", line 295, in connect powervm- next.common t.start_client() powervm- next.common File "/usr/lib/ python2. 6/site- packages/ paramiko/ transport. py", line 451, in start_client powervm- next.common raise e powervm- next.common SSHException: Error reading SSH protocol banner[Errno 24] Too many open files powervm- next.common
2012-12-13 22:31:21 20916 ERROR nova.virt.
2012-12-13 22:31:21 20916 TRACE nova.virt.
2012-12-13 22:31:21 20916 TRACE nova.virt.
2012-12-13 22:31:21 20916 TRACE nova.virt.
2012-12-13 22:31:21 20916 TRACE nova.virt.
2012-12-13 22:31:21 20916 TRACE nova.virt.
2012-12-13 22:31:21 20916 TRACE nova.virt.
2012-12-13 22:31:21 20916 TRACE nova.virt.
2012-12-13 22:31:21 20916 TRACE nova.virt.
2012-12-13 22:31:21 20916 TRACE nova.virt.