So, we can reproduce the bug and in case if beam process will be killed pacemaker detect it only after several minutes, but then pacemaker is able to properly start this service again:
(5-10 minutes after kill -9 <ID of beam process>)
root@node-3:~# ps ax | grep beam 2246 ? Sl 0:06 /usr/lib/erlang/erts-7.1/bin/beam.smp -W w -A 64 -K true -A30 -P 1048576 -K true -B i -- -root /usr/lib/erlang -progname erl -- -home /var/lib/rabbitmq -- -pa /usr/lib/rabbitmq/lib/rabbitmq_server-3.5.6/sbin/../ebin -noshell -noinput -sname rabbit@node-3 -boot start_sasl -config /etc/rabbitmq/rabbitmq -kernel inet_default_connect_options [{nodelay,true}] -rabbit tcp_listeners [{"10.109.1.7",5673}] -sasl errlog_type error -sasl sasl_error_logger false -rabbit error_logger {file,"/<email address hidden>"} -rabbit sasl_error_logger {file,"/<email address hidden>"} -rabbit enabled_plugins_file "/etc/rabbitmq/enabled_plugins" -rabbit plugins_dir "/usr/lib/rabbitmq/lib/rabbitmq_server-3.5.6/sbin/../plugins" -rabbit plugins_expand_dir "/var/lib/rabbitmq/mnesia/rabbit@node-3-plugins-expand" -os_mon start_cpu_sup false -os_mon start_disksup false -os_mon start_memsup false -mnesia dir "/var/lib/rabbitmq/mnesia/rabbit@node-3" 5469 pts/8 S+ 0:00 grep --color=auto beam
So, we can reproduce the bug and in case if beam process will be killed pacemaker detect it only after several minutes, but then pacemaker is able to properly start this service again:
(5-10 minutes after kill -9 <ID of beam process>)
root@node-3:~# ps ax | grep beam erlang/ erts-7. 1/bin/beam. smp -W w -A 64 -K true -A30 -P 1048576 -K true -B i -- -root /usr/lib/erlang -progname erl -- -home /var/lib/rabbitmq -- -pa /usr/lib/ rabbitmq/ lib/rabbitmq_ server- 3.5.6/sbin/ ../ebin -noshell -noinput -sname rabbit@node-3 -boot start_sasl -config /etc/rabbitmq/ rabbitmq -kernel inet_default_ connect_ options [{nodelay,true}] -rabbit tcp_listeners [{"10.109. 1.7",5673} ] -sasl errlog_type error -sasl sasl_error_logger false -rabbit error_logger {file,"/<email address hidden>"} -rabbit sasl_error_logger {file,"/<email address hidden>"} -rabbit enabled_ plugins_ file "/etc/rabbitmq/ enabled_ plugins" -rabbit plugins_dir "/usr/lib/ rabbitmq/ lib/rabbitmq_ server- 3.5.6/sbin/ ../plugins" -rabbit plugins_expand_dir "/var/lib/ rabbitmq/ mnesia/ rabbit@ node-3- plugins- expand" -os_mon start_cpu_sup false -os_mon start_disksup false -os_mon start_memsup false -mnesia dir "/var/lib/ rabbitmq/ mnesia/ rabbit@ node-3"
2246 ? Sl 0:06 /usr/lib/
5469 pts/8 S+ 0:00 grep --color=auto beam