Ok, thanks for the guidance, I've found logs from the controller.
It's still the issue with rabbitmq:
2014-07-07T12:27:22.315162+00:00 err: =CRASH REPORT==== 7-Jul-2014::12:27:13 === 2014-07-07T12:27:22.315162+00:00 err: crasher: 2014-07-07T12:27:22.315162+00:00 err: initial call: gen:init_it/6 2014-07-07T12:27:22.315162+00:00 err: pid: <0.11493.0> 2014-07-07T12:27:22.315162+00:00 err: registered_name: [] 2014-07-07T12:27:22.315162+00:00 err: exception exit: {{function_clause, 2014-07-07T12:27:22.315162+00:00 err: [{rabbit_mirror_queue_slave,terminate, 2014-07-07T12:27:22.315162+00:00 err: [{function_clause, 2014-07-07T12:27:22.315162+00:00 err: [{rabbit_mirror_queue_slave,handle_pre_hibernate, 2014-07-07T12:27:22.315162+00:00 err: [{not_started, 2014-07-07T12:27:22.315162+00:00 err: {amqqueue, 2014-07-07T12:27:22.315162+00:00 err: {resource,<<"/">>,queue, 2014-07-07T12:27:22.315162+00:00 err: <<"l3_agent_fanout_f98ecb4c3ab34472b130bce536715ef0">>}, 2014-07-07T12:27:22.315162+00:00 err: false,true,none, 2014-07-07T12:27:22.315162+00:00 err: [{<<"x-ha-policy">>,longstr,<<"all">>}], 2014-07-07T12:27:22.315162+00:00 err: <0.11493.0>,[],[], 2014-07-07T12:27:22.315162+00:00 err: [{vhost,<<"/">>}, 2014-07-07T12:27:22.315162+00:00 err: {name,<<"ha-all">>}, 2014-07-07T12:27:22.315162+00:00 err: {pattern,<<".">>}, 2014-07-07T12:27:22.315162+00:00 err: {'apply-to',<<"all">>}, 2014-07-07T12:27:22.315162+00:00 err: {definition, 2014-07-07T12:27:22.315162+00:00 err: [{<<"ha-mode">>,<<"all">>}, 2014-07-07T12:27:22.315162+00:00 err: {<<"ha-sync-mode">>,<<"automatic">>}]}, 2014-07-07T12:27:22.315162+00:00 err: {priority,0}], 2014-07-07T12:27:22.315162+00:00 err: [{<0.11495.0>,<0.11493.0>}], 2014-07-07T12:27:22.315162+00:00 err: []}}]}, 2014-07-07T12:27:22.315162+00:00 err: {gen_server2,pre_hibernate,1}, 2014-07-07T12:27:22.315162+00:00 err: {proc_lib,init_p_do_apply,3}]}, 2014-07-07T12:27:22.315162+00:00 err: {not_started, 2014-07-07T12:27:22.315162+00:00 err: {amqqueue, 2014-07-07T12:27:22.315162+00:00 err: {resource,<<"/">>,queue, 2014-07-07T12:27:22.315162+00:00 err: <<"l3_agent_fanout_f98ecb4c3ab34472b130bce536715ef0">>}, 2014-07-07T12:27:22.315162+00:00 err: false,true,none, 2014-07-07T12:27:22.315162+00:00 err: [{<<"x-ha-policy">>,longstr,<<"all">>}], 2014-07-07T12:27:22.315162+00:00 err: <0.11493.0>,[],[], 2014-07-07T12:27:22.315162+00:00 err: [{vhost,<<"/">>}, 2014-07-07T12:27:22.315162+00:00 err: {name,<<"ha-all">>}, 2014-07-07T12:27:22.315162+00:00 err: {pattern,<<".">>}, 2014-07-07T12:27:22.315162+00:00 err: {'apply-to',<<"all">>}, 2014-07-07T12:27:22.315162+00:00 err: {definition, 2014-07-07T12:27:22.315162+00:00 err: [{<<"ha-mode">>,<<"all">>}, 2014-07-07T12:27:22.315162+00:00 err: {<<"ha-sync-mode">>,<<"automatic">>}]}, 2014-07-07T12:27:22.315162+00:00 err: {priority,0}], 2014-07-07T12:27:22.315162+00:00 err: [{<0.11495.0>,<0.11493.0>}], 2014-07-07T12:27:22.315162+00:00 err: []}}]}, 2014-07-07T12:27:22.315162+00:00 err: {gen_server2,terminate,3}, 2014-07-07T12:27:22.315162+00:00 err: {proc_lib,init_p_do_apply,3}]}, 2014-07-07T12:27:22.315162+00:00 err: {gen_server2,call,[<2708.3998.0>,go,infinity]}} 2014-07-07T12:27:22.315162+00:00 err: in function gen_server2:terminate/3 2014-07-07T12:27:22.315162+00:00 err: ancestors: [rabbit_amqqueue_sup,rabbit_sup,<0.72.0>] 2014-07-07T12:27:22.315162+00:00 err: messages: [{'$gen_call', 2014-07-07T12:27:22.315162+00:00 err: {<0.11464.0>,#Ref<0.0.0.51237>}, 2014-07-07T12:27:22.315162+00:00 err: {basic_consume,false,<0.11464.0>,<0.11462.0>,false, 2014-07-07T12:27:22.315162+00:00 err: <<"3">>,false,none,[], 2014-07-07T12:27:22.315162+00:00 err: {'basic.consume_ok',<<"3">>}}}] 2014-07-07T12:27:22.315162+00:00 err: links: [<0.275.0>,<0.11494.0>] 2014-07-07T12:27:22.315162+00:00 err: dictionary: [{{xtype_to_module,direct},rabbit_exchange_type_direct}, 2014-07-07T12:27:22.315162+00:00 err: {guid,{{449189282,1548653523,1088174412,2664687952},0}}] 2014-07-07T12:27:22.315162+00:00 err: trap_exit: true 2014-07-07T12:27:22.315162+00:00 err: status: running 2014-07-07T12:27:22.315162+00:00 err: heap_size: 2584 2014-07-07T12:27:22.315162+00:00 err: stack_size: 24 2014-07-07T12:27:22.315162+00:00 err: reductions: 1857 2014-07-07T12:27:22.315162+00:00 err: neighbours: 2014-07-07T12:27:22.315162+00:00 err: neighbour: [{pid,<0.11495.0>}, 2014-07-07T12:27:22.315162+00:00 err: {registered_name,[]}, 2014-07-07T12:27:22.315162+00:00 err: {initial_call,{gen,init_it, 2014-07-07T12:27:22.315162+00:00 err: ['Argument__1','Argument__2', 2014-07-07T12:27:22.315162+00:00 err: 'Argument__3','Argument__4', 2014-07-07T12:27:22.315162+00:00 err: 'Argument__5','Argument__6']}}, 2014-07-07T12:27:22.315162+00:00 err: {current_function,{erlang,hibernate,3}}, 2014-07-07T12:27:22.315162+00:00 err: {ancestors,[<0.11494.0>,<0.11493.0>,rabbit_amqqueue_sup, 2014-07-07T12:27:22.315162+00:00 err: rabbit_sup,<0.72.0>]}, 2014-07-07T12:27:22.315162+00:00 err: {messages,[]}, 2014-07-07T12:27:22.315162+00:00 err: {links,[<0.11494.0>]}, 2014-07-07T12:27:22.315162+00:00 err: {dictionary,[{random_seed,{1404,8663,19679}}]}, 2014-07-07T12:27:22.315162+00:00 err: {trap_exit,false}, 2014-07-07T12:27:22.315162+00:00 err: {status,waiting}, 2014-07-07T12:27:22.315162+00:00 err: {heap_size,147}, 2014-07-07T12:27:22.315162+00:00 err: {stack_size,0}, 2014-07-07T12:27:22.315162+00:00 err: {reductions,212}] 2014-07-07T12:27:22.315162+00:00 err: neighbour: [{pid,<0.11494.0>}, 2014-07-07T12:27:22.315162+00:00 err: {registered_name,[]}, 2014-07-07T12:27:22.315162+00:00 err: {initial_call,{gen,init_it, 2014-07-07T12:27:22.315162+00:00 err: ['Argument__1','Argument__2', 2014-07-07T12:27:22.315162+00:00 err: 'Argument__3','Argument__4', 2014-07-07T12:27:22.315162+00:00 err: 'Argument__5','Argument__6']}}, 2014-07-07T12:27:22.315162+00:00 err: {current_function,{erlang,hibernate,3}}, 2014-07-07T12:27:22.315162+00:00 err: {ancestors,[<0.11493.0>,rabbit_amqqueue_sup,rabbit_sup, 2014-07-07T12:27:22.315162+00:00 err: <0.72.0>]}, 2014-07-07T12:27:22.315162+00:00 err: {messages,[]}, 2014-07-07T12:27:22.315162+00:00 err: {links,[<0.11493.0>,<0.11495.0>]}, 2014-07-07T12:27:22.315162+00:00 err: {dictionary,[]}, 2014-07-07T12:27:22.315162+00:00 err: {trap_exit,false}, 2014-07-07T12:27:22.315162+00:00 err: {status,waiting}, 2014-07-07T12:27:22.315162+00:00 err: {heap_size,224}, 2014-07-07T12:27:22.315162+00:00 err: {stack_size,0}, 2014-07-07T12:27:22.315162+00:00 err: {reductions,120}]
Ok, thanks for the guidance, I've found logs from the controller.
It's still the issue with rabbitmq:
2014-07- 07T12:27: 22.315162+ 00:00 err: =CRASH REPORT==== 7-Jul-2014: :12:27: 13 === 07T12:27: 22.315162+ 00:00 err: crasher: 07T12:27: 22.315162+ 00:00 err: initial call: gen:init_it/6 07T12:27: 22.315162+ 00:00 err: pid: <0.11493.0> 07T12:27: 22.315162+ 00:00 err: registered_name: [] 07T12:27: 22.315162+ 00:00 err: exception exit: {{function_clause, 07T12:27: 22.315162+ 00:00 err: [{rabbit_ mirror_ queue_slave, terminate, 07T12:27: 22.315162+ 00:00 err: [{function_clause, 07T12:27: 22.315162+ 00:00 err: [{rabbit_ mirror_ queue_slave, handle_ pre_hibernate, 07T12:27: 22.315162+ 00:00 err: [{not_started, 07T12:27: 22.315162+ 00:00 err: {amqqueue, 07T12:27: 22.315162+ 00:00 err: {resource, <<"/">> ,queue, 07T12:27: 22.315162+ 00:00 err: <<"l3_agent_ fanout_ f98ecb4c3ab3447 2b130bce536715e f0">>}, 07T12:27: 22.315162+ 00:00 err: false,true,none, 07T12:27: 22.315162+ 00:00 err: [{<<"x- ha-policy" >>,longstr, <<"all" >>}], 07T12:27: 22.315162+ 00:00 err: <0.11493.0>,[],[], 07T12:27: 22.315162+ 00:00 err: [{vhost,<<"/">>}, 07T12:27: 22.315162+ 00:00 err: {name,< <"ha-all" >>}, 07T12:27: 22.315162+ 00:00 err: {pattern,<<".">>}, 07T12:27: 22.315162+ 00:00 err: {'apply- to',<<" all">>} , 07T12:27: 22.315162+ 00:00 err: {definition, 07T12:27: 22.315162+ 00:00 err: [{<<"ha- mode">> ,<<"all" >>}, 07T12:27: 22.315162+ 00:00 err: {<<"ha- sync-mode" >>,<<"automatic ">>}]}, 07T12:27: 22.315162+ 00:00 err: {priority,0}], 07T12:27: 22.315162+ 00:00 err: [{<0.11495. 0>,<0.11493. 0>}], 07T12:27: 22.315162+ 00:00 err: []}}]}, 07T12:27: 22.315162+ 00:00 err: {gen_server2, pre_hibernate, 1}, 07T12:27: 22.315162+ 00:00 err: {proc_lib, init_p_ do_apply, 3}]}, 07T12:27: 22.315162+ 00:00 err: {not_started, 07T12:27: 22.315162+ 00:00 err: {amqqueue, 07T12:27: 22.315162+ 00:00 err: {resource, <<"/">> ,queue, 07T12:27: 22.315162+ 00:00 err: <<"l3_agent_ fanout_ f98ecb4c3ab3447 2b130bce536715e f0">>}, 07T12:27: 22.315162+ 00:00 err: false,true,none, 07T12:27: 22.315162+ 00:00 err: [{<<"x- ha-policy" >>,longstr, <<"all" >>}], 07T12:27: 22.315162+ 00:00 err: <0.11493.0>,[],[], 07T12:27: 22.315162+ 00:00 err: [{vhost,<<"/">>}, 07T12:27: 22.315162+ 00:00 err: {name,< <"ha-all" >>}, 07T12:27: 22.315162+ 00:00 err: {pattern,<<".">>}, 07T12:27: 22.315162+ 00:00 err: {'apply- to',<<" all">>} , 07T12:27: 22.315162+ 00:00 err: {definition, 07T12:27: 22.315162+ 00:00 err: [{<<"ha- mode">> ,<<"all" >>}, 07T12:27: 22.315162+ 00:00 err: {<<"ha- sync-mode" >>,<<"automatic ">>}]}, 07T12:27: 22.315162+ 00:00 err: {priority,0}], 07T12:27: 22.315162+ 00:00 err: [{<0.11495. 0>,<0.11493. 0>}], 07T12:27: 22.315162+ 00:00 err: []}}]}, 07T12:27: 22.315162+ 00:00 err: {gen_server2, terminate, 3}, 07T12:27: 22.315162+ 00:00 err: {proc_lib, init_p_ do_apply, 3}]}, 07T12:27: 22.315162+ 00:00 err: {gen_server2, call,[< 2708.3998. 0>,go,infinity] }} 07T12:27: 22.315162+ 00:00 err: in function gen_server2: terminate/ 3 07T12:27: 22.315162+ 00:00 err: ancestors: [rabbit_ amqqueue_ sup,rabbit_ sup,<0. 72.0>] 07T12:27: 22.315162+ 00:00 err: messages: [{'$gen_call', 07T12:27: 22.315162+ 00:00 err: {<0.11464. 0>,#Ref< 0.0.0.51237> }, 07T12:27: 22.315162+ 00:00 err: {basic_ consume, false,< 0.11464. 0>,<0.11462. 0>,false, 07T12:27: 22.315162+ 00:00 err: <<"3">> ,false, none,[] , 07T12:27: 22.315162+ 00:00 err: {'basic. consume_ ok',<<" 3">>}}} ] 07T12:27: 22.315162+ 00:00 err: links: [<0.275. 0>,<0.11494. 0>] 07T12:27: 22.315162+ 00:00 err: dictionary: [{{xtype_ to_module, direct} ,rabbit_ exchange_ type_direct} , 07T12:27: 22.315162+ 00:00 err: {guid,{ {449189282, 1548653523, 1088174412, 2664687952} ,0}}] 07T12:27: 22.315162+ 00:00 err: trap_exit: true 07T12:27: 22.315162+ 00:00 err: status: running 07T12:27: 22.315162+ 00:00 err: heap_size: 2584 07T12:27: 22.315162+ 00:00 err: stack_size: 24 07T12:27: 22.315162+ 00:00 err: reductions: 1857 07T12:27: 22.315162+ 00:00 err: neighbours: 07T12:27: 22.315162+ 00:00 err: neighbour: [{pid,<0.11495.0>}, 07T12:27: 22.315162+ 00:00 err: {registered_ name,[] }, 07T12:27: 22.315162+ 00:00 err: {initial_ call,{gen, init_it, 07T12:27: 22.315162+ 00:00 err: ['Argument_ _1','Argument_ _2', 07T12:27: 22.315162+ 00:00 err: 'Argument_ _3','Argument_ _4', 07T12:27: 22.315162+ 00:00 err: 'Argument_ _5','Argument_ _6']}}, 07T12:27: 22.315162+ 00:00 err: {current_ function, {erlang, hibernate, 3}}, 07T12:27: 22.315162+ 00:00 err: {ancestors, [<0.11494. 0>,<0.11493. 0>,rabbit_ amqqueue_ sup, 07T12:27: 22.315162+ 00:00 err: rabbit_ sup,<0. 72.0>]} , 07T12:27: 22.315162+ 00:00 err: {messages,[]}, 07T12:27: 22.315162+ 00:00 err: {links, [<0.11494. 0>]}, 07T12:27: 22.315162+ 00:00 err: {dictionary, [{random_ seed,{1404, 8663,19679} }]}, 07T12:27: 22.315162+ 00:00 err: {trap_exit,false}, 07T12:27: 22.315162+ 00:00 err: {status,waiting}, 07T12:27: 22.315162+ 00:00 err: {heap_size,147}, 07T12:27: 22.315162+ 00:00 err: {stack_size,0}, 07T12:27: 22.315162+ 00:00 err: {reductions,212}] 07T12:27: 22.315162+ 00:00 err: neighbour: [{pid,<0.11494.0>}, 07T12:27: 22.315162+ 00:00 err: {registered_ name,[] }, 07T12:27: 22.315162+ 00:00 err: {initial_ call,{gen, init_it, 07T12:27: 22.315162+ 00:00 err: ['Argument_ _1','Argument_ _2', 07T12:27: 22.315162+ 00:00 err: 'Argument_ _3','Argument_ _4', 07T12:27: 22.315162+ 00:00 err: 'Argument_ _5','Argument_ _6']}}, 07T12:27: 22.315162+ 00:00 err: {current_ function, {erlang, hibernate, 3}}, 07T12:27: 22.315162+ 00:00 err: {ancestors, [<0.11493. 0>,rabbit_ amqqueue_ sup,rabbit_ sup, 07T12:27: 22.315162+ 00:00 err: <0.72.0>]}, 07T12:27: 22.315162+ 00:00 err: {messages,[]}, 07T12:27: 22.315162+ 00:00 err: {links, [<0.11493. 0>,<0.11495. 0>]}, 07T12:27: 22.315162+ 00:00 err: {dictionary,[]}, 07T12:27: 22.315162+ 00:00 err: {trap_exit,false}, 07T12:27: 22.315162+ 00:00 err: {status,waiting}, 07T12:27: 22.315162+ 00:00 err: {heap_size,224}, 07T12:27: 22.315162+ 00:00 err: {stack_size,0}, 07T12:27: 22.315162+ 00:00 err: {reductions,120}]
2014-07-
2014-07-
2014-07-
2014-07-
2014-07-
2014-07-
2014-07-
2014-07-
2014-07-
2014-07-
2014-07-
2014-07-
2014-07-
2014-07-
2014-07-
2014-07-
2014-07-
2014-07-
2014-07-
2014-07-
2014-07-
2014-07-
2014-07-
2014-07-
2014-07-
2014-07-
2014-07-
2014-07-
2014-07-
2014-07-
2014-07-
2014-07-
2014-07-
2014-07-
2014-07-
2014-07-
2014-07-
2014-07-
2014-07-
2014-07-
2014-07-
2014-07-
2014-07-
2014-07-
2014-07-
2014-07-
2014-07-
2014-07-
2014-07-
2014-07-
2014-07-
2014-07-
2014-07-
2014-07-
2014-07-
2014-07-
2014-07-
2014-07-
2014-07-
2014-07-
2014-07-
2014-07-
2014-07-
2014-07-
2014-07-
2014-07-
2014-07-
2014-07-
2014-07-
2014-07-
2014-07-
2014-07-
2014-07-
2014-07-
2014-07-
2014-07-
2014-07-
2014-07-
2014-07-
2014-07-
2014-07-
2014-07-
2014-07-
2014-07-
2014-07-
2014-07-
2014-07-
2014-07-
2014-07-
2014-07-
2014-07-
2014-07-
2014-07-
2014-07-
2014-07-
2014-07-
2014-07-