| 2016-12-08 20:04:03 |
Vinoth Kannan Ganapathy |
description |
Contrail version 3.2, build 8
Multi node HA setup, where openstack, contrail, collector and database on each separate node respectively.
seen collector crash @ RedisProcessorExec::SyncDeleteUVEs
Also so many core file got generated..
(gdb) bt
#0 0x00007f129410ecc9 in raise () from /lib/x86_64-linux-gnu/libc.so.6
#1 0x00007f12941120d8 in abort () from /lib/x86_64-linux-gnu/libc.so.6
#2 0x00007f1294107b86 in ?? () from /lib/x86_64-linux-gnu/libc.so.6
#3 0x00007f1294107c32 in __assert_fail () from /lib/x86_64-linux-gnu/libc.so.6
#4 0x00000000005af30d in RedisProcessorExec::SyncDeleteUVEs (redis_ip=..., redis_port=<optimized out>, redis_password="", source="5b8s35",
node_type="Compute", module="contrail-vrouter-agent", instance_id="0", delReply=std::vector of length 0, capacity 0)
at controller/src/analytics/redis_processor_vizd.cc:229
#5 0x00000000005fb3ed in OpServerProxy::DeleteUVEs (this=0x1b592b0, source="5b8s35", module="contrail-vrouter-agent", node_type="Compute", instance_id="0")
at controller/src/analytics/OpServerProxy.cc:826
#6 0x00000000005a1e31 in SandeshGenerator::DisconnectSession (this=this@entry=0x7f12740f40a0, vsession=vsession@entry=0x1c27820)
at controller/src/analytics/generator.cc:175
#7 0x00000000005903dd in Collector::ReceiveSandeshCtrlMsg (this=0x1b67f60, state_machine=<optimized out>, session=<optimized out>, sandesh=<optimized out>)
at controller/src/analytics/collector.cc:285
#8 0x0000000000740ff0 in SandeshServerConnection::ProcessSandeshCtrlMessage (this=this@entry=0x1c4fa60,
msg="<SandeshHeader><Namespace type=\"string\" identifier=\"1\"></Namespace><Timestamp type=\"i64\" identifier=\"2\">1481220943296491</Timestamp><Module type=\"string\" identifier=\"3\">contrail-vrouter-agent</Module>"..., header=..., sandesh_name="SandeshCtrlClientToServer",
header_offset=header_offset@entry=734) at tools/sandesh/library/cpp/sandesh_connection.cc:172
#9 0x000000000073fd7e in ssm::ServerInit::react (this=this@entry=0x7f126010dee0, event=...) at tools/sandesh/library/cpp/sandesh_state_machine.cc:303
#10 0x00000000007404e5 in react<ssm::ServerInit, boost::statechart::event_base, void const*> (eventType=<synthetic pointer>, evt=..., stt=...)
at /usr/include/boost/statechart/custom_reaction.hpp:42
#11 local_react_impl<boost::mpl::list3<boost::statechart::custom_reaction<ssm::EvSandeshCtrlMessageRecv>, boost::statechart::custom_reaction<ssm::EvSandeshMessageRecv>, boost::statechart::in_state_reaction<ssm::EvTcpDeleteSession, SandeshStateMachine, &SandeshStateMachine::DeleteTcpSession> >, boost::statechart::simple_state<ssm::ServerInit, SandeshStateMachine, boost::mpl::list<mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na>, (boost::statechart::history_mode)0> > (
eventType=0xb8e140 <boost::statechart::detail::id_holder<ssm::EvSandeshCtrlMessageRecv>::idProvider_>, evt=..., stt=...)
at /usr/include/boost/statechart/simple_state.hpp:816
#12 local_react<boost::mpl::list3<boost::statechart::custom_reaction<ssm::EvSandeshCtrlMessageRecv>, boost::statechart::custom_reaction<ssm::EvSandeshMessageRecv>, boost::statechart::in_state_reaction<ssm::EvTcpDeleteSession, SandeshStateMachine, &SandeshStateMachine::DeleteTcpSession> > > (
eventType=0xb8e140 <boost::statechart::detail::id_holder<ssm::EvSandeshCtrlMessageRecv>::idProvider_>, evt=..., this=0x7f126010dee0)
at /usr/include/boost/statechart/simple_state.hpp:851
#13 local_react_impl<boost::mpl::list4<boost::statechart::custom_reaction<ssm::EvTcpClose>, boost::statechart::custom_reaction<ssm::EvSandeshCtrlMessageRecv>, boost::statechart::custom_reaction<ssm::EvSandeshMessageRecv>, boost::statechart::in_state_reaction<ssm::EvTcpDeleteSession, SandeshStateMachine, &SandeshStateMachine::DeleteTcpSession> >, boost::statechart::simple_state<ssm::ServerInit, SandeshStateMachine, boost::mpl::list<mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na>, (boost::statechart::history_mode)0> > (eventType=<optimized out>, evt=..., stt=...) at /usr/include/boost/statechart/simple_state.hpp:820
#14 local_react<boost::mpl::list4<boost::statechart::custom_reaction<ssm::EvTcpClose>, boost::statechart::custom_reaction<ssm::EvSandeshCtrlMessageRecv>, boost::statechart::custom_reaction<ssm::EvSandeshMessageRecv>, boost::statechart::in_state_reaction<ssm::EvTcpDeleteSession, SandeshStateMachine, &SandeshStateMachine::DeleteTcpSession> > > (eventType=<optimized out>, evt=..., this=<optimized out>) at /usr/include/boost/statechart/simple_state.hpp:851
#15 local_react_impl<boost::mpl::list<boost::statechart::transition<ssm::EvStop, ssm::Idle, SandeshStateMachine, &SandeshStateMachine::OnIdle>, boost::statechart::custom_reaction<ssm::EvTcpClose>, boost::statechart::custom_reaction<ssm::EvSandeshCtrlMessageRecv>, boost::statechart::custom_reaction<ssm::EvSandeshMessageRecv>, boost::statechart::in_state_reaction<ssm::EvTcpDeleteSession, SandeshStateMachine, &SandeshStateMachine::DeleteTcpSession> >, boost::statechart::simple_state<ssm::ServerInit, SandeshStateMachine, boost::mpl::list<mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na>, (boost::statechart::history_mode)0> > (
eventType=0xb8e140 <boost::statechart::detail::id_holder<ssm::EvSandeshCtrlMessageRecv>::idProvider_>, evt=..., stt=...)
at /usr/include/boost/statechart/simple_state.hpp:820
#16 local_react<boost::mpl::list<boost::statechart::transition<ssm::EvStop, ssm::Idle, SandeshStateMachine, &SandeshStateMachine::OnIdle>, boost::statechart::custom_reaction<ssm::EvTcpClose>, boost::statechart::custom_reaction<ssm::EvSandeshCtrlMessageRecv>, boost::statechart::custom_reaction<ssm::EvSandeshMessageR---Type <return> to continue, or q <return> to quit---
ecv>, boost::statechart::in_state_reaction<ssm::EvTcpDeleteSession, SandeshStateMachine, &SandeshStateMachine::DeleteTcpSession> > > (
eventType=0xb8e140 <boost::statechart::detail::id_holder<ssm::EvSandeshCtrlMessageRecv>::idProvider_>, evt=..., this=0x7f126010dee0)
at /usr/include/boost/statechart/simple_state.hpp:851
#17 boost::statechart::simple_state<ssm::ServerInit, SandeshStateMachine, boost::mpl::list<mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na>, (boost::statechart::history_mode)0>::react_impl (this=0x7f126010dee0, evt=..., eventType=0xb8e140 <boost::statechart::detail::id_holder<ssm::EvSandeshCtrlMessageRecv>::idProvider_>)
at /usr/include/boost/statechart/simple_state.hpp:489
#18 0x000000000073e07b in operator() (this=<synthetic pointer>) at /usr/include/boost/statechart/state_machine.hpp:87
#19 operator()<boost::statechart::detail::send_function<boost::statechart::detail::state_base<std::allocator<void>, boost::statechart::detail::rtti_policy>, boost::statechart::event_base, void const*>, boost::statechart::state_machine<SandeshStateMachine, ssm::Idle>::exception_event_handler> (this=0x1c2b9c8,
action=...) at /usr/include/boost/statechart/null_exception_translator.hpp:33
#20 boost::statechart::state_machine<SandeshStateMachine, ssm::Idle, std::allocator<void>, boost::statechart::null_exception_translator>::send_event (
this=0x1c2b970, evt=...) at /usr/include/boost/statechart/state_machine.hpp:889
#21 0x0000000000735fb5 in process_event (evt=..., this=0x1c2b970) at /usr/include/boost/statechart/state_machine.hpp:275
#22 SandeshStateMachine::DequeueEvent (this=0x1c2b970, ec=...) at tools/sandesh/library/cpp/sandesh_state_machine.cc:800
#23 0x000000000073d1d7 in operator() (a0=<error reading variable: access outside bounds of object referenced via synthetic pointer>, this=0x7f128970de90)
at /usr/include/boost/function/function_template.hpp:767
#24 QueueTaskRunner<SandeshStateMachine::EventContainer, WorkQueue<SandeshStateMachine::EventContainer> >::RunQueue (this=0x7f1278226700)
at controller/src/base/queue_task.h:92
#25 0x000000000046a2bf in TaskImpl::execute (this=0x7f128d10b340) at controller/src/base/task.cc:262
#26 0x00007f1295699b3a in ?? () from /usr/lib/libtbb.so.2
#27 0x00007f1295695816 in ?? () from /usr/lib/libtbb.so.2
#28 0x00007f1295694f4b in ?? () from /usr/lib/libtbb.so.2
#29 0x00007f12956910ff in ?? () from /usr/lib/libtbb.so.2
#30 0x00007f12956912f9 in ?? () from /usr/lib/libtbb.so.2
#31 0x00007f12958b5182 in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#32 0x00007f12941d247d in clone () from /lib/x86_64-linux-gnu/libc.so.6 |
Contrail version 3.2, build 8
Multi node HA setup, where openstack, contrail, collector and database on each separate node respectively.
seen collector crash @ RedisProcessorExec::SyncDeleteUVEs
Also so many core file got generated..
root@5b8s30-vm3:~# contrail-status
== Contrail Analytics ==
supervisor-analytics: active
contrail-alarm-gen:0 active
contrail-analytics-api active
contrail-analytics-nodemgr active
contrail-collector failed
contrail-query-engine active
contrail-snmp-collector active
contrail-topology active
========Run time service failures=============
/var/crashes/core.contrail-collec.13777.5b8s30-vm3.1481220943
/var/crashes/core.contrail-collec.21972.5b8s30-vm3.1481220968
/var/crashes/core.contrail-collec.19114.5b8s30-vm3.1481221224
/var/crashes/core.contrail-collec.19207.5b8s30-vm3.1481221228
/var/crashes/core.contrail-collec.19071.5b8s30-vm3.1481221169
/var/crashes/core.contrail-collec.19226.5b8s30-vm3.1481221231
/var/crashes/core.contrail-collec.18995.5b8s30-vm3.1481221127
/var/crashes/core.contrail-collec.25292.5b8s30-vm3.1481220978
/var/crashes/core.contrail-collec.17522.5b8s30-vm3.1481220952
/var/crashes/core.contrail-collec.18975.5b8s30-vm3.1481221123
/var/crashes/core.contrail-collec.20898.5b8s30-vm3.1481220960
/var/crashes/core.contrail-collec.18955.5b8s30-vm3.1481221120
/var/crashes/core.contrail-collec.19264.5b8s30-vm3.1481221240
/var/crashes/core.contrail-collec.19595.5b8s30-vm3.1481220956
/var/crashes/core.contrail-collec.19091.5b8s30-vm3.1481221171
/var/crashes/core.contrail-collec.24636.5b8s30-vm3.1481220975
/var/crashes/core.contrail-collec.19014.5b8s30-vm3.1481221164
/var/crashes/core.contrail-collec.19245.5b8s30-vm3.1481221235
/var/crashes/core.contrail-collec.23392.5b8s30-vm3.1481220973
/var/crashes/core.contrail-collec.18752.5b8s30-vm3.1481221118
(gdb) bt
#0 0x00007f129410ecc9 in raise () from /lib/x86_64-linux-gnu/libc.so.6
#1 0x00007f12941120d8 in abort () from /lib/x86_64-linux-gnu/libc.so.6
#2 0x00007f1294107b86 in ?? () from /lib/x86_64-linux-gnu/libc.so.6
#3 0x00007f1294107c32 in __assert_fail () from /lib/x86_64-linux-gnu/libc.so.6
#4 0x00000000005af30d in RedisProcessorExec::SyncDeleteUVEs (redis_ip=..., redis_port=<optimized out>, redis_password="", source="5b8s35",
node_type="Compute", module="contrail-vrouter-agent", instance_id="0", delReply=std::vector of length 0, capacity 0)
at controller/src/analytics/redis_processor_vizd.cc:229
#5 0x00000000005fb3ed in OpServerProxy::DeleteUVEs (this=0x1b592b0, source="5b8s35", module="contrail-vrouter-agent", node_type="Compute", instance_id="0")
at controller/src/analytics/OpServerProxy.cc:826
#6 0x00000000005a1e31 in SandeshGenerator::DisconnectSession (this=this@entry=0x7f12740f40a0, vsession=vsession@entry=0x1c27820)
at controller/src/analytics/generator.cc:175
#7 0x00000000005903dd in Collector::ReceiveSandeshCtrlMsg (this=0x1b67f60, state_machine=<optimized out>, session=<optimized out>, sandesh=<optimized out>)
at controller/src/analytics/collector.cc:285
#8 0x0000000000740ff0 in SandeshServerConnection::ProcessSandeshCtrlMessage (this=this@entry=0x1c4fa60,
msg="<SandeshHeader><Namespace type=\"string\" identifier=\"1\"></Namespace><Timestamp type=\"i64\" identifier=\"2\">1481220943296491</Timestamp><Module type=\"string\" identifier=\"3\">contrail-vrouter-agent</Module>"..., header=..., sandesh_name="SandeshCtrlClientToServer",
header_offset=header_offset@entry=734) at tools/sandesh/library/cpp/sandesh_connection.cc:172
#9 0x000000000073fd7e in ssm::ServerInit::react (this=this@entry=0x7f126010dee0, event=...) at tools/sandesh/library/cpp/sandesh_state_machine.cc:303
#10 0x00000000007404e5 in react<ssm::ServerInit, boost::statechart::event_base, void const*> (eventType=<synthetic pointer>, evt=..., stt=...)
at /usr/include/boost/statechart/custom_reaction.hpp:42
#11 local_react_impl<boost::mpl::list3<boost::statechart::custom_reaction<ssm::EvSandeshCtrlMessageRecv>, boost::statechart::custom_reaction<ssm::EvSandeshMessageRecv>, boost::statechart::in_state_reaction<ssm::EvTcpDeleteSession, SandeshStateMachine, &SandeshStateMachine::DeleteTcpSession> >, boost::statechart::simple_state<ssm::ServerInit, SandeshStateMachine, boost::mpl::list<mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na>, (boost::statechart::history_mode)0> > (
eventType=0xb8e140 <boost::statechart::detail::id_holder<ssm::EvSandeshCtrlMessageRecv>::idProvider_>, evt=..., stt=...)
at /usr/include/boost/statechart/simple_state.hpp:816
#12 local_react<boost::mpl::list3<boost::statechart::custom_reaction<ssm::EvSandeshCtrlMessageRecv>, boost::statechart::custom_reaction<ssm::EvSandeshMessageRecv>, boost::statechart::in_state_reaction<ssm::EvTcpDeleteSession, SandeshStateMachine, &SandeshStateMachine::DeleteTcpSession> > > (
eventType=0xb8e140 <boost::statechart::detail::id_holder<ssm::EvSandeshCtrlMessageRecv>::idProvider_>, evt=..., this=0x7f126010dee0)
at /usr/include/boost/statechart/simple_state.hpp:851
#13 local_react_impl<boost::mpl::list4<boost::statechart::custom_reaction<ssm::EvTcpClose>, boost::statechart::custom_reaction<ssm::EvSandeshCtrlMessageRecv>, boost::statechart::custom_reaction<ssm::EvSandeshMessageRecv>, boost::statechart::in_state_reaction<ssm::EvTcpDeleteSession, SandeshStateMachine, &SandeshStateMachine::DeleteTcpSession> >, boost::statechart::simple_state<ssm::ServerInit, SandeshStateMachine, boost::mpl::list<mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na>, (boost::statechart::history_mode)0> > (eventType=<optimized out>, evt=..., stt=...) at /usr/include/boost/statechart/simple_state.hpp:820
#14 local_react<boost::mpl::list4<boost::statechart::custom_reaction<ssm::EvTcpClose>, boost::statechart::custom_reaction<ssm::EvSandeshCtrlMessageRecv>, boost::statechart::custom_reaction<ssm::EvSandeshMessageRecv>, boost::statechart::in_state_reaction<ssm::EvTcpDeleteSession, SandeshStateMachine, &SandeshStateMachine::DeleteTcpSession> > > (eventType=<optimized out>, evt=..., this=<optimized out>) at /usr/include/boost/statechart/simple_state.hpp:851
#15 local_react_impl<boost::mpl::list<boost::statechart::transition<ssm::EvStop, ssm::Idle, SandeshStateMachine, &SandeshStateMachine::OnIdle>, boost::statechart::custom_reaction<ssm::EvTcpClose>, boost::statechart::custom_reaction<ssm::EvSandeshCtrlMessageRecv>, boost::statechart::custom_reaction<ssm::EvSandeshMessageRecv>, boost::statechart::in_state_reaction<ssm::EvTcpDeleteSession, SandeshStateMachine, &SandeshStateMachine::DeleteTcpSession> >, boost::statechart::simple_state<ssm::ServerInit, SandeshStateMachine, boost::mpl::list<mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na>, (boost::statechart::history_mode)0> > (
eventType=0xb8e140 <boost::statechart::detail::id_holder<ssm::EvSandeshCtrlMessageRecv>::idProvider_>, evt=..., stt=...)
at /usr/include/boost/statechart/simple_state.hpp:820
#16 local_react<boost::mpl::list<boost::statechart::transition<ssm::EvStop, ssm::Idle, SandeshStateMachine, &SandeshStateMachine::OnIdle>, boost::statechart::custom_reaction<ssm::EvTcpClose>, boost::statechart::custom_reaction<ssm::EvSandeshCtrlMessageRecv>, boost::statechart::custom_reaction<ssm::EvSandeshMessageR---Type <return> to continue, or q <return> to quit---
ecv>, boost::statechart::in_state_reaction<ssm::EvTcpDeleteSession, SandeshStateMachine, &SandeshStateMachine::DeleteTcpSession> > > (
eventType=0xb8e140 <boost::statechart::detail::id_holder<ssm::EvSandeshCtrlMessageRecv>::idProvider_>, evt=..., this=0x7f126010dee0)
at /usr/include/boost/statechart/simple_state.hpp:851
#17 boost::statechart::simple_state<ssm::ServerInit, SandeshStateMachine, boost::mpl::list<mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na>, (boost::statechart::history_mode)0>::react_impl (this=0x7f126010dee0, evt=..., eventType=0xb8e140 <boost::statechart::detail::id_holder<ssm::EvSandeshCtrlMessageRecv>::idProvider_>)
at /usr/include/boost/statechart/simple_state.hpp:489
#18 0x000000000073e07b in operator() (this=<synthetic pointer>) at /usr/include/boost/statechart/state_machine.hpp:87
#19 operator()<boost::statechart::detail::send_function<boost::statechart::detail::state_base<std::allocator<void>, boost::statechart::detail::rtti_policy>, boost::statechart::event_base, void const*>, boost::statechart::state_machine<SandeshStateMachine, ssm::Idle>::exception_event_handler> (this=0x1c2b9c8,
action=...) at /usr/include/boost/statechart/null_exception_translator.hpp:33
#20 boost::statechart::state_machine<SandeshStateMachine, ssm::Idle, std::allocator<void>, boost::statechart::null_exception_translator>::send_event (
this=0x1c2b970, evt=...) at /usr/include/boost/statechart/state_machine.hpp:889
#21 0x0000000000735fb5 in process_event (evt=..., this=0x1c2b970) at /usr/include/boost/statechart/state_machine.hpp:275
#22 SandeshStateMachine::DequeueEvent (this=0x1c2b970, ec=...) at tools/sandesh/library/cpp/sandesh_state_machine.cc:800
#23 0x000000000073d1d7 in operator() (a0=<error reading variable: access outside bounds of object referenced via synthetic pointer>, this=0x7f128970de90)
at /usr/include/boost/function/function_template.hpp:767
#24 QueueTaskRunner<SandeshStateMachine::EventContainer, WorkQueue<SandeshStateMachine::EventContainer> >::RunQueue (this=0x7f1278226700)
at controller/src/base/queue_task.h:92
#25 0x000000000046a2bf in TaskImpl::execute (this=0x7f128d10b340) at controller/src/base/task.cc:262
#26 0x00007f1295699b3a in ?? () from /usr/lib/libtbb.so.2
#27 0x00007f1295695816 in ?? () from /usr/lib/libtbb.so.2
#28 0x00007f1295694f4b in ?? () from /usr/lib/libtbb.so.2
#29 0x00007f12956910ff in ?? () from /usr/lib/libtbb.so.2
#30 0x00007f12956912f9 in ?? () from /usr/lib/libtbb.so.2
#31 0x00007f12958b5182 in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#32 0x00007f12941d247d in clone () from /lib/x86_64-linux-gnu/libc.so.6 |
|
