Contrail version 3.2, build 8
Multi node HA setup, where openstack, contrail, collector and database on each separate node respectively.
seen collector crash @ RedisProcessorExec::SyncDeleteUVEs Also so many core file got generated..
(gdb) bt #0 0x00007f129410ecc9 in raise () from /lib/x86_64-linux-gnu/libc.so.6 #1 0x00007f12941120d8 in abort () from /lib/x86_64-linux-gnu/libc.so.6 #2 0x00007f1294107b86 in ?? () from /lib/x86_64-linux-gnu/libc.so.6 #3 0x00007f1294107c32 in __assert_fail () from /lib/x86_64-linux-gnu/libc.so.6 #4 0x00000000005af30d in RedisProcessorExec::SyncDeleteUVEs (redis_ip=..., redis_port=<optimized out>, redis_password="", source="5b8s35", node_type="Compute", module="contrail-vrouter-agent", instance_id="0", delReply=std::vector of length 0, capacity 0) at controller/src/analytics/redis_processor_vizd.cc:229 #5 0x00000000005fb3ed in OpServerProxy::DeleteUVEs (this=0x1b592b0, source="5b8s35", module="contrail-vrouter-agent", node_type="Compute", instance_id="0") at controller/src/analytics/OpServerProxy.cc:826 #6 0x00000000005a1e31 in SandeshGenerator::DisconnectSession (this=this@entry=0x7f12740f40a0, vsession=vsession@entry=0x1c27820) at controller/src/analytics/generator.cc:175 #7 0x00000000005903dd in Collector::ReceiveSandeshCtrlMsg (this=0x1b67f60, state_machine=<optimized out>, session=<optimized out>, sandesh=<optimized out>) at controller/src/analytics/collector.cc:285 #8 0x0000000000740ff0 in SandeshServerConnection::ProcessSandeshCtrlMessage (this=this@entry=0x1c4fa60, msg="<SandeshHeader><Namespace type=\"string\" identifier=\"1\"></Namespace><Timestamp type=\"i64\" identifier=\"2\">1481220943296491</Timestamp><Module type=\"string\" identifier=\"3\">contrail-vrouter-agent</Module>"..., header=..., sandesh_name="SandeshCtrlClientToServer", header_offset=header_offset@entry=734) at tools/sandesh/library/cpp/sandesh_connection.cc:172 #9 0x000000000073fd7e in ssm::ServerInit::react (this=this@entry=0x7f126010dee0, event=...) at tools/sandesh/library/cpp/sandesh_state_machine.cc:303 #10 0x00000000007404e5 in react<ssm::ServerInit, boost::statechart::event_base, void const*> (eventType=<synthetic pointer>, evt=..., stt=...) at /usr/include/boost/statechart/custom_reaction.hpp:42 #11 local_react_impl<boost::mpl::list3<boost::statechart::custom_reaction<ssm::EvSandeshCtrlMessageRecv>, boost::statechart::custom_reaction<ssm::EvSandeshMessageRecv>, boost::statechart::in_state_reaction<ssm::EvTcpDeleteSession, SandeshStateMachine, &SandeshStateMachine::DeleteTcpSession> >, boost::statechart::simple_state<ssm::ServerInit, SandeshStateMachine, boost::mpl::list<mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na>, (boost::statechart::history_mode)0> > ( eventType=0xb8e140 <boost::statechart::detail::id_holder<ssm::EvSandeshCtrlMessageRecv>::idProvider_>, evt=..., stt=...) at /usr/include/boost/statechart/simple_state.hpp:816 #12 local_react<boost::mpl::list3<boost::statechart::custom_reaction<ssm::EvSandeshCtrlMessageRecv>, boost::statechart::custom_reaction<ssm::EvSandeshMessageRecv>, boost::statechart::in_state_reaction<ssm::EvTcpDeleteSession, SandeshStateMachine, &SandeshStateMachine::DeleteTcpSession> > > ( eventType=0xb8e140 <boost::statechart::detail::id_holder<ssm::EvSandeshCtrlMessageRecv>::idProvider_>, evt=..., this=0x7f126010dee0) at /usr/include/boost/statechart/simple_state.hpp:851 #13 local_react_impl<boost::mpl::list4<boost::statechart::custom_reaction<ssm::EvTcpClose>, boost::statechart::custom_reaction<ssm::EvSandeshCtrlMessageRecv>, boost::statechart::custom_reaction<ssm::EvSandeshMessageRecv>, boost::statechart::in_state_reaction<ssm::EvTcpDeleteSession, SandeshStateMachine, &SandeshStateMachine::DeleteTcpSession> >, boost::statechart::simple_state<ssm::ServerInit, SandeshStateMachine, boost::mpl::list<mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na>, (boost::statechart::history_mode)0> > (eventType=<optimized out>, evt=..., stt=...) at /usr/include/boost/statechart/simple_state.hpp:820 #14 local_react<boost::mpl::list4<boost::statechart::custom_reaction<ssm::EvTcpClose>, boost::statechart::custom_reaction<ssm::EvSandeshCtrlMessageRecv>, boost::statechart::custom_reaction<ssm::EvSandeshMessageRecv>, boost::statechart::in_state_reaction<ssm::EvTcpDeleteSession, SandeshStateMachine, &SandeshStateMachine::DeleteTcpSession> > > (eventType=<optimized out>, evt=..., this=<optimized out>) at /usr/include/boost/statechart/simple_state.hpp:851 #15 local_react_impl<boost::mpl::list<boost::statechart::transition<ssm::EvStop, ssm::Idle, SandeshStateMachine, &SandeshStateMachine::OnIdle>, boost::statechart::custom_reaction<ssm::EvTcpClose>, boost::statechart::custom_reaction<ssm::EvSandeshCtrlMessageRecv>, boost::statechart::custom_reaction<ssm::EvSandeshMessageRecv>, boost::statechart::in_state_reaction<ssm::EvTcpDeleteSession, SandeshStateMachine, &SandeshStateMachine::DeleteTcpSession> >, boost::statechart::simple_state<ssm::ServerInit, SandeshStateMachine, boost::mpl::list<mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na>, (boost::statechart::history_mode)0> > ( eventType=0xb8e140 <boost::statechart::detail::id_holder<ssm::EvSandeshCtrlMessageRecv>::idProvider_>, evt=..., stt=...) at /usr/include/boost/statechart/simple_state.hpp:820 #16 local_react<boost::mpl::list<boost::statechart::transition<ssm::EvStop, ssm::Idle, SandeshStateMachine, &SandeshStateMachine::OnIdle>, boost::statechart::custom_reaction<ssm::EvTcpClose>, boost::statechart::custom_reaction<ssm::EvSandeshCtrlMessageRecv>, boost::statechart::custom_reaction<ssm::EvSandeshMessageR---Type <return> to continue, or q <return> to quit--- ecv>, boost::statechart::in_state_reaction<ssm::EvTcpDeleteSession, SandeshStateMachine, &SandeshStateMachine::DeleteTcpSession> > > ( eventType=0xb8e140 <boost::statechart::detail::id_holder<ssm::EvSandeshCtrlMessageRecv>::idProvider_>, evt=..., this=0x7f126010dee0) at /usr/include/boost/statechart/simple_state.hpp:851 #17 boost::statechart::simple_state<ssm::ServerInit, SandeshStateMachine, boost::mpl::list<mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na>, (boost::statechart::history_mode)0>::react_impl (this=0x7f126010dee0, evt=..., eventType=0xb8e140 <boost::statechart::detail::id_holder<ssm::EvSandeshCtrlMessageRecv>::idProvider_>) at /usr/include/boost/statechart/simple_state.hpp:489 #18 0x000000000073e07b in operator() (this=<synthetic pointer>) at /usr/include/boost/statechart/state_machine.hpp:87 #19 operator()<boost::statechart::detail::send_function<boost::statechart::detail::state_base<std::allocator<void>, boost::statechart::detail::rtti_policy>, boost::statechart::event_base, void const*>, boost::statechart::state_machine<SandeshStateMachine, ssm::Idle>::exception_event_handler> (this=0x1c2b9c8, action=...) at /usr/include/boost/statechart/null_exception_translator.hpp:33 #20 boost::statechart::state_machine<SandeshStateMachine, ssm::Idle, std::allocator<void>, boost::statechart::null_exception_translator>::send_event ( this=0x1c2b970, evt=...) at /usr/include/boost/statechart/state_machine.hpp:889 #21 0x0000000000735fb5 in process_event (evt=..., this=0x1c2b970) at /usr/include/boost/statechart/state_machine.hpp:275 #22 SandeshStateMachine::DequeueEvent (this=0x1c2b970, ec=...) at tools/sandesh/library/cpp/sandesh_state_machine.cc:800 #23 0x000000000073d1d7 in operator() (a0=<error reading variable: access outside bounds of object referenced via synthetic pointer>, this=0x7f128970de90) at /usr/include/boost/function/function_template.hpp:767 #24 QueueTaskRunner<SandeshStateMachine::EventContainer, WorkQueue<SandeshStateMachine::EventContainer> >::RunQueue (this=0x7f1278226700) at controller/src/base/queue_task.h:92 #25 0x000000000046a2bf in TaskImpl::execute (this=0x7f128d10b340) at controller/src/base/task.cc:262 #26 0x00007f1295699b3a in ?? () from /usr/lib/libtbb.so.2 #27 0x00007f1295695816 in ?? () from /usr/lib/libtbb.so.2 #28 0x00007f1295694f4b in ?? () from /usr/lib/libtbb.so.2 #29 0x00007f12956910ff in ?? () from /usr/lib/libtbb.so.2 #30 0x00007f12956912f9 in ?? () from /usr/lib/libtbb.so.2 #31 0x00007f12958b5182 in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0 #32 0x00007f12941d247d in clone () from /lib/x86_64-linux-gnu/libc.so.6
Contrail version 3.2, build 8
Multi node HA setup, where openstack, contrail, collector and database on each separate node respectively.
seen collector crash @ RedisProcessorE xec::SyncDelete UVEs
Also so many core file got generated..
(gdb) bt 64-linux- gnu/libc. so.6 64-linux- gnu/libc. so.6 64-linux- gnu/libc. so.6 64-linux- gnu/libc. so.6 xec::SyncDelete UVEs (redis_ip=..., redis_port= <optimized out>, redis_password="", source="5b8s35", type="Compute" , module= "contrail- vrouter- agent", instance_id="0", delReply= std::vector of length 0, capacity 0) src/analytics/ redis_processor _vizd.cc: 229 :DeleteUVEs (this=0x1b592b0, source="5b8s35", module= "contrail- vrouter- agent", node_type= "Compute" , instance_id="0") src/analytics/ OpServerProxy. cc:826 r::DisconnectSe ssion (this=this@ entry=0x7f12740 f40a0, vsession= vsession@ entry=0x1c27820 ) src/analytics/ generator. cc:175 :ReceiveSandesh CtrlMsg (this=0x1b67f60, state_machine= <optimized out>, session=<optimized out>, sandesh=<optimized out>) src/analytics/ collector. cc:285 nnection: :ProcessSandesh CtrlMessage (this=this@ entry=0x1c4fa60 , "<SandeshHeader ><Namespace type=\"string\" identifier= \"1\">< /Namespace> <Timestamp type=\"i64\" identifier= \"2\">148122094 3296491< /Timestamp> <Module type=\"string\" identifier= \"3\">contrail- vrouter- agent</ Module> "..., header=..., sandesh_ name="SandeshCt rlClientToServe r", offset= header_ offset@ entry=734) at tools/sandesh/ library/ cpp/sandesh_ connection. cc:172 ::react (this=this@ entry=0x7f12601 0dee0, event=...) at tools/sandesh/ library/ cpp/sandesh_ state_machine. cc:303 :ServerInit, boost:: statechart: :event_ base, void const*> (eventType= <synthetic pointer>, evt=..., stt=...) boost/statechar t/custom_ reaction. hpp:42 impl<boost: :mpl::list3< boost:: statechart: :custom_ reaction< ssm::EvSandeshC trlMessageRecv> , boost:: statechart: :custom_ reaction< ssm::EvSandeshM essageRecv> , boost:: statechart: :in_state_ reaction< ssm::EvTcpDelet eSession, SandeshStateMac hine, &SandeshStateMa chine:: DeleteTcpSessio n> >, boost:: statechart: :simple_ state<ssm: :ServerInit, SandeshStateMac hine, boost:: mpl::list< mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na>, (boost: :statechart: :history_ mode)0> > ( 0xb8e140 <boost: :statechart: :detail: :id_holder< ssm::EvSandeshC trlMessageRecv> ::idProvider_ >, evt=..., stt=...) boost/statechar t/simple_ state.hpp: 816 boost:: mpl::list3< boost:: statechart: :custom_ reaction< ssm::EvSandeshC trlMessageRecv> , boost:: statechart: :custom_ reaction< ssm::EvSandeshM essageRecv> , boost:: statechart: :in_state_ reaction< ssm::EvTcpDelet eSession, SandeshStateMac hine, &SandeshStateMa chine:: DeleteTcpSessio n> > > ( 0xb8e140 <boost: :statechart: :detail: :id_holder< ssm::EvSandeshC trlMessageRecv> ::idProvider_ >, evt=..., this=0x7f126010 dee0) boost/statechar t/simple_ state.hpp: 851 impl<boost: :mpl::list4< boost:: statechart: :custom_ reaction< ssm::EvTcpClose >, boost:: statechart: :custom_ reaction< ssm::EvSandeshC trlMessageRecv> , boost:: statechart: :custom_ reaction< ssm::EvSandeshM essageRecv> , boost:: statechart: :in_state_ reaction< ssm::EvTcpDelet eSession, SandeshStateMac hine, &SandeshStateMa chine:: DeleteTcpSessio n> >, boost:: statechart: :simple_ state<ssm: :ServerInit, SandeshStateMac hine, boost:: mpl::list< mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na>, (boost: :statechart: :history_ mode)0> > (eventType= <optimized out>, evt=..., stt=...) at /usr/include/ boost/statechar t/simple_ state.hpp: 820 boost:: mpl::list4< boost:: statechart: :custom_ reaction< ssm::EvTcpClose >, boost:: statechart: :custom_ reaction< ssm::EvSandeshC trlMessageRecv> , boost:: statechart: :custom_ reaction< ssm::EvSandeshM essageRecv> , boost:: statechart: :in_state_ reaction< ssm::EvTcpDelet eSession, SandeshStateMac hine, &SandeshStateMa chine:: DeleteTcpSessio n> > > (eventType= <optimized out>, evt=..., this=<optimized out>) at /usr/include/ boost/statechar t/simple_ state.hpp: 851 impl<boost: :mpl::list< boost:: statechart: :transition< ssm::EvStop, ssm::Idle, SandeshStateMac hine, &SandeshStateMa chine:: OnIdle> , boost:: statechart: :custom_ reaction< ssm::EvTcpClose >, boost:: statechart: :custom_ reaction< ssm::EvSandeshC trlMessageRecv> , boost:: statechart: :custom_ reaction< ssm::EvSandeshM essageRecv> , boost:: statechart: :in_state_ reaction< ssm::EvTcpDelet eSession, SandeshStateMac hine, &SandeshStateMa chine:: DeleteTcpSessio n> >, boost:: statechart: :simple_ state<ssm: :ServerInit, SandeshStateMac hine, boost:: mpl::list< mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na>, (boost: :statechart: :history_ mode)0> > ( 0xb8e140 <boost: :statechart: :detail: :id_holder< ssm::EvSandeshC trlMessageRecv> ::idProvider_ >, evt=..., stt=...) boost/statechar t/simple_ state.hpp: 820 boost:: mpl::list< boost:: statechart: :transition< ssm::EvStop, ssm::Idle, SandeshStateMac hine, &SandeshStateMa chine:: OnIdle> , boost:: statechart: :custom_ reaction< ssm::EvTcpClose >, boost:: statechart: :custom_ reaction< ssm::EvSandeshC trlMessageRecv> , boost:: statechart: :custom_ reaction< ssm::EvSandeshM essageR- --Type <return> to continue, or q <return> to quit--- statechart: :in_state_ reaction< ssm::EvTcpDelet eSession, SandeshStateMac hine, &SandeshStateMa chine:: DeleteTcpSessio n> > > ( 0xb8e140 <boost: :statechart: :detail: :id_holder< ssm::EvSandeshC trlMessageRecv> ::idProvider_ >, evt=..., this=0x7f126010 dee0) boost/statechar t/simple_ state.hpp: 851 statechart: :simple_ state<ssm: :ServerInit, SandeshStateMac hine, boost:: mpl::list< mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na>, (boost: :statechart: :history_ mode)0> ::react_ impl (this=0x7f12601 0dee0, evt=..., eventType=0xb8e140 <boost: :statechart: :detail: :id_holder< ssm::EvSandeshC trlMessageRecv> ::idProvider_ >) boost/statechar t/simple_ state.hpp: 489 boost/statechar t/state_ machine. hpp:87 )<boost: :statechart: :detail: :send_function< boost:: statechart: :detail: :state_ base<std: :allocator< void>, boost:: statechart: :detail: :rtti_policy> , boost:: statechart: :event_ base, void const*>, boost:: statechart: :state_ machine< SandeshStateMac hine, ssm::Idle> ::exception_ event_handler> (this=0x1c2b9c8, boost/statechar t/null_ exception_ translator. hpp:33 statechart: :state_ machine< SandeshStateMac hine, ssm::Idle, std::allocator< void>, boost:: statechart: :null_exception _translator> ::send_ event ( boost/statechar t/state_ machine. hpp:889 boost/statechar t/state_ machine. hpp:275 hine::DequeueEv ent (this=0x1c2b970, ec=...) at tools/sandesh/ library/ cpp/sandesh_ state_machine. cc:800 de90) boost/function/ function_ template. hpp:767 <SandeshStateMa chine:: EventContainer, WorkQueue< SandeshStateMac hine::EventCont ainer> >::RunQueue (this=0x7f12782 26700) src/base/ queue_task. h:92 0b340) at controller/ src/base/ task.cc: 262 libtbb. so.2 libtbb. so.2 libtbb. so.2 libtbb. so.2 libtbb. so.2 64-linux- gnu/libpthread. so.0 64-linux- gnu/libc. so.6
#0 0x00007f129410ecc9 in raise () from /lib/x86_
#1 0x00007f12941120d8 in abort () from /lib/x86_
#2 0x00007f1294107b86 in ?? () from /lib/x86_
#3 0x00007f1294107c32 in __assert_fail () from /lib/x86_
#4 0x00000000005af30d in RedisProcessorE
node_
at controller/
#5 0x00000000005fb3ed in OpServerProxy:
at controller/
#6 0x00000000005a1e31 in SandeshGenerato
at controller/
#7 0x00000000005903dd in Collector:
at controller/
#8 0x0000000000740ff0 in SandeshServerCo
msg=
header_
#9 0x000000000073fd7e in ssm::ServerInit
#10 0x00000000007404e5 in react<ssm:
at /usr/include/
#11 local_react_
eventType=
at /usr/include/
#12 local_react<
eventType=
at /usr/include/
#13 local_react_
#14 local_react<
#15 local_react_
eventType=
at /usr/include/
#16 local_react<
ecv>, boost::
eventType=
at /usr/include/
#17 boost::
at /usr/include/
#18 0x000000000073e07b in operator() (this=<synthetic pointer>) at /usr/include/
#19 operator(
action=...) at /usr/include/
#20 boost::
this=0x1c2b970, evt=...) at /usr/include/
#21 0x0000000000735fb5 in process_event (evt=..., this=0x1c2b970) at /usr/include/
#22 SandeshStateMac
#23 0x000000000073d1d7 in operator() (a0=<error reading variable: access outside bounds of object referenced via synthetic pointer>, this=0x7f128970
at /usr/include/
#24 QueueTaskRunner
at controller/
#25 0x000000000046a2bf in TaskImpl::execute (this=0x7f128d1
#26 0x00007f1295699b3a in ?? () from /usr/lib/
#27 0x00007f1295695816 in ?? () from /usr/lib/
#28 0x00007f1295694f4b in ?? () from /usr/lib/
#29 0x00007f12956910ff in ?? () from /usr/lib/
#30 0x00007f12956912f9 in ?? () from /usr/lib/
#31 0x00007f12958b5182 in start_thread () from /lib/x86_
#32 0x00007f12941d247d in clone () from /lib/x86_