2.21.1 #15
Under traffic load (high throughput/CPS), agent crash with this bt is seen:
[Thread debugging using libthread_db enabled] Using host libthread_db library "/lib/x86_64-linux-gnu/libthread_db.so.1". Core was generated by `/usr/bin/contrail-vrouter-agent'. Program terminated with signal SIGSEGV, Segmentation fault. #0 endpoint (other=..., this=0x7f5cffbfd340) at /usr/include/boost/asio/ip/detail/endpoint.hpp:48 48 /usr/include/boost/asio/ip/detail/endpoint.hpp: No such file or directory. (gdb) bt #0 endpoint (other=..., this=0x7f5cffbfd340) at /usr/include/boost/asio/ip/detail/endpoint.hpp:48 #1 basic_endpoint (other=..., this=0x7f5cffbfd340) at /usr/include/boost/asio/ip/basic_endpoint.hpp:97 #2 remote_endpoint (this=0x0) at controller/src/io/tcp_session.h:129 #3 scm::Connect::ToIdle (this=this@entry=0x7f5d1c6465c0, state_machine=state_machine@entry=0x7f5d400877f0, event_name=event_name@entry=0x10875ee "EvConnectTimerExpired") at tools/sandesh/library/cpp/sandesh_client_sm.cc:422 #4 0x0000000000f50438 in scm::Connect::react (this=this@entry=0x7f5d1c6465c0, event=...) at tools/sandesh/library/cpp/sandesh_client_sm.cc:370 #5 0x0000000000f516b8 in react<scm::Connect, boost::statechart::event_base, void const*> (eventType=<synthetic pointer>, evt=..., stt=...) at /usr/include/boost/statechart/custom_reaction.hpp:42 #6 local_react_impl<boost::mpl::list7<boost::statechart::custom_reaction<scm::EvConnectTimerExpired>, boost::statechart::custom_reaction<scm::EvTcpConnected>, boost::statechart::custom_reaction<scm::EvTcpConnectFail>, boost::statechart::custom_reaction<scm::EvTcpClose>, boost::statechart::custom_reaction<scm::EvDiscUpdate>, boost::statechart::in_state_reaction<scm::EvSandeshSend, SandeshClientSMImpl, &SandeshClientSMImpl::ReleaseSandesh>, boost::statechart::in_state_reaction<scm::EvTcpDeleteSession, SandeshClientSMImpl, &SandeshClientSMImpl::DeleteTcpSession> >, boost::statechart::simple_state<scm::Connect, SandeshClientSMImpl, boost::mpl::list<mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na>, (boost::statechart::history_mode)0> > (eventType=<optimized out>, evt=..., stt=...) at /usr/include/boost/statechart/simple_state.hpp:816 #7 local_react<boost::mpl::list7<boost::statechart::custom_reaction<scm::EvConnectTimerExpired>, boost::statechart::custom_reaction<scm::EvTcpConnected>, boost::statechart::custom_reaction<scm::EvTcpConnectFail>, boost::statechart::custom_reaction<scm::EvTcpClose>, boost::statechart::custom_reaction<scm::EvDiscUpdate>, boost::statechart::in_state_reaction<scm::EvSandeshSend, SandeshClientSMImpl, &SandeshClientSMImpl::ReleaseSandesh>, boost::statechart::in_state_reaction<scm::EvTcpDeleteSession, SandeshClientSMImpl, &SandeshClientSMImpl::DeleteTcpSession> > > ( eventType=<optimized out>, evt=..., this=<optimized out>) at /usr/include/boost/statechart/simple_state.hpp:851 #8 local_react_impl<boost::mpl::list<boost::statechart::transition<scm::EvStop, scm::Idle, SandeshClientSMImpl, &SandeshClientSMImpl::OnIdle>, boost::statechart::custom_reaction<scm::EvConnectTimerExpired>, boost::statechart::custom_reaction<scm::EvTcpConnected>, boost::statechart::custom_reaction<scm::EvTcpConnectFail>, boost::statechart::custom_reaction<scm::EvTcpClose>, boost::statechart::custom_reaction<scm::EvDiscUpdate>, boost::statechart::in_state_reaction<scm::EvSandeshSend, SandeshClientSMImpl, &SandeshClientSMImpl::ReleaseSandesh>, boost::statechart::in_state_reaction<scm::EvTcpDeleteSession, SandeshClientSMImpl, &SandeshClientSMImpl::DeleteTcpSession> >, boost::statechart::simple_state<scm::Connect, SandeshClientSMImpl, boost::mpl::list<mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na>, (boost::statechart::history_mode)0> > ( eventType=0x153eb80 <boost::statechart::detail::id_holder<scm::EvConnectTimerExpired>::idProvider_>, evt=..., stt=...) at /usr/include/boost/statechart/simple_state.hpp:820 #9 local_react<boost::mpl::list<boost::statechart::transition<scm::EvStop, scm::Idle, SandeshClientSMImpl, &SandeshClientSMImpl::OnIdle>, boost::statechart::custom_reaction<scm::EvConnectTimerExpired>, boost::statechart::custom_reaction<scm::EvTcpConnected>, boost::statechart::custom_reaction<scm::EvTcpConnectFail>, boost::statechart::custom_reaction<scm::EvTcpClose>, boost::statechart::custom_reaction<scm::EvDiscUpdate>, boost::statechart::in_state_reaction<scm::EvSandeshSend, SandeshClientSMImpl, &SandeshClientSMImpl::ReleaseSandesh>, boost::statechart::in_state_reaction<scm::EvTcpDeleteSession, SandeshClientSMImpl, &SandeshClientSMImpl::DeleteTcpSession> > > ( eventType=0x153eb80 <boost::statechart::detail::id_holder<scm::EvConnectTimerExpired>::idProvider_>, evt=..., this=0x7f5d1c6465c0) at /usr/include/boost/statechart/simple_state.hpp:851 #10 boost::statechart::simple_state<scm::Connect, SandeshClientSMImpl, boost::mpl::list<mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na>, (boost::statechart::history_mode)0>::react_impl (this=0x7f5d1c6465c0, evt=..., ---Type <return> to continue, or q <return> to quit--- eventType=0x153eb80 <boost::statechart::detail::id_holder<scm::EvConnectTimerExpired>::idProvider_>) at /usr/include/boost/statechart/simple_state.hpp:489 #11 0x0000000000f4c69b in operator() (this=<synthetic pointer>) at /usr/include/boost/statechart/state_machine.hpp:87 #12 operator()<boost::statechart::detail::send_function<boost::statechart::detail::state_base<std::allocator<void>, boost::statechart::detail::rtti_policy>, boost::statechart::event_base, void const*>, boost::statechart::state_machine<SandeshClientSMImpl, scm::Idle>::exception_event_handler> (this=0x7f5d400878b0, action=...) at /usr/include/boost/statechart/null_exception_translator.hpp:33 #13 boost::statechart::state_machine<SandeshClientSMImpl, scm::Idle, std::allocator<void>, boost::statechart::null_exception_translator>::send_event (this=0x7f5d40087858, evt=...) at /usr/include/boost/statechart/state_machine.hpp:889 #14 0x0000000000f43ade in process_event (evt=..., this=0x7f5d40087858) at /usr/include/boost/statechart/state_machine.hpp:275 #15 SandeshClientSMImpl::DequeueEvent (this=0x7f5d400877f0, ec=...) at tools/sandesh/library/cpp/sandesh_client_sm.cc:871 #16 0x0000000000f481e2 in operator() (a1=..., p=0x7f5d400877f0, this=0x7f5cffbfda28) at /usr/include/boost/bind/mem_fn_template.hpp:165 #17 operator()<bool, boost::_mfi::mf1<bool, SandeshClientSMImpl, SandeshClientSMImpl::EventContainer>, boost::_bi::list1<SandeshClientSMImpl::EventContainer&> > (a=<synthetic pointer>, f=..., this=0x7f5cffbfda38) at /usr/include/boost/bind/bind.hpp:303 #18 operator()<SandeshClientSMImpl::EventContainer> (a1=..., this=0x7f5cffbfda28) at /usr/include/boost/bind/bind_template.hpp:32 #19 boost::detail::function::function_obj_invoker1<boost::_bi::bind_t<bool, boost::_mfi::mf1<bool, SandeshClientSMImpl, SandeshClientSMImpl::EventContainer>, boost::_bi::list2<boost::_bi::value<SandeshClientSMImpl*>, boost::arg<1> > >, bool, SandeshClientSMImpl::EventContainer>::invoke (function_obj_ptr=..., a0=...) at /usr/include/boost/function/function_template.hpp:132 #20 0x0000000000f4b887 in operator() (a0=<error reading variable: access outside bounds of object referenced via synthetic pointer>, this=0x7f5cffbfda20) at /usr/include/boost/function/function_template.hpp:767 #21 QueueTaskRunner<SandeshClientSMImpl::EventContainer, WorkQueue<SandeshClientSMImpl::EventContainer> >::RunQueue (this=0x7f5cc8539650) at controller/src/base/queue_task.h:81 #22 0x0000000000fdd8b0 in TaskImpl::execute (this=0x7f5d47527c40) at controller/src/base/task.cc:238 #23 0x00007f5d4e9a0b3a in ?? () from /usr/lib/libtbb.so.2 #24 0x00007f5d4e99c816 in ?? () from /usr/lib/libtbb.so.2 #25 0x00007f5d4e99bf4b in ?? () from /usr/lib/libtbb.so.2 #26 0x00007f5d4e9980ff in ?? () from /usr/lib/libtbb.so.2 #27 0x00007f5d4e9982f9 in ?? () from /usr/lib/libtbb.so.2 #28 0x00007f5d4ebbc182 in start_thread (arg=0x7f5cffbfe700) at pthread_create.c:312 #29 0x00007f5d4de9547d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111
2.21.1 #15
Under traffic load (high throughput/CPS), agent crash with this bt is seen:
[Thread debugging using libthread_db enabled] 64-linux- gnu/libthread_ db.so.1" . contrail- vrouter- agent'. d340) at /usr/include/ boost/asio/ ip/detail/ endpoint. hpp:48 boost/asio/ ip/detail/ endpoint. hpp: No such file or directory. d340) at /usr/include/ boost/asio/ ip/detail/ endpoint. hpp:48 d340) at /usr/include/ boost/asio/ ip/basic_ endpoint. hpp:97 src/io/ tcp_session. h:129 :ToIdle (this=this@ entry=0x7f5d1c6 465c0, state_machine= state_machine@ entry=0x7f5d400 877f0, name=event_ name@entry= 0x10875ee "EvConnectTimer Expired" ) at tools/sandesh/ library/ cpp/sandesh_ client_ sm.cc:422 entry=0x7f5d1c6 465c0, event=...) library/ cpp/sandesh_ client_ sm.cc:370 statechart: :event_ base, void const*> (eventType= <synthetic pointer>, evt=..., stt=...) boost/statechar t/custom_ reaction. hpp:42 impl<boost: :mpl::list7< boost:: statechart: :custom_ reaction< scm::EvConnectT imerExpired> , boost:: statechart: :custom_ reaction< scm::EvTcpConne cted>, boost:: statechart: :custom_ reaction< scm::EvTcpConne ctFail> , boost:: statechart: :custom_ reaction< scm::EvTcpClose >, boost:: statechart: :custom_ reaction< scm::EvDiscUpda te>, boost:: statechart: :in_state_ reaction< scm::EvSandeshS end, SandeshClientSM Impl, &SandeshClientS MImpl:: ReleaseSandesh> , boost:: statechart: :in_state_ reaction< scm::EvTcpDelet eSession, SandeshClientSM Impl, &SandeshClientS MImpl:: DeleteTcpSessio n> >, boost:: statechart: :simple_ state<scm: :Connect, SandeshClientSM Impl, boost:: mpl::list< mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na>, (boost: :statechart: :history_ mode)0> > (eventType= <optimized out>, evt=..., stt=...) at /usr/include/ boost/statechar t/simple_ state.hpp: 816 boost:: mpl::list7< boost:: statechart: :custom_ reaction< scm::EvConnectT imerExpired> , boost:: statechart: :custom_ reaction< scm::EvTcpConne cted>, boost:: statechart: :custom_ reaction< scm::EvTcpConne ctFail> , boost:: statechart: :custom_ reaction< scm::EvTcpClose >, boost:: statechart: :custom_ reaction< scm::EvDiscUpda te>, boost:: statechart: :in_state_ reaction< scm::EvSandeshS end, SandeshClientSM Impl, &SandeshClientS MImpl:: ReleaseSandesh> , boost:: statechart: :in_state_ reaction< scm::EvTcpDelet eSession, SandeshClientSM Impl, &SandeshClientS MImpl:: DeleteTcpSessio n> > > ( <optimized out>, evt=..., this=<optimized out>) at /usr/include/ boost/statechar t/simple_ state.hpp: 851 impl<boost: :mpl::list< boost:: statechart: :transition< scm::EvStop, scm::Idle, SandeshClientSM Impl, &SandeshClientS MImpl:: OnIdle> , boost:: statechart: :custom_ reaction< scm::EvConnectT imerExpired> , boost:: statechart: :custom_ reaction< scm::EvTcpConne cted>, boost:: statechart: :custom_ reaction< scm::EvTcpConne ctFail> , boost:: statechart: :custom_ reaction< scm::EvTcpClose >, boost:: statechart: :custom_ reaction< scm::EvDiscUpda te>, boost:: statechart: :in_state_ reaction< scm::EvSandeshS end, SandeshClientSM Impl, &SandeshClientS MImpl:: ReleaseSandesh> , boost:: statechart: :in_state_ reaction< scm::EvTcpDelet eSession, SandeshClientSM Impl, &SandeshClientS MImpl:: DeleteTcpSessio n> >, boost:: statechart: :simple_ state<scm: :Connect, SandeshClientSM Impl, boost:: mpl::list< mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na>, (boost: :statechart: :history_ mode)0> > ( 0x153eb80 <boost: :statechart: :detail: :id_holder< scm::EvConnectT imerExpired> ::idProvider_ >, evt=..., stt=...) boost/statechar t/simple_ state.hpp: 820 boost:: mpl::list< boost:: statechart: :transition< scm::EvStop, scm::Idle, SandeshClientSM Impl, &SandeshClientS MImpl:: OnIdle> , boost:: statechart: :custom_ reaction< scm::EvConnectT imerExpired> , boost:: statechart: :custom_ reaction< scm::EvTcpConne cted>, boost:: statechart: :custom_ reaction< scm::EvTcpConne ctFail> , boost:: statechart: :custom_ reaction< scm::EvTcpClose >, boost:: statechart: :custom_ reaction< scm::EvDiscUpda te>, boost:: statechart: :in_state_ reaction< scm::EvSandeshS end, SandeshClientSM Impl, &SandeshClientS MImpl:: ReleaseSandesh> , boost:: statechart: :in_state_ reaction< scm::EvTcpDelet eSession, SandeshClientSM Impl, &SandeshClientS MImpl:: DeleteTcpSessio n> > > ( 0x153eb80 <boost: :statechart: :detail: :id_holder< scm::EvConnectT imerExpired> ::idProvider_ >, evt=..., this=0x7f5d1c64 65c0) boost/statechar t/simple_ state.hpp: 851 statechart: :simple_ state<scm: :Connect, SandeshClientSM Impl, boost:: mpl::list< mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na>, (boost: :statechart: :history_ mode)0> ::react_ impl (this=0x7f5d1c6 465c0, evt=..., 0x153eb80 <boost: :statechart: :detail: :id_holder< scm::EvConnectT imerExpired> ::idProvider_ >) boost/statechar t/simple_ state.hpp: 489 boost/statechar t/state_ machine. hpp:87 )<boost: :statechart: :detail: :send_function< boost:: statechart: :detail: :state_ base<std: :allocator< void>, boost:: statechart: :detail: :rtti_policy> , boost:: statechart: :event_ base, void const*>, boost:: statechart: :state_ machine< SandeshClientSM Impl, scm::Idle> ::exception_ event_handler> (this=0x7f5d400 878b0, action=...) at /usr/include/ boost/statechar t/null_ exception_ translator. hpp:33 statechart: :state_ machine< SandeshClientSM Impl, scm::Idle, std::allocator< void>, boost:: statechart: :null_exception _translator> ::send_ event (this=0x7f5d400 87858, evt=...) at /usr/include/ boost/statechar t/state_ machine. hpp:889 7858) at /usr/include/ boost/statechar t/state_ machine. hpp:275 Impl::DequeueEv ent (this=0x7f5d400 877f0, ec=...) at tools/sandesh/ library/ cpp/sandesh_ client_ sm.cc:871 da28) at /usr/include/ boost/bind/ mem_fn_ template. hpp:165 _mfi::mf1< bool, SandeshClientSM Impl, SandeshClientSM Impl::EventCont ainer>, boost:: _bi::list1< SandeshClientSM Impl::EventCont ainer&> > (a=<synthetic pointer>, f=..., this=0x7f5cffbf da38) at /usr/include/ boost/bind/ bind.hpp: 303 )<SandeshClient SMImpl: :EventContainer > (a1=..., this=0x7f5cffbf da28) at /usr/include/ boost/bind/ bind_template. hpp:32 detail: :function: :function_ obj_invoker1< boost:: _bi::bind_ t<bool, boost:: _mfi::mf1< bool, SandeshClientSM Impl, SandeshClientSM Impl::EventCont ainer>, boost:: _bi::list2< boost:: _bi::value< SandeshClientSM Impl*>, boost::arg<1> > >, bool, SandeshClientSM Impl::EventCont ainer>: :invoke (function_ obj_ptr= ..., a0=...) at /usr/include/ boost/function/ function_ template. hpp:132 0x7f5cffbfda20) at /usr/include/ boost/function/ function_ template. hpp:767 <SandeshClientS MImpl:: EventContainer, WorkQueue< SandeshClientSM Impl::EventCont ainer> >::RunQueue (this=0x7f5cc85 39650) src/base/ queue_task. h:81 27c40) at controller/ src/base/ task.cc: 238 libtbb. so.2 libtbb. so.2 libtbb. so.2 libtbb. so.2 libtbb. so.2 e700) at pthread_ create. c:312 unix/sysv/ linux/x86_ 64/clone. S:111
Using host libthread_db library "/lib/x86_
Core was generated by `/usr/bin/
Program terminated with signal SIGSEGV, Segmentation fault.
#0 endpoint (other=..., this=0x7f5cffbf
48 /usr/include/
(gdb) bt
#0 endpoint (other=..., this=0x7f5cffbf
#1 basic_endpoint (other=..., this=0x7f5cffbf
#2 remote_endpoint (this=0x0) at controller/
#3 scm::Connect:
event_
#4 0x0000000000f50438 in scm::Connect::react (this=this@
at tools/sandesh/
#5 0x0000000000f516b8 in react<scm::Connect, boost::
at /usr/include/
#6 local_react_
#7 local_react<
eventType=
#8 local_react_
eventType=
at /usr/include/
#9 local_react<
eventType=
at /usr/include/
#10 boost::
---Type <return> to continue, or q <return> to quit---
eventType=
at /usr/include/
#11 0x0000000000f4c69b in operator() (this=<synthetic pointer>) at /usr/include/
#12 operator(
#13 boost::
#14 0x0000000000f43ade in process_event (evt=..., this=0x7f5d4008
#15 SandeshClientSM
#16 0x0000000000f481e2 in operator() (a1=..., p=0x7f5d400877f0, this=0x7f5cffbf
#17 operator()<bool, boost::
#18 operator(
#19 boost::
#20 0x0000000000f4b887 in operator() (a0=<error reading variable: access outside bounds of object referenced via synthetic pointer>,
this=
#21 QueueTaskRunner
at controller/
#22 0x0000000000fdd8b0 in TaskImpl::execute (this=0x7f5d475
#23 0x00007f5d4e9a0b3a in ?? () from /usr/lib/
#24 0x00007f5d4e99c816 in ?? () from /usr/lib/
#25 0x00007f5d4e99bf4b in ?? () from /usr/lib/
#26 0x00007f5d4e9980ff in ?? () from /usr/lib/
#27 0x00007f5d4e9982f9 in ?? () from /usr/lib/
#28 0x00007f5d4ebbc182 in start_thread (arg=0x7f5cffbf
#29 0x00007f5d4de9547d in clone () at ../sysdeps/