| 2016-01-06 17:20:51 |
amit surana |
description |
2.21.1 #15
Under traffic load (high throughput/CPS), agent crash with this bt is seen:
[Thread debugging using libthread_db enabled]
Using host libthread_db library "/lib/x86_64-linux-gnu/libthread_db.so.1".
Core was generated by `/usr/bin/contrail-vrouter-agent'.
Program terminated with signal SIGSEGV, Segmentation fault.
#0 endpoint (other=..., this=0x7f5cffbfd340) at /usr/include/boost/asio/ip/detail/endpoint.hpp:48
48 /usr/include/boost/asio/ip/detail/endpoint.hpp: No such file or directory.
(gdb) bt
#0 endpoint (other=..., this=0x7f5cffbfd340) at /usr/include/boost/asio/ip/detail/endpoint.hpp:48
#1 basic_endpoint (other=..., this=0x7f5cffbfd340) at /usr/include/boost/asio/ip/basic_endpoint.hpp:97
#2 remote_endpoint (this=0x0) at controller/src/io/tcp_session.h:129
#3 scm::Connect::ToIdle (this=this@entry=0x7f5d1c6465c0, state_machine=state_machine@entry=0x7f5d400877f0,
event_name=event_name@entry=0x10875ee "EvConnectTimerExpired") at tools/sandesh/library/cpp/sandesh_client_sm.cc:422
#4 0x0000000000f50438 in scm::Connect::react (this=this@entry=0x7f5d1c6465c0, event=...)
at tools/sandesh/library/cpp/sandesh_client_sm.cc:370
#5 0x0000000000f516b8 in react<scm::Connect, boost::statechart::event_base, void const*> (eventType=<synthetic pointer>, evt=..., stt=...)
at /usr/include/boost/statechart/custom_reaction.hpp:42
#6 local_react_impl<boost::mpl::list7<boost::statechart::custom_reaction<scm::EvConnectTimerExpired>, boost::statechart::custom_reaction<scm::EvTcpConnected>, boost::statechart::custom_reaction<scm::EvTcpConnectFail>, boost::statechart::custom_reaction<scm::EvTcpClose>, boost::statechart::custom_reaction<scm::EvDiscUpdate>, boost::statechart::in_state_reaction<scm::EvSandeshSend, SandeshClientSMImpl, &SandeshClientSMImpl::ReleaseSandesh>, boost::statechart::in_state_reaction<scm::EvTcpDeleteSession, SandeshClientSMImpl, &SandeshClientSMImpl::DeleteTcpSession> >, boost::statechart::simple_state<scm::Connect, SandeshClientSMImpl, boost::mpl::list<mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na>, (boost::statechart::history_mode)0> > (eventType=<optimized out>, evt=..., stt=...) at /usr/include/boost/statechart/simple_state.hpp:816
#7 local_react<boost::mpl::list7<boost::statechart::custom_reaction<scm::EvConnectTimerExpired>, boost::statechart::custom_reaction<scm::EvTcpConnected>, boost::statechart::custom_reaction<scm::EvTcpConnectFail>, boost::statechart::custom_reaction<scm::EvTcpClose>, boost::statechart::custom_reaction<scm::EvDiscUpdate>, boost::statechart::in_state_reaction<scm::EvSandeshSend, SandeshClientSMImpl, &SandeshClientSMImpl::ReleaseSandesh>, boost::statechart::in_state_reaction<scm::EvTcpDeleteSession, SandeshClientSMImpl, &SandeshClientSMImpl::DeleteTcpSession> > > (
eventType=<optimized out>, evt=..., this=<optimized out>) at /usr/include/boost/statechart/simple_state.hpp:851
#8 local_react_impl<boost::mpl::list<boost::statechart::transition<scm::EvStop, scm::Idle, SandeshClientSMImpl, &SandeshClientSMImpl::OnIdle>, boost::statechart::custom_reaction<scm::EvConnectTimerExpired>, boost::statechart::custom_reaction<scm::EvTcpConnected>, boost::statechart::custom_reaction<scm::EvTcpConnectFail>, boost::statechart::custom_reaction<scm::EvTcpClose>, boost::statechart::custom_reaction<scm::EvDiscUpdate>, boost::statechart::in_state_reaction<scm::EvSandeshSend, SandeshClientSMImpl, &SandeshClientSMImpl::ReleaseSandesh>, boost::statechart::in_state_reaction<scm::EvTcpDeleteSession, SandeshClientSMImpl, &SandeshClientSMImpl::DeleteTcpSession> >, boost::statechart::simple_state<scm::Connect, SandeshClientSMImpl, boost::mpl::list<mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na>, (boost::statechart::history_mode)0> > (
eventType=0x153eb80 <boost::statechart::detail::id_holder<scm::EvConnectTimerExpired>::idProvider_>, evt=..., stt=...)
at /usr/include/boost/statechart/simple_state.hpp:820
#9 local_react<boost::mpl::list<boost::statechart::transition<scm::EvStop, scm::Idle, SandeshClientSMImpl, &SandeshClientSMImpl::OnIdle>, boost::statechart::custom_reaction<scm::EvConnectTimerExpired>, boost::statechart::custom_reaction<scm::EvTcpConnected>, boost::statechart::custom_reaction<scm::EvTcpConnectFail>, boost::statechart::custom_reaction<scm::EvTcpClose>, boost::statechart::custom_reaction<scm::EvDiscUpdate>, boost::statechart::in_state_reaction<scm::EvSandeshSend, SandeshClientSMImpl, &SandeshClientSMImpl::ReleaseSandesh>, boost::statechart::in_state_reaction<scm::EvTcpDeleteSession, SandeshClientSMImpl, &SandeshClientSMImpl::DeleteTcpSession> > > (
eventType=0x153eb80 <boost::statechart::detail::id_holder<scm::EvConnectTimerExpired>::idProvider_>, evt=..., this=0x7f5d1c6465c0)
at /usr/include/boost/statechart/simple_state.hpp:851
#10 boost::statechart::simple_state<scm::Connect, SandeshClientSMImpl, boost::mpl::list<mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na>, (boost::statechart::history_mode)0>::react_impl (this=0x7f5d1c6465c0, evt=...,
---Type <return> to continue, or q <return> to quit---
eventType=0x153eb80 <boost::statechart::detail::id_holder<scm::EvConnectTimerExpired>::idProvider_>)
at /usr/include/boost/statechart/simple_state.hpp:489
#11 0x0000000000f4c69b in operator() (this=<synthetic pointer>) at /usr/include/boost/statechart/state_machine.hpp:87
#12 operator()<boost::statechart::detail::send_function<boost::statechart::detail::state_base<std::allocator<void>, boost::statechart::detail::rtti_policy>, boost::statechart::event_base, void const*>, boost::statechart::state_machine<SandeshClientSMImpl, scm::Idle>::exception_event_handler> (this=0x7f5d400878b0, action=...) at /usr/include/boost/statechart/null_exception_translator.hpp:33
#13 boost::statechart::state_machine<SandeshClientSMImpl, scm::Idle, std::allocator<void>, boost::statechart::null_exception_translator>::send_event (this=0x7f5d40087858, evt=...) at /usr/include/boost/statechart/state_machine.hpp:889
#14 0x0000000000f43ade in process_event (evt=..., this=0x7f5d40087858) at /usr/include/boost/statechart/state_machine.hpp:275
#15 SandeshClientSMImpl::DequeueEvent (this=0x7f5d400877f0, ec=...) at tools/sandesh/library/cpp/sandesh_client_sm.cc:871
#16 0x0000000000f481e2 in operator() (a1=..., p=0x7f5d400877f0, this=0x7f5cffbfda28) at /usr/include/boost/bind/mem_fn_template.hpp:165
#17 operator()<bool, boost::_mfi::mf1<bool, SandeshClientSMImpl, SandeshClientSMImpl::EventContainer>, boost::_bi::list1<SandeshClientSMImpl::EventContainer&> > (a=<synthetic pointer>, f=..., this=0x7f5cffbfda38) at /usr/include/boost/bind/bind.hpp:303
#18 operator()<SandeshClientSMImpl::EventContainer> (a1=..., this=0x7f5cffbfda28) at /usr/include/boost/bind/bind_template.hpp:32
#19 boost::detail::function::function_obj_invoker1<boost::_bi::bind_t<bool, boost::_mfi::mf1<bool, SandeshClientSMImpl, SandeshClientSMImpl::EventContainer>, boost::_bi::list2<boost::_bi::value<SandeshClientSMImpl*>, boost::arg<1> > >, bool, SandeshClientSMImpl::EventContainer>::invoke (function_obj_ptr=..., a0=...) at /usr/include/boost/function/function_template.hpp:132
#20 0x0000000000f4b887 in operator() (a0=<error reading variable: access outside bounds of object referenced via synthetic pointer>,
this=0x7f5cffbfda20) at /usr/include/boost/function/function_template.hpp:767
#21 QueueTaskRunner<SandeshClientSMImpl::EventContainer, WorkQueue<SandeshClientSMImpl::EventContainer> >::RunQueue (this=0x7f5cc8539650)
at controller/src/base/queue_task.h:81
#22 0x0000000000fdd8b0 in TaskImpl::execute (this=0x7f5d47527c40) at controller/src/base/task.cc:238
#23 0x00007f5d4e9a0b3a in ?? () from /usr/lib/libtbb.so.2
#24 0x00007f5d4e99c816 in ?? () from /usr/lib/libtbb.so.2
#25 0x00007f5d4e99bf4b in ?? () from /usr/lib/libtbb.so.2
#26 0x00007f5d4e9980ff in ?? () from /usr/lib/libtbb.so.2
#27 0x00007f5d4e9982f9 in ?? () from /usr/lib/libtbb.so.2
#28 0x00007f5d4ebbc182 in start_thread (arg=0x7f5cffbfe700) at pthread_create.c:312
#29 0x00007f5d4de9547d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111 |
2.21.1 #15
core file location: 10.84.5.112:/cs-shared/bugs/1531550/core*.1531550
Under traffic load (high throughput/CPS), agent crash with this bt is seen:
[Thread debugging using libthread_db enabled]
Using host libthread_db library "/lib/x86_64-linux-gnu/libthread_db.so.1".
Core was generated by `/usr/bin/contrail-vrouter-agent'.
Program terminated with signal SIGSEGV, Segmentation fault.
#0 endpoint (other=..., this=0x7f5cffbfd340) at /usr/include/boost/asio/ip/detail/endpoint.hpp:48
48 /usr/include/boost/asio/ip/detail/endpoint.hpp: No such file or directory.
(gdb) bt
#0 endpoint (other=..., this=0x7f5cffbfd340) at /usr/include/boost/asio/ip/detail/endpoint.hpp:48
#1 basic_endpoint (other=..., this=0x7f5cffbfd340) at /usr/include/boost/asio/ip/basic_endpoint.hpp:97
#2 remote_endpoint (this=0x0) at controller/src/io/tcp_session.h:129
#3 scm::Connect::ToIdle (this=this@entry=0x7f5d1c6465c0, state_machine=state_machine@entry=0x7f5d400877f0,
event_name=event_name@entry=0x10875ee "EvConnectTimerExpired") at tools/sandesh/library/cpp/sandesh_client_sm.cc:422
#4 0x0000000000f50438 in scm::Connect::react (this=this@entry=0x7f5d1c6465c0, event=...)
at tools/sandesh/library/cpp/sandesh_client_sm.cc:370
#5 0x0000000000f516b8 in react<scm::Connect, boost::statechart::event_base, void const*> (eventType=<synthetic pointer>, evt=..., stt=...)
at /usr/include/boost/statechart/custom_reaction.hpp:42
#6 local_react_impl<boost::mpl::list7<boost::statechart::custom_reaction<scm::EvConnectTimerExpired>, boost::statechart::custom_reaction<scm::EvTcpConnected>, boost::statechart::custom_reaction<scm::EvTcpConnectFail>, boost::statechart::custom_reaction<scm::EvTcpClose>, boost::statechart::custom_reaction<scm::EvDiscUpdate>, boost::statechart::in_state_reaction<scm::EvSandeshSend, SandeshClientSMImpl, &SandeshClientSMImpl::ReleaseSandesh>, boost::statechart::in_state_reaction<scm::EvTcpDeleteSession, SandeshClientSMImpl, &SandeshClientSMImpl::DeleteTcpSession> >, boost::statechart::simple_state<scm::Connect, SandeshClientSMImpl, boost::mpl::list<mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na>, (boost::statechart::history_mode)0> > (eventType=<optimized out>, evt=..., stt=...) at /usr/include/boost/statechart/simple_state.hpp:816
#7 local_react<boost::mpl::list7<boost::statechart::custom_reaction<scm::EvConnectTimerExpired>, boost::statechart::custom_reaction<scm::EvTcpConnected>, boost::statechart::custom_reaction<scm::EvTcpConnectFail>, boost::statechart::custom_reaction<scm::EvTcpClose>, boost::statechart::custom_reaction<scm::EvDiscUpdate>, boost::statechart::in_state_reaction<scm::EvSandeshSend, SandeshClientSMImpl, &SandeshClientSMImpl::ReleaseSandesh>, boost::statechart::in_state_reaction<scm::EvTcpDeleteSession, SandeshClientSMImpl, &SandeshClientSMImpl::DeleteTcpSession> > > (
eventType=<optimized out>, evt=..., this=<optimized out>) at /usr/include/boost/statechart/simple_state.hpp:851
#8 local_react_impl<boost::mpl::list<boost::statechart::transition<scm::EvStop, scm::Idle, SandeshClientSMImpl, &SandeshClientSMImpl::OnIdle>, boost::statechart::custom_reaction<scm::EvConnectTimerExpired>, boost::statechart::custom_reaction<scm::EvTcpConnected>, boost::statechart::custom_reaction<scm::EvTcpConnectFail>, boost::statechart::custom_reaction<scm::EvTcpClose>, boost::statechart::custom_reaction<scm::EvDiscUpdate>, boost::statechart::in_state_reaction<scm::EvSandeshSend, SandeshClientSMImpl, &SandeshClientSMImpl::ReleaseSandesh>, boost::statechart::in_state_reaction<scm::EvTcpDeleteSession, SandeshClientSMImpl, &SandeshClientSMImpl::DeleteTcpSession> >, boost::statechart::simple_state<scm::Connect, SandeshClientSMImpl, boost::mpl::list<mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na>, (boost::statechart::history_mode)0> > (
eventType=0x153eb80 <boost::statechart::detail::id_holder<scm::EvConnectTimerExpired>::idProvider_>, evt=..., stt=...)
at /usr/include/boost/statechart/simple_state.hpp:820
#9 local_react<boost::mpl::list<boost::statechart::transition<scm::EvStop, scm::Idle, SandeshClientSMImpl, &SandeshClientSMImpl::OnIdle>, boost::statechart::custom_reaction<scm::EvConnectTimerExpired>, boost::statechart::custom_reaction<scm::EvTcpConnected>, boost::statechart::custom_reaction<scm::EvTcpConnectFail>, boost::statechart::custom_reaction<scm::EvTcpClose>, boost::statechart::custom_reaction<scm::EvDiscUpdate>, boost::statechart::in_state_reaction<scm::EvSandeshSend, SandeshClientSMImpl, &SandeshClientSMImpl::ReleaseSandesh>, boost::statechart::in_state_reaction<scm::EvTcpDeleteSession, SandeshClientSMImpl, &SandeshClientSMImpl::DeleteTcpSession> > > (
eventType=0x153eb80 <boost::statechart::detail::id_holder<scm::EvConnectTimerExpired>::idProvider_>, evt=..., this=0x7f5d1c6465c0)
at /usr/include/boost/statechart/simple_state.hpp:851
#10 boost::statechart::simple_state<scm::Connect, SandeshClientSMImpl, boost::mpl::list<mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na>, (boost::statechart::history_mode)0>::react_impl (this=0x7f5d1c6465c0, evt=...,
---Type <return> to continue, or q <return> to quit---
eventType=0x153eb80 <boost::statechart::detail::id_holder<scm::EvConnectTimerExpired>::idProvider_>)
at /usr/include/boost/statechart/simple_state.hpp:489
#11 0x0000000000f4c69b in operator() (this=<synthetic pointer>) at /usr/include/boost/statechart/state_machine.hpp:87
#12 operator()<boost::statechart::detail::send_function<boost::statechart::detail::state_base<std::allocator<void>, boost::statechart::detail::rtti_policy>, boost::statechart::event_base, void const*>, boost::statechart::state_machine<SandeshClientSMImpl, scm::Idle>::exception_event_handler> (this=0x7f5d400878b0, action=...) at /usr/include/boost/statechart/null_exception_translator.hpp:33
#13 boost::statechart::state_machine<SandeshClientSMImpl, scm::Idle, std::allocator<void>, boost::statechart::null_exception_translator>::send_event (this=0x7f5d40087858, evt=...) at /usr/include/boost/statechart/state_machine.hpp:889
#14 0x0000000000f43ade in process_event (evt=..., this=0x7f5d40087858) at /usr/include/boost/statechart/state_machine.hpp:275
#15 SandeshClientSMImpl::DequeueEvent (this=0x7f5d400877f0, ec=...) at tools/sandesh/library/cpp/sandesh_client_sm.cc:871
#16 0x0000000000f481e2 in operator() (a1=..., p=0x7f5d400877f0, this=0x7f5cffbfda28) at /usr/include/boost/bind/mem_fn_template.hpp:165
#17 operator()<bool, boost::_mfi::mf1<bool, SandeshClientSMImpl, SandeshClientSMImpl::EventContainer>, boost::_bi::list1<SandeshClientSMImpl::EventContainer&> > (a=<synthetic pointer>, f=..., this=0x7f5cffbfda38) at /usr/include/boost/bind/bind.hpp:303
#18 operator()<SandeshClientSMImpl::EventContainer> (a1=..., this=0x7f5cffbfda28) at /usr/include/boost/bind/bind_template.hpp:32
#19 boost::detail::function::function_obj_invoker1<boost::_bi::bind_t<bool, boost::_mfi::mf1<bool, SandeshClientSMImpl, SandeshClientSMImpl::EventContainer>, boost::_bi::list2<boost::_bi::value<SandeshClientSMImpl*>, boost::arg<1> > >, bool, SandeshClientSMImpl::EventContainer>::invoke (function_obj_ptr=..., a0=...) at /usr/include/boost/function/function_template.hpp:132
#20 0x0000000000f4b887 in operator() (a0=<error reading variable: access outside bounds of object referenced via synthetic pointer>,
this=0x7f5cffbfda20) at /usr/include/boost/function/function_template.hpp:767
#21 QueueTaskRunner<SandeshClientSMImpl::EventContainer, WorkQueue<SandeshClientSMImpl::EventContainer> >::RunQueue (this=0x7f5cc8539650)
at controller/src/base/queue_task.h:81
#22 0x0000000000fdd8b0 in TaskImpl::execute (this=0x7f5d47527c40) at controller/src/base/task.cc:238
#23 0x00007f5d4e9a0b3a in ?? () from /usr/lib/libtbb.so.2
#24 0x00007f5d4e99c816 in ?? () from /usr/lib/libtbb.so.2
#25 0x00007f5d4e99bf4b in ?? () from /usr/lib/libtbb.so.2
#26 0x00007f5d4e9980ff in ?? () from /usr/lib/libtbb.so.2
#27 0x00007f5d4e9982f9 in ?? () from /usr/lib/libtbb.so.2
#28 0x00007f5d4ebbc182 in start_thread (arg=0x7f5cffbfe700) at pthread_create.c:312
#29 0x00007f5d4de9547d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111 |
|
