control-node crash during Sandesh message send (in TCP)

[Ananth Suryanarayana]

/github-build/R2.0/41/centos65/icehouse/store/sandbox/build/debug/control-node/contrail-control
Core in /cs-shared/bugs/<this-id>/.

Do not know any thing else that happened during this crash

(gdb) bt
#0 0x00002b08fd3433a0 in pthread_mutex_lock () from /lib64/libpthread.so.0
#1 0x0000000000437f8e in tbb::mutex::lock (this=0x20) at build/include/tbb/mutex.h:164
#2 0x0000000000437f40 in tbb::mutex::scoped_lock::acquire (this=0x7ffff9a07a10, mutex=...) at build/include/tbb/mutex.h:105
#3 0x0000000000437efb in tbb::mutex::scoped_lock::scoped_lock (this=0x7ffff9a07a10, mutex=...) at build/include/tbb/mutex.h:91
#4 0x00000000007cb7d8 in TcpSession::IsClosed (this=0x0) at controller/src/io/tcp_session.h:113
#5 0x0000000000b3f2c7 in SandeshClient::SendSandesh (this=0x2dcc140, snh=0x2dcf830) at tools/sandesh/library/cpp/sandesh_client.cc:117
#6 0x0000000000b283b5 in Sandesh::SendEnqueue (this=0x2dcf830) at tools/sandesh/library/cpp/sandesh.cc:565
#7 0x0000000000b28546 in Sandesh::Dispatch (this=0x2dcf830, sconn=0x0) at tools/sandesh/library/cpp/sandesh.cc:585
#8 0x0000000000a5b2a4 in IFMapServerConnection::Send (category="IFMapPeer", level=SandeshLevel::SYS_DEBUG, file=
    "controller/src/ifmap/client/ifmap_channel.cc", line=204, str1="Retrying connection to Ifmap-server.", str2="")
    at build/debug/ifmap/ifmap_log_types.h:1042
#9 0x0000000000a4a24a in IFMapChannel::ReconnectPreparationInMainThr (this=0x2dcdce0) at controller/src/ifmap/client/ifmap_channel.cc:203
#10 0x0000000000a6ffd9 in boost::_mfi::mf0<void, IFMapChannel>::operator() (this=0x7ffff9a07fb0, p=0x2dcdce0)
    at build/include/boost/bind/mem_fn_template.hpp:49
#11 0x0000000000a6e868 in boost::_bi::list1<boost::_bi::value<IFMapChannel*> >::operator()<boost::_mfi::mf0<void, IFMapChannel>, boost::_bi::list0> (this=0x7ffff9a07fc0, f=..., a=...) at build/include/boost/bind/bind.hpp:253
#12 0x0000000000a6cf2b in boost::_bi::bind_t<void, boost::_mfi::mf0<void, IFMapChannel>, boost::_bi::list1<boost::_bi::value<IFMapChannel*> > >::operator() (this=0x7ffff9a07fb0) at build/include/boost/bind/bind_template.hpp:20
#13 0x0000000000a6b78f in boost::asio::asio_handler_invoke<boost::_bi::bind_t<void, boost::_mfi::mf0<void, IFMapChannel>, boost::_bi::list1<boost::_bi::value<IFMapChannel*> > > > (function=...) at build/include/boost/asio/handler_invoke_hook.hpp:64
#14 0x0000000000a684d2 in boost_asio_handler_invoke_helpers::invoke<boost::_bi::bind_t<void, boost::_mfi::mf0<void, IFMapChannel>, boost::_bi::list1<boost::_bi::value<IFMapChannel*> > >, boost::_bi::bind_t<void, boost::_mfi::mf0<void, IFMapChannel>, boost::_bi::list1<boost::_bi::value<IFMapChannel*> > > > (function=..., context=...) at build/include/boost/asio/detail/handler_invoke_helpers.hpp:39
#15 0x0000000000a65c9a in boost::asio::detail::completion_handler<boost::_bi::bind_t<void, boost::_mfi::mf0<void, IFMapChannel>, boost::_bi::list1<boost::_bi::value<IFMapChannel*> > > >::do_complete (owner=0x2d6fc70, base=0x2b0904057440)
    at build/include/boost/asio/detail/completion_handler.hpp:67
#16 0x000000000058d16c in boost::asio::detail::task_io_service_operation::complete (this=0x2b0904057440, owner=..., ec=..., bytes_transferred=0)
    at build/include/boost/asio/detail/task_io_service_operation.hpp:37
#17 0x0000000000a5797c in boost::asio::detail::strand_service::do_complete (owner=0x2d6fc70, base=0x2dd6560, ec=...)
    at build/include/boost/asio/detail/impl/strand_service.ipp:160
#18 0x000000000058d16c in boost::asio::detail::task_io_service_operation::complete (this=0x2dd6560, owner=..., ec=..., bytes_transferred=0)
    at build/include/boost/asio/detail/task_io_service_operation.hpp:37
#19 0x000000000058e05d in boost::asio::detail::task_io_service::do_run_one (this=0x2d6fc70, lock=..., this_thread=..., private_op_queue=...,
    ec=...) at build/include/boost/asio/detail/impl/task_io_service.ipp:366
#20 0x000000000058dc88 in boost::asio::detail::task_io_service::run (this=0x2d6fc70, ec=...)
    at build/include/boost/asio/detail/impl/task_io_service.ipp:146
#21 0x000000000058e3d1 in boost::asio::io_service::run (this=0x1373550, ec=...) at build/include/boost/asio/impl/io_service.ipp:68
#22 0x0000000000c61985 in EventManager::Run (this=0x1373550) at controller/src/io/event_manager.cc:32
#23 0x0000000000577592 in main (argc=1, argv=0x7ffff9a09588) at controller/src/control-node/main.cc:584
(gdb)

[Ananth Suryanarayana (anantha-l)]

contrail-dns had also crashed, with the same backtrace

(gdb) bt
#0 0x00002af1ed06e3a0 in pthread_mutex_lock () from /lib64/libpthread.so.0
#1 0x0000000000ac7f7e in tbb::mutex::lock (this=0x20) at build/include/tbb/mutex.h:164
#2 0x0000000000ac7f30 in tbb::mutex::scoped_lock::acquire (this=0x7fff5ef5ea80, mutex=...) at build/include/tbb/mutex.h:105
#3 0x0000000000ac7eeb in tbb::mutex::scoped_lock::scoped_lock (this=0x7fff5ef5ea80, mutex=...) at build/include/tbb/mutex.h:91
#4 0x0000000000e25c3a in TcpSession::IsClosed (this=0x0) at controller/src/io/tcp_session.h:113
#5 0x0000000000e23edb in SandeshClient::SendSandesh (this=0x2140590, snh=0x21a26e0) at tools/sandesh/library/cpp/sandesh_client.cc:117
#6 0x0000000000e0ce21 in Sandesh::SendEnqueue (this=0x21a26e0) at tools/sandesh/library/cpp/sandesh.cc:565
#7 0x0000000000e0cfb2 in Sandesh::Dispatch (this=0x21a26e0, sconn=0x0) at tools/sandesh/library/cpp/sandesh.cc:585
#8 0x0000000000d18d80 in IFMapServerConnection::Send (category="IFMapPeer", level=SandeshLevel::SYS_DEBUG, file=
    "controller/src/ifmap/client/ifmap_channel.cc", line=204, str1="Retrying connection to Ifmap-server.", str2="")
    at build/debug/ifmap/ifmap_log_types.h:1042
#9 0x0000000000d08622 in IFMapChannel::ReconnectPreparationInMainThr (this=0x2141f80) at controller/src/ifmap/client/ifmap_channel.cc:203
#10 0x0000000000d2f5a5 in boost::_mfi::mf0<void, IFMapChannel>::operator() (this=0x7fff5ef5f020, p=0x2141f80)
    at build/include/boost/bind/mem_fn_template.hpp:49
#11 0x0000000000d2dea2 in boost::_bi::list1<boost::_bi::value<IFMapChannel*> >::operator()<boost::_mfi::mf0<void, IFMapChannel>, boost::_bi::list0> (this=0x7fff5ef5f030, f=..., a=...) at build/include/boost/bind/bind.hpp:253
#12 0x0000000000d2c401 in boost::_bi::bind_t<void, boost::_mfi::mf0<void, IFMapChannel>, boost::_bi::list1<boost::_bi::value<IFMapChannel*> > >::operator() (this=0x7fff5ef5f020) at build/include/boost/bind/bind_template.hpp:20
#13 0x0000000000d2a9ed in boost::asio::asio_handler_invoke<boost::_bi::bind_t<void, boost::_mfi::mf0<void, IFMapChannel>, boost::_bi::list1<boost::_bi::value<IFMapChannel*> > > > (function=...) at build/include/boost/asio/handler_invoke_hook.hpp:64
#14 0x0000000000d27542 in boost_asio_handler_invoke_helpers::invoke<boost::_bi::bind_t<void, boost::_mfi::mf0<void, IFMapChannel>, boost::_bi::list1<boost::_bi::value<IFMapChannel*> > >, boost::_bi::bind_t<void, boost::_mfi::mf0<void, IFMapChannel>, boost::_bi::list1<boost::_bi::value<IFMapChannel*> > > > (function=..., context=...) at build/include/boost/asio/detail/handler_invoke_helpers.hpp:39
#15 0x0000000000d24c42 in boost::asio::detail::completion_handler<boost::_bi::bind_t<void, boost::_mfi::mf0<void, IFMapChannel>, boost::_bi::list1<boost::_bi::value<IFMapChannel*> > > >::do_complete (owner=0x20f9270, base=0x2af200025ba0)
    at build/include/boost/asio/detail/completion_handler.hpp:67
#16 0x0000000000c11f6c in boost::asio::detail::task_io_service_operation::complete (this=0x2af200025ba0, owner=..., ec=..., bytes_transferred=0)
    at build/include/boost/asio/detail/task_io_service_operation.hpp:37
#17 0x0000000000d15460 in boost::asio::detail::strand_serv...

[Megh Bhatt (meghb)]

Fixed in latest