contrail-tor-agent is struck in tbb::mutex::lock called from TcpSession::IsClosed. On looking at the lock, it seems that there is an attempt to take lock on a free'd TcpSession. (gdb) bt #0 __lll_lock_wait () at ../nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S:135 #1 0x00007f76c3b28657 in _L_lock_909 () from /lib/x86_64-linux-gnu/libpthread.so.0 #2 0x00007f76c3b28480 in __GI___pthread_mutex_lock (mutex=0x7f76980e6b98) at ../nptl/pthread_mutex_lock.c:79 #3 0x000000000100c7ec in tbb::mutex::lock (this=0x7f76980e6b98) at /usr/include/tbb/mutex.h:164 #4 0x000000000100c79e in tbb::mutex::scoped_lock::acquire (this=0x7fff60b7fc40, mutex=...) at /usr/include/tbb/mutex.h:105 #5 0x000000000100c759 in tbb::mutex::scoped_lock::scoped_lock (this=0x7fff60b7fc40, mutex=...) at /usr/include/tbb/mutex.h:91 #6 0x00000000012fce92 in TcpSession::IsClosed (this=0x7f76980e6b90) at controller/src/io/tcp_session.h:119 #7 0x0000000001859d81 in SandeshClient::SendSandesh (this=0x7f76b4070f70, snh=0x3dcdc40) at tools/sandesh/library/cpp/sandesh_client.cc:117 #8 0x000000000183f7e3 in Sandesh::SendEnqueue (this=0x3dcdc40) at tools/sandesh/library/cpp/sandesh.cc:591 #9 0x000000000183fa36 in Sandesh::Dispatch (this=0x3dcdc40, sconn=0x0) at tools/sandesh/library/cpp/sandesh.cc:611 #10 0x0000000001616f2e in XmppEventLog::Send (category=..., level=SandeshLevel::SYS_NOTICE, file=..., line=1314, f1=..., f2=..., f3=..., f4=...) at build/debug/xmpp/sandesh/xmpp_state_machine_sandesh_types.h:504 #11 0x000000000161305d in XmppStateMachine::OnSessionEvent (this=0x7f765c00e5b0, session=0x7f7694088de0, event=TcpSession::CONNECT_FAILED) at controller/src/xmpp/xmpp_state_machine.cc:1311 #12 0x0000000001631886 in boost::_mfi::mf2::operator() (this=0x7f7694088e98, p=0x7f765c00e5b0, a1=0x7f7694088de0, a2=TcpSession::CONNECT_FAILED) at /usr/include/boost/bind/mem_fn_template.hpp:280 #13 0x00000000016300ca in boost::_bi::list3, boost::arg<1>, boost::arg<2> >::operator(), boost::_bi::list2 > (this=0x7f7694088ea8, f=..., a=...) at /usr/include/boost/bind/bind.hpp:392 #14 0x000000000162edd8 in boost::_bi::bind_t, boost::_bi::list3, boost::arg<1>, boost::arg<2> > >::operator() (this=0x7f7694088e98, a1=@0x7fff60b80160: 0x7f7694088de0, a2=@0x7fff60b8015c: TcpSession::CONNECT_FAILED) at /usr/include/boost/bind/bind_template.hpp:61 #15 0x000000000162def1 in boost::detail::function::void_function_obj_invoker2, boost::_bi::list3, boost::arg<1>, boost::arg<2> > >, void, TcpSession*, TcpSession::Event>::invoke (function_obj_ptr=..., a0=0x7f7694088de0, a1=TcpSession::CONNECT_FAILED) at /usr/include/boost/function/function_template.hpp:153 #16 0x000000000182e424 in boost::function2::operator() (this=0x7f7694088e90, a0=0x7f7694088de0, a1=TcpSession::CONNECT_FAILED) at /usr/include/boost/function/function_template.hpp:767 #17 0x0000000001825028 in TcpSession::ConnectFailed (this=0x7f7694088de0) at controller/src/io/tcp_session.cc:278 #18 0x00000000018181a3 in TcpServer::ConnectHandler (this=0x7f765c00d2e0, server=..., session=..., error=...) at controller/src/io/tcp_server.cc:388 #19 0x0000000001822f35 in boost::_mfi::mf3, boost::intrusive_ptr, boost::system::error_code const&>::operator() (this=0x7fff60b806c0, p=0x7f765c00d2e0, a1=..., a2=..., a3=...) at /usr/include/boost/bind/mem_fn_template.hpp:393 #20 0x0000000001822e11 in boost::_bi::list4, boost::_bi::value >, boost::_bi::value >, boost::arg<1> (*)()>::operator(), boost::intrusive_ptr, boost::system::error_code const&>, boost::_bi::list1 > (this=0x7fff60b806d0, f=..., a=...) at /usr/include/boost/bind/bind.hpp:457 #21 0x0000000001822d30 in boost::_bi::bind_t, boost::intrusive_ptr, boost::system::error_code const&>, boost::_bi::list4, boost::_bi::value >, boost::_bi::value >, boost::arg<1> (*)()> >::operator() (this=0x7fff60b806c0, a1=...) at /usr/include/boost/bind/bind_template.hpp:47 #22 0x0000000001822bc1 in boost::asio::detail::binder1, boost::intrusive_ptr, boost::system::error_code const&>, boost::_bi::list4, boost::_bi::value >, boost::_bi::value >, boost::arg<1> (*)()> >, boost::system::error_code>::operator() (this=0x7fff60b806c0) at /usr/include/boost/asio/detail/bind_handler.hpp:47 #23 0x0000000001822ad9 in boost::asio::asio_handler_invoke to continue, or q to quit--- er, boost::intrusive_ptr, boost::intrusive_ptr, boost::system::error_code const&>, boost::_bi::list4, boost::_bi::value >, boost::_bi::value >, boost::arg<1> (*)()> >, boost::system::error_code> > (function=...) at /usr/include/boost/asio/handler_invoke_hook.hpp:64 #24 0x000000000182294e in boost_asio_handler_invoke_helpers::invoke, boost::intrusive_ptr, boost::system::error_code const&>, boost::_bi::list4, boost::_bi::value >, boost::_bi::value >, boost::arg<1> (*)()> >, boost::system::error_code>, boost::_bi::bind_t, boost::intrusive_ptr, boost::system::error_code const&>, boost::_bi::list4, boost::_bi::value >, boost::_bi::value >, boost::arg<1> (*)()> > > (function=..., context=...) at /usr/include/boost/asio/detail/handler_invoke_helpers.hpp:37 #25 0x000000000182275b in boost::asio::detail::asio_handler_invoke, boost::intrusive_ptr, boost::system::error_code const&>, boost::_bi::list4, boost::_bi::value >, boost::_bi::value >, boost::arg<1> (*)()> >, boost::system::error_code>, boost::_bi::bind_t, boost::intrusive_ptr, boost::system::error_code const&>, boost::_bi::list4, boost::_bi::value >, boost::_bi::value >, boost::arg<1> (*)()> >, boost::system::error_code> (function=..., this_handler=0x7fff60b807b0) at /usr/include/boost/asio/detail/bind_handler.hpp:88 #26 0x00000000018222bc in boost_asio_handler_invoke_helpers::invoke, boost::intrusive_ptr, boost::system::error_code const&>, boost::_bi::list4, boost::_bi::value >, boost::_bi::value >, boost::arg<1> (*)()> >, boost::system::error_code>, boost::asio::detail::binder1, boost::intrusive_ptr, boost::system::error_code const&>, boost::_bi::list4, boost::_bi::value >, boost::_bi::value >, boost::arg<1> (*)()> >, boost::system::error_code> > (function=..., context=...) at /usr/include/boost/asio/detail/handler_invoke_helpers.hpp:37 #27 0x00000000018218a0 in boost::asio::detail::reactive_socket_connect_op, boost::intrusive_ptr, boost::system::error_code const&>, boost::_bi::list4, boost::_bi::value >, boost::_bi::value >, boost::arg<1> (*)()> > >::do_complete (owner=0x3da16f0, base=0x7f7694204620) at /usr/include/boost/asio/detail/reactive_socket_connect_op.hpp:100 #28 0x000000000119a8b8 in boost::asio::detail::task_io_service_operation::complete (this=0x7f7694204620, owner=..., ec=..., bytes_transferred=0) at /usr/include/boost/asio/detail/task_io_service_operation.hpp:37 #29 0x00000000011f6b9f in boost::asio::detail::epoll_reactor::descriptor_state::do_complete (owner=0x3da16f0, base=0x3db02e0, ec=..., bytes_transferred=29) at /usr/include/boost/asio/detail/impl/epoll_reactor.ipp:651 #30 0x000000000119a8b8 in boost::asio::detail::task_io_service_operation::complete (this=0x3db02e0, owner=..., ec=..., bytes_transferred=29) at /usr/include/boost/asio/detail/task_io_service_operation.hpp:37 #31 0x000000000119b933 in boost::asio::detail::task_io_service::do_run_one (this=0x3da16f0, lock=..., this_thread=..., ec=...) at /usr/include/boost/asio/detail/impl/task_io_service.ipp:384 #32 0x000000000119b493 in boost::asio::detail::task_io_service::run (this=0x3da16f0, ec=...) at /usr/include/boost/asio/detail/impl/task_io_service.ipp:153 #33 0x00000000017f914b in boost::asio::io_service::run (this=0x3da16d0, ec=...) at /usr/include/boost/asio/impl/io_service.ipp:66 #34 0x00000000017f7e62 in EventManager::RunWithExceptionHandling (this=0x3da16d0) at controller/src/io/event_manager.cc:51 #35 0x00000000012d1a19 in main (argc=3, argv=0x7fff60b81508) at controller/src/vnsw/agent/ovs_tor_agent/main.cc:75 (gdb) frame 6 #6 0x00000000012fce92 in TcpSession::IsClosed (this=0x7f76980e6b90) at controller/src/io/tcp_session.h:119 119 controller/src/io/tcp_session.h: No such file or directory. (gdb) p refcount_ $8 = { > = { > = { my_storage = { my_value = 0 } }, }, } (gdb) (gdb) frame 3 #3 0x000000000100c7ec in tbb::mutex::lock (this=0x7f76980e6b98) at /usr/include/tbb/mutex.h:164 164 /usr/include/tbb/mutex.h: No such file or directory. (gdb) p *this $11 = { static is_rw_mutex = false, static is_recursive_mutex = false, static is_fair_mutex = false, impl = { __data = { __lock = 2, __count = 32630, __owner = 0, __nusers = 0, __kind = -1, __spins = 0, __elision = 0, __list = { __prev = 0x0, __next = 0x0 } }, __size = "\002\000\000\000v\177\000\000\000\000\000\000\000\000\000\000\377\377\377\377", '\000' , __align = 140144782868482 } } (gdb)