Redhat-7.3-osp8- 3.0.2.0-51~liberty:collector crash:OpServerProxy::DeleteUVEs

[shajuvk]

Trace:
=======
#0 0x00002b5562edc1d7 in raise () from /lib64/libc.so.6
#1 0x00002b5562edd8c8 in abort () from /lib64/libc.so.6
#2 0x00002b5562ed5146 in __assert_fail_base () from /lib64/libc.so.6
#3 0x00002b5562ed51f2 in __assert_fail () from /lib64/libc.so.6
#4 0x0000000000638328 in OpServerProxy::DeleteUVEs(std::string const&, std::string const&, std::string const&, std::string const&) ()
#5 0x00000000005b4ab9 in SandeshGenerator::DisconnectSession(VizSession*) ()
#6 0x00000000005921d2 in Collector::ReceiveSandeshCtrlMsg(SandeshStateMachine*, SandeshSession*, Sandesh const*) ()
#7 0x00000000007d4152 in SandeshServerConnection::ProcessSandeshCtrlMessage(std::string const&, SandeshHeader const&, std::string, unsigned int) ()
#8 0x00000000007c31ef in ssm::ServerInit::react(ssm::EvSandeshCtrlMessageRecv const&) ()
#9 0x00000000007d24e9 in boost::statechart::detail::reaction_result boost::statechart::custom_reaction<ssm::EvSandeshCtrlMessageRecv>::react<ssm::ServerInit, boost::statechart::event_base, void const*>(ssm::ServerInit&, boost::statechart::event_base const&, void const* const&) ()
#10 0x00000000007d2195 in boost::statechart::detail::reaction_result boost::statechart::simple_state<ssm::ServerInit, SandeshStateMachine, boost::mpl::list<mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na>, (boost::statechart::history_mode)0>::local_react_impl_non_empty::local_react_impl<boost::mpl::list3<boost::statechart::custom_reaction<ssm::EvSandeshCtrlMessageRecv>, boost::statechart::custom_reaction<ssm::EvSandeshMessageRecv>, boost::statechart::in_state_reaction<ssm::EvTcpDeleteSession, SandeshStateMachine, &(void SandeshStateMachine::DeleteTcpSession<ssm::EvTcpDeleteSession>(ssm::EvTcpDeleteSession const&))> >, boost::statechart::simple_state<ssm::ServerInit, SandeshStateMachine, boost::mpl::list<mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na>, (boost::statechart::history_mode)0> >(boost::statechart::simple_state<ssm::ServerInit, SandeshStateMachine, boost::mpl::list<mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na>, (boost::statechart::history_mode)0>&, boost::statechart::event_base const&, void const*) ()
#11 0x00000000007d1e73 in boost::statechart::detail::reaction_result boost::statechart::simple_state<ssm::ServerInit, SandeshStateMachine, boost::mpl::list<mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na>, (boost::statechart::history_mode)0>::local_react<boost::mpl::list3<boost::statechart::custom_reaction<ssm::EvSandeshCtrlMessageRecv>, boost::statechart::custom_reaction<ssm::EvSandeshMessageRecv>, boost::statechart::in_state_reaction<ssm::EvTcpDeleteSession, SandeshStateMachine, &(void SandeshStateMachine::DeleteTcpSession<ssm::EvTcpDeleteSession>(ssm::EvTcpDeleteSession const&))> > >(boost::statechart::event_base const&, void const*) ()
#12 0x00000000007d1a3f in boost::statechart::detail::reaction_result boost::statechart::simple_state<ssm::ServerInit, SandeshStateMachine, boost::mpl::list<mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na>, (boost::statechart::history_mode)0>::local_react_impl_non_empty::local_react_impl<boost::mpl::list4<boost::statechart::custom_reaction<ssm::EvTcpClose>, boost::statechart::custom_reaction<ssm::EvSandeshCtrlMessageRecv>, boost::statechart::custom_reaction<ssm::EvSandeshMessageRecv>, boost::statechart::in_state_reaction<ssm::EvTcpDeleteSession, SandeshStateMachine, &(void SandeshStateMachine::DeleteTcpSession<ssm::EvTcpDeleteSession>(ssm::EvTcpDeleteSession const&))> >, boost::statechart::simple_state<ssm::ServerInit, SandeshStateMachine, boost::mpl::list<mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na>, (boost::statechart::history_mode)0> >(boost::statechart::simple_state<ssm::ServerInit, SandeshStateMachine, boost::mpl::list<mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na>, (boost::statechart::history_mode)0>&, boost::statechart::event_base const&, void const*) ()
#13 0x00000000007d1247 in boost::statechart::detail::reaction_result boost::statechart::simple_state<ssm::ServerInit, SandeshStateMachine, boost::mpl::list<mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na>, (boost::statechart::history_mode)0>::local_react<boost::mpl::list4<boost::statechart::custom_reaction<ssm::EvTcpClose>, boost::statechart::custom_reaction<ssm::EvSandeshCtrlMessageRecv>, boost::statechart::custom_reaction<ssm::EvSandeshMessageRecv>, boost::statechart::in_state_reaction<ssm::EvTcpDeleteSession, SandeshStateMachine, &(void SandeshStateMachine::DeleteTcpSession<ssm::EvTcpDeleteSession>(ssm::EvTcpDeleteSession const&))> > >(boost::statechart::event_base const&, void const*) ()
#14 0x00000000007d09be in boost::statechart::detail::reaction_result boost::statechart::simple_state<ssm::ServerInit, SandeshStateMachine, boost::mpl::list<mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na>, (boost::statechart::history_mode)0>::local_react_impl_non_empty::local_react_impl<boost::mpl::list<boost::statechart::transition<ssm::EvStop, ssm::Idle, SandeshStateMachine, &(void SandeshStateMachine::OnIdle<ssm::EvStop>(ssm::EvStop const&))>, boost::statechart::custom_reaction<ssm::EvTcpClose>, boost::statechart::custom_reaction<ssm::EvSandeshCtrlMessageRecv>, boost::statechart::custom_reaction<ssm::EvSandeshMessageRecv>, boost::statechart::in_state_reaction<ssm::EvTcpDeleteSession, SandeshStateMachine, &(void SandeshStateMachine::DeleteTcpSession<ssm::EvTcpDeleteSession>(ssm::EvTcpDeleteSession const&))>, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na>, boost::statechart::simple_state<ssm::ServerInit, SandeshStateMachine, boost::mpl::list<mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na>, (boost::statechart::history_mode)0> >---Type <return> to continue, or q <return> to quit---
(boost::statechart::simple_state<ssm::ServerInit, SandeshStateMachine, boost::mpl::list<mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na>, (boost::statechart::history_mode)0>&, boost::statechart::event_base const&, void const*) ()
#15 0x00000000007cfe21 in boost::statechart::detail::reaction_result boost::statechart::simple_state<ssm::ServerInit, SandeshStateMachine, boost::mpl::list<mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na>, (boost::statechart::history_mode)0>::local_react<boost::mpl::list<boost::statechart::transition<ssm::EvStop, ssm::Idle, SandeshStateMachine, &(void SandeshStateMachine::OnIdle<ssm::EvStop>(ssm::EvStop const&))>, boost::statechart::custom_reaction<ssm::EvTcpClose>, boost::statechart::custom_reaction<ssm::EvSandeshCtrlMessageRecv>, boost::statechart::custom_reaction<ssm::EvSandeshMessageRecv>, boost::statechart::in_state_reaction<ssm::EvTcpDeleteSession, SandeshStateMachine, &(void SandeshStateMachine::DeleteTcpSession<ssm::EvTcpDeleteSession>(ssm::EvTcpDeleteSession const&))>, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na> >(boost::statechart::event_base const&, void const*) ()
#16 0x00000000007ceceb in boost::statechart::simple_state<ssm::ServerInit, SandeshStateMachine, boost::mpl::list<mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na>, (boost::statechart::history_mode)0>::react_impl(boost::statechart::event_base const&, void const*) ()
#17 0x0000000000798495 in boost::statechart::detail::send_function<boost::statechart::detail::state_base<std::allocator<void>, boost::statechart::detail::rtti_policy>, boost::statechart::event_base, void const*>::operator()() ()
#18 0x00000000007c9a54 in boost::statechart::detail::safe_reaction_result boost::statechart::null_exception_translator::operator()<boost::statechart::detail::send_function<boost::statechart::detail::state_base<std::allocator<void>, boost::statechart::detail::rtti_policy>, boost::statechart::event_base, void const*>, boost::statechart::state_machine<SandeshStateMachine, ssm::Idle, std::allocator<void>, boost::statechart::null_exception_translator>::exception_event_handler>(boost::statechart::detail::send_function<boost::statechart::detail::state_base<std::allocator<void>, boost::statechart::detail::rtti_policy>, boost::statechart::event_base, void const*>, boost::statechart::state_machine<SandeshStateMachine, ssm::Idle, std::allocator<void>, boost::statechart::null_exception_translator>::exception_event_handler) ()
#19 0x00000000007c812a in boost::statechart::state_machine<SandeshStateMachine, ssm::Idle, std::allocator<void>, boost::statechart::null_exception_translator>::send_event(boost::statechart::event_base const&) ()
#20 0x00000000007c5bdf in boost::statechart::state_machine<SandeshStateMachine, ssm::Idle, std::allocator<void>, boost::statechart::null_exception_translator>::process_event(boost::statechart::event_base const&) ()
#21 0x00000000007c080d in SandeshStateMachine::DequeueEvent(SandeshStateMachine::EventContainer&) ()
#22 0x00000000007cd93e in boost::_mfi::mf1<bool, SandeshStateMachine, SandeshStateMachine::EventContainer&>::operator()(SandeshStateMachine*, SandeshStateMachine::EventContainer&) const ()
#23 0x00000000007cc332 in bool boost::_bi::list2<boost::_bi::value<SandeshStateMachine*>, boost::arg<1> >::operator()<bool, boost::_mfi::mf1<bool, SandeshStateMachine, SandeshStateMachine::EventContainer&>, boost::_bi::list1<SandeshStateMachine::EventContainer&> >(boost::_bi::type<bool>, boost::_mfi::mf1<bool, SandeshStateMachine, SandeshStateMachine::EventContainer&>&, boost::_bi::list1<SandeshStateMachine::EventContainer&>&, long) ()
#24 0x00000000007cb424 in bool boost::_bi::bind_t<bool, boost::_mfi::mf1<bool, SandeshStateMachine, SandeshStateMachine::EventContainer&>, boost::_bi::list2<boost::_bi::value<SandeshStateMachine*>, boost::arg<1> > >::operator()<SandeshStateMachine::EventContainer>(SandeshStateMachine::EventContainer&) ()
#25 0x00000000007ca2e6 in boost::detail::function::function_obj_invoker1<boost::_bi::bind_t<bool, boost::_mfi::mf1<bool, SandeshStateMachine, SandeshStateMachine::EventContainer&>, boost::_bi::list2<boost::_bi::value<SandeshStateMachine*>, boost::arg<1> > >, bool, SandeshStateMachine::EventContainer>::invoke(boost::detail::function::function_buffer&, SandeshStateMachine::EventContainer) ()
#26 0x00000000007d07b7 in boost::function1<bool, SandeshStateMachine::EventContainer>::operator()(SandeshStateMachine::EventContainer) const ()
#27 0x00000000007cfac6 in QueueTaskRunner<SandeshStateMachine::EventContainer, WorkQueue<SandeshStateMachine::EventContainer> >::RunQueue() ()
#28 0x00000000007ceb92 in QueueTaskRunner<SandeshStateMachine::EventContainer, WorkQueue<SandeshStateMachine::EventContainer> >::Run() ()
#29 0x000000000045791a in TaskImpl::execute() ()
#30 0x00002b5561c42440 in tbb::internal::custom_scheduler<tbb::internal::IntelSchedulerTraits>::local_wait_for_all (this=0x2b5574000e80, parent=...,
    child=0x2b558003b3c0) at /ecbuilds/PipeLine/sb/third_party/tbb40_20111130oss/src/tbb/custom_scheduler.h:449
#31 0x00002b5561c392e4 in tbb::internal::arena::process (this=0x2a4eb00, s=...) at /ecbuilds/PipeLine/sb/third_party/tbb40_20111130oss/src/tbb/arena.cpp:99
#32 0x00002b5561c37c24 in tbb::internal::market::process (this=0x2a52a00, j=...) at /ecbuilds/PipeLine/sb/third_party/tbb40_20111130oss/src/tbb/market.cpp:393
#33 0x00002b5561c329c2 in tbb::internal::rml::private_worker::run (this=0x2a53680)
    at /ecbuilds/PipeLine/sb/third_party/tbb40_20111130oss/src/tbb/private_server.cpp:263
#34 0x00002b5561c32864 in tbb::internal::rml::private_worker::thread_routine (arg=0x2a53680)
    at /ecbuilds/PipeLine/sb/third_party/tbb40_20111130oss/src/tbb/private_server.cpp:231
#35 0x00002b55619fddc5 in start_thread () from /lib64/libpthread.so.0
#36 0x00002b5562f9e73d in clone () from /lib64/libc.so.6

[shajuvk (shajuvk)]

core dump : /cs-shared/bugs/1661775

[Arvind (arvindv)]

Can I have the redis-server log, collector log ?

Also was it reproducible in 3.2, 3.1 or do u suspect it could be ?

There was a fix made in 3.2 in recent past in the same code path. So it will help
if u can confirm if you reproduced it.

https://bugs.launchpad.net/juniperopenstack/+bug/1670908