This crash was seen on 40 vRouters (78 node cluster) after few minutes of solution script run.
2.23-123.el7.centos kilo.
full core here:
10.84.5.112:/cs-shared/bugs/<bug-id>/
(gdb) bt
#0 0x00002b9faf1085f7 in raise () from /lib64/libc.so.6
#1 0x00002b9faf109ce8 in abort () from /lib64/libc.so.6
#2 0x00002b9faf101566 in __assert_fail_base () from /lib64/libc.so.6
#3 0x00002b9faf101612 in __assert_fail () from /lib64/libc.so.6
#4 0x00000000016e784a in FlowTable::InsertByIndex (this=0x2b9fc0041ba0, flow_handle=466461, flow=0x2ba0405eb260) at controller/src/vnsw/agent/pkt/flow_table.cc:2103
#5 0x00000000016ef4d1 in FlowTable::AddIndexFlowInfo (this=0x2b9fc0041ba0, fe=0x2ba0405eb260, flow_handle=466461) at controller/src/vnsw/agent/pkt/flow_table.cc:3355
#6 0x00000000016e23d4 in FlowTable::Add (this=0x2b9fc0041ba0, flow=0x2ba0405eb260, rflow=0x2ba04021e620, update=false) at controller/src/vnsw/agent/pkt/flow_table.cc:1065
#7 0x0000000001728de9 in PktFlowInfo::Add (this=0x2ba0094037b0, pkt=0x314dfe0, in=0x2ba009403730, out=0x2ba009403770) at controller/src/vnsw/agent/pkt/pkt_flow_info.cc:1514
#8 0x0000000001737509 in FlowHandler::Run (this=0x2ba040066a10) at controller/src/vnsw/agent/pkt/flow_handler.cc:80
#9 0x0000000001732854 in Proto::ProcessProto (this=0x2b9fc003f830, msg_info=...) at controller/src/vnsw/agent/pkt/proto.cc:47
#10 0x0000000001735296 in boost::_mfi::mf1<bool, Proto, boost::shared_ptr<PktInfo> >::operator() (this=0x2ba009403b68, p=0x2b9fc003f830, a1=...) at /usr/include/boost/bind/mem_fn_template.hpp:165
#11 0x0000000001734c42 in boost::_bi::list2<boost::_bi::value<Proto*>, boost::arg<1> >::operator()<bool, boost::_mfi::mf1<bool, Proto, boost::shared_ptr<PktInfo> >, boost::_bi::list1<boost::shared_ptr<PktInfo>&> > (this=0x2ba009403b78, f=..., a=...)
at /usr/include/boost/bind/bind.hpp:303
#12 0x00000000017345b6 in boost::_bi::bind_t<bool, boost::_mfi::mf1<bool, Proto, boost::shared_ptr<PktInfo> >, boost::_bi::list2<boost::_bi::value<Proto*>, boost::arg<1> > >::operator()<boost::shared_ptr<PktInfo> > (this=0x2ba009403b68, a1=...)
at /usr/include/boost/bind/bind_template.hpp:32
#13 0x00000000017341bd in boost::detail::function::function_obj_invoker1<boost::_bi::bind_t<bool, boost::_mfi::mf1<bool, Proto, boost::shared_ptr<PktInfo> >, boost::_bi::list2<boost::_bi::value<Proto*>, boost::arg<1> > >, bool, boost::shared_ptr<PktInfo> >::invoke
(function_obj_ptr=..., a0=...) at /usr/include/boost/function/function_template.hpp:132
#14 0x000000000172377a in boost::function1<bool, boost::shared_ptr<PktInfo> >::operator() (this=0x2ba009403b60, a0=...) at /usr/include/boost/function/function_template.hpp:767
#15 0x00000000017356ea in QueueTaskRunner<boost::shared_ptr<PktInfo>, WorkQueue<boost::shared_ptr<PktInfo> > >::RunQueue (this=0x314e390) at controller/src/base/queue_task.h:87
#16 0x00000000017354f0 in QueueTaskRunner<boost::shared_ptr<PktInfo>, WorkQueue<boost::shared_ptr<PktInfo> > >::Run (this=0x314e390) at controller/src/base/queue_task.h:66
#17 0x0000000001d8f860 in TaskImpl::execute (this=0x2b9fb4f63d40) at controller/src/base/task.cc:253
#18 0x00002b9fae69f8ca in tbb::internal::custom_scheduler<tbb::internal::IntelSchedulerTraits>::local_wait_for_all(tbb::task&, tbb::task*) () from /lib64/libtbb.so.2
#19 0x00002b9fae69b5b6 in tbb::internal::arena::process(tbb::internal::generic_scheduler&) () from /lib64/libtbb.so.2
#20 0x00002b9fae69ac8b in tbb::internal::market::process(rml::job&) () from /lib64/libtbb.so.2
#21 0x00002b9fae69867f in tbb::internal::rml::private_worker::run() () from /lib64/libtbb.so.2
#22 0x00002b9fae698879 in tbb::internal::rml::private_worker::thread_routine(void*) () from /lib64/libtbb.so.2
#23 0x00002b9fae469dc5 in start_thread () from /lib64/libpthread.so.0
#24 0x00002b9faf1c921d in clone () from /lib64/libc.so.6
(gdb) p ((FlowEntryPtr) 0x2ba05e509010)[466461]ok….
Review in progress for https:/
Submitter: Naveen N (<email address hidden>)