Two cores here: 1. vizd Issue is that Db_SetInitDone() is not implemented in CqlIf and hence generator can connect and issue Db_AddColumn which will not find table meta and core root@a7s9:~# gdb /var/tmp/vizd /var/tmp/core.contrail-collec.2089.nodeg20.1450683984 GNU gdb (Ubuntu 7.7-0ubuntu3) 7.7 Copyright (C) 2014 Free Software Foundation, Inc. License GPLv3+: GNU GPL version 3 or later This is free software: you are free to change and redistribute it. There is NO WARRANTY, to the extent permitted by law. Type "show copying" and "show warranty" for details. This GDB was configured as "x86_64-linux-gnu". Type "show configuration" for configuration details. For bug reporting instructions, please see: . Find the GDB manual and other documentation resources online at: . For help, type "help". Type "apropos word" to search for commands related to "word"... Reading symbols from /var/tmp/vizd...done. warning: core file may not match specified executable file. [New LWP 2890] [New LWP 3940] [New LWP 2228] [New LWP 3941] [New LWP 2891] [New LWP 2089] [New LWP 2884] [New LWP 2885] [Thread debugging using libthread_db enabled] Using host libthread_db library "/lib/x86_64-linux-gnu/libthread_db.so.1". Core was generated by `/usr/bin/contrail-collector'. Program terminated with signal SIGABRT, Aborted. #0 0x00007f7d37044cc9 in raise () from /lib/x86_64-linux-gnu/libc.so.6 (gdb) bt #0 0x00007f7d37044cc9 in raise () from /lib/x86_64-linux-gnu/libc.so.6 #1 0x00007f7d370480d8 in abort () from /lib/x86_64-linux-gnu/libc.so.6 #2 0x00007f7d3703db86 in ?? () from /lib/x86_64-linux-gnu/libc.so.6 #3 0x00007f7d3703dc32 in __assert_fail () from /lib/x86_64-linux-gnu/libc.so.6 #4 0x0000000000416600 in cass::cql::CqlIf::CqlIfImpl::IsTableStatic (table=..., this=) at controller/src/database/cassandra/cql/cql_if.cc:878 #5 0x0000000000560e08 in IsTableStatic (table=..., this=) at /usr/include/c++/4.8/bits/basic_string.h:539 #6 InsertIntoTableSync (consistency=CASS_CONSISTENCY_ONE, v_columns=..., this=) at controller/src/database/cassandra/cql/cql_if.cc:893 #7 cass::cql::CqlIf::Db_AddColumnSync (this=, cl=...) at controller/src/database/cassandra/cql/cql_if.cc:1152 #8 0x000000000055ea09 in cass::cql::CqlIf::Db_AddColumn (this=, cl=...) at controller/src/database/cassandra/cql/cql_if.cc:1148 #9 0x000000000052e528 in DbHandler::StatTableWrite (this=this@entry=0x104fa00, t2=t2@entry=172935007, statName=..., statAttr=..., ptag=..., stag=..., t1=t1@entry=0, unm=..., jsonline=..., ttl=ttl@entry=172800) at controller/src/analytics/db_handler.cc:863 #10 0x0000000000530018 in DbHandler::StatTableInsertTtl (this=this@entry=0x104fa00, ts=ts@entry=1450683984478939, statName=..., statAttr=..., attribs_tag=..., attribs=..., ttl=ttl@entry=172800) at controller/src/analytics/db_handler.cc:1019 #11 0x000000000053125f in DbHandler::FieldNamesTableInsert (this=this@entry=0x104fa00, timestamp=1450683984478939, table_prefix=..., field_name=..., field_val=..., ttl=ttl@entry=172800) at controller/src/analytics/db_handler.cc:661 #12 0x00000000005337e9 in DbHandler::MessageTableInsert (this=0x104fa00, vmsgp=) at controller/src/analytics/db_handler.cc:592 #13 0x00000000004c39ad in Ruleeng::rule_execute (this=0x104c080, vmsgp=0x7f7d2eff6ad0, uveproc=, db=0x104fa00) at controller/src/analytics/ruleeng.cc:670 #14 0x00000000004e6946 in operator() (a2=, a1=true, a0=0x7f7d2eff6ad0, this=0x7f7d2eff68e0) at /usr/include/boost/function/function_template.hpp:767 #15 SandeshGenerator::ProcessRules (this=, vmsg=0x7f7d2eff6ad0, rsc=) at controller/src/analytics/generator.cc:261 #16 0x00000000004cf2af in Collector::ReceiveSandeshMsg (this=0x1055da0, session=, msg=0x7f7d1003a0c0, rsc=) at controller/src/analytics/collector.cc:194 #17 0x0000000000648020 in SandeshServerConnection::ProcessSandeshMessage (this=0x10cef50, msg=, resource=) at tools/sandesh/library/cpp/sandesh_connection.cc:191 #18 0x00000000006451ec in react (event=..., this=0x7f7d10000a90) at tools/sandesh/library/cpp/sandesh_state_machine.cc:371 #19 react (eventType=, evt=..., stt=...) at /usr/include/boost/statechart/custom_reaction.hpp:42 #20 local_react_impl, boost::statechart::in_state_reaction, boost::statechart::custom_reaction >, boost::statechart::simple_state, (boost::statechart::history_mode)0> > (eventType=0xa67b50 ::idProvider_>, evt=..., stt=...) at /usr/include/boost/statechart/simple_state.hpp:816 #21 local_react, boost::statechart::in_state_reaction, boost::statechart::custom_reaction > > (eventType=0xa67b50 ::idProvider_>, evt=..., this=0x7f7d10000a90) at /usr/include/boost/statechart/simple_state.hpp:851 #22 local_react_impl, boost::statechart::custom_reaction, boost::statechart::in_state_reaction, boost::statechart::custom_reaction >, boost::statechart::simple_state, (boost::statechart::history_mode)0> > (eventType=, evt=..., stt=...) at /usr/include/boost/statechart/simple_state.hpp:820 #23 local_react, boost::statechart::custom_reaction, boost::statechart::in_state_reaction, boost::statechart::custom_reaction > > (eventType=, evt=..., this=) at /usr/include/boost/statechart/simple_state.hpp:851 #24 local_react_impl, boost::statechart::custom_reaction, boost::statechart::custom_reaction, boost::statechart::in_state_reaction, boost::statechart::custom_reaction >, boost::statechart::simple_state, (boost::statechart::history_mode)0> > ( eventType=0xa67b50 ::idProvider_>, evt=..., stt=...) at /usr/include/boost/statechart/simple_state.hpp:820 #25 local_react, boost::statechart::custom_reaction, boost::statechart::custom_reaction, boost::statechart::in_state_reaction, boost::statechart::custom_reaction > > ( eventType=0xa67b50 ::idProvider_>, evt=..., this=0x7f7d10000a90) at /usr/include/boost/statechart/simple_state.hpp:851 #26 boost::statechart::simple_state, (boost::statechart::history_mode)0>::react_impl (this=0x7f7d10000a90, evt=..., eventType=0xa67b50 ::idProvider_>) at /usr/include/boost/statechart/simple_state.hpp:489 #27 0x0000000000644bdb in operator() (this=) at /usr/include/boost/statechart/state_machine.hpp:87 #28 operator(), boost::statechart::detail::rtti_policy>, boost::statechart::event_base, void const*>, boost::statechart::state_machine::exception_event_handler> (this=0x10cf038, action=...) at /usr/include/boost/statechart/null_exception_translator.hpp:33 #29 boost::statechart::state_machine, boost::statechart::null_exception_translator>::send_event (this=0x10cefe0, evt=...) at /usr/include/boost/statechart/state_machine.hpp:889 #30 0x000000000063cea5 in process_event (evt=..., this=0x10cefe0) at /usr/include/boost/statechart/state_machine.hpp:275 #31 SandeshStateMachine::DequeueEvent (this=0x10cefe0, ec=...) at tools/sandesh/library/cpp/sandesh_state_machine.cc:733 #32 0x0000000000643d87 in operator() (a0=, this=0x7f7d2eff70e0) at /usr/include/boost/function/function_template.hpp:767 #33 QueueTaskRunner >::RunQueue (this=0x7f7d1000fc60) at controller/src/base/queue_task.h:81 #34 0x00000000006abdc0 in TaskImpl::execute (this=0x7f7d31e1b740) at controller/src/base/task.cc:238 #35 0x00007f7d385cfb3a in ?? () from /usr/lib/libtbb.so.2 #36 0x00007f7d385cb816 in ?? () from /usr/lib/libtbb.so.2 #37 0x00007f7d385caf4b in ?? () from /usr/lib/libtbb.so.2 #38 0x00007f7d385c70ff in ?? () from /usr/lib/libtbb.so.2 #39 0x00007f7d385c72f9 in ?? () from /usr/lib/libtbb.so.2 #40 0x00007f7d387eb182 in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0 #41 0x00007f7d3710847d in clone () from /lib/x86_64-linux-gnu/libc.so.6 (gdb) info threads Id Target Id Frame 8 Thread 0x7f7d2e3f6700 (LWP 2885) 0x00007f7d370f982d in read () from /lib/x86_64-linux-gnu/libc.so.6 7 Thread 0x7f7d2e7f7700 (LWP 2884) 0x00007f7d387ef414 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0 6 Thread 0x7f7d3a178100 (LWP 2089) 0x00007f7d37108b13 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6 5 Thread 0x7f7d2dff5700 (LWP 2891) 0x00007f7d370cef3d in nanosleep () from /lib/x86_64-linux-gnu/libc.so.6 4 Thread 0x7f7d2cb65700 (LWP 3941) 0x00007f7d37102859 in syscall () from /lib/x86_64-linux-gnu/libc.so.6 3 Thread 0x7f7d30a4d700 (LWP 2228) 0x00007f7d370cef3d in nanosleep () from /lib/x86_64-linux-gnu/libc.so.6 2 Thread 0x7f7d2f7f9700 (LWP 3940) 0x00007f7d37102859 in syscall () from /lib/x86_64-linux-gnu/libc.so.6 * 1 Thread 0x7f7d2eff8700 (LWP 2890) 0x00007f7d37044cc9 in raise () from /lib/x86_64-linux-gnu/libc.so.6 (gdb) thread apply all bt Thread 8 (Thread 0x7f7d2e3f6700 (LWP 2885)): #0 0x00007f7d370f982d in read () from /lib/x86_64-linux-gnu/libc.so.6 #1 0x00007f7d370886a0 in _IO_file_underflow () from /lib/x86_64-linux-gnu/libc.so.6 #2 0x00007f7d3708962e in _IO_default_uflow () from /lib/x86_64-linux-gnu/libc.so.6 #3 0x00007f7d3707d344 in _IO_getline_info () from /lib/x86_64-linux-gnu/libc.so.6 #4 0x00007f7d3707c2c6 in fgets () from /lib/x86_64-linux-gnu/libc.so.6 #5 0x0000000000694e26 in fgets (__stream=0x7f7d18018560, __n=512, __s=0x7f7d2e3f46b0 "") at /usr/include/x86_64-linux-gnu/bits/stdio2.h:263 #6 MiscUtils::GetVersionInfoInternal (cmd=..., rpm_version=..., build_num=...) at controller/src/base/misc_utils.cc:85 #7 0x00000000006956ff in MiscUtils::GetContrailVersionInfo (id=id@entry=MiscUtils::Analytics, rpm_version=..., build_num=...) at controller/src/base/misc_utils.cc:126 #8 0x00000000006964fb in MiscUtils::GetBuildInfo (id=id@entry=MiscUtils::Analytics, build_info=..., result=...) at controller/src/base/misc_utils.cc:135 #9 0x0000000000522b08 in CollectorVersion (version=...) at controller/src/analytics/main.cc:55 #10 CollectorSummaryLogger (collector=0x1055da0, hostname=..., osp=osp@entry=0x1049ac0) at controller/src/analytics/main.cc:102 #11 0x0000000000523381 in CollectorInfoLogger (ctx=...) at controller/src/analytics/main.cc:128 #12 0x00000000006af6df in operator() (this=) at /usr/include/boost/function/function_template.hpp:767 #13 TaskTrigger::WorkerTask::Run (this=0x7f7d18015680) at controller/src/base/task_trigger.cc:19 #14 0x00000000006abdc0 in TaskImpl::execute (this=0x7f7d31e33640) at controller/src/base/task.cc:238 #15 0x00007f7d385cfb3a in ?? () from /usr/lib/libtbb.so.2 #16 0x00007f7d385cb816 in ?? () from /usr/lib/libtbb.so.2 #17 0x00007f7d385caf4b in ?? () from /usr/lib/libtbb.so.2 #18 0x00007f7d385c70ff in ?? () from /usr/lib/libtbb.so.2 #19 0x00007f7d385c72f9 in ?? () from /usr/lib/libtbb.so.2 #20 0x00007f7d387eb182 in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0 #21 0x00007f7d3710847d in clone () from /lib/x86_64-linux-gnu/libc.so.6 Thread 7 (Thread 0x7f7d2e7f7700 (LWP 2884)): #0 0x00007f7d387ef414 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0 #1 0x00007f7d36e00329 in uv_cond_wait () from /usr/lib/x86_64-linux-gnu/libuv.so.1 #2 0x00007f7d39d2a44b in cass::Future::wait() () from /usr/lib/x86_64-linux-gnu/libcassandra.so.2 #3 0x0000000000560064 in cass::cql::impl::ExecuteQuerySyncInternal (session=, query=query@entry=0x7f7d2e7f5b10 "CREATE KEYSPACE IF NOT EXISTS \"ContrailAnalyticsCql\" WITH replication = { 'class' : 'SimpleStrategy', 'replication_factor' : 2 }", result=result@entry=0x0, consistency=consistency@entry=CASS_CONSISTENCY_QUORUM) at controller/src/database/cassandra/cql/cql_if.cc:544 #4 0x0000000000564e16 in ExecuteQuerySync (consistency=CASS_CONSISTENCY_QUORUM, query=0x7f7d2e7f5b10 "CREATE KEYSPACE IF NOT EXISTS \"ContrailAnalyticsCql\" WITH replication = { 'class' : 'SimpleStrategy', 'replication_factor' : 2 }", session=) at controller/src/database/cassandra/cql/cql_if.cc:561 #5 cass::cql::CqlIf::CqlIfImpl::CreateKeyspaceIfNotExistsSync (this=0x1047b10, keyspace=..., replication_factor=..., consistency=consistency@entry=CASS_CONSISTENCY_QUORUM) at controller/src/database/cassandra/cql/cql_if.cc:825 #6 0x00000000005605ba in cass::cql::CqlIf::Db_AddSetTablespace (this=this@entry=0x104fdf0, tablespace=..., replication_factor=...) at controller/src/database/cassandra/cql/cql_if.cc:1122 #7 0x000000000052b923 in DbHandler::Initialize (this=this@entry=0x104fa00, instance=instance@entry=-1) at controller/src/analytics/db_handler.cc:307 #8 0x0000000000534958 in Init (initial=true, instance=-1, this=0x104fa00) at controller/src/analytics/db_handler.cc:290 #9 DbHandlerInitializer::Initialize (this=0x104f8a0) at controller/src/analytics/db_handler.cc:1599 #10 0x0000000000534e49 in DbHandlerInitializer::InitTimerExpired (this=) at controller/src/analytics/db_handler.cc:1637 #11 0x00000000006b28f9 in operator() (this=) at /usr/include/boost/function/function_template.hpp:767 #12 Timer::TimerTask::Run (this=0x1047a20) at controller/src/base/timer.cc:42 #13 0x00000000006abdc0 in TaskImpl::execute (this=0x7f7d31e53440) at controller/src/base/task.cc:238 #14 0x00007f7d385cfb3a in ?? () from /usr/lib/libtbb.so.2 #15 0x00007f7d385cb816 in ?? () from /usr/lib/libtbb.so.2 #16 0x00007f7d385caf4b in ?? () from /usr/lib/libtbb.so.2 #17 0x00007f7d385c70ff in ?? () from /usr/lib/libtbb.so.2 #18 0x00007f7d385c72f9 in ?? () from /usr/lib/libtbb.so.2 #19 0x00007f7d387eb182 in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0 #20 0x00007f7d3710847d in clone () from /lib/x86_64-linux-gnu/libc.so.6 Thread 6 (Thread 0x7f7d3a178100 (LWP 2089)): #0 0x00007f7d37108b13 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6 #1 0x00000000005800a5 in boost::asio::detail::epoll_reactor::run (this=0x104ad00, block=, ops=...) at /usr/include/boost/asio/detail/impl/epoll_reactor.ipp:392 #2 0x0000000000582652 in do_run_one (ec=..., this_thread=..., lock=..., this=0x101aee0) at /usr/include/boost/asio/detail/impl/task_io_service.ipp:368 #3 boost::asio::detail::task_io_service::run (this=0x101aee0, ec=...) at /usr/include/boost/asio/detail/impl/task_io_service.ipp:153 #4 0x00000000005a21a8 in run (this=0x1032510, ec=...) at /usr/include/boost/asio/impl/io_service.ipp:66 #5 EventManager::Run (this=0x1032510) at controller/src/io/event_manager.cc:32 #6 0x000000000042073c in main (argc=, argv=) at controller/src/analytics/main.cc:417 Thread 5 (Thread 0x7f7d2dff5700 (LWP 2891)): #0 0x00007f7d370cef3d in nanosleep () from /lib/x86_64-linux-gnu/libc.so.6 #1 0x00007f7d371004a4 in usleep () from /lib/x86_64-linux-gnu/libc.so.6 ---Type to continue, or q to quit--- #2 0x00007f7d392f09ba in rd_kafka_wait_destroyed () from /usr/lib/x86_64-linux-gnu/librdkafka.so.1 #3 0x0000000000547ce5 in StopKafka (this=0x104b1c0) at controller/src/analytics/OpServerProxy.cc:547 #4 OpServerProxy::OpServerImpl::KafkaTimer (this=0x104b1c0) at controller/src/analytics/OpServerProxy.cc:473 #5 0x00000000006b28f9 in operator() (this=) at /usr/include/boost/function/function_template.hpp:767 #6 Timer::TimerTask::Run (this=0x1047a80) at controller/src/base/timer.cc:42 #7 0x00000000006abdc0 in TaskImpl::execute (this=0x7f7d31e53b40) at controller/src/base/task.cc:238 #8 0x00007f7d385cfb3a in ?? () from /usr/lib/libtbb.so.2 #9 0x00007f7d385cb816 in ?? () from /usr/lib/libtbb.so.2 #10 0x00007f7d385caf4b in ?? () from /usr/lib/libtbb.so.2 #11 0x00007f7d385c70ff in ?? () from /usr/lib/libtbb.so.2 #12 0x00007f7d385c72f9 in ?? () from /usr/lib/libtbb.so.2 #13 0x00007f7d387eb182 in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0 #14 0x00007f7d3710847d in clone () from /lib/x86_64-linux-gnu/libc.so.6 Thread 4 (Thread 0x7f7d2cb65700 (LWP 3941)): #0 0x00007f7d37102859 in syscall () from /lib/x86_64-linux-gnu/libc.so.6 #1 0x00007f7d36e0480a in ?? () from /usr/lib/x86_64-linux-gnu/libuv.so.1 #2 0x00007f7d36e02de7 in ?? () from /usr/lib/x86_64-linux-gnu/libuv.so.1 #3 0x00007f7d36df5cf2 in uv_run () from /usr/lib/x86_64-linux-gnu/libuv.so.1 #4 0x00007f7d39d2a505 in cass::LoopThread::on_run_internal(void*) () from /usr/lib/x86_64-linux-gnu/libcassandra.so.2 #5 0x00007f7d36dffe37 in ?? () from /usr/lib/x86_64-linux-gnu/libuv.so.1 #6 0x00007f7d387eb182 in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0 #7 0x00007f7d3710847d in clone () from /lib/x86_64-linux-gnu/libc.so.6 Thread 3 (Thread 0x7f7d30a4d700 (LWP 2228)): #0 0x00007f7d370cef3d in nanosleep () from /lib/x86_64-linux-gnu/libc.so.6 #1 0x00007f7d370cedd4 in sleep () from /lib/x86_64-linux-gnu/libc.so.6 #2 0x00007f7d3930070a in ?? () from /usr/lib/x86_64-linux-gnu/librdkafka.so.1 #3 0x00007f7d387eb182 in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0 #4 0x00007f7d3710847d in clone () from /lib/x86_64-linux-gnu/libc.so.6 Thread 2 (Thread 0x7f7d2f7f9700 (LWP 3940)): #0 0x00007f7d37102859 in syscall () from /lib/x86_64-linux-gnu/libc.so.6 #1 0x00007f7d36e0480a in ?? () from /usr/lib/x86_64-linux-gnu/libuv.so.1 #2 0x00007f7d36e02de7 in ?? () from /usr/lib/x86_64-linux-gnu/libuv.so.1 #3 0x00007f7d36df5cf2 in uv_run () from /usr/lib/x86_64-linux-gnu/libuv.so.1 #4 0x00007f7d39d2a505 in cass::LoopThread::on_run_internal(void*) () from /usr/lib/x86_64-linux-gnu/libcassandra.so.2 #5 0x00007f7d36dffe37 in ?? () from /usr/lib/x86_64-linux-gnu/libuv.so.1 #6 0x00007f7d387eb182 in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0 #7 0x00007f7d3710847d in clone () from /lib/x86_64-linux-gnu/libc.so.6 Thread 1 (Thread 0x7f7d2eff8700 (LWP 2890)): #0 0x00007f7d37044cc9 in raise () from /lib/x86_64-linux-gnu/libc.so.6 #1 0x00007f7d370480d8 in abort () from /lib/x86_64-linux-gnu/libc.so.6 #2 0x00007f7d3703db86 in ?? () from /lib/x86_64-linux-gnu/libc.so.6 #3 0x00007f7d3703dc32 in __assert_fail () from /lib/x86_64-linux-gnu/libc.so.6 #4 0x0000000000416600 in cass::cql::CqlIf::CqlIfImpl::IsTableStatic (table=..., this=) at controller/src/database/cassandra/cql/cql_if.cc:878 #5 0x0000000000560e08 in IsTableStatic (table=..., this=) at /usr/include/c++/4.8/bits/basic_string.h:539 #6 InsertIntoTableSync (consistency=CASS_CONSISTENCY_ONE, v_columns=..., this=) at controller/src/database/cassandra/cql/cql_if.cc:893 #7 cass::cql::CqlIf::Db_AddColumnSync (this=, cl=...) at controller/src/database/cassandra/cql/cql_if.cc:1152 #8 0x000000000055ea09 in cass::cql::CqlIf::Db_AddColumn (this=, cl=...) at controller/src/database/cassandra/cql/cql_if.cc:1148 #9 0x000000000052e528 in DbHandler::StatTableWrite (this=this@entry=0x104fa00, t2=t2@entry=172935007, statName=..., statAttr=..., ptag=..., stag=..., t1=t1@entry=0, unm=..., jsonline=..., ttl=ttl@entry=172800) at controller/src/analytics/db_handler.cc:863 #10 0x0000000000530018 in DbHandler::StatTableInsertTtl (this=this@entry=0x104fa00, ts=ts@entry=1450683984478939, statName=..., statAttr=..., attribs_tag=..., attribs=..., ttl=ttl@entry=172800) at controller/src/analytics/db_handler.cc:1019 #11 0x000000000053125f in DbHandler::FieldNamesTableInsert (this=this@entry=0x104fa00, timestamp=1450683984478939, table_prefix=..., field_name=..., field_val=..., ttl=ttl@entry=172800) at controller/src/analytics/db_handler.cc:661 #12 0x00000000005337e9 in DbHandler::MessageTableInsert (this=0x104fa00, vmsgp=) at controller/src/analytics/db_handler.cc:592 #13 0x00000000004c39ad in Ruleeng::rule_execute (this=0x104c080, vmsgp=0x7f7d2eff6ad0, uveproc=, db=0x104fa00) at controller/src/analytics/ruleeng.cc:670 #14 0x00000000004e6946 in operator() (a2=, a1=true, a0=0x7f7d2eff6ad0, this=0x7f7d2eff68e0) at /usr/include/boost/function/function_template.hpp:767 #15 SandeshGenerator::ProcessRules (this=, vmsg=0x7f7d2eff6ad0, rsc=) at controller/src/analytics/generator.cc:261 #16 0x00000000004cf2af in Collector::ReceiveSandeshMsg (this=0x1055da0, session=, msg=0x7f7d1003a0c0, rsc=) at controller/src/analytics/collector.cc:194 #17 0x0000000000648020 in SandeshServerConnection::ProcessSandeshMessage (this=0x10cef50, msg=, resource=) at tools/sandesh/library/cpp/sandesh_connection.cc:191 #18 0x00000000006451ec in react (event=..., this=0x7f7d10000a90) at tools/sandesh/library/cpp/sandesh_state_machine.cc:371 #19 react (eventType=, evt=..., stt=...) at /usr/include/boost/statechart/custom_reaction.hpp:42 ---Type to continue, or q to quit--- #20 local_react_impl, boost::statechart::in_state_reaction, boost::statechart::custom_reaction >, boost::statechart::simple_state, (boost::statechart::history_mode)0> > (eventType=0xa67b50 ::idProvider_>, evt=..., stt=...) at /usr/include/boost/statechart/simple_state.hpp:816 #21 local_react, boost::statechart::in_state_reaction, boost::statechart::custom_reaction > > (eventType=0xa67b50 ::idProvider_>, evt=..., this=0x7f7d10000a90) at /usr/include/boost/statechart/simple_state.hpp:851 #22 local_react_impl, boost::statechart::custom_reaction, boost::statechart::in_state_reaction, boost::statechart::custom_reaction >, boost::statechart::simple_state, (boost::statechart::history_mode)0> > (eventType=, evt=..., stt=...) at /usr/include/boost/statechart/simple_state.hpp:820 #23 local_react, boost::statechart::custom_reaction, boost::statechart::in_state_reaction, boost::statechart::custom_reaction > > (eventType=, evt=..., this=) at /usr/include/boost/statechart/simple_state.hpp:851 #24 local_react_impl, boost::statechart::custom_reaction, boost::statechart::custom_reaction, boost::statechart::in_state_reaction, boost::statechart::custom_reaction >, boost::statechart::simple_state, (boost::statechart::history_mode)0> > ( eventType=0xa67b50 ::idProvider_>, evt=..., stt=...) at /usr/include/boost/statechart/simple_state.hpp:820 #25 local_react, boost::statechart::custom_reaction, boost::statechart::custom_reaction, boost::statechart::in_state_reaction, boost::statechart::custom_reaction > > ( eventType=0xa67b50 ::idProvider_>, evt=..., this=0x7f7d10000a90) at /usr/include/boost/statechart/simple_state.hpp:851 #26 boost::statechart::simple_state, (boost::statechart::history_mode)0>::react_impl (this=0x7f7d10000a90, evt=..., eventType=0xa67b50 ::idProvider_>) at /usr/include/boost/statechart/simple_state.hpp:489 #27 0x0000000000644bdb in operator() (this=) at /usr/include/boost/statechart/state_machine.hpp:87 #28 operator(), boost::statechart::detail::rtti_policy>, boost::statechart::event_base, void const*>, boost::statechart::state_machine::exception_event_handler> (this=0x10cf038, action=...) at /usr/include/boost/statechart/null_exception_translator.hpp:33 #29 boost::statechart::state_machine, boost::statechart::null_exception_translator>::send_event (this=0x10cefe0, evt=...) at /usr/include/boost/statechart/state_machine.hpp:889 #30 0x000000000063cea5 in process_event (evt=..., this=0x10cefe0) at /usr/include/boost/statechart/state_machine.hpp:275 #31 SandeshStateMachine::DequeueEvent (this=0x10cefe0, ec=...) at tools/sandesh/library/cpp/sandesh_state_machine.cc:733 #32 0x0000000000643d87 in operator() (a0=, this=0x7f7d2eff70e0) at /usr/include/boost/function/function_template.hpp:767 #33 QueueTaskRunner >::RunQueue (this=0x7f7d1000fc60) at controller/src/base/queue_task.h:81 #34 0x00000000006abdc0 in TaskImpl::execute (this=0x7f7d31e1b740) at controller/src/base/task.cc:238 #35 0x00007f7d385cfb3a in ?? () from /usr/lib/libtbb.so.2 #36 0x00007f7d385cb816 in ?? () from /usr/lib/libtbb.so.2 #37 0x00007f7d385caf4b in ?? () from /usr/lib/libtbb.so.2 #38 0x00007f7d385c70ff in ?? () from /usr/lib/libtbb.so.2 #39 0x00007f7d385c72f9 in ?? () from /usr/lib/libtbb.so.2 #40 0x00007f7d387eb182 in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0 #41 0x00007f7d3710847d in clone () from /lib/x86_64-linux-gnu/libc.so.6 (gdb) q Need to implement Db_SetInitDone, just checking session_state_ in CqlIfImpl to be CONNECTED is not enough. 2. contrail-query-engine core contrail-query-engine can come up before contrail-collector and hence it can do a Db_GetRow() of not created SYSTEM_OBJECT_TABLE and hence the table meta will be NULL. This is a valid scenario which should be handled by failing the Db_GetRow without the assert. root@a7s9:~# gdb /var/tmp/qed /var/tmp/nodeg20_core.contrail-query-.22973.nodeg20.1450680158 GNU gdb (Ubuntu 7.7-0ubuntu3) 7.7 Copyright (C) 2014 Free Software Foundation, Inc. License GPLv3+: GNU GPL version 3 or later This is free software: you are free to change and redistribute it. There is NO WARRANTY, to the extent permitted by law. Type "show copying" and "show warranty" for details. This GDB was configured as "x86_64-linux-gnu". Type "show configuration" for configuration details. For bug reporting instructions, please see: . Find the GDB manual and other documentation resources online at: . For help, type "help". Type "apropos word" to search for commands related to "word"... Reading symbols from /var/tmp/qed...done. warning: core file may not match specified executable file. [New LWP 22973] [New LWP 22977] [New LWP 22978] [New LWP 22980] [New LWP 22981] [New LWP 22982] [New LWP 22983] [New LWP 22986] [New LWP 22992] [Thread debugging using libthread_db enabled] Using host libthread_db library "/lib/x86_64-linux-gnu/libthread_db.so.1". Core was generated by `/usr/bin/contrail-query-engine'. Program terminated with signal SIGABRT, Aborted. #0 0x00007f9b8947acc9 in raise () from /lib/x86_64-linux-gnu/libc.so.6 (gdb) bt #0 0x00007f9b8947acc9 in raise () from /lib/x86_64-linux-gnu/libc.so.6 #1 0x00007f9b8947e0d8 in abort () from /lib/x86_64-linux-gnu/libc.so.6 #2 0x00007f9b89473b86 in ?? () from /lib/x86_64-linux-gnu/libc.so.6 #3 0x00007f9b89473c32 in __assert_fail () from /lib/x86_64-linux-gnu/libc.so.6 #4 0x0000000000418518 in cass::cql::CqlIf::CqlIfImpl::IsTableStatic (table=..., this=) at controller/src/database/cassandra/cql/cql_if.cc:878 #5 0x000000000053c40f in IsTableStatic (table=..., this=) at controller/src/database/cassandra/cql/cql_if.cc:918 #6 SelectFromTableSync (consistency=CASS_CONSISTENCY_ONE, out=, rkey=..., cfname=..., this=) at controller/src/database/cassandra/cql/cql_if.cc:910 #7 cass::cql::CqlIf::Db_GetRow (this=, out=, cfname=..., rowkey=...) at controller/src/database/cassandra/cql/cql_if.cc:1159 #8 0x00000000004a0ffb in QueryEngine::QueryEngine (this=0x12285c0, evm=, cassandra_ips=..., cassandra_ports=..., redis_ip=..., redis_port=, redis_password=..., max_tasks=8, max_slice=100, cassandra_user=..., cassandra_password=..., use_cql=true) at controller/src/query_engine/query.cc:1268 #9 0x000000000041c4c5 in main (argc=, argv=) at controller/src/query_engine/qed.cc:301 (gdb) info threads Id Target Id Frame 9 Thread 0x7f9b6b7fe700 (LWP 22992) 0x00007f9b89538859 in syscall () from /lib/x86_64-linux-gnu/libc.so.6 8 Thread 0x7f9b6bfff700 (LWP 22986) 0x00007f9b89538859 in syscall () from /lib/x86_64-linux-gnu/libc.so.6 7 Thread 0x7f9b80c96700 (LWP 22983) 0x00007f9b89538859 in syscall () from /lib/x86_64-linux-gnu/libc.so.6 6 Thread 0x7f9b81097700 (LWP 22982) 0x00007f9b89538859 in syscall () from /lib/x86_64-linux-gnu/libc.so.6 5 Thread 0x7f9b81498700 (LWP 22981) 0x00007f9b89538859 in syscall () from /lib/x86_64-linux-gnu/libc.so.6 4 Thread 0x7f9b81899700 (LWP 22980) 0x00007f9b89538859 in syscall () from /lib/x86_64-linux-gnu/libc.so.6 3 Thread 0x7f9b81c9a700 (LWP 22978) 0x00007f9b89538859 in syscall () from /lib/x86_64-linux-gnu/libc.so.6 2 Thread 0x7f9b8209b700 (LWP 22977) 0x00007f9b89538859 in syscall () from /lib/x86_64-linux-gnu/libc.so.6 * 1 Thread 0x7f9b8b9c47c0 (LWP 22973) 0x00007f9b8947acc9 in raise () from /lib/x86_64-linux-gnu/libc.so.6 (gdb) thread apply all bt Thread 9 (Thread 0x7f9b6b7fe700 (LWP 22992)): #0 0x00007f9b89538859 in syscall () from /lib/x86_64-linux-gnu/libc.so.6 #1 0x00007f9b8923a80a in ?? () from /usr/lib/x86_64-linux-gnu/libuv.so.1 #2 0x00007f9b89238de7 in ?? () from /usr/lib/x86_64-linux-gnu/libuv.so.1 #3 0x00007f9b8922bcf2 in uv_run () from /usr/lib/x86_64-linux-gnu/libuv.so.1 #4 0x00007f9b8b577505 in cass::LoopThread::on_run_internal(void*) () from /usr/lib/x86_64-linux-gnu/libcassandra.so.2 #5 0x00007f9b89235e37 in ?? () from /usr/lib/x86_64-linux-gnu/libuv.so.1 #6 0x00007f9b8a46d182 in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0 #7 0x00007f9b8953e47d in clone () from /lib/x86_64-linux-gnu/libc.so.6 Thread 8 (Thread 0x7f9b6bfff700 (LWP 22986)): #0 0x00007f9b89538859 in syscall () from /lib/x86_64-linux-gnu/libc.so.6 #1 0x00007f9b8923a80a in ?? () from /usr/lib/x86_64-linux-gnu/libuv.so.1 #2 0x00007f9b89238de7 in ?? () from /usr/lib/x86_64-linux-gnu/libuv.so.1 #3 0x00007f9b8922bcf2 in uv_run () from /usr/lib/x86_64-linux-gnu/libuv.so.1 #4 0x00007f9b8b577505 in cass::LoopThread::on_run_internal(void*) () from /usr/lib/x86_64-linux-gnu/libcassandra.so.2 #5 0x00007f9b89235e37 in ?? () from /usr/lib/x86_64-linux-gnu/libuv.so.1 #6 0x00007f9b8a46d182 in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0 #7 0x00007f9b8953e47d in clone () from /lib/x86_64-linux-gnu/libc.so.6 Thread 7 (Thread 0x7f9b80c96700 (LWP 22983)): #0 0x00007f9b89538859 in syscall () from /lib/x86_64-linux-gnu/libc.so.6 #1 0x00007f9b8a2492db in ?? () from /usr/lib/libtbb.so.2 #2 0x00007f9b8a2492f9 in ?? () from /usr/lib/libtbb.so.2 #3 0x00007f9b8a46d182 in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0 #4 0x00007f9b8953e47d in clone () from /lib/x86_64-linux-gnu/libc.so.6 Thread 6 (Thread 0x7f9b81097700 (LWP 22982)): #0 0x00007f9b89538859 in syscall () from /lib/x86_64-linux-gnu/libc.so.6 #1 0x00007f9b8a2492db in ?? () from /usr/lib/libtbb.so.2 #2 0x00007f9b8a2492f9 in ?? () from /usr/lib/libtbb.so.2 #3 0x00007f9b8a46d182 in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0 #4 0x00007f9b8953e47d in clone () from /lib/x86_64-linux-gnu/libc.so.6 Thread 5 (Thread 0x7f9b81498700 (LWP 22981)): #0 0x00007f9b89538859 in syscall () from /lib/x86_64-linux-gnu/libc.so.6 #1 0x00007f9b8a2492db in ?? () from /usr/lib/libtbb.so.2 #2 0x00007f9b8a2492f9 in ?? () from /usr/lib/libtbb.so.2 #3 0x00007f9b8a46d182 in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0 #4 0x00007f9b8953e47d in clone () from /lib/x86_64-linux-gnu/libc.so.6 Thread 4 (Thread 0x7f9b81899700 (LWP 22980)): #0 0x00007f9b89538859 in syscall () from /lib/x86_64-linux-gnu/libc.so.6 #1 0x00007f9b8a2492db in ?? () from /usr/lib/libtbb.so.2 #2 0x00007f9b8a2492f9 in ?? () from /usr/lib/libtbb.so.2 #3 0x00007f9b8a46d182 in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0 #4 0x00007f9b8953e47d in clone () from /lib/x86_64-linux-gnu/libc.so.6 Thread 3 (Thread 0x7f9b81c9a700 (LWP 22978)): #0 0x00007f9b89538859 in syscall () from /lib/x86_64-linux-gnu/libc.so.6 #1 0x00007f9b8a2492db in ?? () from /usr/lib/libtbb.so.2 #2 0x00007f9b8a2492f9 in ?? () from /usr/lib/libtbb.so.2 #3 0x00007f9b8a46d182 in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0 #4 0x00007f9b8953e47d in clone () from /lib/x86_64-linux-gnu/libc.so.6 Thread 2 (Thread 0x7f9b8209b700 (LWP 22977)): #0 0x00007f9b89538859 in syscall () from /lib/x86_64-linux-gnu/libc.so.6 #1 0x00007f9b8a2492db in ?? () from /usr/lib/libtbb.so.2 #2 0x00007f9b8a2492f9 in ?? () from /usr/lib/libtbb.so.2 #3 0x00007f9b8a46d182 in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0 #4 0x00007f9b8953e47d in clone () from /lib/x86_64-linux-gnu/libc.so.6 ---Type to continue, or q to quit---q Quit (gdb) bt #0 0x00007f9b8947acc9 in raise () from /lib/x86_64-linux-gnu/libc.so.6 #1 0x00007f9b8947e0d8 in abort () from /lib/x86_64-linux-gnu/libc.so.6 #2 0x00007f9b89473b86 in ?? () from /lib/x86_64-linux-gnu/libc.so.6 #3 0x00007f9b89473c32 in __assert_fail () from /lib/x86_64-linux-gnu/libc.so.6 #4 0x0000000000418518 in cass::cql::CqlIf::CqlIfImpl::IsTableStatic (table=..., this=) at controller/src/database/cassandra/cql/cql_if.cc:878 #5 0x000000000053c40f in IsTableStatic (table=..., this=) at controller/src/database/cassandra/cql/cql_if.cc:918 #6 SelectFromTableSync (consistency=CASS_CONSISTENCY_ONE, out=, rkey=..., cfname=..., this=) at controller/src/database/cassandra/cql/cql_if.cc:910 #7 cass::cql::CqlIf::Db_GetRow (this=, out=, cfname=..., rowkey=...) at controller/src/database/cassandra/cql/cql_if.cc:1159 #8 0x00000000004a0ffb in QueryEngine::QueryEngine (this=0x12285c0, evm=, cassandra_ips=..., cassandra_ports=..., redis_ip=..., redis_port=, redis_password=..., max_tasks=8, max_slice=100, cassandra_user=..., cassandra_password=..., use_cql=true) at controller/src/query_engine/query.cc:1268 #9 0x000000000041c4c5 in main (argc=, argv=) at controller/src/query_engine/qed.cc:301 (gdb) f 7 #7 cass::cql::CqlIf::Db_GetRow (this=, out=, cfname=..., rowkey=...) at controller/src/database/cassandra/cql/cql_if.cc:1159 1159 controller/src/database/cassandra/cql/cql_if.cc: No such file or directory. (gdb) p this $1 = (gdb) p this->impl_ value has been optimized out (gdb) p this->impl_->session_state_ value has been optimized out (gdb) list 1154 in controller/src/database/cassandra/cql/cql_if.cc (gdb) f 6 #6 SelectFromTableSync (consistency=CASS_CONSISTENCY_ONE, out=, rkey=..., cfname=..., this=) at controller/src/database/cassandra/cql/cql_if.cc:910 910 in controller/src/database/cassandra/cql/cql_if.cc (gdb) p session_ value has been optimized out (gdb) p session_.px value has been optimized out (gdb) q root@a7s9:~# ls root@a7s9:~# cd /var/tmp root@a7s9:/var/tmp# ls core.contrail-collec.2089.nodeg20.1450683984 nodeg20_core.contrail-query-.22973.nodeg20.1450680158 nodeg20_core.contrail-query-.23027.nodeg20.1450680160.gz qed vizd root@a7s9:/var/tmp# gunzip nodeg20_core.contrail-query-.23027.nodeg20.1450680160.gz root@a7s9:/var/tmp# gdb /var/tmp/qed /var/tmp/nodeg20_core.contrail-query-.23027.nodeg20.1450680160 GNU gdb (Ubuntu 7.7-0ubuntu3) 7.7 Copyright (C) 2014 Free Software Foundation, Inc. License GPLv3+: GNU GPL version 3 or later This is free software: you are free to change and redistribute it. There is NO WARRANTY, to the extent permitted by law. Type "show copying" and "show warranty" for details. This GDB was configured as "x86_64-linux-gnu". Type "show configuration" for configuration details. For bug reporting instructions, please see: . Find the GDB manual and other documentation resources online at: . For help, type "help". Type "apropos word" to search for commands related to "word"... Reading symbols from /var/tmp/qed...done. warning: core file may not match specified executable file. [New LWP 23027] [New LWP 23028] [New LWP 23038] [New LWP 23029] [New LWP 23031] [New LWP 23032] [New LWP 23033] [New LWP 23035] [New LWP 23045] [New LWP 23041] [New LWP 23042] [New LWP 23039] [New LWP 23044] [New LWP 23030] [New LWP 23043] [New LWP 23040] [New LWP 23037] [Thread debugging using libthread_db enabled] Using host libthread_db library "/lib/x86_64-linux-gnu/libthread_db.so.1". Core was generated by `/usr/bin/contrail-query-engine'. Program terminated with signal SIGABRT, Aborted. #0 0x00007f84ecbe4cc9 in raise () from /lib/x86_64-linux-gnu/libc.so.6 (gdb) bt #0 0x00007f84ecbe4cc9 in raise () from /lib/x86_64-linux-gnu/libc.so.6 #1 0x00007f84ecbe80d8 in abort () from /lib/x86_64-linux-gnu/libc.so.6 #2 0x00007f84ecbddb86 in ?? () from /lib/x86_64-linux-gnu/libc.so.6 #3 0x00007f84ecbddc32 in __assert_fail () from /lib/x86_64-linux-gnu/libc.so.6 #4 0x0000000000418518 in cass::cql::CqlIf::CqlIfImpl::IsTableStatic (table=..., this=) at controller/src/database/cassandra/cql/cql_if.cc:878 #5 0x000000000053c40f in IsTableStatic (table=..., this=) at controller/src/database/cassandra/cql/cql_if.cc:918 #6 SelectFromTableSync (consistency=CASS_CONSISTENCY_ONE, out=, rkey=..., cfname=..., this=) at controller/src/database/cassandra/cql/cql_if.cc:910 #7 cass::cql::CqlIf::Db_GetRow (this=, out=, cfname=..., rowkey=...) at controller/src/database/cassandra/cql/cql_if.cc:1159 #8 0x00000000004a0ffb in QueryEngine::QueryEngine (this=0xdad0a0, evm=, cassandra_ips=..., cassandra_ports=..., redis_ip=..., redis_port=, redis_password=..., max_tasks=8, max_slice=100, cassandra_user=..., cassandra_password=..., use_cql=true) at controller/src/query_engine/query.cc:1268 #9 0x000000000041c4c5 in main (argc=, argv=) at controller/src/query_engine/qed.cc:301 (gdb) inof threads Undefined command: "inof". Try "help". (gdb) info threads Id Target Id Frame 17 Thread 0x7f84dfbfe700 (LWP 23037) 0x00007f84ecca2859 in syscall () from /lib/x86_64-linux-gnu/libc.so.6 16 Thread 0x7f84df7fd700 (LWP 23040) 0x00007f84ecca2859 in syscall () from /lib/x86_64-linux-gnu/libc.so.6 15 Thread 0x7f84de7f9700 (LWP 23043) 0x00007f84ecca2859 in syscall () from /lib/x86_64-linux-gnu/libc.so.6 14 Thread 0x7f84dffff700 (LWP 23030) 0x00007f84ecca2859 in syscall () from /lib/x86_64-linux-gnu/libc.so.6 13 Thread 0x7f84ddda5700 (LWP 23044) 0x00007f84ecca2859 in syscall () from /lib/x86_64-linux-gnu/libc.so.6 12 Thread 0x7f84df3fc700 (LWP 23039) 0x00007f84ecca2859 in syscall () from /lib/x86_64-linux-gnu/libc.so.6 11 Thread 0x7f84de3f8700 (LWP 23042) 0x00007f84ecca2859 in syscall () from /lib/x86_64-linux-gnu/libc.so.6 10 Thread 0x7f84debfa700 (LWP 23041) 0x00007f84ecca2859 in syscall () from /lib/x86_64-linux-gnu/libc.so.6 9 Thread 0x7f84dd5a4700 (LWP 23045) 0x00007f84ecca2859 in syscall () from /lib/x86_64-linux-gnu/libc.so.6 8 Thread 0x7f84e4400700 (LWP 23035) 0x00007f84ecca2859 in syscall () from /lib/x86_64-linux-gnu/libc.so.6 7 Thread 0x7f84e4c02700 (LWP 23033) 0x00007f84ecca2859 in syscall () from /lib/x86_64-linux-gnu/libc.so.6 6 Thread 0x7f84e4801700 (LWP 23032) 0x00007f84ecca2859 in syscall () from /lib/x86_64-linux-gnu/libc.so.6 5 Thread 0x7f84e5003700 (LWP 23031) 0x00007f84ecca2859 in syscall () from /lib/x86_64-linux-gnu/libc.so.6 4 Thread 0x7f84e5404700 (LWP 23029) 0x00007f84ecca2859 in syscall () from /lib/x86_64-linux-gnu/libc.so.6 3 Thread 0x7f84deffb700 (LWP 23038) 0x00007f84ecca2859 in syscall () from /lib/x86_64-linux-gnu/libc.so.6 2 Thread 0x7f84e5805700 (LWP 23028) 0x00007f84ecca2859 in syscall () from /lib/x86_64-linux-gnu/libc.so.6 * 1 Thread 0x7f84ef12e7c0 (LWP 23027) 0x00007f84ecbe4cc9 in raise () from /lib/x86_64-linux-gnu/libc.so.6 (gdb) thread apply all bt Thread 17 (Thread 0x7f84dfbfe700 (LWP 23037)): #0 0x00007f84ecca2859 in syscall () from /lib/x86_64-linux-gnu/libc.so.6 #1 0x00007f84ed9b32db in ?? () from /usr/lib/libtbb.so.2 #2 0x00007f84ed9b32f9 in ?? () from /usr/lib/libtbb.so.2 #3 0x00007f84edbd7182 in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0 #4 0x00007f84ecca847d in clone () from /lib/x86_64-linux-gnu/libc.so.6 Thread 16 (Thread 0x7f84df7fd700 (LWP 23040)): #0 0x00007f84ecca2859 in syscall () from /lib/x86_64-linux-gnu/libc.so.6 #1 0x00007f84ed9b32db in ?? () from /usr/lib/libtbb.so.2 #2 0x00007f84ed9b32f9 in ?? () from /usr/lib/libtbb.so.2 #3 0x00007f84edbd7182 in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0 #4 0x00007f84ecca847d in clone () from /lib/x86_64-linux-gnu/libc.so.6 Thread 15 (Thread 0x7f84de7f9700 (LWP 23043)): #0 0x00007f84ecca2859 in syscall () from /lib/x86_64-linux-gnu/libc.so.6 #1 0x00007f84ed9b32db in ?? () from /usr/lib/libtbb.so.2 #2 0x00007f84ed9b32f9 in ?? () from /usr/lib/libtbb.so.2 #3 0x00007f84edbd7182 in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0 #4 0x00007f84ecca847d in clone () from /lib/x86_64-linux-gnu/libc.so.6 Thread 14 (Thread 0x7f84dffff700 (LWP 23030)): #0 0x00007f84ecca2859 in syscall () from /lib/x86_64-linux-gnu/libc.so.6 #1 0x00007f84ed9b32db in ?? () from /usr/lib/libtbb.so.2 #2 0x00007f84ed9b32f9 in ?? () from /usr/lib/libtbb.so.2 #3 0x00007f84edbd7182 in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0 #4 0x00007f84ecca847d in clone () from /lib/x86_64-linux-gnu/libc.so.6 Thread 13 (Thread 0x7f84ddda5700 (LWP 23044)): #0 0x00007f84ecca2859 in syscall () from /lib/x86_64-linux-gnu/libc.so.6 #1 0x00007f84ec9a480a in ?? () from /usr/lib/x86_64-linux-gnu/libuv.so.1 #2 0x00007f84ec9a2de7 in ?? () from /usr/lib/x86_64-linux-gnu/libuv.so.1 #3 0x00007f84ec995cf2 in uv_run () from /usr/lib/x86_64-linux-gnu/libuv.so.1 #4 0x00007f84eece1505 in cass::LoopThread::on_run_internal(void*) () from /usr/lib/x86_64-linux-gnu/libcassandra.so.2 #5 0x00007f84ec99fe37 in ?? () from /usr/lib/x86_64-linux-gnu/libuv.so.1 #6 0x00007f84edbd7182 in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0 #7 0x00007f84ecca847d in clone () from /lib/x86_64-linux-gnu/libc.so.6 Thread 12 (Thread 0x7f84df3fc700 (LWP 23039)): #0 0x00007f84ecca2859 in syscall () from /lib/x86_64-linux-gnu/libc.so.6 #1 0x00007f84ed9b32db in ?? () from /usr/lib/libtbb.so.2 #2 0x00007f84ed9b32f9 in ?? () from /usr/lib/libtbb.so.2 #3 0x00007f84edbd7182 in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0 #4 0x00007f84ecca847d in clone () from /lib/x86_64-linux-gnu/libc.so.6 Thread 11 (Thread 0x7f84de3f8700 (LWP 23042)): #0 0x00007f84ecca2859 in syscall () from /lib/x86_64-linux-gnu/libc.so.6 #1 0x00007f84ed9b32db in ?? () from /usr/lib/libtbb.so.2 #2 0x00007f84ed9b32f9 in ?? () from /usr/lib/libtbb.so.2 #3 0x00007f84edbd7182 in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0 #4 0x00007f84ecca847d in clone () from /lib/x86_64-linux-gnu/libc.so.6 Thread 10 (Thread 0x7f84debfa700 (LWP 23041)): #0 0x00007f84ecca2859 in syscall () from /lib/x86_64-linux-gnu/libc.so.6 #1 0x00007f84ed9b32db in ?? () from /usr/lib/libtbb.so.2 #2 0x00007f84ed9b32f9 in ?? () from /usr/lib/libtbb.so.2 #3 0x00007f84edbd7182 in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0 #4 0x00007f84ecca847d in clone () from /lib/x86_64-linux-gnu/libc.so.6 Thread 9 (Thread 0x7f84dd5a4700 (LWP 23045)): #0 0x00007f84ecca2859 in syscall () from /lib/x86_64-linux-gnu/libc.so.6 ---Type to continue, or q to quit--- #1 0x00007f84ec9a480a in ?? () from /usr/lib/x86_64-linux-gnu/libuv.so.1 #2 0x00007f84ec9a2de7 in ?? () from /usr/lib/x86_64-linux-gnu/libuv.so.1 #3 0x00007f84ec995cf2 in uv_run () from /usr/lib/x86_64-linux-gnu/libuv.so.1 #4 0x00007f84eece1505 in cass::LoopThread::on_run_internal(void*) () from /usr/lib/x86_64-linux-gnu/libcassandra.so.2 #5 0x00007f84ec99fe37 in ?? () from /usr/lib/x86_64-linux-gnu/libuv.so.1 #6 0x00007f84edbd7182 in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0 #7 0x00007f84ecca847d in clone () from /lib/x86_64-linux-gnu/libc.so.6 Thread 8 (Thread 0x7f84e4400700 (LWP 23035)): #0 0x00007f84ecca2859 in syscall () from /lib/x86_64-linux-gnu/libc.so.6 #1 0x00007f84ed9b32db in ?? () from /usr/lib/libtbb.so.2 #2 0x00007f84ed9b32f9 in ?? () from /usr/lib/libtbb.so.2 #3 0x00007f84edbd7182 in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0 #4 0x00007f84ecca847d in clone () from /lib/x86_64-linux-gnu/libc.so.6 Thread 7 (Thread 0x7f84e4c02700 (LWP 23033)): #0 0x00007f84ecca2859 in syscall () from /lib/x86_64-linux-gnu/libc.so.6 #1 0x00007f84ed9b32db in ?? () from /usr/lib/libtbb.so.2 #2 0x00007f84ed9b32f9 in ?? () from /usr/lib/libtbb.so.2 #3 0x00007f84edbd7182 in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0 #4 0x00007f84ecca847d in clone () from /lib/x86_64-linux-gnu/libc.so.6 Thread 6 (Thread 0x7f84e4801700 (LWP 23032)): #0 0x00007f84ecca2859 in syscall () from /lib/x86_64-linux-gnu/libc.so.6 #1 0x00007f84ed9b32db in ?? () from /usr/lib/libtbb.so.2 #2 0x00007f84ed9b32f9 in ?? () from /usr/lib/libtbb.so.2 #3 0x00007f84edbd7182 in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0 #4 0x00007f84ecca847d in clone () from /lib/x86_64-linux-gnu/libc.so.6 Thread 5 (Thread 0x7f84e5003700 (LWP 23031)): #0 0x00007f84ecca2859 in syscall () from /lib/x86_64-linux-gnu/libc.so.6 #1 0x00007f84ed9b32db in ?? () from /usr/lib/libtbb.so.2 #2 0x00007f84ed9b32f9 in ?? () from /usr/lib/libtbb.so.2 #3 0x00007f84edbd7182 in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0 #4 0x00007f84ecca847d in clone () from /lib/x86_64-linux-gnu/libc.so.6 Thread 4 (Thread 0x7f84e5404700 (LWP 23029)): #0 0x00007f84ecca2859 in syscall () from /lib/x86_64-linux-gnu/libc.so.6 #1 0x00007f84ed9b32db in ?? () from /usr/lib/libtbb.so.2 #2 0x00007f84ed9b32f9 in ?? () from /usr/lib/libtbb.so.2 #3 0x00007f84edbd7182 in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0 #4 0x00007f84ecca847d in clone () from /lib/x86_64-linux-gnu/libc.so.6 Thread 3 (Thread 0x7f84deffb700 (LWP 23038)): #0 0x00007f84ecca2859 in syscall () from /lib/x86_64-linux-gnu/libc.so.6 #1 0x00007f84ed9b32db in ?? () from /usr/lib/libtbb.so.2 #2 0x00007f84ed9b32f9 in ?? () from /usr/lib/libtbb.so.2 #3 0x00007f84edbd7182 in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0 #4 0x00007f84ecca847d in clone () from /lib/x86_64-linux-gnu/libc.so.6 Thread 2 (Thread 0x7f84e5805700 (LWP 23028)): #0 0x00007f84ecca2859 in syscall () from /lib/x86_64-linux-gnu/libc.so.6 #1 0x00007f84ed9b32db in ?? () from /usr/lib/libtbb.so.2 #2 0x00007f84ed9b32f9 in ?? () from /usr/lib/libtbb.so.2 #3 0x00007f84edbd7182 in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0 #4 0x00007f84ecca847d in clone () from /lib/x86_64-linux-gnu/libc.so.6 Thread 1 (Thread 0x7f84ef12e7c0 (LWP 23027)): #0 0x00007f84ecbe4cc9 in raise () from /lib/x86_64-linux-gnu/libc.so.6 #1 0x00007f84ecbe80d8 in abort () from /lib/x86_64-linux-gnu/libc.so.6 #2 0x00007f84ecbddb86 in ?? () from /lib/x86_64-linux-gnu/libc.so.6 #3 0x00007f84ecbddc32 in __assert_fail () from /lib/x86_64-linux-gnu/libc.so.6 ---Type to continue, or q to quit--- #4 0x0000000000418518 in cass::cql::CqlIf::CqlIfImpl::IsTableStatic (table=..., this=) at controller/src/database/cassandra/cql/cql_if.cc:878 #5 0x000000000053c40f in IsTableStatic (table=..., this=) at controller/src/database/cassandra/cql/cql_if.cc:918 #6 SelectFromTableSync (consistency=CASS_CONSISTENCY_ONE, out=, rkey=..., cfname=..., this=) at controller/src/database/cassandra/cql/cql_if.cc:910 #7 cass::cql::CqlIf::Db_GetRow (this=, out=, cfname=..., rowkey=...) at controller/src/database/cassandra/cql/cql_if.cc:1159 #8 0x00000000004a0ffb in QueryEngine::QueryEngine (this=0xdad0a0, evm=, cassandra_ips=..., cassandra_ports=..., redis_ip=..., redis_port=, redis_password=..., max_tasks=8, max_slice=100, cassandra_user=..., cassandra_password=..., use_cql=true) at controller/src/query_engine/query.cc:1268 #9 0x000000000041c4c5 in main (argc=, argv=) at controller/src/query_engine/qed.cc:301 (gdb) f 9 #9 0x000000000041c4c5 in main (argc=, argv=) at controller/src/query_engine/qed.cc:301 301 controller/src/query_engine/qed.cc: No such file or directory. (gdb) info locals module = Module::QUERY_ENGINE instance_id = {static npos = , _M_dataplus = {> = {<__gnu_cxx::new_allocator> = {}, }, _M_p = 0xd7ee48 "0"}} cassandra_servers = {, std::allocator >, std::allocator, std::allocator > > >> = { _M_impl = {, std::allocator > >> = {<__gnu_cxx::new_allocator, std::allocator > >> = {}, }, _M_start = 0xda1910, _M_finish = 0xda1928, _M_end_of_storage = 0xda1928}}, } cassandra_ips = {, std::allocator >, std::allocator, std::allocator > > >> = { _M_impl = {, std::allocator > >> = {<__gnu_cxx::new_allocator, std::allocator > >> = {}, }, _M_start = 0xdacfa0, _M_finish = 0xdacfb8, _M_end_of_storage = 0xdacfc0}}, } qe = {px = 0x0} options = {config_file_ = {, std::allocator >, std::allocator, std::allocator > > >> = { _M_impl = {, std::allocator > >> = {<__gnu_cxx::new_allocator, std::allocator > >> = {}, }, _M_start = 0xd653e0, _M_finish = 0xd653e8, _M_end_of_storage = 0xd653e8}}, }, discovery_server_ = {static npos = , _M_dataplus = {> = {<__gnu_cxx::new_allocator> = {}, }, _M_p = 0xa1dbf8 ""}}, discovery_port_ = 5998, redis_server_ = {static npos = , _M_dataplus = {> = {<__gnu_cxx::new_allocator> = {}, }, _M_p = 0xda8648 "127.0.0.1"}}, redis_port_ = 6379, redis_password_ = {static npos = , _M_dataplus = {> = {<__gnu_cxx::new_allocator> = {}, }, _M_p = 0xa1dbf8 ""}}, hostname_ = {static npos = , _M_dataplus = {> = {<__gnu_cxx::new_allocator> = {}, }, _M_p = 0xd74558 "nodeg20"}}, host_ip_ = {static npos = , _M_dataplus = {> = {<__gnu_cxx::new_allocator> = {}, }, _M_p = 0xda8608 "$__contrail_host_ip__"}}, http_server_port_ = 8091, log_category_ = {static npos = , _M_dataplus = {> = {<__gnu_cxx::new_allocator> = {}, }, _M_p = 0xa1dbf8 ""}}, log_property_file_ = {static npos = , _M_dataplus = {> = {<__gnu_cxx::new_allocator> = {}, }, _M_p = 0xa1dbf8 ""}}, log_disable_ = false, log_file_ = {static npos = , _M_dataplus = {> = {<__gnu_cxx::new_allocator> = {}, }, _M_p = 0xda8888 "/var/log/contrail/contrail-query-engine.log"}}, log_files_count_ = 10, log_file_size_ = 1048576, log_level_ = { static npos = , _M_dataplus = {> = {<__gnu_cxx::new_allocator> = {}, }, _M_p = 0xda8858 "SYS_NOTICE"}}, log_local_ = true, use_syslog_ = false, syslog_facility_ = { static npos = , _M_dataplus = {> = {<__gnu_cxx::new_allocator> = {}, }, _M_p = 0xda0108 "LOG_LOCAL0"}}, start_time_ = 0, max_tasks_ = 0, max_slice_ = 100, test_mode_ = false, analytics_data_ttl_ = 0, send_ratelimit_ = 0, cassandra_server_list_ = {, std::allocator >, std::allocator, std::allocator > > >> = { _M_impl = {, std::allocator > >> = {<__gnu_cxx::new_allocator, std::allocator > >> = {}, }, _M_start = 0xda8700, _M_finish = 0xda8718, _M_end_of_storage = 0xda8720}}, }, collector_server_list_ = {, std::allocator >, std::allocator, std::allocator > > >> = { _M_impl = {, std::allocator > >> = {<__gnu_cxx::new_allocator, std::allocator > >> = {}, }, _M_start = 0xda8390, _M_finish = 0xda8398, _M_end_of_storage = 0xda8398}}, }, config_file_options_ = {static m_default_line_length = 80, m_caption = {static npos = , _M_dataplus = {> = {<__gnu_cxx::new_allocator> = {}, }, _M_p = 0xa1dbf8 ""}}, m_line_length = 80, m_min_description_length = 40, m_options = {, std::allocator > >> = { _M_impl = { >> = {<__gnu_cxx::new_allocator >> = {}, }, _M_start = 0xda2760, _M_finish = 0xda2920, _M_end_of_storage = 0xda2960}}, }, belong_to_group = { >> = { _M_impl = {> = {<__gnu_cxx::new_allocator> = {}, }, _M_start = { = {> = {}, _M_p = 0xda2520, _M_offset = 0}, }, _M_finish = { = {> = {}, _M_p = 0xda2520, _M_offset = 28}, }, _M_end_of_storage = 0xda2528}}, }, groups = {, std::allocator > >> = { _M_impl = { >> = {<__gnu_cxx::new_allocator >> = {}, }, _M_start = 0xda2650, _M_finish = 0xda2670, _M_end_of_storage = 0xda2670}}, }}, cassandra_user_ = {static npos = , _M_dataplus = {> = {<__gnu_cxx::new_allocator> = {}, }, _M_p = 0xa1dbf8 ""}}, cassandra_password_ = {static npos = , _M_dataplus = {> = {<__gnu_cxx::new_allocator> = {}, }, _M_p = 0xa1dbf8 ""}}} module_name = {static npos = , _M_dataplus = {> = {<__gnu_cxx::new_allocator> = {}, }, _M_p = 0xd7b308 "contrail-query-engine"}} csf = {, std::allocator >, unsigned char, boost::function >)> >> = { = {vtable = 0x0, functor = { obj_ptr = 0x7f84ecf6d060, type = {type = 0x7f84ecf6d060, const_qualified = 128, volatile_qualified = 10}, func_ptr = 0x7f84ecf6d060, bound_memfunc_ptr = {memfunc_ptr = (void (boost::detail::function::X::*)(boost::detail::function::X * const, int)) 0x7f84ecf6d060, this adjustment 14158464, obj_ptr = 0xd80aac}, obj_ref = {obj_ptr = 0x7f84ecf6d060, is_const_qualified = 128, is_volatile_qualified = 10}, data = 96 '`'}}, static args = , static arity = }, } use_cql = true log_property_file = {static npos = , _M_dataplus = {> = {<__gnu_cxx::new_allocator> = {}, }, _M_p = 0xa1dbf8 ""}} ds_client = 0x0 dss_ep = {impl_ = {data_ = {base = {sa_family = 2, sa_data = '\000' }, v4 = {sin_family = 2, sin_port = 0, sin_addr = {s_addr = 0}, sin_zero = "\000\000\000\000\000\000\000"}, v6 = {sin6_family = 2, sin6_port = 0, sin6_flowinfo = 0, sin6_addr = { __in6_u = {__u6_addr8 = '\000' , __u6_addr16 = {0, 0, 0, 0, 0, 0, 0, 0}, __u6_addr32 = {0, 0, 0, 0}}}, sin6_scope_id = 0}}}} max_tasks = 8 node_type = success = cassandra_ports = { >> = {_M_impl = {> = {<__gnu_cxx::new_allocator> = {}, }, _M_start = 0xdacf50, _M_finish = 0xdacf5c, _M_end_of_storage = 0xdacf60}}, } css = evm = {io_service_ = { = {}, service_registry_ = 0xd80a30, impl_ = @0xd74490}, shutdown_ = false, mutex_ = {flag = 0 '\000', static is_rw_mutex = false, static is_recursive_mutex = false, static is_fair_mutex = false}} error = {m_val = 0, m_cat = } (gdb) p qe $1 = {px = 0x0} (gdb) bt #0 0x00007f84ecbe4cc9 in raise () from /lib/x86_64-linux-gnu/libc.so.6 #1 0x00007f84ecbe80d8 in abort () from /lib/x86_64-linux-gnu/libc.so.6 #2 0x00007f84ecbddb86 in ?? () from /lib/x86_64-linux-gnu/libc.so.6 #3 0x00007f84ecbddc32 in __assert_fail () from /lib/x86_64-linux-gnu/libc.so.6 #4 0x0000000000418518 in cass::cql::CqlIf::CqlIfImpl::IsTableStatic (table=..., this=) at controller/src/database/cassandra/cql/cql_if.cc:878 #5 0x000000000053c40f in IsTableStatic (table=..., this=) at controller/src/database/cassandra/cql/cql_if.cc:918 #6 SelectFromTableSync (consistency=CASS_CONSISTENCY_ONE, out=, rkey=..., cfname=..., this=) at controller/src/database/cassandra/cql/cql_if.cc:910 #7 cass::cql::CqlIf::Db_GetRow (this=, out=, cfname=..., rowkey=...) at controller/src/database/cassandra/cql/cql_if.cc:1159 #8 0x00000000004a0ffb in QueryEngine::QueryEngine (this=0xdad0a0, evm=, cassandra_ips=..., cassandra_ports=..., redis_ip=..., redis_port=, redis_password=..., max_tasks=8, max_slice=100, cassandra_user=..., cassandra_password=..., use_cql=true) at controller/src/query_engine/query.cc:1268 #9 0x000000000041c4c5 in main (argc=, argv=) at controller/src/query_engine/qed.cc:301 (gdb) f 8 #8 0x00000000004a0ffb in QueryEngine::QueryEngine (this=0xdad0a0, evm=, cassandra_ips=..., cassandra_ports=..., redis_ip=..., redis_port=, redis_password=..., max_tasks=8, max_slice=100, cassandra_user=..., cassandra_password=..., use_cql=true) at controller/src/query_engine/query.cc:1268 1268 controller/src/query_engine/query.cc: No such file or directory. (gdb) p dbif_ $2 = {px = 0xdaffd0, pn = {pi_ = 0xdb0fc0}} (gdb) p dbif_.px $3 = (boost::shared_ptr::element_type *) 0xdaffd0 (gdb) p *dbif_.px $4 = {_vptr.GenDbIf = 0x6f39d0 } (gdb) set print object on (gdb) p *dbif_.px $5 = (cass::cql::CqlIf) { = {_vptr.GenDbIf = 0x6f39d0 }, impl_ = 0xdafff0} (gdb) p *dbif_.px->impl_ There is no member or method named impl_. (gdb) p *dbif_.px.impl_ There is no member or method named impl_. (gdb) p *dbif_.px->impl_ There is no member or method named impl_. (gdb) p dbif_.px->impl_ There is no member or method named impl_. (gdb) p dbif_.px $6 = (boost::shared_ptr::element_type *) 0xdaffd0 (gdb) p *dbif_.px $7 = (cass::cql::CqlIf) { = {_vptr.GenDbIf = 0x6f39d0 }, impl_ = 0xdafff0} (gdb) p *dbif_.px->impl_ There is no member or method named impl_. (gdb) p dbif_.px->impl_ There is no member or method named impl_. (gdb) p dbif_.px.impl_ There is no member or method named impl_. (gdb) p (cass::cql::CqlIf::CqlIfImpl *)0xdafff0 $8 = (cass::cql::CqlIf::CqlIfImpl *) 0xdafff0 (gdb) p *(cass::cql::CqlIf::CqlIfImpl *)0xdafff0 $9 = (cass::cql::CqlIf::CqlIfImpl) {_vptr.CqlIfImpl = 0x6f39b0 , static kQCreateKeyspaceIfNotExists = 0x6f07c0 "CREATE KEYSPACE IF NOT EXISTS \"%s\" WITH replication = { 'class' : 'SimpleStrategy', 'replication_factor' : %s }", static kQUseKeyspace = 0x6f049a "USE \"%s\"", static kTaskName = 0x6f048a "CqlIfImpl::Task", static kTaskInstance = -1, static kReconnectInterval = 5000, evm_ = 0x7fff6082cc60, cluster_ = {> = {px = 0xdb0080, pn = {pi_ = 0xdabfa0}}, }, session_ = {> = {px = 0xdb0240, pn = { pi_ = 0xdb0bf0}}, }, session_state_ = {> = {my_storage = {my_value = cass::cql::CqlIf::CqlIfImpl::SessionState::CONNECTED}}, }, reconnect_timer_ = 0xdb0cd0, connect_cb_ = {> = { = {vtable = 0x0, functor = {obj_ptr = 0x0, type = {type = 0x0, const_qualified = false, volatile_qualified = false}, func_ptr = 0x0, bound_memfunc_ptr = {memfunc_ptr = NULL, obj_ptr = 0x0}, obj_ref = {obj_ptr = 0x0, is_const_qualified = false, is_volatile_qualified = false}, data = 0 '\000'}}, > = {}, static args = , static arity = }, }, disconnect_cb_ = {> = { = {vtable = 0x0, functor = {obj_ptr = 0x0, type = {type = 0x0, const_qualified = false, volatile_qualified = false}, func_ptr = 0x0, bound_memfunc_ptr = {memfunc_ptr = NULL, obj_ptr = 0x0}, obj_ref = {obj_ptr = 0x0, is_const_qualified = false, is_volatile_qualified = false}, data = 0 '\000'}}, > = {}, static args = , static arity = }, }, keyspace_ = {static npos = , _M_dataplus = {> = {<__gnu_cxx::new_allocator> = {}, }, _M_p = 0xd63968 "ContrailAnalyticsCql"}}} (gdb) q root@a7s9:/var/tmp# gdb /var/tmp/qed /var/tmp/nodeg20_core.contrail-query-.23027.nodeg20.1450680160 GNU gdb (Ubuntu 7.7-0ubuntu3) 7.7 Copyright (C) 2014 Free Software Foundation, Inc. License GPLv3+: GNU GPL version 3 or later This is free software: you are free to change and redistribute it. There is NO WARRANTY, to the extent permitted by law. Type "show copying" and "show warranty" for details. This GDB was configured as "x86_64-linux-gnu". Type "show configuration" for configuration details. For bug reporting instructions, please see: . Find the GDB manual and other documentation resources online at: . For help, type "help". Type "apropos word" to search for commands related to "word"... Reading symbols from /var/tmp/qed...done. warning: core file may not match specified executable file. [New LWP 23027] [New LWP 23028] [New LWP 23038] [New LWP 23029] [New LWP 23031] [New LWP 23032] [New LWP 23033] [New LWP 23035] [New LWP 23045] [New LWP 23041] [New LWP 23042] [New LWP 23039] [New LWP 23044] [New LWP 23030] [New LWP 23043] [New LWP 23040] [New LWP 23037] [Thread debugging using libthread_db enabled] Using host libthread_db library "/lib/x86_64-linux-gnu/libthread_db.so.1". Core was generated by `/usr/bin/contrail-query-engine'. Program terminated with signal SIGABRT, Aborted. #0 0x00007f84ecbe4cc9 in raise () from /lib/x86_64-linux-gnu/libc.so.6 (gdb) bt #0 0x00007f84ecbe4cc9 in raise () from /lib/x86_64-linux-gnu/libc.so.6 #1 0x00007f84ecbe80d8 in abort () from /lib/x86_64-linux-gnu/libc.so.6 #2 0x00007f84ecbddb86 in ?? () from /lib/x86_64-linux-gnu/libc.so.6 #3 0x00007f84ecbddc32 in __assert_fail () from /lib/x86_64-linux-gnu/libc.so.6 #4 0x0000000000418518 in cass::cql::CqlIf::CqlIfImpl::IsTableStatic (table=..., this=) at controller/src/database/cassandra/cql/cql_if.cc:878 #5 0x000000000053c40f in IsTableStatic (table=..., this=) at controller/src/database/cassandra/cql/cql_if.cc:918 #6 SelectFromTableSync (consistency=CASS_CONSISTENCY_ONE, out=, rkey=..., cfname=..., this=) at controller/src/database/cassandra/cql/cql_if.cc:910 #7 cass::cql::CqlIf::Db_GetRow (this=, out=, cfname=..., rowkey=...) at controller/src/database/cassandra/cql/cql_if.cc:1159 #8 0x00000000004a0ffb in QueryEngine::QueryEngine (this=0xdad0a0, evm=, cassandra_ips=..., cassandra_ports=..., redis_ip=..., redis_port=, redis_password=..., max_tasks=8, max_slice=100, cassandra_user=..., cassandra_password=..., use_cql=true) at controller/src/query_engine/query.cc:1268 #9 0x000000000041c4c5 in main (argc=, argv=) at controller/src/query_engine/qed.cc:301 (gdb) f 4 #4 0x0000000000418518 in cass::cql::CqlIf::CqlIfImpl::IsTableStatic (table=..., this=) at controller/src/database/cassandra/cql/cql_if.cc:878 878 controller/src/database/cassandra/cql/cql_if.cc: No such file or directory. (gdb) p table $1 = (const std::string &) @0x7fff6082c6e0: {static npos = , _M_dataplus = {> = {<__gnu_cxx::new_allocator> = {}, }, _M_p = 0xd63f98 "SystemObjectTable"}} (gdb) bt #0 0x00007f84ecbe4cc9 in raise () from /lib/x86_64-linux-gnu/libc.so.6 #1 0x00007f84ecbe80d8 in abort () from /lib/x86_64-linux-gnu/libc.so.6 #2 0x00007f84ecbddb86 in ?? () from /lib/x86_64-linux-gnu/libc.so.6 #3 0x00007f84ecbddc32 in __assert_fail () from /lib/x86_64-linux-gnu/libc.so.6 #4 0x0000000000418518 in cass::cql::CqlIf::CqlIfImpl::IsTableStatic (table=..., this=) at controller/src/database/cassandra/cql/cql_if.cc:878 #5 0x000000000053c40f in IsTableStatic (table=..., this=) at controller/src/database/cassandra/cql/cql_if.cc:918 #6 SelectFromTableSync (consistency=CASS_CONSISTENCY_ONE, out=, rkey=..., cfname=..., this=) at controller/src/database/cassandra/cql/cql_if.cc:910 #7 cass::cql::CqlIf::Db_GetRow (this=, out=, cfname=..., rowkey=...) at controller/src/database/cassandra/cql/cql_if.cc:1159 #8 0x00000000004a0ffb in QueryEngine::QueryEngine (this=0xdad0a0, evm=, cassandra_ips=..., cassandra_ports=..., redis_ip=..., redis_port=, redis_password=..., max_tasks=8, max_slice=100, cassandra_user=..., cassandra_password=..., use_cql=true) at controller/src/query_engine/query.cc:1268 #9 0x000000000041c4c5 in main (argc=, argv=) at controller/src/query_engine/qed.cc:301 (gdb) f 9 #9 0x000000000041c4c5 in main (argc=, argv=) at controller/src/query_engine/qed.cc:301 301 controller/src/query_engine/qed.cc: No such file or directory. (gdb) p dbif_ No symbol "dbif_" in current context. (gdb) f 8 #8 0x00000000004a0ffb in QueryEngine::QueryEngine (this=0xdad0a0, evm=, cassandra_ips=..., cassandra_ports=..., redis_ip=..., redis_port=, redis_password=..., max_tasks=8, max_slice=100, cassandra_user=..., cassandra_password=..., use_cql=true) at controller/src/query_engine/query.cc:1268 1268 controller/src/query_engine/query.cc: No such file or directory. (gdb) p dbif_ $2 = {px = 0xdaffd0, pn = {pi_ = 0xdb0fc0}} (gdb) p dbif_.px $3 = (boost::shared_ptr::element_type *) 0xdaffd0 (gdb) set print element on No symbol "on" in current context. (gdb) set print elements on No symbol "on" in current context. (gdb) set print elements Argument required (integer to set it to, or "unlimited".). (gdb) set print object on (gdb) p dbif_.px $4 = (boost::shared_ptr::element_type *) 0xdaffd0 (gdb) p *dbif_.px $5 = (cass::cql::CqlIf) { = {_vptr.GenDbIf = 0x6f39d0 }, impl_ = 0xdafff0} (gdb) p *dbif_.px->impl_ There is no member or method named impl_. (gdb) p *((cass::cql::CqlIf)dbif_.px)->impl_ $6 = (boost::detail::sp_counted_impl_p) {_vptr.CqlIfImpl = 0x6c9290 +16>, static kQCreateKeyspaceIfNotExists = 0x6f07c0 "CREATE KEYSPACE IF NOT EXISTS \"%s\" WITH replication = { 'class' : 'SimpleStrategy', 'replication_factor' : %s }", static kQUseKeyspace = 0x6f049a "USE \"%s\"", static kTaskName = 0x6f048a "CqlIfImpl::Task", static kTaskInstance = -1, static kReconnectInterval = 5000, evm_ = 0x100000001, cluster_ = {> = {px = 0xdaffd0, pn = {pi_ = 0x31}}, }, session_ = {> = {px = 0xd, pn = { pi_ = 0xd}}, }, session_state_ = {> = {my_storage = {my_value = cass::cql::CqlIf::CqlIfImpl::SessionState::CONNECT_PENDING}}, }, reconnect_timer_ = 0x322e3430322e3031, connect_cb_ = {> = { = {vtable = 0x33352e3731, functor = {obj_ptr = 0x31, type = {type = 0x31, const_qualified = 168, volatile_qualified = 15}, func_ptr = 0x31, bound_memfunc_ptr = { memfunc_ptr = &virtual table offset 48, this adjustment 14356392, obj_ptr = 0xdb0ff8}, obj_ref = {obj_ptr = 0x31, is_const_qualified = 168, is_volatile_qualified = 15}, data = 49 '1'}}, > = {}, static args = , static arity = }, }, disconnect_cb_ = {> = { = {vtable = 0xdb1058, functor = {obj_ptr = 0x0, type = {type = 0x0, const_qualified = false, volatile_qualified = false}, func_ptr = 0x0, bound_memfunc_ptr = {memfunc_ptr = NULL, obj_ptr = 0x31}, obj_ref = {obj_ptr = 0x0, is_const_qualified = false, is_volatile_qualified = false}, data = 0 '\000'}}, > = {}, static args = , static arity = }, }, keyspace_ = {static npos = , _M_dataplus = {> = {<__gnu_cxx::new_allocator> = {}, }, _M_p = 0xd }}} (gdb) p *((cass::cql::CqlIf::CqlIfImpl)0xdafff0) Invalid cast. (gdb) p *(cass::cql::CqlIf::CqlIfImpl)0xdafff0 Invalid cast. (gdb) p (cass::cql::CqlIf::CqlIfImpl)0xdafff0 Invalid cast. (gdb) p (cass::cql::CqlIf::CqlIfImpl *)0xdafff0 $7 = (cass::cql::CqlIf::CqlIfImpl *) 0xdafff0 (gdb) p *(cass::cql::CqlIf::CqlIfImpl *)0xdafff0 $8 = (cass::cql::CqlIf::CqlIfImpl) {_vptr.CqlIfImpl = 0x6f39b0 , static kQCreateKeyspaceIfNotExists = 0x6f07c0 "CREATE KEYSPACE IF NOT EXISTS \"%s\" WITH replication = { 'class' : 'SimpleStrategy', 'replication_factor' : %s }", static kQUseKeyspace = 0x6f049a "USE \"%s\"", static kTaskName = 0x6f048a "CqlIfImpl::Task", static kTaskInstance = -1, static kReconnectInterval = 5000, evm_ = 0x7fff6082cc60, cluster_ = {> = {px = 0xdb0080, pn = {pi_ = 0xdabfa0}}, }, session_ = {> = {px = 0xdb0240, pn = { pi_ = 0xdb0bf0}}, }, session_state_ = {> = {my_storage = {my_value = cass::cql::CqlIf::CqlIfImpl::SessionState::CONNECTED}}, }, reconnect_timer_ = 0xdb0cd0, connect_cb_ = {> = { = {vtable = 0x0, functor = {obj_ptr = 0x0, type = {type = 0x0, const_qualified = false, volatile_qualified = false}, func_ptr = 0x0, bound_memfunc_ptr = { memfunc_ptr = NULL, obj_ptr = 0x0}, obj_ref = {obj_ptr = 0x0, is_const_qualified = false, is_volatile_qualified = false}, data = 0 '\000'}}, > = {}, static args = , static arity = }, }, disconnect_cb_ = {> = { = {vtable = 0x0, functor = {obj_ptr = 0x0, type = {type = 0x0, const_qualified = false, volatile_qualified = false}, func_ptr = 0x0, bound_memfunc_ptr = {memfunc_ptr = NULL, obj_ptr = 0x0}, obj_ref = {obj_ptr = 0x0, is_const_qualified = false, is_volatile_qualified = false}, data = 0 '\000'}}, > = {}, static args = , static arity = }, }, keyspace_ = {static npos = , _M_dataplus = {> = {<__gnu_cxx::new_allocator> = {}, }, _M_p = 0xd63968 "ContrailAnalyticsCql"}}} (gdb) p *((cass::cql::CqlIf::CqlIfImpl*)dbif_.px)->impl_ $9 = (cass::cql::CqlIf::CqlIfImpl) {_vptr.CqlIfImpl = 0x6f39b0 , static kQCreateKeyspaceIfNotExists = 0x6f07c0 "CREATE KEYSPACE IF NOT EXISTS \"%s\" WITH replication = { 'class' : 'SimpleStrategy', 'replication_factor' : %s }", static kQUseKeyspace = 0x6f049a "USE \"%s\"", static kTaskName = 0x6f048a "CqlIfImpl::Task", static kTaskInstance = -1, static kReconnectInterval = 5000, evm_ = 0x7fff6082cc60, cluster_ = {> = {px = 0xdb0080, pn = {pi_ = 0xdabfa0}}, }, session_ = {> = {px = 0xdb0240, pn = { pi_ = 0xdb0bf0}}, }, session_state_ = {> = {my_storage = {my_value = cass::cql::CqlIf::CqlIfImpl::SessionState::CONNECTED}}, }, reconnect_timer_ = 0xdb0cd0, connect_cb_ = {> = { = {vtable = 0x0, functor = {obj_ptr = 0x0, type = {type = 0x0, const_qualified = false, volatile_qualified = false}, func_ptr = 0x0, bound_memfunc_ptr = { memfunc_ptr = NULL, obj_ptr = 0x0}, obj_ref = {obj_ptr = 0x0, is_const_qualified = false, is_volatile_qualified = false}, data = 0 '\000'}}, > = {}, static args = , static arity = }, }, disconnect_cb_ = {> = { = {vtable = 0x0, functor = {obj_ptr = 0x0, type = {type = 0x0, const_qualified = false, volatile_qualified = false}, func_ptr = 0x0, bound_memfunc_ptr = {memfunc_ptr = NULL, obj_ptr = 0x0}, obj_ref = {obj_ptr = 0x0, is_const_qualified = false, is_volatile_qualified = false}, data = 0 '\000'}}, > = {}, static args = , static arity = }, }, keyspace_ = {static npos = , _M_dataplus = {> = {<__gnu_cxx::new_allocator> = {}, }, _M_p = 0xd63968 "ContrailAnalyticsCql"}}} (gdb)