2.21.1 #15.
core file is in 10.84.5.112:/auto/cs-shared/bugs/1531712/core.contrail-collec.11692.csol1-node3.1452045454
in traffic test ( high throughput/cps ) contrail-controller crash is seen with following backtrace.
(gdb) bt
#0 0x00007fb137625cc9 in __GI_raise (sig=sig@entry=6) at ../nptl/sysdeps/unix/sysv/linux/raise.c:56
#1 0x00007fb1376290d8 in __GI_abort () at abort.c:89
#2 0x00007fb13761eb86 in __assert_fail_base (
fmt=0x7fb13776f830 "%s%s%s:%u: %s%sAssertion `%s' failed.\n%n",
assertion=assertion@entry=0x76b92c "reply->type != 6",
file=file@entry=0x76ba00 "controller/src/analytics/OpServerProxy.cc", line=line@entry=357,
function=function@entry=0x76c9e0 <OpServerProxy::OpServerImpl::processorCallbackProcess(redisAsyncContext const*, void*, void*)::__PRETTY_FUNCTION__> "void OpServerProxy::OpServerImpl::processorCallbackProcess(const redisAsyncContext*, void*, void*)") at assert.c:92
#3 0x00007fb13761ec32 in __GI___assert_fail (assertion=0x76b92c "reply->type != 6",
file=0x76ba00 "controller/src/analytics/OpServerProxy.cc", line=357,
function=0x76c9e0 <OpServerProxy::OpServerImpl::processorCallbackProcess(redisAsyncContext const*, void*, void*)::__PRETTY_FUNCTION__> "void OpServerProxy::OpServerImpl::processorCallbackProcess(const redisAsyncContext*, void*, void*)") at assert.c:101
#4 0x000000000052dbe7 in OpServerProxy::OpServerImpl::processorCallbackProcess (
this=<optimized out>, c=<optimized out>, r=<optimized out>, privdata=<optimized out>)
at controller/src/analytics/OpServerProxy.cc:357
#5 0x00000000004dad3a in operator() (a2=0x0, a1=0xe14e60, a0=0xcffcd0, this=0x7fff6f59d5d0)
at /usr/include/boost/function/function_template.hpp:767
#6 RedisAsyncConnection::RAC_AsyncCmdCallback (c=0xcffcd0, r=0xe14e60, privdata=0x0)
at controller/src/analytics/redis_connection.cc:239
#7 0x000000000074c853 in __redisRunCallback (cb=0x7fff6f59d6f0, cb=0x7fff6f59d6f0,
reply=<optimized out>, ac=0xcffcd0) at build/third_party/hiredis/src/async.c:219
#8 redisProcessCallbacks (ac=0xcffcd0) at build/third_party/hiredis/src/async.c:417
#9 0x000000000074dc79 in redisBoostClient::handle_read (this=0xdb0ec0, ec=...)
at build/third_party/hiredis/hiredis-boostasio-adapter/boostasio.cpp:62
---Type <return> to continue, or q <return> to quit---
#10 0x000000000074e344 in call<boost::shared_ptr<redisBoostClient>, boost::system::error_code> (
b1=<synthetic pointer>, u=..., this=<optimized out>)
at /usr/include/boost/bind/mem_fn_template.hpp:156
#11 operator()<boost::shared_ptr<redisBoostClient> > (a1=..., u=..., this=<optimized out>)
at /usr/include/boost/bind/mem_fn_template.hpp:171
#12 operator()<boost::_mfi::mf1<void, redisBoostClient, boost::system::error_code>, boost::_bi::list2<const boost::system::error_code&, long unsigned int const&> > (a=<synthetic pointer>, f=...,
this=<optimized out>) at /usr/include/boost/bind/bind.hpp:313
#13 operator()<boost::system::error_code, long unsigned int> (a2=<optimized out>, a1=...,
this=<optimized out>) at /usr/include/boost/bind/bind_template.hpp:102
#14 operator() (this=<optimized out>) at /usr/include/boost/asio/detail/bind_handler.hpp:127
#15 asio_handler_invoke<boost::asio::detail::binder2<boost::_bi::bind_t<void, boost::_mfi::mf1<void, redisBoostClient, boost::system::error_code>, boost::_bi::list2<boost::_bi::value<boost::shared_ptr<redisBoostClient> >, boost::arg<1> (*)()> >, boost::system::error_code, unsigned long> > (function=...)
at /usr/include/boost/asio/handler_invoke_hook.hpp:64
#16 invoke<boost::asio::detail::binder2<boost::_bi::bind_t<void, boost::_mfi::mf1<void, redisBoostClient, boost::system::error_code>, boost::_bi::list2<boost::_bi::value<boost::shared_ptr<redisBoostClient> >, boost::arg<1> (*)()> >, boost::system::error_code, unsigned long>, boost::_bi::bind_t<void, boost::_mfi::mf1<void, redisBoostClient, boost::system::error_code>, boost::_bi::list2<boost::_bi::value<boost::shared_ptr<redisBoostClient> >, boost::arg<1> (*)()> > > (context=..., function=...)
at /usr/include/boost/asio/detail/handler_invoke_helpers.hpp:37
#17 boost::asio::detail::reactive_null_buffers_op<boost::_bi::bind_t<void, boost::_mfi::mf1<void, redisBoostClient, boost::system::error_code>, boost::_bi::list2<boost::_bi::value<boost::shared_ptr<redisBoostClient> >, boost::arg<1> (*)()> > >::do_complete (owner=<optimized out>, base=<optimized out>)
at /usr/include/boost/asio/detail/reactive_null_buffers_op.hpp:75
#18 0x000000000058d37f in complete (bytes_transferred=0, ec=..., owner=..., this=<optimized out>)
---Type <return> to continue, or q <return> to quit---
at /usr/include/boost/asio/detail/task_io_service_operation.hpp:37
#19 boost::asio::detail::epoll_reactor::descriptor_state::do_complete (owner=0xce3540,
base=0xcfefc0, ec=..., bytes_transferred=<optimized out>)
at /usr/include/boost/asio/detail/impl/epoll_reactor.ipp:651
#20 0x0000000000562777 in complete (bytes_transferred=5, ec=..., owner=..., this=0xcfefc0)
at /usr/include/boost/asio/detail/task_io_service_operation.hpp:37
#21 do_run_one (ec=..., this_thread=..., lock=..., this=0xce3540)
at /usr/include/boost/asio/detail/impl/task_io_service.ipp:384
#22 boost::asio::detail::task_io_service::run (this=0xce3540, ec=...)
at /usr/include/boost/asio/detail/impl/task_io_service.ipp:153
#23 0x000000000057fa58 in run (this=0xccda00, ec=...)
at /usr/include/boost/asio/impl/io_service.ipp:66
#24 EventManager::Run (this=0xccda00) at controller/src/io/event_manager.cc:32
#25 0x00000000004218cb in main (argc=<optimized out>, argv=<optimized out>)
at controller/src/analytics/main.cc:412
(gdb)
From the code, this is when this assert can happen:
// If redis returns error for async request, then perhaps it
// is busy executing a script and it has reached the maximum
// execution time limit.
This timeout for script execution is set to 15 seconds.