Juniper Openstack

Comment 3 for bug 1456853

Revision history for this message

Megh Bhatt (meghb) wrote on 2015-05-20:

Provided access to the box.

root@ubuntu-cn:~# gdb /var/tmp/vizd /var/crashes/core.contrail-collec.2768.ubuntu-cn.1431399756
GNU gdb (Ubuntu 7.7-0ubuntu3) 7.7
Copyright (C) 2014 Free Software Foundation, Inc.
License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html>
This is free software: you are free to change and redistribute it.
There is NO WARRANTY, to the extent permitted by law. Type "show copying"
and "show warranty" for details.
This GDB was configured as "x86_64-linux-gnu".
Type "show configuration" for configuration details.
For bug reporting instructions, please see:
<http://www.gnu.org/software/gdb/bugs/>.
Find the GDB manual and other documentation resources online at:
<http://www.gnu.org/software/gdb/documentation/>.
For help, type "help".
Type "apropos word" to search for commands related to "word"...
Reading symbols from /var/tmp/vizd...done.

warning: core file may not match specified executable file.
[New LWP 2768]
[New LWP 3217]
[New LWP 3228]
[New LWP 3218]
[New LWP 3229]
[New LWP 3219]
[New LWP 3227]
[New LWP 3220]
[New LWP 3226]
[Thread debugging using libthread_db enabled]
Using host libthread_db library "/lib/x86_64-linux-gnu/libthread_db.so.1".
Core was generated by `/usr/bin/contrail-collector'.
Program terminated with signal SIGABRT, Aborted.
#0 0x00007f55c8335cc9 in raise () from /lib/x86_64-linux-gnu/libc.so.6
(gdb) bt
#0 0x00007f55c8335cc9 in raise () from /lib/x86_64-linux-gnu/libc.so.6
#1 0x00007f55c83390d8 in abort () from /lib/x86_64-linux-gnu/libc.so.6
#2 0x00007f55c832eb86 in ?? () from /lib/x86_64-linux-gnu/libc.so.6
#3 0x00007f55c832ec32 in __assert_fail () from /lib/x86_64-linux-gnu/libc.so.6
#4 0x0000000000529aea in OpServerProxy::OpServerImpl::toConnectCallbackProcess (this=<optimized out>, c=<optimized out>, r=<optimized out>, privdata=<optimized out>)
    at controller/src/analytics/OpServerProxy.cc:242
#5 0x00000000004d707a in operator() (a2=0x0, a1=0x2947050, a0=0x294b030, this=0x7fffe1727120) at /usr/include/boost/function/function_template.hpp:767
#6 RedisAsyncConnection::RAC_AsyncCmdCallback (c=0x294b030, r=0x2947050, privdata=0x0) at controller/src/analytics/redis_connection.cc:239
#7 0x00000000007423f3 in __redisRunCallback (cb=0x7fffe1727240, cb=0x7fffe1727240, reply=<optimized out>, ac=0x294b030) at build/third_party/hiredis/src/async.c:219
#8 redisProcessCallbacks (ac=0x294b030) at build/third_party/hiredis/src/async.c:417
#9 0x0000000000743819 in redisBoostClient::handle_read (this=0x294afe0, ec=...) at build/third_party/hiredis/hiredis-boostasio-adapter/boostasio.cpp:62
#10 0x0000000000743ee4 in call<boost::shared_ptr<redisBoostClient>, boost::system::error_code> (b1=<synthetic pointer>, u=..., this=<optimized out>) at /usr/include/boost/bind/mem_fn_template.hpp:156
#11 operator()<boost::shared_ptr<redisBoostClient> > (a1=..., u=..., this=<optimized out>) at /usr/include/boost/bind/mem_fn_template.hpp:171
#12 operator()<boost::_mfi::mf1<void, redisBoostClient, boost::system::error_code>, boost::_bi::list2<const boost::system::error_code&, long unsigned int const&> > (a=<synthetic pointer>, f=...,
    this=<optimized out>) at /usr/include/boost/bind/bind.hpp:313
#13 operator()<boost::system::error_code, long unsigned int> (a2=<optimized out>, a1=..., this=<optimized out>) at /usr/include/boost/bind/bind_template.hpp:102
#14 operator() (this=<optimized out>) at /usr/include/boost/asio/detail/bind_handler.hpp:127
#15 asio_handler_invoke<boost::asio::detail::binder2<boost::_bi::bind_t<void, boost::_mfi::mf1<void, redisBoostClient, boost::system::error_code>, boost::_bi::list2<boost::_bi::value<boost::shared_ptr<redisBoostClient> >, boost::arg<1> (*)()> >, boost::system::error_code, unsigned long> > (function=...) at /usr/include/boost/asio/handler_invoke_hook.hpp:64
#16 invoke<boost::asio::detail::binder2<boost::_bi::bind_t<void, boost::_mfi::mf1<void, redisBoostClient, boost::system::error_code>, boost::_bi::list2<boost::_bi::value<boost::shared_ptr<redisBoostClient> >, boost::arg<1> (*)()> >, boost::system::error_code, unsigned long>, boost::_bi::bind_t<void, boost::_mfi::mf1<void, redisBoostClient, boost::system::error_code>, boost::_bi::list2<boost::_bi::value<boost::shared_ptr<redisBoostClient> >, boost::arg<1> (*)()> > > (context=..., function=...) at /usr/include/boost/asio/detail/handler_invoke_helpers.hpp:37
#17 boost::asio::detail::reactive_null_buffers_op<boost::_bi::bind_t<void, boost::_mfi::mf1<void, redisBoostClient, boost::system::error_code>, boost::_bi::list2<boost::_bi::value<boost::shared_ptr<redisBoostClient> >, boost::arg<1> (*)()> > >::do_complete (owner=<optimized out>, base=<optimized out>) at /usr/include/boost/asio/detail/reactive_null_buffers_op.hpp:75
#18 0x0000000000584a6f in complete (bytes_transferred=0, ec=..., owner=..., this=<optimized out>) at /usr/include/boost/asio/detail/task_io_service_operation.hpp:37
#19 boost::asio::detail::epoll_reactor::descriptor_state::do_complete (owner=0x28d91f0, base=0x28f37d0, ec=..., bytes_transferred=<optimized out>)
    at /usr/include/boost/asio/detail/impl/epoll_reactor.ipp:651
#20 0x000000000055bc27 in complete (bytes_transferred=5, ec=..., owner=..., this=0x28f37d0) at /usr/include/boost/asio/detail/task_io_service_operation.hpp:37
#21 do_run_one (ec=..., this_thread=..., lock=..., this=0x28d91f0) at /usr/include/boost/asio/detail/impl/task_io_service.ipp:384
#22 boost::asio::detail::task_io_service::run (this=0x28d91f0, ec=...) at /usr/include/boost/asio/detail/impl/task_io_service.ipp:153
#23 0x00000000005772b8 in run (this=0x28bbf50, ec=...) at /usr/include/boost/asio/impl/io_service.ipp:66
#24 EventManager::Run (this=0x28bbf50) at controller/src/io/event_manager.cc:32
#25 0x0000000000421062 in main (argc=<optimized out>, argv=<optimized out>) at controller/src/analytics/main.cc:407
(gdb) f 4
#4 0x0000000000529aea in OpServerProxy::OpServerImpl::toConnectCallbackProcess (this=<optimized out>, c=<optimized out>, r=<optimized out>, privdata=<optimized out>)
    at controller/src/analytics/OpServerProxy.cc:242
242 controller/src/analytics/OpServerProxy.cc: No such file or directory.
(gdb) p reply
$1 = <optimized out>
(gdb) p reply.type
$2 = <optimized out>
(gdb) p r
$3 = <optimized out>
(gdb) 6
Undefined command: "6". Try "help".
(gdb) f 6
#6 RedisAsyncConnection::RAC_AsyncCmdCallback (c=0x294b030, r=0x2947050, privdata=0x0) at controller/src/analytics/redis_connection.cc:239
239 controller/src/analytics/redis_connection.cc: No such file or directory.
(gdb) p r
$4 = (void *) 0x2947050
(gdb) p (redisReply*)r
$5 = (redisReply *) 0x2947050
(gdb) p *$5
$6 = {type = 6, integer = 0, len = 46, str = 0x28f2dc0 "LOADING Redis is loading the dataset in memory", elements = 0, element = 0x0}
(gdb)

Looks like redis is initializing and hence returning the above error leading to assert. We should retry. Also need to check why is redis loading the database in memory

Provided access to the box.

root@ubuntu-cn:~# gdb /var/tmp/vizd /var/crashes/core.contrail-collec.2768.ubuntu-cn.1431399756
GNU gdb (Ubuntu 7.7-0ubuntu3) 7.7
Copyright (C) 2014 Free Software Foundation, Inc.
License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html>
This is free software: you are free to change and redistribute it.
There is NO WARRANTY, to the extent permitted by law.  Type "show copying"
and "show warranty" for details.
This GDB was configured as "x86_64-linux-gnu".
Type "show configuration" for configuration details.
For bug reporting instructions, please see:
<http://www.gnu.org/software/gdb/bugs/>.
Find the GDB manual and other documentation resources online at:
<http://www.gnu.org/software/gdb/documentation/>.
For help, type "help".
Type "apropos word" to search for commands related to "word"...
Reading symbols from /var/tmp/vizd...done.

warning: core file may not match specified executable file.
[New LWP 2768]
[New LWP 3217]
[New LWP 3228]
[New LWP 3218]
[New LWP 3229]
[New LWP 3219]
[New LWP 3227]
[New LWP 3220]
[New LWP 3226]
[Thread debugging using libthread_db enabled]
Using host libthread_db library "/lib/x86_64-linux-gnu/libthread_db.so.1".
Core was generated by `/usr/bin/contrail-collector'.
Program terminated with signal SIGABRT, Aborted.
#0  0x00007f55c8335cc9 in raise () from /lib/x86_64-linux-gnu/libc.so.6
(gdb) bt
#0  0x00007f55c8335cc9 in raise () from /lib/x86_64-linux-gnu/libc.so.6
#1  0x00007f55c83390d8 in abort () from /lib/x86_64-linux-gnu/libc.so.6
#2  0x00007f55c832eb86 in ?? () from /lib/x86_64-linux-gnu/libc.so.6
#3  0x00007f55c832ec32 in __assert_fail () from /lib/x86_64-linux-gnu/libc.so.6
#4  0x0000000000529aea in OpServerProxy::OpServerImpl::toConnectCallbackProcess (this=<optimized out>, c=<optimized out>, r=<optimized out>, privdata=<optimized out>)
    at controller/src/analytics/OpServerProxy.cc:242
#5  0x00000000004d707a in operator() (a2=0x0, a1=0x2947050, a0=0x294b030, this=0x7fffe1727120) at /usr/include/boost/function/function_template.hpp:767
#6  RedisAsyncConnection::RAC_AsyncCmdCallback (c=0x294b030, r=0x2947050, privdata=0x0) at controller/src/analytics/redis_connection.cc:239
#7  0x00000000007423f3 in __redisRunCallback (cb=0x7fffe1727240, cb=0x7fffe1727240, reply=<optimized out>, ac=0x294b030) at build/third_party/hiredis/src/async.c:219
#8  redisProcessCallbacks (ac=0x294b030) at build/third_party/hiredis/src/async.c:417
#9  0x0000000000743819 in redisBoostClient::handle_read (this=0x294afe0, ec=...) at build/third_party/hiredis/hiredis-boostasio-adapter/boostasio.cpp:62
#10 0x0000000000743ee4 in call<boost::shared_ptr<redisBoostClient>, boost::system::error_code> (b1=<synthetic pointer>, u=..., this=<optimized out>) at /usr/include/boost/bind/mem_fn_template.hpp:156
#11 operator()<boost::shared_ptr<redisBoostClient> > (a1=..., u=..., this=<optimized out>) at /usr/include/boost/bind/mem_fn_template.hpp:171
#12 operator()<boost::_mfi::mf1<void, redisBoostClient, boost::system::error_code>, boost::_bi::list2<const boost::system::error_code&, long unsigned int const&> > (a=<synthetic pointer>, f=...,
    this=<optimized out>) at /usr/include/boost/bind/bind.hpp:313
#13 operator()<boost::system::error_code, long unsigned int> (a2=<optimized out>, a1=..., this=<optimized out>) at /usr/include/boost/bind/bind_template.hpp:102
#14 operator() (this=<optimized out>) at /usr/include/boost/asio/detail/bind_handler.hpp:127
#15 asio_handler_invoke<boost::asio::detail::binder2<boost::_bi::bind_t<void, boost::_mfi::mf1<void, redisBoostClient, boost::system::error_code>, boost::_bi::list2<boost::_bi::value<boost::shared_ptr<redisBoostClient> >, boost::arg<1> (*)()> >, boost::system::error_code, unsigned long> > (function=...) at /usr/include/boost/asio/handler_invoke_hook.hpp:64
#16 invoke<boost::asio::detail::binder2<boost::_bi::bind_t<void, boost::_mfi::mf1<void, redisBoostClient, boost::system::error_code>, boost::_bi::list2<boost::_bi::value<boost::shared_ptr<redisBoostClient> >, boost::arg<1> (*)()> >, boost::system::error_code, unsigned long>, boost::_bi::bind_t<void, boost::_mfi::mf1<void, redisBoostClient, boost::system::error_code>, boost::_bi::list2<boost::_bi::value<boost::shared_ptr<redisBoostClient> >, boost::arg<1> (*)()> > > (context=..., function=...) at /usr/include/boost/asio/detail/handler_invoke_helpers.hpp:37
#17 boost::asio::detail::reactive_null_buffers_op<boost::_bi::bind_t<void, boost::_mfi::mf1<void, redisBoostClient, boost::system::error_code>, boost::_bi::list2<boost::_bi::value<boost::shared_ptr<redisBoostClient> >, boost::arg<1> (*)()> > >::do_complete (owner=<optimized out>, base=<optimized out>) at /usr/include/boost/asio/detail/reactive_null_buffers_op.hpp:75
#18 0x0000000000584a6f in complete (bytes_transferred=0, ec=..., owner=..., this=<optimized out>) at /usr/include/boost/asio/detail/task_io_service_operation.hpp:37
#19 boost::asio::detail::epoll_reactor::descriptor_state::do_complete (owner=0x28d91f0, base=0x28f37d0, ec=..., bytes_transferred=<optimized out>)
    at /usr/include/boost/asio/detail/impl/epoll_reactor.ipp:651
#20 0x000000000055bc27 in complete (bytes_transferred=5, ec=..., owner=..., this=0x28f37d0) at /usr/include/boost/asio/detail/task_io_service_operation.hpp:37
#21 do_run_one (ec=..., this_thread=..., lock=..., this=0x28d91f0) at /usr/include/boost/asio/detail/impl/task_io_service.ipp:384
#22 boost::asio::detail::task_io_service::run (this=0x28d91f0, ec=...) at /usr/include/boost/asio/detail/impl/task_io_service.ipp:153
#23 0x00000000005772b8 in run (this=0x28bbf50, ec=...) at /usr/include/boost/asio/impl/io_service.ipp:66
#24 EventManager::Run (this=0x28bbf50) at controller/src/io/event_manager.cc:32
#25 0x0000000000421062 in main (argc=<optimized out>, argv=<optimized out>) at controller/src/analytics/main.cc:407
(gdb) f 4
#4  0x0000000000529aea in OpServerProxy::OpServerImpl::toConnectCallbackProcess (this=<optimized out>, c=<optimized out>, r=<optimized out>, privdata=<optimized out>)
    at controller/src/analytics/OpServerProxy.cc:242
242	controller/src/analytics/OpServerProxy.cc: No such file or directory.
(gdb) p reply
$1 = <optimized out>
(gdb) p reply.type
$2 = <optimized out>
(gdb) p r
$3 = <optimized out>
(gdb)  6
Undefined command: "6".  Try "help".
(gdb) f 6
#6  RedisAsyncConnection::RAC_AsyncCmdCallback (c=0x294b030, r=0x2947050, privdata=0x0) at controller/src/analytics/redis_connection.cc:239
239	controller/src/analytics/redis_connection.cc: No such file or directory.
(gdb) p r
$4 = (void *) 0x2947050
(gdb) p (redisReply*)r
$5 = (redisReply *) 0x2947050
(gdb) p *$5
$6 = {type = 6, integer = 0, len = 46, str = 0x28f2dc0 "LOADING Redis is loading the dataset in memory", elements = 0, element = 0x0}
(gdb)

Looks like redis is initializing and hence returning the above error leading to assert. We should retry. Also need to check why is redis loading the database in memory