| 2015-02-23 04:36:59 |
Nischal Sheth |
description |
Segmentation fault with the following backtrace when running bgp_server_test.
We ran into a similar issue earlier and worked around it by not allocating memory.
Root cause is an issue in how gperftools tries to determine the backtrace. The
problem happens because gperftools expects all code (including system libraries)
to be compiled to use stack frames (with rbp, the base pointer register). In this
case, the string library functions were not compile to enable stack frames, so they
did not push the base pointer register onto the stack. This causes a crash when
gperftools tcmalloc() attempts to retrieve the stack trace for a malloc() call.
Fix would be to install libunwind and rebuild gperftools to reflect that, so that it
doesn’t rely on the base pointer register any more.
#0 GetStackTrace (result=0x7f8c6507c8c0, max_depth=42, skip_count=0) at /build/nsheth/default/third_party/gperftools-2.1/src/stacktrace_x86-inl.h:325
#1 0x00007f8c6931aa86 in MallocHook_GetCallerStackTrace (result=0x7f8c6507ca50, max_depth=32, skip_count=<optimized out>) at /build/nsheth/default/third_party/gperftools-2.1/src/malloc_hook.cc:665
#2 0x00007f8c6930c34f in NewHook (ptr=0x416ca40, size=57) at /build/nsheth/default/third_party/gperftools-2.1/src/heap-checker.cc:575
#3 0x00007f8c6931af3b in MallocHook::InvokeNewHookSlow (p=0x416ca40, s=57) at /build/nsheth/default/third_party/gperftools-2.1/src/malloc_hook.cc:525
#4 0x00007f8c69328a27 in InvokeNewHook (s=57, p=<optimized out>) at /build/nsheth/default/third_party/gperftools-2.1/src/malloc_hook-inl.h:161
#5 tc_new (size=57) at /build/nsheth/default/third_party/gperftools-2.1/src/tcmalloc.cc:1607
#6 0x00007f8c6908ea89 in std::string::_Rep::_S_create(unsigned long, unsigned long, std::allocator<char> const&) () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6
#7 0x00007f8c69090495 in char* std::string::_S_construct<char const*>(char const*, char const*, std::allocator<char> const&, std::forward_iterator_tag) () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6
#8 0x00007f8c690905e3 in std::basic_string<char, std::char_traits<char>, std::allocator<char> >::basic_string(char const*, std::allocator<char> const&) () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6
#9 0x0000000000fcb90b in BgpServer::ConfigUpdater::BgpServerConfigUpdate (this=0x3e04298, instance_name="default-domain:default-project:ip-fabric:__default__", params=...)
at controller/src/bgp/bgp_server.cc:142
#10 0x0000000000fcb16a in BgpServer::ConfigUpdater::ProcessProtocolConfig (this=0x3e04298, protocol_config=0x40ebb30, event=BgpConfigManager::CFG_CHANGE) at controller/src/bgp/bgp_server.cc:75
#11 0x0000000000fd2a45 in boost::_mfi::mf2<void, BgpServer::ConfigUpdater, BgpProtocolConfig const*, BgpConfigManager::EventType>::operator() (this=0x40e9280, p=0x3e04298, a1=0x40ebb30,
a2=BgpConfigManager::CFG_CHANGE) at build/include/boost/bind/mem_fn_template.hpp:280
#12 0x0000000000fd21ae in boost::_bi::list3<boost::_bi::value<BgpServer::ConfigUpdater*>, boost::arg<1>, boost::arg<2> >::operator()<boost::_mfi::mf2<void, BgpServer::ConfigUpdater, BgpProtocolConfig const*, BgpConfigManager::EventType>, boost::_bi::list2<BgpProtocolConfig const*&, BgpConfigManager::EventType&> > (this=0x40e9290, f=..., a=...) at build/include/boost/bind/bind.hpp:392
#13 0x0000000000fd190e in boost::_bi::bind_t<void, boost::_mfi::mf2<void, BgpServer::ConfigUpdater, BgpProtocolConfig const*, BgpConfigManager::EventType>, boost::_bi::list3<boost::_bi::value<BgpServer::ConfigUpdater*>, boost::arg<1>, boost::arg<2> > >::operator()<BgpProtocolConfig const*, BgpConfigManager::EventType> (this=0x40e9280, a1=@0x7f8c6507d3c0: 0x40ebb30,
a2=@0x7f8c6507d3bc: BgpConfigManager::CFG_CHANGE) at build/include/boost/bind/bind_template.hpp:61
#14 0x0000000000fd101e in boost::detail::function::void_function_obj_invoker2<boost::_bi::bind_t<void, boost::_mfi::mf2<void, BgpServer::ConfigUpdater, BgpProtocolConfig const*, BgpConfigManager::EventType>, boost::_bi::list3<boost::_bi::value<BgpServer::ConfigUpdater*>, boost::arg<1>, boost::arg<2> > >, void, BgpProtocolConfig const*, BgpConfigManager::EventType>::invoke (function_obj_ptr=...,
a0=0x40ebb30, a1=BgpConfigManager::CFG_CHANGE) at build/include/boost/function/function_template.hpp:153
#15 0x0000000000eaec78 in boost::function2<void, BgpProtocolConfig const*, BgpConfigManager::EventType>::operator() (this=0x40e9278, a0=0x40ebb30, a1=BgpConfigManager::CFG_CHANGE)
at build/include/boost/function/function_template.hpp:763
#16 0x0000000000ea4ac5 in BgpConfigManager::Notify<BgpProtocolConfig> (this=0x40e9200, config=0x40ebb30, event=BgpConfigManager::CFG_CHANGE) at controller/src/bgp/bgp_config.cc:63
#17 0x0000000000eaa298 in BgpConfigManager::ProcessBgpProtocol (this=0x40e9200, delta=...) at controller/src/bgp/bgp_config.cc:1089
#18 0x0000000000eaaae1 in BgpConfigManager::ProcessBgpRouter (this=0x40e9200, delta=...) at controller/src/bgp/bgp_config.cc:1138
#19 0x0000000000eb667b in boost::_mfi::mf1<void, BgpConfigManager, IFMapConfigListener::ConfigDelta const&>::operator() (this=0x411ad00, p=0x40e9200, a1=...)
at build/include/boost/bind/mem_fn_template.hpp:165
#20 0x0000000000eb6383 in boost::_bi::list2<boost::_bi::value<BgpConfigManager*>, boost::arg<1> >::operator()<boost::_mfi::mf1<void, BgpConfigManager, IFMapConfigListener::ConfigDelta const&>, boost::_bi::list1<IFMapConfigListener::ConfigDelta const&> > (this=0x411ad10, f=..., a=...) at build/include/boost/bind/bind.hpp:313
#21 0x0000000000eb5ed6 in boost::_bi::bind_t<void, boost::_mfi::mf1<void, BgpConfigManager, IFMapConfigListener::ConfigDelta const&>, boost::_bi::list2<boost::_bi::value<BgpConfigManager*>, boost::arg<1> > >::operator()<IFMapConfigListener::ConfigDelta> (this=0x411ad00, a1=...) at build/include/boost/bind/bind_template.hpp:47
#22 0x0000000000eb5974 in boost::detail::function::void_function_obj_invoker1<boost::_bi::bind_t<void, boost::_mfi::mf1<void, BgpConfigManager, IFMapConfigListener::ConfigDelta const&>, boost::_bi::list2<boost::_bi::value<BgpConfigManager*>, boost::arg<1> > >, void, IFMapConfigListener::ConfigDelta const&>::invoke (function_obj_ptr=..., a0=...) at build/include/boost/function/function_template.hpp:153
#23 0x0000000000eb0289 in boost::function1<void, IFMapConfigListener::ConfigDelta const&>::operator() (this=0x411acf8, a0=...) at build/include/boost/function/function_template.hpp:763
#24 0x0000000000eab6ab in BgpConfigManager::ProcessChanges (this=0x40e9200, change_list=std::vector of length 3, capacity 4 = {...}) at controller/src/bgp/bgp_config.cc:1225
#25 0x0000000000eab781 in BgpConfigManager::ConfigHandler (this=0x40e9200) at controller/src/bgp/bgp_config.cc:1239
#26 0x0000000000eb6290 in boost::_mfi::mf0<bool, BgpConfigManager>::operator() (this=0x40e92c0, p=0x40e9200) at build/include/boost/bind/mem_fn_template.hpp:49
#27 0x0000000000eb5de3 in boost::_bi::list1<boost::_bi::value<BgpConfigManager*> >::operator()<bool, boost::_mfi::mf0<bool, BgpConfigManager>, boost::_bi::list0> (this=0x40e92d0, f=..., a=...)
at build/include/boost/bind/bind.hpp:243
#28 0x0000000000eb582d in boost::_bi::bind_t<bool, boost::_mfi::mf0<bool, BgpConfigManager>, boost::_bi::list1<boost::_bi::value<BgpConfigManager*> > >::operator() (this=0x40e92c0)
at build/include/boost/bind/bind_template.hpp:20
#29 0x0000000000eb50df in boost::detail::function::function_obj_invoker0<boost::_bi::bind_t<bool, boost::_mfi::mf0<bool, BgpConfigManager>, boost::_bi::list1<boost::_bi::value<BgpConfigManager*> > >, bool>::invoke (function_obj_ptr=...) at build/include/boost/function/function_template.hpp:132
#30 0x0000000000d8e3c2 in boost::function0<bool>::operator() (this=0x40e92b8) at build/include/boost/function/function_template.hpp:763
#31 0x000000000131be21 in TaskTrigger::WorkerTask::Run (this=0x404e5d0) at controller/src/base/task_trigger.cc:19
#32 0x000000000130ab4d in TaskImpl::execute (this=0x40209c0) at controller/src/base/task.cc:232
#33 0x00007f8c69a7bece in tbb::internal::custom_scheduler<tbb::internal::IntelSchedulerTraits>::local_wait_for_all (this=0x3f38d80, parent=..., child=0x3e56e40)
at /build/nsheth/default/third_party/tbb40_20111130oss/src/tbb/custom_scheduler.h:449
#34 0x00007f8c69a72e0b in tbb::internal::arena::process (this=0x3e92d00, s=...) at /build/nsheth/default/third_party/tbb40_20111130oss/src/tbb/arena.cpp:99
#35 0x00007f8c69a716f2 in tbb::internal::market::process (this=0x3e5a500, j=...) at /build/nsheth/default/third_party/tbb40_20111130oss/src/tbb/market.cpp:393
#36 0x00007f8c69a6c3ce in tbb::internal::rml::private_worker::run (this=0x3e8c480) at /build/nsheth/default/third_party/tbb40_20111130oss/src/tbb/private_server.cpp:263
#37 0x00007f8c69a6c270 in tbb::internal::rml::private_worker::thread_routine (arg=0x3e8c480) at /build/nsheth/default/third_party/tbb40_20111130oss/src/tbb/private_server.cpp:231
#38 0x00007f8c69ca5e9a in start_thread (arg=0x7f8c6507e700) at pthread_create.c:308
#39 0x00007f8c688132ed in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#40 0x0000000000000000 in ?? () |
Segmentation fault with following backtrace when running bgp_server_test.
Root cause is an issue in how gperftools tries to determine the backtrace.
The problem happens because gperftools expects all code (including system
libraries) to be compiled to use stack frames (with rbp, the base pointer
register). In this case, the string library functions were not compile to
enable stack frames, so they did not push the base pointer register onto
the stack. This causes a crash when gperftools tcmalloc() attempts to
retrieve the stack trace for a malloc() call.
Fix would be to install libunwind and rebuild gperftools to reflect that,
so that it doesn’t rely on the base pointer register any more.
#0 GetStackTrace (result=0x7f8c6507c8c0, max_depth=42, skip_count=0) at /build/nsheth/default/third_party/gperftools-2.1/src/stacktrace_x86-inl.h:325
#1 0x00007f8c6931aa86 in MallocHook_GetCallerStackTrace (result=0x7f8c6507ca50, max_depth=32, skip_count=<optimized out>) at /build/nsheth/default/third_party/gperftools-2.1/src/malloc_hook.cc:665
#2 0x00007f8c6930c34f in NewHook (ptr=0x416ca40, size=57) at /build/nsheth/default/third_party/gperftools-2.1/src/heap-checker.cc:575
#3 0x00007f8c6931af3b in MallocHook::InvokeNewHookSlow (p=0x416ca40, s=57) at /build/nsheth/default/third_party/gperftools-2.1/src/malloc_hook.cc:525
#4 0x00007f8c69328a27 in InvokeNewHook (s=57, p=<optimized out>) at /build/nsheth/default/third_party/gperftools-2.1/src/malloc_hook-inl.h:161
#5 tc_new (size=57) at /build/nsheth/default/third_party/gperftools-2.1/src/tcmalloc.cc:1607
#6 0x00007f8c6908ea89 in std::string::_Rep::_S_create(unsigned long, unsigned long, std::allocator<char> const&) () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6
#7 0x00007f8c69090495 in char* std::string::_S_construct<char const*>(char const*, char const*, std::allocator<char> const&, std::forward_iterator_tag) () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6
#8 0x00007f8c690905e3 in std::basic_string<char, std::char_traits<char>, std::allocator<char> >::basic_string(char const*, std::allocator<char> const&) () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6
#9 0x0000000000fcb90b in BgpServer::ConfigUpdater::BgpServerConfigUpdate (this=0x3e04298, instance_name="default-domain:default-project:ip-fabric:__default__", params=...)
at controller/src/bgp/bgp_server.cc:142
#10 0x0000000000fcb16a in BgpServer::ConfigUpdater::ProcessProtocolConfig (this=0x3e04298, protocol_config=0x40ebb30, event=BgpConfigManager::CFG_CHANGE) at controller/src/bgp/bgp_server.cc:75
#11 0x0000000000fd2a45 in boost::_mfi::mf2<void, BgpServer::ConfigUpdater, BgpProtocolConfig const*, BgpConfigManager::EventType>::operator() (this=0x40e9280, p=0x3e04298, a1=0x40ebb30,
a2=BgpConfigManager::CFG_CHANGE) at build/include/boost/bind/mem_fn_template.hpp:280
#12 0x0000000000fd21ae in boost::_bi::list3<boost::_bi::value<BgpServer::ConfigUpdater*>, boost::arg<1>, boost::arg<2> >::operator()<boost::_mfi::mf2<void, BgpServer::ConfigUpdater, BgpProtocolConfig const*, BgpConfigManager::EventType>, boost::_bi::list2<BgpProtocolConfig const*&, BgpConfigManager::EventType&> > (this=0x40e9290, f=..., a=...) at build/include/boost/bind/bind.hpp:392
#13 0x0000000000fd190e in boost::_bi::bind_t<void, boost::_mfi::mf2<void, BgpServer::ConfigUpdater, BgpProtocolConfig const*, BgpConfigManager::EventType>, boost::_bi::list3<boost::_bi::value<BgpServer::ConfigUpdater*>, boost::arg<1>, boost::arg<2> > >::operator()<BgpProtocolConfig const*, BgpConfigManager::EventType> (this=0x40e9280, a1=@0x7f8c6507d3c0: 0x40ebb30,
a2=@0x7f8c6507d3bc: BgpConfigManager::CFG_CHANGE) at build/include/boost/bind/bind_template.hpp:61
#14 0x0000000000fd101e in boost::detail::function::void_function_obj_invoker2<boost::_bi::bind_t<void, boost::_mfi::mf2<void, BgpServer::ConfigUpdater, BgpProtocolConfig const*, BgpConfigManager::EventType>, boost::_bi::list3<boost::_bi::value<BgpServer::ConfigUpdater*>, boost::arg<1>, boost::arg<2> > >, void, BgpProtocolConfig const*, BgpConfigManager::EventType>::invoke (function_obj_ptr=...,
a0=0x40ebb30, a1=BgpConfigManager::CFG_CHANGE) at build/include/boost/function/function_template.hpp:153
#15 0x0000000000eaec78 in boost::function2<void, BgpProtocolConfig const*, BgpConfigManager::EventType>::operator() (this=0x40e9278, a0=0x40ebb30, a1=BgpConfigManager::CFG_CHANGE)
at build/include/boost/function/function_template.hpp:763
#16 0x0000000000ea4ac5 in BgpConfigManager::Notify<BgpProtocolConfig> (this=0x40e9200, config=0x40ebb30, event=BgpConfigManager::CFG_CHANGE) at controller/src/bgp/bgp_config.cc:63
#17 0x0000000000eaa298 in BgpConfigManager::ProcessBgpProtocol (this=0x40e9200, delta=...) at controller/src/bgp/bgp_config.cc:1089
#18 0x0000000000eaaae1 in BgpConfigManager::ProcessBgpRouter (this=0x40e9200, delta=...) at controller/src/bgp/bgp_config.cc:1138
#19 0x0000000000eb667b in boost::_mfi::mf1<void, BgpConfigManager, IFMapConfigListener::ConfigDelta const&>::operator() (this=0x411ad00, p=0x40e9200, a1=...)
at build/include/boost/bind/mem_fn_template.hpp:165
#20 0x0000000000eb6383 in boost::_bi::list2<boost::_bi::value<BgpConfigManager*>, boost::arg<1> >::operator()<boost::_mfi::mf1<void, BgpConfigManager, IFMapConfigListener::ConfigDelta const&>, boost::_bi::list1<IFMapConfigListener::ConfigDelta const&> > (this=0x411ad10, f=..., a=...) at build/include/boost/bind/bind.hpp:313
#21 0x0000000000eb5ed6 in boost::_bi::bind_t<void, boost::_mfi::mf1<void, BgpConfigManager, IFMapConfigListener::ConfigDelta const&>, boost::_bi::list2<boost::_bi::value<BgpConfigManager*>, boost::arg<1> > >::operator()<IFMapConfigListener::ConfigDelta> (this=0x411ad00, a1=...) at build/include/boost/bind/bind_template.hpp:47
#22 0x0000000000eb5974 in boost::detail::function::void_function_obj_invoker1<boost::_bi::bind_t<void, boost::_mfi::mf1<void, BgpConfigManager, IFMapConfigListener::ConfigDelta const&>, boost::_bi::list2<boost::_bi::value<BgpConfigManager*>, boost::arg<1> > >, void, IFMapConfigListener::ConfigDelta const&>::invoke (function_obj_ptr=..., a0=...) at build/include/boost/function/function_template.hpp:153
#23 0x0000000000eb0289 in boost::function1<void, IFMapConfigListener::ConfigDelta const&>::operator() (this=0x411acf8, a0=...) at build/include/boost/function/function_template.hpp:763
#24 0x0000000000eab6ab in BgpConfigManager::ProcessChanges (this=0x40e9200, change_list=std::vector of length 3, capacity 4 = {...}) at controller/src/bgp/bgp_config.cc:1225
#25 0x0000000000eab781 in BgpConfigManager::ConfigHandler (this=0x40e9200) at controller/src/bgp/bgp_config.cc:1239
#26 0x0000000000eb6290 in boost::_mfi::mf0<bool, BgpConfigManager>::operator() (this=0x40e92c0, p=0x40e9200) at build/include/boost/bind/mem_fn_template.hpp:49
#27 0x0000000000eb5de3 in boost::_bi::list1<boost::_bi::value<BgpConfigManager*> >::operator()<bool, boost::_mfi::mf0<bool, BgpConfigManager>, boost::_bi::list0> (this=0x40e92d0, f=..., a=...)
at build/include/boost/bind/bind.hpp:243
#28 0x0000000000eb582d in boost::_bi::bind_t<bool, boost::_mfi::mf0<bool, BgpConfigManager>, boost::_bi::list1<boost::_bi::value<BgpConfigManager*> > >::operator() (this=0x40e92c0)
at build/include/boost/bind/bind_template.hpp:20
#29 0x0000000000eb50df in boost::detail::function::function_obj_invoker0<boost::_bi::bind_t<bool, boost::_mfi::mf0<bool, BgpConfigManager>, boost::_bi::list1<boost::_bi::value<BgpConfigManager*> > >, bool>::invoke (function_obj_ptr=...) at build/include/boost/function/function_template.hpp:132
#30 0x0000000000d8e3c2 in boost::function0<bool>::operator() (this=0x40e92b8) at build/include/boost/function/function_template.hpp:763
#31 0x000000000131be21 in TaskTrigger::WorkerTask::Run (this=0x404e5d0) at controller/src/base/task_trigger.cc:19
#32 0x000000000130ab4d in TaskImpl::execute (this=0x40209c0) at controller/src/base/task.cc:232
#33 0x00007f8c69a7bece in tbb::internal::custom_scheduler<tbb::internal::IntelSchedulerTraits>::local_wait_for_all (this=0x3f38d80, parent=..., child=0x3e56e40)
at /build/nsheth/default/third_party/tbb40_20111130oss/src/tbb/custom_scheduler.h:449
#34 0x00007f8c69a72e0b in tbb::internal::arena::process (this=0x3e92d00, s=...) at /build/nsheth/default/third_party/tbb40_20111130oss/src/tbb/arena.cpp:99
#35 0x00007f8c69a716f2 in tbb::internal::market::process (this=0x3e5a500, j=...) at /build/nsheth/default/third_party/tbb40_20111130oss/src/tbb/market.cpp:393
#36 0x00007f8c69a6c3ce in tbb::internal::rml::private_worker::run (this=0x3e8c480) at /build/nsheth/default/third_party/tbb40_20111130oss/src/tbb/private_server.cpp:263
#37 0x00007f8c69a6c270 in tbb::internal::rml::private_worker::thread_routine (arg=0x3e8c480) at /build/nsheth/default/third_party/tbb40_20111130oss/src/tbb/private_server.cpp:231
#38 0x00007f8c69ca5e9a in start_thread (arg=0x7f8c6507e700) at pthread_create.c:308
#39 0x00007f8c688132ed in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#40 0x0000000000000000 in ?? () |
|
