Vcenter-as-compute: Query-engine crashes @ std::string::compare

[Sarath]

This seen on Vcenter-compute sanity R3.2 #33

[New LWP 16497]
[Thread debugging using libthread_db enabled]
Using host libthread_db library "/lib/x86_64-linux-gnu/libthread_db.so.1".
Core was generated by `/usr/bin/contrail-query-engine --conf_file /etc/contrail/contrail-query-engine.'.
Program terminated with signal SIGSEGV, Segmentation fault.
#0 0x00007f1fc1783e7c in std::string::compare(std::string const&) const () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6
(gdb) bt
#0 0x00007f1fc1783e7c in std::string::compare(std::string const&) const () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6
#1 0x000000000050a140 in operator< <char, std::char_traits<char>, std::allocator<char> > (
    __rhs="default-domain:ctest-TestECMPwithSVMChangeIPv6-53059416:ctest-left_ctest-TestECMPwithSVMChangeIPv6-53059416-47773667", __lhs=...) at /usr/include/c++/4.8/bits/basic_string.h:2571
#2 operator()<std::basic_string<char> > (this=<optimized out>, rhs="default-domain:ctest-TestECMPwithSVMChangeIPv6-53059416:ctest-left_ctest-TestECMPwithSVMChangeIPv6-53059416-47773667",
    lhs=...) at /usr/include/boost/variant/variant.hpp:980
#3 operator()<std::basic_string<char> > (rhs_content="default-domain:ctest-TestECMPwithSVMChangeIPv6-53059416:ctest-left_ctest-TestECMPwithSVMChangeIPv6-53059416-47773667",
    this=<optimized out>) at /usr/include/boost/variant/variant.hpp:948
#4 internal_visit<std::basic_string<char> const> (operand="default-domain:ctest-TestECMPwithSVMChangeIPv6-53059416:ctest-left_ctest-TestECMPwithSVMChangeIPv6-53059416-47773667",
    this=0x7f1fb2bfa0b0) at /usr/include/boost/variant/variant.hpp:1017
#5 visitation_impl_invoke_impl<boost::detail::variant::invoke_visitor<boost::detail::variant::comparer<boost::variant<boost::blank, std::basic_string<char>, unsigned long, unsigned int, boost::uuids::uuid, unsigned char, unsigned short, double, boost::asio::ip::address>, boost::detail::variant::less_comp> >, void const*, std::basic_string<char> > (storage=0x7f1f8c014790,
    visitor=...) at /usr/include/boost/variant/detail/visitation_impl.hpp:130
#6 visitation_impl_invoke<boost::detail::variant::invoke_visitor<boost::detail::variant::comparer<boost::variant<boost::blank, std::basic_string<char>, unsigned long, unsigned int, boost::uuids::uuid, unsigned char, unsigned short, double, boost::asio::ip::address>, boost::detail::variant::less_comp> >, void const*, std::basic_string<char>, boost::variant<boost::blank, std::basic_string<char>, unsigned long, unsigned int, boost::uuids::uuid, unsigned char, unsigned short, double, boost::asio::ip::address>::has_fallback_type_> (internal_which=1, t=0x0,
    storage=0x7f1f8c014790, visitor=...) at /usr/include/boost/variant/detail/visitation_impl.hpp:173
#7 boost::detail::variant::visitation_impl<mpl_::int_<0>, boost::detail::variant::visitation_impl_step<boost::mpl::l_iter<boost::mpl::l_item<mpl_::long_<9l>, boost::blank, boost::mpl::l_item<mpl_::long_<8l>, std::string, boost::mpl::l_item<mpl_::long_<7l>, unsigned long, boost::mpl::l_item<mpl_::long_<6l>, unsigned int, boost::mpl::l_item<mpl_::long_<5l>, boost::uuids::uuid, boost::mpl::l_item<mpl_::long_<4l>, unsigned char, boost::mpl::l_item<mpl_::long_<3l>, unsigned short, boost::mpl::l_item<mpl_::long_<2l>, double, boost::mpl::l_item<mpl_::long_<1l>, boost::asio::ip::address, boost::mpl::l_end> > > > > > > > > >, boost::mpl::l_iter<boost::mpl::l_end> >, boost::detail::variant::invoke_visitor<boost::detail::variant::comparer<boost::variant<boost::blank, std::string, unsigned long, unsigned int, boost::uuids::uuid, unsigned char, unsigned short, double, boost::asio::ip::address, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_>, boost::detail::variant::less_comp> >, void const*, boost::variant<boost::blank, std::string, unsigned long, unsigned int, boost::uuids::uuid, unsigned char, unsigned short, double, boost::asio::ip::address, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_>::has_fallback_type_> (internal_which=internal_which@entry=1, logical_which=logical_which@entry=1, visitor=...,
    storage=storage@entry=0x7f1f8c014790, no_backup_flag=no_backup_flag@entry=...) at /usr/include/boost/variant/detail/visitation_impl.hpp:256
#8 0x000000000050790a in internal_apply_visitor_impl<boost::detail::variant::invoke_visitor<boost::detail::variant::comparer<boost::variant<boost::blank, std::basic_string<char>, unsigned long, unsigned int, boost::uuids::uuid, unsigned char, unsigned short, double, boost::asio::ip::address>, boost::detail::variant::less_comp> >, void const*> (storage=0x7f1f8c014790,
    visitor=..., logical_which=1, internal_which=1) at /usr/include/boost/variant/variant.hpp:2326
#9 internal_apply_visitor<boost::detail::variant::invoke_visitor<boost::detail::variant::comparer<boost::variant<boost::blank, std::basic_string<char>, unsigned long, unsigned int, boost::uuids::uuid, unsigned char, unsigned short, double, boost::asio::ip::address>, boost::detail::variant::less_comp> > > (visitor=..., this=0x7f1f8c014788)
    at /usr/include/boost/variant/variant.hpp:2348
#10 apply_visitor<boost::detail::variant::comparer<boost::variant<boost::blank, std::basic_string<char>, unsigned long, unsigned int, boost::uuids::uuid, unsigned char, unsigned short, double, boost::asio::ip::address>, boost::detail::variant::less_comp> > (visitor=..., this=0x7f1f8c014788) at /usr/include/boost/variant/variant.hpp:2370
#11 operator< (rhs=..., this=0x7f1f8c050718) at /usr/include/boost/variant/variant.hpp:2284
#12 query_result_unit_t::operator< (this=this@entry=0x7f1f8c216320, rhs=...) at controller/src/query_engine/set_operation.cc:19
#13 0x000000000050892e in operator< (rhs=..., this=0x7f1f8c216320) at /usr/include/c++/4.8/bits/stl_algo.h:5892
#14 set_intersection<__gnu_cxx::__normal_iterator<query_result_unit_t*, std::vector<query_result_unit_t> >, __gnu_cxx::__normal_iterator<query_result_unit_t*, std::vector<query_result_unit_t> >, std::back_insert_iterator<std::vector<query_result_unit_t> > > (__result=..., __last2=..., __first2=..., __last1=..., __first1=...) at /usr/include/c++/4.8/bits/stl_algo.h:5894
#15 SetOperationUnit::op_and (qi="6cb21c1c-100a-11e7-b278-00000a540d20", res=std::vector of length 405, capacity 405 = {...}, inp=std::vector of length 2, capacity 2 = {...})
    at controller/src/query_engine/set_operation.cc:43
#16 0x000000000052abe6 in WhereQuery::subquery_processed (this=0x7f1f942d5270, subquery=<optimized out>) at controller/src/query_engine/where_query.cc:1101
#17 0x00000000004b673f in DbQueryUnit::WPCompleteCb (this=0x7f1f940e82e0, wp=0x7f1f94047250, ret_code=<optimized out>) at controller/src/query_engine/db_query.cc:252
#18 0x000000000048b97c in boost::function1<void, bool>::operator() (this=<optimized out>, a0=<optimized out>) at /usr/include/boost/function/function_template.hpp:767
#19 0x00000000004b9329 in WorkPipeline<DbQueryUnit::Input, DbQueryUnit::Output, DbQueryUnit::Output, DbQueryUnit::Output, DbQueryUnit::Output, DbQueryUnit::Output, DbQueryUnit::Output>::NextStage<0, DbQueryUnit::Output> (this=0x7f1f94047250) at controller/src/base/work_pipeline-inl.h:191
#20 0x00000000004b9b5d in WorkPipeline<DbQueryUnit::Input, DbQueryUnit::Output, DbQueryUnit::Output, DbQueryUnit::Output, DbQueryUnit::Output, DbQueryUnit::Output, DbQueryUnit::Output>::WorkStageCb (this=0x7f1f94047250, stage=1476403880, ret_code=24) at controller/src/base/work_pipeline-inl.h:148
---Type <return> to continue, or q <return> to quit---
#21 0x00000000004b7ba7 in operator() (a0=true, this=0x7f1f94047170) at /usr/include/boost/function/function_template.hpp:767
#22 WorkStage<DbQueryUnit::Input, DbQueryUnit::Output, std::vector<query_result_unit_t, std::allocator<query_result_unit_t> >, DbQueryUnit::Stage0Out>::Runner (this=0x7f1f94047160)
    at controller/src/base/work_pipeline-inl.h:75
#23 0x0000000000484159 in operator() (this=<optimized out>) at /usr/include/boost/function/function_template.hpp:767
#24 PipelineWorker::Run (this=<optimized out>) at controller/src/base/work_processor-inl.h:27
#25 0x000000000045c8ff in TaskImpl::execute (this=0x7f1fb9dc6740) at controller/src/base/task.cc:262
#26 0x00007f1fc1bf6b3a in ?? () from /usr/lib/libtbb.so.2
#27 0x00007f1fc1bf2816 in ?? () from /usr/lib/libtbb.so.2
#28 0x00007f1fc1bf1f4b in ?? () from /usr/lib/libtbb.so.2
#29 0x00007f1fc1bee0ff in ?? () from /usr/lib/libtbb.so.2
#30 0x00007f1fc1bee2f9 in ?? () from /usr/lib/libtbb.so.2
#31 0x00007f1fc1e12184 in start_thread (arg=0x7f1fb2bfb700) at pthread_create.c:312
#32 0x00007f1fc0ee337d in __ecvt_r (value=0, ndigit=0, decpt=0x0, sign=0x0, buf=0x7f1fb2bfb9c0 "\300\331\177\263\037\177", len=139774119622400) at efgcvt_r.c:218
#33 0x0000000000000000 in ?? ()
(gdb) quit
root@a6s32:~#
root@a6s32:~#
root@a6s32:~# ls -l /var/crashes/core.contrail-query-.15781.a6s32.1490302262
-rw------- 1 contrail contrail 259383296 Mar 23 13:51 /var/crashes/core.contrail-query-.15781.a6s32.1490302262
root@a6s32:~#

[Sarath (nsarath)]

Please find Cores/Logs @

nsarath@ubuntu-build04:/auto/cores/1675598$ ls -l
total 1276988
-rwxrwxrwx 1 nsarath test 259383296 Mar 23 18:17 core.contrail-query-.15781.a6s32.1490302262
-rwxrwxrwx 1 nsarath test 347607040 Mar 23 18:16 Ctrl-A-log.tar
-rwxrwxrwx 1 nsarath test 408350720 Mar 23 18:16 Ctrl-B-log.tar
-rwxrwxrwx 1 nsarath test 221071360 Mar 23 18:16 Ctrl-C-log.tar
-rwxrwxrwx 1 nsarath test 20408320 Mar 23 18:16 Esxi-1-log.tar
-rwxrwxrwx 1 nsarath test 20264960 Mar 23 18:16 Esxi-2-log.tar
-rwxrwxrwx 1 nsarath test 25384960 Mar 23 18:15 Kvm-1-log.tar

[OpenContrail Admin (ci-admin-f)]

Review in progress for https://review.opencontrail.org/29957
Submitter: Arvind (<email address hidden>)

[Arvind (arvindv)]

When < overloaded fn is called to compare the query_result_unit_t from the different subquery results. It iterates through the structure and compares. We can see that one of the iterator is not set.
(gdb) frame 12
#12 query_result_unit_t::operator< (this=this@entry=0x7f1f8c216320, rhs=...)
    at controller/src/query_engine/set_operation.cc:19
19 controller/src/query_engine/set_operation.cc: No such file or directory.
(gdb) print it
$1 = (boost::variant<...>) which (1) = std::basic_string<char, std::char_traits<char>, std::allocator<char> > value = <error reading variable: Cannot access memory at address 0x28>
(gdb) print jt
$2 = (boost::variant<...>) which (1) = std::basic_string<char, std::char_traits<char>, std::allocator<char> > value = "default-domain:ctest-TestECMPwithSVMChangeIPv6-53059416:ctest-left_ctest-TestECMPwithSVMChangeIPv6-53059416-47773667"

This was the actual argument passed to the overloaded fn:

(gdb) frame 14
#14 set_intersection<__gnu_cxx::__normal_iterator<query_result_unit_t*, std::vector<query_result_unit_t> >, __gnu_cxx::__normal_iterator<query_result_unit_t*, std::vector<query_result_unit_t> >, std::back_insert_iterator<std::vector<query_result_unit_t> > > (__result=..., __last2=..., __first2={
  timestamp = 1490302039966492,
  info = std::vector of length 1, capacity 1 = {(boost::variant<...>) which (4) = boost::uuids::uuid value = (boost::uuids::uuid) 1c35cd20-2703-4855-8295-b3410c2859e3}
}, __last1=..., __first1={
  timestamp = 1490302039966492,
  info = std::vector of length 2, capacity 2 = {(boost::variant<...>) which (4) = boost::uuids::uuid value = (boost::uuids::uuid) 1c35cd20-2703-4855-8295-b3410c2859e3, (boost::variant<...>) which (1) = std::basic_string<char, std::char_traits<char>, std::allocator<char> > value = "default-domain:ctest-TestECMPwithSVMChangeIPv6-53059416:ctest-left_ctest-TestECMPwithSVMChangeIPv6-53059416-47773667"}
}) at /usr/include/c++/4.8/bits/stl_algo.h:5894
5894 else if (*__first2 < *__first1)

[OpenContrail Admin (ci-admin-f)]

Reviewed: https://review.opencontrail.org/29957
Committed: http://github.org/Juniper/contrail-controller/commit/742650719518b26b6d9bf7e4d43d8599ae5ff498
Submitter: Zuul (<email address hidden>)
Branch: R3.2

commit 742650719518b26b6d9bf7e4d43d8599ae5ff498
Author: arvindvis <email address hidden>
Date: Wed Mar 29 20:14:23 2017 -0700

In for loop, separating the variables in if condition is not the
way to compare the conditions (If comma separated only the last
specified condition gets executed). This is not desirable, because
terator it can point to results from MessageTableIndex queries and
jt can point to resultss from ObjectTable and their dimensionality
is not same and this can result in crash.
Closes-Bug:#1675598

Change-Id: I8af6ef7166900ea4705715fd317d2655d2c3b984

[OpenContrail Admin (ci-admin-f)]

Review in progress for https://review.opencontrail.org/29990
Submitter: Arvind (<email address hidden>)

[OpenContrail Admin (ci-admin-f)]

Reviewed: https://review.opencontrail.org/29990
Committed: http://github.org/Juniper/contrail-controller/commit/19e95440fdbe5dbc8c37f42e0c2bd9bc1a12eff3
Submitter: Zuul (<email address hidden>)
Branch: master

commit 19e95440fdbe5dbc8c37f42e0c2bd9bc1a12eff3
Author: arvindvis <email address hidden>
Date: Wed Mar 29 20:14:23 2017 -0700

In for loop, separating the variables in if condition is not the
way to compare the conditions (If comma separated only the last
specified condition gets executed). This is not desirable, because
terator it can point to results from MessageTableIndex queries and
jt can point to resultss from ObjectTable and their dimensionality
is not same and this can result in crash.
Closes-Bug:#1675598

Change-Id: I8af6ef7166900ea4705715fd317d2655d2c3b984
(cherry picked from commit 742650719518b26b6d9bf7e4d43d8599ae5ff498)