| 2016-10-18 23:00:09 |
Senthilnathan Murugappan |
description |
Have 500 odd vms in the compute and agent crashes continuously with below trace.
FYI, the other two computes with 250 odd vms each are stable.
BTW, all the vms are launched in a single VN who has 1000 subnets and each subnet has one VM.
contrail-vrouter-agent: controller/src/vnsw/agent/vrouter/ksync/nexthop_ksync.cc:888: int NHKSyncEntry::Encode(sandesh_op::type, char*, int): Assertion `encode_len <= buf_len' failed.
the cores are copied on to /cs-shared/bugs/<bugid>/
(gdb) bt
#0 0x00007effb46aecc9 in __GI_raise (sig=sig@entry=6) at ../nptl/sysdeps/unix/sysv/linux/raise.c:56
#1 0x00007effb46b20d8 in __GI_abort () at abort.c:89
#2 0x00007effb46a7b86 in __assert_fail_base (fmt=0x7effb47f8830 "%s%s%s:%u: %s%sAssertion `%s' failed.\n%n", assertion=assertion@entry=0x13867b6 "encode_len <= buf_len",
file=file@entry=0x1389b10 "controller/src/vnsw/agent/vrouter/ksync/nexthop_ksync.cc", line=line@entry=888,
function=function@entry=0x1389e60 <NHKSyncEntry::Encode(sandesh_op::type, char*, int)::__PRETTY_FUNCTION__> "int NHKSyncEntry::Encode(sandesh_op::type, char*, int)") at assert.c:92
#3 0x00007effb46a7c32 in __GI___assert_fail (assertion=0x13867b6 "encode_len <= buf_len", file=0x1389b10 "controller/src/vnsw/agent/vrouter/ksync/nexthop_ksync.cc", line=888,
function=0x1389e60 <NHKSyncEntry::Encode(sandesh_op::type, char*, int)::__PRETTY_FUNCTION__> "int NHKSyncEntry::Encode(sandesh_op::type, char*, int)") at assert.c:101
#4 0x0000000000dacbee in NHKSyncEntry::Encode (this=0x7effa4c50590, op=<optimized out>, buf=0x7effa9491470 "", buf_len=4096) at controller/src/vnsw/agent/vrouter/ksync/nexthop_ksync.cc:888
#5 0x0000000000dacd73 in NHKSyncEntry::AddMsg (this=0x7effa4c50590, buf=0x7effa9491470 "", buf_len=4096) at controller/src/vnsw/agent/vrouter/ksync/nexthop_ksync.cc:1002
#6 0x0000000000dcea21 in KSyncNetlinkDBEntry::Add (this=0x7effa4c50590) at controller/src/ksync/ksync_netlink.cc:88
#7 0x0000000000e80dd5 in KSyncSM_Add (obj=0x7eff9400b450, entry=0x7effa4c50590) at controller/src/ksync/ksync_object.cc:753
#8 0x0000000000e81dd5 in KSyncObject::NotifyEvent (this=0x7eff9400b450, entry=0x7effa4c50590, event=KSyncEntry::ADD_CHANGE_REQ) at controller/src/ksync/ksync_object.cc:1347
#9 0x0000000000e8339c in KSyncDBObject::Notify (this=0x7eff9400b450, partition=<optimized out>, e=0x7eff80876638) at controller/src/ksync/ksync_object.cc:499
#10 0x00000000011a3b3a in operator() (a1=0x7eff80876638, a0=0x7eff954235a0, this=0x7effadd34aa0) at /usr/include/boost/function/function_template.hpp:767
#11 RunNotify (entry=0x7eff80876638, tpart=0x7eff954235a0, this=0x7eff95419560) at controller/src/db/db_table.cc:115
#12 DBTableBase::RunNotify (this=<optimized out>, tpart=tpart@entry=0x7eff954235a0, entry=entry@entry=0x7eff80876638) at controller/src/db/db_table.cc:207
#13 0x00000000011a6348 in DBTablePartBase::RunNotify (this=this@entry=0x7eff954235a0) at controller/src/db/db_table_partition.cc:47
#14 0x00000000011a1f7e in DBPartition::QueueRunner::Run (this=0x7eff840047e0) at controller/src/db/db_partition.cc:215
#15 0x00000000012e4dff in TaskImpl::execute (this=0x7effadef6340) at controller/src/base/task.cc:262
#16 0x00007effb527db3a in ?? () from /usr/lib/libtbb.so.2
#17 0x00007effb5279816 in ?? () from /usr/lib/libtbb.so.2
#18 0x00007effb5278f4b in ?? () from /usr/lib/libtbb.so.2
#19 0x00007effb52750ff in ?? () from /usr/lib/libtbb.so.2
#20 0x00007effb52752f9 in ?? () from /usr/lib/libtbb.so.2
#21 0x00007effb5499182 in start_thread (arg=0x7effadd35700) at pthread_create.c:312
#22 0x00007effb477247d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111
(gdb) p buf_len
$1 = 4096
(gdb) p buf
$2 = 0x7effa9491470 ""
(gdb) p encode_len
$3 = <optimized out>
(gdb) p encoder
$4 = {
<SandeshBuffer> = {
<Sandesh> = {
_vptr.Sandesh = 0x1384550 <vtable for vr_nexthop_req+16>,
static response_callback_ = {
<boost::function1<void, Sandesh*>> = {
<boost::function_base> = {
vtable = 0x0,
functor = {
obj_ptr = 0x0,
type = {
type = 0x0,
const_qualified = false,
volatile_qualified = false
},
func_ptr = 0x0,
bound_memfunc_ptr = {
memfunc_ptr = NULL,
obj_ptr = 0x0
},
obj_ref = {
obj_ptr = 0x0,
is_const_qualified = false,
is_volatile_qualified = false
},
data = 0 '\000'
}
},
<std::unary_function<Sandesh*, void>> = {<No data fields>},
members of boost::function1<void, Sandesh*>:
static args = <optimized out>,
static arity = <optimized out>
}, <No data fields>},
static client_ = 0x7eff7cf44df0,
static role_ = Sandesh::SandeshRole::Generator,
static module_ = {
static npos = <optimized out>,
_M_dataplus = {
<std::allocator<char>> = {
<__gnu_cxx::new_allocator<char>> = {<No data fields>}, <No data fields>},
members of std::basic_string<char, std::char_traits<char>, std::allocator<char> >::_Alloc_hider:
_M_p = 0x342f798 "contrail-vrouter-agent"
}
},
static source_ = {
static npos = <optimized out>,
_M_dataplus = {
<std::allocator<char>> = {
<__gnu_cxx::new_allocator<char>> = {<No data fields>}, <No data fields>},
members of std::basic_string<char, std::char_traits<char>, std::allocator<char> >::_Alloc_hider:
_M_p = 0x3446f48 "a2s36"
}
},
static node_type_ = {
static npos = <optimized out>,
_M_dataplus = {
<std::allocator<char>> = {
<__gnu_cxx::new_allocator<char>> = {<No data fields>}, <No data fields>},
members of std::basic_string<char, std::char_traits<char>, std::allocator<char> >::_Alloc_hider:
_M_p = 0x34312d8 "Compute"
}
},
static instance_id_ = {
static npos = <optimized out>,
_M_dataplus = {
<std::allocator<char>> = {
<__gnu_cxx::new_allocator<char>> = {<No data fields>}, <No data fields>},
members of std::basic_string<char, std::char_traits<char>, std::allocator<char> >::_Alloc_hider:
_M_p = 0x3431a18 "0"
}
},
static http_port_ = 8085,
static recv_queue_ = {
_M_ptr = 0x7eff7cf44c40
},
static recv_task_id_ = 11,
static client_context_ = 0x0,
static module_context_ = {
_M_t = {
_M_impl = {
<std::allocator<std::_Rb_tree_node<std::pair<std::basic_string<char, std::char_traits<char>, std::allocator<char> > const, SandeshContext*> > >> = {
<__gnu_cxx::new_allocator<std::_Rb_tree_node<std::pair<std::basic_string<char, std::char_traits<char>, std::allocator<char> > const, SandeshContext*> > >> = {<No data fields>}, <No data fields>},
members of std::_Rb_tree<std::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::pair<std::basic_string<char, std::char_traits<char>, std::allocator<char> > const, SandeshContext*>, std::_Select1st<std::pair<std::basic_string<char, std::char_traits<char>, std::allocator<char> > const, SandeshContext*> >, std::less<std::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::allocator<std::pair<std::basic_string<char, std::char_traits<char>, std::allocator<char> > const, SandeshContext*> > >::_Rb_tree_impl<std::less<std::basic_string<char, std::char_traits<char>, std::allocator<char> > >, false>:
_M_key_compare = {
<std::binary_function<std::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::basic_string<char, std::char_traits<char>, std::allocator<char> >, bool>> = {<No data fields>}, <No data fields>},
_M_header = {
_M_color = std::_S_red,
_M_parent = 0x0,
_M_left = 0x1923c08 <Sandesh::module_context_+8>,
_M_right = 0x1923c08 <Sandesh::module_context_+8>
},
_M_node_count = 0
}
}
},
static enable_local_log_ = true,
static enable_flow_log_ = false,
static logging_level_ = SandeshLevel::SYS_NOTICE,
static logging_ut_level_ = SandeshLevel::UT_DEBUG,
static logging_category_ = {
static npos = <optimized out>,
_M_dataplus = {
<std::allocator<char>> = {
<__gnu_cxx::new_allocator<char>> = {<No data fields>}, <No data fields>},
members of std::basic_string<char, std::char_traits<char>, std::allocator<char> >::_Alloc_hider:
_M_p = 0x1911078 <std::string::_Rep::_S_empty_rep_storage@@GLIBCXX_3.4+24> ""
}
},
static enable_trace_print_ = false,
static connect_to_collector_ = true,
static event_manager_ = 0x34502f0,
static send_queue_enabled_ = true,
static sending_level_ = SandeshLevel::INVALID,
static msg_stats_ = {
detail_type_stats_map_ = {
<boost::ptr_map_adapter<SandeshMessageTypeStats, std::map<std::basic_string<char, std::char_traits<char>, std::allocator<char> >, void*, std::less<std::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::allocator<std::pair<std::basic_string<char, std::char_traits<char>, std::allocator<char> > const, void*> > >, boost::heap_clone_allocator, true>> = {
<boost::ptr_container_detail::ptr_map_adapter_base<SandeshMessageTypeStats, std::map<std::basic_string<char, std::char_traits<char>, std::allocator<char> >, void*, std::less<std::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::allocator<std::pair<std::basic_string<char, std::char_traits<char>, std::allocator<char> > const, void*> > >, boost::heap_clone_allocator, true>> = {
<boost::ptr_container_detail::associative_ptr_container<boost::ptr_container_detail::map_config<SandeshMessageTypeStats, std::map<std::basic_string<char, std::char_traits<char>, std::allocator<char> >, void*, std::less<std::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::allocator<std::pair<std::basic_string<char, std::char_traits<char>, std::allocator<char> > const, void*> > >, true>, boost::heap_clone_allocator>> = {
<boost::ptr_container_detail::reversible_ptr_container<boost::ptr_container_detail::map_config<SandeshMessageTypeStats, std::map<std::basic_string<char, std::char_traits<char>, std::allocator<char> >, void*, std::less<std::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::allocator<std::pair<std::basic_string<char, std::char_traits<char>, std::allocator<char> > const, void*> > >, true>, boost::heap_clone_allocator>> = {
static allow_null = false,
c_ = {
_M_t = {
_M_impl = {
<std::allocator<std::_Rb_tree_node<std::pair<std::basic_string<char, std::char_traits<char>, std::allocator<char> > const, void*> > >> = {
<__gnu_cxx::new_allocator<std::_Rb_tree_node<std::pair<std::basic_string<char, std::char_traits<char>, std::allocator<char> > const, void*> > >> = {<No data fields>}, <No data fields>},
members of std::_Rb_tree<std::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::pair<std::basic_string<char, std::char_traits<char>, std::allocator<char> > const, void*>, std::_Select1st<std::pair<std::basic_string<char, std::char_traits<char>, std::allocator<char> > const, void*> >, std::less<std::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::allocator<std::pair<std::basic_string<char, std::char_traits<char>, std::allocator<char> > const, void*> > >::_Rb_tree_impl<std::less<std::basic_string<char, std::char_traits<char>, std::allocator<char> > >, false>:
_M_key_compare = {
<std::binary_function<std::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::basic_string<char, std::char_traits<char>, std::allocator<char> >, bool>> = {<No data fields>}, <No data fields>},
_M_header = {
_M_color = std::_S_red,
_M_parent = 0x7eff7cf54460,
_M_left = 0x7effa0001420,
_M_right = 0x7effa808e4c0
},
_M_node_count = 21
}
}
}
}, <No data fields>}, <No data fields>}, <No data fields>}, <No data fields>},
detail_agg_stats_ = {
_vptr.SandeshMessageStats = 0x1415c70 <vtable for SandeshMessageStats+16>,
static ascii_fingerprint = 0x1433a00 "F82EE8FB1ED9259367155C92B2A8210E",
static binary_fingerprint = "\370.\350\373\036\331%\223g\025\\\222\262\250!\016",
messages_sent = 4053,
bytes_sent = 13059012,
messages_received = 1,
bytes_received = 940,
messages_sent_dropped = 34,
bytes_sent_dropped = 0,
messages_received_dropped = 0,
bytes_received_dropped = 0,
messages_sent_dropped_no_queue = 0,
messages_sent_dropped_no_client = 2,
messages_sent_dropped_no_session = 0,
messages_sent_dropped_queue_level = 9,
messages_sent_dropped_client_send_failed = 0,
messages_sent_dropped_session_not_connected = 0,
messages_sent_dropped_header_write_failed = 0,
messages_sent_dropped_write_failed = 0,
messages_sent_dropped_wrong_client_sm_state = 23,
messages_sent_dropped_validation_failed = 0,
messages_sent_dropped_rate_limited = 0,
bytes_sent_dropped_no_queue = 0,
bytes_sent_dropped_no_client = 0,
bytes_sent_dropped_no_session = 0,
bytes_sent_dropped_queue_level = 0,
bytes_sent_dropped_client_send_failed = 0,
bytes_sent_dropped_session_not_connected = 0,
bytes_sent_dropped_header_write_failed = 0,
bytes_sent_dropped_write_failed = 0,
bytes_sent_dropped_wrong_client_sm_state = 0,
bytes_sent_dropped_validation_failed = 0,
bytes_sent_dropped_rate_limited = 0,
messages_received_dropped_no_queue = 0,
messages_received_dropped_queue_level = 0,
messages_received_dropped_create_failed = 0,
messages_received_dropped_control_msg_failed = 0,
messages_received_dropped_decoding_failed = 0,
bytes_received_dropped_no_queue = 0,
bytes_received_dropped_queue_level = 0,
bytes_received_dropped_create_failed = 0,
bytes_received_dropped_control_msg_failed = 0,
bytes_received_dropped_decoding_failed = 0,
__isset = {
messages_sent = false,
bytes_sent = false,
messages_received = false,
bytes_received = false,
messages_sent_dropped = false,
bytes_sent_dropped = false,
messages_received_dropped = false,
bytes_received_dropped = false,
messages_sent_dropped_no_queue = false,
messages_sent_dropped_no_client = false,
messages_sent_dropped_no_session = false,
messages_sent_dropped_queue_level = false,
messages_sent_dropped_client_send_failed = false,
messages_sent_dropped_session_not_connected = false,
messages_sent_dropped_header_write_failed = false,
messages_sent_dropped_write_failed = false,
messages_sent_dropped_wrong_client_sm_state = false,
messages_sent_dropped_validation_failed = false,
messages_sent_dropped_rate_limited = false,
bytes_sent_dropped_no_queue = false,
bytes_sent_dropped_no_client = false,
bytes_sent_dropped_no_session = false,
bytes_sent_dropped_queue_level = false,
bytes_sent_dropped_client_send_failed = false,
bytes_sent_dropped_session_not_connected = false,
bytes_sent_dropped_header_write_failed = false,
bytes_sent_dropped_write_failed = false,
bytes_sent_dropped_wrong_client_sm_state = false,
bytes_sent_dropped_validation_failed = false,
bytes_sent_dropped_rate_limited = false,
messages_received_dropped_no_queue = false,
messages_received_dropped_queue_level = false,
messages_received_dropped_create_failed = false,
messages_received_dropped_control_msg_failed = false,
messages_received_dropped_decoding_failed = false,
bytes_received_dropped_no_queue = false,
bytes_received_dropped_queue_level = false,
bytes_received_dropped_create_failed = false,
bytes_received_dropped_control_msg_failed = false,
bytes_received_dropped_decoding_failed = false
}
}
},
static stats_mutex_ = {
static is_rw_mutex = false,
static is_recursive_mutex = false,
static is_fair_mutex = false,
impl = {
__data = {
__lock = 0,
__count = 0,
__owner = 0,
__nusers = 0,
__kind = 0,
__spins = 0,
__elision = 0,
__list = {
__prev = 0x0,
__next = 0x0
}
},
__size = '\000' <repeats 39 times>,
__align = 0
}
},
static logger_ = {<No data fields>},
static disable_flow_collection_ = false,
seqnum_ = 2426,
context_ = {
static npos = <optimized out>,
_M_dataplus = {
<std::allocator<char>> = {
<__gnu_cxx::new_allocator<char>> = {<No data fields>}, <No data fields>},
members of std::basic_string<char, std::char_traits<char>, std::allocator<char> >::_Alloc_hider:
_M_p = 0x1911078 <std::string::_Rep::_S_empty_rep_storage@@GLIBCXX_3.4+24> ""
}
},
timestamp_ = 0,
scope_ = {
static npos = <optimized out>,
_M_dataplus = {
<std::allocator<char>> = {
<__gnu_cxx::new_allocator<char>> = {<No data fields>}, <No data fields>},
members of std::basic_string<char, std::char_traits<char>, std::allocator<char> >::_Alloc_hider:
_M_p = 0x1911078 <std::string::_Rep::_S_empty_rep_storage@@GLIBCXX_3.4+24> ""
}
},
type_ = SandeshType::BUFFER,
hints_ = 0,
level_ = SandeshLevel::INVALID,
category_ = {
static npos = <optimized out>,
_M_dataplus = {
<std::allocator<char>> = {
<__gnu_cxx::new_allocator<char>> = {<No data fields>}, <No data fields>},
members of std::basic_string<char, std::char_traits<char>, std::allocator<char> >::_Alloc_hider:
_M_p = 0x1911078 <std::string::_Rep::_S_empty_rep_storage@@GLIBCXX_3.4+24> ""
}
},
name_ = {
static npos = <optimized out>,
_M_dataplus = {
<std::allocator<char>> = {
<__gnu_cxx::new_allocator<char>> = {<No data fields>}, <No data fields>},
members of std::basic_string<char, std::char_traits<char>, std::allocator<char> >::_Alloc_hider:
_M_p = 0x7effa930d6f8 "vr_nexthop_req"
}
},
static sandesh_send_ratelimit_ = {
<tbb::internal::atomic_impl_with_arithmetic<unsigned int, unsigned int, char>> = {
<tbb::internal::atomic_impl<unsigned int>> = {
my_storage = {
my_value = 10
}
}, <No data fields>}, <No data fields>}
}, <No data fields>},
members of vr_nexthop_req:
static ascii_fingerprint = 0x1383400 "3215EA4B5433215282CCCF8A58B450BA",
static binary_fingerprint = "2\025\352KT3!R\202\314ϊX\264P\272",
__isset = {
h_op = false,
nhr_type = false,
nhr_family = false,
nhr_id = false,
nhr_rid = false,
nhr_encap_oif_id = false,
nhr_encap_len = false,
nhr_encap_family = false,
nhr_vrf = false,
nhr_tun_sip = false,
nhr_tun_dip = false,
nhr_tun_sport = false,
nhr_tun_dport = false,
nhr_ref_cnt = false,
nhr_marker = false,
nhr_flags = false,
nhr_encap = false,
nhr_nh_list = false,
nhr_label = false,
nhr_label_list = false,
nhr_nh_count = false,
nhr_tun_sip6 = false,
nhr_tun_dip6 = false
},
static reg = {
<SandeshBaseFactory> = {<No data fields>},
members of SandeshDerivedRegister<vr_nexthop_req>:
name_ = {
static npos = <optimized out>,
_M_dataplus = {
<std::allocator<char>> = {
<__gnu_cxx::new_allocator<char>> = {<No data fields>}, <No data fields>},
members of std::basic_string<char, std::char_traits<char>, std::allocator<char> >::_Alloc_hider:
_M_p = 0x3356048 "vr_nexthop_req"
}
}
},
h_op = sandesh_op::ADD,
nhr_type = 6 '\006',
nhr_family = 2 '\002',
nhr_id = 3563,
nhr_rid = 0,
nhr_encap_oif_id = 0,
nhr_encap_len = 0,
nhr_encap_family = 2054,
nhr_vrf = 1,
nhr_tun_sip = 0,
nhr_tun_dip = 0,
nhr_tun_sport = 0,
nhr_tun_dport = 0,
nhr_ref_cnt = 0,
nhr_marker = 0,
nhr_flags = 8193,
nhr_encap = {
<std::_Vector_base<signed char, std::allocator<signed char> >> = {
_M_impl = {
<std::allocator<signed char>> = {
<__gnu_cxx::new_allocator<signed char>> = {<No data fields>}, <No data fields>},
members of std::_Vector_base<signed char, std::allocator<signed char> >::_Vector_impl:
_M_start = 0x0,
_M_finish = 0x0,
_M_end_of_storage = 0x0
}
}, <No data fields>},
nhr_nh_list = {
<std::_Vector_base<int, std::allocator<int> >> = {
_M_impl = {
<std::allocator<int>> = {
<__gnu_cxx::new_allocator<int>> = {<No data fields>}, <No data fields>},
members of std::_Vector_base<int, std::allocator<int> >::_Vector_impl:
_M_start = 0x7effa8b2a330,
_M_finish = 0x7effa8b2abec,
_M_end_of_storage = 0x7effa8b2abec
}
}, <No data fields>},
nhr_label = -1,
nhr_label_list = {
<std::_Vector_base<int, std::allocator<int> >> = {
_M_impl = {
<std::allocator<int>> = {
<__gnu_cxx::new_allocator<int>> = {<No data fields>}, <No data fields>},
members of std::_Vector_base<int, std::allocator<int> >::_Vector_impl:
_M_start = 0x7effa89ff8d0,
_M_finish = 0x7effa8a0018c,
_M_end_of_storage = 0x7effa8a0018c
}
}, <No data fields>},
nhr_nh_count = 0,
nhr_tun_sip6 = {
<std::_Vector_base<signed char, std::allocator<signed char> >> = {
_M_impl = {
<std::allocator<signed char>> = {
<__gnu_cxx::new_allocator<signed char>> = {<No data fields>}, <No data fields>},
members of std::_Vector_base<signed char, std::allocator<signed char> >::_Vector_impl:
_M_start = 0x0,
_M_finish = 0x0,
_M_end_of_storage = 0x0
}
}, <No data fields>},
nhr_tun_dip6 = {
<std::_Vector_base<signed char, std::allocator<signed char> >> = {
_M_impl = {
<std::allocator<signed char>> = {
<__gnu_cxx::new_allocator<signed char>> = {<No data fields>}, <No data fields>},
members of std::_Vector_base<signed char, std::allocator<signed char> >::_Vector_impl:
_M_start = 0x0,
_M_finish = 0x0,
_M_end_of_storage = 0x0
}
}, <No data fields>},
static lseqnum_ = 2427,
static versionsig_ = 1273632050,
static name_ = <optimized out>
} |
Have 500 odd vms in the compute and agent crashes continuously with below trace.
FYI, the other two computes with 250 odd vms each are stable.
BTW, all the vms are launched in a single VN who has 1000 subnets and each subnet has one VM.
contrail-version:
contrail-install-packages 3.1.0.0-20~kilo 20
contrail-vrouter-agent: controller/src/vnsw/agent/vrouter/ksync/nexthop_ksync.cc:888: int NHKSyncEntry::Encode(sandesh_op::type, char*, int): Assertion `encode_len <= buf_len' failed.
the cores are copied on to /cs-shared/bugs/1634676/
(gdb) bt
#0 0x00007effb46aecc9 in __GI_raise (sig=sig@entry=6) at ../nptl/sysdeps/unix/sysv/linux/raise.c:56
#1 0x00007effb46b20d8 in __GI_abort () at abort.c:89
#2 0x00007effb46a7b86 in __assert_fail_base (fmt=0x7effb47f8830 "%s%s%s:%u: %s%sAssertion `%s' failed.\n%n", assertion=assertion@entry=0x13867b6 "encode_len <= buf_len",
file=file@entry=0x1389b10 "controller/src/vnsw/agent/vrouter/ksync/nexthop_ksync.cc", line=line@entry=888,
function=function@entry=0x1389e60 <NHKSyncEntry::Encode(sandesh_op::type, char*, int)::__PRETTY_FUNCTION__> "int NHKSyncEntry::Encode(sandesh_op::type, char*, int)") at assert.c:92
#3 0x00007effb46a7c32 in __GI___assert_fail (assertion=0x13867b6 "encode_len <= buf_len", file=0x1389b10 "controller/src/vnsw/agent/vrouter/ksync/nexthop_ksync.cc", line=888,
function=0x1389e60 <NHKSyncEntry::Encode(sandesh_op::type, char*, int)::__PRETTY_FUNCTION__> "int NHKSyncEntry::Encode(sandesh_op::type, char*, int)") at assert.c:101
#4 0x0000000000dacbee in NHKSyncEntry::Encode (this=0x7effa4c50590, op=<optimized out>, buf=0x7effa9491470 "", buf_len=4096) at controller/src/vnsw/agent/vrouter/ksync/nexthop_ksync.cc:888
#5 0x0000000000dacd73 in NHKSyncEntry::AddMsg (this=0x7effa4c50590, buf=0x7effa9491470 "", buf_len=4096) at controller/src/vnsw/agent/vrouter/ksync/nexthop_ksync.cc:1002
#6 0x0000000000dcea21 in KSyncNetlinkDBEntry::Add (this=0x7effa4c50590) at controller/src/ksync/ksync_netlink.cc:88
#7 0x0000000000e80dd5 in KSyncSM_Add (obj=0x7eff9400b450, entry=0x7effa4c50590) at controller/src/ksync/ksync_object.cc:753
#8 0x0000000000e81dd5 in KSyncObject::NotifyEvent (this=0x7eff9400b450, entry=0x7effa4c50590, event=KSyncEntry::ADD_CHANGE_REQ) at controller/src/ksync/ksync_object.cc:1347
#9 0x0000000000e8339c in KSyncDBObject::Notify (this=0x7eff9400b450, partition=<optimized out>, e=0x7eff80876638) at controller/src/ksync/ksync_object.cc:499
#10 0x00000000011a3b3a in operator() (a1=0x7eff80876638, a0=0x7eff954235a0, this=0x7effadd34aa0) at /usr/include/boost/function/function_template.hpp:767
#11 RunNotify (entry=0x7eff80876638, tpart=0x7eff954235a0, this=0x7eff95419560) at controller/src/db/db_table.cc:115
#12 DBTableBase::RunNotify (this=<optimized out>, tpart=tpart@entry=0x7eff954235a0, entry=entry@entry=0x7eff80876638) at controller/src/db/db_table.cc:207
#13 0x00000000011a6348 in DBTablePartBase::RunNotify (this=this@entry=0x7eff954235a0) at controller/src/db/db_table_partition.cc:47
#14 0x00000000011a1f7e in DBPartition::QueueRunner::Run (this=0x7eff840047e0) at controller/src/db/db_partition.cc:215
#15 0x00000000012e4dff in TaskImpl::execute (this=0x7effadef6340) at controller/src/base/task.cc:262
#16 0x00007effb527db3a in ?? () from /usr/lib/libtbb.so.2
#17 0x00007effb5279816 in ?? () from /usr/lib/libtbb.so.2
#18 0x00007effb5278f4b in ?? () from /usr/lib/libtbb.so.2
#19 0x00007effb52750ff in ?? () from /usr/lib/libtbb.so.2
#20 0x00007effb52752f9 in ?? () from /usr/lib/libtbb.so.2
#21 0x00007effb5499182 in start_thread (arg=0x7effadd35700) at pthread_create.c:312
#22 0x00007effb477247d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111
(gdb) p buf_len
$1 = 4096
(gdb) p buf
$2 = 0x7effa9491470 ""
(gdb) p encode_len
$3 = <optimized out>
(gdb) p encoder
$4 = {
<SandeshBuffer> = {
<Sandesh> = {
_vptr.Sandesh = 0x1384550 <vtable for vr_nexthop_req+16>,
static response_callback_ = {
<boost::function1<void, Sandesh*>> = {
<boost::function_base> = {
vtable = 0x0,
functor = {
obj_ptr = 0x0,
type = {
type = 0x0,
const_qualified = false,
volatile_qualified = false
},
func_ptr = 0x0,
bound_memfunc_ptr = {
memfunc_ptr = NULL,
obj_ptr = 0x0
},
obj_ref = {
obj_ptr = 0x0,
is_const_qualified = false,
is_volatile_qualified = false
},
data = 0 '\000'
}
},
<std::unary_function<Sandesh*, void>> = {<No data fields>},
members of boost::function1<void, Sandesh*>:
static args = <optimized out>,
static arity = <optimized out>
}, <No data fields>},
static client_ = 0x7eff7cf44df0,
static role_ = Sandesh::SandeshRole::Generator,
static module_ = {
static npos = <optimized out>,
_M_dataplus = {
<std::allocator<char>> = {
<__gnu_cxx::new_allocator<char>> = {<No data fields>}, <No data fields>},
members of std::basic_string<char, std::char_traits<char>, std::allocator<char> >::_Alloc_hider:
_M_p = 0x342f798 "contrail-vrouter-agent"
}
},
static source_ = {
static npos = <optimized out>,
_M_dataplus = {
<std::allocator<char>> = {
<__gnu_cxx::new_allocator<char>> = {<No data fields>}, <No data fields>},
members of std::basic_string<char, std::char_traits<char>, std::allocator<char> >::_Alloc_hider:
_M_p = 0x3446f48 "a2s36"
}
},
static node_type_ = {
static npos = <optimized out>,
_M_dataplus = {
<std::allocator<char>> = {
<__gnu_cxx::new_allocator<char>> = {<No data fields>}, <No data fields>},
members of std::basic_string<char, std::char_traits<char>, std::allocator<char> >::_Alloc_hider:
_M_p = 0x34312d8 "Compute"
}
},
static instance_id_ = {
static npos = <optimized out>,
_M_dataplus = {
<std::allocator<char>> = {
<__gnu_cxx::new_allocator<char>> = {<No data fields>}, <No data fields>},
members of std::basic_string<char, std::char_traits<char>, std::allocator<char> >::_Alloc_hider:
_M_p = 0x3431a18 "0"
}
},
static http_port_ = 8085,
static recv_queue_ = {
_M_ptr = 0x7eff7cf44c40
},
static recv_task_id_ = 11,
static client_context_ = 0x0,
static module_context_ = {
_M_t = {
_M_impl = {
<std::allocator<std::_Rb_tree_node<std::pair<std::basic_string<char, std::char_traits<char>, std::allocator<char> > const, SandeshContext*> > >> = {
<__gnu_cxx::new_allocator<std::_Rb_tree_node<std::pair<std::basic_string<char, std::char_traits<char>, std::allocator<char> > const, SandeshContext*> > >> = {<No data fields>}, <No data fields>},
members of std::_Rb_tree<std::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::pair<std::basic_string<char, std::char_traits<char>, std::allocator<char> > const, SandeshContext*>, std::_Select1st<std::pair<std::basic_string<char, std::char_traits<char>, std::allocator<char> > const, SandeshContext*> >, std::less<std::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::allocator<std::pair<std::basic_string<char, std::char_traits<char>, std::allocator<char> > const, SandeshContext*> > >::_Rb_tree_impl<std::less<std::basic_string<char, std::char_traits<char>, std::allocator<char> > >, false>:
_M_key_compare = {
<std::binary_function<std::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::basic_string<char, std::char_traits<char>, std::allocator<char> >, bool>> = {<No data fields>}, <No data fields>},
_M_header = {
_M_color = std::_S_red,
_M_parent = 0x0,
_M_left = 0x1923c08 <Sandesh::module_context_+8>,
_M_right = 0x1923c08 <Sandesh::module_context_+8>
},
_M_node_count = 0
}
}
},
static enable_local_log_ = true,
static enable_flow_log_ = false,
static logging_level_ = SandeshLevel::SYS_NOTICE,
static logging_ut_level_ = SandeshLevel::UT_DEBUG,
static logging_category_ = {
static npos = <optimized out>,
_M_dataplus = {
<std::allocator<char>> = {
<__gnu_cxx::new_allocator<char>> = {<No data fields>}, <No data fields>},
members of std::basic_string<char, std::char_traits<char>, std::allocator<char> >::_Alloc_hider:
_M_p = 0x1911078 <std::string::_Rep::_S_empty_rep_storage@@GLIBCXX_3.4+24> ""
}
},
static enable_trace_print_ = false,
static connect_to_collector_ = true,
static event_manager_ = 0x34502f0,
static send_queue_enabled_ = true,
static sending_level_ = SandeshLevel::INVALID,
static msg_stats_ = {
detail_type_stats_map_ = {
<boost::ptr_map_adapter<SandeshMessageTypeStats, std::map<std::basic_string<char, std::char_traits<char>, std::allocator<char> >, void*, std::less<std::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::allocator<std::pair<std::basic_string<char, std::char_traits<char>, std::allocator<char> > const, void*> > >, boost::heap_clone_allocator, true>> = {
<boost::ptr_container_detail::ptr_map_adapter_base<SandeshMessageTypeStats, std::map<std::basic_string<char, std::char_traits<char>, std::allocator<char> >, void*, std::less<std::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::allocator<std::pair<std::basic_string<char, std::char_traits<char>, std::allocator<char> > const, void*> > >, boost::heap_clone_allocator, true>> = {
<boost::ptr_container_detail::associative_ptr_container<boost::ptr_container_detail::map_config<SandeshMessageTypeStats, std::map<std::basic_string<char, std::char_traits<char>, std::allocator<char> >, void*, std::less<std::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::allocator<std::pair<std::basic_string<char, std::char_traits<char>, std::allocator<char> > const, void*> > >, true>, boost::heap_clone_allocator>> = {
<boost::ptr_container_detail::reversible_ptr_container<boost::ptr_container_detail::map_config<SandeshMessageTypeStats, std::map<std::basic_string<char, std::char_traits<char>, std::allocator<char> >, void*, std::less<std::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::allocator<std::pair<std::basic_string<char, std::char_traits<char>, std::allocator<char> > const, void*> > >, true>, boost::heap_clone_allocator>> = {
static allow_null = false,
c_ = {
_M_t = {
_M_impl = {
<std::allocator<std::_Rb_tree_node<std::pair<std::basic_string<char, std::char_traits<char>, std::allocator<char> > const, void*> > >> = {
<__gnu_cxx::new_allocator<std::_Rb_tree_node<std::pair<std::basic_string<char, std::char_traits<char>, std::allocator<char> > const, void*> > >> = {<No data fields>}, <No data fields>},
members of std::_Rb_tree<std::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::pair<std::basic_string<char, std::char_traits<char>, std::allocator<char> > const, void*>, std::_Select1st<std::pair<std::basic_string<char, std::char_traits<char>, std::allocator<char> > const, void*> >, std::less<std::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::allocator<std::pair<std::basic_string<char, std::char_traits<char>, std::allocator<char> > const, void*> > >::_Rb_tree_impl<std::less<std::basic_string<char, std::char_traits<char>, std::allocator<char> > >, false>:
_M_key_compare = {
<std::binary_function<std::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::basic_string<char, std::char_traits<char>, std::allocator<char> >, bool>> = {<No data fields>}, <No data fields>},
_M_header = {
_M_color = std::_S_red,
_M_parent = 0x7eff7cf54460,
_M_left = 0x7effa0001420,
_M_right = 0x7effa808e4c0
},
_M_node_count = 21
}
}
}
}, <No data fields>}, <No data fields>}, <No data fields>}, <No data fields>},
detail_agg_stats_ = {
_vptr.SandeshMessageStats = 0x1415c70 <vtable for SandeshMessageStats+16>,
static ascii_fingerprint = 0x1433a00 "F82EE8FB1ED9259367155C92B2A8210E",
static binary_fingerprint = "\370.\350\373\036\331%\223g\025\\\222\262\250!\016",
messages_sent = 4053,
bytes_sent = 13059012,
messages_received = 1,
bytes_received = 940,
messages_sent_dropped = 34,
bytes_sent_dropped = 0,
messages_received_dropped = 0,
bytes_received_dropped = 0,
messages_sent_dropped_no_queue = 0,
messages_sent_dropped_no_client = 2,
messages_sent_dropped_no_session = 0,
messages_sent_dropped_queue_level = 9,
messages_sent_dropped_client_send_failed = 0,
messages_sent_dropped_session_not_connected = 0,
messages_sent_dropped_header_write_failed = 0,
messages_sent_dropped_write_failed = 0,
messages_sent_dropped_wrong_client_sm_state = 23,
messages_sent_dropped_validation_failed = 0,
messages_sent_dropped_rate_limited = 0,
bytes_sent_dropped_no_queue = 0,
bytes_sent_dropped_no_client = 0,
bytes_sent_dropped_no_session = 0,
bytes_sent_dropped_queue_level = 0,
bytes_sent_dropped_client_send_failed = 0,
bytes_sent_dropped_session_not_connected = 0,
bytes_sent_dropped_header_write_failed = 0,
bytes_sent_dropped_write_failed = 0,
bytes_sent_dropped_wrong_client_sm_state = 0,
bytes_sent_dropped_validation_failed = 0,
bytes_sent_dropped_rate_limited = 0,
messages_received_dropped_no_queue = 0,
messages_received_dropped_queue_level = 0,
messages_received_dropped_create_failed = 0,
messages_received_dropped_control_msg_failed = 0,
messages_received_dropped_decoding_failed = 0,
bytes_received_dropped_no_queue = 0,
bytes_received_dropped_queue_level = 0,
bytes_received_dropped_create_failed = 0,
bytes_received_dropped_control_msg_failed = 0,
bytes_received_dropped_decoding_failed = 0,
__isset = {
messages_sent = false,
bytes_sent = false,
messages_received = false,
bytes_received = false,
messages_sent_dropped = false,
bytes_sent_dropped = false,
messages_received_dropped = false,
bytes_received_dropped = false,
messages_sent_dropped_no_queue = false,
messages_sent_dropped_no_client = false,
messages_sent_dropped_no_session = false,
messages_sent_dropped_queue_level = false,
messages_sent_dropped_client_send_failed = false,
messages_sent_dropped_session_not_connected = false,
messages_sent_dropped_header_write_failed = false,
messages_sent_dropped_write_failed = false,
messages_sent_dropped_wrong_client_sm_state = false,
messages_sent_dropped_validation_failed = false,
messages_sent_dropped_rate_limited = false,
bytes_sent_dropped_no_queue = false,
bytes_sent_dropped_no_client = false,
bytes_sent_dropped_no_session = false,
bytes_sent_dropped_queue_level = false,
bytes_sent_dropped_client_send_failed = false,
bytes_sent_dropped_session_not_connected = false,
bytes_sent_dropped_header_write_failed = false,
bytes_sent_dropped_write_failed = false,
bytes_sent_dropped_wrong_client_sm_state = false,
bytes_sent_dropped_validation_failed = false,
bytes_sent_dropped_rate_limited = false,
messages_received_dropped_no_queue = false,
messages_received_dropped_queue_level = false,
messages_received_dropped_create_failed = false,
messages_received_dropped_control_msg_failed = false,
messages_received_dropped_decoding_failed = false,
bytes_received_dropped_no_queue = false,
bytes_received_dropped_queue_level = false,
bytes_received_dropped_create_failed = false,
bytes_received_dropped_control_msg_failed = false,
bytes_received_dropped_decoding_failed = false
}
}
},
static stats_mutex_ = {
static is_rw_mutex = false,
static is_recursive_mutex = false,
static is_fair_mutex = false,
impl = {
__data = {
__lock = 0,
__count = 0,
__owner = 0,
__nusers = 0,
__kind = 0,
__spins = 0,
__elision = 0,
__list = {
__prev = 0x0,
__next = 0x0
}
},
__size = '\000' <repeats 39 times>,
__align = 0
}
},
static logger_ = {<No data fields>},
static disable_flow_collection_ = false,
seqnum_ = 2426,
context_ = {
static npos = <optimized out>,
_M_dataplus = {
<std::allocator<char>> = {
<__gnu_cxx::new_allocator<char>> = {<No data fields>}, <No data fields>},
members of std::basic_string<char, std::char_traits<char>, std::allocator<char> >::_Alloc_hider:
_M_p = 0x1911078 <std::string::_Rep::_S_empty_rep_storage@@GLIBCXX_3.4+24> ""
}
},
timestamp_ = 0,
scope_ = {
static npos = <optimized out>,
_M_dataplus = {
<std::allocator<char>> = {
<__gnu_cxx::new_allocator<char>> = {<No data fields>}, <No data fields>},
members of std::basic_string<char, std::char_traits<char>, std::allocator<char> >::_Alloc_hider:
_M_p = 0x1911078 <std::string::_Rep::_S_empty_rep_storage@@GLIBCXX_3.4+24> ""
}
},
type_ = SandeshType::BUFFER,
hints_ = 0,
level_ = SandeshLevel::INVALID,
category_ = {
static npos = <optimized out>,
_M_dataplus = {
<std::allocator<char>> = {
<__gnu_cxx::new_allocator<char>> = {<No data fields>}, <No data fields>},
members of std::basic_string<char, std::char_traits<char>, std::allocator<char> >::_Alloc_hider:
_M_p = 0x1911078 <std::string::_Rep::_S_empty_rep_storage@@GLIBCXX_3.4+24> ""
}
},
name_ = {
static npos = <optimized out>,
_M_dataplus = {
<std::allocator<char>> = {
<__gnu_cxx::new_allocator<char>> = {<No data fields>}, <No data fields>},
members of std::basic_string<char, std::char_traits<char>, std::allocator<char> >::_Alloc_hider:
_M_p = 0x7effa930d6f8 "vr_nexthop_req"
}
},
static sandesh_send_ratelimit_ = {
<tbb::internal::atomic_impl_with_arithmetic<unsigned int, unsigned int, char>> = {
<tbb::internal::atomic_impl<unsigned int>> = {
my_storage = {
my_value = 10
}
}, <No data fields>}, <No data fields>}
}, <No data fields>},
members of vr_nexthop_req:
static ascii_fingerprint = 0x1383400 "3215EA4B5433215282CCCF8A58B450BA",
static binary_fingerprint = "2\025\352KT3!R\202\314ϊX\264P\272",
__isset = {
h_op = false,
nhr_type = false,
nhr_family = false,
nhr_id = false,
nhr_rid = false,
nhr_encap_oif_id = false,
nhr_encap_len = false,
nhr_encap_family = false,
nhr_vrf = false,
nhr_tun_sip = false,
nhr_tun_dip = false,
nhr_tun_sport = false,
nhr_tun_dport = false,
nhr_ref_cnt = false,
nhr_marker = false,
nhr_flags = false,
nhr_encap = false,
nhr_nh_list = false,
nhr_label = false,
nhr_label_list = false,
nhr_nh_count = false,
nhr_tun_sip6 = false,
nhr_tun_dip6 = false
},
static reg = {
<SandeshBaseFactory> = {<No data fields>},
members of SandeshDerivedRegister<vr_nexthop_req>:
name_ = {
static npos = <optimized out>,
_M_dataplus = {
<std::allocator<char>> = {
<__gnu_cxx::new_allocator<char>> = {<No data fields>}, <No data fields>},
members of std::basic_string<char, std::char_traits<char>, std::allocator<char> >::_Alloc_hider:
_M_p = 0x3356048 "vr_nexthop_req"
}
}
},
h_op = sandesh_op::ADD,
nhr_type = 6 '\006',
nhr_family = 2 '\002',
nhr_id = 3563,
nhr_rid = 0,
nhr_encap_oif_id = 0,
nhr_encap_len = 0,
nhr_encap_family = 2054,
nhr_vrf = 1,
nhr_tun_sip = 0,
nhr_tun_dip = 0,
nhr_tun_sport = 0,
nhr_tun_dport = 0,
nhr_ref_cnt = 0,
nhr_marker = 0,
nhr_flags = 8193,
nhr_encap = {
<std::_Vector_base<signed char, std::allocator<signed char> >> = {
_M_impl = {
<std::allocator<signed char>> = {
<__gnu_cxx::new_allocator<signed char>> = {<No data fields>}, <No data fields>},
members of std::_Vector_base<signed char, std::allocator<signed char> >::_Vector_impl:
_M_start = 0x0,
_M_finish = 0x0,
_M_end_of_storage = 0x0
}
}, <No data fields>},
nhr_nh_list = {
<std::_Vector_base<int, std::allocator<int> >> = {
_M_impl = {
<std::allocator<int>> = {
<__gnu_cxx::new_allocator<int>> = {<No data fields>}, <No data fields>},
members of std::_Vector_base<int, std::allocator<int> >::_Vector_impl:
_M_start = 0x7effa8b2a330,
_M_finish = 0x7effa8b2abec,
_M_end_of_storage = 0x7effa8b2abec
}
}, <No data fields>},
nhr_label = -1,
nhr_label_list = {
<std::_Vector_base<int, std::allocator<int> >> = {
_M_impl = {
<std::allocator<int>> = {
<__gnu_cxx::new_allocator<int>> = {<No data fields>}, <No data fields>},
members of std::_Vector_base<int, std::allocator<int> >::_Vector_impl:
_M_start = 0x7effa89ff8d0,
_M_finish = 0x7effa8a0018c,
_M_end_of_storage = 0x7effa8a0018c
}
}, <No data fields>},
nhr_nh_count = 0,
nhr_tun_sip6 = {
<std::_Vector_base<signed char, std::allocator<signed char> >> = {
_M_impl = {
<std::allocator<signed char>> = {
<__gnu_cxx::new_allocator<signed char>> = {<No data fields>}, <No data fields>},
members of std::_Vector_base<signed char, std::allocator<signed char> >::_Vector_impl:
_M_start = 0x0,
_M_finish = 0x0,
_M_end_of_storage = 0x0
}
}, <No data fields>},
nhr_tun_dip6 = {
<std::_Vector_base<signed char, std::allocator<signed char> >> = {
_M_impl = {
<std::allocator<signed char>> = {
<__gnu_cxx::new_allocator<signed char>> = {<No data fields>}, <No data fields>},
members of std::_Vector_base<signed char, std::allocator<signed char> >::_Vector_impl:
_M_start = 0x0,
_M_finish = 0x0,
_M_end_of_storage = 0x0
}
}, <No data fields>},
static lseqnum_ = 2427,
static versionsig_ = 1273632050,
static name_ = <optimized out>
} |
|
