agent continuous crash @ NHKSyncEntry::Encode(sandesh_op::type, char*, int): Assertion `encode_len <= buf_len' failed.
Affects | Status | Importance | Assigned to | Milestone | ||
---|---|---|---|---|---|---|
Juniper Openstack | Status tracked in Trunk | |||||
R3.0 |
Fix Committed
|
High
|
Hari Prasad Killi | |||
R3.1 |
Fix Committed
|
High
|
Hari Prasad Killi | |||
R3.2 |
Fix Committed
|
High
|
Hari Prasad Killi | |||
Trunk |
Fix Committed
|
High
|
Hari Prasad Killi |
Bug Description
Have 500 odd vms in the compute and agent crashes continuously with below trace.
FYI, the other two computes with 250 odd vms each are stable.
BTW, all the vms are launched in a single VN who has 1000 subnets and each subnet has one VM.
contrail-version:
contrail-
contrail-
the cores are copied on to /cs-shared/
(gdb) bt
#0 0x00007effb46aecc9 in __GI_raise (sig=sig@entry=6) at ../nptl/
#1 0x00007effb46b20d8 in __GI_abort () at abort.c:89
#2 0x00007effb46a7b86 in __assert_fail_base (fmt=0x7effb47f8830 "%s%s%s:%u: %s%sAssertion `%s' failed.\n%n", assertion=
file=
function=
#3 0x00007effb46a7c32 in __GI___assert_fail (assertion=
function=
#4 0x0000000000dacbee in NHKSyncEntry:
#5 0x0000000000dacd73 in NHKSyncEntry:
#6 0x0000000000dcea21 in KSyncNetlinkDBE
#7 0x0000000000e80dd5 in KSyncSM_Add (obj=0x7eff9400
#8 0x0000000000e81dd5 in KSyncObject:
#9 0x0000000000e8339c in KSyncDBObject:
#10 0x00000000011a3b3a in operator() (a1=0x7eff80876638, a0=0x7eff954235a0, this=0x7effadd3
#11 RunNotify (entry=
#12 DBTableBase:
#13 0x00000000011a6348 in DBTablePartBase
#14 0x00000000011a1f7e in DBPartition:
#15 0x00000000012e4dff in TaskImpl::execute (this=0x7effade
#16 0x00007effb527db3a in ?? () from /usr/lib/
#17 0x00007effb5279816 in ?? () from /usr/lib/
#18 0x00007effb5278f4b in ?? () from /usr/lib/
#19 0x00007effb52750ff in ?? () from /usr/lib/
#20 0x00007effb52752f9 in ?? () from /usr/lib/
#21 0x00007effb5499182 in start_thread (arg=0x7effadd3
#22 0x00007effb477247d in clone () at ../sysdeps/
(gdb) p buf_len
$1 = 4096
(gdb) p buf
$2 = 0x7effa9491470 ""
(gdb) p encode_len
$3 = <optimized out>
(gdb) p encoder
$4 = {
<SandeshBuffer> = {
<Sandesh> = {
_vptr.Sandesh = 0x1384550 <vtable for vr_nexthop_req+16>,
static response_callback_ = {
vtable = 0x0,
functor = {
type = {
},
},
},
data = 0 '\000'
}
},
members of boost::
static args = <optimized out>,
static arity = <optimized out>
}, <No data fields>},
static client_ = 0x7eff7cf44df0,
static role_ = Sandesh:
static module_ = {
static npos = <optimized out>,
_M_dataplus = {
members of std::basic_
_M_p = 0x342f798 "contrail-
}
},
static source_ = {
static npos = <optimized out>,
_M_dataplus = {
members of std::basic_
_M_p = 0x3446f48 "a2s36"
}
},
static node_type_ = {
static npos = <optimized out>,
_M_dataplus = {
members of std::basic_
_M_p = 0x34312d8 "Compute"
}
},
static instance_id_ = {
static npos = <optimized out>,
_M_dataplus = {
members of std::basic_
_M_p = 0x3431a18 "0"
}
},
static http_port_ = 8085,
static recv_queue_ = {
_M_ptr = 0x7eff7cf44c40
},
static recv_task_id_ = 11,
static client_context_ = 0x0,
static module_context_ = {
_M_t = {
_M_impl = {
members of std::_Rb_
},
}
}
},
static enable_local_log_ = true,
static enable_flow_log_ = false,
static logging_level_ = SandeshLevel:
static logging_ut_level_ = SandeshLevel:
static logging_category_ = {
static npos = <optimized out>,
_M_dataplus = {
members of std::basic_
_M_p = 0x1911078 <std::string:
}
},
static enable_trace_print_ = false,
static connect_
static event_manager_ = 0x34502f0,
static send_queue_enabled_ = true,
static sending_level_ = SandeshLevel:
static msg_stats_ = {
}
}, <No data fields>}, <No data fields>}, <No data fields>}, <No data fields>},
static ascii_fingerprint = 0x1433a00 "F82EE8FB1ED925
static binary_fingerprint = "\370.\
__isset = {
}
}
},
static stats_mutex_ = {
static is_rw_mutex = false,
static is_recursive_mutex = false,
static is_fair_mutex = false,
impl = {
__data = {
__lock = 0,
__count = 0,
__owner = 0,
__kind = 0,
__spins = 0,
__list = {
}
},
__size = '\000' <repeats 39 times>,
__align = 0
}
},
static logger_ = {<No data fields>},
static disable_
seqnum_ = 2426,
context_ = {
static npos = <optimized out>,
_M_dataplus = {
members of std::basic_
_M_p = 0x1911078 <std::string:
}
},
timestamp_ = 0,
scope_ = {
static npos = <optimized out>,
_M_dataplus = {
members of std::basic_
_M_p = 0x1911078 <std::string:
}
},
type_ = SandeshType:
hints_ = 0,
level_ = SandeshLevel:
category_ = {
static npos = <optimized out>,
_M_dataplus = {
members of std::basic_
_M_p = 0x1911078 <std::string:
}
},
name_ = {
static npos = <optimized out>,
_M_dataplus = {
members of std::basic_
_M_p = 0x7effa930d6f8 "vr_nexthop_req"
}
},
static sandesh_
}
}, <No data fields>}, <No data fields>}
}, <No data fields>},
members of vr_nexthop_req:
static ascii_fingerprint = 0x1383400 "3215EA4B543321
static binary_fingerprint = "2\025\
__isset = {
h_op = false,
nhr_type = false,
nhr_family = false,
nhr_id = false,
nhr_rid = false,
nhr_
nhr_encap_len = false,
nhr_
nhr_vrf = false,
nhr_tun_sip = false,
nhr_tun_dip = false,
nhr_tun_sport = false,
nhr_tun_dport = false,
nhr_ref_cnt = false,
nhr_marker = false,
nhr_flags = false,
nhr_encap = false,
nhr_nh_list = false,
nhr_label = false,
nhr_label_list = false,
nhr_nh_count = false,
nhr_tun_sip6 = false,
nhr_tun_dip6 = false
},
static reg = {
<SandeshBas
members of SandeshDerivedR
name_ = {
static npos = <optimized out>,
_M_dataplus = {
members of std::basic_
_M_p = 0x3356048 "vr_nexthop_req"
}
}
},
h_op = sandesh_op::ADD,
nhr_type = 6 '\006',
nhr_family = 2 '\002',
nhr_id = 3563,
nhr_rid = 0,
nhr_encap_oif_id = 0,
nhr_encap_len = 0,
nhr_encap_family = 2054,
nhr_vrf = 1,
nhr_tun_sip = 0,
nhr_tun_dip = 0,
nhr_tun_sport = 0,
nhr_tun_dport = 0,
nhr_ref_cnt = 0,
nhr_marker = 0,
nhr_flags = 8193,
nhr_encap = {
<std:
_M_impl = {
members of std::_Vector_
_M_start = 0x0,
_M_finish = 0x0,
}
}, <No data fields>},
nhr_nh_list = {
<std:
_M_impl = {
members of std::_Vector_
_M_start = 0x7effa8b2a330,
_M_finish = 0x7effa8b2abec,
}
}, <No data fields>},
nhr_label = -1,
nhr_label_list = {
<std:
_M_impl = {
members of std::_Vector_
_M_start = 0x7effa89ff8d0,
_M_finish = 0x7effa8a0018c,
}
}, <No data fields>},
nhr_nh_count = 0,
nhr_tun_sip6 = {
<std:
_M_impl = {
members of std::_Vector_
_M_start = 0x0,
_M_finish = 0x0,
}
}, <No data fields>},
nhr_tun_dip6 = {
<std:
_M_impl = {
members of std::_Vector_
_M_start = 0x0,
_M_finish = 0x0,
}
}, <No data fields>},
static lseqnum_ = 2427,
static versionsig_ = 1273632050,
static name_ = <optimized out>
}
description: | updated |
Review in progress for https:/ /review. opencontrail. org/25092
Submitter: Hari Prasad Killi (<email address hidden>)