Vrouter agent crash in virtual tbb::task* TaskImpl::execute()") at assert.c:101

[Vinod Nair]

Agent crashed in virtual tbb::task* TaskImpl::execute()") at assert.c:101 soon after starting traffic from a dpodk vm

ore was generated by `/usr/bin/contrail-vrouter-agent'.
Program terminated with signal SIGABRT, Aborted.
#0 0x00007fd2ee0bbc37 in __GI_raise (sig=sig@entry=6) at ../nptl/sysdeps/unix/sysv/linux/raise.c:56
56 ../nptl/sysdeps/unix/sysv/linux/raise.c: No such file or directory.
(gdb) bt
#0 0x00007fd2ee0bbc37 in __GI_raise (sig=sig@entry=6) at ../nptl/sysdeps/unix/sysv/linux/raise.c:56
#1 0x00007fd2ee0bf028 in __GI_abort () at abort.c:89
#2 0x00007fd2ee0b4bf6 in __assert_fail_base (fmt=0x7fd2ee2053b8 "%s%s%s:%u: %s%sAssertion `%s' failed.\n%n", assertion=assertion@entry=0x1350495 "0",
    file=file@entry=0x1467860 "controller/src/base/task.cc", line=line@entry=291,
    function=function@entry=0x14691e0 <TaskImpl::execute()::__PRETTY_FUNCTION__> "virtual tbb::task* TaskImpl::execute()") at assert.c:92
#3 0x00007fd2ee0b4ca2 in __GI___assert_fail (assertion=0x1350495 "0", file=0x1467860 "controller/src/base/task.cc", line=291,
    function=0x14691e0 <TaskImpl::execute()::__PRETTY_FUNCTION__> "virtual tbb::task* TaskImpl::execute()") at assert.c:101
#4 0x00000000012fbd73 in TaskImpl::execute (this=<optimized out>) at controller/src/base/task.cc:291
#5 0x00007fd2eec8ab3a in ?? () from /usr/lib/libtbb.so.2
#6 0x00007fd2eec86816 in ?? () from /usr/lib/libtbb.so.2
#7 0x00007fd2eec85f4b in ?? () from /usr/lib/libtbb.so.2
#8 0x00007fd2eec820ff in ?? () from /usr/lib/libtbb.so.2
#9 0x00007fd2eec822f9 in ?? () from /usr/lib/libtbb.so.2
#10 0x00007fd2eeea6184 in start_thread (arg=0x7fd2e6119700) at pthread_create.c:312
#11 0x00007fd2ee17f37d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111
(gdb) bt full
#0 0x00007fd2ee0bbc37 in __GI_raise (sig=sig@entry=6) at ../nptl/sysdeps/unix/sysv/linux/raise.c:56
        resultvar = 0
        pid = 3683
        selftid = 3704
#1 0x00007fd2ee0bf028 in __GI_abort () at abort.c:89
        save_stage = 2
        act = {__sigaction_handler = {sa_handler = 0x7fffb01a1e5e, sa_sigaction = 0x7fffb01a1e5e}, sa_mask = {__val = {140543914940676, 21395552, 291, 4294967295,
              140543913583955, 0, 140543779768912, 26438784, 21395552, 290, 0, 0, 0, 21474836480, 140543959871488, 140543914955704}}, sa_flags = 20251797,
          sa_restorer = 0x14691e0 <TaskImpl::execute()::__PRETTY_FUNCTION__>}
        sigs = {__val = {32, 0 <repeats 15 times>}}
#2 0x00007fd2ee0b4bf6 in __assert_fail_base (fmt=0x7fd2ee2053b8 "%s%s%s:%u: %s%sAssertion `%s' failed.\n%n", assertion=assertion@entry=0x1350495 "0",
    file=file@entry=0x1467860 "controller/src/base/task.cc", line=line@entry=291,
    function=function@entry=0x14691e0 <TaskImpl::execute()::__PRETTY_FUNCTION__> "virtual tbb::task* TaskImpl::execute()") at assert.c:92
        str = 0x7fd2c81e3770 ""
        total = 4096
#3 0x00007fd2ee0b4ca2 in __GI___assert_fail (assertion=0x1350495 "0", file=0x1467860 "controller/src/base/task.cc", line=291,
    function=0x14691e0 <TaskImpl::execute()::__PRETTY_FUNCTION__> "virtual tbb::task* TaskImpl::execute()") at assert.c:101
No locals.
#4 0x00000000012fbd73 in TaskImpl::execute (this=<optimized out>) at controller/src/base/task.cc:291
        what = "std::bad_alloc"
        e = <optimized out>
        running = <optimized out>
        __PRETTY_FUNCTION__ = "virtual tbb::task* TaskImpl::execute()"
#5 0x00007fd2eec8ab3a in ?? () from /usr/lib/libtbb.so.2
No symbol table info available.
#6 0x00007fd2eec86816 in ?? () from /usr/lib/libtbb.so.2
---Type <return> to continue, or q <return> to quit---
No symbol table info available.
#7 0x00007fd2eec85f4b in ?? () from /usr/lib/libtbb.so.2
No symbol table info available.
#8 0x00007fd2eec820ff in ?? () from /usr/lib/libtbb.so.2
No symbol table info available.
#9 0x00007fd2eec822f9 in ?? () from /usr/lib/libtbb.so.2
No symbol table info available.
#10 0x00007fd2eeea6184 in start_thread (arg=0x7fd2e6119700) at pthread_create.c:312
        __res = <optimized out>
        pd = 0x7fd2e6119700
        now = <optimized out>
        unwind_buf = {cancel_jmp_buf = {{jmp_buf = {140543779772160, 8186071939107276645, 0, 0, 140543779772864, 140543779772160, -8196822705613813915, -8196805470752454811},
              mask_was_saved = 0}}, priv = {pad = {0x0, 0x0, 0x0, 0x0}, data = {prev = 0x0, cleanup = 0x0, canceltype = 0}}}
        not_first_call = <optimized out>
        pagesize_m1 = <optimized out>
        sp = <optimized out>
        freesize = <optimized out>
        __PRETTY_FUNCTION__ = "start_thread"
#11 0x00007fd2ee17f37d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111

Core in /cs-shared/bugs/
Version: 3.2.1.0-22 Mitaka 14.04.5

[Hari Prasad Killi (haripk)]

"std::bad_alloc" exception.

[Vinod Nair (vinodnair)]

See similar crashes in 3.2.B25 also
[Thread debugging using libthread_db enabled]
Using host libthread_db library "/lib/x86_64-linux-gnu/libthread_db.so.1".
Core was generated by `/usr/bin/contrail-vrouter-agent'.
Program terminated with signal SIGABRT, Aborted.
#0 0x00007f3de84e5c37 in __GI_raise (sig=sig@entry=6) at ../nptl/sysdeps/unix/sysv/linux/raise.c:56
56 ../nptl/sysdeps/unix/sysv/linux/raise.c: No such file or directory.
(gdb) bt
#0 0x00007f3de84e5c37 in __GI_raise (sig=sig@entry=6) at ../nptl/sysdeps/unix/sysv/linux/raise.c:56
#1 0x00007f3de84e9028 in __GI_abort () at abort.c:89
#2 0x00007f3de84debf6 in __assert_fail_base (fmt=0x7f3de862f3b8 "%s%s%s:%u: %s%sAssertion `%s' failed.\n%n", assertion=assertion@entry=0x1353ad5 "0",
    file=file@entry=0x146ba00 "controller/src/base/task.cc", line=line@entry=291,
    function=function@entry=0x146d380 <TaskImpl::execute()::__PRETTY_FUNCTION__> "virtual tbb::task* TaskImpl::execute()") at assert.c:92
#3 0x00007f3de84deca2 in __GI___assert_fail (assertion=0x1353ad5 "0", file=0x146ba00 "controller/src/base/task.cc", line=291,
    function=0x146d380 <TaskImpl::execute()::__PRETTY_FUNCTION__> "virtual tbb::task* TaskImpl::execute()") at assert.c:101
#4 0x00000000012ff3c3 in TaskImpl::execute (this=<optimized out>) at controller/src/base/task.cc:291
#5 0x00007f3de90b4b3a in ?? () from /usr/lib/libtbb.so.2
#6 0x00007f3de90b0816 in ?? () from /usr/lib/libtbb.so.2
#7 0x00007f3de90aff4b in ?? () from /usr/lib/libtbb.so.2
#8 0x00007f3de90ac0ff in ?? () from /usr/lib/libtbb.so.2
#9 0x00007f3de90ac2f9 in ?? () from /usr/lib/libtbb.so.2
#10 0x00007f3de92d0184 in start_thread (arg=0x7f3dcbbfe700) at pthread_create.c:312
#11 0x00007f3de85a937d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111