root@sriov:~/var/log# cd coredump/ root@sriov:~/var/log/coredump# ls core.contrail-vroute.24365.rdm5r16c015.rdm5b.cci.att.com.1504119226 core.contrail-vroute.37795.rdm5r16c015.rdm5b.cci.att.com.1504046049 core.contrail-vroute.5744.rdm5r16c015.rdm5b.cci.att.com.1504110311 core.contrail-vroute.34655.rdm5r16c015.rdm5b.cci.att.com.1504116100 core.contrail-vroute.41156.rdm5r16c015.rdm5b.cci.att.com.1504128467 root@sriov:~/var/log/coredump# date -d @1504119226 Wed Aug 30 11:53:46 PDT 2017 root@sriov:~/var/log/coredump# date -d @1504046049 Tue Aug 29 15:34:09 PDT 2017 root@sriov:~/var/log/coredump# date -d @1504110311 Wed Aug 30 09:25:11 PDT 2017 root@sriov:~/var/log/coredump# date -d @1504116100 Wed Aug 30 11:01:40 PDT 2017 root@sriov:~/var/log/coredump# date -d @1504128467 Wed Aug 30 14:27:47 PDT 2017 root@sriov:~/var/log/coredump# root@sriov:~/var/log/coredump# strings core.contrail-vroute.37795.rdm5r16c015.rdm5b.cci.att.com.1504046049 | grep build-info {"build-info": [{"build-time": "2017-06-29 09:59:25.240327", "build-hostname": "ubuntu", "build-user": "contrail-builder", "build-version": "3.2.3.2"}]} {"build-info":[{"build-time":"2017-06-29 09:59:25.240327","build-hostname":"ubuntu","build-user":"contrail-builder","build-version":"3.2.3.2","build-id":"3.2.3.2-6","build-number":"6"}]} {"build-infoA {"build-info":[{"build-time":"2017-06-29 09:59:25.240327","build-hostname":"ubuntu","build-user":"contrail-builder","build-version":"3.2.3.2","build-id":"3.2.3.2-6","build-number":"6"}]} {"build-info": [{"build-time": "2017-06-29 10:47:18.086620", "build-hostname": "ubuntu", "build-user": "contrail-builder", "build-version": "3.2.3.2"}]} root@sriov:~/var/log/coredump# contrail-version Package Version Build-ID | Repo | Package Name -------------------------------------- ------------------------------ ---------------------------------- contrail-analytics 3.2.3.2-6 6 contrail-config 3.2.3.2-6 6 contrail-config-openstack 3.2.3.2-6 6 contrail-control 3.2.3.2-6 6 contrail-database-common 3.2.3.2-6 6 contrail-dns 3.2.3.2-6 6 contrail-docs 3.2.3.2-6 6 contrail-f5 3.2.3.2-6 6 contrail-fabric-utils 3.2.3.2-6 6 contrail-heat 3.2.3.2-6 6 contrail-install-packages 3.2.3.2-6~mitaka 6 contrail-lib 3.2.3.2-6 6 contrail-nodemgr 3.2.3.2-6 6 contrail-nova-networkapi 3.2.3.2-6 6 contrail-nova-vif 3.2.3.2-6 6 contrail-openstack 3.2.3.2-6 6 contrail-openstack-analytics 3.2.3.2-6 6 contrail-openstack-config 3.2.3.2-6 6 contrail-openstack-control 3.2.3.2-6 6 contrail-openstack-dashboard 3.2.3.2-6 6 contrail-openstack-database 3.2.3.2-6 6 contrail-openstack-vrouter 3.2.3.2-6 6 contrail-openstack-webui 3.2.3.2-6 6 contrail-setup 3.2.3.2-6 6 contrail-utils 3.2.3.2-6 6 contrail-vrouter-agent 3.2.3.2-6 6 contrail-vrouter-common 3.2.3.2-6 6 contrail-vrouter-dkms 3.2.3.2-6 6 contrail-vrouter-init 3.2.3.2-6 6 contrail-vrouter-utils 3.2.3.2-6 6 contrail-web-controller 3.2.3.2-6 6 contrail-web-core 3.2.3.2-6 6 ifmap-python-client 0.1-2 6 ifmap-server 0.3.2-1contrail3 6 neutron-plugin-contrail 3.2.3.2-6 6 nova-api 2:13.0.0-0ubuntu2~cloud0.1contrail1 6 nova-common 2:13.0.0-0ubuntu2~cloud0.1contrail1 6 nova-compute 2:13.0.0-0ubuntu2~cloud0.1contrail1 6 nova-compute-kvm 2:13.0.0-0ubuntu2~cloud0.1contrail1 6 nova-compute-libvirt 2:13.0.0-0ubuntu2~cloud0.1contrail1 6 nova-conductor 2:13.0.0-0ubuntu2~cloud0.1contrail1 6 nova-console 2:13.0.0-0ubuntu2~cloud0.1contrail1 6 nova-consoleauth 2:13.0.0-0ubuntu2~cloud0.1contrail1 6 nova-novncproxy 2:13.0.0-0ubuntu2~cloud0.1contrail1 6 nova-scheduler 2:13.0.0-0ubuntu2~cloud0.1contrail1 6 python-contrail 3.2.3.2-6 6 python-contrail-vrouter-api 3.2.3.2-6 6 python-neutronclient 1:4.1.1-2~cloud0.2contrail 6 python-nova 2:13.0.0-0ubuntu2~cloud0.1contrail1 6 python-opencontrail-vrouter-netns 3.2.3.2-6 6 root@sriov:~/var/log/coredump# root@sriov:~/var/log/coredump# root@sriov:~/var/log/coredump# root@sriov:~/var/log/coredump# gdb `which contrail-vrouter-agent` core.contrail-vroute.24365.rdm5r16c015.rdm5b.cci.att.com.1504119226 GNU gdb (Ubuntu 7.7.1-0ubuntu5~14.04.3) 7.7.1 Copyright (C) 2014 Free Software Foundation, Inc. License GPLv3+: GNU GPL version 3 or later This is free software: you are free to change and redistribute it. There is NO WARRANTY, to the extent permitted by law. Type "show copying" and "show warranty" for details. This GDB was configured as "x86_64-linux-gnu". Type "show configuration" for configuration details. For bug reporting instructions, please see: . Find the GDB manual and other documentation resources online at: . For help, type "help". Type "apropos word" to search for commands related to "word"... Reading symbols from /usr/bin/contrail-vrouter-agent...(no debugging symbols found)...done. BFD: Warning: /root/var/log/coredump/core.contrail-vroute.24365.rdm5r16c015.rdm5b.cci.att.com.1504119226 is truncated: expected core file size >= 1510924288, found: 364838912. warning: core file may not match specified executable file. [New LWP 24383] [New LWP 24365] [New LWP 24384] [New LWP 24386] [New LWP 24382] [New LWP 24388] [New LWP 24387] [New LWP 24385] [New LWP 24381] Cannot access memory at address 0x7fa1c45081a8 Cannot access memory at address 0x7fa1c45081a0 (gdb) bt Python Exception Cannot access memory at address 0x7fa1ba541758: #0 0x00007fa1c18e1c37 in ?? () Cannot access memory at address 0x7fa1ba541758 (gdb) quit root@sriov:~/var/log/coredump# gdb `which contrail-vrouter-agent` core.contrail-vroute.37795.rdm5r16c015.rdm5b.cci.att.com.1504046049 GNU gdb (Ubuntu 7.7.1-0ubuntu5~14.04.3) 7.7.1 Copyright (C) 2014 Free Software Foundation, Inc. License GPLv3+: GNU GPL version 3 or later This is free software: you are free to change and redistribute it. There is NO WARRANTY, to the extent permitted by law. Type "show copying" and "show warranty" for details. This GDB was configured as "x86_64-linux-gnu". Type "show configuration" for configuration details. For bug reporting instructions, please see: . Find the GDB manual and other documentation resources online at: . For help, type "help". Type "apropos word" to search for commands related to "word"... Reading symbols from /usr/bin/contrail-vrouter-agent...(no debugging symbols found)...done. warning: core file may not match specified executable file. [New LWP 37815] [New LWP 37818] [New LWP 37795] [New LWP 37819] [New LWP 37814] [New LWP 37812] [New LWP 37813] [New LWP 37816] [New LWP 37817] [Thread debugging using libthread_db enabled] Using host libthread_db library "/lib/x86_64-linux-gnu/libthread_db.so.1". Core was generated by `/usr/bin/contrail-vrouter-agent'. Program terminated with signal SIGABRT, Aborted. #0 0x00007f1d7dd5bc37 in raise () from /lib/x86_64-linux-gnu/libc.so.6 (gdb) bt #0 0x00007f1d7dd5bc37 in raise () from /lib/x86_64-linux-gnu/libc.so.6 #1 0x00007f1d7dd5f028 in abort () from /lib/x86_64-linux-gnu/libc.so.6 #2 0x00007f1d7dd54bf6 in ?? () from /lib/x86_64-linux-gnu/libc.so.6 #3 0x00007f1d7dd54ca2 in __assert_fail () from /lib/x86_64-linux-gnu/libc.so.6 #4 0x0000000000cec455 in FlowEntryFreeList::Free(FlowEntry*) () #5 0x0000000000cdd528 in intrusive_ptr_release(FlowEntry*) () #6 0x0000000000cf7199 in FlowMgmtRequest::~FlowMgmtRequest() () #7 0x000000000087bcce in boost::detail::sp_counted_base::release() () #8 0x0000000000cfd435 in QueueTaskRunner, WorkQueue > >::RunQueue() () #9 0x000000000130560f in TaskImpl::execute() () #10 0x00007f1d7e92ab3a in ?? () from /usr/lib/libtbb.so.2 #11 0x00007f1d7e926816 in ?? () from /usr/lib/libtbb.so.2 #12 0x00007f1d7e925f4b in ?? () from /usr/lib/libtbb.so.2 #13 0x00007f1d7e9220ff in ?? () from /usr/lib/libtbb.so.2 #14 0x00007f1d7e9222f9 in ?? () from /usr/lib/libtbb.so.2 #15 0x00007f1d7eb46184 in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0 #16 0x00007f1d7de1f37d in clone () from /lib/x86_64-linux-gnu/libc.so.6 (gdb) quit root@sriov:~/var/log/coredump# ls core.contrail-vroute.24365.rdm5r16c015.rdm5b.cci.att.com.1504119226 core.contrail-vroute.37795.rdm5r16c015.rdm5b.cci.att.com.1504046049 core.contrail-vroute.5744.rdm5r16c015.rdm5b.cci.att.com.1504110311 core.contrail-vroute.34655.rdm5r16c015.rdm5b.cci.att.com.1504116100 core.contrail-vroute.41156.rdm5r16c015.rdm5b.cci.att.com.1504128467 root@sriov:~/var/log/coredump# gdb `which contrail-vrouter-agent` core.contrail-vroute.5744.rdm5r16c015.rdm5b.cci.att.com.1504110311 GNU gdb (Ubuntu 7.7.1-0ubuntu5~14.04.3) 7.7.1 Copyright (C) 2014 Free Software Foundation, Inc. License GPLv3+: GNU GPL version 3 or later This is free software: you are free to change and redistribute it. There is NO WARRANTY, to the extent permitted by law. Type "show copying" and "show warranty" for details. This GDB was configured as "x86_64-linux-gnu". Type "show configuration" for configuration details. For bug reporting instructions, please see: . Find the GDB manual and other documentation resources online at: . For help, type "help". Type "apropos word" to search for commands related to "word"... Reading symbols from /usr/bin/contrail-vrouter-agent...(no debugging symbols found)...done. warning: core file may not match specified executable file. [New LWP 5764] [New LWP 5744] [New LWP 5762] [New LWP 5761] [New LWP 5768] [New LWP 5763] [New LWP 5766] [New LWP 5767] [New LWP 5765] [Thread debugging using libthread_db enabled] Using host libthread_db library "/lib/x86_64-linux-gnu/libthread_db.so.1". Core was generated by `/usr/bin/contrail-vrouter-agent'. Program terminated with signal SIGABRT, Aborted. #0 0x00007fac410a1c37 in raise () from /lib/x86_64-linux-gnu/libc.so.6 (gdb) bt #0 0x00007fac410a1c37 in raise () from /lib/x86_64-linux-gnu/libc.so.6 #1 0x00007fac410a5028 in abort () from /lib/x86_64-linux-gnu/libc.so.6 #2 0x00007fac4109abf6 in ?? () from /lib/x86_64-linux-gnu/libc.so.6 #3 0x00007fac4109aca2 in __assert_fail () from /lib/x86_64-linux-gnu/libc.so.6 #4 0x0000000000cec455 in FlowEntryFreeList::Free(FlowEntry*) () #5 0x0000000000cdd528 in intrusive_ptr_release(FlowEntry*) () #6 0x0000000000cf7199 in FlowMgmtRequest::~FlowMgmtRequest() () #7 0x000000000087bcce in boost::detail::sp_counted_base::release() () #8 0x0000000000cfd435 in QueueTaskRunner, WorkQueue > >::RunQueue() () #9 0x000000000130560f in TaskImpl::execute() () #10 0x00007fac41c70b3a in ?? () from /usr/lib/libtbb.so.2 #11 0x00007fac41c6c816 in ?? () from /usr/lib/libtbb.so.2 #12 0x00007fac41c6bf4b in ?? () from /usr/lib/libtbb.so.2 #13 0x00007fac41c680ff in ?? () from /usr/lib/libtbb.so.2 #14 0x00007fac41c682f9 in ?? () from /usr/lib/libtbb.so.2 #15 0x00007fac41e8c184 in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0 #16 0x00007fac4116537d in clone () from /lib/x86_64-linux-gnu/libc.so.6 (gdb) quit root@sriov:~/var/log/coredump# gdb `which contrail-vrouter-agent` core.contrail-vroute.34655.rdm5r16c015.rdm5b.cci.att.com.1504116100 GNU gdb (Ubuntu 7.7.1-0ubuntu5~14.04.3) 7.7.1 Copyright (C) 2014 Free Software Foundation, Inc. License GPLv3+: GNU GPL version 3 or later This is free software: you are free to change and redistribute it. There is NO WARRANTY, to the extent permitted by law. Type "show copying" and "show warranty" for details. This GDB was configured as "x86_64-linux-gnu". Type "show configuration" for configuration details. For bug reporting instructions, please see: . Find the GDB manual and other documentation resources online at: . For help, type "help". Type "apropos word" to search for commands related to "word"... Reading symbols from /usr/bin/contrail-vrouter-agent...(no debugging symbols found)...done. warning: core file may not match specified executable file. [New LWP 34672] [New LWP 34676] [New LWP 34673] [New LWP 34655] [New LWP 34678] [New LWP 34677] [New LWP 34674] [New LWP 34675] [New LWP 34679] [Thread debugging using libthread_db enabled] Using host libthread_db library "/lib/x86_64-linux-gnu/libthread_db.so.1". Core was generated by `/usr/bin/contrail-vrouter-agent'. Program terminated with signal SIGABRT, Aborted. #0 0x00007f133a6c5c37 in raise () from /lib/x86_64-linux-gnu/libc.so.6 (gdb) bt #0 0x00007f133a6c5c37 in raise () from /lib/x86_64-linux-gnu/libc.so.6 #1 0x00007f133a6c9028 in abort () from /lib/x86_64-linux-gnu/libc.so.6 #2 0x00007f133a6bebf6 in ?? () from /lib/x86_64-linux-gnu/libc.so.6 #3 0x00007f133a6beca2 in __assert_fail () from /lib/x86_64-linux-gnu/libc.so.6 #4 0x0000000000cec455 in FlowEntryFreeList::Free(FlowEntry*) () #5 0x0000000000cdd528 in intrusive_ptr_release(FlowEntry*) () #6 0x0000000000cf7199 in FlowMgmtRequest::~FlowMgmtRequest() () #7 0x000000000087bcce in boost::detail::sp_counted_base::release() () #8 0x0000000000cfd435 in QueueTaskRunner, WorkQueue > >::RunQueue() () #9 0x000000000130560f in TaskImpl::execute() () #10 0x00007f133b294b3a in ?? () from /usr/lib/libtbb.so.2 #11 0x00007f133b290816 in ?? () from /usr/lib/libtbb.so.2 #12 0x00007f133b28ff4b in ?? () from /usr/lib/libtbb.so.2 #13 0x00007f133b28c0ff in ?? () from /usr/lib/libtbb.so.2 #14 0x00007f133b28c2f9 in ?? () from /usr/lib/libtbb.so.2 #15 0x00007f133b4b0184 in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0 #16 0x00007f133a78937d in clone () from /lib/x86_64-linux-gnu/libc.so.6 (gdb) quit root@sriov:~/var/log/coredump# ls core.contrail-vroute.24365.rdm5r16c015.rdm5b.cci.att.com.1504119226 core.contrail-vroute.37795.rdm5r16c015.rdm5b.cci.att.com.1504046049 core.contrail-vroute.5744.rdm5r16c015.rdm5b.cci.att.com.1504110311 core.contrail-vroute.34655.rdm5r16c015.rdm5b.cci.att.com.1504116100 core.contrail-vroute.41156.rdm5r16c015.rdm5b.cci.att.com.1504128467 root@sriov:~/var/log/coredump# gdb `which contrail-vrouter-agent` core.contrail-vroute.41156.rdm5r16c015.rdm5b.cci.att.com.1504128467 GNU gdb (Ubuntu 7.7.1-0ubuntu5~14.04.3) 7.7.1 Copyright (C) 2014 Free Software Foundation, Inc. License GPLv3+: GNU GPL version 3 or later This is free software: you are free to change and redistribute it. There is NO WARRANTY, to the extent permitted by law. Type "show copying" and "show warranty" for details. This GDB was configured as "x86_64-linux-gnu". Type "show configuration" for configuration details. For bug reporting instructions, please see: . Find the GDB manual and other documentation resources online at: . For help, type "help". Type "apropos word" to search for commands related to "word"... Reading symbols from /usr/bin/contrail-vrouter-agent...(no debugging symbols found)...done. "/root/var/log/coredump/core.contrail-vroute.41156.rdm5r16c015.rdm5b.cci.att.com.1504128467" is not a core dump: File format not recognized (gdb) bt No stack. (gdb)