#Grrr... from http://wiki.python.org/moin/DebuggingWithGdb :
#Also, with gcc 4.5.2 on Ubuntu (at least) the macros fail because the call_function routine appears to be between PyEval_EvalFrameEx and PyEval_EvalCodeEx so the macro fails with No symbol "co" in current context..
#Recompiling python with make "CFLAGS=-g -fno-inline -fno-strict-aliasing" solves this problem.
# So let's build our own python...
sudo apt-get install dpkg-dev
sudo apt-get build-dep python2.7
mkdir -p ~/apts
pushd ~/apts
apt-get source python2.7
cd python2.7-2.7.2/
#Edit configure.in; add -f -fno-inline to the -fno-strict-aliasing
vim configure.in
Change:
BASECFLAGS="$BASECFLAGS -fno-strict-aliasing"
To:
BASECFLAGS="$BASECFLAGS -fno-strict-aliasing -fno-inline -g"
# Build the package, skip tests
export DEB_BUILD_OPTIONS="nocheck notest nobench parallel=8"
# That should work, but I haven't actually tried with parallel
export DEB_BUILD_OPTIONS="nocheck notest nobench"
dpkg-buildpackage
I have a python with gdb support, and have a nice stack trace:
(gdb) pystack nova/nova/ utils.py (745): to_primitive nova/nova/ utils.py (736): to_primitive nova/nova/ utils.py (751): to_primitive nova/nova/ utils.py (730): to_primitive nova/nova/ utils.py (745): to_primitive nova/nova/ utils.py (736): to_primitive nova/nova/ utils.py (751): to_primitive nova/nova/ utils.py (736): to_primitive nova/nova/ utils.py (751): to_primitive nova/nova/ utils.py (730): to_primitive nova/nova/ utils.py (736): to_primitive nova/nova/ notifier/ api.py (118): notify nova/nova/ exception. py (136): wrapped nova/nova/ compute/ manager. py (141): decorated_function nova/nova/ exception. py (112): wrapped nova/nova/ compute/ manager. py (394): _shutdown_ instance_ even_if_ deleted nova/nova/ compute/ manager. py (443): _run_instance nova/nova/ compute/ manager. py (644): run_instance nova/nova/ compute/ manager. py (170): decorated_function nova/nova/ exception. py (112): wrapped nova/nova/ rpc/amqp. py (250): _process_data nova/nova/ exception. py (112): wrapped python2. 7/dist- packages/ eventlet/ greenpool. py (80): _spawn_n_impl python2. 7/dist- packages/ eventlet/ hubs/epolls. py (59): do_poll python2. 7/dist- packages/ eventlet/ hubs/poll. py (97): wait python2. 7/dist- packages/ eventlet/ hubs/hub. py (226): run python2. 7/dist- packages/ eventlet/ hubs/hub. py (177): switch python2. 7/dist- packages/ eventlet/ event.py (116): wait python2. 7/dist- packages/ eventlet/ greenthread. py (166): wait nova/nova/ service. py (131): wait nova/nova/ service. py (411): wait nova/bin/ nova-compute (54): <module>
/opt/stack/
/opt/stack/
/opt/stack/
/opt/stack/
/opt/stack/
/opt/stack/
/opt/stack/
/opt/stack/
/opt/stack/
/opt/stack/
/opt/stack/
/opt/stack/
/opt/stack/
/opt/stack/
/opt/stack/
/opt/stack/
/opt/stack/
/opt/stack/
/opt/stack/
/opt/stack/
/opt/stack/
/opt/stack/
/usr/lib/
/usr/lib/
/usr/lib/
/usr/lib/
/usr/lib/
/usr/lib/
/usr/lib/
/opt/stack/
/opt/stack/
/opt/stack/
So this does look like bug 934575. I merged trunk to get the patch, and no more crashes. Thanks Russell for the help + for the fix.
Here are my notes on how to get a stack trace with Python & GDB:
# I'm presuming root, I think...
sudo bash
# You probably want gdb :-)
sudo apt-get install gdb
# Useful symbols
sudo apt-get install python2.7-dbg
sudo apt-get install libc6-dbg
sudo apt-get install python-greenlet-dbg
#Grrr... from http:// wiki.python. org/moin/ DebuggingWithGd b : aliasing" solves this problem.
#Also, with gcc 4.5.2 on Ubuntu (at least) the macros fail because the call_function routine appears to be between PyEval_EvalFrameEx and PyEval_EvalCodeEx so the macro fails with No symbol "co" in current context..
#Recompiling python with make "CFLAGS=-g -fno-inline -fno-strict-
# So let's build our own python...
sudo apt-get install dpkg-dev
sudo apt-get build-dep python2.7
mkdir -p ~/apts aliasing S="$BASECFLAGS -fno-strict- aliasing" S="$BASECFLAGS -fno-strict- aliasing -fno-inline -g"
pushd ~/apts
apt-get source python2.7
cd python2.7-2.7.2/
#Edit configure.in; add -f -fno-inline to the -fno-strict-
vim configure.in
Change:
BASECFLAG
To:
BASECFLAG
# Build the package, skip tests OPTIONS= "nocheck notest nobench parallel=8" OPTIONS= "nocheck notest nobench"
export DEB_BUILD_
# That should work, but I haven't actually tried with parallel
export DEB_BUILD_
dpkg-buildpackage
cd .. 7_2.7.2- 5ubuntu1_ amd64.deb 7-dbg_2. 7.2-5ubuntu1_ amd64.deb 7_2.7.2- 5ubuntu1_ amd64.deb 7-minimal_ 2.7.2-5ubuntu1_ amd64.deb
dpkg -i python2.
dpkg -i python2.
dpkg -i libpython2.
dpkg -i python2.
# Should now show today's date...
python
popd
#Become root
sudo bash
cd ~ svn.python. org/view/ *checkout* /python/ branches/ release27- maint/Misc/ gdbinit"
wget "http://
mv gdbinit .gdbinit
# Change mode so gdb will trust it
chmod 700 .gdbinit
sudo chown root .gdbinit
# Shouldn't show any warnings about gdb init
sudo gdb
quit
ps -ef | grep compute
stack@openstack1:~$ ps -ef | grep comp nova/bin/ nova-compute nova/bin/ nova-compute
root 17372 5350 0 11:01 pts/10 00:00:00 sg libvirtd /opt/stack/
stack 17373 17372 10 11:01 pts/10 00:00:00 python /opt/stack/
stack 17379 4561 0 11:01 pts/0 00:00:00 grep --color=auto comp
sudo gdb python 17373
pystack
# Should have a python stack