Comment 10 for bug 18847

The problem is as follows:

These two calls happen to pthread_mutex_lock() without an unlock call inbetween.

Breakpoint 3, 0xb77fdc42 in pthread_mutex_lock () from
/lib/tls/i686/cmov/libpthread.so.0
(gdb) bt
#0 0xb77fdc42 in pthread_mutex_lock () from /lib/tls/i686/cmov/libpthread.so.0
#1 0xb784a5e7 in _XInternalLockDisplay (dpy=0x8058c98, wskip=32) at
../../src/locking.c:506
#2 0xb7841c42 in _XWaitForReadable (dpy=0x8058c98) at ../../src/XlibInt.c:504
#3 0xb7842072 in _XRead (dpy=0x8058c98, data=0xbfb3893c
"\027G\uffff\uffff\uffffNk\uffff\204\uffff\uffff\uffffP\uffff\005\b\230\214\005\b\uffff\211\uffff\uffff0\v\uffff\uffff\001",
size=32)
    at ../../src/XlibInt.c:1080
#4 0xb7842fe8 in _XReply (dpy=0x8058c98, rep=0xbfb3893c, extra=0, discard=0) at
../../src/XlibInt.c:1712
#5 0xb76bada2 in XShmQueryVersion (dpy=0x8058c98, majorVersion=0x8056c50,
minorVersion=0x8056c50, sharedPixmaps=0x8056c50)
    at ../../src/XShm.c:189
#6 0xb7bd1eb8 in _gdk_windowing_image_init (display=0x805d950) at
gdkimage-x11.c:211
#7 0xb7bbc669 in IA__gdk_display_open (display_name=0x8056c50 "") at
gdkdisplay-x11.c:312
#8 0xb7b9cf03 in IA__gdk_display_open_default_libgtk_only () at gdk.c:272
#9 0xb7d45da5 in IA__gtk_init_check (argc=0x8056c50, argv=0x8056c50) at
gtkmain.c:713
#10 0xb7d45dd8 in IA__gtk_init (argc=0x8056c50, argv=0x8056c50) at
gtkmain.c:748#11 0x08048655 in main ()
(gdb) c
Continuing.

Breakpoint 3, 0xb77fdc42 in pthread_mutex_lock () from
/lib/tls/i686/cmov/libpthread.so.0
(gdb) bt
#0 0xb77fdc42 in pthread_mutex_lock () from /lib/tls/i686/cmov/libpthread.so.0
#1 0xb784a59b in _XLockDisplay (dpy=0x8058c98) at ../../src/locking.c:481
#2 0xb7837b36 in XQueryExtension (dpy=0x8058c98, name=0xb7c07883
"XInputExtension", major_opcode=0x8056c50,
    first_event=0x8056c50, first_error=0x8056c50) at ../../src/QuExt.c:46
#3 0xb7be8bd5 in _gdk_input_common_init (display=0x805d950, include_core=0) at
gdkinput-x11.c:394
#4 0xb7be9ce9 in _gdk_input_init (display=0x805d950) at gdkinput-xfree.c:41
#5 0xb7bbc67f in IA__gdk_display_open (display_name=0x8056c50 "\001") at
gdkdisplay-x11.c:314
#6 0xb7b9cf03 in IA__gdk_display_open_default_libgtk_only () at gdk.c:272
#7 0xb7d45da5 in IA__gtk_init_check (argc=0x8056c50, argv=0x8056c50) at
gtkmain.c:713
#8 0xb7d45dd8 in IA__gtk_init (argc=0x8056c50, argv=0x8056c50) at
gtkmain.c:748#9 0x08048655 in main ()

This is caused by the fact that _XReply and friends assume that the X display is
already locked and do things like this:

  Unlock( dpy ); <-- does nothing if not already locked.
  poll();
  Lock( dpy );

So even if the display wasn't locked when _XReply was invoked, it will be locked
when it returns. XShmQueryVersion appears to know this and makes the proper
locking calls on entry and the proper unlocking calls on exit. However, in the
version of the library that I have installed, the calls are never made. I can't
explain this.

Here's a dump of the assembly for the XShmQueryVersion that I have installed
right now

d4383d697f04ef43e93c5e1752e107bd usr/lib/libXext.so.6.4.1 from libxext6
version 1:6.4.3-2

00006d21 <XShmQueryVersion>:
    6d21: 55 push %ebp
    6d22: 89 e5 mov %esp,%ebp
    6d24: 57 push %edi
    6d25: 56 push %esi
    6d26: 53 push %ebx
    6d27: 83 ec 3c sub $0x3c,%esp
    6d2a: e8 e5 b8 ff ff call 2614 <_XVIDtoVisual@plt+0x3c>
    6d2f: 81 c3 d9 43 00 00 add $0x43d9,%ebx
    6d35: 8b 7d 08 mov 0x8(%ebp),%edi
    6d38: 89 f8 mov %edi,%eax
    6d3a: e8 e1 fc ff ff call 6a20 <XShapeGetRectangles+0x21b>
    6d3f: 89 c6 mov %eax,%esi
    6d41: 85 c0 test %eax,%eax
    6d43: 0f 84 b1 00 00 00 je 6dfa <XShmQueryVersion+0xd9>
    6d49: 8b 50 08 mov 0x8(%eax),%edx
    6d4c: 85 d2 test %edx,%edx
    6d4e: 0f 84 a6 00 00 00 je 6dfa <XShmQueryVersion+0xd9>
    6d54: 8b 57 6c mov 0x6c(%edi),%edx
    6d57: 8d 42 04 lea 0x4(%edx),%eax
    6d5a: 3b 47 70 cmp 0x70(%edi),%eax
    6d5d: 0f 87 b3 00 00 00 ja 6e16 <XShmQueryVersion+0xf5>
    6d63: 89 57 64 mov %edx,0x64(%edi)
    6d66: c6 02 00 movb $0x0,(%edx)
    6d69: 66 c7 42 02 01 00 movw $0x1,0x2(%edx)
    6d6f: 83 47 6c 04 addl $0x4,0x6c(%edi)
    6d73: 83 47 60 01 addl $0x1,0x60(%edi)
    6d77: 8b 46 08 mov 0x8(%esi),%eax
    6d7a: 8b 40 04 mov 0x4(%eax),%eax
    6d7d: 88 02 mov %al,(%edx)
    6d7f: c6 42 01 00 movb $0x0,0x1(%edx)
    6d83: 8d 45 d4 lea 0xffffffd4(%ebp),%eax
    6d86: c7 44 24 0c 00 00 00 movl $0x0,0xc(%esp)
    6d8d: 00
    6d8e: c7 44 24 08 00 00 00 movl $0x0,0x8(%esp)
    6d95: 00
    6d96: 89 44 24 04 mov %eax,0x4(%esp)
    6d9a: 89 3c 24 mov %edi,(%esp)
    6d9d: e8 86 b6 ff ff call 2428 <_XReply@plt>
    6da2: 85 c0 test %eax,%eax
    6da4: 75 16 jne 6dbc <XShmQueryVersion+0x9b>
    6da6: 8b 57 7c mov 0x7c(%edi),%edx
    6da9: 85 d2 test %edx,%edx
    6dab: 74 07 je 6db4 <XShmQueryVersion+0x93>
    6dad: 89 3c 24 mov %edi,(%esp)
    6db0: ff d2 call *%edx
    6db2: 31 c0 xor %eax,%eax
    6db4: 83 c4 3c add $0x3c,%esp
    6db7: 5b pop %ebx
    6db8: 5e pop %esi
    6db9: 5f pop %edi
    6dba: 5d pop %ebp
    6dbb: c3 ret
    6dbc: 0f b7 55 dc movzwl 0xffffffdc(%ebp),%edx
    6dc0: 8b 45 0c mov 0xc(%ebp),%eax
    6dc3: 89 10 mov %edx,(%eax)
    6dc5: 0f b7 55 de movzwl 0xffffffde(%ebp),%edx
    6dc9: 8b 45 10 mov 0x10(%ebp),%eax
    6dcc: 89 10 mov %edx,(%eax)
    6dce: 31 d2 xor %edx,%edx
    6dd0: 80 7d d5 00 cmpb $0x0,0xffffffd5(%ebp)
    6dd4: 0f 95 c2 setne %dl
    6dd7: 8b 45 14 mov 0x14(%ebp),%eax
    6dda: 89 10 mov %edx,(%eax)
    6ddc: 8b 57 7c mov 0x7c(%edi),%edx
    6ddf: b8 01 00 00 00 mov $0x1,%eax
    6de4: 85 d2 test %edx,%edx
    6de6: 74 cc je 6db4 <XShmQueryVersion+0x93>
    6de8: 89 3c 24 mov %edi,(%esp)
    6deb: ff d2 call *%edx
    6ded: b8 01 00 00 00 mov $0x1,%eax
    6df2: 83 c4 3c add $0x3c,%esp
    6df5: 5b pop %ebx
    6df6: 5e pop %esi
    6df7: 5f pop %edi
    6df8: 5d pop %ebp
    6df9: c3 ret
    6dfa: 8b 83 98 03 00 00 mov 0x398(%ebx),%eax
    6e00: 89 44 24 04 mov %eax,0x4(%esp)
    6e04: 89 3c 24 mov %edi,(%esp)
    6e07: e8 7c b7 ff ff call 2588 <XMissingExtension@plt>
    6e0c: 31 c0 xor %eax,%eax
    6e0e: 83 c4 3c add $0x3c,%esp
    6e11: 5b pop %ebx
    6e12: 5e pop %esi
    6e13: 5f pop %edi
    6e14: 5d pop %ebp
    6e15: c3 ret
    6e16: 89 3c 24 mov %edi,(%esp)
    6e19: e8 aa b6 ff ff call 24c8 <_XFlush@plt>
    6e1e: 8b 57 6c mov 0x6c(%edi),%edx
    6e21: e9 3d ff ff ff jmp 6d63 <XShmQueryVersion+0x42>

The X(Un)LockDisplay() are callbacks invoked on a member of the dpy structure.
The call should look something like this:

    7e44: 8b 86 d0 04 00 00 mov 0x4d0(%esi),%eax
get dpy->lock
    7e4a: 85 c0 test %eax,%eax
if( !dpy->lock )
    7e4c: 74 09 je 7e57 <XShmQueryVersion+0x47>
then don't make the call
    7e4e: 83 ec 0c sub $0xc,%esp
put some space on the stack
    7e51: 56 push %esi
push the 'dpy' variable as the only parameter to _XLockDisplay
    7e52: ff 10 call *(%eax)
make the call

There is nothing to this effect in the above assembly dump. When I compile the
code for myself, however (apt-get source libxext6, fakeroot dpkg-buildpackage)
then my version of XShmQueryVersion contains the above code. Installing my copy
of libXext fixes the problem and my test programs (and totem) run fine.

Not really sure what to say about this... maybe the version of GCC used to
compile the libXext package currently in breezy removed some code that it
shouldn't have. GCC version here is gcc (GCC) 4.0.2 20050720 (prerelease)
(Debian 4.0.1-2ubuntu3).