Print backtraces in case an assert fails inside xlib/xcb.

As you know there are some nasty libs / apps doing locking
incorrectly. In order to improve the information given to the user
when he encounters such a situation (people don't run apps in gdb
normally) I created the patch attached.
It's very non-intrusive (and affects only xlib/xcb, Josh told me on
irc that it could be useful for other areas too, personally I don't
think that it's really needed at other places ...).

Some same outputs and the discussion of them:

    lxuser@pdln:/tmp$ ./main
    Got a backtrace:
    #0 /tmp/usr/lib/libxcb-xlib.so.0 [0xb7f9d728]
    #1 /tmp/usr/lib/libxcb-xlib.so.0(xcb_xlib_unlock+0x31) [0xb7f9d861]
    #2 ./test.so(function_a+0x11) [0xb7f9f3fd]
    #3 ./test.so(function_b+0x11) [0xb7f9f410]
    #4 ./main [0x80484a7]
    #5 /lib/libc.so.6(__libc_start_main+0xdc) [0xb7e60ebc]
    #6 ./main [0x80483f1]
    main: xcb_xlib.c:82: xcb_xlib_unlock: Assertion `c->xlib.lock' failed.
    Aborted

That's kinda the normal situation.

    lxuser@pdln:/tmp$ ./main
    Got a backtrace:
    #0 /tmp/usr/lib/libxcb-xlib.so.0 [0xb7f90728]
    #1 /tmp/usr/lib/libxcb-xlib.so.0(xcb_xlib_unlock+0x31) [0xb7f90861]
    #2 /tmp/test.so [0xb7f923cd]
    #3 /tmp/test.so(function_b+0x11) [0xb7f923e0]
    #4 ./main [0x80484ab]
    #5 /lib/libc.so.6(__libc_start_main+0xdc) [0xb7e53ebc]
    #6 ./main [0x80483f1]
    main: xcb_xlib.c:82: xcb_xlib_unlock: Assertion `c->xlib.lock' failed.
    Aborted

There are two possible reasons that the name doesn't appear in #2:
a) a hidden symbol or a symbol with statical linkage in a library
b) a symbol in an app not compiled with -rdynamic.
But in both cases you still know _where_ the caller is.

Note that in this example test.so was compiled with
-fomit-frame-pointer; this isn't an issue as _one_ (= the caller)
stack trace is still valid (as long as you don't have the insane idea
to compile xcb with -fo-f-p).

Another issue that may appear is "tail call elimination" (some entries
are mysteriously missing; this is quite ugly, but you still get enough
information so that you can do something useful with the issue e.g. by
disassembling the relevant parts with gdb).

Signed-off-by: Jamey Sharp <jamey@minilop.net>

2 files changed, 37 insertions, 3 deletions

diff --git a/configure.ac b/configure.ac
index d73e51f..b446e7f 100644
--- a/configure.ac
+++ b/configure.ac
@@ -48,6 +48,8 @@ PKG_CHECK_MODULES(XDMCP, xdmcp,
 		], [$XDMCP_LIBS]),
 	[AC_MSG_RESULT(no)])
 
+AC_CHECK_HEADER([execinfo.h], [AC_DEFINE(HAVE_BACKTRACE,1,[Has backtrace*() needed for retrieving stack traces])])
+
 AC_SUBST(NEEDED)
 
 # Find the xcb-proto protocol descriptions
diff --git a/src/xcb_xlib.c b/src/xcb_xlib.c
index 59f972c..07d530c 100644
--- a/src/xcb_xlib.c
+++ b/src/xcb_xlib.c
@@ -28,6 +28,38 @@
 
 #include <assert.h>
 
+#ifdef HAVE_BACKTRACE
+#include <execinfo.h>
+#include <stdio.h>
+#include <stdlib.h>
+#endif
+
+static void xcb_xlib_printbt(void)
+{
+#ifdef HAVE_BACKTRACE
+	void *array[20];
+	int size;
+	char **strings;
+	int i;
+
+	size = backtrace(array, 20);
+	strings = backtrace_symbols(array, size);
+
+	printf("Got a backtrace:\n");
+
+	for (i = 0; i < size; ++i)
+		printf("#%i %s\n", i, strings[i]);
+
+	free(strings);
+#endif
+}
+
+#ifndef NDEBUG
+#define xcb_assert(x) do { if (!(x)) { xcb_xlib_printbt(); assert(x); } } while(0)
+#else
+#define xcb_assert(x)
+#endif
+
 unsigned int xcb_get_request_sent(xcb_connection_t *c)
 {
     if(c->has_error)
@@ -38,7 +70,7 @@ unsigned int xcb_get_request_sent(xcb_connection_t *c)
 void xcb_xlib_lock(xcb_connection_t *c)
 {
     _xcb_lock_io(c);
-    assert(!c->xlib.lock);
+    xcb_assert(!c->xlib.lock);
     c->xlib.lock = 1;
     c->xlib.thread = pthread_self();
     _xcb_unlock_io(c);
@@ -47,8 +79,8 @@ void xcb_xlib_lock(xcb_connection_t *c)
 void xcb_xlib_unlock(xcb_connection_t *c)
 {
     _xcb_lock_io(c);
-    assert(c->xlib.lock);
-    assert(pthread_equal(c->xlib.thread, pthread_self()));
+    xcb_assert(c->xlib.lock);
+    xcb_assert(pthread_equal(c->xlib.thread, pthread_self()));
     c->xlib.lock = 0;
     pthread_cond_broadcast(&c->xlib.cond);
     _xcb_unlock_io(c);