[test] Preparatory work for running under memfault.

In order to run under memfault, the framework is first extended to handle
running concurrent tests - i.e. multi-threading. (Not that this is a
requirement for memfault, instead it shares a common goal of storing
per-test data).  To that end all the global data is moved into a per-test
context and the targets are adjusted to avoid overlap on shared, global
resources (such as output files and frame buffers). In order to preserve
the simplicity of the standard draw routines, the context is not passed
explicitly as a parameter to the routines, but is instead attached to the
cairo_t via the user_data.

For the masochist, to enable the tests to be run across multiple threads
simply set the environment variable CAIRO_TEST_NUM_THREADS to the desired
number.

In the long run, we can hope the need for memfault (runtime testing of
error paths) will be mitigated by static analysis. A promising candidate
for this task would appear to be http://hal.cs.berkeley.edu/cil/.

1 files changed, 456 insertions, 236 deletions

diff --git a/test/cairo-test.c b/test/cairo-test.c
index 4fd817f08..8e70f9508 100644
--- a/test/cairo-test.c
+++ b/test/cairo-test.c
@@ -31,10 +31,6 @@
 #include <stdlib.h>
 #include <stdarg.h>
 #include <ctype.h>
-#include <setjmp.h>
-#ifdef HAVE_SIGNAL_H
-#include <signal.h>
-#endif
 #if HAVE_FEENABLEEXCEPT
 #include <fenv.h>
 #endif
@@ -47,6 +43,9 @@
 #if HAVE_FCFINI
 #include <fontconfig/fontconfig.h>
 #endif
+#ifdef HAVE_PTHREAD_H
+#include <pthread.h>
+#endif
 
 #include "cairo-test.h"
 
@@ -66,8 +65,10 @@
 #define TRUE !FALSE
 #endif
 
+static cairo_user_data_key_t _cairo_test_context_key;
+
 static void
-xunlink (const char *pathname);
+xunlink (const cairo_test_context_t *ctx, const char *pathname);
 
 static const char *fail_face = "", *normal_face = "";
 
@@ -101,20 +102,11 @@ static const char *vector_ignored_tests[] = {
     NULL
 };
 
-/* Static data is messy, but we're coding for tests here, not a
- * general-purpose library, and it keeps the tests cleaner to avoid a
- * context object there, (though not a whole lot). */
-FILE *cairo_test_log_file = NULL;
-const char *srcdir;
-const char *refdir;
-
-/* Used to catch crashes in a test, such that we report it as such and
- * continue testing, although one crasher may already have corrupted memory in
- * an nonrecoverable fashion. */
-jmp_buf jmpbuf;
-
-void
-cairo_test_init (const char *test_name)
+static void
+_cairo_test_init (cairo_test_context_t *ctx,
+		  const cairo_test_t *test,
+		  const char *test_name,
+		  cairo_test_status_t expectation)
 {
     char *log_name;
 
@@ -122,23 +114,68 @@ cairo_test_init (const char *test_name)
     feenableexcept (FE_DIVBYZERO | FE_INVALID | FE_OVERFLOW);
 #endif
 
+    ctx->test = test;
+    ctx->test_name = test_name;
+    ctx->expectation = expectation;
+
     xasprintf (&log_name, "%s%s", test_name, CAIRO_TEST_LOG_SUFFIX);
-    xunlink (log_name);
+    xunlink (NULL, log_name);
 
-    cairo_test_log_file = fopen (log_name, "a");
-    if (cairo_test_log_file == NULL) {
+    ctx->log_file = fopen (log_name, "a");
+    if (ctx->log_file == NULL) {
 	fprintf (stderr, "Error opening log file: %s\n", log_name);
-	cairo_test_log_file = stderr;
+	ctx->log_file = stderr;
     }
     free (log_name);
 
+    ctx->srcdir = getenv ("srcdir");
+    if (ctx->srcdir == NULL)
+	ctx->srcdir = ".";
+
+    ctx->refdir = getenv ("CAIRO_REF_DIR");
+
+    ctx->thread = 0;
+
+    {
+	int tmp_num_targets;
+	cairo_bool_t tmp_limited_targets;
+	ctx->targets_to_test = cairo_boilerplate_get_targets (&tmp_num_targets, &tmp_limited_targets);
+	ctx->num_targets = tmp_num_targets;
+	ctx->limited_targets = tmp_limited_targets;
+    }
+
     printf ("\nTESTING %s\n", test_name);
 }
 
 void
-cairo_test_fini (void)
+cairo_test_init (cairo_test_context_t *ctx,
+		 const char *test_name)
 {
-    fclose (cairo_test_log_file);
+    _cairo_test_init (ctx, NULL, test_name, CAIRO_TEST_SUCCESS);
+}
+
+static void
+cairo_test_init_thread (cairo_test_context_t *ctx, cairo_test_context_t *master, int thread)
+{
+    *ctx = *master;
+    ctx->thread = thread;
+}
+
+void
+cairo_test_fini (cairo_test_context_t *ctx)
+{
+    if (ctx->thread != 0)
+	return;
+
+    if (ctx->log_file == NULL)
+	return;
+
+    if (ctx->log_file != stderr)
+	fclose (ctx->log_file);
+    ctx->log_file = NULL;
+
+    cairo_boilerplate_free_targets (ctx->targets_to_test);
+
     cairo_debug_reset_static_data ();
 #if HAVE_FCFINI
     FcFini ();
@@ -146,10 +183,10 @@ cairo_test_fini (void)
 }
 
 void
-cairo_test_log (const char *fmt, ...)
+cairo_test_log (const cairo_test_context_t *ctx, const char *fmt, ...)
 {
     va_list va;
-    FILE *file = cairo_test_log_file ? cairo_test_log_file : stderr;
+    FILE *file = ctx && ctx->log_file ? ctx->log_file : stderr;
 
     va_start (va, fmt);
     vfprintf (file, fmt, va);
@@ -157,7 +194,8 @@ cairo_test_log (const char *fmt, ...)
 }
 
 void
-cairo_test_log_path (const cairo_path_t *path)
+cairo_test_log_path (const cairo_test_context_t *ctx,
+		     const cairo_path_t *path)
 {
   int i;
 
@@ -165,21 +203,25 @@ cairo_test_log_path (const cairo_path_t *path)
     cairo_path_data_t *data = &path->data[i];
     switch (data->header.type) {
     case CAIRO_PATH_MOVE_TO:
-	cairo_test_log ("    cairo_move_to (cr, %g, %g);\n",
+	cairo_test_log (ctx,
+		        "    cairo_move_to (cr, %g, %g);\n",
 			data[1].point.x, data[1].point.y);
         break;
     case CAIRO_PATH_LINE_TO:
-	cairo_test_log ("    cairo_line_to (cr, %g, %g);\n",
+	cairo_test_log (ctx,
+		        "    cairo_line_to (cr, %g, %g);\n",
 			data[1].point.x, data[1].point.y);
 	break;
     case CAIRO_PATH_CURVE_TO:
-	cairo_test_log ("    cairo_curve_to (cr, %g, %g, %g, %g, %g, %g);\n",
+	cairo_test_log (ctx,
+		        "    cairo_curve_to (cr, %g, %g, %g, %g, %g, %g);\n",
 			data[1].point.x, data[1].point.y,
 			data[2].point.x, data[2].point.y,
 			data[3].point.x, data[3].point.y);
 	break;
     case CAIRO_PATH_CLOSE_PATH:
-	cairo_test_log ("    cairo_close_path (cr);\n\n");
+	cairo_test_log (ctx,
+		        "    cairo_close_path (cr);\n\n");
 	break;
     default:
 	assert (0);
@@ -188,25 +230,26 @@ cairo_test_log_path (const cairo_path_t *path)
 }
 
 static void
-xunlink (const char *pathname)
+xunlink (const cairo_test_context_t *ctx, const char *pathname)
 {
     if (unlink (pathname) < 0 && errno != ENOENT) {
-	cairo_test_log ("Error: Cannot remove %s: %s\n",
+	cairo_test_log (ctx, "Error: Cannot remove %s: %s\n",
 			pathname, strerror (errno));
 	exit (1);
     }
 }
 
 static char *
-cairo_ref_name_for_test_target_format (const char *test_name,
+cairo_ref_name_for_test_target_format (const cairo_test_context_t *ctx,
+	                               const char *test_name,
 				       const char *target_name,
 				       const char *format)
 {
     char *ref_name = NULL;
 
     /* First look for a previous build for comparison. */
-    if (refdir) {
-	xasprintf (&ref_name, "%s/%s-%s-%s%s", refdir,
+    if (ctx->refdir != NULL) {
+	xasprintf (&ref_name, "%s/%s-%s-%s%s", ctx->refdir,
 		   test_name,
 		   target_name,
 		   format,
@@ -218,7 +261,7 @@ cairo_ref_name_for_test_target_format (const char *test_name,
     }
 
     /* Next look for a target/format-specific reference image. */
-    xasprintf (&ref_name, "%s/%s-%s-%s%s", srcdir,
+    xasprintf (&ref_name, "%s/%s-%s-%s%s", ctx->srcdir,
 	       test_name,
 	       target_name,
 	       format,
@@ -229,7 +272,7 @@ cairo_ref_name_for_test_target_format (const char *test_name,
 	goto done;
 
     /* Next, look for target-specific reference image. */
-    xasprintf (&ref_name, "%s/%s-%s%s", srcdir,
+    xasprintf (&ref_name, "%s/%s-%s%s", ctx->srcdir,
 	       test_name,
 	       target_name,
 	       CAIRO_TEST_REF_SUFFIX);
@@ -239,7 +282,7 @@ cairo_ref_name_for_test_target_format (const char *test_name,
 	goto done;
 
     /* Next, look for format-specific reference image. */
-    xasprintf (&ref_name, "%s/%s-%s%s", srcdir,
+    xasprintf (&ref_name, "%s/%s-%s%s", ctx->srcdir,
 	       test_name,
 	       format,
 	       CAIRO_TEST_REF_SUFFIX);
@@ -249,7 +292,7 @@ cairo_ref_name_for_test_target_format (const char *test_name,
 	goto done;
 
     /* Finally, look for the standard reference image. */
-    xasprintf (&ref_name, "%s/%s%s", srcdir,
+    xasprintf (&ref_name, "%s/%s%s", ctx->srcdir,
 	       test_name,
 	       CAIRO_TEST_REF_SUFFIX);
     if (access (ref_name, F_OK) != 0)
@@ -264,10 +307,14 @@ done:
 }
 
 static cairo_bool_t
-cairo_test_target_has_similar (const cairo_test_t *test, cairo_boilerplate_target_t *target)
+cairo_test_target_has_similar (const cairo_test_context_t *ctx,
+			       const cairo_boilerplate_target_t *target)
 {
     cairo_surface_t *surface;
-    cairo_bool_t has_similar = FALSE;
+    cairo_bool_t has_similar;
+    cairo_t * cr;
+    cairo_surface_t *similar;
+    void *closure;
 
     /* ignore image intermediate targets */
     if (target->expected_type == CAIRO_SURFACE_TYPE_IMAGE)
@@ -276,106 +323,140 @@ cairo_test_target_has_similar (const cairo_test_t *test, cairo_boilerplate_targe
     if (getenv ("CAIRO_TEST_IGNORE_SIMILAR"))
 	return FALSE;
 
-    surface = (target->create_surface) (test->name,
+    surface = (target->create_surface) (ctx->test->name,
 					target->content,
-					test->width,
-					test->height,
+					ctx->test->width,
+					ctx->test->height,
+					ctx->test->width + 25 * NUM_DEVICE_OFFSETS,
+					ctx->test->height + 25 * NUM_DEVICE_OFFSETS,
 					CAIRO_BOILERPLATE_MODE_TEST,
-					&target->closure);
-    if (surface != NULL) {
-	cairo_t * cr = cairo_create (surface);
-	cairo_surface_t *similar;
-	cairo_push_group_with_content (cr, cairo_boilerplate_content (target->content));
-	similar = cairo_get_group_target (cr);
-	has_similar = cairo_surface_get_type (similar) == cairo_surface_get_type (surface);
-
-	cairo_destroy (cr);
-	cairo_surface_destroy (surface);
-
-	if (target->cleanup)
-	    target->cleanup (target->closure);
-    }
+					0,
+					&closure);
+    if (surface == NULL)
+	return FALSE;
+
+    cr = cairo_create (surface);
+    cairo_push_group_with_content (cr,
+				   cairo_boilerplate_content (target->content));
+    similar = cairo_get_group_target (cr);
+
+    has_similar = cairo_surface_get_type (similar) == cairo_surface_get_type (surface);
+
+    cairo_destroy (cr);
+    cairo_surface_destroy (surface);
+
+    if (target->cleanup)
+	target->cleanup (closure);
 
     return has_similar;
 }
 
 static cairo_test_status_t
-cairo_test_for_target (cairo_test_t			 *test,
-		       cairo_boilerplate_target_t	 *target,
+cairo_test_for_target (const cairo_test_context_t		 *ctx,
+		       const cairo_boilerplate_target_t	 *target,
 		       int				  dev_offset,
 		       cairo_bool_t                       similar)
 {
     cairo_test_status_t status;
     cairo_surface_t *surface = NULL;
     cairo_t *cr;
+    const char *no_offset_str = "";
     char *png_name, *ref_name, *diff_name, *offset_str;
-    cairo_test_status_t ret = CAIRO_TEST_SUCCESS;
+    cairo_test_status_t ret;
     cairo_content_t expected_content;
     cairo_font_options_t *font_options;
     const char *format;
+    cairo_bool_t have_output = FALSE;
+    cairo_bool_t have_result = FALSE;
+    void *closure;
+    int width, height;
 
     /* Get the strings ready that we'll need. */
     format = cairo_boilerplate_content_name (target->content);
     if (dev_offset)
 	xasprintf (&offset_str, "-%d", dev_offset);
     else
-	offset_str = strdup("");
+	offset_str = (char *) no_offset_str;
 
-    xasprintf (&png_name, "%s-%s-%s%s%s%s",
-	       test->name,
-	       target->name,
-	       format,
-	       similar ? "-similar" : "",
-	       offset_str, CAIRO_TEST_PNG_SUFFIX);
-    ref_name = cairo_ref_name_for_test_target_format (test->name, target->name, format);
-    xasprintf (&diff_name, "%s-%s-%s%s%s%s",
-	       test->name,
-	       target->name,
-	       format,
-	       similar ? "-similar" : "",
-	       offset_str, CAIRO_TEST_DIFF_SUFFIX);
+    ref_name = cairo_ref_name_for_test_target_format (ctx, ctx->test->name, target->name, format);
+    if (ctx->thread == 0) {
+	xasprintf (&png_name, "%s-%s-%s%s%s%s",
+		   ctx->test->name,
+		   target->name,
+		   format,
+		   similar ? "-similar" : "",
+		   offset_str, CAIRO_TEST_PNG_SUFFIX);
+	xasprintf (&diff_name, "%s-%s-%s%s%s%s",
+		   ctx->test->name,
+		   target->name,
+		   format,
+		   similar ? "-similar" : "",
+		   offset_str, CAIRO_TEST_DIFF_SUFFIX);
+    } else {
+	xasprintf (&png_name, "%s-%s-%s%s%s-%d%s",
+		   ctx->test->name,
+		   target->name,
+		   format,
+		   similar ? "-similar" : "",
+		   offset_str,
+		   ctx->thread,
+		   CAIRO_TEST_PNG_SUFFIX);
+	xasprintf (&diff_name, "%s-%s-%s%s%s-%d%s",
+		   ctx->test->name,
+		   target->name,
+		   format,
+		   similar ? "-similar" : "",
+		   offset_str,
+		   ctx->thread,
+		   CAIRO_TEST_DIFF_SUFFIX);
+    }
 
     if (target->is_vector) {
 	int i;
 
 	for (i = 0; vector_ignored_tests[i] != NULL; i++)
-	    if (strcmp (test->name, vector_ignored_tests[i]) == 0) {
-		cairo_test_log ("Error: Skipping for vector target %s\n", target->name);
+	    if (strcmp (ctx->test->name, vector_ignored_tests[i]) == 0) {
+		cairo_test_log (ctx, "Error: Skipping for vector target %s\n", target->name);
 		ret = CAIRO_TEST_UNTESTED;
 		goto UNWIND_STRINGS;
 	    }
     }
 
-    if (ret == CAIRO_TEST_SUCCESS) {
-	/* Run the actual drawing code. */
-
-	if (test->width && test->height) {
-	    test->width += dev_offset;
-	    test->height += dev_offset;
-	}
-
-	surface = (target->create_surface) (test->name,
-					    target->content,
-					    test->width,
-					    test->height,
-					    CAIRO_BOILERPLATE_MODE_TEST,
-					    &target->closure);
-
-	if (test->width && test->height) {
-	    test->width -= dev_offset;
-	    test->height -= dev_offset;;
-	}
+    width = ctx->test->width;
+    height = ctx->test->height;
+    if (width && height) {
+	width += dev_offset;
+	height += dev_offset;
     }
 
+    have_output = FALSE;
+    have_result = FALSE;
+
+    /* Run the actual drawing code. */
+    ret = CAIRO_TEST_SUCCESS;
+    surface = (target->create_surface) (ctx->test->name,
+					target->content,
+					width, height,
+					ctx->test->width + 25 * NUM_DEVICE_OFFSETS,
+					ctx->test->height + 25 * NUM_DEVICE_OFFSETS,
+					CAIRO_BOILERPLATE_MODE_TEST,
+					ctx->thread,
+					&closure);
     if (surface == NULL) {
-	cairo_test_log ("Error: Failed to set %s target\n", target->name);
+	cairo_test_log (ctx, "Error: Failed to set %s target\n", target->name);
 	ret = CAIRO_TEST_UNTESTED;
 	goto UNWIND_STRINGS;
     }
 
+    if (cairo_surface_status (surface)) {
+	cairo_test_log (ctx, "Error: Created an error surface\n");
+	ret = CAIRO_TEST_FAILURE;
+	goto UNWIND_STRINGS;
+    }
+
     /* Check that we created a surface of the expected type. */
     if (cairo_surface_get_type (surface) != target->expected_type) {
-	cairo_test_log ("Error: Created surface is of type %d (expected %d)\n",
+	cairo_test_log (ctx, "Error: Created surface is of type %d (expected %d)\n",
 			cairo_surface_get_type (surface), target->expected_type);
 	ret = CAIRO_TEST_FAILURE;
 	goto UNWIND_SURFACE;
@@ -387,7 +468,7 @@ cairo_test_for_target (cairo_test_t			 *test,
     expected_content = cairo_boilerplate_content (target->content);
 
     if (cairo_surface_get_content (surface) != expected_content) {
-	cairo_test_log ("Error: Created surface has content %d (expected %d)\n",
+	cairo_test_log (ctx, "Error: Created surface has content %d (expected %d)\n",
 			cairo_surface_get_content (surface), expected_content);
 	ret = CAIRO_TEST_FAILURE;
 	goto UNWIND_SURFACE;
@@ -396,6 +477,11 @@ cairo_test_for_target (cairo_test_t			 *test,
     cairo_surface_set_device_offset (surface, dev_offset, dev_offset);
 
     cr = cairo_create (surface);
+    if (cairo_set_user_data (cr, &_cairo_test_context_key, (void*) ctx, NULL)) {
+	ret = CAIRO_TEST_FAILURE;
+	goto UNWIND_CAIRO;
+    }
+
     if (similar)
 	cairo_push_group_with_content (cr, expected_content);
 
@@ -416,40 +502,49 @@ cairo_test_for_target (cairo_test_t			 *test,
     cairo_font_options_destroy (font_options);
 
     cairo_save (cr);
-    status = (test->draw) (cr, test->width, test->height);
+    status = (ctx->test->draw) (cr, ctx->test->width, ctx->test->height);
     cairo_restore (cr);
 
-    /* Then, check all the different ways it could fail. */
-    if (status) {
-	cairo_test_log ("Error: Function under test failed\n");
-	ret = status;
-	goto UNWIND_CAIRO;
-    }
-
     if (similar) {
 	cairo_pop_group_to_source (cr);
 	cairo_set_operator (cr, CAIRO_OPERATOR_SOURCE);
 	cairo_paint (cr);
     }
 
+    /* Then, check all the different ways it could fail. */
+    if (status) {
+	cairo_test_log (ctx, "Error: Function under test failed\n");
+	ret = status;
+	goto UNWIND_CAIRO;
+    }
+
     if (cairo_status (cr) != CAIRO_STATUS_SUCCESS) {
-	cairo_test_log ("Error: Function under test left cairo status in an error state: %s\n",
+	cairo_test_log (ctx, "Error: Function under test left cairo status in an error state: %s\n",
 			cairo_status_to_string (cairo_status (cr)));
 	ret = CAIRO_TEST_FAILURE;
 	goto UNWIND_CAIRO;
     }
 
     /* Skip image check for tests with no image (width,height == 0,0) */
-    if (test->width != 0 && test->height != 0) {
+    if (ctx->test->width != 0 && ctx->test->height != 0) {
 	buffer_diff_result_t result;
 	cairo_status_t diff_status;
-	xunlink (png_name);
-	(target->write_to_png) (surface, png_name);
 
-	cairo_test_log ("OUTPUT: %s\n", png_name);
+	xunlink (ctx, png_name);
+
+	diff_status = (target->write_to_png) (surface, png_name);
+	if (diff_status) {
+	    cairo_test_log (ctx, "Error: Failed to compare images: %s\n",
+			    cairo_status_to_string (diff_status));
+	    ret = CAIRO_TEST_FAILURE;
+	    goto UNWIND_CAIRO;
+	}
+	have_output = TRUE;
+
 	if (!ref_name) {
-	    cairo_test_log ("Error: Cannot find reference image for %s/%s-%s-%s%s\n",srcdir,
-			    test->name,
+	    cairo_test_log (ctx, "Error: Cannot find reference image for %s/%s-%s-%s%s\n",
+		            ctx->srcdir,
+			    ctx->test->name,
 			    target->name,
 			    format,
 			    CAIRO_TEST_REF_SUFFIX);
@@ -457,22 +552,23 @@ cairo_test_for_target (cairo_test_t			 *test,
 	    goto UNWIND_CAIRO;
 	}
 
-	cairo_test_log ("REFERENCE: %s\n", ref_name);
-	cairo_test_log ("DIFFERENCE: %s\n", diff_name);
-
 	if (target->content == CAIRO_TEST_CONTENT_COLOR_ALPHA_FLATTENED) {
-	    diff_status= image_diff_flattened (png_name, ref_name, diff_name,
+	    diff_status = image_diff_flattened (ctx,
+		                               png_name, ref_name, diff_name,
 					       dev_offset, dev_offset, 0, 0, &result);
 	} else {
-	    diff_status = image_diff (png_name, ref_name, diff_name,
+	    diff_status = image_diff (ctx,
+		                      png_name, ref_name, diff_name,
 				      dev_offset, dev_offset, 0, 0, &result);
 	}
 	if (diff_status) {
-	    cairo_test_log ("Error: Failed to compare images: %s\n",
+	    cairo_test_log (ctx, "Error: Failed to compare images: %s\n",
 			    cairo_status_to_string (diff_status));
 	    ret = CAIRO_TEST_FAILURE;
 	    goto UNWIND_CAIRO;
 	}
+
+	have_result = TRUE;
 	if (result.pixels_changed && result.max_diff > target->error_tolerance) {
 	    ret = CAIRO_TEST_FAILURE;
 	    goto UNWIND_CAIRO;
@@ -484,10 +580,19 @@ UNWIND_CAIRO:
 UNWIND_SURFACE:
     cairo_surface_destroy (surface);
 
-    cairo_debug_reset_static_data ();
-
     if (target->cleanup)
-	target->cleanup (target->closure);
+	target->cleanup (closure);
+
+    if (ctx->thread == 0) {
+	if (have_output)
+	    cairo_test_log (ctx, "OUTPUT: %s\n", png_name);
+
+	if (have_result) {
+	    cairo_test_log (ctx,
+		            "REFERENCE: %s\nDIFFERENCE: %s\n",
+			    ref_name, diff_name);
+	}
+    }
 
 UNWIND_STRINGS:
     if (png_name)
@@ -496,13 +601,20 @@ UNWIND_STRINGS:
       free (ref_name);
     if (diff_name)
       free (diff_name);
-    if (offset_str)
+    if (offset_str != no_offset_str)
       free (offset_str);
 
     return ret;
 }
 
-#ifdef HAVE_SIGNAL_H
+#if defined(HAVE_SIGNAL_H) && defined(HAVE_SETJMP_H)
+#include <signal.h>
+#include <setjmp.h>
+/* Used to catch crashes in a test, so that we report it as such and
+ * continue testing, although one crasher may already have corrupted memory in
+ * an nonrecoverable fashion. */
+static jmp_buf jmpbuf;
+
 static void
 segfault_handler (int signal)
 {
@@ -511,21 +623,16 @@ segfault_handler (int signal)
 #endif
 
 static cairo_test_status_t
-cairo_test_expecting (cairo_test_t *test,
-		      cairo_test_status_t expectation)
+cairo_test_run (cairo_test_context_t *ctx)
 {
     /* we use volatile here to make sure values are not clobbered
      * by longjmp */
-    volatile size_t i, j, num_targets, similar, has_similar;
-    volatile cairo_bool_t limited_targets = FALSE, print_fail_on_stdout = TRUE;
-#ifdef HAVE_SIGNAL_H
-    void (*old_segfault_handler)(int);
-#endif
+    volatile size_t i, j;
+    volatile cairo_bool_t print_fail_on_stdout = ctx->thread == 0;
     volatile cairo_test_status_t status, ret;
-    cairo_boilerplate_target_t ** volatile targets_to_test;
 
 #ifdef HAVE_UNISTD_H
-    if (isatty (2)) {
+    if (ctx->thread == 0 && isatty (2)) {
 	fail_face = "\033[41m\033[37m\033[1m";
 	normal_face = "\033[m";
 	if (isatty (1))
@@ -533,25 +640,6 @@ cairo_test_expecting (cairo_test_t *test,
     }
 #endif
 
-    srcdir = getenv ("srcdir");
-    if (!srcdir)
-	srcdir = ".";
-    refdir = getenv ("CAIRO_REF_DIR");
-
-    cairo_test_init (test->name);
-    printf ("%s\n", test->description);
-
-    if (expectation == CAIRO_TEST_FAILURE)
-    printf ("Expecting failure\n");
-
-    {
-	int tmp_num_targets;
-	cairo_bool_t tmp_limited_targets;
-	targets_to_test = cairo_boilerplate_get_targets (&tmp_num_targets, &tmp_limited_targets);
-	num_targets = tmp_num_targets;
-	limited_targets = tmp_limited_targets;
-    }
-
     /* The intended logic here is that we return overall SUCCESS
      * iff. there is at least one tested backend and that all tested
      * backends return SUCCESS, OR, there's backends were manually
@@ -570,90 +658,217 @@ cairo_test_expecting (cairo_test_t *test,
      * Also, on a crash, run no further tests.
      */
     status = ret = CAIRO_TEST_UNTESTED;
-    for (i = 0; i < num_targets; i++) {
+    for (i = 0; i < ctx->num_targets && status != CAIRO_TEST_CRASHED; i++) {
+	const cairo_boilerplate_target_t * volatile target = ctx->targets_to_test[(i + ctx->thread) % ctx->num_targets];
+
 	for (j = 0; j < NUM_DEVICE_OFFSETS; j++) {
-	    cairo_boilerplate_target_t * volatile target = targets_to_test[i];
-	    volatile int dev_offset = j * 25;
-	    has_similar = cairo_test_target_has_similar (test, target);
+	    volatile int dev_offset = ((j + ctx->thread) % NUM_DEVICE_OFFSETS) * 25;
+	    volatile int similar, has_similar;
+
+	    has_similar = cairo_test_target_has_similar (ctx, target);
 	    for (similar = 0; similar <= has_similar ; similar++) {
-		cairo_test_log ("Testing %s with %s%s target (dev offset %d)\n", test->name, similar ? " (similar)" : "", target->name, dev_offset);
-		printf ("%s-%s-%s [%d]%s:\t", test->name, target->name,
-			cairo_boilerplate_content_name (target->content),
-			dev_offset,
-			similar ? " (similar)": "");
-
-#ifdef HAVE_SIGNAL_H
-		/* Set up a checkpoint to get back to in case of segfaults. */
-		old_segfault_handler = signal (SIGSEGV, segfault_handler);
-		if (0 == setjmp (jmpbuf))
+		cairo_test_log (ctx, "Testing %s with %s%s target (dev offset %d)\n", ctx->test_name, similar ? " (similar)" : "", target->name, dev_offset);
+		if (ctx->thread == 0) {
+		    printf ("%s-%s-%s [%d]%s:\t", ctx->test->name, target->name,
+			    cairo_boilerplate_content_name (target->content),
+			    dev_offset,
+			    similar ? " (similar)": "");
+		    fflush (stdout);
+		}
+
+#if defined(HAVE_SIGNAL_H) && defined(HAVE_SETJMP_H)
+		if (ctx->thread == 0) {
+		    void (* volatile old_segfault_handler)(int);
+		    void (* volatile old_sigpipe_handler)(int);
+
+		    /* Set up a checkpoint to get back to in case of segfaults. */
+#ifdef SIGSEGV
+		    old_segfault_handler = signal (SIGSEGV, segfault_handler);
+#endif
+#ifdef SIGPIPE
+		    old_sigpipe_handler = signal (SIGPIPE, segfault_handler);
+#endif
+		    if (0 == setjmp (jmpbuf))
+			status = cairo_test_for_target (ctx, target, dev_offset, similar);
+		    else
+			status = CAIRO_TEST_CRASHED;
+#ifdef SIGSEGV
+		    signal (SIGSEGV, old_segfault_handler);
 #endif
-		    status = cairo_test_for_target (test, target, dev_offset, similar);
-#ifdef HAVE_SIGNAL_H
-		else
-		    status = CAIRO_TEST_CRASHED;
-		signal (SIGSEGV, old_segfault_handler);
+#ifdef SIGPIPE
+		    signal (SIGPIPE, old_sigpipe_handler);
+#endif
+		} else {
+		    status = cairo_test_for_target (ctx, target, dev_offset, similar);
+		}
+#else
+		status = cairo_test_for_target (ctx, target, dev_offset, similar);
 #endif
 
-		cairo_test_log ("TEST: %s TARGET: %s FORMAT: %s OFFSET: %d SIMILAR: %d RESULT: ",
-				test->name, target->name,
-				cairo_boilerplate_content_name (target->content),
-				dev_offset, similar);
-
-		switch (status) {
-		case CAIRO_TEST_SUCCESS:
-		    printf ("PASS\n");
-		    cairo_test_log ("PASS\n");
-		    if (ret == CAIRO_TEST_UNTESTED)
-			ret = CAIRO_TEST_SUCCESS;
-		    break;
-		case CAIRO_TEST_UNTESTED:
-		    printf ("UNTESTED\n");
-		    cairo_test_log ("UNTESTED\n");
-		    break;
-		case CAIRO_TEST_CRASHED:
-		    if (print_fail_on_stdout) {
-			printf ("!!!CRASHED!!!\n");
-		    } else {
+		if (ctx->thread == 0) {
+		    cairo_test_log (ctx,
+			    "TEST: %s TARGET: %s FORMAT: %s OFFSET: %d SIMILAR: %d RESULT: ",
+				    ctx->test->name, target->name,
+				    cairo_boilerplate_content_name (target->content),
+				    dev_offset, similar);
+		    switch (status) {
+		    case CAIRO_TEST_SUCCESS:
+			printf ("PASS\n");
+			cairo_test_log (ctx, "PASS\n");
+			if (ret == CAIRO_TEST_UNTESTED)
+			    ret = CAIRO_TEST_SUCCESS;
+			break;
+		    case CAIRO_TEST_UNTESTED:
+			printf ("UNTESTED\n");
+			cairo_test_log (ctx, "UNTESTED\n");
+			break;
+		    case CAIRO_TEST_CRASHED:
+			if (print_fail_on_stdout) {
+			    printf ("!!!CRASHED!!!\n");
+			} else {
 			/* eat the test name */
 			printf ("\r");
 			fflush (stdout);
-		    }
-		    cairo_test_log ("CRASHED\n");
-		    fprintf (stderr, "%s-%s-%s [%d]%s:\t%s!!!CRASHED!!!%s\n",
-			     test->name, target->name,
-			     cairo_boilerplate_content_name (target->content), dev_offset, similar ? " (similar)" : "",
-			     fail_face, normal_face);
-		    ret = CAIRO_TEST_FAILURE;
-		    break;
-		default:
-		case CAIRO_TEST_FAILURE:
-		    if (expectation == CAIRO_TEST_FAILURE) {
-			printf ("XFAIL\n");
-			cairo_test_log ("XFAIL\n");
-		    } else {
-			if (print_fail_on_stdout) {
-			    printf ("FAIL\n");
-			} else {
-			    /* eat the test name */
-			    printf ("\r");
-			    fflush (stdout);
 			}
-			fprintf (stderr, "%s-%s-%s [%d]%s:\t%sFAIL%s\n",
-				 test->name, target->name,
+			cairo_test_log (ctx, "CRASHED\n");
+			fprintf (stderr, "%s-%s-%s [%d]%s:\t%s!!!CRASHED!!!%s\n",
+				 ctx->test->name, target->name,
 				 cairo_boilerplate_content_name (target->content), dev_offset, similar ? " (similar)" : "",
 				 fail_face, normal_face);
-			cairo_test_log ("FAIL\n");
+			ret = CAIRO_TEST_FAILURE;
+			break;
+		    default:
+		    case CAIRO_TEST_FAILURE:
+			if (ctx->expectation == CAIRO_TEST_FAILURE) {
+			    printf ("XFAIL\n");
+			    cairo_test_log (ctx, "XFAIL\n");
+			} else {
+			    if (print_fail_on_stdout) {
+				printf ("FAIL\n");
+			    } else {
+				/* eat the test name */
+				printf ("\r");
+				fflush (stdout);
+			    }
+			    fprintf (stderr, "%s-%s-%s [%d]%s:\t%sFAIL%s\n",
+				     ctx->test->name, target->name,
+				     cairo_boilerplate_content_name (target->content), dev_offset, similar ? " (similar)" : "",
+				     fail_face, normal_face);
+			    cairo_test_log (ctx, "FAIL\n");
+			}
+			ret = CAIRO_TEST_FAILURE;
+			break;
+		    }
+		    fflush (stdout);
+		} else {
+#ifdef HAVE_FLOCKFILE
+		    flockfile (stdout);
+#endif
+		    printf ("%s-%s-%s %d [%d]:\t",
+			    ctx->test->name, target->name,
+			    cairo_boilerplate_content_name (target->content),
+			    ctx->thread,
+			    dev_offset);
+		    switch (status) {
+		    case CAIRO_TEST_SUCCESS:
+			printf ("PASS\n");
+			break;
+		    case CAIRO_TEST_UNTESTED:
+			printf ("UNTESTED\n");
+			break;
+		    case CAIRO_TEST_CRASHED:
+			printf ("!!!CRASHED!!!\n");
+			ret = CAIRO_TEST_FAILURE;
+			break;
+		    default:
+		    case CAIRO_TEST_FAILURE:
+			if (ctx->expectation == CAIRO_TEST_FAILURE) {
+			    printf ("XFAIL\n");
+			} else {
+			    printf ("FAIL\n");
+			}
+			ret = CAIRO_TEST_FAILURE;
+			break;
 		    }
-		    ret = status;
-		    break;
-		}
 
-		if (status == CAIRO_TEST_CRASHED)
-		    goto out;
+		    fflush (stdout);
+#ifdef HAVE_FLOCKFILE
+		    funlockfile (stdout);
+#endif
+		}
 	    }
 	}
     }
-out:
+
+    return ret;
+}
+
+#ifdef HAVE_PTHREAD_H
+typedef struct _cairo_test_thread {
+    pthread_t thread;
+    cairo_test_context_t *ctx;
+    size_t id;
+} cairo_test_thread_t;
+
+static void *
+cairo_test_run_threaded (void *closure)
+{
+    cairo_test_thread_t *arg = closure;
+    cairo_test_context_t ctx;
+    cairo_test_status_t ret;
+
+    cairo_test_init_thread (&ctx, arg->ctx, arg->id);
+
+    ret = cairo_test_run (&ctx);
+
+    cairo_test_fini (&ctx);
+
+    return (void *) ret;
+}
+#endif
+
+
+static cairo_test_status_t
+cairo_test_expecting (const cairo_test_t *test,
+		      cairo_test_status_t expectation)
+{
+    cairo_test_context_t ctx;
+    cairo_test_status_t ret = CAIRO_TEST_SUCCESS;
+    size_t num_threads;
+
+    _cairo_test_init (&ctx, test, test->name, expectation);
+    printf ("%s\n", test->description);
+
+    if (expectation == CAIRO_TEST_FAILURE)
+	printf ("Expecting failure\n");
+
+#ifdef HAVE_PTHREAD_H
+    num_threads = 0;
+    if (getenv ("CAIRO_TEST_NUM_THREADS"))
+	num_threads = atoi (getenv ("CAIRO_TEST_NUM_THREADS"));
+    if (num_threads > 1) {
+	cairo_test_thread_t *threads;
+	size_t n;
+
+	threads = xmalloc (sizeof (cairo_test_thread_t) * num_threads);
+	for (n = 0; n < num_threads; n++) {
+	    threads[n].ctx = &ctx;
+	    threads[n].id = n + 1;
+	    pthread_create (&threads[n].thread, NULL,
+		    cairo_test_run_threaded, &threads[n]);
+	}
+	for (n = 0; n < num_threads; n++) {
+	    void *tmp;
+	    pthread_join (threads[n].thread, &tmp);
+	    if (ret == CAIRO_TEST_SUCCESS)
+		ret = (cairo_test_status_t) tmp;
+	}
+	free (threads);
+    }
+
+    if (ret == CAIRO_TEST_SUCCESS)
+#endif
+	ret = cairo_test_run (&ctx);
 
     if (ret != CAIRO_TEST_SUCCESS)
         printf ("Check %s%s out for more information.\n", test->name, CAIRO_TEST_LOG_SUFFIX);
@@ -661,7 +876,7 @@ out:
     /* if the set of targets to test was limited using CAIRO_TEST_TARGET, we
      * behave slightly differently, to ensure that limiting the targets does
      * not increase the number of tests failing. */
-    if (limited_targets) {
+    if (ctx.limited_targets) {
 
 	/* if all untested, success */
 	if (ret == CAIRO_TEST_UNTESTED) {
@@ -680,21 +895,18 @@ out:
 	}
 
     } else {
-
 	if (ret == CAIRO_TEST_UNTESTED)
 	    ret = CAIRO_TEST_FAILURE;
 
     }
 
-    cairo_test_fini ();
-
-    cairo_boilerplate_free_targets (targets_to_test);
+    cairo_test_fini (&ctx);
 
     return ret;
 }
 
 cairo_test_status_t
-cairo_test (cairo_test_t *test)
+cairo_test (const cairo_test_t *test)
 {
     cairo_test_status_t expectation = CAIRO_TEST_SUCCESS;
     const char *xfails;
@@ -726,20 +938,26 @@ cairo_test (cairo_test_t *test)
     return cairo_test_expecting (test, expectation);
 }
 
+const cairo_test_context_t *
+cairo_test_get_context (cairo_t *cr)
+{
+    return cairo_get_user_data (cr, &_cairo_test_context_key);
+}
+
 cairo_surface_t *
-cairo_test_create_surface_from_png (const char *filename)
+cairo_test_create_surface_from_png (const cairo_test_context_t *ctx,
+	                            const char *filename)
 {
     cairo_surface_t *image;
-    char *srcdir = getenv ("srcdir");
 
     image = cairo_image_surface_create_from_png (filename);
     if (cairo_surface_status(image)) {
         /* expect not found when running with srcdir != builddir
          * such as when 'make distcheck' is run
          */
-	if (srcdir) {
+	if (ctx->srcdir) {
 	    char *srcdir_filename;
-	    xasprintf (&srcdir_filename, "%s/%s", srcdir, filename);
+	    xasprintf (&srcdir_filename, "%s/%s", ctx->srcdir, filename);
 	    image = cairo_image_surface_create_from_png (srcdir_filename);
 	    free (srcdir_filename);
 	}
@@ -749,12 +967,13 @@ cairo_test_create_surface_from_png (const char *filename)
 }
 
 cairo_pattern_t *
-cairo_test_create_pattern_from_png (const char *filename)
+cairo_test_create_pattern_from_png (const cairo_test_context_t *ctx,
+	                            const char *filename)
 {
     cairo_surface_t *image;
     cairo_pattern_t *pattern;
 
-    image = cairo_test_create_surface_from_png (filename);
+    image = cairo_test_create_surface_from_png (ctx, filename);
 
     pattern = cairo_pattern_create_for_surface (image);
 
@@ -807,7 +1026,8 @@ cairo_test_paint_checkered (cairo_t *cr)
     cairo_paint (cr);
     cairo_restore (cr);
 
+    status = cairo_surface_status (check);
     cairo_surface_destroy (check);
 
-    return CAIRO_STATUS_SUCCESS;
+    return status;
 }