[cairo-test] Repeat tests for similar surfaces.

Having noticed strange discrepancies creeping into similar surfaces
whilst working on the xlib backend, I thought it wise to also run
the test harness against similar targets. For consistency, only
targets whose similar surface use the same backend are included.

This can be disabled by exporting CAIRO_TEST_IGNORE_SIMILAR=1.

1 files changed, 113 insertions, 60 deletions

diff --git a/test/cairo-test.c b/test/cairo-test.c
index 500bb162..de8ed1bc 100755
--- a/test/cairo-test.c
+++ b/test/cairo-test.c
@@ -204,10 +204,47 @@ done:
     return ref_name;
 }
 
+static cairo_bool_t
+cairo_test_target_has_similar (const cairo_test_t *test, cairo_boilerplate_target_t *target)
+{
+    cairo_surface_t *surface;
+    cairo_bool_t has_similar = FALSE;
+
+    /* ignore image intermediate targets */
+    if (target->expected_type == CAIRO_SURFACE_TYPE_IMAGE)
+	return FALSE;
+
+    if (getenv ("CAIRO_TEST_IGNORE_SIMILAR"))
+	return FALSE;
+
+    surface = (target->create_surface) (test->name,
+					target->content,
+					test->width,
+					test->height,
+					CAIRO_BOILERPLATE_MODE_TEST,
+					&target->closure);
+    if (surface != NULL) {
+	cairo_t * cr = cairo_create (surface);
+	cairo_surface_t *similar;
+	cairo_push_group_with_content (cr, target->content);
+	similar = cairo_get_target (cr);
+	has_similar = cairo_surface_get_type (similar) == cairo_surface_get_type (surface);
+
+	cairo_destroy (cr);
+	cairo_surface_destroy (surface);
+
+	if (target->cleanup)
+	    target->cleanup (target->closure);
+    }
+
+    return has_similar;
+}
+
 static cairo_test_status_t
 cairo_test_for_target (cairo_test_t			 *test,
 		       cairo_boilerplate_target_t	 *target,
-		       int				  dev_offset)
+		       int				  dev_offset,
+		       cairo_bool_t                       similar)
 {
     cairo_test_status_t status;
     cairo_surface_t *surface = NULL;
@@ -225,16 +262,18 @@ cairo_test_for_target (cairo_test_t			 *test,
     else
 	offset_str = strdup("");
 
-    xasprintf (&png_name, "%s-%s-%s%s%s",
+    xasprintf (&png_name, "%s-%s-%s%s%s%s",
 	       test->name,
 	       target->name,
 	       format,
+	       similar ? "-similar" : "",
 	       offset_str, CAIRO_TEST_PNG_SUFFIX);
     ref_name = cairo_ref_name_for_test_target_format (test->name, target->name, format);
-    xasprintf (&diff_name, "%s-%s-%s%s%s",
+    xasprintf (&diff_name, "%s-%s-%s%s%s%s",
 	       test->name,
 	       target->name,
 	       format,
+	       similar ? "-similar" : "",
 	       offset_str, CAIRO_TEST_DIFF_SUFFIX);
 
     if (target->is_vector) {
@@ -301,6 +340,8 @@ cairo_test_for_target (cairo_test_t			 *test,
     cairo_surface_set_device_offset (surface, dev_offset, dev_offset);
 
     cr = cairo_create (surface);
+    if (similar)
+	cairo_push_group_with_content (cr, target->content);
 
     /* Clear to transparent (or black) depending on whether the target
      * surface supports alpha. */
@@ -327,6 +368,11 @@ cairo_test_for_target (cairo_test_t			 *test,
 	goto UNWIND_CAIRO;
     }
 
+    if (similar) {
+	cairo_pop_group_to_source (cr);
+	cairo_paint (cr);
+    }
+
     if (cairo_status (cr) != CAIRO_STATUS_SUCCESS) {
 	cairo_test_log ("Error: Function under test left cairo status in an error state: %s\n",
 			cairo_status_to_string (cairo_status (cr)));
@@ -409,7 +455,7 @@ cairo_test_expecting (cairo_test_t *test,
 {
     /* we use volatile here to make sure values are not clobbered
      * by longjmp */
-    volatile size_t i, j, num_targets;
+    volatile size_t i, j, num_targets, similar, has_similar;
     volatile cairo_bool_t limited_targets = FALSE, print_fail_on_stdout = TRUE;
 #ifdef HAVE_SIGNAL_H
     void (*old_segfault_handler)(int);
@@ -462,83 +508,90 @@ cairo_test_expecting (cairo_test_t *test,
      * Also, on a crash, run no further tests.
      */
     status = ret = CAIRO_TEST_UNTESTED;
-    for (i = 0; i < num_targets && status != CAIRO_TEST_CRASHED; i++) {
+    for (i = 0; i < num_targets; i++) {
 	for (j = 0; j < NUM_DEVICE_OFFSETS; j++) {
 	    cairo_boilerplate_target_t * volatile target = targets_to_test[i];
 	    volatile int dev_offset = j * 25;
-
-	    cairo_test_log ("Testing %s with %s target (dev offset %d)\n", test->name, target->name, dev_offset);
-	    printf ("%s-%s-%s [%d]:\t", test->name, target->name,
-		    cairo_boilerplate_content_name (target->content),
-		    dev_offset);
+	    has_similar = cairo_test_target_has_similar (test, target);
+	    for (similar = 0; similar <= has_similar ; similar++) {
+		cairo_test_log ("Testing %s with %s%s target (dev offset %d)\n", test->name, similar ? " (similar)" : "", target->name, dev_offset);
+		printf ("%s-%s-%s [%d]%s:\t", test->name, target->name,
+			cairo_boilerplate_content_name (target->content),
+			dev_offset,
+			similar ? " (similar)": "");
 
 #ifdef HAVE_SIGNAL_H
-	    /* Set up a checkpoint to get back to in case of segfaults. */
-	    old_segfault_handler = signal (SIGSEGV, segfault_handler);
-	    if (0 == setjmp (jmpbuf))
+		/* Set up a checkpoint to get back to in case of segfaults. */
+		old_segfault_handler = signal (SIGSEGV, segfault_handler);
+		if (0 == setjmp (jmpbuf))
 #endif
-		status = cairo_test_for_target (test, target, dev_offset);
+		    status = cairo_test_for_target (test, target, dev_offset, similar);
 #ifdef HAVE_SIGNAL_H
-	    else
-	        status = CAIRO_TEST_CRASHED;
-	    signal (SIGSEGV, old_segfault_handler);
+		else
+		    status = CAIRO_TEST_CRASHED;
+		signal (SIGSEGV, old_segfault_handler);
 #endif
 
-	    cairo_test_log ("TEST: %s TARGET: %s FORMAT: %s OFFSET: %d RESULT: ",
-			    test->name, target->name,
-			    cairo_boilerplate_content_name (target->content),
-			    dev_offset);
-
-	    switch (status) {
-	    case CAIRO_TEST_SUCCESS:
-		printf ("PASS\n");
-		cairo_test_log ("PASS\n");
-		if (ret == CAIRO_TEST_UNTESTED)
-		    ret = CAIRO_TEST_SUCCESS;
-		break;
-	    case CAIRO_TEST_UNTESTED:
-		printf ("UNTESTED\n");
-		cairo_test_log ("UNTESTED\n");
-		break;
-	    case CAIRO_TEST_CRASHED:
-		if (print_fail_on_stdout) {
-		    printf ("!!!CRASHED!!!\n");
-		} else {
-		    /* eat the test name */
-		    printf ("\r");
-		    fflush (stdout);
-		}
-		cairo_test_log ("CRASHED\n");
-		fprintf (stderr, "%s-%s-%s [%d]:\t%s!!!CRASHED!!!%s\n",
-			 test->name, target->name,
-			 cairo_boilerplate_content_name (target->content), dev_offset,
-			 fail_face, normal_face);
-		ret = CAIRO_TEST_FAILURE;
-		break;
-	    default:
-	    case CAIRO_TEST_FAILURE:
-		if (expectation == CAIRO_TEST_FAILURE) {
-		    printf ("XFAIL\n");
-		    cairo_test_log ("XFAIL\n");
-		} else {
+		cairo_test_log ("TEST: %s TARGET: %s FORMAT: %s OFFSET: %d SIMILAR: %d RESULT: ",
+				test->name, target->name,
+				cairo_boilerplate_content_name (target->content),
+				dev_offset, similar);
+
+		switch (status) {
+		case CAIRO_TEST_SUCCESS:
+		    printf ("PASS\n");
+		    cairo_test_log ("PASS\n");
+		    if (ret == CAIRO_TEST_UNTESTED)
+			ret = CAIRO_TEST_SUCCESS;
+		    break;
+		case CAIRO_TEST_UNTESTED:
+		    printf ("UNTESTED\n");
+		    cairo_test_log ("UNTESTED\n");
+		    break;
+		case CAIRO_TEST_CRASHED:
 		    if (print_fail_on_stdout) {
-			printf ("FAIL\n");
+			printf ("!!!CRASHED!!!\n");
 		    } else {
 			/* eat the test name */
 			printf ("\r");
 			fflush (stdout);
 		    }
-		    fprintf (stderr, "%s-%s-%s [%d]:\t%sFAIL%s\n",
+		    cairo_test_log ("CRASHED\n");
+		    fprintf (stderr, "%s-%s-%s [%d]%s:\t%s!!!CRASHED!!!%s\n",
 			     test->name, target->name,
-			     cairo_boilerplate_content_name (target->content), dev_offset,
+			     cairo_boilerplate_content_name (target->content), dev_offset, similar ? " (similar)" : "",
 			     fail_face, normal_face);
-		    cairo_test_log ("FAIL\n");
+		    ret = CAIRO_TEST_FAILURE;
+		    break;
+		default:
+		case CAIRO_TEST_FAILURE:
+		    if (expectation == CAIRO_TEST_FAILURE) {
+			printf ("XFAIL\n");
+			cairo_test_log ("XFAIL\n");
+		    } else {
+			if (print_fail_on_stdout) {
+			    printf ("FAIL\n");
+			} else {
+			    /* eat the test name */
+			    printf ("\r");
+			    fflush (stdout);
+			}
+			fprintf (stderr, "%s-%s-%s [%d]%s:\t%sFAIL%s\n",
+				 test->name, target->name,
+				 cairo_boilerplate_content_name (target->content), dev_offset, similar ? " (similar)" : "",
+				 fail_face, normal_face);
+			cairo_test_log ("FAIL\n");
+		    }
+		    ret = status;
+		    break;
 		}
-		ret = status;
-		break;
+
+		if (status == CAIRO_TEST_CRASHED)
+		    goto out;
 	    }
 	}
     }
+out:
 
     if (ret != CAIRO_TEST_SUCCESS)
         printf ("Check %s%s out for more information.\n", test->name, CAIRO_TEST_LOG_SUFFIX);