From abaef9bfbe11302beb6481547632b44be9d5be8d Mon Sep 17 00:00:00 2001
From: Chris Wilson <chris@chris-wilson.co.uk>
Date: Wed, 9 Sep 2009 13:57:06 +0100
Subject: [test] Compare a failure against the image output

If a backend fails in exactly the same way as the image, then we can
safely assume that the failure is systematic and not an error in the
backend, so change the result to XFAIL.
---
 test/cairo-test.c | 46 ++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 46 insertions(+)

diff --git a/test/cairo-test.c b/test/cairo-test.c
index 55e28dc7..1f495af6 100644
--- a/test/cairo-test.c
+++ b/test/cairo-test.c
@@ -1369,6 +1369,52 @@ REPEAT:
 	    cairo_test_copy_file (test_filename, pass_filename);
 	}
 
+	/* If failed, compare against the current image output,
+	 * and attempt to detect systematic failures.
+	 */
+	if (ret == CAIRO_TEST_FAILURE) {
+	    char *image_out_path;
+
+	    image_out_path =
+		cairo_test_reference_filename (ctx,
+					       base_name,
+					       ctx->test_name,
+					       "image",
+					       "image",
+					       format,
+					       CAIRO_TEST_OUT_SUFFIX,
+					       CAIRO_TEST_PNG_EXTENSION);
+	    if (image_out_path != NULL) {
+		if (cairo_test_files_equal (out_png_path,
+					    image_out_path))
+		{
+		    ret = CAIRO_TEST_XFAILURE;
+		}
+		else
+		{
+		    ref_image =
+			cairo_image_surface_create_from_png (image_out_path);
+		    if (cairo_surface_status (ref_image) == CAIRO_STATUS_SUCCESS)
+		    {
+			diff_status = image_diff (ctx,
+						  test_image, ref_image,
+						  diff_image,
+						  &result);
+			if (diff_status == CAIRO_STATUS_SUCCESS &&
+			    (result.pixels_changed == 0 ||
+			     result.max_diff > target->error_tolerance))
+			{
+			    ret = CAIRO_TEST_XFAILURE;
+			}
+
+			cairo_surface_destroy (ref_image);
+		    }
+		}
+
+		free (image_out_path);
+	    }
+	}
+
 	cairo_surface_destroy (test_image);
 	cairo_surface_destroy (diff_image);
     }
-- 
cgit v1.2.3