Improve instructions for when to add a test to the XFAIL list.

Print explanations for expected failures.

5 files changed, 36 insertions, 16 deletions

diff --git a/test/Makefile.am b/test/Makefile.am
index e6e7d859..c47789ac 100644
--- a/test/Makefile.am
+++ b/test/Makefile.am
@@ -1,4 +1,4 @@
-# All new test cases go here
+# All test cases go here
 TESTS = 		\
 clip-twice		\
 coverage		\
@@ -22,9 +22,9 @@ translate-show-surface	\
 trap-clip		\
 user-data
 
-# And all new tests go here too. I really don't like having to repeat
-# this list. Anyone know a good way to avoid it? Can I use a wildcard
-# here?
+# All tests which have a reference image go here.
+# I really don't like having to repeat this list. Anyone know a good
+# way to avoid it? Can I use a wildcard here?
 EXTRA_DIST =			\
 create-for-png-ref.png		\
 fill-rule-ref.png		\
@@ -43,17 +43,19 @@ transforms-ref.png		\
 translate-show-surface-ref.png	\
 trap-clip-ref.png
 
-# Once we can draw the text-rotate.c test case correctly, we should
-# create and add text-rotate-ref.png to the list of reference PNGs.
-
-# This list is only for known bugs (not regressions). We do need to
-# fix these before the next release, but they are expected to fail for
-# now, so they don't need to hold up any new code commit.
+# Any test for which the code committed to CVS is expected to fail
+# should be listed here.
+#
+# This way, we can avoid being bothered by reports of bugs we are
+# aware of, but users can still report when tests start behaving in
+# unexpected ways on their system.
+#
+# Of course, before any "release" of cairo we should eliminate
+# everything from this list by fixing the bugs. (We don't necessarily
+# have to be that strict for "snapshots" though.)
 #
-# When new bugs are found in committed code they can be listed
-# here. New failures due to local, uncommitted code changes are
-# regression bugs that should not be listed here. Instead they should
-# be fixed before the code is committed.
+# Also, any test listed here should call cairo_test_expect_failure and
+# provide an explanation for the expected failure.
 XFAIL_TESTS =		\
 pixman-rotate		\
 text-rotate
diff --git a/test/cairo-test.c b/test/cairo-test.c
index 48b6ec14..487d3dfb 100644
--- a/test/cairo-test.c
+++ b/test/cairo-test.c
@@ -98,6 +98,15 @@ xunlink (const char *pathname)
 }
 
 cairo_test_status_t
+cairo_test_expect_failure (cairo_test_t		      *test, 
+			   cairo_test_draw_function_t  draw,
+			   const char		      *because)
+{
+    printf ("%s is expected to fail %s\n", test->name, because);
+    return cairo_test (test, draw);
+}
+
+cairo_test_status_t
 cairo_test (cairo_test_t *test, cairo_test_draw_function_t draw)
 {
     cairo_test_status_t status;
diff --git a/test/cairo-test.h b/test/cairo-test.h
index 6bf9111d..27efacc0 100644
--- a/test/cairo-test.h
+++ b/test/cairo-test.h
@@ -47,6 +47,11 @@ typedef cairo_test_status_t  (*cairo_test_draw_function_t) (cairo_t *cr, int wid
 cairo_test_status_t
 cairo_test (cairo_test_t *test, cairo_test_draw_function_t draw);
 
+cairo_test_status_t
+cairo_test_expect_failure (cairo_test_t		      *test, 
+			   cairo_test_draw_function_t  draw,
+			   const char		      *reason);
+
 cairo_pattern_t *
 cairo_test_create_png_pattern (cairo_t *cr, const char *filename);
 
diff --git a/test/pixman-rotate.c b/test/pixman-rotate.c
index a5e52d2c..af3b5610 100644
--- a/test/pixman-rotate.c
+++ b/test/pixman-rotate.c
@@ -76,5 +76,7 @@ draw (cairo_t *cr, int width, int height)
 int
 main (void)
 {
-    return cairo_test (&test, draw);
+    return cairo_test_expect_failure (&test, draw,
+				      "because of a known off-by-one bug "
+				      "when rotating a pixman image");
 }
diff --git a/test/text-rotate.c b/test/text-rotate.c
index 13b2d99d..3d5f4d3c 100644
--- a/test/text-rotate.c
+++ b/test/text-rotate.c
@@ -115,5 +115,7 @@ draw (cairo_t *cr, int width, int height)
 int
 main (void)
 {
-    return cairo_test (&test, draw);
+    return cairo_test_expect_failure (&test, draw,
+				      "because of known glyph positioning bugs "
+				      "with rotated text");
 }