test: Run some XTS5 integration tests against Xvfb if possible.

By default the tests will be skipped.  However, if you set XTEST_DIR
to the repo of a built X Test Suite and PIGLIT_DIR to a piglit repo
(no build necessary), make check will run piglit's xts-render tests
against Xvfb.

We could run more of XTS5, but I haven't spent the time identifying
what additional subset would be worth running, since much of it is
only really testing the client libraries.  We want to make sure that
we keep the runtime down, and this subset of the test suite took 92
seconds according to piglit.

Reviewed-by: Adam Jackson <ajax@redhat.com>

3 files changed, 135 insertions, 3 deletions

diff --git a/test/Makefile.am b/test/Makefile.am
index f5e54680b..5a35e2bb1 100644
--- a/test/Makefile.am
+++ b/test/Makefile.am
@@ -12,8 +12,23 @@ endif
 endif
 check_LTLIBRARIES = libxservertest.la
 
-TESTS=$(noinst_PROGRAMS)
-TESTS_ENVIRONMENT = $(XORG_MALLOC_DEBUG_ENV)
+if XVFB
+XVFB_TESTS = scripts/xvfb-piglit.sh
+endif
+
+SCRIPT_TESTS = \
+	$(XVFB_TESTS) \
+	$(NULL)
+
+TESTS = \
+	$(noinst_PROGRAMS) \
+	$(SCRIPT_TESTS) \
+	$(NULL)
+
+TESTS_ENVIRONMENT = \
+	XSERVER_DIR=$(abs_top_builddir) \
+	$(XORG_MALLOC_DEBUG_ENV) \
+	$(NULL)
 
 AM_CFLAGS = $(DIX_CFLAGS) @XORG_CFLAGS@
 AM_CPPFLAGS = $(XORG_INCS)
@@ -139,5 +154,9 @@ endif
 libxservertest_la_DEPENDENCIES = $(libxservertest_la_LIBADD)
 endif
 
-EXTRA_DIST = ddxstubs.c
+EXTRA_DIST = \
+	$(SCRIPT_TESTS) \
+	scripts/xinit-piglit-session.sh \
+	ddxstubs.c \
+	$(NULL)
 
diff --git a/test/scripts/xinit-piglit-session.sh b/test/scripts/xinit-piglit-session.sh
new file mode 100755
index 000000000..0f0c7943d
--- /dev/null
+++ b/test/scripts/xinit-piglit-session.sh
@@ -0,0 +1,47 @@
+#!/bin/sh
+
+# .xinitrc replacement to run piglit and exit.
+#
+# Note that piglit will run many processes against the server, so
+# running the server with -noreset is recommended to improve runtime.
+
+set -e
+
+if test "x$PIGLIT_DIR" = "x"; then
+    echo "PIGLIT_DIR must be set to the directory of the piglit repository."
+    exit 1
+fi
+
+if test "x$PIGLIT_RESULTS_DIR" = "x"; then
+    echo "PIGLIT_RESULTS_DIR must be defined"
+    exit 1
+fi
+
+if test "x$XTEST_DIR" = "x"; then
+    echo "XTEST_DIR must be set to the root of the built xtest tree."
+    exit 1
+fi
+
+cd $PIGLIT_DIR
+
+# Write the piglit.conf we'll use for our testing.  Don't use the
+# default piglit.conf name because that may overwrite a local
+# piglit.conf.
+PIGLITCONF=piglit-xserver-test.conf
+cat <<EOF > $PIGLITCONF
+[xts]
+path=$XTEST_DIR
+EOF
+
+# Skip some tests that are failing at the time of importing the script.
+#    "REPORT: min_bounds, rbearing was 0, expecting 2"
+PIGLIT_ARGS="$PIGLIT_ARGS -x xlistfontswithinfo@3"
+PIGLIT_ARGS="$PIGLIT_ARGS -x xlistfontswithinfo@4"
+PIGLIT_ARGS="$PIGLIT_ARGS -x xloadqueryfont@1"
+PIGLIT_ARGS="$PIGLIT_ARGS -x xqueryfont@1"
+PIGLIT_ARGS="$PIGLIT_ARGS -x xqueryfont@2"
+#    "REPORT: Incorrect pixel on inside of area at point (0, 0): 0x0 != 0x1"
+PIGLIT_ARGS="$PIGLIT_ARGS -x xgetimage@7"
+PIGLIT_ARGS="$PIGLIT_ARGS -x xgetsubimage@7"
+
+exec ./piglit-run.py xts-render -f $PIGLITCONF $PIGLIT_ARGS $PIGLIT_RESULTS_DIR
diff --git a/test/scripts/xvfb-piglit.sh b/test/scripts/xvfb-piglit.sh
new file mode 100755
index 000000000..2a4e94052
--- /dev/null
+++ b/test/scripts/xvfb-piglit.sh
@@ -0,0 +1,66 @@
+#!/bin/sh
+
+set -e
+
+if test "x$XTEST_DIR" = "x"; then
+    echo "XTEST_DIR must be set to the directory of the xtest repository."
+    # Exit as a "skip" so make check works even without piglit.
+    exit 77
+fi
+
+if test "x$PIGLIT_DIR" = "x"; then
+    echo "PIGLIT_DIR must be set to the directory of the piglit repository."
+    # Exit as a "skip" so make check works even without piglit.
+    exit 77
+fi
+
+if test "x$XSERVER_DIR" = "x"; then
+    echo "XSERVER_DIR must be set to the directory of the xserver repository."
+    # Exit as a real failure because it should always be set.
+    exit 1
+fi
+
+export PIGLIT_RESULTS_DIR=$PIGLIT_DIR/results/xvfb
+
+startx \
+    $XSERVER_DIR/test/scripts/xinit-piglit-session.sh \
+    -- \
+    $XSERVER_DIR/hw/vfb/Xvfb \
+        -noreset \
+        -screen scrn 1280x1024x24
+
+# Write out piglit-summaries.
+SHORT_SUMMARY=$PIGLIT_RESULTS_DIR/summary
+LONG_SUMMARY=$PIGLIT_RESULTS_DIR/long-summary
+$PIGLIT_DIR/piglit-summary.py -s $PIGLIT_RESULTS_DIR > $SHORT_SUMMARY
+$PIGLIT_DIR/piglit-summary.py $PIGLIT_RESULTS_DIR > $LONG_SUMMARY
+
+# Write the short summary to make check's log file.
+cat $SHORT_SUMMARY
+
+# Parse the piglit summary to decide on our exit status.
+status=0
+# "pass: 0" would mean no tests actually ran.
+if grep "pass:.*0" $SHORT_SUMMARY > /dev/null; then
+    status=1
+fi
+# Fails or crashes should be failures from make check's perspective.
+if ! grep "fail:.*0" $SHORT_SUMMARY > /dev/null; then
+    status=1
+fi
+if ! grep "crash:.*0" $SHORT_SUMMARY > /dev/null; then
+    status=1
+fi
+
+if test $status != 0; then
+    $PIGLIT_DIR/piglit-summary-html.py \
+	--overwrite \
+	$PIGLIT_RESULTS_DIR/html \
+	$PIGLIT_RESULTS_DIR
+
+    echo "Some piglit tests failed."
+    echo "The list of failing tests can be found in $LONG_SUMMARY."
+    echo "An html page of the failing tests can be found at $PIGLIT_RESULTS_DIR/html/problems.html"
+fi
+
+exit $status