From deae9c7e846e244e5d62b2dcfb6663fde0e12cbb Mon Sep 17 00:00:00 2001 From: Eric Anholt Date: Wed, 31 Aug 2016 17:00:00 -0700 Subject: test: Run some XTS5 integration tests against Xvfb if possible. By default the tests will be skipped. However, if you set XTEST_DIR to the repo of a built X Test Suite and PIGLIT_DIR to a piglit repo (no build necessary), make check will run piglit's xts-render tests against Xvfb. We could run more of XTS5, but I haven't spent the time identifying what additional subset would be worth running, since much of it is only really testing the client libraries. We want to make sure that we keep the runtime down, and this subset of the test suite took 92 seconds according to piglit. Reviewed-by: Adam Jackson --- test/Makefile.am | 25 ++++++++++++-- test/scripts/xinit-piglit-session.sh | 47 +++++++++++++++++++++++++ test/scripts/xvfb-piglit.sh | 66 ++++++++++++++++++++++++++++++++++++ 3 files changed, 135 insertions(+), 3 deletions(-) create mode 100755 test/scripts/xinit-piglit-session.sh create mode 100755 test/scripts/xvfb-piglit.sh (limited to 'test') diff --git a/test/Makefile.am b/test/Makefile.am index f5e54680b..5a35e2bb1 100644 --- a/test/Makefile.am +++ b/test/Makefile.am @@ -12,8 +12,23 @@ endif endif check_LTLIBRARIES = libxservertest.la -TESTS=$(noinst_PROGRAMS) -TESTS_ENVIRONMENT = $(XORG_MALLOC_DEBUG_ENV) +if XVFB +XVFB_TESTS = scripts/xvfb-piglit.sh +endif + +SCRIPT_TESTS = \ + $(XVFB_TESTS) \ + $(NULL) + +TESTS = \ + $(noinst_PROGRAMS) \ + $(SCRIPT_TESTS) \ + $(NULL) + +TESTS_ENVIRONMENT = \ + XSERVER_DIR=$(abs_top_builddir) \ + $(XORG_MALLOC_DEBUG_ENV) \ + $(NULL) AM_CFLAGS = $(DIX_CFLAGS) @XORG_CFLAGS@ AM_CPPFLAGS = $(XORG_INCS) @@ -139,5 +154,9 @@ endif libxservertest_la_DEPENDENCIES = $(libxservertest_la_LIBADD) endif -EXTRA_DIST = ddxstubs.c +EXTRA_DIST = \ + $(SCRIPT_TESTS) \ + scripts/xinit-piglit-session.sh \ + ddxstubs.c \ + $(NULL) diff --git a/test/scripts/xinit-piglit-session.sh b/test/scripts/xinit-piglit-session.sh new file mode 100755 index 000000000..0f0c7943d --- /dev/null +++ b/test/scripts/xinit-piglit-session.sh @@ -0,0 +1,47 @@ +#!/bin/sh + +# .xinitrc replacement to run piglit and exit. +# +# Note that piglit will run many processes against the server, so +# running the server with -noreset is recommended to improve runtime. + +set -e + +if test "x$PIGLIT_DIR" = "x"; then + echo "PIGLIT_DIR must be set to the directory of the piglit repository." + exit 1 +fi + +if test "x$PIGLIT_RESULTS_DIR" = "x"; then + echo "PIGLIT_RESULTS_DIR must be defined" + exit 1 +fi + +if test "x$XTEST_DIR" = "x"; then + echo "XTEST_DIR must be set to the root of the built xtest tree." + exit 1 +fi + +cd $PIGLIT_DIR + +# Write the piglit.conf we'll use for our testing. Don't use the +# default piglit.conf name because that may overwrite a local +# piglit.conf. +PIGLITCONF=piglit-xserver-test.conf +cat < $PIGLITCONF +[xts] +path=$XTEST_DIR +EOF + +# Skip some tests that are failing at the time of importing the script. +# "REPORT: min_bounds, rbearing was 0, expecting 2" +PIGLIT_ARGS="$PIGLIT_ARGS -x xlistfontswithinfo@3" +PIGLIT_ARGS="$PIGLIT_ARGS -x xlistfontswithinfo@4" +PIGLIT_ARGS="$PIGLIT_ARGS -x xloadqueryfont@1" +PIGLIT_ARGS="$PIGLIT_ARGS -x xqueryfont@1" +PIGLIT_ARGS="$PIGLIT_ARGS -x xqueryfont@2" +# "REPORT: Incorrect pixel on inside of area at point (0, 0): 0x0 != 0x1" +PIGLIT_ARGS="$PIGLIT_ARGS -x xgetimage@7" +PIGLIT_ARGS="$PIGLIT_ARGS -x xgetsubimage@7" + +exec ./piglit-run.py xts-render -f $PIGLITCONF $PIGLIT_ARGS $PIGLIT_RESULTS_DIR diff --git a/test/scripts/xvfb-piglit.sh b/test/scripts/xvfb-piglit.sh new file mode 100755 index 000000000..2a4e94052 --- /dev/null +++ b/test/scripts/xvfb-piglit.sh @@ -0,0 +1,66 @@ +#!/bin/sh + +set -e + +if test "x$XTEST_DIR" = "x"; then + echo "XTEST_DIR must be set to the directory of the xtest repository." + # Exit as a "skip" so make check works even without piglit. + exit 77 +fi + +if test "x$PIGLIT_DIR" = "x"; then + echo "PIGLIT_DIR must be set to the directory of the piglit repository." + # Exit as a "skip" so make check works even without piglit. + exit 77 +fi + +if test "x$XSERVER_DIR" = "x"; then + echo "XSERVER_DIR must be set to the directory of the xserver repository." + # Exit as a real failure because it should always be set. + exit 1 +fi + +export PIGLIT_RESULTS_DIR=$PIGLIT_DIR/results/xvfb + +startx \ + $XSERVER_DIR/test/scripts/xinit-piglit-session.sh \ + -- \ + $XSERVER_DIR/hw/vfb/Xvfb \ + -noreset \ + -screen scrn 1280x1024x24 + +# Write out piglit-summaries. +SHORT_SUMMARY=$PIGLIT_RESULTS_DIR/summary +LONG_SUMMARY=$PIGLIT_RESULTS_DIR/long-summary +$PIGLIT_DIR/piglit-summary.py -s $PIGLIT_RESULTS_DIR > $SHORT_SUMMARY +$PIGLIT_DIR/piglit-summary.py $PIGLIT_RESULTS_DIR > $LONG_SUMMARY + +# Write the short summary to make check's log file. +cat $SHORT_SUMMARY + +# Parse the piglit summary to decide on our exit status. +status=0 +# "pass: 0" would mean no tests actually ran. +if grep "pass:.*0" $SHORT_SUMMARY > /dev/null; then + status=1 +fi +# Fails or crashes should be failures from make check's perspective. +if ! grep "fail:.*0" $SHORT_SUMMARY > /dev/null; then + status=1 +fi +if ! grep "crash:.*0" $SHORT_SUMMARY > /dev/null; then + status=1 +fi + +if test $status != 0; then + $PIGLIT_DIR/piglit-summary-html.py \ + --overwrite \ + $PIGLIT_RESULTS_DIR/html \ + $PIGLIT_RESULTS_DIR + + echo "Some piglit tests failed." + echo "The list of failing tests can be found in $LONG_SUMMARY." + echo "An html page of the failing tests can be found at $PIGLIT_RESULTS_DIR/html/problems.html" +fi + +exit $status -- cgit v1.2.3