Build a tests file for the X Test suite.xts

Build the test suite externally, then drop a symlink from that xtest
top level directory into bin/

4 files changed, 210 insertions, 1 deletions

diff --git a/framework/exectest.py b/framework/exectest.py
index c7b9c2cef..8271c4db3 100644
--- a/framework/exectest.py
+++ b/framework/exectest.py
@@ -198,7 +198,7 @@ class ExecTest(Test):
 
             results['info'] = unicode("Returncode: {0}\n\nErrors:\n{1}\n\n"
                                       "Output:\n{2}").format(returncode,
-                                                             err, out)
+                                                             err, out) + results['info']
             results['returncode'] = returncode
             results['command'] = ' '.join(self.command)
             results['dmesg'] = dmesg_diff
diff --git a/framework/summary.py b/framework/summary.py
index 6ee12268d..6a76d885a 100644
--- a/framework/summary.py
+++ b/framework/summary.py
@@ -382,6 +382,24 @@ class Summary:
             for key, value in each.tests.iteritems():
                 temp_path = path.join(destination, each.name, path.dirname(key))
 
+                images = value.get('images')
+
+                images_table = ''
+                if images:
+                    images_table += '<table>'
+                    images_table += '  <tr>'
+                    images_table += '    <td></td>'
+                    images_table += '    <td>reference</td>'
+                    images_table += '    <td>rendered</td>'
+                    images_table += '  </tr>'
+                    for image in images:
+                        images_table += '  <tr>'
+                        images_table += '    <td>{0}</td>'.format(image['image_desc'])
+                        images_table += '    <td><img src="file://{0}"></td>'.format(image['image_ref'])
+                        images_table += '    <td><img src="file://{0}"></td>'.format(image['image_render'])
+                        images_table += '  </tr>'
+                    images_table += '</table>'
+
                 if value['result'] not in exclude:
                     # os.makedirs is very annoying, it throws an OSError if
                     # the path requested already exists, so do this check to
@@ -404,6 +422,7 @@ class Summary:
                             info=value.get('info', 'None'),
                             traceback=value.get('traceback', 'None'),
                             command=value.get('command', 'None'),
+                            images_table=images_table,
                             dmesg=value.get('dmesg', 'None'),
                             css=path.relpath(result_css, temp_path),
                             index=path.relpath(index, temp_path)))
diff --git a/templates/test_result.mako b/templates/test_result.mako
index 490c00951..f66d189aa 100644
--- a/templates/test_result.mako
+++ b/templates/test_result.mako
@@ -29,6 +29,13 @@
         <td>${time}</b>
       </tr>
       <tr>
+        <td>Images</td>
+        <td>
+	  ${images_table}
+        </td>
+      </tr>
+
+      <tr>
         <td>Info</td>
         <td>
           <pre>${info | h}</pre>
diff --git a/tests/xts.py b/tests/xts.py
new file mode 100644
index 000000000..ffb0e74c6
--- /dev/null
+++ b/tests/xts.py
@@ -0,0 +1,183 @@
+#
+# Copyright (c) 2012 Intel Corporation
+#
+# Permission is hereby granted, free of charge, to any person
+# obtaining a copy of this software and associated documentation
+# files (the "Software"), to deal in the Software without
+# restriction, including without limitation the rights to use,
+# copy, modify, merge, publish, distribute, sublicense, and/or
+# sell copies of the Software, and to permit persons to whom the
+# Software is furnished to do so, subject to the following
+# conditions:
+#
+# This permission notice shall be included in all copies or
+# substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
+# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
+# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+# PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE AUTHOR(S) BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
+# AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF
+# OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+# DEALINGS IN THE SOFTWARE.
+
+import os
+import re
+import sys
+import subprocess
+
+from framework.core import *
+from framework.exectest import *
+
+#############################################################################
+##### CairoTest: Execute an intel-gpu-tools test
+#####
+##### To use this, create a cairo-test-suite symlink in piglit/bin
+##### which points to the cairo-test-suite binary from cairo/tests.
+##### compiled tests. Piglit will automatically add all tests into the
+##### 'cairo' category.
+#############################################################################
+
+x_test_suite = os.path.join(testBinDir, 'xtest')
+if not os.path.exists(x_test_suite):
+    print "xtest symlink not found!"
+    sys.exit(0)
+
+profile = TestProfile()
+
+class xts_test(ExecTest):
+    def __init__(self, name, testname, testnum):
+        ExecTest.__init__(self, ['./' + os.path.basename(name), '-i', str(testnum)])
+        self.testname = '{0}-{1}'.format(testname, testnum)
+        self.cwd = os.path.dirname(os.path.realpath(name))
+        self.test_results_file = os.path.join(self.cwd, self.testname)
+        self.env["TET_RESFILE"] = self.test_results_file
+        self.env["XT_RESET_DELAY"] = '0'
+        self.env["XT_FONTPATH_GOOD"] = '/usr/share/fonts/X11/misc'
+        self.env["XT_FONTPATH"] = os.path.join(x_test_suite, 'xts5', 'fonts')
+
+        # Possibly unnecessary?
+        self.env["XT_LOCAL"] = 'Yes'
+        self.env["XT_TCP"] = 'No'
+        self.env["XT_DISPLAYHOST"] = ''
+
+    def process_log_for_images(self, log):
+        images = []
+
+        for line in log.splitlines():
+            m = re.search('See file (Err[0-9]+.err)', line)
+            if m is not None:
+                # Can we parse any other useful information out to
+                # give a better description of each image?
+                desc = m.group(1)
+
+                # The error logs are text, with a header with width,
+                # height, and depth, then run-length-encoded pixel
+                # values (in hexadecimal).  Use xtsttopng to convert
+                # the error log to a pair of PNGs so we can put them
+                # in the summary.
+                command = ['xtsttopng', os.path.join(self.cwd, m.group(1))]
+                try:
+                    proc = subprocess.Popen(command,
+                                            stdout=subprocess.PIPE,
+                                            stderr=subprocess.PIPE,
+                                            cwd = self.cwd,
+                                            universal_newlines=True)
+
+                    out = proc.communicate()[0]
+
+                    cwd = os.getcwd()
+
+                    # Each Err*.err log contains a rendered image, and
+                    # a reference image that it was compared to.  We
+                    # relocate the to our tree with more useful names.
+                    # (Otherwise, since tests generate error logs with
+                    # numbers sequentially starting from 0, each
+                    # subtest with an error would overwrite the
+                    # previous test's images).
+                    #
+                    # XXX: Of course, we're overwriting them between
+                    # separate piglit runs, because I'm dropping them
+                    # at the top level instead of in our results
+                    # directory.  Need to plumb that output directory
+                    # in to this module.
+                    ref_path = '{0}/{1}-{2}-ref.png'.format(cwd, self.testname, m.group(1))
+                    render_path = '{0}/{1}-{2}-render.png'.format(cwd, self.testname, m.group(1))
+                    found_ref = False
+                    found_render = False
+                    for i in out.splitlines():
+                        if not found_render:
+                            os.rename(os.path.join(self.cwd, i), render_path)
+                            found_render = True
+                        else:
+                            os.rename(os.path.join(self.cwd, i), ref_path)
+                            assert not found_ref
+                            found_ref = True
+                    images.append(dict(image_desc = desc,
+                                       image_ref = ref_path,
+                                       image_render = render_path))
+                except OSError as e:
+                    images.append(dict(image_desc = 'image processing failed'))
+
+        return images
+
+    def interpretResult(self, out, returncode, results, dmesg):
+
+        log = ''
+        try:
+            with open(self.test_results_file) as f:
+                log = f.read()
+                os.remove(self.test_results_file)
+                results['info'] = log
+        except IOError:
+            results['info'] = "No results file found"
+
+        if returncode == 0:
+            if re.search('FAIL', out) is not None:
+                results['result'] = 'fail'
+            elif re.search('PASS', out) is not None:
+                results['result'] = 'pass'
+            else:
+                results['result'] = 'fail'
+        elif returncode == 77:
+            results['result'] = 'skip'
+        elif returncode == 1:
+            if re.search('Could not open all VSW5 fonts', log):
+                results['result'] = 'warn'
+            else:
+                results['result'] = 'fail'
+        else:
+            results['result'] = 'fail'
+
+        results['images'] = self.process_log_for_images(log)
+
+        return out
+
+def add_xts_m_files(path):
+    """Add all tests represented by .m files."""
+    fs_path = os.path.join(x_test_suite, path)
+
+    for filename in os.listdir(fs_path):
+        filepath = os.path.join(fs_path, filename)
+        if os.path.isdir(filepath):
+            add_xts_m_files(os.path.join(path, filename))
+        else:
+            ext = filename.rsplit('.')[-1]
+            if ext != 'm':
+                continue
+            testname = filename[0:-(len(ext) + 1)]  # +1 for '.'
+
+            # Each 'ASSERTION' line in the .m file corresponds to a
+            # generated subtest (even if one block of code has
+            # multiple asssertions).
+            i = 1
+            with open(filepath) as f:
+                for line in f:
+                    if not re.match('>>ASSERTION', line):
+                        continue
+
+                    profile.tests['{0}/{1}/{2}'.format(path, testname, i)] = xts_test(os.path.join(fs_path, testname), testname, i)
+                    i = i + 1
+
+add_xts_m_files('xts5')