framework: add a dEQP runner that can run without process isolation

1 files changed, 141 insertions, 61 deletions

diff --git a/framework/test/deqp.py b/framework/test/deqp.py
index 8627feabb..f62d731b1 100644
--- a/framework/test/deqp.py
+++ b/framework/test/deqp.py
@@ -22,7 +22,9 @@ from __future__ import (
     absolute_import, division, print_function, unicode_literals
 )
 import abc
+import collections
 import os
+import re
 import subprocess
 try:
     from lxml import etree as et
@@ -35,7 +37,8 @@ from six.moves import range
 from framework import core, grouptools, exceptions
 from framework import options
 from framework.profile import TestProfile
-from framework.test.base import Test, is_crash_returncode, TestRunError
+from framework.results import TestResult
+from framework.test.base import Test, is_crash_returncode, TestRunError, TestRunner
 
 __all__ = [
     'DEQPBaseTest',
@@ -82,10 +85,11 @@ def select_source(bin_, filename, mustpass, extra_args):
 def make_profile(test_list, test_class):
     """Create a TestProfile instance."""
     profile = TestProfile()
+    runner = DEQPTestRunner()
     for testname in test_list:
         # deqp uses '.' as the testgroup separator.
         piglit_name = testname.replace('.', grouptools.SEPARATOR)
-        profile.test_list[piglit_name] = test_class(testname)
+        profile.test_list[piglit_name] = test_class(testname, runner)
 
     return profile
 
@@ -153,16 +157,6 @@ def iter_deqp_test_cases(case_file):
 
 @six.add_metaclass(abc.ABCMeta)
 class DEQPBaseTest(Test):
-    __RESULT_MAP = {
-        "Pass": "pass",
-        "Fail": "fail",
-        "QualityWarning": "warn",
-        "InternalError": "fail",
-        "Crash": "crash",
-        "NotSupported": "skip",
-        "ResourceError": "crash",
-    }
-
     @abc.abstractproperty
     def deqp_bin(self):
         """The path to the exectuable."""
@@ -177,52 +171,138 @@ class DEQPBaseTest(Test):
         """
         return _EXTRA_ARGS
 
-    def __init__(self, case_name):
-        command = [self.deqp_bin, '--deqp-case=' + case_name]
-
-        super(DEQPBaseTest, self).__init__(command)
-
-        # dEQP's working directory must be the same as that of the executable,
-        # otherwise it cannot find its data files (2014-12-07).
-        # This must be called after super or super will overwrite it
-        self.cwd = os.path.dirname(self.deqp_bin)
-
-    @Test.command.getter
-    def command(self):
-        """Return the command plus any extra arguments."""
-        command = super(DEQPBaseTest, self).command
-        return command + self.extra_args
-
-    def __find_map(self, result):
-        """Run over the lines and set the result."""
-        # splitting this into a separate function allows us to return cleanly,
-        # otherwise this requires some break/else/continue madness
-        for line in result.out.split('\n'):
-            line = line.lstrip()
-            for k, v in six.iteritems(self.__RESULT_MAP):
-                if line.startswith(k):
-                    result.result = v
-                    return
-
-    def interpret_result(self, result):
-        if is_crash_returncode(result.returncode):
-            result.result = 'crash'
-        elif result.returncode != 0:
-            result.result = 'fail'
-        else:
-            self.__find_map(result)
-
-        # We failed to parse the test output. Fallback to 'fail'.
-        if result.result == 'notrun':
-            result.result = 'fail'
-
-    # def _run_command(self, *args, **kwargs):
-    #     """Rerun the command if X11 connection failure happens."""
-    #     for _ in range(5):
-    #         super(DEQPBaseTest, self)._run_command(*args, **kwargs)
-    #         x_err_msg = "FATAL ERROR: Failed to open display"
-    #         if x_err_msg in self.result.err or x_err_msg in self.result.out:
-    #             continue
-    #         return
-
-    #     raise TestRunError('Failed to connect to X server 5 times', 'fail')
+    def __init__(self, case_name, runner):
+        # Use only the case name as command. The real command line will be
+        # built by the runner.
+        super(DEQPBaseTest, self).__init__([case_name], runner=runner)
+
+
+class DEQPTestRunner(TestRunner):
+    """Runner for dEQP tests. Supports running multiple tests at a time.
+    """
+    __RESULT_MAP = {
+        "Pass": "pass",
+        "Fail": "fail",
+        "QualityWarning": "warn",
+        "InternalError": "fail",
+        "Crash": "crash",
+        "NotSupported": "skip",
+        "ResourceError": "crash",
+    }
+
+    RE_test_case = re.compile(r'Test case \'(.*)\'..')
+    RE_result = re.compile(r'  (' + '|'.join(six.iterkeys(__RESULT_MAP)) + r') \(.*\)')
+
+    @TestRunner.max_tests.getter
+    def max_tests(self):
+        """Limit the number of tests so that the progress indicator is still useful
+        in typical runs and we can also still force concurrency. Plus, this is
+        likely to behave better in the case of system hangs.
+        """
+        return 100
+
+    def _build_case_trie(self, case_names):
+        """Turn a list of case names into the trie format expected by dEQP."""
+        def make_trie():
+            return collections.defaultdict(make_trie)
+        root = make_trie()
+
+        for case_name in case_names:
+            node = root
+            for name in case_name.split('.'):
+                node = node[name]
+
+        def format_trie(trie):
+            if len(trie) == 0:
+                return ''
+            return '{' + ','.join(k + format_trie(v) for k, v in six.iteritems(trie)) + '}'
+
+        return format_trie(root)
+
+    def _run_tests(self, results, tests):
+        prog = None
+        extra_args = None
+        case_names = {}
+
+        for test in tests:
+            assert isinstance(test, DEQPBaseTest)
+
+            test_prog = test.deqp_bin
+            test_extra_args = test.extra_args
+
+            if prog is None:
+                prog = test_prog
+            elif prog != test_prog:
+                raise exceptions.PiglitInternalError(
+                    'dEQP binaries must match for tests to be run in the same command!\n')
+
+            if extra_args is None:
+                extra_args = test_extra_args
+            elif len(extra_args) != len(test_extra_args) or \
+                 any(a != b for a, b in zip(extra_args, test_extra_args)):
+                raise exceptions.PiglitInternalError(
+                    'Extra arguments must match for tests to be run in the same command!\n')
+
+            case_names[test.command[0]] = test
+
+        case_name_trie = self._build_case_trie(six.iterkeys(case_names))
+        self._run_command(
+            results,
+            [prog, '--deqp-caselist=' + case_name_trie] + extra_args,
+            # dEQP's working directory must be the same as that of the executable,
+            # otherwise it cannot find its data files (2014-12-07).
+            cwd=os.path.dirname(prog))
+
+        test = None
+        out_all = []
+        out = out_all
+        for each in results.out.split('\n'):
+            out.append(each)
+
+            m = self.RE_test_case.search(each)
+            if m is not None:
+                if test is not None:
+                    out.append('PIGLIT: Failed to parse test result of {}'.format(test.name))
+                    results.set_result(test.name, 'fail')
+                    out_all += out
+
+                test = case_names[m.group(1)]
+                out = []
+            else:
+                m = self.RE_result.search(each)
+                if m is not None:
+                    if m.group(1) not in self.__RESULT_MAP:
+                        out.append('PIGLIT: Unknown result status: {}'.format(m.group(1)))
+                        status = 'fail'
+                    else:
+                        status = self.__RESULT_MAP[m.group(1)]
+
+                    if test is not None:
+                        result = TestResult(status)
+                        result.root = test.name
+                        result.returncode = 0
+                        result.out = '\n'.join(out)
+
+                        test.interpret_result(result)
+                        results.set_result(test.name, result.result)
+                        out_all.append(result.out)
+
+                        test = None
+                        out = out_all
+                    else:
+                        out.append('PIGLIT: Unexpected result status: {}'.format(m.group(1)))
+
+        if is_crash_returncode(results.returncode):
+            if test is not None:
+                results.set_result(test.name, 'crash')
+                out_all += out
+                test = None
+            else:
+                results.result = 'crash'
+        elif test is not None:
+            out.append('PIGLIT: Failed to parse test result of {}'.format(test.name))
+            results.set_result(test.name, 'fail')
+            out_all += out
+            test = None
+
+        results.out = '\n'.join(out_all)