framework: Add class for running multiple shader_tests in a single process

This adds a class based on the ReducedProcessMixin that allows the
python layer to understand the output of shader_runner when it runs more
than one test per process.

Signed-off-by: Dylan Baker <dylanx.c.baker@intel.com>

2 files changed, 132 insertions, 1 deletions

diff --git a/framework/test/shader_test.py b/framework/test/shader_test.py
index 2b14f171a..e72e2ec58 100644
--- a/framework/test/shader_test.py
+++ b/framework/test/shader_test.py
@@ -27,9 +27,12 @@ from __future__ import (
     absolute_import, division, print_function, unicode_literals
 )
 import io
+import os
 import re
 
 from framework import exceptions
+from framework import status
+from .base import ReducedProcessMixin
 from .opengl import FastSkipMixin
 from .piglit_test import PiglitBaseTest
 
@@ -168,3 +171,70 @@ class ShaderTest(FastSkipMixin, PiglitBaseTest):
     def command(self):
         """ Add -auto to the test command """
         return self._command + ['-auto']
+
+
+class MultiShaderTest(ReducedProcessMixin, PiglitBaseTest):
+    """A Shader class that can run more than one test at a time.
+
+    This class can call shader_runner with multiple shader_files at a time, and
+    interpret the results, as well as handle pre-mature exit through crashes or
+    from breaking import assupmtions in the utils about skipping.
+
+    Arguments:
+    filenames -- a list of absolute paths to shader test files
+    """
+
+    def __init__(self, filenames):
+        # TODO fast skip.
+        parser = Parser(filenames[0])
+        parser.parse()
+        prog = parser.prog
+        files = [parser.filename]
+
+        for each in filenames[1:]:
+            parser = Parser(each)
+            parser.parse()
+            assert parser.prog == prog
+            files.append(parser.filename)
+
+        super(MultiShaderTest, self).__init__(
+            [prog] + files,
+            subtests=[os.path.basename(os.path.splitext(f)[0]).lower()
+                      for f in filenames],
+            run_concurrent=True)
+
+    @PiglitBaseTest.command.getter  # pylint: disable=no-member
+    def command(self):
+        """Add -auto to the test command."""
+        return self._command + ['-auto', '-report-subtests']
+
+    def _is_subtest(self, line):
+        return line.startswith('PIGLIT TEST:')
+
+    def _resume(self, current):
+        command = [self.command[0]]
+        command.extend(self.command[current + 1:])
+        return command
+
+    def _stop_status(self):
+        # If the lower level framework skips then return a status for that
+        # subtest as skip, and resume.
+        if self.result.out.endswith('PIGLIT: {"result": "skip" }\n'):
+            return status.SKIP
+        if self.result.returncode > 0:
+            return status.FAIL
+        return status.CRASH
+
+    def _is_cherry(self):
+        # Due to the way that piglt is architected if a particular feature
+        # isn't supported it causes the test to exit with status 0. There is no
+        # straightforward way to fix this, so we work around it by looking for
+        # the message that feature provides and marking the test as not
+        # "cherry" when it is found at the *end* of stdout. (We don't want to
+        # match other places or we'll end up in an infinite loop)
+        return (
+            self.result.returncode == 0 and not
+            self.result.out.endswith(
+                'not supported on this implementation\n') and not
+            self.result.out.endswith(
+                'PIGLIT: {"result": "skip" }\n'))
diff --git a/unittests/framework/test/test_shader_test.py b/unittests/framework/test/test_shader_test.py
index 9cf8b6042..c62aee3f8 100644
--- a/unittests/framework/test/test_shader_test.py
+++ b/unittests/framework/test/test_shader_test.py
@@ -33,7 +33,6 @@ except ImportError:
 import pytest
 import six
 
-from framework import exceptions
 from framework.test import shader_test
 
 # pylint: disable=invalid-name,no-self-use
@@ -204,3 +203,65 @@ def test_command_add_auto(tmpdir):
     test = shader_test.ShaderTest(six.text_type(p))
 
     assert '-auto' in test.command
+
+
+class TestMultiShaderTest(object):
+    """Tests for the MultiShaderTest class."""
+
+    class TestConstructor(object):
+        """Tests for the constructor object."""
+
+        @pytest.fixture
+        def inst(self, tmpdir):
+            """A fixture that creates an instance to test."""
+            one = tmpdir.join('foo.shader_test')
+            one.write(textwrap.dedent("""\
+                [require]
+                GLSL >= 3.0
+
+                [vertex shader]"""))
+            two = tmpdir.join('bar.shader_test')
+            two.write(textwrap.dedent("""\
+                [require]
+                GLSL >= 4.0
+
+                [vertex shader]"""))
+
+            return shader_test.MultiShaderTest(
+                [six.text_type(one), six.text_type(two)])
+
+        def test_prog(self, inst):
+            assert os.path.basename(inst.command[0]) == 'shader_runner'
+
+        def test_filenames(self, inst):
+            assert os.path.basename(inst.command[1]) == 'foo.shader_test'
+            assert os.path.basename(inst.command[2]) == 'bar.shader_test'
+
+        def test_extra(self, inst):
+            assert inst.command[3] == '-auto'
+
+    @pytest.fixture
+    def inst(self, tmpdir):
+        """A fixture that creates an instance to test."""
+        one = tmpdir.join('foo.shader_test')
+        one.write(textwrap.dedent("""\
+            [require]
+            GLSL >= 3.0
+
+            [vertex shader]"""))
+        two = tmpdir.join('bar.shader_test')
+        two.write(textwrap.dedent("""\
+            [require]
+            GLSL >= 4.0
+
+            [vertex shader]"""))
+
+        return shader_test.MultiShaderTest(
+            [six.text_type(one), six.text_type(two)])
+
+    def test_resume(self, inst):
+        actual = inst._resume(1)  # pylint: disable=protected-access
+        assert os.path.basename(actual[0]) == 'shader_runner'
+        assert os.path.basename(actual[1]) == 'bar.shader_test'
+        assert os.path.basename(actual[2]) == '-auto'
+