framework/test/opengl.py: Add FastSkipMixin which checks extensions

This Mixin provides a way for OpenGL tests to skip very fast. Currently
it only applies to GL extensions, but will be extended to cover GLSL
version requirements and GL version requirements (and ES)>

This is split into a separate module because it's going to grow into a
fairly large amount of code (mostly around querying wflinfo).

Signed-off-by: Dylan Baker <dylanx.c.baker@intel.com>

3 files changed, 398 insertions, 1 deletions

diff --git a/framework/test/opengl.py b/framework/test/opengl.py
new file mode 100644
index 000000000..3485d3a6b
--- /dev/null
+++ b/framework/test/opengl.py
@@ -0,0 +1,206 @@
+# Copyright (c) 2015 Intel Corporation
+
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+"""Mixins for OpenGL derived tests."""
+
+from __future__ import absolute_import, division, print_function
+import errno
+import os
+import subprocess
+
+from framework import exceptions, core
+from framework.options import OPTIONS
+from .base import TestIsSkip
+
+# pylint: disable=too-few-public-methods
+
+__all__ = [
+    'FastSkipMixin',
+]
+
+
+class StopWflinfo(exceptions.PiglitException):
+    """Exception called when wlfinfo getter should stop."""
+    def __init__(self, reason):
+        super(StopWflinfo, self).__init__()
+        self.reason = reason
+
+
+class WflInfo(object):
+    """Class representing platform information as provided by wflinfo.
+
+    The design of this is odd to say the least, it's basically a bag with some
+    lazy property evaluators in it, used to avoid calculating the values
+    provided by wflinfo more than once.
+
+    The problems:
+    - Needs to be shared with all subclasses
+    - Needs to evaluate only once
+    - cannot evaluate until user sets OPTIONS.env['PIGLIT_PLATFORM']
+
+    This solves all of that, and is
+
+    """
+    __shared_state = {}
+    def __new__(cls, *args, **kwargs):
+        # Implement the borg pattern:
+        # https://code.activestate.com/recipes/66531-singleton-we-dont-need-no-stinkin-singleton-the-bo/
+        #
+        # This is something like a singleton, but much easier to implement
+        self = super(WflInfo, cls).__new__(cls, *args, **kwargs)
+        self.__dict__ = cls.__shared_state
+        return self
+
+    @staticmethod
+    def __call_wflinfo(opts):
+        """Helper to call wflinfo and reduce code duplication.
+
+        This catches and handles CalledProcessError and OSError.ernno == 2
+        gracefully: it passes them to allow platforms without a particular
+        gl/gles version or wflinfo (resepctively) to work.
+
+        Arguments:
+        opts -- arguments to pass to wflinfo other than verbose and platform
+
+        """
+        with open(os.devnull, 'w') as d:
+            try:
+                raw = subprocess.check_output(
+                    ['wflinfo',
+                     '--platform', OPTIONS.env['PIGLIT_PLATFORM']] + opts,
+                    stderr=d)
+            except subprocess.CalledProcessError:
+                # When we hit this error it usually going to be because we have
+                # an incompatible platform/profile combination
+                raise StopWflinfo('Called')
+            except OSError as e:
+                # If we get a 'no wflinfo' warning then just return
+                if e.errno == errno.ENOENT:
+                    raise StopWflinfo('OSError')
+                raise
+        return raw
+
+    @staticmethod
+    def __getline(lines, name):
+        """Find a line in a list return it."""
+        for line in lines:
+            if line.startswith(name):
+                return line
+        raise Exception('Unreachable')
+
+    @core.lazy_property
+    def gl_extensions(self):
+        """Call wflinfo to get opengl extensions.
+
+        This provides a very conservative set of extensions, it provides every
+        extension from gles1, 2 and 3 and from GL both core and compat profile
+        as a single set. This may let a few tests execute that will still skip
+        manually, but it helps to ensure that this method never skips when it
+        shouldn't.
+
+        """
+        _trim = len('OpenGL extensions: ')
+        all_ = set()
+
+        def helper(const, vars_):
+            """Helper function to reduce code duplication."""
+            # This is a pretty fragile function but it really does help with
+            # duplication
+            for var in vars_:
+                try:
+                    ret = self.__call_wflinfo(const + [var])
+                except StopWflinfo as e:
+                    # This means tat the particular api or profile is
+                    # unsupported
+                    if e.reason == 'Called':
+                        continue
+                    else:
+                        raise
+                all_.update(set(self.__getline(
+                    ret.split('\n'), 'OpenGL extensions')[_trim:].split()))
+
+        try:
+            helper(['--verbose', '--api'], ['gles1', 'gles2', 'gles3'])
+            helper(['--verbose', '--api', 'gl', '--profile'],
+                   ['core', 'compat', 'none'])
+        except StopWflinfo as e:
+            # Handle wflinfo not being installed by returning an empty set. This
+            # will essentially make FastSkipMixin a no-op.
+            if e.reason == 'OSError':
+                return set()
+            raise
+
+        return {e.strip() for e in all_}
+
+
+class FastSkipMixin(object):
+    """Fast test skipping for OpenGL based suites.
+
+    This provides an is_skip() method which will skip the test if an of it's
+    requirements are not met.
+
+    It also provides new attributes:
+    gl_reqruied -- This is a set of extensions that are required for running
+                   the extension.
+    gl_version -- A float that is the required version number for an OpenGL
+                  test.
+    gles_version -- A float that is the required version number for an OpenGL
+                    ES test
+    glsl_version -- A float that is the required version number of OpenGL
+                    Shader Language for a test
+    glsl_ES_version -- A float that is the required version number of OpenGL ES
+                       Shader Language for a test
+
+    This requires wflinfo to be installed and accessible to provide it's
+    functionality, however, it will no-op if wflinfo is not accessible.
+
+    The design of this function is conservative. The design goal is that it
+    it is better to run a few tests that could have been skipped, than to skip
+    all the tests that could have, but also a few that should have run.
+
+    """
+    # XXX: This still gets called once for each thread. (4 times with 4
+    # threads), this is a synchronization issue and I don't know how to stop it
+    # other than querying each value before starting the thread pool.
+    __info = WflInfo()
+
+    def __init__(self, *args, **kwargs):
+        super(FastSkipMixin, self).__init__(*args, **kwargs)
+        self.gl_required = set()
+        self.gl_version = None
+        self.gles_version = None
+        self.glsl_version = None
+        self.glsl_es_version = None
+
+    def is_skip(self):
+        """Skip this test if any of it's feature requirements are unmet.
+
+        If no extensions were calculated (if wflinfo isn't installed) then run
+        all tests.
+
+        """
+        if self.__info.gl_extensions:
+            for extension in self.gl_required:
+                if extension not in self.__info.gl_extensions:
+                    raise TestIsSkip(
+                        'Test requires extension {} '
+                        'which is not available'.format(extension))
+
+        super(FastSkipMixin, self).is_skip()
diff --git a/framework/tests/base_tests.py b/framework/tests/base_tests.py
index a7afd255e..c005273a3 100644
--- a/framework/tests/base_tests.py
+++ b/framework/tests/base_tests.py
@@ -28,7 +28,10 @@ from nose.plugins.attrib import attr
 
 import framework.tests.utils as utils
 from framework.test.base import (
-    Test, WindowResizeMixin, ValgrindMixin, TestRunError
+    Test,
+    TestRunError,
+    ValgrindMixin,
+    WindowResizeMixin,
 )
 from framework.tests.status_tests import PROBLEMS, STATUSES
 from framework.options import _Options as Options
diff --git a/framework/tests/opengl_tests.py b/framework/tests/opengl_tests.py
new file mode 100644
index 000000000..aa427380c
--- /dev/null
+++ b/framework/tests/opengl_tests.py
@@ -0,0 +1,188 @@
+# Copyright (c) 2015 Intel Corporation
+
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+"""Test the opengl module."""
+
+from __future__ import absolute_import, division, print_function
+import subprocess
+
+import mock
+import nose.tools as nt
+
+import framework.tests.utils as utils
+from framework.test import opengl
+from framework.test.base import TestIsSkip
+
+# pylint: disable=invalid-name,protected-access,line-too-long,pointless-statement,attribute-defined-outside-init
+
+
+class TestWflInfo(object):
+    """Tests for the WflInfo class."""
+    __patchers = []
+
+    def setup(self):
+        """Setup each instance, patching necissary bits."""
+        self._test = opengl.WflInfo()
+        self.__patchers.append(mock.patch.dict(
+            'framework.test.opengl.OPTIONS.env',
+            {'PIGLIT_PLATFORM': 'foo'}))
+        self.__patchers.append(mock.patch(
+            'framework.test.opengl.WflInfo._WflInfo__shared_state', {}))
+
+        for f in self.__patchers:
+            f.start()
+
+    def teardown(self):
+        for f in self.__patchers:
+            f.stop()
+
+    def test_gl_extension(self):
+        """test.opengl.WflInfo.gl_extensions: Provides list of gl extensions"""
+        rv = 'foo\nbar\nboink\nOpenGL extensions: GL_foobar GL_ham_sandwhich\n'
+        expected = set(['GL_foobar', 'GL_ham_sandwhich'])
+
+        with mock.patch('framework.test.opengl.subprocess.check_output',
+                        mock.Mock(return_value=rv)):
+            nt.eq_(expected, self._test.gl_extensions)
+
+
+class TestWflInfoSError(object):
+    """Tests for the Wflinfo functions to handle OSErrors."""
+    __patchers = []
+
+    @classmethod
+    def setup_class(cls):
+        """Setup the class, patching as necissary."""
+        cls.__patchers.append(mock.patch.dict(
+            'framework.test.opengl.OPTIONS.env',
+            {'PIGLIT_PLATFORM': 'foo'}))
+        cls.__patchers.append(mock.patch(
+            'framework.test.opengl.subprocess.check_output',
+            mock.Mock(side_effect=OSError(2, 'foo'))))
+        cls.__patchers.append(mock.patch(
+            'framework.test.opengl.WflInfo._WflInfo__shared_state', {}))
+
+        for f in cls.__patchers:
+            f.start()
+
+    def setup(self):
+        self.inst = opengl.WflInfo()
+
+    @classmethod
+    def teardown_class(cls):
+        for f in cls.__patchers:
+            f.stop()
+
+    @utils.not_raises(OSError)
+    def test_gl_extensions(self):
+        """test.opengl.WflInfo.gl_extensions: Handles OSError "no file" gracefully"""
+        self.inst.gl_extensions
+
+
+class TestWflInfoCalledProcessError(object):
+    """Tests for the WflInfo functions to handle OSErrors."""
+    __patchers = []
+
+    @classmethod
+    def setup_class(cls):
+        """Setup the class, patching as necissary."""
+        cls.__patchers.append(mock.patch.dict(
+            'framework.test.opengl.OPTIONS.env',
+            {'PIGLIT_PLATFORM': 'foo'}))
+        cls.__patchers.append(mock.patch(
+            'framework.test.opengl.subprocess.check_output',
+            mock.Mock(side_effect=subprocess.CalledProcessError(1, 'foo'))))
+        cls.__patchers.append(mock.patch(
+            'framework.test.opengl.WflInfo._WflInfo__shared_state', {}))
+
+        for f in cls.__patchers:
+            f.start()
+
+    @classmethod
+    def teardown_class(cls):
+        for f in cls.__patchers:
+            f.stop()
+
+    def setup(self):
+        self.inst = opengl.WflInfo()
+
+    @utils.not_raises(subprocess.CalledProcessError)
+    def test_gl_extensions(self):
+        """test.opengl.WflInfo.gl_extensions: Handles CalledProcessError gracefully"""
+        self.inst.gl_extensions
+
+
+class TestFastSkipMixin(object):
+    """Tests for the FastSkipMixin class."""
+    __patchers = []
+
+    @classmethod
+    def setup_class(cls):
+        """Create a Class with FastSkipMixin, but patch various bits."""
+        class _Test(opengl.FastSkipMixin, utils.Test):
+            pass
+
+        cls._class = _Test
+
+        _mock_wflinfo = mock.Mock(spec=opengl.WflInfo)
+        _mock_wflinfo.gl_version = 3.3
+        _mock_wflinfo.gles_version = 3.0
+        _mock_wflinfo.glsl_version = 3.3
+        _mock_wflinfo.glsl_es_version = 2.0
+        _mock_wflinfo.gl_extensions = set(['bar'])
+
+        cls.__patchers.append(mock.patch.object(
+            _Test, '_FastSkipMixin__info', _mock_wflinfo))
+
+        for patcher in cls.__patchers:
+            patcher.start()
+
+    @classmethod
+    def teardown_class(cls):
+        for patcher in cls.__patchers:
+            patcher.stop()
+
+    def setup(self):
+        self.test = self._class(['foo'])
+
+    @nt.raises(TestIsSkip)
+    def test_should_skip(self):
+        """test.opengl.FastSkipMixin.is_skip: Skips when requires is missing from extensions"""
+        self.test.gl_required.add('foobar')
+        self.test.is_skip()
+
+    @utils.not_raises(TestIsSkip)
+    def test_should_not_skip(self):
+        """test.opengl.FastSkipMixin.is_skip: runs when requires is in extensions"""
+        self.test.gl_required.add('bar')
+        self.test.is_skip()
+
+    @utils.not_raises(TestIsSkip)
+    def test_extension_empty(self):
+        """test.opengl.FastSkipMixin.is_skip: if extensions are empty test runs"""
+        self.test.gl_required.add('foobar')
+        with mock.patch.object(self.test._FastSkipMixin__info, 'gl_extensions',  # pylint: disable=no-member
+                               None):
+            self.test.is_skip()
+
+    @utils.not_raises(TestIsSkip)
+    def test_requires_empty(self):
+        """test.opengl.FastSkipMixin.is_skip: if gl_requires is empty test runs"""
+        self.test.is_skip()