tests: use meta profiles

This is a rather sizeable commit, I'm sorry for that. Basically there
was no clean way I could come up with to make the change from a single
python file to multiple xml files.

The reason we want this is to to avoid duplication amongst profiles (and
very long build times), by allowing a single profile to be included in
multiple other profiles. This means that the all profile is now composed
of the "glslparser", "shader", and "opengl" (everything in all that's
not glslparser, asmparser, or shader tests). Running `piglit run all
foo` still produces the same results. This also allows us to mix native
and non-native tests so that traditional targets like quick_cl still
work.

Tested-by: Rafael Antognolli <rafael.antognolli@intel.com>

17 files changed, 196 insertions, 171 deletions

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 4259ec832..dc80ece9a 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -533,7 +533,7 @@ install (
 install (
 	DIRECTORY tests
 	DESTINATION ${PIGLIT_INSTALL_LIBDIR}
-	FILES_MATCHING REGEX ".*\\.(py|program_test|shader_test|frag|vert|geom|tesc|tese|comp|ktx|cl|txt|inc)$"
+	FILES_MATCHING REGEX ".*\\.(xml|py|program_test|shader_test|frag|vert|geom|tesc|tese|comp|ktx|cl|txt|inc)$"
 	REGEX "CMakeFiles|CMakeLists" EXCLUDE
 )
 
diff --git a/tests/all.meta.xml b/tests/all.meta.xml
new file mode 100644
index 000000000..c0e8311c2
--- /dev/null
+++ b/tests/all.meta.xml
@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="utf-8"?>
+<PiglitMetaProfile>
+  <Profile>opengl</Profile>
+  <Profile>shader</Profile>
+  <Profile>glslparser</Profile>
+</PiglitMetaProfile>
diff --git a/tests/cpu.meta.xml b/tests/cpu.meta.xml
new file mode 100644
index 000000000..7ee5e8ebd
--- /dev/null
+++ b/tests/cpu.meta.xml
@@ -0,0 +1,4 @@
+<?xml version="1.0" encoding="utf-8"?>
+<PiglitMetaProfile>
+  <Profile>glslparser</Profile>
+</PiglitMetaProfile>
diff --git a/tests/cpu.py b/tests/cpu.py
deleted file mode 100644
index 65d999062..000000000
--- a/tests/cpu.py
+++ /dev/null
@@ -1,31 +0,0 @@
-"""Profile that removes all GPU based tests.
-
-This profile is the inverse of gpu.py.
-
-It runs GLSLParserTests, asmparsertests, and ARB_vertex_program and
-ARB_fragment_program tests only.
-
-Using driver specific overrides these can be forced to run on arbitrary
-hardware.
-
-"""
-
-from __future__ import (
-    absolute_import, division, print_function, unicode_literals
-)
-from tests.quick import profile as _profile
-from framework.test import GLSLParserTest
-
-__all__ = ['profile']
-
-profile = _profile.copy()  # pylint: disable=invalid-name
-
-
-def filter_gpu(name, test):
-    """Remove all tests that are run on the GPU."""
-    if isinstance(test, GLSLParserTest) or name.startswith('asmparsertest'):
-        return True
-    return False
-
-
-profile.filters.append(filter_gpu)
diff --git a/tests/glslparser.py b/tests/glslparser.py
index 5d0facbc2..9d50dabdf 100644
--- a/tests/glslparser.py
+++ b/tests/glslparser.py
@@ -3,12 +3,58 @@
 from __future__ import (
     absolute_import, division, print_function, unicode_literals
 )
+import os
 
-from framework.test import GLSLParserTest
-from tests.all import profile as _profile
+from framework import grouptools
+from framework.profile import TestProfile
+from framework.test.glsl_parser_test import GLSLParserTest, GLSLParserNoConfigError
+from framework.test.piglit_test import ASMParserTest, ROOT_DIR
+from .py_modules.constants import GENERATED_TESTS_DIR, TESTS_DIR
 
 __all__ = ['profile']
 
-profile = _profile.copy()  # pylint: disable=invalid-name
+profile = TestProfile()
 
-profile.filters.append(lambda _, t: isinstance(t, GLSLParserTest))
+# Find and add all shader tests.
+basepath = os.path.normpath(os.path.join(TESTS_DIR, '..'))
+for basedir in [TESTS_DIR, GENERATED_TESTS_DIR]:
+    for dirpath, _, filenames in os.walk(basedir):
+        groupname = grouptools.from_path(os.path.relpath(dirpath, basedir))
+        for filename in filenames:
+            testname, ext = os.path.splitext(filename)
+            if ext in ['.vert', '.tesc', '.tese', '.geom', '.frag', '.comp']:
+                try:
+                    test = GLSLParserTest.new(
+                        os.path.join(os.path.relpath(dirpath, basepath), filename))
+                except GLSLParserNoConfigError:
+                    # In the event that there is no config assume that it is a
+                    # legacy test, and continue
+                    continue
+
+                # For glslparser tests you can have multiple tests with the
+                # same name, but a different stage, so keep the extension.
+                testname = filename
+            else:
+                continue
+
+            group = grouptools.join(groupname, testname)
+            assert group not in profile.test_list, group
+
+            profile.test_list[group] = test
+
+# Collect and add all asmparsertests
+for basedir in [TESTS_DIR, GENERATED_TESTS_DIR]:
+    _basedir = os.path.join(basedir, 'asmparsertest', 'shaders')
+    for dirpath, _, filenames in os.walk(_basedir):
+        base_group = grouptools.from_path(os.path.join(
+            'asmparsertest', os.path.relpath(dirpath, _basedir)))
+        type_ = os.path.basename(dirpath)
+
+        dirname = os.path.relpath(dirpath, ROOT_DIR)
+        for filename in filenames:
+            if not os.path.splitext(filename)[1] == '.txt':
+                continue
+
+            group = grouptools.join(base_group, filename)
+            profile.test_list[group] = ASMParserTest(
+                type_, os.path.join(dirname, filename))
diff --git a/tests/gpu.meta.xml b/tests/gpu.meta.xml
new file mode 100644
index 000000000..709c89b51
--- /dev/null
+++ b/tests/gpu.meta.xml
@@ -0,0 +1,5 @@
+<?xml version="1.0" encoding="utf-8"?>
+<PiglitMetaProfile>
+  <Profile>quick_gl</Profile>
+  <Profile>quick_shader</Profile>
+</PiglitMetaProfile>
diff --git a/tests/gpu.py b/tests/gpu.py
deleted file mode 100644
index fce550c67..000000000
--- a/tests/gpu.py
+++ /dev/null
@@ -1,18 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# quick.tests minus compiler tests.
-
-from __future__ import (
-    absolute_import, division, print_function, unicode_literals
-)
-
-from tests.quick import profile as _profile
-from framework.test import GLSLParserTest
-
-__all__ = ['profile']
-
-profile = _profile.copy()  # pylint: disable=invalid-name
-
-# Remove all parser tests, as they are compiler test
-profile.filters.append(lambda p, t: not isinstance(t, GLSLParserTest))
-profile.filters.append(lambda n, _: not n.startswith('asmparsertest'))
diff --git a/tests/llvmpipe.meta.xml b/tests/llvmpipe.meta.xml
new file mode 100644
index 000000000..86348788b
--- /dev/null
+++ b/tests/llvmpipe.meta.xml
@@ -0,0 +1,5 @@
+<?xml version="1.0" encoding="utf-8"?>
+<PiglitMetaProfile>
+  <Profile>llvmpipe_gl</Profile>
+  <Profile>quick_shader</Profile>
+</PiglitMetaProfile>
diff --git a/tests/llvmpipe.py b/tests/llvmpipe_gl.py
index 7ccdff3c4..71b41433c 100644
--- a/tests/llvmpipe.py
+++ b/tests/llvmpipe_gl.py
@@ -8,7 +8,7 @@ import platform
 import sys
 
 from framework.grouptools import join
-from tests.gpu import profile as _profile
+from tests.quick_gl import profile as _profile
 
 __all__ = ['profile']
 
diff --git a/tests/no_error.py b/tests/no_error.py
index 3e063130c..9b53c6cb4 100644
--- a/tests/no_error.py
+++ b/tests/no_error.py
@@ -3,26 +3,27 @@
 from __future__ import (
      absolute_import, division, print_function, unicode_literals
 )
+import itertools
 
 import six
 
-from tests.gpu import profile as _profile
+from tests.quick_gl import profile as _profile1
+from tests.quick_shader import profile as _profile2
 from framework.test import PiglitGLTest
 from framework.test.shader_test import ShaderTest, MultiShaderTest
-from framework.profile import TestDict
+from framework.profile import TestProfile
 
 __all__ = ['profile']
 
-profile = _profile.copy()  # pylint: disable=invalid-name
-
-# Save the old test_list, but create a new one to contain the modified tests
-old_test_list = profile.test_list
-profile.test_list = TestDict()
+# Save the filters from the original profiles to a new profile
+profile = TestProfile()
+profile.filters = _profile1.filters + _profile2.filters
 
 # Add a modified version of each PiglitGLTest as a khr_no_error variant.
 # Shader runner doesn't explitly test for expected errors so we add shader
 # tests as is. We actively filter GLSL parser and any other type of tests.
-for name, test in six.iteritems(old_test_list):
+for name, test in itertools.chain(six.iteritems(_profile1.test_list),
+                                  six.iteritems(_profile2.test_list)):
     if isinstance(test, (PiglitGLTest, ShaderTest, MultiShaderTest)):
         profile.test_list['{} khr_no_error'.format(name)] = test
         test.command += ['-khr_no_error']
diff --git a/tests/quick_cl.py b/tests/opencl_foreign.py
index 831e8fda0..76efaf6d4 100644
--- a/tests/quick_cl.py
+++ b/tests/opencl_foreign.py
@@ -1,6 +1,6 @@
-#!/usr/bin/env python
-#
+# encoding=utf-8
 # Copyright 2014 Advanced Micro Devices, Inc.
+# Copyright © 2018 Intel Corporation
 #
 # Permission is hereby granted, free of charge, to any person obtaining a
 # copy of this software and associated documentation files (the "Software"),
@@ -20,18 +20,15 @@
 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 # SOFTWARE.
-#
-# Authors: Tom Stellard <thomas.stellard@amd.com>
-#
 
 from __future__ import (
     absolute_import, division, print_function, unicode_literals
 )
 
-from tests.cl import profile as _profile
+from framework.profile import TestProfile
 from framework.test import add_opencv_tests, add_oclconform_tests
 
-profile = _profile.copy()  # pylint: disable=invalid-name
+profile = TestProfile()
 
 add_opencv_tests(profile)
 add_oclconform_tests(profile)
diff --git a/tests/all.py b/tests/opengl.py
index 018c021d7..8b1776569 100644
--- a/tests/all.py
+++ b/tests/opengl.py
@@ -4,12 +4,10 @@
 from __future__ import (
     absolute_import, division, print_function, unicode_literals
 )
-import collections
 import itertools
 import os
 import platform
 
-import six
 from six.moves import range
 
 from framework import grouptools
@@ -18,18 +16,13 @@ from framework import options
 from framework import wflinfo
 from framework.profile import TestProfile
 from framework.driver_classifier import DriverClassifier
-from framework.test import (PiglitGLTest, PiglitBaseTest,
-                            GLSLParserTest, GLSLParserNoConfigError)
-from framework.test.shader_test import ShaderTest, MultiShaderTest
 from framework.test.piglit_test import (
-    ASMParserTest, BuiltInConstantsTest, ROOT_DIR
+    PiglitGLTest, PiglitBaseTest, BuiltInConstantsTest
 )
 from .py_modules.constants import TESTS_DIR, GENERATED_TESTS_DIR
 
 __all__ = ['profile']
 
-PROCESS_ISOLATION = options.OPTIONS.process_isolation
-
 # Disable bad hanging indent errors in pylint
 # There is a bug in pylint which causes the profile.test_list.group_manager to
 # be tagged as bad hanging indent, even though it seems to be correct (and
@@ -213,8 +206,6 @@ def power_set(s):
 # Collecting all tests
 profile = TestProfile()  # pylint: disable=invalid-name
 
-shader_tests = collections.defaultdict(list)
-
 wfl_info = wflinfo.WflInfo()
 
 
@@ -295,73 +286,6 @@ def walk_filter_dir_tree(root):
     return retval
 
 
-# Find and add all shader tests.
-for basedir in [TESTS_DIR, GENERATED_TESTS_DIR]:
-    if os.environ.get("PIGLIT_FILTER_DIRECTORIES"):
-        files = walk_filter_dir_tree(basedir)
-    else:
-        files = os.walk(basedir)
-    for dirpath, _, filenames in files:
-        groupname = grouptools.from_path(os.path.relpath(dirpath, basedir))
-        for filename in filenames:
-            testname, ext = os.path.splitext(filename)
-            dirname = os.path.relpath(dirpath, ROOT_DIR)
-            if ext == '.shader_test':
-                if PROCESS_ISOLATION:
-                    test = ShaderTest.new(os.path.join(dirname, filename))
-                else:
-                    shader_tests[groupname].append(os.path.join(dirpath, filename))
-                    continue
-            elif ext in ['.vert', '.tesc', '.tese', '.geom', '.frag', '.comp']:
-                try:
-                    test = GLSLParserTest.new(os.path.join(dirname, filename))
-                except GLSLParserNoConfigError:
-                    # In the event that there is no config assume that it is a
-                    # legacy test, and continue
-                    continue
-
-                # For glslparser tests you can have multiple tests with the
-                # same name, but a different stage, so keep the extension.
-                testname = filename
-            else:
-                continue
-
-            group = grouptools.join(groupname, testname)
-            assert group not in profile.test_list, group
-
-            profile.test_list[group] = test
-
-# Because we need to handle duplicate group names in TESTS and GENERATED_TESTS
-# this dictionary is constructed, then added to the actual test dictionary.
-for group, files in six.iteritems(shader_tests):
-    assert group not in profile.test_list, 'duplicate group: {}'.format(group)
-    # If there is only one file in the directory use a normal shader_test.
-    # Otherwise use a MultiShaderTest
-    if len(files) == 1:
-        group = grouptools.join(
-            group, os.path.basename(os.path.splitext(files[0])[0]))
-        profile.test_list[group] = ShaderTest.new(files[0])
-    else:
-        profile.test_list[group] = MultiShaderTest.new(files)
-
-
-# Collect and add all asmparsertests
-for basedir in [TESTS_DIR, GENERATED_TESTS_DIR]:
-    _basedir = os.path.join(basedir, 'asmparsertest', 'shaders')
-    for dirpath, _, filenames in os.walk(_basedir):
-        base_group = grouptools.from_path(os.path.join(
-            'asmparsertest', os.path.relpath(dirpath, _basedir)))
-        type_ = os.path.basename(dirpath)
-
-        dirname = os.path.relpath(dirpath, ROOT_DIR)
-        for filename in filenames:
-            if not os.path.splitext(filename)[1] == '.txt':
-                continue
-
-            group = grouptools.join(base_group, filename)
-            profile.test_list[group] = ASMParserTest(
-                type_, os.path.join(dirname, filename))
-
 # Find and add all apitrace tests.
 classifier = DriverClassifier()
 for basedir in [os.path.join(TESTS_DIR, 'apitrace', 'traces')]:
diff --git a/tests/quick.meta.xml b/tests/quick.meta.xml
new file mode 100644
index 000000000..fed58159b
--- /dev/null
+++ b/tests/quick.meta.xml
@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="utf-8"?>
+<PiglitMetaProfile>
+  <Profile>quick_gl</Profile>
+  <Profile>quick_shader</Profile>
+  <Profile>glslparser</Profile>
+</PiglitMetaProfile>
diff --git a/tests/quick_cl.meta.xml b/tests/quick_cl.meta.xml
new file mode 100644
index 000000000..830f5c09a
--- /dev/null
+++ b/tests/quick_cl.meta.xml
@@ -0,0 +1,5 @@
+<?xml version="1.0" encoding="utf-8"?>
+<PiglitMetaProfile>
+  <Profile>opencl_foreign</Profile>
+  <Profile>cl</Profile>
+</PiglitMetaProfile>
diff --git a/tests/quick.py b/tests/quick_gl.py
index 2d05d7a44..c6501904c 100644
--- a/tests/quick.py
+++ b/tests/quick_gl.py
@@ -13,12 +13,10 @@ don't want that level of exhaustiveness, so this filter removes 80% in a random
 from __future__ import (
     absolute_import, division, print_function, unicode_literals
 )
-import random
 
 from framework import grouptools
 from framework.test import PiglitGLTest
-from framework.test.shader_test import ShaderTest
-from tests.all import profile as _profile
+from tests.opengl import profile as _profile
 
 __all__ = ['profile']
 
@@ -26,20 +24,6 @@ __all__ = ['profile']
 # pylint: disable=bad-continuation
 
 
-class FilterVsIn(object):
-    """Filter out 80% of the Vertex Attrib 64 vs_in tests."""
-
-    def __init__(self):
-        self.random = random.Random()
-        self.random.seed(42)
-
-    def __call__(self, name, test):
-        if isinstance(test, ShaderTest) and 'vs_in' in grouptools.split(name):
-            # 20%
-            return self.random.random() <= .2
-        return True
-
-
 profile = _profile.copy()  # pylint: disable=invalid-name
 
 # Set the --quick flag on a few image_load_store_tests
@@ -92,7 +76,3 @@ with profile.test_list.group_manager(
         g(['arb_texture_multisample-large-float-texture', '--array',
            '--fp16'] + size_arg,
           'large-float-texture-array-fp16', run_concurrent=False)
-
-# These take too long
-profile.filters.append(lambda n, _: '-explosion' not in n)
-profile.filters.append(FilterVsIn())
diff --git a/tests/quick_shader.py b/tests/quick_shader.py
new file mode 100644
index 000000000..c1b1d7938
--- /dev/null
+++ b/tests/quick_shader.py
@@ -0,0 +1,53 @@
+# encoding=utf-8
+# Copyright © 2018 Intel Coproration
+
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+from __future__ import (
+    absolute_import, division, print_function, unicode_literals
+)
+import random
+
+from framework import grouptools
+from .shader import profile as _profile
+
+__all__ = ['profile']
+
+profile = _profile.copy()
+
+
+# There are too many of theset, it's simply exhaustive to run them all.
+class FilterVsIn(object):
+    """Filter out 80% of the Vertex Attrib 64 vs_in tests."""
+
+    def __init__(self):
+        self.random = random.Random()
+        self.random.seed(42)
+
+    def __call__(self, name, _):
+        if 'vs_in' in grouptools.split(name):
+            # 20%
+            return self.random.random() <= .2
+        return True
+
+
+profile.filters.append(FilterVsIn())
+
+# These take too long
+profile.filters.append(lambda n, _: '-explosion' not in n)
diff --git a/tests/shader.py b/tests/shader.py
index d283a577c..0396c4c8d 100644
--- a/tests/shader.py
+++ b/tests/shader.py
@@ -3,12 +3,54 @@
 from __future__ import (
     absolute_import, division, print_function, unicode_literals
 )
+import collections
+import os
 
+import six
+
+from framework.options import OPTIONS
+from framework import grouptools
+from framework.profile import TestProfile
 from framework.test.shader_test import ShaderTest, MultiShaderTest
-from tests.all import profile as _profile
+from .py_modules.constants import GENERATED_TESTS_DIR, TESTS_DIR
 
 __all__ = ['profile']
 
-profile = _profile.copy()  # pylint: disable=invalid-name
+profile = TestProfile()
+
+shader_tests = collections.defaultdict(list)
+
+# Find and add all shader tests.
+basepath = os.path.normpath(os.path.join(TESTS_DIR, '..'))
+for basedir in [TESTS_DIR, GENERATED_TESTS_DIR]:
+    for dirpath, _, filenames in os.walk(basedir):
+        groupname = grouptools.from_path(os.path.relpath(dirpath, basedir))
+        for filename in filenames:
+            testname, ext = os.path.splitext(filename)
+            if ext == '.shader_test':
+                dirname = os.path.relpath(dirpath, basepath)
+                if OPTIONS.process_isolation:
+                    test = ShaderTest.new(os.path.join(dirname, filename))
+                else:
+                    shader_tests[groupname].append(os.path.join(dirname, filename))
+                    continue
+            else:
+                continue
+
+            group = grouptools.join(groupname, testname)
+            assert group not in profile.test_list, group
+
+            profile.test_list[group] = test
 
-profile.filters.append(lambda _, t: isinstance(t, (ShaderTest, MultiShaderTest)))
+# Because we need to handle duplicate group names in TESTS and GENERATED_TESTS
+# this dictionary is constructed, then added to the actual test dictionary.
+for group, files in six.iteritems(shader_tests):
+    assert group not in profile.test_list, 'duplicate group: {}'.format(group)
+    # If there is only one file in the directory use a normal shader_test.
+    # Otherwise use a MultiShaderTest
+    if len(files) == 1:
+        group = grouptools.join(
+            group, os.path.basename(os.path.splitext(files[0])[0]))
+        profile.test_list[group] = ShaderTest.new(files[0])
+    else:
+        profile.test_list[group] = MultiShaderTest.new(files)