framework: add commandline option for deqp-mustpass

Rather than relying on just setting the option in the configuration file
turning on the functionality add a switch.

Signed-off-by: Dylan Baker <dylanx.c.baker@intel.com>

6 files changed, 25 insertions, 4 deletions

diff --git a/framework/options.py b/framework/options.py
index 28dcf9301..94a80840b 100644
--- a/framework/options.py
+++ b/framework/options.py
@@ -180,8 +180,9 @@ class _Options(object):  # pylint: disable=too-many-instance-attributes
     dmesg -- True if dmesg checking is desired. This forces concurrency off
     monitored -- True if monitoring is desired. This forces concurrency off
     env -- environment variables set for each test before run
-
+    deqp_mustpass -- True to enable the use of the deqp mustpass list feature.
     """
+
     include_filter = _ReListDescriptor('_include_filter', type_=_FilterReList)
     exclude_filter = _ReListDescriptor('_exclude_filter', type_=_FilterReList)
 
@@ -195,6 +196,7 @@ class _Options(object):  # pylint: disable=too-many-instance-attributes
         self.dmesg = False
         self.monitored = False
         self.sync = False
+        self.deqp_mustpass = False
 
         # env is used to set some base environment variables that are not going
         # to change across runs, without sending them to os.environ which is
diff --git a/framework/programs/run.py b/framework/programs/run.py
index 686f37ec4..7c8660cd7 100644
--- a/framework/programs/run.py
+++ b/framework/programs/run.py
@@ -172,6 +172,12 @@ def _run_parser(input_):
                         dest='overwrite',
                         action='store_true',
                         help='If the results_path already exists, delete it')
+    parser.add_argument('--deqp-mustpass-list',
+                        dest='deqp_mustpass',
+                        action='store_true',
+                        help='Run only the tests in the deqp mustpass list '
+                             'when running a deqp gles{2,3,31} profile, '
+                             'otherwise run all tests.')
     parser.add_argument("test_profile",
                         metavar="<Profile path(s)>",
                         nargs='+',
@@ -252,6 +258,7 @@ def run(input_):
     options.OPTIONS.dmesg = args.dmesg
     options.OPTIONS.monitored = args.monitored
     options.OPTIONS.sync = args.sync
+    options.OPTIONS.deqp_mustpass = args.deqp_mustpass
 
     # Set the platform to pass to waffle
     options.OPTIONS.env['PIGLIT_PLATFORM'] = args.platform
@@ -338,6 +345,7 @@ def resume(input_):
     options.OPTIONS.dmesg = results.options['dmesg']
     options.OPTIONS.monitored = results.options['monitored']
     options.OPTIONS.sync = results.options['sync']
+    options.OPTIONS.deqp_mustpass = results.options['deqp_mustpass']
 
     core.get_config(args.config_file)
 
diff --git a/framework/test/deqp.py b/framework/test/deqp.py
index cd094c98a..c3452b4d9 100644
--- a/framework/test/deqp.py
+++ b/framework/test/deqp.py
@@ -33,6 +33,7 @@ import six
 from six.moves import range
 
 from framework import core, grouptools, exceptions
+from framework import options
 from framework.profile import TestProfile
 from framework.test.base import Test, is_crash_returncode, TestRunError
 
@@ -71,7 +72,7 @@ _EXTRA_ARGS = get_option('PIGLIT_DEQP_EXTRA_ARGS',
 
 def select_source(bin_, filename, mustpass, extra_args):
     """Return either the mustpass list or the generated list."""
-    if mustpass is not None:
+    if options.OPTIONS.deqp_mustpass:
         return gen_mustpass_tests(mustpass)
     else:
         return iter_deqp_test_cases(
diff --git a/tests/deqp_gles2.py b/tests/deqp_gles2.py
index 29d995520..518a4e04c 100644
--- a/tests/deqp_gles2.py
+++ b/tests/deqp_gles2.py
@@ -25,6 +25,7 @@ from __future__ import (
 )
 
 from framework.test import deqp
+from framework.options import OPTIONS
 
 __all__ = ['profile']
 
@@ -34,7 +35,8 @@ _DEQP_GLES2_BIN = deqp.get_option('PIGLIT_DEQP_GLES2_BIN',
                                   required=True)
 
 _DEQP_MUSTPASS = deqp.get_option('PIGLIT_DEQP2_MUSTPASS',
-                                 ('deqp-gles2', 'mustpasslist'))
+                                 ('deqp-gles2', 'mustpasslist'),
+                                 required=OPTIONS.deqp_mustpass)
 
 _EXTRA_ARGS = deqp.get_option('PIGLIT_DEQP_GLES2_EXTRA_ARGS',
                               ('deqp-gles2', 'extra_args'),
diff --git a/tests/deqp_gles3.py b/tests/deqp_gles3.py
index 970feb103..3843deb4d 100644
--- a/tests/deqp_gles3.py
+++ b/tests/deqp_gles3.py
@@ -27,6 +27,8 @@ import os
 import warnings
 
 from framework.test import deqp
+from framework.options import OPTIONS
+from framework import exceptions
 
 __all__ = ['profile']
 
@@ -46,6 +48,10 @@ if os.environ.get('PIGLIT_DEQP_MUSTPASS') is not None:
         'and will be removed. You should update and scripts using the old '
         'environment variable')
 
+if OPTIONS.deqp_mustpass and not _DEQP_MUSTPASS:
+    raise exceptions.PiglitFatalError(
+        'Use of mustpasslist requested, but no mustpasslist provided.')
+
 _EXTRA_ARGS = deqp.get_option('PIGLIT_DEQP_GLES3_EXTRA_ARGS',
                               ('deqp-gles3', 'extra_args'),
                               default='').split()
diff --git a/tests/deqp_gles31.py b/tests/deqp_gles31.py
index f74bdfeb0..7021e8966 100644
--- a/tests/deqp_gles31.py
+++ b/tests/deqp_gles31.py
@@ -25,6 +25,7 @@ from __future__ import (
 )
 
 from framework.test import deqp
+from framework.options import OPTIONS
 
 __all__ = ['profile']
 
@@ -34,7 +35,8 @@ _DEQP_GLES31_BIN = deqp.get_option('PIGLIT_DEQP_GLES31_BIN',
                                    required=True)
 
 _DEQP_MUSTPASS = deqp.get_option('PIGLIT_DEQP31_MUSTPASS',
-                                 ('deqp-gles31', 'mustpasslist'))
+                                 ('deqp-gles31', 'mustpasslist'),
+                                 required=OPTIONS.deqp_mustpass)
 
 _EXTRA_ARGS = deqp.get_option('PIGLIT_DEQP_GLES31_EXTRA_ARGS',
                               ('deqp-gles31', 'extra_args'),