framework: Add mustpasslist support to deqp-gles{2,31}

This also changes the environment variable of deqp-gles3, but adds a
deprecation warning if using the old environment variable and honors it.
At some point it should be removed. There is also a small change in the
default value of the mustpass variable in the conf file to reflect the
modern location of the mustpass lists.

Signed-off-by: Dylan Baker <dylanx.c.baker@intel.com>

4 files changed, 38 insertions, 13 deletions

diff --git a/piglit.conf.example b/piglit.conf.example
index d8dd3fc67..671526869 100644
--- a/piglit.conf.example
+++ b/piglit.conf.example
@@ -57,7 +57,12 @@ testB
 ; overrides the value set here.
 ;extra_args=--deqp-visibility hidden
 
-;[deqp-gles3]
+; Path to the test case list of CTS for deqp-gles2. You can also set this with
+; the environment variable PIGLIT_DEQP2_MUSTPASS. Piglit will run the subset of
+; dEQP-GLES2 tests if this option is set.
+;mustpasslist=/home/knuth/src/deqp/android/cts/com.drawelements.deqp.gles2.xml
+
+[deqp-gles3]
 ;
 ; Path to the deqp-gles3 executable. You can also set this with the environment
 ; variable PIGLIT_DEQP_GLES3_EXE. Piglit will run the dEQP-GLES3 tests if and
@@ -70,10 +75,9 @@ testB
 ;extra_args=--deqp-visibility hidden
 ;
 ; Path to the test case list of CTS for deqp-gles3. You can also set this with
-; the environment variable PIGLIT_DEQP_MUSTPASS. Piglit will run the subset of
-; dEQP-GLES3 tests if and only if this option is set.
-;mustpasslist= \
-; /android/platform/external/deqp/android/cts/com.drawelements.deqp.gles3.xml
+; the environment variable PIGLIT_DEQP3_MUSTPASS. Piglit will run the subset of
+; dEQP-GLES3 tests if this option is set.
+;mustpasslist=/home/knuth/src/deqp/android/cts/com.drawelements.deqp.gles3.xml
 
 [deqp-gles31]
 ; Path to the deqp-gles31 executable
@@ -85,6 +89,11 @@ testB
 ; overrides the value set here.
 ;extra_args=--deqp-visibility hidden
 
+; Path to the test case list of CTS for deqp-gles31. You can also set this with
+; the environment variable PIGLIT_DEQP31_MUSTPASS. Piglit will run the subset of
+; dEQP-GLES31 tests if this option is set.
+;mustpasslist=/home/knuth/src/deqp/android/cts/com.drawelements.deqp.gles31.xml
+
 [deqp-vk]
 ; Path to the deqp-ck executable
 ; Can be overwritten by PIGLIT_DEQP_VK_BIN environment variable
@@ -201,3 +210,5 @@ run_test=./%(test_name)s
 ;type=dmesg
 ;parameters=--level emerg,alert,crit,err,warn,notice
 ;regex=\*ERROR\* ring create req|\*ERROR\* Failed to reset chip|BUG:|Oops:|turning off the locking correctness validator
+
+; vim: ft=dosini
diff --git a/tests/deqp_gles2.py b/tests/deqp_gles2.py
index 4a83e16e2..29d995520 100644
--- a/tests/deqp_gles2.py
+++ b/tests/deqp_gles2.py
@@ -33,6 +33,9 @@ _DEQP_GLES2_BIN = deqp.get_option('PIGLIT_DEQP_GLES2_BIN',
                                   ('deqp-gles2', 'bin'),
                                   required=True)
 
+_DEQP_MUSTPASS = deqp.get_option('PIGLIT_DEQP2_MUSTPASS',
+                                 ('deqp-gles2', 'mustpasslist'))
+
 _EXTRA_ARGS = deqp.get_option('PIGLIT_DEQP_GLES2_EXTRA_ARGS',
                               ('deqp-gles2', 'extra_args'),
                               default='').split()
@@ -49,7 +52,6 @@ class DEQPGLES2Test(deqp.DEQPBaseTest):
 
 
 profile = deqp.make_profile(  # pylint: disable=invalid-name
-    deqp.iter_deqp_test_cases(
-        deqp.gen_caselist_txt(_DEQP_GLES2_BIN, 'dEQP-GLES2-cases.txt',
-                              _EXTRA_ARGS)),
+    deqp.select_source(_DEQP_GLES2_BIN, 'dEQP-GLES2-cases.txt', _DEQP_MUSTPASS,
+                       _EXTRA_ARGS),
     DEQPGLES2Test)
diff --git a/tests/deqp_gles3.py b/tests/deqp_gles3.py
index bfee3442b..970feb103 100644
--- a/tests/deqp_gles3.py
+++ b/tests/deqp_gles3.py
@@ -23,6 +23,8 @@
 from __future__ import (
     absolute_import, division, print_function, unicode_literals
 )
+import os
+import warnings
 
 from framework.test import deqp
 
@@ -33,8 +35,16 @@ _DEQP_GLES3_EXE = deqp.get_option('PIGLIT_DEQP_GLES3_EXE',
                                   ('deqp-gles3', 'exe'),
                                   required=True)
 
-_DEQP_MUSTPASS = deqp.get_option('PIGLIT_DEQP_MUSTPASS',
+_DEQP_MUSTPASS = deqp.get_option('PIGLIT_DEQP3_MUSTPASS',
                                  ('deqp-gles3', 'mustpasslist'))
+if os.environ.get('PIGLIT_DEQP_MUSTPASS') is not None:
+    # see if the old environment variable was set, if it is uses it, and give a
+    # deprecation warning
+    _DEQP_MUSTPASS = os.environ['PIGLIT_DEQP_MUSTPASS']
+    warnings.warn(
+        'PIGLIT_DEQP_MUSTPASS has been replaced by PIGLIT_DEQP3_MUSTPASS '
+        'and will be removed. You should update and scripts using the old '
+        'environment variable')
 
 _EXTRA_ARGS = deqp.get_option('PIGLIT_DEQP_GLES3_EXTRA_ARGS',
                               ('deqp-gles3', 'extra_args'),
@@ -55,6 +65,6 @@ class DEQPGLES3Test(deqp.DEQPBaseTest):
 
 
 profile = deqp.make_profile(  # pylint: disable=invalid-name
-    deqp.select_source(_DEQP_MUSTPASS, _DEQP_GLES3_EXE, 'dEQP-GLES3-cases.txt',
+    deqp.select_source(_DEQP_GLES3_EXE, 'dEQP-GLES3-cases.txt', _DEQP_MUSTPASS,
                        _EXTRA_ARGS),
     DEQPGLES3Test)
diff --git a/tests/deqp_gles31.py b/tests/deqp_gles31.py
index 51188c76b..f74bdfeb0 100644
--- a/tests/deqp_gles31.py
+++ b/tests/deqp_gles31.py
@@ -33,6 +33,9 @@ _DEQP_GLES31_BIN = deqp.get_option('PIGLIT_DEQP_GLES31_BIN',
                                    ('deqp-gles31', 'bin'),
                                    required=True)
 
+_DEQP_MUSTPASS = deqp.get_option('PIGLIT_DEQP31_MUSTPASS',
+                                 ('deqp-gles31', 'mustpasslist'))
+
 _EXTRA_ARGS = deqp.get_option('PIGLIT_DEQP_GLES31_EXTRA_ARGS',
                               ('deqp-gles31', 'extra_args'),
                               default='').split()
@@ -48,7 +51,6 @@ class DEQPGLES31Test(deqp.DEQPBaseTest):
 
 
 profile = deqp.make_profile(  # pylint: disable=invalid-name
-    deqp.iter_deqp_test_cases(
-        deqp.gen_caselist_txt(_DEQP_GLES31_BIN, 'dEQP-GLES31-cases.txt',
-                              _EXTRA_ARGS)),
+    deqp.select_source(_DEQP_GLES31_BIN, 'dEQP-GLES31-cases.txt',
+                       _DEQP_MUSTPASS, _EXTRA_ARGS),
     DEQPGLES31Test)