summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorKenneth Graunke <kenneth@whitecape.org>2012-11-25 03:50:13 -0800
committerKenneth Graunke <kenneth@whitecape.org>2012-11-25 03:50:13 -0800
commit595610b9f4b7b6914fd235322d2bc00d761d25c7 (patch)
treec10c14d5ea6a447451dcc38f87888d9429ff88b0
parent53a5b78d4ed126f1f45f021b55cd3abf3a28ac24 (diff)
new plugin architecture! now loading oglconform test list.
-rw-r--r--framework/config.py40
-rw-r--r--framework/suites.py134
-rw-r--r--framework/test.py54
-rw-r--r--programs/__init__.py0
-rwxr-xr-xprograms/run.py9
-rwxr-xr-xrobyn16
-rw-r--r--robynrc.example22
-rw-r--r--suites/__init__.py39
-rw-r--r--suites/oglconform.py70
9 files changed, 238 insertions, 146 deletions
diff --git a/framework/config.py b/framework/config.py
new file mode 100644
index 0000000..c56e308
--- /dev/null
+++ b/framework/config.py
@@ -0,0 +1,40 @@
+#
+# Copyright © 2012 Intel Corporation
+#
+# Permission is hereby granted, free of charge, to any person obtaining a
+# copy of this software and associated documentation files (the "Software"),
+# to deal in the Software without restriction, including without limitation
+# the rights to use, copy, modify, merge, publish, distribute, sublicense,
+# and/or sell copies of the Software, and to permit persons to whom the
+# Software is furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice (including the next
+# paragraph) shall be included in all copies or substantial portions of the
+# Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+# IN THE SOFTWARE.
+#
+import sys
+import os.path as path
+from configparser import ConfigParser
+import xdg.BaseDirectory as xdg
+
+__all__ = ['load']
+
+def load():
+ config = ConfigParser()
+
+ rcdir = xdg.load_first_config('robyn')
+ if rcdir:
+ rcfile = path.join(xdg.load_first_config('robyn'), 'robynrc')
+ config.read(rcfile)
+ else:
+ print('warning: could not read ~/.config/robyn/robynrc.')
+
+ return config
diff --git a/framework/suites.py b/framework/suites.py
deleted file mode 100644
index 5b892c5..0000000
--- a/framework/suites.py
+++ /dev/null
@@ -1,134 +0,0 @@
-#
-# Copyright © 2012 Intel Corporation
-#
-# Permission is hereby granted, free of charge, to any person obtaining a
-# copy of this software and associated documentation files (the "Software"),
-# to deal in the Software without restriction, including without limitation
-# the rights to use, copy, modify, merge, publish, distribute, sublicense,
-# and/or sell copies of the Software, and to permit persons to whom the
-# Software is furnished to do so, subject to the following conditions:
-#
-# The above copyright notice and this permission notice (including the next
-# paragraph) shall be included in all copies or substantial portions of the
-# Software.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
-# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
-# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
-# IN THE SOFTWARE.
-#
-import json
-import xml.etree.ElementTree as ET
-import os.path as path
-import re
-
-#import framework.test
-
-#__all__ = ['TestSuite']
-
-#############################################################################
-##### Test suite descriptions
-#############################################################################
-
-class TestSuite:
- """
- Description of a test suite.
-
- For now, this contains the following information:
- - Name
- - Description
- - Absolute path to the XML test list file
- - How to invoke tests
- - How to interpret tests results
- - How to pass specific flags such as platform, quick mode, and so on.
- - Dictionary of tests
- """
- def __init__(self, xmlFile):
- tree = ET.parse(xmlFile)
- suiteElem = tree.getroot()
-
- self.name = suiteElem.attrib['name']
- self.description = suiteElem.attrib['description']
- self.testlist_file = path.abspath(path.join(path.dirname(xmlFile.name), suiteElem.attrib['tests']))
-
- resultElem = suiteElem.find('result')
-
- self.status_default = resultElem.attrib['default']
- self.status_re = {}
- for rx in resultElem:
- self.status_re[rx.attrib['status']] = re.compile(rx.text)
-
- # XXX: parse <platforms> and <quick>
-
- self.tests = {}
-
- def interpretResult(self, output):
- """
- Interpret a test's output and return its result status.
-
- Applies the suite's regular expressions to the output, seeing which
- one matches. If none match, returns the suite's default status.
-
- The result will be either 'pass', 'fail', or 'skip'.
- """
- for status, re in self.status_re.items():
- if re.search(output) is not None:
- return status
-
- return self.status_default
-
-#############################################################################
-##### Configuration file reading
-#############################################################################
-
-def availableSuites():
- """
- Read the ~/.robynrc file and fetch a list of available test suites.
-
- Returns a dictionary whose keys are suite names, and whose items are
- absolute paths to the JSON suite description file.
-
- Does not create TestSuite objects and populate the test suite information.
- """
- # XXX: load list of available suites from ~/.robynrc or such
- return {'piglit': '/home/kwg/Projects/piglit/robyn-piglit.xml',
- 'glean': '/home/kwg/Projects/piglit/robyn-glean.xml'}
-
-### [str] x bool -> {str -> TestSuite}
-def loadSuiteInfo(desiredSuites, loadTests):
- """
- Load TestSuite descriptions for a list of requested suites.
- """
- suiteFiles = availableSuites()
-
- suiteInfo = {}
-
- for name in desiredSuites:
- if name not in suiteFiles:
- print('error: unknown test suite "%s".\n' % name)
- print(' Available Suites:')
- for s in suiteFiles.keys():
- print(' -', s)
- return {}
-
- with open(suiteFiles[name], 'r') as fp:
- suiteInfo[name] = TestSuite(fp)
-
- if loadTests:
- loadTestList(suiteInfo[name])
-
- return suiteInfo
-
-### TestSuite -> None
-def loadTestList(suite):
- """Actually load the tests."""
-
- with open(suite.testlist_file) as fp:
- js = json.load(fp)
-
- suite.tests = js
-
- return None
diff --git a/framework/test.py b/framework/test.py
new file mode 100644
index 0000000..cdcb905
--- /dev/null
+++ b/framework/test.py
@@ -0,0 +1,54 @@
+#
+# Copyright © 2012 Intel Corporation
+#
+# Permission is hereby granted, free of charge, to any person obtaining a
+# copy of this software and associated documentation files (the "Software"),
+# to deal in the Software without restriction, including without limitation
+# the rights to use, copy, modify, merge, publish, distribute, sublicense,
+# and/or sell copies of the Software, and to permit persons to whom the
+# Software is furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice (including the next
+# paragraph) shall be included in all copies or substantial portions of the
+# Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+# IN THE SOFTWARE.
+#
+
+from framework.process import runProgram
+
+__all__ = ['runProgram']
+
+#class TestResult:
+ #def __init__(self, status, valgrind_status, exitcode, out, err):
+ # visuals??????
+
+
+class Test:
+ def __init__(self, command):
+ self.command = command
+
+ def run(self):
+ out, err, exitcode, finished = runProgram(self.command, timeout)
+
+ return {'out': out, 'err': err, 'exitcode': exitcode,
+ 'status': self.resultStatus(out, err, exitcode, finished)}
+
+ def resultStatus(self, out, err, exitcode, finished):
+ if not finished:
+ return 'timeout'
+ elif exitcode in (-5, -6, -8, -10, -11, -1073741819, -1073741676):
+ # Unix: -5/SIGTRAP, -6/SIGABRT, -8/SIGFPE, -10/SIGUSR1, -11/SIGSEGV
+ # Windows: 0xc0000005 EXCEPTION_ACCESS_VIOLATION,
+ # 0xc0000094 EXCEPTION_INT_DIVIDE_BY_ZERO
+ return 'crash'
+
+ return self.suite.interpretResult(out)
+
+
diff --git a/programs/__init__.py b/programs/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/programs/__init__.py
diff --git a/programs/run.py b/programs/run.py
index 544bb79..d435cbe 100755
--- a/programs/run.py
+++ b/programs/run.py
@@ -32,7 +32,7 @@ import os.path as path
import sys
import framework.database
-import framework.suites
+import suites
def parseArguments(argv):
p = ArgumentParser(prog='robyn run', description='A GPU test runner')
@@ -58,13 +58,12 @@ def parseArguments(argv):
p.add_argument('suites', nargs='+')
return p.parse_args(argv)
-
-def main(argv):
+def main(argv, config):
options = parseArguments(argv)
framework.database.load()
- suites = framework.suites.loadSuiteInfo(options.suites, True)
+ tests = suites.loadTestLists(config, options.suites)
- for test in sorted(suites['piglit'].tests.keys()):
+ for test in sorted(tests['oglconform'].keys()):
print(test)
if __name__ == '__main__':
diff --git a/robyn b/robyn
index d06f552..d4f2ffc 100755
--- a/robyn
+++ b/robyn
@@ -21,11 +21,9 @@
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
# IN THE SOFTWARE.
#
-import programs.run
import sys
-import time
-from threading import Thread, Timer
-import subprocess
+import programs.run
+import framework.config
def usage():
print(
@@ -34,22 +32,26 @@ def usage():
Valid commands are:
Basic:
- - run Run tests/continue an existing test run
+ - run Run tests
+ - resume Resume an in-progress test run
- report Generate a report on the results.
Suite management:
- list-suites List known test suites
- add-suite Add a new test suite to your .robynrc
-See '%(prog)s help <command>' for more information on a specific command. """ % { 'prog': sys.argv[0] })
+See '%(prog)s help <command>' for more information on a specific command.""" % { 'prog': sys.argv[0] })
sys.exit(0)
if len(sys.argv) < 2:
usage()
+config = framework.config.load()
+
cmd = sys.argv[1]
+args = sys.argv[2:]
if cmd == 'run':
- programs.run.main(sys.argv[2:])
+ programs.run.main(args, config)
else:
usage()
diff --git a/robynrc.example b/robynrc.example
new file mode 100644
index 0000000..9e1b7a1
--- /dev/null
+++ b/robynrc.example
@@ -0,0 +1,22 @@
+[piglit]
+basedir = /home/kwg/Projects/piglit
+
+[es2conform]
+basedir = /home/kwg/Projects/ogles2conform/GTF_ES/glsl/GTF
+
+[es3conform]
+basedir = /home/kwg/Projects/gles3conform/GTF_ES/glsl/GTF
+
+[oglconform]
+binary = /home/kwg/Projects/oglconform/dump/linux/debug64/OGLconform/oglconform64
+
+[options]
+platform = glx # x11_egl, wayland, agl, wgl
+
+threads = 1 # No concurrency
+
+# Detect intermittent tests
+recheck-fail = 5
+recheck-crash = 0
+recheck-hung = 0
+recheck-pass = 0
diff --git a/suites/__init__.py b/suites/__init__.py
new file mode 100644
index 0000000..f7d5dd3
--- /dev/null
+++ b/suites/__init__.py
@@ -0,0 +1,39 @@
+import os
+import os.path as path
+import importlib
+from glob import iglob
+
+__all__ = ['loadTestLists']
+
+def loadPlugins(config):
+ '''
+ Find and import all submodules in the current package's directory.
+
+ Rather than making everyone enumerate all the types of tests, they can
+ simply drop .py files in here. As long as they have a generateTestList()
+ function, they'll be loaded and work.
+ '''
+ plugins = {}
+ suitesDir = path.dirname(path.realpath(__file__))
+ for pyfile in iglob(path.join(suitesDir, '[!_]*.py')):
+ p = path.basename(pyfile)[:-3]
+ plugin = importlib.import_module('suites.' + p)
+ if plugin.init(config):
+ plugins[p] = plugin
+
+ return plugins
+
+
+def loadTestLists(config, desiredSuites):
+ '''Generate the necessary test lists!
+
+ desiredSuites may be None (to load all test suites).
+ '''
+ suites = loadPlugins(config)
+ tests = {}
+ for suite, plugin in suites.items():
+ if not desiredSuites or suite in desiredSuites:
+ tests[suite] = plugin.makeTestList()
+
+ return tests
+
diff --git a/suites/oglconform.py b/suites/oglconform.py
new file mode 100644
index 0000000..86d0bc2
--- /dev/null
+++ b/suites/oglconform.py
@@ -0,0 +1,70 @@
+#
+# Copyright © 2012 Intel Corporation
+#
+# Permission is hereby granted, free of charge, to any person obtaining a
+# copy of this software and associated documentation files (the "Software"),
+# to deal in the Software without restriction, including without limitation
+# the rights to use, copy, modify, merge, publish, distribute, sublicense,
+# and/or sell copies of the Software, and to permit persons to whom the
+# Software is furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice (including the next
+# paragraph) shall be included in all copies or substantial portions of the
+# Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+# IN THE SOFTWARE.
+#
+
+import re
+from subprocess import call, DEVNULL
+from framework.test import Test
+
+__all__ = ['init', 'makeTestList']
+
+###
+## Plugin initialization
+###
+oglc = None
+def init(config):
+ global oglc
+ try:
+ # Get the oglconform binary path from the configuration file
+ block = config['oglconform']
+ oglc = block['binary']
+ return True
+ except KeyError:
+ return False
+
+skip_re = re.compile(r'Total Not run: 1|no test in schedule is compat|GLSL [134].[0-5]0 is not supported|wont be scheduled due to lack of compatible fbconfig')
+pass_re = re.compile(r'Total Passed : 1')
+
+class OGLCTest(Test):
+ def __init__(self, oglc, category, subtest):
+ Test.__init__(self, [oglc, '-minFmt', '-v', '4', '-test', category, subtest])
+
+ def interpretResult(self, out, err, exitcode):
+ if skip_re.search(out) is not None:
+ return 'skip'
+ elif pass_re.search(out) is not None:
+ return 'pass'
+
+ return 'fail'
+
+def makeTestList():
+ tempfile = '/tmp/oglc.tests'
+ call([oglc, '-generateTestList', tempfile], stdout=DEVNULL, stderr=DEVNULL)
+ with open(tempfile) as f:
+ testlist = f.read().splitlines()
+
+ d = {}
+ for l in testlist:
+ category, test = l.split()
+ d['oglconform/' + category + '/' + test] = OGLCTest(oglc, category, test)
+
+ return d