summaryrefslogtreecommitdiff
path: root/regtest
diff options
context:
space:
mode:
authorCarlos Garcia Campos <cgarcia@igalia.com>2017-12-21 13:25:37 +0100
committerCarlos Garcia Campos <carlosgc@gnome.org>2017-12-21 15:22:31 +0100
commit48de59a92a5f88d10a1a53abf7f15f578104e1ed (patch)
tree71dd2b958eeea9cf8a5c8e4ca3580cffce21e55b /regtest
parent79a096a48c735ac301ae4d7439bddf630a559a96 (diff)
regtest: Add an option to exit after n failures
Note that running jobs are not cancelled when max failures is reached, so we usually end up getting more failures than the maximum.
Diffstat (limited to 'regtest')
-rw-r--r--regtest/HTMLReport.py2
-rw-r--r--regtest/Printer.py15
-rw-r--r--regtest/TestRun.py46
-rw-r--r--regtest/commands/run-tests.py8
4 files changed, 68 insertions, 3 deletions
diff --git a/regtest/HTMLReport.py b/regtest/HTMLReport.py
index f582e74d..b4840db2 100644
--- a/regtest/HTMLReport.py
+++ b/regtest/HTMLReport.py
@@ -278,6 +278,8 @@ class HTMLReport:
def create(self, launch_browser):
html = "<html><body><a name='top'></a>"
+ if os.path.exists(os.path.join(self._outdir, '.exited_early')):
+ html += "<p style='border: 3px solid red; font-weight: bold; display: inline-block; padding: 3px'>Testing exited early</p>"
if self.config.backends:
backends = [get_backend(name) for name in self.config.backends]
else:
diff --git a/regtest/Printer.py b/regtest/Printer.py
index 1de693d4..23ef7b3f 100644
--- a/regtest/Printer.py
+++ b/regtest/Printer.py
@@ -33,6 +33,7 @@ class Printer:
self._stream = sys.stdout
self._rewrite = self._stream.isatty() and not self._verbose
self._current_line_len = 0
+ self._blocked = 0
self._lock = RLock()
@@ -60,17 +61,23 @@ class Printer:
self.printout_ln(msg)
with self._lock:
+ if self._blocked > 0:
+ return
self._erase_current_line()
self._print(msg)
self._current_line_len = len(msg[msg.rfind('\n') + 1:])
def printout_ln(self, msg=''):
with self._lock:
+ if self._blocked > 0:
+ return
self._erase_current_line()
self._print(self._ensure_new_line(msg))
def printerr(self, msg):
with self._lock:
+ if self._blocked > 0:
+ return
self.stderr.write(self._ensure_new_line(msg))
self.stderr.flush()
@@ -84,6 +91,14 @@ class Printer:
if self._verbose:
self.printout_ln(msg)
+ def block(self):
+ with self._lock:
+ self._blocked += 1
+
+ def unblock(self):
+ with self._lock:
+ self._blocked -= 1
+
def get_printer():
try:
instance = Printer()
diff --git a/regtest/TestRun.py b/regtest/TestRun.py
index e344cc7d..d2a0ed07 100644
--- a/regtest/TestRun.py
+++ b/regtest/TestRun.py
@@ -23,21 +23,24 @@ from Printer import get_printer
import sys
import os
import errno
+import shutil
from InterruptibleQueue import InterruptibleQueue
from threading import Thread, RLock
class TestRun:
- def __init__(self, docsdir, refsdir, outdir):
+ def __init__(self, docsdir, refsdir, outdir, max_failures = None):
self._docsdir = docsdir
self._refsdir = refsdir
self._outdir = outdir
+ self._max_failures = max_failures
self._skip = get_skipped_tests(docsdir)
self._passwords = get_passwords(docsdir)
self.config = Config()
self.printer = get_printer()
self._total_tests = 1
+ self._exited_early = False
# Results
self._n_tests = 0
@@ -171,13 +174,42 @@ class TestRun:
for backend in backends:
self.test(refs_path, doc_path, out_path, backend, password)
+ def _should_exit_early(self):
+ if self._max_failures is None:
+ return False
+
+ def _len(tests):
+ retval = 0
+ for backend in tests:
+ retval += len(tests[backend])
+ return retval
+
+ with self._lock:
+ if _len(self._failed) + _len(self._crashed) + _len(self._failed_status_error) >= self._max_failures:
+ if not self._exited_early:
+ self.printer.printout_ln('Exiting early after %d failures, waiting for running jobs to finish...' % self._max_failures)
+ self.printer.block()
+ self._exited_early = True
+ return True
+ return False
+
def _worker_thread(self):
while True:
doc = self._queue.get()
- self.run_test(doc)
+ if not self._should_exit_early():
+ self.run_test(doc)
self._queue.task_done()
def run_tests(self, tests = []):
+ # Clean the output dir.
+ try:
+ shutil.rmtree(self._outdir)
+ except OSError as e:
+ if e.errno != errno.ENOENT:
+ raise
+ except:
+ raise
+
if not tests:
docs, total_docs = get_document_paths_from_dir(self._docsdir)
else:
@@ -229,6 +261,13 @@ class TestRun:
else:
for doc in docs:
self.run_test(doc)
+ if self._should_exit_early():
+ break
+
+ if self._exited_early:
+ open(os.path.join(self._outdir, '.exited_early'), 'w').close()
+ self.printer.unblock()
+ return -self._max_failures
return int(self._n_passed != self._n_run)
@@ -236,6 +275,9 @@ class TestRun:
self.printer.printout_ln()
if self._n_run:
+ if self._exited_early:
+ self.printer.printout_ln("Testing exited early")
+ self.printer.printout_ln()
self.printer.printout_ln("%d tests passed (%.2f%%)" % (self._n_passed, (self._n_passed * 100.) / self._n_run))
self.printer.printout_ln()
diff --git a/regtest/commands/run-tests.py b/regtest/commands/run-tests.py
index 5f1914fe..d8df26cc 100644
--- a/regtest/commands/run-tests.py
+++ b/regtest/commands/run-tests.py
@@ -51,6 +51,9 @@ class RunTests(Command):
parser.add_argument('--update-refs',
action = 'store_true', dest = 'update_refs', default = False,
help = 'Update references for failed tests')
+ parser.add_argument('--exit-after-n-failures',
+ action = 'store', dest = 'max_failures', type=int, default = None,
+ help = 'Exit after N failures. Ignored if --update-refs is present too')
parser.add_argument('tests', metavar = 'TEST', nargs = '+',
help = 'Tests directory or individual test to run')
@@ -77,7 +80,10 @@ class RunTests(Command):
if docs_dir is None:
docs_dir = os.path.commonprefix(docs).rpartition(os.path.sep)[0]
- tests = TestRun(docs_dir, options['refs_dir'], options['out_dir'])
+ max_failures = None
+ if not config.update_refs:
+ max_failures = options['max_failures']
+ tests = TestRun(docs_dir, options['refs_dir'], options['out_dir'], max_failures)
status = tests.run_tests(docs)
tests.summary()
get_printer().printout_ln("Tests run in %s" % (t.elapsed_str()))