From 48de59a92a5f88d10a1a53abf7f15f578104e1ed Mon Sep 17 00:00:00 2001 From: Carlos Garcia Campos Date: Thu, 21 Dec 2017 13:25:37 +0100 Subject: regtest: Add an option to exit after n failures Note that running jobs are not cancelled when max failures is reached, so we usually end up getting more failures than the maximum. --- regtest/HTMLReport.py | 2 ++ regtest/Printer.py | 15 ++++++++++++++ regtest/TestRun.py | 46 +++++++++++++++++++++++++++++++++++++++++-- regtest/commands/run-tests.py | 8 +++++++- 4 files changed, 68 insertions(+), 3 deletions(-) (limited to 'regtest') diff --git a/regtest/HTMLReport.py b/regtest/HTMLReport.py index f582e74d..b4840db2 100644 --- a/regtest/HTMLReport.py +++ b/regtest/HTMLReport.py @@ -278,6 +278,8 @@ class HTMLReport: def create(self, launch_browser): html = "" + if os.path.exists(os.path.join(self._outdir, '.exited_early')): + html += "

Testing exited early

" if self.config.backends: backends = [get_backend(name) for name in self.config.backends] else: diff --git a/regtest/Printer.py b/regtest/Printer.py index 1de693d4..23ef7b3f 100644 --- a/regtest/Printer.py +++ b/regtest/Printer.py @@ -33,6 +33,7 @@ class Printer: self._stream = sys.stdout self._rewrite = self._stream.isatty() and not self._verbose self._current_line_len = 0 + self._blocked = 0 self._lock = RLock() @@ -60,17 +61,23 @@ class Printer: self.printout_ln(msg) with self._lock: + if self._blocked > 0: + return self._erase_current_line() self._print(msg) self._current_line_len = len(msg[msg.rfind('\n') + 1:]) def printout_ln(self, msg=''): with self._lock: + if self._blocked > 0: + return self._erase_current_line() self._print(self._ensure_new_line(msg)) def printerr(self, msg): with self._lock: + if self._blocked > 0: + return self.stderr.write(self._ensure_new_line(msg)) self.stderr.flush() @@ -84,6 +91,14 @@ class Printer: if self._verbose: self.printout_ln(msg) + def block(self): + with self._lock: + self._blocked += 1 + + def unblock(self): + with self._lock: + self._blocked -= 1 + def get_printer(): try: instance = Printer() diff --git a/regtest/TestRun.py b/regtest/TestRun.py index e344cc7d..d2a0ed07 100644 --- a/regtest/TestRun.py +++ b/regtest/TestRun.py @@ -23,21 +23,24 @@ from Printer import get_printer import sys import os import errno +import shutil from InterruptibleQueue import InterruptibleQueue from threading import Thread, RLock class TestRun: - def __init__(self, docsdir, refsdir, outdir): + def __init__(self, docsdir, refsdir, outdir, max_failures = None): self._docsdir = docsdir self._refsdir = refsdir self._outdir = outdir + self._max_failures = max_failures self._skip = get_skipped_tests(docsdir) self._passwords = get_passwords(docsdir) self.config = Config() self.printer = get_printer() self._total_tests = 1 + self._exited_early = False # Results self._n_tests = 0 @@ -171,13 +174,42 @@ class TestRun: for backend in backends: self.test(refs_path, doc_path, out_path, backend, password) + def _should_exit_early(self): + if self._max_failures is None: + return False + + def _len(tests): + retval = 0 + for backend in tests: + retval += len(tests[backend]) + return retval + + with self._lock: + if _len(self._failed) + _len(self._crashed) + _len(self._failed_status_error) >= self._max_failures: + if not self._exited_early: + self.printer.printout_ln('Exiting early after %d failures, waiting for running jobs to finish...' % self._max_failures) + self.printer.block() + self._exited_early = True + return True + return False + def _worker_thread(self): while True: doc = self._queue.get() - self.run_test(doc) + if not self._should_exit_early(): + self.run_test(doc) self._queue.task_done() def run_tests(self, tests = []): + # Clean the output dir. + try: + shutil.rmtree(self._outdir) + except OSError as e: + if e.errno != errno.ENOENT: + raise + except: + raise + if not tests: docs, total_docs = get_document_paths_from_dir(self._docsdir) else: @@ -229,6 +261,13 @@ class TestRun: else: for doc in docs: self.run_test(doc) + if self._should_exit_early(): + break + + if self._exited_early: + open(os.path.join(self._outdir, '.exited_early'), 'w').close() + self.printer.unblock() + return -self._max_failures return int(self._n_passed != self._n_run) @@ -236,6 +275,9 @@ class TestRun: self.printer.printout_ln() if self._n_run: + if self._exited_early: + self.printer.printout_ln("Testing exited early") + self.printer.printout_ln() self.printer.printout_ln("%d tests passed (%.2f%%)" % (self._n_passed, (self._n_passed * 100.) / self._n_run)) self.printer.printout_ln() diff --git a/regtest/commands/run-tests.py b/regtest/commands/run-tests.py index 5f1914fe..d8df26cc 100644 --- a/regtest/commands/run-tests.py +++ b/regtest/commands/run-tests.py @@ -51,6 +51,9 @@ class RunTests(Command): parser.add_argument('--update-refs', action = 'store_true', dest = 'update_refs', default = False, help = 'Update references for failed tests') + parser.add_argument('--exit-after-n-failures', + action = 'store', dest = 'max_failures', type=int, default = None, + help = 'Exit after N failures. Ignored if --update-refs is present too') parser.add_argument('tests', metavar = 'TEST', nargs = '+', help = 'Tests directory or individual test to run') @@ -77,7 +80,10 @@ class RunTests(Command): if docs_dir is None: docs_dir = os.path.commonprefix(docs).rpartition(os.path.sep)[0] - tests = TestRun(docs_dir, options['refs_dir'], options['out_dir']) + max_failures = None + if not config.update_refs: + max_failures = options['max_failures'] + tests = TestRun(docs_dir, options['refs_dir'], options['out_dir'], max_failures) status = tests.run_tests(docs) tests.summary() get_printer().printout_ln("Tests run in %s" % (t.elapsed_str())) -- cgit v1.2.3