summaryrefslogtreecommitdiff
path: root/regtest
diff options
context:
space:
mode:
authorCarlos Garcia Campos <carlosgc@gnome.org>2012-09-23 18:15:13 +0200
committerCarlos Garcia Campos <carlosgc@gnome.org>2012-09-23 18:15:13 +0200
commit32bb87419c360a3b14c717c0f385198b70a1b2e7 (patch)
tree16c8fe02a24b999f2d46582d8a9b8af01d41635f /regtest
parentca6afce24aac2ef9d88e215177b11760f7468a6d (diff)
regtest: Reduce the noise of the default output when running tests
Show permanent information only about failed tests, without the details about the failing pages. Previous verbose output is available passing --verbose command line output.
Diffstat (limited to 'regtest')
-rw-r--r--regtest/Printer.py96
-rw-r--r--regtest/TestReferences.py8
-rw-r--r--regtest/TestRun.py35
-rw-r--r--regtest/backends/__init__.py13
-rw-r--r--regtest/commands/create-refs.py3
-rw-r--r--regtest/commands/find-regression.py5
-rw-r--r--regtest/commands/run-tests.py3
-rw-r--r--regtest/main.py3
8 files changed, 137 insertions, 29 deletions
diff --git a/regtest/Printer.py b/regtest/Printer.py
new file mode 100644
index 00000000..68356b5a
--- /dev/null
+++ b/regtest/Printer.py
@@ -0,0 +1,96 @@
+# Printer.py
+#
+# Copyright (C) 2012 Carlos Garcia Campos <carlosgc@gnome.org>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+
+import sys
+from Config import Config
+
+class Printer:
+
+ __single = None
+
+ def __init__(self):
+ if Printer.__single is not None:
+ raise Printer.__single
+
+ self._verbose = Config().verbose
+ self._stream = sys.stdout
+ self._rewrite = self._stream.isatty() #and not self.config.verbose
+ self._current_line = None
+
+ Printer.__single = self
+
+ def _erase_current_line(self):
+ if not self._rewrite or self._current_line is None:
+ return
+
+ line_len = len(self._current_line)
+ self._stream.write('\b' * line_len + ' ' * line_len + '\b' * line_len)
+ self._current_line = None
+
+ def _ensure_new_line(self, msg):
+ if not msg.endswith('\n'):
+ msg += '\n'
+ return msg
+
+ def _print(self, msg):
+ self._stream.write(msg)
+ self._stream.flush()
+
+ def printout(self, msg):
+ self._erase_current_line()
+ self._print(msg)
+ self._current_line = msg[msg.rfind('\n') + 1:]
+
+ def printout_update(self, msg):
+ if self._rewrite and self._current_line is not None:
+ msg = self._current_line + msg
+ elif not self._rewrite:
+ msg = self._ensure_new_line(msg)
+ self.printout(msg)
+
+ def printout_ln(self, msg):
+ if self._current_line is not None:
+ self._current_line = None
+ msg = '\n' + msg
+
+ self._print(self._ensure_new_line(msg))
+
+ def printerr(self, msg):
+ self.stderr.write(self._ensure_new_line(msg))
+ self.stderr.flush()
+
+ def print_test_start(self, msg):
+ self.printout(msg)
+
+ def print_test_result(self, msg):
+ self.printout_update(msg)
+
+ def print_default(self, msg):
+ if self._verbose:
+ self.printout_ln(msg)
+
+def get_printer():
+ try:
+ instance = Printer()
+ except Printer, i:
+ instance = i
+
+ return instance
+
+
+
diff --git a/regtest/TestReferences.py b/regtest/TestReferences.py
index d65d30d4..9a9e9233 100644
--- a/regtest/TestReferences.py
+++ b/regtest/TestReferences.py
@@ -20,6 +20,7 @@ import os
import errno
from backends import get_backend, get_all_backends
from Config import Config
+from Printer import get_printer
from Utils import get_document_paths_from_dir, get_skipped_tests
class TestReferences:
@@ -29,6 +30,7 @@ class TestReferences:
self._refsdir = refsdir
self._skipped = get_skipped_tests(docsdir)
self.config = Config()
+ self.printer = get_printer()
try:
os.makedirs(self._refsdir)
@@ -40,7 +42,7 @@ class TestReferences:
def create_refs_for_file(self, filename, n_doc = 1, total_docs = 1):
if filename in self._skipped:
- print("Skipping test '%s' (%d/%d)" % (os.path.join(self._docsdir, filename), n_doc, total_docs))
+ self.printer.print_default("Skipping test '%s' (%d/%d)" % (os.path.join(self._docsdir, filename), n_doc, total_docs))
return
refs_path = os.path.join(self._refsdir, filename)
@@ -60,9 +62,9 @@ class TestReferences:
for backend in backends:
if not self.config.force and backend.has_results(refs_path):
- print("Results found, skipping '%s' for %s backend (%d/%d)" % (doc_path, backend.get_name(), n_doc, total_docs))
+ self.printer.print_default("Results found, skipping '%s' for %s backend (%d/%d)" % (doc_path, backend.get_name(), n_doc, total_docs))
continue
- print("Creating refs for '%s' using %s backend (%d/%d)" % (doc_path, backend.get_name(), n_doc, total_docs))
+ self.printer.printout_ln("Creating refs for '%s' using %s backend (%d/%d)" % (doc_path, backend.get_name(), n_doc, total_docs))
if backend.create_refs(doc_path, refs_path):
backend.create_checksums(refs_path, self.config.checksums_only)
diff --git a/regtest/TestRun.py b/regtest/TestRun.py
index cffd00b1..24afa2fd 100644
--- a/regtest/TestRun.py
+++ b/regtest/TestRun.py
@@ -19,6 +19,7 @@
from backends import get_backend, get_all_backends
from Config import Config
from Utils import get_document_paths_from_dir, get_skipped_tests
+from Printer import get_printer
import sys
import os
import errno
@@ -31,6 +32,7 @@ class TestRun:
self._outdir = outdir
self._skip = get_skipped_tests(docsdir)
self.config = Config()
+ self.printer = get_printer()
# Results
self._n_tests = 0
@@ -56,12 +58,11 @@ class TestRun:
ref_is_failed = backend.is_failed(refs_path)
if not ref_has_md5 and not ref_is_crashed and not ref_is_failed:
self._skipped.append("%s (%s)" % (doc_path, backend.get_name()))
- print("Reference files not found, skipping '%s' for %s backend" % (doc_path, backend.get_name()))
+ self.printer.print_default("Reference files not found, skipping '%s' for %s backend" % (doc_path, backend.get_name()))
return
self._n_tests += 1
- sys.stdout.write("Testing '%s' using %s backend (%d/%d): " % (doc_path, backend.get_name(), n_doc, total_docs))
- sys.stdout.flush()
+ self.printer.print_test_start("Testing '%s' using %s backend (%d/%d): " % (doc_path, backend.get_name(), n_doc, total_docs))
test_has_md5 = backend.create_refs(doc_path, test_path)
if backend.has_stderr(test_path):
@@ -70,7 +71,7 @@ class TestRun:
if ref_has_md5 and test_has_md5:
if backend.compare_checksums(refs_path, test_path, not self.config.keep_results, self.config.create_diffs, self.config.update_refs):
# FIXME: remove dir if it's empty?
- print("PASS")
+ self.printer.print_test_result("PASS")
self._n_passed += 1
else:
print("FAIL")
@@ -78,32 +79,31 @@ class TestRun:
return
elif test_has_md5:
if ref_is_crashed:
- print("DOES NOT CRASH")
+ self.printer.print_test_result("DOES NOT CRASH")
elif ref_is_failed:
- print("DOES NOT FAIL")
-
+ self.printer.print_test_result("DOES NOT FAIL")
return
test_is_crashed = backend.is_crashed(test_path)
if ref_is_crashed and test_is_crashed:
- print("PASS (Expected crash)")
+ self.printer.print_test_result("PASS (Expected crash)")
self._n_passed += 1
return
test_is_failed = backend.is_failed(test_path)
if ref_is_failed and test_is_failed:
# FIXME: compare status errors
- print("PASS (Expected fail with status error %d)" % (test_is_failed))
+ self.printer.print_test_result("PASS (Expected fail with status error %d)" % (test_is_failed))
self._n_passed += 1
return
if test_is_crashed:
- print("CRASH")
+ self.printer.print_test_result("CRASH")
self._crashed.append("%s (%s)" % (doc_path, backend.get_name()))
return
if test_is_failed:
- print("FAIL (status error %d)" % (test_is_failed))
+ self.printer.print_test_result("FAIL (status error %d)" % (test_is_failed))
self._failed_status_error("%s (%s)" % (doc_path, backend.get_name()))
return
@@ -111,7 +111,7 @@ class TestRun:
if filename in self._skip:
doc_path = os.path.join(self._docsdir, filename)
self._skipped.append("%s" % (doc_path))
- print("Skipping test '%s' (%d/%d)" % (doc_path, n_doc, total_docs))
+ self.printer.print_default("Skipping test '%s' (%d/%d)" % (doc_path, n_doc, total_docs))
return
out_path = os.path.join(self._outdir, filename)
@@ -127,7 +127,7 @@ class TestRun:
if not os.path.isdir(refs_path):
self._skipped.append("%s" % (doc_path))
- print("Reference dir not found for %s, skipping (%d/%d)" % (doc_path, n_doc, total_docs))
+ self.printer.print_default("Reference dir not found for %s, skipping (%d/%d)" % (doc_path, n_doc, total_docs))
return
if self.config.backends:
@@ -147,16 +147,17 @@ class TestRun:
def summary(self):
if not self._n_tests:
- print("No tests run")
+ self.printer.printout_ln("No tests run")
return
- print("Total %d tests" % (self._n_tests))
- print("%d tests passed (%.2f%%)" % (self._n_passed, (self._n_passed * 100.) / self._n_tests))
+ self.printer.printout_ln("Total %d tests" % (self._n_tests))
+ self.printer.printout_ln("%d tests passed (%.2f%%)" % (self._n_passed, (self._n_passed * 100.) / self._n_tests))
def report_tests(test_list, test_type):
n_tests = len(test_list)
if not n_tests:
return
- print("%d tests %s (%.2f%%): %s" % (n_tests, test_type, (n_tests * 100.) / self._n_tests, ", ".join(test_list)))
+ self.printer.printout_ln("%d tests %s (%.2f%%): %s" % (n_tests, test_type, (n_tests * 100.) / self._n_tests, ", ".join(test_list)))
+
report_tests(self._failed, "failed")
report_tests(self._crashed, "crashed")
report_tests(self._failed_status_error, "failed to run")
diff --git a/regtest/backends/__init__.py b/regtest/backends/__init__.py
index 26be0b03..ff6ef849 100644
--- a/regtest/backends/__init__.py
+++ b/regtest/backends/__init__.py
@@ -21,6 +21,7 @@ import os
import shutil
import errno
from Config import Config
+from Printer import get_printer
__all__ = [ 'register_backend',
'get_backend',
@@ -38,6 +39,8 @@ class Backend:
self._diff_ext = diff_ext
self._utilsdir = Config().utils_dir
+ self.printer = get_printer()
+
def get_name(self):
return self._name
@@ -83,7 +86,7 @@ class Backend:
if not basename in tests:
retval = False
- print("%s found in md5 ref file but missing in output dir %s" % (basename, out_path))
+ self.printer.print_default("%s found in md5 ref file but missing in output dir %s" % (basename, out_path))
continue
result_path = os.path.join(out_path, basename)
@@ -99,10 +102,10 @@ class Backend:
if remove_results:
os.remove(result_path)
else:
- print("Differences found in %s" % (basename))
+ self.printer.print_default("Differences found in %s" % (basename))
if create_diffs:
if not os.path.exists(ref_path):
- print("Reference file %s not found, skipping diff for %s" % (ref_path, result_path))
+ self.printer.print_default("Reference file %s not found, skipping diff for %s" % (ref_path, result_path))
else:
try:
self._create_diff(ref_path, result_path)
@@ -112,14 +115,14 @@ class Backend:
if update_refs:
if os.path.exists(ref_path):
- print("Updating image reference %s" % (ref_path))
+ self.printer.print_default("Updating image reference %s" % (ref_path))
shutil.copyfile(result_path, ref_path)
retval = False
md5_file.close()
if update_refs and not retval:
- print("Updating md5 reference %s" % (md5_path))
+ self.printer.print_default("Updating md5 reference %s" % (md5_path))
f = open(md5_path + '.md5.tmp', 'wb')
f.writelines(result_md5)
f.close()
diff --git a/regtest/commands/create-refs.py b/regtest/commands/create-refs.py
index b055703d..d559fb3f 100644
--- a/regtest/commands/create-refs.py
+++ b/regtest/commands/create-refs.py
@@ -20,6 +20,7 @@ from commands import Command, register_command
from TestReferences import TestReferences
from Timer import Timer
from Config import Config
+from Printer import get_printer
import os
import tempfile
@@ -60,6 +61,6 @@ class CreateRefs(Command):
refs.create_refs()
else:
refs.create_refs_for_file(os.path.basename(doc))
- print("Refs created in %s" % (t.elapsed_str()))
+ get_printer().printout_ln("Refs created in %s" % (t.elapsed_str()))
register_command('create-refs', CreateRefs)
diff --git a/regtest/commands/find-regression.py b/regtest/commands/find-regression.py
index 1a46eee8..8f5f8113 100644
--- a/regtest/commands/find-regression.py
+++ b/regtest/commands/find-regression.py
@@ -20,6 +20,7 @@ from commands import Command, register_command
from Bisect import Bisect
from Timer import Timer
from Config import Config
+from Printer import get_printer
import os
import tempfile
@@ -66,12 +67,12 @@ class FindRegression(Command):
doc = options['test']
if not os.path.isfile(doc):
- print("Invalid test %s: not a regulat file" % (doc))
+ get_printer().printerr("Invalid test %s: not a regulat file" % (doc))
return
t = Timer()
bisect = Bisect(options['test'], options['refs_dir'], options['out_dir'])
bisect.run()
- print("Tests run in %s" % (t.elapsed_str()))
+ get_printer().printout_ln("Tests run in %s" % (t.elapsed_str()))
register_command('find-regression', FindRegression)
diff --git a/regtest/commands/run-tests.py b/regtest/commands/run-tests.py
index d05d8158..c5d87f96 100644
--- a/regtest/commands/run-tests.py
+++ b/regtest/commands/run-tests.py
@@ -20,6 +20,7 @@ from commands import Command, register_command
from TestRun import TestRun
from Timer import Timer
from Config import Config
+from Printer import get_printer
import os
import tempfile
@@ -68,6 +69,6 @@ class RunTests(Command):
else:
tests.run_test(os.path.basename(doc))
tests.summary()
- print("Tests run in %s" % (t.elapsed_str()))
+ get_printer().printout_ln("Tests run in %s" % (t.elapsed_str()))
register_command('run-tests', RunTests)
diff --git a/regtest/main.py b/regtest/main.py
index a46a64c2..290c8bc6 100644
--- a/regtest/main.py
+++ b/regtest/main.py
@@ -49,6 +49,9 @@ def main(args):
parser.add_argument('--help-command', metavar = 'COMMAND',
action = HelpAction,
help = 'Show help for a given command')
+ parser.add_argument('-v', '--verbose',
+ action = 'store_true', dest = 'verbose', default = False,
+ help = 'Run in verbose mode')
parser.add_argument('--utils-dir',
action = 'store', dest = 'utils_dir', default = os.path.abspath("../utils"),
help = 'Directory of poppler utils used for the tests')