Add initial poppler regressions test program

14 files changed, 1025 insertions, 0 deletions

diff --git a/regtest/Config.py b/regtest/Config.py
new file mode 100644
index 00000000..197edcea
--- /dev/null
+++ b/regtest/Config.py
@@ -0,0 +1,32 @@
+# Config.py
+#
+# Copyright (C) 2011 Carlos Garcia Campos <carlosgc@gnome.org>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+class Config:
+
+    shared_state = {}
+
+    def __init__(self, config = None):
+        if config is not None:
+            self.__class__.shared_state = config
+        self.__dict__ = self.__class__.shared_state
+
+if __name__ == '__main__':
+    c = Config({'foo' : 25})
+    print c.foo
+    cc = Config()
+    print cc.foo
diff --git a/regtest/TestReferences.py b/regtest/TestReferences.py
new file mode 100644
index 00000000..0af3f8ae
--- /dev/null
+++ b/regtest/TestReferences.py
@@ -0,0 +1,73 @@
+# TestReferences.py
+#
+# Copyright (C) 2011 Carlos Garcia Campos <carlosgc@gnome.org>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+import os
+import errno
+from backends import get_backend, get_all_backends
+from Config import Config
+
+class TestReferences:
+
+    def __init__(self, docsdir, refsdir):
+        self._docsdir = docsdir
+        self._refsdir = refsdir
+        self.config = Config()
+
+        try:
+            os.makedirs(self._refsdir)
+        except OSError, e:
+            if e.errno != errno.EEXIST:
+                raise
+        except:
+            raise
+
+    def create_refs_for_file(self, filename):
+        refs_path = os.path.join(self._refsdir, filename)
+        try:
+            os.makedirs(refs_path)
+        except OSError, e:
+            if e.errno != errno.EEXIST:
+                raise
+        except:
+            raise
+        doc_path = os.path.join(self._docsdir, filename)
+
+        if self.config.backends:
+            backends = [get_backend(name) for name in self.config.backends]
+        else:
+            backends = get_all_backends()
+
+        for backend in backends:
+            if not self.config.force and backend.has_md5(refs_path):
+                print "Checksum file found, skipping '%s' for %s backend" % (doc_path, backend.get_name())
+                continue
+            print "Creating refs for '%s' using %s backend" % (doc_path, backend.get_name())
+            if backend.create_refs(doc_path, refs_path):
+                backend.create_checksums(refs_path, self.config.checksums_only)
+
+    def create_refs(self):
+        for root, dirs, files in os.walk(self._docsdir, False):
+            for entry in files:
+                if not entry.lower().endswith('.pdf'):
+                    continue
+
+                test_path = os.path.join(root[len(self._docsdir):], entry)
+                self.create_refs_for_file(test_path.lstrip(os.path.sep))
+
+
+
diff --git a/regtest/TestRun.py b/regtest/TestRun.py
new file mode 100644
index 00000000..ffeb48d4
--- /dev/null
+++ b/regtest/TestRun.py
@@ -0,0 +1,156 @@
+# TestRun.py
+#
+# Copyright (C) 2011 Carlos Garcia Campos <carlosgc@gnome.org>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+from backends import get_backend, get_all_backends
+from Config import Config
+import sys
+import os
+import errno
+
+class TestRun:
+
+    def __init__(self, docsdir, refsdir, outdir):
+        self._docsdir = docsdir
+        self._refsdir = refsdir
+        self._outdir = outdir
+        self.config = Config()
+
+        # Results
+        self._n_tests = 0
+        self._n_passed = 0
+        self._failed = []
+        self._crashed = []
+        self._failed_status_error = []
+        self._stderr = []
+
+        try:
+            os.makedirs(self._outdir);
+        except OSError, e:
+            if e.errno != errno.EEXIST:
+                raise
+        except:
+            raise
+
+    def test(self, refs_path, doc_path, test_path, backend):
+        # First check whether there are test results for the backend
+        ref_has_md5 = backend.has_md5(refs_path)
+        ref_is_crashed = backend.is_crashed(refs_path)
+        ref_is_failed = backend.is_failed(refs_path)
+        if not ref_has_md5 and not ref_is_crashed and not ref_is_failed:
+            print "Reference files not found, skipping '%s' for %s backend" % (doc_path, backend.get_name())
+            return
+
+        self._n_tests += 1
+        sys.stdout.write("Testing '%s' using %s backend: " % (doc_path, backend.get_name()))
+        sys.stdout.flush()
+        test_has_md5 = backend.create_refs(doc_path, test_path)
+
+        if backend.has_stderr(test_path):
+            self._stderr.append("%s (%s)" % (doc_path, backend.get_name()))
+
+        if ref_has_md5 and test_has_md5:
+            if backend.compare_checksums(refs_path, test_path, not self.config.keep_results, self.config.create_diffs):
+                # FIXME: remove dir if it's empty?
+                print "PASS"
+                self._n_passed += 1
+            else:
+                print "FAIL"
+                self._failed.append("%s (%s)" % (doc_path, backend.get_name()))
+            return
+        elif test_has_md5:
+            if ref_is_crashed:
+                print "DOES NOT CRASH"
+            elif ref_is_failed:
+                print "DOES NOT FAIL"
+
+            return
+
+        test_is_crashed = backend.is_crashed(test_path)
+        if ref_is_crashed and test_is_crashed:
+            print "PASS (Expected crash)"
+            self._n_passed += 1
+            return
+
+        test_is_failed = backend.is_failed(test_path)
+        if ref_is_failed and test_is_failed:
+            # FIXME: compare status errors
+            print "PASS (Expected fail with status error %d)" % (test_is_failed)
+            self._n_passed += 1
+            return
+
+        if test_is_crashed:
+            print "CRASH"
+            self._crashed.append("%s (%s)" % (doc_path, backend.get_name()))
+            return
+
+        if test_is_failed:
+            print "FAIL (status error %d)" % (test_is_failed)
+            self._failed_status_error("%s (%s)" % (doc_path, backend.get_name()))
+            return
+
+    def run_test(self, filename):
+        out_path = os.path.join(self._outdir, filename)
+        try:
+            os.makedirs(out_path)
+        except OSError, e:
+            if e.errno != errno.EEXIST:
+                raise
+        except:
+            raise
+        doc_path = os.path.join(self._docsdir, filename)
+        refs_path = os.path.join(self._refsdir, filename)
+
+        if not os.path.isdir(refs_path):
+            print "Reference dir not found for %s, skipping" % (doc_path)
+            return
+
+        if self.config.backends:
+            backends = [get_backend(name) for name in self.config.backends]
+        else:
+            backends = get_all_backends()
+
+        for backend in backends:
+            self.test(refs_path, doc_path, out_path, backend)
+
+    def run_tests(self):
+        for root, dirs, files in os.walk(self._docsdir, False):
+              for entry in files:
+                  if not entry.lower().endswith('.pdf'):
+                      continue
+
+                  test_path = os.path.join(root[len(self._docsdir):], entry)
+                  self.run_test(test_path.lstrip(os.path.sep))
+
+    def summary(self):
+        if not self._n_tests:
+            print "No tests run"
+            return
+
+        print "Total %d tests" % (self._n_tests)
+        print "%d tests passed (%.2f%%)" % (self._n_passed, (self._n_passed * 100.) / self._n_tests)
+        def report_tests(test_list, test_type):
+            n_tests = len(test_list)
+            if not n_tests:
+                return
+            print "%d tests %s (%.2f%%): %s" % (n_tests, test_type, (n_tests * 100.) / self._n_tests, ", ".join(test_list))
+        report_tests(self._failed, "failed")
+        report_tests(self._crashed, "crashed")
+        report_tests(self._failed_status_error, "failed to run")
+        report_tests(self._stderr, "have stderr output")
+
+
diff --git a/regtest/Timer.py b/regtest/Timer.py
new file mode 100644
index 00000000..102af965
--- /dev/null
+++ b/regtest/Timer.py
@@ -0,0 +1,73 @@
+# Timer.py
+#
+# Copyright (C) 2011 Carlos Garcia Campos <carlosgc@gnome.org>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+from time import time, strftime, gmtime
+
+class Timer:
+
+    def __init__(self, start = True):
+        self._stop = None
+        if start:
+            self.start()
+        else:
+            self._start = None
+
+    def start(self):
+        self._start = time()
+
+    def stop(self):
+        self._stop = time()
+
+    def elapsed(self):
+        if self._start is None:
+            return 0
+
+        if self._stop is None:
+            return time() - self._start
+
+        return self._stop - self._start
+
+    def elapsed_str(self):
+        h, m, s = [int(i) for i in strftime('%H:%M:%S', gmtime(self.elapsed())).split(':')]
+        retval = "%d seconds" % (s)
+        if h == 0 and m == 0:
+            return retval
+
+        retval = "%d minutes and %s" % (m, retval)
+        if h == 0:
+            return retval
+
+        retval = "%d hours, %s" % (h, retval)
+        return retval
+
+
+if __name__ == '__main__':
+    from time import sleep
+
+    t = Timer()
+    sleep(5)
+    print "Elapsed: %s" % (t.elapsed_str())
+    sleep(1)
+    print "Elapsed: %s" % (t.elapsed_str())
+
+    t.start()
+    sleep(2)
+    t.stop()
+    print "Elapsed: %s" % (t.elapsed_str())
+    sleep(2)
+    print "Elapsed: %s" % (t.elapsed_str())
diff --git a/regtest/backends/__init__.py b/regtest/backends/__init__.py
new file mode 100644
index 00000000..e730eaa7
--- /dev/null
+++ b/regtest/backends/__init__.py
@@ -0,0 +1,220 @@
+# backends
+#
+# Copyright (C) 2011 Carlos Garcia Campos <carlosgc@gnome.org>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+from hashlib import md5
+import os
+from Config import Config
+
+__all__ = [ 'register_backend',
+            'get_backend',
+            'get_all_backends',
+            'UnknownBackendError',
+            'Backend' ]
+
+class UnknownBackendError(Exception):
+    '''Unknown backend type'''
+
+class Backend:
+
+    def __init__(self, name):
+        self._name = name
+        self._utilsdir = Config().utils_dir
+
+    def get_name(self):
+        return self._name
+
+    def create_checksums(self, refs_path, delete_refs = False):
+        path = os.path.join(refs_path, self._name)
+        md5_file = open(path + '.md5', 'w')
+
+        for entry in os.listdir(refs_path):
+            if not entry.startswith(self._name) or entry.endswith('.md5'):
+                continue
+            ref_path = os.path.join(refs_path, entry)
+            f = open(ref_path, 'r')
+            md5_file.write("%s %s\n" % (md5(f.read()).hexdigest(), ref_path))
+            f.close()
+            if delete_refs:
+                os.remove(ref_path)
+
+        md5_file.close()
+
+    def compare_checksums(self, refs_path, out_path, remove_results = True, create_diffs = True):
+        retval = True
+
+        md5_path = os.path.join(refs_path, self._name)
+        md5_file = open(md5_path + '.md5', 'r')
+        tests = os.listdir(out_path)
+
+        for line in md5_file.readlines():
+            md5sum, ref_path = line.strip('\n').split(' ')
+            basename = os.path.basename(ref_path)
+            if not basename in tests:
+                retval = False
+                print "%s found in md5 ref file but missing in output dir %s" % (basename, out_path)
+                continue
+
+            result_path = os.path.join(out_path, basename)
+            f = open(result_path, 'r')
+            matched = md5sum == md5(f.read()).hexdigest()
+            f.close()
+            if matched:
+                if remove_results:
+                    os.remove(result_path)
+            else:
+                print "Differences found in %s" % (basename)
+                if create_diffs:
+                    if not os.path.exists(ref_path):
+                        print "Reference file %s not found, skipping diff for %s" % (ref_path, result_path)
+                    else:
+                        try:
+                            self._create_diff(ref_path, result_path)
+                        except NotImplementedError:
+                            # Diff not supported by backend
+                            pass
+                retval = False
+        md5_file.close()
+
+        return retval
+
+    def has_md5(self, test_path):
+        return os.path.exists(os.path.join(test_path, self._name + '.md5'))
+
+    def is_crashed(self, test_path):
+        return os.path.exists(os.path.join(test_path, self._name + '.crashed'))
+
+    def is_failed(self, test_path):
+        failed_path = os.path.join(test_path, self._name + '.failed')
+        if not os.path.exists(failed_path):
+            return 0
+
+        f = open(failed_path, 'r')
+        status = int(f.read())
+        f.close()
+
+        return status
+
+    def has_stderr(self, test_path):
+        return os.path.exists(os.path.join(test_path, self._name + '.stderr'))
+
+    def __create_stderr_file(self, stderr, out_path):
+        if not stderr:
+            return
+        stderr_file = open(out_path + '.stderr', 'w')
+        stderr_file.write(stderr)
+        stderr_file.close()
+
+    def __create_failed_file_if_needed(self, status, out_path):
+        if os.WIFEXITED(status) or os.WEXITSTATUS(status) == 0:
+            return False
+
+        failed_file = open(out_path + '.failed', 'w')
+        failed_file.write("%d" % (os.WEXITSTATUS(status)))
+        failed_file.close()
+
+        return True
+
+    def _check_exit_status(self, p, out_path):
+        status = p.wait()
+
+        stderr = p.stderr.read()
+        self.__create_stderr_file(stderr, out_path)
+
+        if not os.WIFEXITED(status):
+            open(out_path + '.crashed', 'w').close()
+            return False
+
+        if self.__create_failed_file_if_needed(status, out_path):
+            return False
+
+        return True
+
+    def _check_exit_status2(self, p1, p2, out_path):
+        status1 = p1.wait()
+        status2 = p2.wait()
+
+        p1_stderr = p1.stderr.read()
+        p2_stderr = p2.stderr.read()
+        if p1_stderr or p2_stderr:
+            self.__create_stderr_file(p1_stderr + p2_stderr, out_path)
+
+        if not os.WIFEXITED(status1) or not os.WIFEXITED(status2):
+            open(out_path + '.crashed', 'w').close()
+            return False
+
+        if self.__create_failed_file_if_needed(status1, out_path):
+            return False
+        if self.__create_failed_file_if_needed(status2, out_path):
+            return False
+
+        return True
+
+    def _diff_png(self, ref_path, result_path):
+        try:
+            import Image, ImageChops
+        except ImportError:
+            raise NotImplementedError
+
+        ref = Image.open(ref_path)
+        result = Image.open(result_path)
+        diff = ImageChops.difference(ref, result)
+        diff.save(result_path + '.diff', 'png')
+
+    def _create_diff(self, ref_path, result_path):
+        raise NotImplementedError
+
+    def create_refs(self, doc_path, refs_path):
+        raise NotImplementedError
+
+_backends = {}
+def register_backend(backend_name, backend_class):
+    _backends[backend_name] = backend_class
+
+def _get_backend(backend_name):
+    if backend_name not in _backends:
+        try:
+            __import__('backends.%s' % backend_name)
+        except ImportError:
+            pass
+
+    if backend_name not in _backends:
+        raise UnknownBackendError('Backend %s does not exist' % backend_name)
+
+    return _backends[backend_name]
+
+def get_backend(backend_name):
+    backend_class = _get_backend(backend_name)
+    return backend_class(backend_name)
+
+def get_all_backends():
+    backends = []
+
+    thisdir = os.path.abspath(os.path.dirname(__file__))
+    for fname in os.listdir(os.path.join(thisdir)):
+        name, ext = os.path.splitext(fname)
+        if not ext == '.py':
+            continue
+        try:
+            __import__('backends.%s' % name)
+        except ImportError:
+            continue
+
+        if name in _backends:
+            backends.append(_backends[name](name))
+
+    return backends
diff --git a/regtest/backends/cairo.py b/regtest/backends/cairo.py
new file mode 100644
index 00000000..0ec361f8
--- /dev/null
+++ b/regtest/backends/cairo.py
@@ -0,0 +1,39 @@
+# cairo.py
+#
+# Copyright (C) 2011 Carlos Garcia Campos <carlosgc@gnome.org>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+from backends import Backend, register_backend
+import subprocess
+import os
+
+class Cairo(Backend):
+
+    def __init__(self, name):
+        Backend.__init__(self, name)
+        self._pdftocairo = os.path.join(self._utilsdir, 'pdftocairo');
+
+    def create_refs(self, doc_path, refs_path):
+        out_path = os.path.join(refs_path, 'cairo')
+        p1 = subprocess.Popen([self._pdftocairo, '-cropbox', '-e', '-png', doc_path, out_path], stderr = subprocess.PIPE)
+        p2 = subprocess.Popen([self._pdftocairo, '-cropbox', '-o', '-png', doc_path, out_path], stderr = subprocess.PIPE)
+        return self._check_exit_status2(p1, p2, out_path)
+
+    def _create_diff(self, ref_path, result_path):
+        self._diff_png(ref_path, result_path)
+
+register_backend('cairo', Cairo)
+
diff --git a/regtest/backends/postscript.py b/regtest/backends/postscript.py
new file mode 100644
index 00000000..62360029
--- /dev/null
+++ b/regtest/backends/postscript.py
@@ -0,0 +1,35 @@
+# postscript.py
+#
+# Copyright (C) 2011 Carlos Garcia Campos <carlosgc@gnome.org>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+from backends import Backend, register_backend
+import subprocess
+import os
+
+class PostScript(Backend):
+
+    def __init__(self, name):
+        Backend.__init__(self, name)
+        self._pdftops = os.path.join(self._utilsdir, 'pdftops');
+
+    def create_refs(self, doc_path, refs_path):
+        out_path = os.path.join(refs_path, 'postscript')
+        p = subprocess.Popen([self._pdftops, doc_path, out_path + '.ps'], stderr = subprocess.PIPE)
+        return self._check_exit_status(p, out_path)
+
+register_backend('postscript', PostScript)
+
diff --git a/regtest/backends/splash.py b/regtest/backends/splash.py
new file mode 100644
index 00000000..bc5e448f
--- /dev/null
+++ b/regtest/backends/splash.py
@@ -0,0 +1,39 @@
+# splash.py
+#
+# Copyright (C) 2011 Carlos Garcia Campos <carlosgc@gnome.org>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+from backends import Backend, register_backend
+import subprocess
+import os
+
+class Splash(Backend):
+
+    def __init__(self, name):
+        Backend.__init__(self, name)
+        self._pdftoppm = os.path.join(self._utilsdir, 'pdftoppm');
+
+    def create_refs(self, doc_path, refs_path):
+        out_path = os.path.join(refs_path, 'splash')
+        p1 = subprocess.Popen([self._pdftoppm, '-cropbox', '-e', '-png', doc_path, out_path], stderr = subprocess.PIPE)
+        p2 = subprocess.Popen([self._pdftoppm, '-cropbox', '-o', '-png', doc_path, out_path], stderr = subprocess.PIPE)
+        return self._check_exit_status2(p1, p2, out_path)
+
+    def _create_diff(self, ref_path, result_path):
+        self._diff_png(ref_path, result_path)
+
+register_backend('splash', Splash)
+
diff --git a/regtest/backends/text.py b/regtest/backends/text.py
new file mode 100644
index 00000000..14e7c030
--- /dev/null
+++ b/regtest/backends/text.py
@@ -0,0 +1,48 @@
+# text.py
+#
+# Copyright (C) 2011 Carlos Garcia Campos <carlosgc@gnome.org>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+from backends import Backend, register_backend
+import subprocess
+import os
+
+class Text(Backend):
+
+    def __init__(self, name):
+        Backend.__init__(self, name)
+        self._pdftotext = os.path.join(self._utilsdir, 'pdftotext');
+
+    def create_refs(self, doc_path, refs_path):
+        out_path = os.path.join(refs_path, 'text')
+        p = subprocess.Popen([self._pdftotext, doc_path, out_path + '.txt'], stderr = subprocess.PIPE)
+        return self._check_exit_status(p, out_path)
+
+    def _create_diff(self, ref_path, result_path):
+        import difflib
+
+        ref = open(ref_path, 'r')
+        result = open(result_path, 'r')
+        diff = difflib.unified_diff(ref.readlines(), result.readlines(), ref_path, result_path)
+        ref.close()
+        result.close()
+
+        diff_file = open(result_path + '.diff', 'w')
+        diff_file.writelines(diff)
+        diff_file.close()
+
+register_backend('text', Text)
+
diff --git a/regtest/commands/__init__.py b/regtest/commands/__init__.py
new file mode 100644
index 00000000..93cec6c5
--- /dev/null
+++ b/regtest/commands/__init__.py
@@ -0,0 +1,93 @@
+# commands
+#
+# Copyright (C) 2011 Carlos Garcia Campos <carlosgc@gnome.org>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+import argparse
+
+__all__ = [ 'register_command',
+            'print_help',
+            'run',
+            'UnknownCommandError',
+            'Command' ]
+
+class UnknownCommandError(Exception):
+    '''Unknown command'''
+
+class Command:
+
+    name = None
+    usage_args = '[ options ... ]'
+    description = None
+
+    def __init__(self):
+        self._parser = argparse.ArgumentParser(
+            description = self.description,
+            prog = 'poppler-regtest %s' % (self.name),
+            usage = 'poppler-regtest %s %s' % (self.name, self.usage_args))
+
+    def _get_args_parser(self):
+        return self._parser
+
+    def execute(self, args):
+        ns = self._parser.parse_args(args)
+        self.run(vars(ns))
+
+    def run(self, options):
+        raise NotImplementedError
+
+_commands = {}
+def register_command(command_name, command_class):
+    _commands[command_name] = command_class
+
+def _get_command(command_name):
+    if command_name not in _commands:
+        try:
+            __import__('commands.%s' % command_name)
+        except ImportError:
+            pass
+
+    if command_name not in _commands:
+        raise UnknownCommandError('Invalid %s command' % command_name)
+
+    return _commands[command_name]
+
+def run(args):
+    command_class = _get_command(args[0])
+    command = command_class()
+    command.execute(args[1:])
+
+def print_help():
+    import os
+    thisdir = os.path.abspath(os.path.dirname(__file__))
+
+    for fname in os.listdir(os.path.join(thisdir)):
+        name, ext = os.path.splitext(fname)
+        if not ext == '.py':
+            continue
+        try:
+            __import__('commands.%s' % name)
+        except ImportError:
+            pass
+
+    print "Commands are:"
+    commands = [(x.name, x.description) for x in _commands.values()]
+    commands.sort()
+    for name, description in commands:
+        print "  %-15s %s" % (name, description)
+
+    print
+    print "For more information run 'poppler-regtest --help-command <command>'"
diff --git a/regtest/commands/create-refs.py b/regtest/commands/create-refs.py
new file mode 100644
index 00000000..4c5034b9
--- /dev/null
+++ b/regtest/commands/create-refs.py
@@ -0,0 +1,65 @@
+# create-refs.py
+#
+# Copyright (C) 2011 Carlos Garcia Campos <carlosgc@gnome.org>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+from commands import Command, register_command
+from TestReferences import TestReferences
+from Timer import Timer
+from Config import Config
+import os
+import tempfile
+
+class CreateRefs(Command):
+
+    name = 'create-refs'
+    usage_args = '[ options ... ] documents ... '
+    description = 'Create references for tests'
+
+    def __init__(self):
+        Command.__init__(self)
+        parser = self._get_args_parser()
+        parser.add_argument('--refs-dir',
+                            action = 'store', dest = 'refs_dir', default = os.path.join(tempfile.gettempdir(), 'refs'),
+                            help = 'Directory where the references will be created')
+        parser.add_argument('-f', '--force',
+                            action = 'store_true', dest = 'force', default = False,
+                            help = 'Create references again for tests that already have references')
+        parser.add_argument('-c', '--checksums-only',
+                            action = 'store_true', dest = 'checksums_only', default = False,
+                            help = 'Leave only checksum files in references dir, other files will be deleted')
+        parser.add_argument('documents', nargs='*')
+
+    def run(self, options):
+        config = Config()
+        config.force = options['force']
+        config.checksums_only = options['checksums_only']
+
+        t = Timer()
+        for doc in options['documents']:
+            if os.path.isdir(doc):
+                docs_dir = doc
+            else:
+                docs_dir = os.path.dirname(doc)
+
+            refs = TestReferences(docs_dir, options['refs_dir'])
+            if doc == docs_dir:
+                refs.create_refs()
+            else:
+                refs.create_refs_for_file(os.path.basename(doc))
+        print "Refs created in %s" % (t.elapsed_str())
+
+register_command('create-refs', CreateRefs)
diff --git a/regtest/commands/run-tests.py b/regtest/commands/run-tests.py
new file mode 100644
index 00000000..b97b34b3
--- /dev/null
+++ b/regtest/commands/run-tests.py
@@ -0,0 +1,69 @@
+# run-tests.py
+#
+# Copyright (C) 2011 Carlos Garcia Campos <carlosgc@gnome.org>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+from commands import Command, register_command
+from TestRun import TestRun
+from Timer import Timer
+from Config import Config
+import os
+import tempfile
+
+class RunTests(Command):
+
+    name = 'run-tests'
+    usage_args = '[ options ... ] documents ... '
+    description = 'Run tests for documents'
+
+    def __init__(self):
+        Command.__init__(self)
+        parser = self._get_args_parser()
+        parser.add_argument('--refs-dir',
+                            action = 'store', dest = 'refs_dir', default = os.path.join(tempfile.gettempdir(), 'refs'),
+                            help = 'Directory containing the references')
+        parser.add_argument('-o', '--out-dir',
+                            action = 'store', dest = 'out_dir', default = os.path.join(tempfile.gettempdir(), 'out'),
+                            help = 'Directory where test results will be created')
+        parser.add_argument('--keep-results',
+                            action = 'store_true', dest = 'keep_results', default = False,
+                            help = 'Do not remove result files for passing tests')
+        parser.add_argument('--create-diffs',
+                            action = 'store_true', dest = 'create_diffs', default = False,
+                            help = 'Create diff files for failed tests')
+        parser.add_argument('documents', nargs='*')
+
+    def run(self, options):
+        config = Config()
+        config.keep_results = options['keep_results']
+        config.create_diffs = options['create_diffs']
+
+        t = Timer()
+        for doc in options['documents']:
+            if os.path.isdir(doc):
+                docs_dir = doc
+            else:
+                docs_dir = os.path.dirname(doc)
+
+            tests = TestRun(docs_dir, options['refs_dir'], options['out_dir'])
+            if doc == docs_dir:
+                tests.run_tests()
+            else:
+                tests.run_test(os.path.basename(doc))
+            tests.summary()
+        print "Tests run in %s" % (t.elapsed_str())
+
+register_command('run-tests', RunTests)
diff --git a/regtest/main.py b/regtest/main.py
new file mode 100644
index 00000000..5784f5aa
--- /dev/null
+++ b/regtest/main.py
@@ -0,0 +1,77 @@
+# main.py
+#
+# Copyright (C) 2011 Carlos Garcia Campos <carlosgc@gnome.org>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+import sys
+import argparse
+import commands
+import backends
+import os
+from Config import Config
+
+class ListAction(argparse.Action):
+    def __call__(self, parser, namespace, values, option_string = None):
+        setattr(namespace, self.dest, values.split(','))
+
+class HelpAction(argparse.Action):
+    def __call__(self, parser, namespace, values, option_string = None):
+        if option_string == '--help-command':
+            commands.run([values, '--help'])
+            sys.exit(0)
+
+        parser.print_help()
+        commands.print_help()
+
+        sys.exit(0)
+
+def main(args):
+    parser = argparse.ArgumentParser(
+        description = 'Poppler regression tests',
+        prog = 'poppler-regtest',
+        usage = '%(prog)s [options ...] command [command-options ...] tests ...',
+        add_help = False)
+    parser.add_argument('-h', '--help',
+                        action = HelpAction, nargs = 0)
+    parser.add_argument('--help-command', metavar = 'COMMAND',
+                        action = HelpAction,
+                        help = 'Show help for a given command')
+    parser.add_argument('--utils-dir', action = 'store', dest = 'utils_dir', default = os.path.abspath("../utils"),
+                        help = 'Directory of poppler utils used for the tests')
+    parser.add_argument('-b', '--backends',
+                        action = ListAction, dest = 'backends',
+                        help = 'List of backends that will be used (separated by comma)')
+
+    ns, args = parser.parse_known_args(args)
+    if not args:
+        parser.print_help()
+        sys.exit(0)
+
+    Config(vars(ns))
+    try:
+        commands.run(args)
+    except commands.UnknownCommandError:
+        sys.stderr.write("Unknown command: %s\n" % (args[0]))
+        commands.print_help()
+        sys.exit(1)
+    except backends.UnknownBackendError, e:
+        sys.stderr.write(str(e) + "\n")
+        sys.stdout.write("Backends are: %s\n" % (", ".join([backend.get_name() for backend in backends.get_all_backends()])))
+        sys.exit(1)
+
+if __name__ == '__main__':
+    import sys
+    main(sys.argv[1:])
diff --git a/regtest/poppler-regtest b/regtest/poppler-regtest
new file mode 100755
index 00000000..fb1e1261
--- /dev/null
+++ b/regtest/poppler-regtest
@@ -0,0 +1,6 @@
+#!/usr/bin/env python
+
+import sys
+import main
+
+main.main(sys.argv[1:])