summaryrefslogtreecommitdiff
path: root/regtest
diff options
context:
space:
mode:
authorCarlos Garcia Campos <carlosgc@gnome.org>2013-11-29 10:07:16 +0100
committerCarlos Garcia Campos <carlosgc@gnome.org>2013-11-29 10:07:16 +0100
commit817cc333ca8009998f2099583fd0a2fc703f3db3 (patch)
tree1f7bb8a9d36f6175f734d73cd36ce7433421753c /regtest
parentf8f82f1cc3a948239a05d7762210a3f244299db6 (diff)
regtest: Do not buffer stderr output
Some buggy documents can produce a huge stderr output because of parsing errors or whatever. We could give a file directly to Popen to write the stderr file, but we only want to create the file when there's output, because it's what we use to know whether the command produced output or not. So, instead of buffering the whole output and then write it to the file, now we read from the pipe while the command is running, writing the output in chunks to the file. This improves a lot the memory consumption when running some tests.
Diffstat (limited to 'regtest')
-rw-r--r--regtest/backends/__init__.py40
1 files changed, 30 insertions, 10 deletions
diff --git a/regtest/backends/__init__.py b/regtest/backends/__init__.py
index aa120228..b57d8aa4 100644
--- a/regtest/backends/__init__.py
+++ b/regtest/backends/__init__.py
@@ -18,6 +18,7 @@
import hashlib
import os
+import select
import shutil
import errno
from Config import Config
@@ -193,13 +194,6 @@ class Backend:
return False
return os.path.exists(test_result + self._diff_ext)
- def __create_stderr_file(self, stderr, out_path):
- if not stderr:
- return
- stderr_file = open(out_path + '.stderr', 'wb')
- stderr_file.write(stderr)
- stderr_file.close()
-
def __create_failed_file_if_needed(self, status, out_path):
if os.WIFEXITED(status) or os.WEXITSTATUS(status) == 0:
return False
@@ -210,10 +204,36 @@ class Backend:
return True
- def _check_exit_status(self, p, out_path):
- stderr = p.stderr.read()
- self.__create_stderr_file(stderr, out_path)
+ def __redirect_stderr_to_file(self, fd, out_path):
+ stderr_file = None
+ read_set = [fd]
+ while read_set:
+ try:
+ rlist, wlist, xlist = select.select(read_set, [], [])
+ except select.error as e:
+ continue
+
+ if fd in rlist:
+ try:
+ chunk = os.read(fd, 1024)
+ except OSError as e:
+ if e.errno == errno.EIO:
+ # Child process finished.
+ chunk = ''
+ else:
+ raise e
+ if chunk:
+ if stderr_file is None:
+ stderr_file = open(out_path + '.stderr', 'wb')
+ stderr_file.write(chunk)
+ else:
+ read_set.remove(fd)
+ if stderr_file is not None:
+ stderr_file.close()
+
+ def _check_exit_status(self, p, out_path):
+ self.__redirect_stderr_to_file(p.stderr.fileno(), out_path)
status = p.wait()
if not os.WIFEXITED(status):