From 817cc333ca8009998f2099583fd0a2fc703f3db3 Mon Sep 17 00:00:00 2001
From: Carlos Garcia Campos <carlosgc@gnome.org>
Date: Fri, 29 Nov 2013 10:07:16 +0100
Subject: regtest: Do not buffer stderr output

Some buggy documents can produce a huge stderr output because of parsing
errors or whatever. We could give a file directly to Popen to write the
stderr file, but we only want to create the file when there's output,
because it's what we use to know whether the command produced output or
not. So, instead of buffering the whole output and then write it to the
file, now we read from the pipe while the command is running, writing
the output in chunks to the file. This improves a lot the memory
consumption when running some tests.
---
 regtest/backends/__init__.py | 40 ++++++++++++++++++++++++++++++----------
 1 file changed, 30 insertions(+), 10 deletions(-)

(limited to 'regtest')

diff --git a/regtest/backends/__init__.py b/regtest/backends/__init__.py
index aa120228..b57d8aa4 100644
--- a/regtest/backends/__init__.py
+++ b/regtest/backends/__init__.py
@@ -18,6 +18,7 @@
 
 import hashlib
 import os
+import select
 import shutil
 import errno
 from Config import Config
@@ -193,13 +194,6 @@ class Backend:
             return False
         return os.path.exists(test_result + self._diff_ext)
 
-    def __create_stderr_file(self, stderr, out_path):
-        if not stderr:
-            return
-        stderr_file = open(out_path + '.stderr', 'wb')
-        stderr_file.write(stderr)
-        stderr_file.close()
-
     def __create_failed_file_if_needed(self, status, out_path):
         if os.WIFEXITED(status) or os.WEXITSTATUS(status) == 0:
             return False
@@ -210,10 +204,36 @@ class Backend:
 
         return True
 
-    def _check_exit_status(self, p, out_path):
-        stderr = p.stderr.read()
-        self.__create_stderr_file(stderr, out_path)
+    def __redirect_stderr_to_file(self, fd, out_path):
+        stderr_file = None
+        read_set = [fd]
+        while read_set:
+            try:
+                rlist, wlist, xlist = select.select(read_set, [], [])
+            except select.error as e:
+                continue
+
+            if fd in rlist:
+                try:
+                    chunk = os.read(fd, 1024)
+                except OSError as e:
+                    if e.errno == errno.EIO:
+                        # Child process finished.
+                        chunk = ''
+                    else:
+                        raise e
+                if chunk:
+                    if stderr_file is None:
+                        stderr_file = open(out_path + '.stderr', 'wb')
+                    stderr_file.write(chunk)
+                else:
+                    read_set.remove(fd)
 
+        if stderr_file is not None:
+            stderr_file.close()
+
+    def _check_exit_status(self, p, out_path):
+        self.__redirect_stderr_to_file(p.stderr.fileno(), out_path)
         status = p.wait()
 
         if not os.WIFEXITED(status):
-- 
cgit v1.2.3