Fixed Bug where long outputs to stderr during build process caused a hang

fdroidserver/build.py

@@ -27,12 +27,36 @@ import tarfile
 import traceback
 import time
 import json
+import Queue
+import threading
 from optparse import OptionParser
 
 import common
 from common import BuildException
 from common import VCSException
 
+class AsynchronousFileReader(threading.Thread):
+    '''
+    Helper class to implement asynchronous reading of a file
+    in a separate thread. Pushes read lines on a queue to
+    be consumed in another thread.
+    '''
+ 
+    def __init__(self, fd, queue):
+        assert isinstance(queue, Queue.Queue)
+        assert callable(fd.readline)
+        threading.Thread.__init__(self)
+        self._fd = fd
+        self._queue = queue
+ 
+    def run(self):
+        '''The body of the tread: read lines and put them on the queue.'''
+        for line in iter(self._fd.readline, ''):
+            self._queue.put(line)
+ 
+    def eof(self):
+        '''Check whether there is no more content to expect.'''
+        return not self.is_alive() and self._queue.empty()
 
 def get_builder_vm_id():
     vd = os.path.join('builder', '.vagrant')
@@ -448,20 +472,36 @@ def build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_d
 
         p = subprocess.Popen(['bash', '-x', '-c', cmd], cwd=root_dir,
                 stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-        for line in iter(p.stdout.readline, ''):
+        
-            if verbose:
+        stdout_queue = Queue.Queue()
-                # Output directly to console
+        stdout_reader = AsynchronousFileReader(p.stdout, stdout_queue)
-                sys.stdout.write(line)
+        stdout_reader.start()
-                sys.stdout.flush()
+        stderr_queue = Queue.Queue()
-            else:
+        stderr_reader = AsynchronousFileReader(p.stderr, stderr_queue)
-                output += line
+        stderr_reader.start()
-        for line in iter(p.stderr.readline, ''):
+        
-            if verbose:
+        # Check the queues if we received some output (until there is nothing more to get).
-                # Output directly to console
+        while not stdout_reader.eof() or not stderr_reader.eof():
-                sys.stdout.write(line)
+            # Show what we received from standard output.
-                sys.stdout.flush()
+            while not stdout_queue.empty():
-            else:
+                line = stdout_queue.get()
-                error += line
+                if verbose:
+                    # Output directly to console
+                    sys.stdout.write(line)
+                    sys.stdout.flush()
+                else:
+                    error += line
+ 
+            # Show what we received from standard error.
+            while not stderr_queue.empty():
+                line = stderr_queue.get()
+                if verbose:
+                    # Output directly to console
+                    sys.stdout.write(line)
+                    sys.stdout.flush()
+                else:
+                    error += line
+
         p.communicate()
         if p.returncode != 0:
             raise BuildException("Error running build command for %s:%s" %