From 148959c1ce38a807a9ea986ca1bf2d336c47ac71 Mon Sep 17 00:00:00 2001 From: Kostis Anagnostopoulos Date: Mon, 26 Sep 2016 01:36:57 +0200 Subject: [PATCH] apveyor, #519: FIX incomplete Popen pump + The code in `_read_lines_from_fno()` was reading the stream only once per invocation, so when input was larger than `mmap.PAGESIZE`, bytes were forgotten in the stream. + Also set deamon-threads. --- git/cmd.py | 18 ++++++++++++++---- git/test/test_git.py | 4 +++- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/git/cmd.py b/git/cmd.py index 1cc656bf5..2e5ce6260 100644 --- a/git/cmd.py +++ b/git/cmd.py @@ -69,7 +69,7 @@ def _bchr(c): # Documentation ## @{ -def handle_process_output(process, stdout_handler, stderr_handler, finalizer): +def 9(process, stdout_handler, stderr_handler, finalizer): """Registers for notifications to lean that process output is ready to read, and dispatches lines to the respective line handlers. We are able to handle carriage returns in case progress is sent by that mean. For performance reasons, we only apply this to stderr. @@ -193,14 +193,24 @@ def _deplete_buffer(fno, handler, buf_list, wg=None): else: # Oh ... probably we are on windows. select.select() can only handle sockets, we have files # The only reliable way to do this now is to use threads and wait for both to finish + def _handle_lines(fd, handler, wg): + for line in fd: + line = line.decode(defenc) + if line and handler: + handler(line) + if wg: + wg.done() + # Since the finalizer is expected to wait, we don't have to introduce our own wait primitive # NO: It's not enough unfortunately, and we will have to sync the threads wg = WaitGroup() - for fno, (handler, buf_list) in fdmap.items(): + for fd, handler in zip((process.stdout, process.stderr), + (stdout_handler, stderr_handler)): wg.add(1) - t = threading.Thread(target=lambda: _deplete_buffer(fno, handler, buf_list, wg)) + t = threading.Thread(target=_handle_lines, args=(fd, handler, wg)) + t.setDaemon(True) t.start() - # end + # NOTE: Just joining threads can possibly fail as there is a gap between .start() and when it's # actually started, which could make the wait() call to just return because the thread is not yet # active diff --git a/git/test/test_git.py b/git/test/test_git.py index 534539d78..1695993ac 100644 --- a/git/test/test_git.py +++ b/git/test/test_git.py @@ -238,7 +238,9 @@ def counter_stderr(line): stdin=None, stdout=subprocess.PIPE, stderr=subprocess.PIPE, - shell=False) + shell=False, + creationflags=Git.CREATE_NO_WINDOW if sys.platform == 'win32' else 0, + ) handle_process_output(proc, counter_stdout, counter_stderr, lambda proc: proc.wait())