Last active
April 30, 2016 18:09
-
-
Save HakShak/e6f42a88c7f3fac4a48bf1e0605101f4 to your computer and use it in GitHub Desktop.
Subprocess with stdout and stderr control
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| import subprocess | |
| import logging | |
| import threading | |
| import Queue | |
| import time | |
| def queue_stream(queue, stream): | |
| with stream: | |
| for line in iter(stream.readline, b''): | |
| report = line.rstrip() | |
| if report != "": #fuck off empty lines. | |
| queue.put(report) | |
| #This blocks while trying to stream stdout and stderr to logging/files | |
| def run_process(logPrefix, cmd, workingDirectory=None, logFilePath=None): | |
| logFile = None | |
| if logFilePath: | |
| logFile = open(logFilePath, "w") | |
| try: | |
| logging.debug("Running process: %s", cmd) | |
| p = subprocess.Popen( | |
| cmd, | |
| stdout = subprocess.PIPE, | |
| stderr = subprocess.PIPE, | |
| bufsize = 1, #Only buffer on line so we dont have to wait for all the thigns. | |
| cwd = workingDirectory | |
| ) | |
| outQueue = Queue.Queue() | |
| errQueue = Queue.Queue() | |
| outThread = threading.Thread(target = queue_stream, args = (outQueue, p.stdout)) | |
| outThread.start() | |
| errThread = threading.Thread(target = queue_stream, args = (errQueue, p.stderr)) | |
| errThread.start() | |
| #go until the process returns and both queues are not empty | |
| #this way we don't miss any logs | |
| while p.poll() is None or not outQueue.empty() or not errQueue.empty(): | |
| try: | |
| while not outQueue.empty(): #Drain the queue or else the process will outrun us | |
| outLine = outQueue.get_nowait() | |
| logging.debug('%s: %s' % (logPrefix, outLine)) | |
| if logFile: | |
| logFile.write(outLine) | |
| outQueue.task_done() | |
| except Queue.Empty: | |
| pass | |
| try: | |
| while not errQueue.empty(): #Drain the queue or else the process will outrun us | |
| errLine = errQueue.get_nowait() | |
| logging.error('%s: %s' % (logPrefix, errLine)) | |
| if logFile: | |
| logFile.write(errLine) | |
| errQueue.task_done() | |
| except Queue.Empty: | |
| pass | |
| time.sleep(0.1) #don't gobble CPU | |
| outThread.join(10) | |
| errThread.join(10) | |
| finally: | |
| if logFile: | |
| logFile.close() | |
| return p |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment