josiekre
8/21/2019 - 9:26 PM

Capture subprocess output with logging module in Python

Capture subprocess output with logging module in Python

import logging
import subprocess
import time


def log_subprocess_output(pid, process, logger):
    while True:
        output = process.stdout.readline().decode()
        if output:
            logger.debug('PID: ' + pid + ' ' + output)
        else:
            break


def main(logger):
    all_subprocesses = []
    iter_list = ['2', '4', '6']
    for i in iter_list:
        logger.info('Starting Rscript for %s', i)
        all_subprocesses.append(
            # Use Popen() over run() for running in parallel loops as run() is
            # blocking
            subprocess.Popen(
                ['Rscript', 'testing_subprocess.R'],
                stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
                check=True
            )
        )

    # Wait for all subprocesses from loop to finish
    exit_codes = [p.wait() for process in all_subprocesses]

    # Log everything
    for i, process in enumerate(all_subprocesses):
        log_subprocess_output(iter_list[i], process, logger)


if __name__ == '__main__':
    # Create logger ---
    logger = logging.getLogger('test-logging')
    logger.setLevel(logging.DEBUG)
    # Create file handler
    log_path = 'log/debug-subprocess-logging_' + str(int(time.time())) + '.log'
    fh = logging.FileHandler(log_path)
    fh.setLevel(logging.DEBUG)
    # Create console handler
    ch = logging.StreamHandler()
    ch.setLevel(logging.DEBUG)
    # Create formatter and add it to the handlers
    formatter = logging.Formatter('%(asctime)s %(name)-12s %(levelname)-8s %(message)s')
    fh.setFormatter(formatter)
    ch.setFormatter(formatter)
    # Add the handlers to the logger
    logger.addHandler(fh)
    logger.addHandler(ch)

    try:
        main(logger)
    except Exception:
        logger.exception("Fatal error in main()")
print("Going to sleep...")

for (i in seq(1, 10000000, 1)) {
  j = 1 + i
}

j