Capture subprocess output with logging module in Python
import logging
import subprocess
import time
def log_subprocess_output(pid, process, logger):
while True:
output = process.stdout.readline().decode()
if output:
logger.debug('PID: ' + pid + ' ' + output)
else:
break
def main(logger):
all_subprocesses = []
iter_list = ['2', '4', '6']
for i in iter_list:
logger.info('Starting Rscript for %s', i)
all_subprocesses.append(
# Use Popen() over run() for running in parallel loops as run() is
# blocking
subprocess.Popen(
['Rscript', 'testing_subprocess.R'],
stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
check=True
)
)
# Wait for all subprocesses from loop to finish
exit_codes = [p.wait() for process in all_subprocesses]
# Log everything
for i, process in enumerate(all_subprocesses):
log_subprocess_output(iter_list[i], process, logger)
if __name__ == '__main__':
# Create logger ---
logger = logging.getLogger('test-logging')
logger.setLevel(logging.DEBUG)
# Create file handler
log_path = 'log/debug-subprocess-logging_' + str(int(time.time())) + '.log'
fh = logging.FileHandler(log_path)
fh.setLevel(logging.DEBUG)
# Create console handler
ch = logging.StreamHandler()
ch.setLevel(logging.DEBUG)
# Create formatter and add it to the handlers
formatter = logging.Formatter('%(asctime)s %(name)-12s %(levelname)-8s %(message)s')
fh.setFormatter(formatter)
ch.setFormatter(formatter)
# Add the handlers to the logger
logger.addHandler(fh)
logger.addHandler(ch)
try:
main(logger)
except Exception:
logger.exception("Fatal error in main()")
print("Going to sleep...")
for (i in seq(1, 10000000, 1)) {
j = 1 + i
}
j