seljydef
1/22/2019 - 6:29 AM

Python data converter template

Python data converter template

""" Convert each record """
import csv
import sys
from functools import partial


INPUT_FIELDS = (
    'col1',
    'col2',
    'col3',
)

OUTPUT_FIELDS = (
    'col4',
    'col5',
)


def create_reader():
    """ Create DictReader. """
    return csv.DictReader(sys.stdin, fieldnames=INPUT_FIELDS)


def create_writer():
    """ Create DictWriter. """
    return csv.DictWriter(sys.stdout, fieldnames=OUTPUT_FIELDS)


def run_pipeline(steps, rows):
    """ process steps. """
    tmp = rows
    for step in steps:
        tmp = step(tmp)


def convert(rows):
    """
    DO SOMETHING.
    """
    return rows


def fit_to_output_format(rows):
    """ Fit to the output format. """

    for row in rows:
        yield {
            'row4': row['row1'],
            'row5': row['row2'],

        }


def process(reader, writer):
    """ Main process """

    steps = (
        convert,
        fit_to_output_format,
        partial(filter, lambda x: x['row1'] != ''),
        writer.writerows,
        )
    run_pipeline(steps, reader)


def main():
    """ Main """

    reader = create_reader()
    writer = create_writer()

    process(reader, writer)


if __name__ == '__main__':
    main()