sumit
2/13/2020 - 2:34 AM

parallel processing in for loop python

### METHOD1
from tqdm.contrib.concurrent import process_map,thread_map
fund_matrices = process_map(get_F_matrix, points, max_workers=10)
fund_matrices = thread_map(get_F_matrix, points, max_workers=4)

### METHOD2
from multiprocessing import Pool
# from multiprocessing.pool import ThreadPool as Pool # use this if you are pooling in class https://stackoverflow.com/questions/8804830/python-multiprocessing-picklingerror-cant-pickle-type-function
import os

def process_image(name):
    print(name)
    return

if __name__ == '__main__':
    data_inputs = ['a','b','c']
    pool = Pool(os.cpu_count())                         # Create a multiprocessing Pool
    try:
        pool.map(process_image, data_inputs)  # process data_inputs iterable with pool
    finally: # To make sure processes are closed in the end, even if errors happen
        pool.close()
        pool.join()