We want to pass a Queue to the map method of multiprocessing's Pool.
import multiprocessing as mp
def iqueue(queue):
while not queue.empty():
yield queue.get()
def combine_with_queue(queue):
def f(data):
x = data[0] * data[1]
queue.put(x)
return f
data_set = ((5, 1), (5, 2), (5, 3))
# The Manager instance seems to handle the queue better, in testing.
q = mp.Manager().Queue()
f = combine_with_queue(q)
p = mp.Pool(4)
p.map(f, data_set)
p.close()
p.join()
for i in iqueue(q):
print i