Python memoize function calls individually
# Functions are memoized using separate files, allowing parallel execution / use on cluster.
import os, pickle
def memoize(func):
def decorated(*args, **kwargs):
if not os.path.exists('_cache'):
os.makedirs('_cache')
digest = hashlib.md5(pickle.dumps(args)).hexdigest()
cache_fname = '_cache/' + digest + ".pkl"
if os.path.exists(cache_fname):
with open(cache_fname) as f:
cache = pickle.load(f)
else:
cache = func(*args)
# update the cache file
with open(cache_fname, 'wb') as f:
pickle.dump(cache, f)
return cache
return decorated