danielecook
2/2/2017 - 6:41 PM

Python memoize function calls individually

Python memoize function calls individually

# Functions are memoized using separate files, allowing parallel execution / use on cluster.

import os, pickle
def memoize(func):
    def decorated(*args, **kwargs):
        if not os.path.exists('_cache'):
            os.makedirs('_cache')
        digest = hashlib.md5(pickle.dumps(args)).hexdigest()
        cache_fname = '_cache/' + digest + ".pkl"
        if os.path.exists(cache_fname):
            with open(cache_fname) as f:
                cache = pickle.load(f)
        else:
            cache = func(*args)
            # update the cache file
            with open(cache_fname, 'wb') as f:
                pickle.dump(cache, f)
        return cache
    return decorated