Cache Handling Wrapper for Slow File Processing Functions (Python recipe) by Zoran Isailovski

ActiveState Code (http://code.activestate.com/recipes/412719/)

# *** Put this in "fcache.py" ***

import sys, os
try:
   import cPickle as pickle
except ImportError:
   import pickle

######################################################################
## 
## File Utilities
## 
######################################################################

def ftime(filename, default=-sys.maxint):
   if os.path.isfile(filename):
      return os.path.getmtime(filename)
   return default

def fisnewer(path, thenpath):
   return ftime(path) > ftime(thenpath)

def fstem(filename):
   return os.path.splitext(os.path.basename(filename))[0]

def fpickle(filename, content):
   if __debug__: print '==> pickling content to %s' % fstem(filename)
   file = open(filename, 'wb')
   try:
      try:
         pickle.dump(content, file, True)
      finally:
         file.close()
   except:
      os.remove(filename)
      raise

def funpickle(filename):
   if __debug__: print '==> unpickling content from %s' % fstem(filename)
   file = open(filename, 'rb')

   try:
      return pickle.load(file)
   finally:
      file.close()

######################################################################
## 
## Introspection Helpers
## 
######################################################################


def definingModuleFile(func):
   return func.func_code.co_filename

def definingModuleName(func):
   return fstem(definingModuleFile(func))

def qualifiedIdentifier(func):
   m = definingModuleName(func)
   try:
      c = func.im_class
   except AttributeError:
      return '%s.%s' % (m, func.__name__)
   else:
      return '%s.%s.%s' % (m, c.__name__, func.__name__)

def defaultCacheDirectory():
   return os.path.join(os.path.dirname(__file__), '@cache')

######################################################################
## 
## Encoding Functions used to generate a cache file name
## 
######################################################################

def hashhex(s):
   return hex(hash(s))[2:]

def md5hex(s):
   import md5
   return md5.new(s).hexdigest()

######################################################################
## 
## Cache Handling
## 
######################################################################

def cacheFileStem(func, args, encode=hashhex):
   id = encode(repr(args))
   return r'%s-%s' % (qualifiedIdentifier(func), id)

def shouldRebuild(target, sources):
   for each in sources:
      if fisnewer(each, target): return True
   return False

class CacheManager:
   def __init__(self, cacheDir=None, cacheFileExt='.cache', encode=hashhex):
      self._dir = cacheDir or defaultCacheDirectory()
      if not os.path.isdir(self._dir): os.makedirs(self._dir)
      self._ext = cacheFileExt
      self._encode = encode
      self._cachefiles = []
   def cacheFilePath(self, func, *args):
      filename = cacheFileStem(func, args, encode=self._encode)
      return os.path.join(self._dir, filename) + self._ext
   def deleteCacheFiles(self):
      for each in self._cachefiles[:]:
         if os.path.isfile(each):
            os.remove(each)
            self._cachefiles.remove(each)
   def wrap(self, func):
      def call(*sources):
         sources = map(os.path.abspath, sources)
         cachefile = self.cacheFilePath(func, *sources)
         if shouldRebuild(cachefile, sources):
            result = func(*sources)
            fpickle(cachefile, result)
            self._cachefiles.append(cachefile)
         else:
            result = funpickle(cachefile)
         return result
      return call


# --------------------------------------------------------------------
# *** Put this in a separate file, say "fcache_test.py" ***

if __name__ == '__main__':
   import sys
   import fcache

   # create source files
   for i in range(1, 5): open('file%d.txt' % i, 'wt').close()

   # define a processing function
   def processFiles(file1, file2):
      s = 'processing files %r and %r ...' % (file1, file2)
      print s
      return s

   # define a processing method
   class FileProcessor:
      def processFiles(self, file1, file2):
         return processFiles(file1, file2)

   processor = FileProcessor()

   # create a cache manager
   cm = fcache.CacheManager()

   # let's have a look at the generated cache file names
   print
   print fcache.cacheFileStem(processFiles, ('file1.txt', 'file2.txt'), encode=fcache.hashhex)
   print fcache.cacheFileStem(processor.processFiles, ('file1.txt', 'file2.txt'), encode=fcache.hashhex)

   # wrap the processing function
   f = cm.wrap(processFiles)

   # see what happens when a function is repeatedly again with the same arguments
   print
   result = f('file1.txt', 'file2.txt'); print 'result:', result
   result = f('file3.txt', 'file4.txt'); print 'result:', result
   result = f('file1.txt', 'file2.txt'); print 'result:', result
   result = f('file1.txt', 'file2.txt'); print 'result:', result

   # delete this sessions cache files, if you don't need them later
   # (normaly, you would leave them for later recycling)
   cm.deleteCacheFiles()

   # delete test source files
   for i in range(1, 5): os.remove('file%d.txt' % i)

# --------------------------------------------------------------------