Welcome, guest | Sign In | My Account | Store | Cart
# *** Put this in "fcache.py" ***

import sys, os
try:
   
import cPickle as pickle
except ImportError:
   
import pickle

######################################################################
##
## File Utilities
##
######################################################################

def ftime(filename, default=-sys.maxint):
   
if os.path.isfile(filename):
     
return os.path.getmtime(filename)
   
return default

def fisnewer(path, thenpath):
   
return ftime(path) > ftime(thenpath)

def fstem(filename):
   
return os.path.splitext(os.path.basename(filename))[0]

def fpickle(filename, content):
   
if __debug__: print '==> pickling content to %s' % fstem(filename)
   file
= open(filename, 'wb')
   
try:
     
try:
         pickle
.dump(content, file, True)
     
finally:
         file
.close()
   
except:
      os
.remove(filename)
     
raise

def funpickle(filename):
   
if __debug__: print '==> unpickling content from %s' % fstem(filename)
   file
= open(filename, 'rb')

   
try:
     
return pickle.load(file)
   
finally:
      file
.close()

######################################################################
##
## Introspection Helpers
##
######################################################################


def definingModuleFile(func):
   
return func.func_code.co_filename

def definingModuleName(func):
   
return fstem(definingModuleFile(func))

def qualifiedIdentifier(func):
   m
= definingModuleName(func)
   
try:
      c
= func.im_class
   
except AttributeError:
     
return '%s.%s' % (m, func.__name__)
   
else:
     
return '%s.%s.%s' % (m, c.__name__, func.__name__)

def defaultCacheDirectory():
   
return os.path.join(os.path.dirname(__file__), '@cache')

######################################################################
##
## Encoding Functions used to generate a cache file name
##
######################################################################

def hashhex(s):
   
return hex(hash(s))[2:]

def md5hex(s):
   
import md5
   
return md5.new(s).hexdigest()

######################################################################
##
## Cache Handling
##
######################################################################

def cacheFileStem(func, args, encode=hashhex):
   id
= encode(repr(args))
   
return r'%s-%s' % (qualifiedIdentifier(func), id)

def shouldRebuild(target, sources):
   
for each in sources:
     
if fisnewer(each, target): return True
   
return False

class CacheManager:
   
def __init__(self, cacheDir=None, cacheFileExt='.cache', encode=hashhex):
     
self._dir = cacheDir or defaultCacheDirectory()
     
if not os.path.isdir(self._dir): os.makedirs(self._dir)
     
self._ext = cacheFileExt
     
self._encode = encode
     
self._cachefiles = []
   
def cacheFilePath(self, func, *args):
      filename
= cacheFileStem(func, args, encode=self._encode)
     
return os.path.join(self._dir, filename) + self._ext
   
def deleteCacheFiles(self):
     
for each in self._cachefiles[:]:
         
if os.path.isfile(each):
            os
.remove(each)
           
self._cachefiles.remove(each)
   
def wrap(self, func):
     
def call(*sources):
         sources
= map(os.path.abspath, sources)
         cachefile
= self.cacheFilePath(func, *sources)
         
if shouldRebuild(cachefile, sources):
            result
= func(*sources)
            fpickle
(cachefile, result)
           
self._cachefiles.append(cachefile)
         
else:
            result
= funpickle(cachefile)
         
return result
     
return call


# --------------------------------------------------------------------
# *** Put this in a separate file, say "fcache_test.py" ***

if __name__ == '__main__':
   
import sys
   
import fcache

   
# create source files
   
for i in range(1, 5): open('file%d.txt' % i, 'wt').close()

   
# define a processing function
   
def processFiles(file1, file2):
      s
= 'processing files %r and %r ...' % (file1, file2)
     
print s
     
return s

   
# define a processing method
   
class FileProcessor:
     
def processFiles(self, file1, file2):
         
return processFiles(file1, file2)

   processor
= FileProcessor()

   
# create a cache manager
   cm
= fcache.CacheManager()

   
# let's have a look at the generated cache file names
   
print
   
print fcache.cacheFileStem(processFiles, ('file1.txt', 'file2.txt'), encode=fcache.hashhex)
   
print fcache.cacheFileStem(processor.processFiles, ('file1.txt', 'file2.txt'), encode=fcache.hashhex)

   
# wrap the processing function
   f
= cm.wrap(processFiles)

   
# see what happens when a function is repeatedly again with the same arguments
   
print
   result
= f('file1.txt', 'file2.txt'); print 'result:', result
   result
= f('file3.txt', 'file4.txt'); print 'result:', result
   result
= f('file1.txt', 'file2.txt'); print 'result:', result
   result
= f('file1.txt', 'file2.txt'); print 'result:', result

   
# delete this sessions cache files, if you don't need them later
   
# (normaly, you would leave them for later recycling)
   cm
.deleteCacheFiles()

   
# delete test source files
   
for i in range(1, 5): os.remove('file%d.txt' % i)

# --------------------------------------------------------------------

History

  • revision 7 (18 years ago)
  • previous revisions are not available