[Zope-CVS] CVS: Products/FileCacheManager - FileCache.py:1.4

Chris McDonough chrism at plope.com
Sat Aug 28 20:58:29 EDT 2004


Update of /cvs-repository/Products/FileCacheManager
In directory cvs.zope.org:/tmp/cvs-serv6020

Modified Files:
	FileCache.py 
Log Message:
Add debugging flag.

Add user-settable tempfile path.

Improve some log messages.

At transaction commit, when file is moved to its rightful place, make all containing directories as necessary.

The CacheEntry's sortKey is now based on its timestamp.  If it isn't, cache operations happen in an indeterminate order.  This is problematic because one cache operation could delete a file while the other one creates the same file (ala ZCacheable_invalidate, then ZCacheable_set), and you want them to happen in the order that the programmer intended.


=== Products/FileCacheManager/FileCache.py 1.3 => 1.4 ===
--- Products/FileCacheManager/FileCache.py:1.3	Thu Aug 19 15:37:30 2004
+++ Products/FileCacheManager/FileCache.py	Sat Aug 28 20:58:29 2004
@@ -13,6 +13,8 @@
 ##############################################################################
 
 import os
+import sys
+import time
 import tempfile
 import md5
 
@@ -28,8 +30,11 @@
      SecureModuleImporter
 from Products.PageTemplates.TALES import CompilerError
 
+_debug = 0
+
 class FileCache(RAMCache):
     """ A cache that caches rendered content to the filesystem """
+    _tempfile_path = ''
 
     def __init__(self, path='/tmp'):
         # self.cache maps physical ZODB paths to disk files.
@@ -38,6 +43,7 @@
         self.setDir(path)
         self._makeDirs()
         self._naming_expr = None
+        self._tempfile_path = ''
 
     def _fileName(self, ob):
         """ Compute a filename based on an MD5 hash: doesn't preserve
@@ -67,31 +73,37 @@
 
     def setNamingExpression(self, naming_expression=''):
         """ Set a new file naming expression """
-        msg = ''
-
         if naming_expression:
-            try:
-                self._naming_expr = getEngine().compile(naming_expression)
-            except CompilerError:
-                msg = 'Invalid TAL expression "%s"' % naming_expression
+            self._naming_expr = getEngine().compile(naming_expression)
         else:
             self._naming_expr = None
 
-        return msg
+    def setTempfileDir(self, path=''):
+        self._tempfile_path = path
 
     def getDir(self):
         """ Retrieve the filesystem directory used for caching """
         return self._dir
 
+    def getTempfileDir(self):
+        """ Retrieve the filesystem directory used for caching """
+        return self._tempfile_path
+
     def setDir(self, dir):
         """ Set the filesystem diriectory to use for caching """
         self._dir = dir
         self._makeDirs()
 
+    def setTempfileDir(self, dir):
+        """ Retrieve the filesystem directory used for caching """
+        self._tempfile_path = dir
+
     def ZCache_invalidate(self, ob):
         """ Invalidate cache entries that apply to ob """
+        if _debug:
+            print "ZCache_invalidate called"
         fname = self._fileName(ob)
-        cache_entry = FileCacheEntry(fname)
+        cache_entry = FileCacheEntry(fname, self._tempfile_path)
         cache_entry.remove()
 
     def ZCache_get( self, ob, view_name='', keywords=None
@@ -103,29 +115,32 @@
         try:
             fiter = filestream_iterator(fname, 'rb')
         except IOError:
-            # couldn't get the actual cache
-            zLOG.LOG('FileCacheManager', zLOG.INFO,
-                     'Failed to retrieve cache for %s' % \
-                     '/'.join(ob.getPhysicalPath())
-                     )
+            # couldn't get the actual cached file
+            msg = ('Failed to retrieve cached file for "%s" using "%s" '
+                   'as a filename' % ('/'.join(ob.getPhysicalPath()), fname))
+            zLOG.LOG('FileCacheManager', zLOG.INFO, msg, error=sys.exc_info())
             return default
 
+        if _debug:
+            print ("returning filestream iterator for %s" %
+                   '/'.join(ob.getPhysicalPath()))
         return fiter
 
     def ZCache_set( self, ob, data=None, view_name=''
                   , keywords=None, mtime_func=None
                   ):
         """ Sets a cache entry. """
+        if _debug:
+            print "ZCache_set called"
         fname = self._fileName(ob)
 
         if data is None:
             # maybe it's a File or an Image, grab the data
             data = ob.data
 
-        cache_entry = FileCacheEntry(fname)
+        cache_entry = FileCacheEntry(fname, self._tempfile_path)
         cache_entry.write(data)
 
-
     def _makeDirs(self):
         """ Make sure we have somewhere to put files. """
         chars = '0123456789abcdef'
@@ -135,22 +150,29 @@
                 dirpath = os.path.join(self._dir, '%s%s' % (char, char2))
                 try:
                     os.makedirs(dirpath)
-                except OSError:
-                    zLOG.LOG('FileCacheManager', zLOG.PROBLEM,
-                             'unable to create directory %s' % dirpath)
+                except OSError, why:
+                    # it might have already been created
+                    if not os.path.isdir(dirpath):
+                        zLOG.LOG('FileCacheManager', zLOG.PROBLEM,
+                                 'unable to create directory %s' % dirpath,
+                                 error=sys.exc_info())
 
 
 class FileCacheEntry:
     """ Class representing a cache file """
 
-    def __init__(self, path):
+    makedirs = True # make directories as necessary
+
+    def __init__(self, path, tmpdir):
         """ Instantiate a new instance """
         self.path = path
+        self.tmpdir = tmpdir
         self.temppath = None
         self.writelock = allocate_lock()
         self.op_remove = 0
         self.op_write = 0
         self._transaction_done = 0
+        self.timestamp = time.time()
 
         # Register as Transaction Manager so my transaction hooks get called.
         get_transaction().register(self)
@@ -165,7 +187,10 @@
         """ Write a cache file to disk """
         try:
             # Open a tempfile to safely dump data into
-            fd, tempname = tempfile.mkstemp()
+            if self.tmpdir:
+                fd, tempname = tempfile.mkstemp(dir=self.tmpdir)
+            else:
+                fd, tempname = tempfile.mkstemp()
             self.temppath = tempname
 
             # isinstance won't work on extension class
@@ -240,11 +265,13 @@
                     self.writelock.acquire()
                     if os.path.exists(self.path): # XXX race?
                         os.remove(self.path)
-                except IOError, msg:
+                        if _debug:
+                            print "removed %s" % self.path
+                except IOError:
                      zLOG.LOG( 'FileCacheManager'
                              , zLOG.ERROR
-                             , 'IOError removing file'
-                             , error=msg
+                             , 'IOError removing file "%s"' % self.path
+                             , error=sys.exc_info()
                              )
             finally:
                 self.writelock.release()
@@ -254,14 +281,42 @@
             # Now rename the tempfile to reflect the desired name.
             # This may fail if they are not on the same filesystem.
             try:
-                self.writelock.acquire()
                 try:
+                    self.writelock.acquire()
+                    parent = os.path.realpath(os.path.dirname(self.path))
+                    if not os.path.isdir(parent) and os.path.exists(parent):
+                        zLOG.LOG(
+                            'FileCacheManager',
+                            zLOG.ERROR,
+                            'IOError renaming file "%s" to "%s"' % (
+                            self.temppath, self.path),
+                            'The destination path exists'
+                            )
+                        return
+                    
+                    if self.makedirs and not os.path.isdir(parent):
+                        try:
+                            os.makedirs(parent)
+                        except IOError:
+                            zLOG.LOG(
+                                'FileCacheManager',
+                                zLOG.ERROR,
+                                'IOError making dirs for file "%s"'% self.path,
+                                error=sys.exc_info()
+                                )
+                            return
+                            
+                    # XXX Windows will fail here if target path exists, shrug
                     os.rename(self.temppath, self.path)
-                except OSError:
-                    # Windows fails if fname exists.
-                    # If this doesn't work, tough noogies
-                    os.unlink(self.path)
-                    os.rename(tempname, self.path)
+                    if _debug:
+                        print "wrote %s" % self.path
+                except IOError:
+                     zLOG.LOG( 'FileCacheManager'
+                             , zLOG.ERROR
+                             , 'IOError renaming file "%s" to "%s"' % (
+                                        self.temppath, self.path)
+                             , error=sys.exc_info()
+                             )
             finally:
                 self.writelock.release()
                 self.temppath = None
@@ -270,8 +325,9 @@
 
     def sortKey(self, *ignored):
         """ The sortKey method is used for recent ZODB compatibility which
-            needs to have a known commit order for lock acquisition.
-            I don't care about commit order, so return the constant 1
+            needs to have a known commit order for lock acquisition.  We use
+            our timestamp, which will ensure that the order in which file
+            cache entries are created will be the way the operations they
+            imply are played back.
         """
-        return 1
-
+        return self.timestamp



More information about the Zope-CVS mailing list