[Zope-CVS] CVS: Products/AdaptableStorage/gateway_fs - ShortLivedCache.py:1.1.2.1 FSAutoId.py:1.4.2.5 FSClassificationSection.py:1.7.2.5 FSConnection.py:1.6.2.6 FSDirectoryItems.py:1.8.2.5 public.py:1.3.2.5

Christian Zagrodnick cz@gocept.com
Thu, 6 Feb 2003 08:01:38 -0500


Update of /cvs-repository/Products/AdaptableStorage/gateway_fs
In directory cvs.zope.org:/tmp/cvs-serv14413/gateway_fs

Modified Files:
      Tag: zagy-patches
	FSAutoId.py FSClassificationSection.py FSConnection.py 
	FSDirectoryItems.py public.py 
Added Files:
      Tag: zagy-patches
	ShortLivedCache.py 
Log Message:
merging HEAD into zagy-patches branch

=== Added File Products/AdaptableStorage/gateway_fs/ShortLivedCache.py ===
##############################################################################
#
# Copyright (c) 2003 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL).  A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
"""Simple short-lived object cache.

$Id: ShortLivedCache.py,v 1.1.2.1 2003/02/06 13:01:05 zagy Exp $
"""

from time import time


class ShortLivedCache:

    def __init__(self, lifetime=1):
        # The default lifetime is 1 second.
        self.lifetime = lifetime
        self.data = {}
        self.expiration = time() + lifetime

    def get(self, key, default=None):
        now = time()
        if now >= self.expiration:
            self.data.clear()
            return default
        res = self.data.get(key, default)
        return res

    def set(self, key, value):
        now = time()
        if now >= self.expiration:
            self.data.clear()
            self.expiration = now + self.lifetime
        self.data[key] = value

    def invalidate(self, key):
        try:
            del self.data[key]
        except KeyError:
            pass

    def clear(self):
        self.data.clear()



=== Products/AdaptableStorage/gateway_fs/FSAutoId.py 1.4.2.4 => 1.4.2.5 ===


=== Products/AdaptableStorage/gateway_fs/FSClassificationSection.py 1.7.2.4 => 1.7.2.5 ===
--- Products/AdaptableStorage/gateway_fs/FSClassificationSection.py:1.7.2.4	Tue Feb  4 12:29:36 2003
+++ Products/AdaptableStorage/gateway_fs/FSClassificationSection.py	Thu Feb  6 08:01:05 2003
@@ -16,6 +16,8 @@
 $Id$
 """
 
+import os
+
 from mapper_public import IGateway, FieldSchema
 
 
@@ -50,17 +52,21 @@
                 if '=' in line:
                     k, v = line.split('=', 1)
                     classification[k.strip()] = v.strip()
-        classification['filename'] = self.getIdFrom(event)
+        filename = self.getIdFrom(event)
+        classification['extension'] = os.path.splitext(filename)[1]
         return classification, text.strip()
 
     def store(self, event, state):
         # state is a classification
+        p = event.getKeychain()[-1]
         items = state.items()
         items.sort()
         text = []
         for k, v in items:
-            text.append('%s=%s' % (k, v))
+            if k == 'extension':
+                self.fs_conn.suggestExtension(p, v)
+            else:
+                text.append('%s=%s' % (k, v))
         text = '\n'.join(text)
-        p = event.getKeychain()[-1]
         self.fs_conn.writeSection(p, 'classification', text)
         return text.strip()


=== Products/AdaptableStorage/gateway_fs/FSConnection.py 1.6.2.5 => 1.6.2.6 ===
--- Products/AdaptableStorage/gateway_fs/FSConnection.py:1.6.2.5	Tue Feb  4 12:48:34 2003
+++ Products/AdaptableStorage/gateway_fs/FSConnection.py	Thu Feb  6 08:01:05 2003
@@ -24,6 +24,7 @@
 from interfaces.public import IFSConnection
 from exceptions import FSWriteError
 from mapper_public import ITPCConnection, NoStateFoundError
+from ShortLivedCache import ShortLivedCache
 
 
 # Try to decipher this regular expression ;-)
@@ -34,6 +35,8 @@
 
 NODE_TYPE_SECTION = '@node_type'  # Data is 'f' (file) or 'd' (directory)
 DATA_SECTION = '@data'  # Data is a string (file) or list of names (directory)
+SUGGESTED_EXTENSION_SECTION = '@s_ext'  # The suggested filename extension
+OBJECT_NAMES_SECTION = 'object_names'  # For directories
 
 
 class FSConnection:
@@ -50,16 +53,88 @@
 
     basepath = ''
 
-    def __init__(self, basepath, hidden_filename_prefix='.', 
-            hidden_filenames='$f^'):
-        # $f^ never matches anything
+    def __init__(self, basepath, metadata_prefix='.', hidden_filenames='_'):
         self.basepath = basepath
-        self.hidden_filename_prefix = hidden_filename_prefix
-        self.hidden_filenames = re.compile(hidden_filenames)
+        self.metadata_prefix = metadata_prefix
+        self.hidden_re = re.compile(hidden_filenames)
         self._final = 0
         # _pending holds the data to be written.
         # _pending: { subpath string -> { section_name -> data } }
         self._pending = {}
+        self._props_cache = ShortLivedCache()
+        self._dir_cache = ShortLivedCache()
+
+
+    def computeDirectoryContents(self, path, ignore_error=0):
+        """Computes and returns intermediate directory contents info.
+
+        Returns (filenames, object_names, translations).
+        """
+        res = self._dir_cache.get(path)
+        if res is not None:
+            return res
+
+        filenames = []
+        obj_names = []
+        trans = {}     # { base name -> filename with extension or None }
+        try:
+            fns = os.listdir(path)
+        except OSError:
+            if ignore_error:
+                return (filenames, obj_names, trans)
+        metadata_prefix = self.metadata_prefix
+        for fn in fns:
+            if (not fn
+                or fn.startswith(metadata_prefix)
+                or self.hidden_re.match(fn) is not None):
+                continue
+            filenames.append(fn)
+
+        props = self.getPropertiesFromFile(path)
+        text = props.get(OBJECT_NAMES_SECTION)
+        if text:
+            # Prepare a dictionary of translations.
+            for fn in filenames:
+                if '.' in fn:
+                    base, ext = fn.split('.', 1)
+                    if trans.has_key(base):
+                        # Name collision: two or more files have the same base
+                        # name.  Don't use an extension for this name.
+                        trans[base] = None
+                    else:
+                        trans[base] = fn
+                else:
+                    trans[fn] = None
+            obj_names = [line.strip() for line in text.split('\n')]
+            for obj_name in obj_names:
+                if '.' in obj_name:
+                    base, ext = obj_name.split('.', 1)
+                    trans[base] = None
+
+        res = (filenames, obj_names, trans)
+        self._dir_cache.set(path, res)
+        return res
+        
+
+    def listDirectoryAsMapping(self, path, ignore_error=0):
+        """Returns the translated filenames at path.
+
+        The ignore_error flag makes this method return an empty
+        dictionary if the directory is not found.
+
+        Returns {filename -> obj_name}.
+        """
+        filenames, obj_names, trans = self.computeDirectoryContents(
+            path, ignore_error)
+        res = {}
+        for fn in filenames:
+            res[fn] = fn
+        # Translate names.
+        for obj_name in obj_names:
+            fn = trans.get(obj_name)
+            if fn:
+                res[fn] = obj_name
+        return res
 
 
     def expandPath(self, subpath):
@@ -70,48 +145,45 @@
         else:
             # unchanged.
             path = subpath
+        if not os.path.exists(path):
+            dir_path, obj_name = os.path.split(path)
+            if '.' not in obj_name:
+                # This object might have an automatic filename extension.
+                filenames, obj_names, trans = self.computeDirectoryContents(
+                    dir_path, 1)
+                fn = trans.get(obj_name)
+                if fn is not None:
+                    # Use the filename with an extension.
+                    path = os.path.join(dir_path, fn)
         return path
 
 
     def checkSectionName(self, section_name):
-        assert isinstance(section_name, StringType)
-        assert '[' not in section_name
-        assert ']' not in section_name
-        assert '\n' not in section_name
-        assert section_name != NODE_TYPE_SECTION
-        assert section_name != DATA_SECTION
-
-
-    def _write(self, subpath, section_name, data):
-        # XXX We should be checking for '..'
-        path = self.expandPath(subpath)
-        # Do some early checking.
-        if os.path.exists(path):
-            v = os.access(path, os.W_OK)
-            if not v:
-                raise FSWriteError(
-                    "Can't get write access to %s" % subpath)
-        self.queue(subpath, section_name, data)
+        if (not isinstance(section_name, StringType)
+            or not section_name
+            or '[' in section_name
+            or ']' in section_name
+            or '\n' in section_name
+            or section_name.startswith('@')
+            or section_name == OBJECT_NAMES_SECTION):
+            raise ValueError, section_name
 
 
     def writeSection(self, subpath, section_name, data):
         self.checkSectionName(section_name)
-        self._write(subpath, section_name, data)
+        self.queue(subpath, section_name, data)
 
 
     def writeNodeType(self, subpath, data):
-        path = self.expandPath(subpath)
-        # Do some early checking.
-        if os.path.exists(path):
-            want_dir = (data == 'd')
-            if (want_dir != (not not os.path.isdir(path))):
-                raise FSWriteError(
-                    "Can't mix file and directory at %s" % subpath)
         self.queue(subpath, NODE_TYPE_SECTION, data)
 
 
     def writeData(self, subpath, data):
-        self._write(subpath, DATA_SECTION, data)
+        self.queue(subpath, DATA_SECTION, data)
+
+
+    def suggestExtension(self, subpath, ext):
+        self.queue(subpath, SUGGESTED_EXTENSION_SECTION, ext)
 
 
     def readSection(self, subpath, section_name, default=None):
@@ -133,17 +205,10 @@
         isdir = os.path.isdir(path)
         # Read either the directory listing or the file contents.
         if isdir:
-            names = []
-            prefix = self.hidden_filename_prefix
-            for name in os.listdir(path):
-                if name.startswith(self.hidden_filename_prefix):
-                    continue
-                if self.hidden_filenames.search(name) is not None:
-                    continue
-                names.append(name)
-            # Return a sequence instead of a string.
-            return names
+            # Return a sequence of object names.
+            return self.listDirectoryAsMapping(path).values()
         else:
+            # Return a string.
             try:
                 f = open(path, 'rb')
             except IOError:
@@ -158,24 +223,30 @@
 
     def getPropertiesPath(self, path):
         if os.path.isdir(path):
-            props_fn = os.path.join(path, self.hidden_filename_prefix +
+            props_fn = os.path.join(path, self.metadata_prefix +
                                     'properties')
         else:
             dirname, filename = os.path.split(path)
-            props_fn = os.path.join(dirname, self.hidden_filename_prefix +
+            props_fn = os.path.join(dirname, self.metadata_prefix +
                                     ('%s.properties' % filename))
         return props_fn
 
 
     def getPropertiesFromFile(self, path):
-        """Read a properties file next to path."""
+        """Reads a properties file next to path."""
+        res = self._props_cache.get(path)
+        if res is not None:
+            return res
+
         props_fn = self.getPropertiesPath(path)
 
         try:
             f = open(props_fn, 'rb')
         except IOError:
             # The file is presumably nonexistent
-            return {}
+            res = {}
+            self._props_cache.set(path, res)
+            return res
         try:
             data = f.read()
         finally:
@@ -199,6 +270,7 @@
                 prev_section_name = match.group(1)
                 pos = match.end()
 
+        self._props_cache.set(path, res)
         return res
 
 
@@ -206,8 +278,23 @@
         # sections is a mapping.
         path = self.expandPath(subpath)
         t = sections[NODE_TYPE_SECTION]
-        if t == 'd' and not os.path.exists(path):
-            os.mkdir(path)
+        if not os.path.exists(path):
+            if t == 'd':
+                os.mkdir(path)
+            else:
+                fn = os.path.split(path)[1]
+                if '.' not in fn:
+                    # This object has no extension and doesn't yet exist.
+                    ext = sections.get(SUGGESTED_EXTENSION_SECTION)
+                    if ext:
+                        # Try to use the suggested extension.
+                        if not ext.startswith('.'):
+                            ext = '.' + ext
+                        p = path + ext
+                        if not os.path.exists(p):
+                            # No file is in the way.
+                            # Use the suggested extension.
+                            path = p
         props_fn = self.getPropertiesPath(path)
         items = sections.items()
         items.sort()
@@ -219,38 +306,37 @@
                 elif name == DATA_SECTION:
                     if t == 'd':
                         # Change the list of subobjects.
-                        # Here we only have to delete.
-                        # Subobjects will be created later.
-                        # XXX we might check for dotted names here.
                         self.removeUnlinkedItems(path, data)
+                        writeSection(props_f, OBJECT_NAMES_SECTION,
+                                     '\n'.join(data))
+                        self.disableConflictingExtensions(subpath, data)
+                        self._dir_cache.invalidate(path)
                     else:
-                        # Change file contents.
+                        # Change the file contents.
                         f = open(path, 'wb')
                         try:
                             f.write(data)
                         finally:
                             f.close()
+                elif name == SUGGESTED_EXTENSION_SECTION:
+                    # This doesn't need to be written.
+                    pass
                 else:
-                    props_f.write('[%s]\n' % name)
-                    props_f.write(data.replace('[', '[['))
-                    if data.endswith('\n'):
-                        props_f.write('\n')
-                    else:
-                        props_f.write('\n\n')
+                    writeSection(props_f, name, data)
         finally:
             props_f.close()
+            self._props_cache.invalidate(path)
+            # The file might be new, so invalidate the directory.
+            self._dir_cache.invalidate(os.path.dirname(path))
 
 
     def removeUnlinkedItems(self, path, names):
+        """Removes unused files/subtrees from a directory."""
         linked = {}
         for name in names:
             linked[name] = 1
-        existing = os.listdir(path)
-        prefix = self.hidden_filename_prefix
-        for fn in existing:
-            if (not fn.startswith(prefix)
-                    and self.hidden_filenames.search(fn) is None
-                    and not linked.get(fn)):
+        for fn, obj_name in self.listDirectoryAsMapping(path).items():
+            if not linked.get(obj_name):
                 item_fn = os.path.join(path, fn)
                 if os.path.isdir(item_fn):
                     rmtree(item_fn)
@@ -261,18 +347,50 @@
                         os.remove(item_pfn)
 
 
+    def disableConflictingExtensions(self, subpath, obj_names):
+        """Fixes collisions before writing files in a directory.
+
+        Enforces the rule: if 'foo.*' is in the
+        database, 'foo' may not have an automatic extension.
+        Enforces by removing suggested extensions.
+        """
+        reserved = {}  # { object name without extension -> 1 }
+        for obj_name in obj_names:
+            if '.' in obj_name:
+                base, ext = obj_name.split('.', 1)
+                reserved[base] = 1
+        if not reserved:
+            # No objects have extensions.
+            return
+
+        while subpath.endswith('/'):
+            subpath = subpath[:-1]
+        for obj_name in obj_names:
+            if reserved.has_key(obj_name):
+                # Prevent obj_name from using an automatic extension.
+                child_subpath = '%s/%s' % (subpath, obj_name)
+                self.queue(child_subpath, SUGGESTED_EXTENSION_SECTION,
+                           '', force=1)
+
+
     def beforeWrite(self, items):
+        """Does some early checking while it's easy to bail out.
+
+        This avoids exceptions during the second phase of transaction commit.
+        """
         non_containers = {}
         for subpath, sections in items:
+            path = self.expandPath(subpath)
+            exists = os.path.exists(path)
+            if exists and not os.access(path, os.W_OK):
+                raise FSWriteError(
+                    "Can't get write access to %s" % subpath)
             # type must be provided and must always be either 'd' or 'f'.
             if (not sections.has_key(NODE_TYPE_SECTION)
                 or not sections.has_key(DATA_SECTION)):
                 raise FSWriteError(
                     'Data or node type not specified for %s' % subpath)
             t = sections[NODE_TYPE_SECTION]
-            if t not in ('d', 'f'):
-                raise FSWriteError(
-                    'node type must be "d" or "f" at %s' % subpath)
             dir = os.path.dirname(subpath)
             if non_containers.get(dir):
                 raise FSWriteError(
@@ -283,26 +401,44 @@
                     'The id %s is not allowed.' % (os.path.split(subpath)[1], ))
                    
             if t == 'f':
+                if exists and os.path.isdir(path):
+                    raise FSWriteError(
+                        "Can't write file data to directory at %s"
+                        % subpath)
                 non_containers[subpath] = 1
                 if not isinstance(sections[DATA_SECTION], StringType):
                     raise FSWriteError(
                         'Data for a file must be a string at %s'
                         % subpath)
-            else:
-                if isinstance(sections[DATA_SECTION], StringType):
+            elif t == 'd':
+                if exists and not os.path.isdir(path):
+                    raise FSWriteError(
+                        "Can't write directory contents to file at %s"
+                        % subpath)
+                items = sections[DATA_SECTION]
+                if isinstance(items, StringType):
                     raise FSWriteError(
                         'Data for a directory must be a list or tuple at %s'
                         % subpath)
+                for item in items:
+                    if (not item
+                        or item.startswith(self.metadata_prefix)
+                        or self.hidden_re.match(item) is not None):
+                        raise FSWriteError(
+                            'Not a legal object name: %s' % repr(item))
+            else:
+                raise FSWriteError(
+                    'Node type must be "d" or "f" at %s' % subpath)
 
 
-    def queue(self, subpath, section_name, data):
+    def queue(self, subpath, section_name, data, force=0):
         """Queues data to be written at commit time"""
         m = self._pending
         sections = m.get(subpath)
         if sections is None:
             sections = {}
             m[subpath] = sections
-        if sections.has_key(section_name):
+        if sections.has_key(section_name) and not force:
             if sections[section_name] != data:
                 raise FSWriteError(
                     'Conflicting data storage at %s (%s)' %
@@ -326,7 +462,8 @@
             os.makedirs(self.basepath)
 
     def begin(self):
-        pass
+        self._props_cache.clear()
+        self._dir_cache.clear()
 
     def vote(self):
         """Do some early verification
@@ -341,6 +478,8 @@
     def reset(self):
         self._final = 0
         self._pending.clear()
+        self._props_cache.clear()
+        self._dir_cache.clear()
 
     def abort(self):
         self.reset()
@@ -358,3 +497,12 @@
     def close(self):
         pass
 
+
+
+def writeSection(props_f, name, data):
+    props_f.write('[%s]\n' % name)
+    props_f.write(data.replace('[', '[['))
+    if data.endswith('\n'):
+        props_f.write('\n')
+    else:
+        props_f.write('\n\n')


=== Products/AdaptableStorage/gateway_fs/FSDirectoryItems.py 1.8.2.4 => 1.8.2.5 ===


=== Products/AdaptableStorage/gateway_fs/public.py 1.3.2.4 => 1.3.2.5 ===