[Zope-CVS] CVS: Products/AdaptableStorage/gateway_fs - FSClassificationSection.py:1.9 FSConnection.py:1.11

Shane Hathaway shane@zope.com
Tue, 4 Feb 2003 23:59:50 -0500


Update of /cvs-repository/Products/AdaptableStorage/gateway_fs
In directory cvs.zope.org:/tmp/cvs-serv22769/gateway_fs

Modified Files:
	FSClassificationSection.py FSConnection.py 
Log Message:
Added an automatic filename extensions feature to FSConnection and
corresponding unit tests.  Also cleaned up minor details in FSConnection.  


=== Products/AdaptableStorage/gateway_fs/FSClassificationSection.py 1.8 => 1.9 ===
--- Products/AdaptableStorage/gateway_fs/FSClassificationSection.py:1.8	Thu Jan  9 09:33:58 2003
+++ Products/AdaptableStorage/gateway_fs/FSClassificationSection.py	Tue Feb  4 23:59:16 2003
@@ -16,6 +16,8 @@
 $Id$
 """
 
+import os
+
 from mapper_public import IGateway, FieldSchema
 
 
@@ -50,17 +52,21 @@
                 if '=' in line:
                     k, v = line.split('=', 1)
                     classification[k.strip()] = v.strip()
-        classification['filename'] = self.getIdFrom(event)
+        filename = self.getIdFrom(event)
+        classification['extension'] = os.path.splitext(filename)[1]
         return classification, text.strip()
 
     def store(self, event, state):
         # state is a classification
+        p = event.getKeychain()[-1]
         items = state.items()
         items.sort()
         text = []
         for k, v in items:
-            text.append('%s=%s' % (k, v))
+            if k == 'extension':
+                self.fs_conn.suggestExtension(p, v)
+            else:
+                text.append('%s=%s' % (k, v))
         text = '\n'.join(text)
-        p = event.getKeychain()[-1]
         self.fs_conn.writeSection(p, 'classification', text)
         return text.strip()


=== Products/AdaptableStorage/gateway_fs/FSConnection.py 1.10 => 1.11 ===
--- Products/AdaptableStorage/gateway_fs/FSConnection.py:1.10	Thu Jan  9 09:33:58 2003
+++ Products/AdaptableStorage/gateway_fs/FSConnection.py	Tue Feb  4 23:59:16 2003
@@ -34,6 +34,8 @@
 
 NODE_TYPE_SECTION = '@node_type'  # Data is 'f' (file) or 'd' (directory)
 DATA_SECTION = '@data'  # Data is a string (file) or list of names (directory)
+SUGGESTED_EXTENSION_SECTION = '@s_ext'  # The suggested filename extension
+OBJECT_NAMES_SECTION = 'object_names'  # For directories
 
 
 class FSConnection:
@@ -59,6 +61,69 @@
         self._pending = {}
 
 
+    def computeDirectoryContents(self, path, ignore_error=0):
+        """Computes and returns intermediate directory contents info.
+
+        Returns (filenames, object_names, translations).
+        """
+        filenames = []
+        obj_names = []
+        trans = {}     # { base name -> filename with extension or None }
+        try:
+            fns = os.listdir(path)
+        except OSError:
+            if ignore_error:
+                return filenames, obj_names, trans
+        hidden_filename_prefix = self.hidden_filename_prefix
+        for fn in fns:
+            if fn.startswith(hidden_filename_prefix):
+                continue
+            filenames.append(fn)
+
+        props = self.getPropertiesFromFile(path)
+        text = props.get(OBJECT_NAMES_SECTION)
+        if text:
+            # Prepare a dictionary of translations.
+            for fn in filenames:
+                if '.' in fn:
+                    base, ext = fn.split('.', 1)
+                    if trans.has_key(base):
+                        # Name collision: two or more files have the same base
+                        # name.  Don't use an extension for this name.
+                        trans[base] = None
+                    else:
+                        trans[base] = fn
+                else:
+                    trans[fn] = None
+            obj_names = [line.strip() for line in text.split('\n')]
+            for obj_name in obj_names:
+                if '.' in obj_name:
+                    base, ext = obj_name.split('.', 1)
+                    trans[base] = None
+        return filenames, obj_names, trans
+        
+
+    def listDirectoryAsMapping(self, path, ignore_error=0):
+        """Returns the translated filenames at path.
+
+        The ignore_error flag makes this method return an empty
+        dictionary if the directory is not found.
+
+        Returns {filename -> obj_name}.
+        """
+        filenames, obj_names, trans = self.computeDirectoryContents(
+            path, ignore_error)
+        res = {}
+        for fn in filenames:
+            res[fn] = fn
+        # Translate names.
+        for obj_name in obj_names:
+            fn = trans.get(obj_name)
+            if fn:
+                res[fn] = obj_name
+        return res
+
+
     def expandPath(self, subpath):
         if self.basepath:
             while subpath.startswith('/') or subpath.startswith('\\'):
@@ -67,48 +132,46 @@
         else:
             # unchanged.
             path = subpath
+        if not os.path.exists(path):
+            dir_path, obj_name = os.path.split(path)
+            if '.' not in obj_name:
+                # This object might have an automatic filename extension.
+                # XXX This is expensive.
+                filenames, obj_names, trans = self.computeDirectoryContents(
+                    dir_path, 1)
+                fn = trans.get(obj_name)
+                if fn is not None:
+                    # Use the filename with an extension.
+                    path = os.path.join(dir_path, fn)
         return path
 
 
     def checkSectionName(self, section_name):
-        assert isinstance(section_name, StringType)
-        assert '[' not in section_name
-        assert ']' not in section_name
-        assert '\n' not in section_name
-        assert section_name != NODE_TYPE_SECTION
-        assert section_name != DATA_SECTION
-
-
-    def _write(self, subpath, section_name, data):
-        # XXX We should be checking for '..'
-        path = self.expandPath(subpath)
-        # Do some early checking.
-        if os.path.exists(path):
-            v = os.access(path, os.W_OK)
-            if not v:
-                raise FSWriteError(
-                    "Can't get write access to %s" % subpath)
-        self.queue(subpath, section_name, data)
+        if (not isinstance(section_name, StringType)
+            or not section_name
+            or '[' in section_name
+            or ']' in section_name
+            or '\n' in section_name
+            or section_name.startswith('@')
+            or section_name == OBJECT_NAMES_SECTION):
+            raise ValueError, section_name
 
 
     def writeSection(self, subpath, section_name, data):
         self.checkSectionName(section_name)
-        self._write(subpath, section_name, data)
+        self.queue(subpath, section_name, data)
 
 
     def writeNodeType(self, subpath, data):
-        path = self.expandPath(subpath)
-        # Do some early checking.
-        if os.path.exists(path):
-            want_dir = (data == 'd')
-            if (want_dir != (not not os.path.isdir(path))):
-                raise FSWriteError(
-                    "Can't mix file and directory at %s" % subpath)
         self.queue(subpath, NODE_TYPE_SECTION, data)
 
 
     def writeData(self, subpath, data):
-        self._write(subpath, DATA_SECTION, data)
+        self.queue(subpath, DATA_SECTION, data)
+
+
+    def suggestExtension(self, subpath, ext):
+        self.queue(subpath, SUGGESTED_EXTENSION_SECTION, ext)
 
 
     def readSection(self, subpath, section_name, default=None):
@@ -130,14 +193,10 @@
         isdir = os.path.isdir(path)
         # Read either the directory listing or the file contents.
         if isdir:
-            names = []
-            prefix = self.hidden_filename_prefix
-            for name in os.listdir(path):
-                if not name.startswith(self.hidden_filename_prefix):
-                    names.append(name)
-            # Return a sequence instead of a string.
-            return names
+            # Return a sequence of object names.
+            return self.listDirectoryAsMapping(path).values()
         else:
+            # Return a string.
             try:
                 f = open(path, 'rb')
             except IOError:
@@ -162,7 +221,7 @@
 
 
     def getPropertiesFromFile(self, path):
-        """Read a properties file next to path."""
+        """Reads a properties file next to path."""
         props_fn = self.getPropertiesPath(path)
 
         try:
@@ -200,8 +259,23 @@
         # sections is a mapping.
         path = self.expandPath(subpath)
         t = sections[NODE_TYPE_SECTION]
-        if t == 'd' and not os.path.exists(path):
-            os.mkdir(path)
+        if not os.path.exists(path):
+            if t == 'd':
+                os.mkdir(path)
+            else:
+                fn = os.path.split(path)[1]
+                if '.' not in fn:
+                    # This object has no extension and doesn't yet exist.
+                    ext = sections.get(SUGGESTED_EXTENSION_SECTION)
+                    if ext:
+                        # Try to use the suggested extension.
+                        if not ext.startswith('.'):
+                            ext = '.' + ext
+                        p = path + ext
+                        if not os.path.exists(p):
+                            # No file is in the way.
+                            # Use the suggested extension.
+                            path = p
         props_fn = self.getPropertiesPath(path)
         items = sections.items()
         items.sort()
@@ -213,36 +287,33 @@
                 elif name == DATA_SECTION:
                     if t == 'd':
                         # Change the list of subobjects.
-                        # Here we only have to delete.
-                        # Subobjects will be created later.
-                        # XXX we might check for dotted names here.
                         self.removeUnlinkedItems(path, data)
+                        writeSection(props_f, OBJECT_NAMES_SECTION,
+                                     '\n'.join(data))
+                        self.disableConflictingExtensions(subpath, data)
                     else:
-                        # Change file contents.
+                        # Change the file contents.
                         f = open(path, 'wb')
                         try:
                             f.write(data)
                         finally:
                             f.close()
+                elif name == SUGGESTED_EXTENSION_SECTION:
+                    # This doesn't need to be written.
+                    pass
                 else:
-                    props_f.write('[%s]\n' % name)
-                    props_f.write(data.replace('[', '[['))
-                    if data.endswith('\n'):
-                        props_f.write('\n')
-                    else:
-                        props_f.write('\n\n')
+                    writeSection(props_f, name, data)
         finally:
             props_f.close()
 
 
     def removeUnlinkedItems(self, path, names):
+        """Removes unused files/subtrees from a directory."""
         linked = {}
         for name in names:
             linked[name] = 1
-        existing = os.listdir(path)
-        prefix = self.hidden_filename_prefix
-        for fn in existing:
-            if not fn.startswith(prefix) and not linked.get(fn):
+        for fn, obj_name in self.listDirectoryAsMapping(path).items():
+            if not linked.get(obj_name):
                 item_fn = os.path.join(path, fn)
                 if os.path.isdir(item_fn):
                     rmtree(item_fn)
@@ -253,43 +324,88 @@
                         os.remove(item_pfn)
 
 
+    def disableConflictingExtensions(self, subpath, obj_names):
+        """Fixes collisions before writing files in a directory.
+
+        Enforces the rule: if 'foo.*' is in the
+        database, 'foo' may not have an automatic extension.
+        Enforces by removing suggested extensions.
+        """
+        reserved = {}  # { object name without extension -> 1 }
+        for obj_name in obj_names:
+            if '.' in obj_name:
+                base, ext = obj_name.split('.', 1)
+                reserved[base] = 1
+        if not reserved:
+            # No objects have extensions.
+            return
+
+        while subpath.endswith('/'):
+            subpath = subpath[:-1]
+        for obj_name in obj_names:
+            if reserved.has_key(obj_name):
+                # obj_name has no extension and must not have an
+                # automatic extension.
+                child_subpath = '%s/%s' % (subpath, obj_name)
+                self.queue(child_subpath, SUGGESTED_EXTENSION_SECTION,
+                           '', force=1)
+
+
     def beforeWrite(self, items):
+        """Does some early checking while it's easy to bail out.
+
+        This avoids exceptions during the second phase of transaction commit.
+        """
         non_containers = {}
         for subpath, sections in items:
+            path = self.expandPath(subpath)
+            exists = os.path.exists(path)
+            if exists and not os.access(path, os.W_OK):
+                raise FSWriteError(
+                    "Can't get write access to %s" % subpath)
             # type must be provided and must always be either 'd' or 'f'.
             if (not sections.has_key(NODE_TYPE_SECTION)
                 or not sections.has_key(DATA_SECTION)):
                 raise FSWriteError(
                     'Data or node type not specified for %s' % subpath)
             t = sections[NODE_TYPE_SECTION]
-            if t not in ('d', 'f'):
-                raise FSWriteError(
-                    'node type must be "d" or "f" at %s' % subpath)
             dir = os.path.dirname(subpath)
             if non_containers.get(dir):
                 raise FSWriteError(
                     "Not a directory: %s" % dir)
             if t == 'f':
+                if exists and os.path.isdir(path):
+                    raise FSWriteError(
+                        "Can't write file data to directory at %s"
+                        % subpath)
                 non_containers[subpath] = 1
                 if not isinstance(sections[DATA_SECTION], StringType):
                     raise FSWriteError(
                         'Data for a file must be a string at %s'
                         % subpath)
-            else:
-                if isinstance(sections[DATA_SECTION], StringType):
+            elif t == 'd':
+                if exists and not os.path.isdir(path):
+                    raise FSWriteError(
+                        "Can't write directory contents to file at %s"
+                        % subpath)
+                items = sections[DATA_SECTION]
+                if isinstance(items, StringType):
                     raise FSWriteError(
                         'Data for a directory must be a list or tuple at %s'
                         % subpath)
+            else:
+                raise FSWriteError(
+                    'Node type must be "d" or "f" at %s' % subpath)
 
 
-    def queue(self, subpath, section_name, data):
+    def queue(self, subpath, section_name, data, force=0):
         """Queues data to be written at commit time"""
         m = self._pending
         sections = m.get(subpath)
         if sections is None:
             sections = {}
             m[subpath] = sections
-        if sections.has_key(section_name):
+        if sections.has_key(section_name) and not force:
             if sections[section_name] != data:
                 raise FSWriteError(
                     'Conflicting data storage at %s (%s)' %
@@ -345,3 +461,12 @@
     def close(self):
         pass
 
+
+
+def writeSection(props_f, name, data):
+    props_f.write('[%s]\n' % name)
+    props_f.write(data.replace('[', '[['))
+    if data.endswith('\n'):
+        props_f.write('\n')
+    else:
+        props_f.write('\n\n')