[Zope-CVS] CVS: Products/Ape/lib/apelib/fs - oidtable.py:1.1.2.1 __init__.py:1.1.10.1 annotated.py:1.4.2.1 base.py:1.7.2.1 classification.py:1.4.2.1 connection.py:1.7.2.1 fileops.py:1.1.8.1 interfaces.py:1.4.2.1 params.py:1.1.10.1 properties.py:1.4.2.1 security.py:1.3.2.1 structure.py:1.7.2.1

Shane Hathaway shane at zope.com
Wed Feb 25 11:04:03 EST 2004


Update of /cvs-repository/Products/Ape/lib/apelib/fs
In directory cvs.zope.org:/tmp/cvs-serv9748/lib/apelib/fs

Modified Files:
      Tag: ape-fs-oid-branch
	__init__.py annotated.py base.py classification.py 
	connection.py fileops.py interfaces.py params.py properties.py 
	security.py structure.py 
Added Files:
      Tag: ape-fs-oid-branch
	oidtable.py 
Log Message:
Created ape-fs-oid-branch, which changes Ape to use OIDs on the filesystem.

This solution will enable us to remove some of Ape's patches and
use the filesystem even more like a relational database.  It has been
difficult to find a clean strategy for doing this, but we're finally almost
there.

More tests pass than fail. ;-)


=== Added File Products/Ape/lib/apelib/fs/oidtable.py ===
##############################################################################
#
# Copyright (c) 2003 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL).  A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
"""OIDTable class.

$Id: oidtable.py,v 1.1.2.1 2004/02/25 16:03:28 shane Exp $
"""

class OIDTable:
    """Table of (parent_oid, filename, child_oid).

    The oid and filename columns form the primary key.  Maintains an index
    on the child_oid column to allow fast reverse lookups.
    """

    def __init__(self):
        self.fwd = {}  # { parent_oid : {filename : child_oid} }
        self.back = {}  # { child_oid : [(parent_oid, filename)] }

    def add(self, parent_oid, filename, child_oid):
        """Adds an association from a parent and filename to a child.
        """
        d = self.fwd.get(parent_oid)
        if d is None:
            d = {}
            self.fwd[parent_oid] = d
        if d.has_key(filename):
            if d[filename] != child_oid:
                raise KeyError(
                    "'%s' already has a child named '%s', with OID '%s'"
                    % (parent_oid, filename, d[filename]))
        else:
            d[filename] = child_oid
        p = self.back.get(child_oid)
        key = (parent_oid, filename)
        if p is None:
            p = [key]
            self.back[child_oid] = p
        elif key not in p:
            p.append(key)

    def remove(self, parent_oid, filename):
        """Removes an association between a parent and a child.
        """
        d = self.fwd.get(parent_oid)
        if not d:
            return
        child_oid = d.get(filename)
        if not child_oid:
            return
        del d[filename]
        if not d:
            del self.fwd[parent_oid]
        p = self.back.get(child_oid)
        key = (parent_oid, filename)
        if key in p:
            p.remove(key)
        if not p:
            del self.back[child_oid]

    def setChildren(self, parent_oid, new_children):
        """Updates all children for a parent.

        new_children is {filename: child_oid}.  Calls self.add() and
        self.remove() to make all changes.
        """
        old_children = self.fwd.get(parent_oid)
        if old_children is not None:
            # The dictionary in the table will change as children are
            # added/removed, so make a copy.
            old_children = old_children.copy()
        else:
            old_children = {}
        for filename, child_oid in new_children.items():
            if old_children.has_key(filename):
                if old_children[filename] != child_oid:
                    # Change this child to a new OID.
                    self.remove(parent_oid, filename)
                    self.add(parent_oid, filename, child_oid)
                del old_children[filename]
            else:
                # Add a new child.
                self.add(parent_oid, filename, child_oid)
        # Remove the filenames left over in old_children.
        for filename, child_oid in old_children.items():
            self.remove(parent_oid, filename)

    def getPath(self, ancestor_oid, descendant_oid):
        """Returns the primary path that connects two OIDs.

        The primary path follows the first parent of each OID.
        """
        parts = []
        back_get = self.back.get
        parts_append = parts.append
        oid = descendant_oid
        while oid != ancestor_oid:
            p = back_get(oid)
            if not p:
                # The primary lineage doesn't reach the ancestor.
                return None
            # Follow only the first parent.
            oid, filename = p[0]
            if oid == descendant_oid:
                # Circular OID chain.
                return None
            parts_append(filename)
        parts.reverse()
        return parts

    def getChildren(self, parent_oid):
        """Returns the children of an OID as a mapping of {filename: oid}.

        Do not modify the return value.
        """
        return self.fwd.get(parent_oid)

    def getParents(self, child_oid):
        """Returns the parents of an OID as a list of (oid, filename).

        Do not modify the return value.
        """
        return self.back.get(child_oid)


=== Products/Ape/lib/apelib/fs/__init__.py 1.1 => 1.1.10.1 ===


=== Products/Ape/lib/apelib/fs/annotated.py 1.4 => 1.4.2.1 ===
--- Products/Ape/lib/apelib/fs/annotated.py:1.4	Fri Feb 20 22:30:39 2004
+++ Products/Ape/lib/apelib/fs/annotated.py	Wed Feb 25 11:03:28 2004
@@ -170,7 +170,7 @@
             return 0
         return 1
 
-    def computeDirectoryContents(self, path, allow_missing=0):
+    def computeContents(self, path, allow_missing=0):
         """Returns the name translations for a directory.  Caches the results.
 
         Returns ({filename: name}, {name: filename}).


=== Products/Ape/lib/apelib/fs/base.py 1.7 => 1.7.2.1 ===


=== Products/Ape/lib/apelib/fs/classification.py 1.4 => 1.4.2.1 ===
--- Products/Ape/lib/apelib/fs/classification.py:1.4	Mon Feb  2 10:07:20 2004
+++ Products/Ape/lib/apelib/fs/classification.py	Wed Feb 25 11:03:28 2004
@@ -31,40 +31,40 @@
 
     def load(self, event):
         fs_conn = self.getConnection(event)
-        p = event.oid
-        classification = {'node_type': fs_conn.readNodeType(p)}
-        text = fs_conn.readAnnotation(p, 'classification', '')
+        oid = event.oid
+        classification = {'node_type': fs_conn.readNodeType(oid)}
+        text = fs_conn.readAnnotation(oid, 'classification', '')
         if text:
             lines = text.split('\n')
             for line in lines:
                 if '=' in line:
                     k, v = line.split('=', 1)
                     classification[k.strip()] = v.strip()
-        classification['extension'] = fs_conn.getExtension(p)
+        classification['extension'] = fs_conn.getExtension(oid)
         return classification, text.strip()
 
     def store(self, event, state):
         # state is a classification
         fs_conn = self.getConnection(event)
-        p = event.oid
+        oid = event.oid
         if event.is_new:
             # Don't overwrite existing data
             try:
-                fs_conn.readNodeType(p)
+                fs_conn.readNodeType(oid)
             except LoadError:
                 # Nothing exists yet.
                 pass
             else:
                 # Something exists.  Don't overwrite it.
-                raise OIDConflictError(p)
+                raise OIDConflictError(oid)
         items = state.items()
         items.sort()
         text = []
         for k, v in items:
             if k == 'extension':
-                fs_conn.suggestExtension(p, v)
+                fs_conn.suggestExtension(oid, v)
             else:
                 text.append('%s=%s' % (k, v))
         text = '\n'.join(text)
-        fs_conn.writeAnnotation(p, 'classification', text)
+        fs_conn.writeAnnotation(oid, 'classification', text)
         return text.strip()


=== Products/Ape/lib/apelib/fs/connection.py 1.7 => 1.7.2.1 ===
--- Products/Ape/lib/apelib/fs/connection.py:1.7	Fri Feb 20 22:30:39 2004
+++ Products/Ape/lib/apelib/fs/connection.py	Wed Feb 25 11:03:28 2004
@@ -11,15 +11,20 @@
 # FOR A PARTICULAR PURPOSE.
 #
 ##############################################################################
-"""Full-featured Filesystem connection class.
+"""Filesystem connection class.
 
 $Id$
 """
 
 from apelib.core.interfaces import ITPCConnection, ISourceRepository, LoadError
-from interfaces import IFSReader, IFSWriter, FSWriteError
+
+from interfaces import IFSReader, IFSWriter, FSReadError, FSWriteError
 from fileops import StandardFileOperations
 from annotated import AnnotatedFilesystem, object_names_ann
+from oidtable import OIDTable
+
+DEBUG = 0
+
 
 # For a node_type_ann, the value is 'f' (file) or 'd' (directory)
 node_type_ann = '@node_type'
@@ -40,86 +45,128 @@
     __implements__ = IFSReader, IFSWriter, ITPCConnection, ISourceRepository
 
     basepath = ''
+    root_oid = '0'
 
     def __init__(self, basepath, annotation_prefix='.', hidden_filenames='_',
                  ops=None):
+        # These attributes are used for both reading and writing.
         self.basepath = basepath
         if ops is None:
             ops = StandardFileOperations()
         self.ops = ops
         self.afs = AnnotatedFilesystem(
             ops, annotation_prefix, hidden_filenames)
-        self._final = 0
+        self.table = OIDTable()
+
+        # These attributes are used only for writing.
+        self._final = 0  # True if second phase of commit.
         # _pending holds the data to be written.
-        # _pending: { subpath string -> { annotation_name -> data } }
+        # _pending: { oid -> { annotation_name -> data } }
         self._pending = {}
+        self._script = None  # [(instruction, *args)]
+        self._tmp_subpaths = {}  # { oid: subpath }
+
+    def reset(self):
+        self._final = 0
+        self._pending.clear()
+        self.afs.clearCache()
+        self._script = None
+        self._tmp_subpaths.clear()
 
     #
     # IFSReader implementation.
     #
 
-    def getPath(self, subpath):
-        if self.basepath:
-            while subpath.startswith('/') or subpath.startswith('\\'):
-                subpath = subpath[1:]
-            path = self.ops.join(self.basepath, subpath)
+    def getSubpath(self, oid):
+        p = self.table.getPath(self.root_oid, oid)
+        if p is None:
+            return self._tmp_subpaths.get(oid)
+        if p and p[0] == 'Application':
+            # Place the Application object at basepath.
+            return p[1:]
         else:
-            # unchanged.
-            path = subpath
-        if not self.ops.exists(path):
-            dir_path, obj_name = self.ops.split(path)
-            if '.' not in obj_name:
-                # This object might have an automatic filename extension.
-                contents = self.afs.computeDirectoryContents(dir_path, 1)
-                fn_to_name, name_to_fn = contents
-                fn = name_to_fn.get(obj_name)
-                if fn:
-                    # Use the filename with an extension.
-                    path = self.ops.join(dir_path, fn)
-        return path
+            # Everything else goes in "_root".
+            return ['_root'] + p
+
+    def getPath(self, oid):
+        p = self.getSubpath(oid)
+        if p is None:
+            raise LoadError(oid)
+        return self.ops.join(self.basepath, *p)
 
-    def readNodeType(self, subpath):
-        path = self.getPath(subpath)
+    def readNodeType(self, oid):
+        path = self.getPath(oid)
         if not self.ops.exists(path):
             raise LoadError("%s does not exist" % path)
         return self.ops.isdir(path) and 'd' or 'f'
 
-    def readData(self, subpath, allow_missing=0, as_text=0):
-        path = self.getPath(subpath)
+    def readData(self, oid, allow_missing=0, as_text=0):
         # Return a string.
         try:
+            path = self.getPath(oid)
             return self.ops.readfile(path, as_text)
-        except IOError:
+        except (LoadError, IOError):
             if allow_missing:
                 return None
             raise
 
-    def readDirectory(self, subpath, allow_missing=0):
-        path = self.getPath(subpath)
-        # Return a sequence of object names.
-        contents = self.afs.computeDirectoryContents(path, allow_missing)
+    def readDirectory(self, oid, allow_missing=0):
+        # Return a sequence of (object_name, child_oid).
+        path = self.getPath(oid)
+        contents = self.afs.computeContents(path, allow_missing)
         fn_to_name, name_to_fn = contents
-        return name_to_fn.keys()
+        children = self.table.getChildren(oid)
+        if children is None:
+            children = {}
+        # Remove vanished children from the OID table.
+        for filename, child_oid in children.items():
+            if not fn_to_name.has_key(filename):
+                self.table.remove(oid, filename)
+                # XXX Need to garbage collect descendants.
+        # Return the current children.
+        return [(objname, children.get(filename))
+                for filename, objname in fn_to_name.items()]
 
-    def readAnnotation(self, subpath, name, default=None):
-        self.afs.checkAnnotationName(name)
-        path = self.getPath(subpath)
+    def readAnnotation(self, oid, name, default=None):
+        path = self.getPath(oid)
         annotations = self.afs.getAnnotations(path)
         return annotations.get(name, default)
 
-    def getExtension(self, subpath):
-        path = self.getPath(subpath)
+    def readObjectName(self, oid):
+        parents = self.table.getParents(oid)
+        parent_oid, filename = parents[0]
+        parent_path = self.getPath(parent_oid)
+        contents = self.afs.computeContents(parent_path)
+        fn_to_name, name_to_fn = contents
+        return fn_to_name[filename]
+
+    def getExtension(self, oid):
+        path = self.getPath(oid)
         stuff, ext = self.ops.splitext(path)
         return ext
 
-    def getModTime(self, subpath, default=0):
+    def assignNew(self, oid, children):
+        """See IFSReader.
+        """
+        dir_path = self.getPath(oid)
+        contents = self.afs.computeContents(dir_path)
+        fn_to_name, name_to_fn = contents
+        existing = self.table.getChildren(oid)
+        for name, child_oid in children.items():
+            assert child_oid
+            if existing.has_key(name) and existing[name] != child_oid:
+                raise FSReadError("assignNew() doesn't change existing OIDs")
+            filename = name_to_fn[name]
+            self.table.add(oid, filename, child_oid)
+
+    def getModTime(self, oid, default=0):
         """Returns the time an object was last modified.
 
         Since objects are split into up to three files, this
         implementation returns the modification time of the most
         recently modified of the three.
         """
-        path = self.getPath(subpath)
+        path = self.getPath(oid)
         extra = self.afs.getAnnotationPaths(path)
         maxtime = -1
         for p in (path,) + tuple(extra):
@@ -134,10 +181,6 @@
             maxtime = default
         return maxtime
 
-    #
-    # ISourceRepository implementation.
-    #
-
     def _get_paths_mtime(self, paths):
         t = []
         for path in paths:
@@ -147,13 +190,17 @@
                 t.append(None)
         return t
 
-    def getPollSources(self, subpath):
-        path = self.getPath(subpath)
+    def getPollSources(self, oid):
+        path = self.getPath(oid)
         extra = self.afs.getAnnotationPaths(path)
         paths = (path,) + tuple(extra)
         t = self._get_paths_mtime(paths)
         return {(self, paths): t}
 
+    #
+    # ISourceRepository implementation.
+    #
+
     def poll(self, sources):
         """ISourceRepository implementation.
 
@@ -171,186 +218,409 @@
     #
     # IFSWriter implementation.
     #
+    def _queue(self, oid, name, data):
+        """Queues data to be written at commit time.
 
-    def writeNodeType(self, subpath, data):
-        self._queue(subpath, node_type_ann, data)
-
-    def writeData(self, subpath, data, as_text=0):
-        self._queue(subpath, data_ann, (data, as_text))
+        'name' is the name of the annotation.
+        """
+        m = self._pending
+        anns = m.get(oid)
+        if anns is None:
+            anns = {}
+            m[oid] = anns
+        if anns.has_key(name):
+            if anns[name] != data:
+                raise FSWriteError(
+                    'Conflicting data storage at %s (%s)' %
+                    (oid, name))
+        else:
+            anns[name] = data
 
-    def writeDirectory(self, subpath, names):
-        self._queue(subpath, file_list_ann, names)
+    def writeNodeType(self, oid, data):
+        if data not in ('d', 'f'):
+            raise FSWriteError(
+                'Node type must be "d" or "f" at %s' % oid)
+        self._queue(oid, node_type_ann, data)
+
+    def writeData(self, oid, data, as_text=0):
+        if not isinstance(data, type('')):
+            raise FSWriteError(
+                'Data for a file must be a string at %s' % oid)
+        self._queue(oid, data_ann, (data, as_text))
+
+    def writeDirectory(self, oid, data):
+        if isinstance(data, type('')):  # XXX Need a better check
+            raise FSWriteError(
+                'Data for a directory must be a list or tuple at %s' % oid)
+        isLegalFilename = self.afs.isLegalFilename
+        for objname, child_oid in data:
+            assert child_oid, "%s lacks a child_oid" % repr(objname)
+            if not isLegalFilename(objname):
+                raise FSWriteError(
+                    'Not a legal object name: %s' % repr(objname))
+        self._queue(oid, file_list_ann, data)
 
-    def writeAnnotation(self, subpath, name, data):
+    def writeAnnotation(self, oid, name, data):
         self.afs.checkAnnotationName(name)
-        self._queue(subpath, name, data)
+        self._queue(oid, name, data)
 
-    def suggestExtension(self, subpath, ext):
-        self._queue(subpath, suggested_extension_ann, ext)
+    def suggestExtension(self, oid, ext):
+        self._queue(oid, suggested_extension_ann, ext)
 
 
-    def _writeFinal(self, subpath, anns):
-        """Performs an actual write of a file or directory to disk.
-        """
-        # anns is a mapping.
-        path = self.getPath(subpath)
-        t = anns[node_type_ann]
-        if not self.ops.exists(path):
-            if t == 'd':
-                self.ops.mkdir(path)
-            else:
-                fn = self.ops.split(path)[1]
-                if '.' not in fn:
-                    # This object has no extension and doesn't yet exist.
-                    ext = anns.get(suggested_extension_ann)
-                    if ext:
-                        # Try to use the suggested extension.
-                        if not ext.startswith('.'):
-                            ext = '.' + ext
-                        p = path + ext
-                        if not self.ops.exists(p):
-                            # No file is in the way.
-                            # Use the suggested extension.
-                            path = p
-        to_write = {}
-        for name, value in anns.items():
-            if (name == node_type_ann
-                or name == suggested_extension_ann):
-                # Doesn't need to be written.
-                continue
-            elif name == data_ann:
-                data, as_text = value
-                self.ops.writefile(path, as_text, data)
-            elif name == file_list_ann:
-                # Change the list of subobjects.
-                self._removeUnlinkedItems(path, value)
-                to_write[object_names_ann] = '\n'.join(value)
-                self._disableConflictingExtensions(subpath, value)
-                self.afs.invalidate(path)
-            else:
-                to_write[name] = value
-        self.afs.writeAnnotations(path, to_write)
+    def _prepareContainerChanges(self, path, data):
+        """Prepares the new dictionary of children for a directory.
 
+        Chooses filenames for all of the directory's children.
+        Prevents filename collisions involving extensions by enforcing
+        the rule that if there is some object named 'foo.*', an object
+        named 'foo' may not have an automatic extension.
 
-    def _removeUnlinkedItems(self, path, names):
-        """Removes unused files/subtrees from a directory."""
-        linked = {}
-        for name in names:
-            linked[name] = 1
-        fn_to_name, name_to_fn = self.afs.computeDirectoryContents(path)
-        for fn, obj_name in fn_to_name.items():
-            if not linked.get(obj_name):
-                item_fn = self.ops.join(path, fn)
-                if self.ops.isdir(item_fn):
-                    self.ops.rmtree(item_fn)
-                else:
-                    self.ops.remove(item_fn)
-                    extra_paths = self.afs.getAnnotationPaths(item_fn)
-                    for p in extra_paths:
-                        if self.ops.exists(p):
-                            self.ops.remove(p)
-
-
-    def _disableConflictingExtensions(self, subpath, obj_names):
-        """Fixes collisions before writing files in a directory.
-
-        Enforces the rule: if 'foo.*' is in the database, 'foo' may
-        not have an automatic extension.  Enforces by un-queuing
-        suggested extensions.
-        """
-        reserved = {}  # { object name without extension -> 1 }
-        for obj_name in obj_names:
-            if '.' in obj_name:
-                base, ext = obj_name.split('.', 1)
+        'path' is a filesystem path or None.  'data' is a list of
+        (objname, child_oid).  Returns {filename: child_oid}.
+        """
+        if path:
+            existing = self.afs.computeContents(path)[1]
+            # existing contains {objname: filename}
+        else:
+            existing = {}
+
+        reserved = {}  # { object name stripped of extension: 1 }
+        for objname, child_oid in data:
+            if '.' in objname:
+                base, ext = objname.split('.', 1)
                 reserved[base] = 1
-        if not reserved:
-            # No objects have extensions.
-            return
-        while subpath.endswith('/'):
-            subpath = subpath[:-1]
-        for obj_name in obj_names:
-            if reserved.has_key(obj_name):
-                # Prevent obj_name from using an automatic extension.
-                child_subpath = '%s/%s' % (subpath, obj_name)
-                self._queue(child_subpath, suggested_extension_ann,
-                            '', force=1)
-
-
-    def _beforeWrite(self, items):
-        """Does some early checking while it's easy to bail out.
-
-        This helps avoid exceptions during the second phase of
-        transaction commit.
-        """
-        non_containers = {}
-        for subpath, anns in items:
-            path = self.getPath(subpath)
-            exists = self.ops.exists(path)
-            if exists and not self.ops.canwrite(path):
+        new_filenames = {}
+        for objname, child_oid in data:
+            filename = objname
+            if not reserved.has_key(objname):
+                # This object is eligible for an automatic extension.
+                fn = existing.get(objname)
+                if fn:
+                    # Use the existing filename.
+                    filename = fn
+                else:
+                    anns = self._pending.get(child_oid)
+                    if anns:
+                        extension = anns.get(suggested_extension_ann)
+                        if extension:
+                            if not extension.startswith('.'):
+                                extension = '.' + extension
+                            filename = objname + extension
+            new_filenames[objname] = filename
+
+        fn_oid = {}
+        for objname, child_oid in data:
+            fn_oid[new_filenames[objname]] = child_oid
+        return fn_oid
+
+
+    def _prepare(self):
+        """Prepares for transaction commit.
+
+        Does some early checking while it's easy to bail out.  This
+        helps avoid exceptions during the second phase of transaction
+        commit.
+        """
+        container_changes = {}  # {oid: {filename: child_oid}}
+        for oid, anns in self._pending.items():
+            if self.table.getParents(oid) or oid == self.root_oid:
+                # This is an existing object.  It has a path.
+                p = self.getSubpath(oid)
+                if p is None:
+                    raise FSWriteError(
+                        "No path known for OID %s" % repr(oid))
+                if p:
+                    info = self.ops.join(*p)
+                    path = self.ops.join(self.basepath, info)
+                else:
+                    info = '/'
+                    path = self.basepath
+                if not self.ops.exists(path):
+                    path = None
+            else:
+                # This is a new object.  It does not have a path yet.
+                path = None
+                info = 'new object: %s' % repr(oid)
+            if path and not self.ops.canwrite(path):
                 raise FSWriteError(
-                    "Can't get write access to %s" % subpath)
+                    "Can't get write access. %s" % info)
+
             # type must be provided and must always be either 'd' or 'f'.
             if not anns.has_key(node_type_ann):
                 raise FSWriteError(
-                    'Node type not specified for %s' % subpath)
+                    'Node type not specified for %s' % info)
             t = anns[node_type_ann]
-            dir = self.ops.dirname(subpath)
-            if non_containers.get(dir):
-                raise FSWriteError(
-                    "Not a directory: %s" % dir)
+            dir = self.ops.dirname(oid)
+
             if t == 'f':
+                # Writing a file
                 data, as_text = anns[data_ann]
                 if anns.has_key(file_list_ann):
                     raise FSWriteError(
-                        "Files can't have directory contents. %s"
-                        % subpath)
-                if exists and self.ops.isdir(path):
-                    raise FSWriteError(
-                        "Can't write file data to directory at %s"
-                        % subpath)
-                non_containers[subpath] = 1
-                if not isinstance(data, type('')):
+                        "Files can't have directory contents. %s" % info)
+                if path and self.ops.isdir(path):
                     raise FSWriteError(
-                        'Data for a file must be a string at %s'
-                        % subpath)
+                        "A directory exists where a file is to be written. %s"
+                        % info)
+
             elif t == 'd':
+                # Writing a directory
                 data = anns[file_list_ann]
                 if anns.has_key(data_ann):
                     raise FSWriteError(
-                        "Directories can't have file data. %s"
-                        % subpath)
-                if exists and not self.ops.isdir(path):
+                        "Directories can't have file data. %s" % info)
+                if path and not self.ops.isdir(path):
                     raise FSWriteError(
-                        "Can't write directory contents to file at %s"
-                        % subpath)
-                if isinstance(data, type('')):
+                        "A file exists where a directory is to be written. %s"
+                        % info)
+                fn_oid = self._prepareContainerChanges(path, data)
+                container_changes[oid] = fn_oid
+
+            else:
+                raise FSWriteError('Node type must be "d" or "f". %s' % info)
+        self._script = self._generateScript(container_changes)
+
+
+    def _generateScript(self, container_changes):
+        """Generates the script for committing the transaction.
+
+        Returns [(instruction, *args)].
+        """
+        # container_changes is {oid: {filename: child_oid}}
+        # script is [(instruction, *args)]
+        script = []
+        script.append(("clearTemp",))
+
+        # Compute the number of times each relevant child_oid is to
+        # be linked or unlinked.
+        # counts is {child_oid: [link_count, unlink_count]}
+        counts = {}
+        def increment(child_oid, index, counts=counts):
+            c = counts.get(child_oid)
+            if c is None:
+                counts[child_oid] = c = [0, 0]
+            c[index] += 1
+
+        for oid, new_children in container_changes.items():
+            old_children = self.table.getChildren(oid)
+            if old_children is None:
+                old_children = {}
+            for filename, child_oid in new_children.items():
+                if old_children.get(filename) == child_oid:
+                    continue  # No change.
+                # Adding a link
+                increment(child_oid, 0)
+                if DEBUG:
+                    print 'fs: add link %s/%s -> %s' % (
+                        oid, filename, child_oid)
+            for filename, child_oid in old_children.items():
+                if new_children.get(filename) == child_oid:
+                    continue  # No change.
+                # Removing a link
+                increment(child_oid, 1)
+                if DEBUG:
+                    print 'fs: del link %s/%s -> %s' % (
+                        oid, filename, child_oid)
+
+        # Add steps to the script to move objects to a temporary directory,
+        # then delete objects.
+        to_delete = []  # [oid]
+        for child_oid, (links, unlinks) in counts.items():
+            if not self.table.getParents(child_oid):
+                # A new object should be added once or not at all.
+                if links > 1:
                     raise FSWriteError(
-                        'Data for a directory must be a list or tuple at %s'
-                        % subpath)
-                for item in data:
-                    if not self.afs.isLegalFilename(item):
-                        raise FSWriteError(
-                            'Not a legal object name: %s' % repr(item))
+                        "Multiple links to %s" % repr(child_oid))
             else:
-                raise FSWriteError(
-                    'Node type must be "d" or "f" at %s' % subpath)
+                # An existing object should be moved, removed, or left alone.
+                if links > 1 or (links > 0 and unlinks < 1):
+                    raise FSWriteError(
+                        "Multiple links to %s" % repr(child_oid))
+                if links > 0:
+                    # Moving.
+                    script.append(("moveToTemp", child_oid))
+                elif unlinks > 0:
+                    # Deleting.
+                    to_delete.append(child_oid)
+
+        for child_oid in to_delete:
+            script.append(("delete", child_oid))
+        script.append(("writeAll", container_changes))
+        if container_changes.has_key(self.root_oid):
+            script.append(("fixupRoot", container_changes[self.root_oid]))
+        script.append(("clearTemp",))
+        return script
 
+    def _do_clearTemp(self):
+        """Script command: zap the temporary directory.
+        """
+        ops = self.ops
+        path = ops.join(self.basepath, '_tmp')
+        if ops.exists(path):
+            self.ops.rmtree(path)
+        self._tmp_subpaths.clear()
 
-    def _queue(self, subpath, name, data, force=0):
-        """Queues data to be written at commit time"""
-        m = self._pending
-        anns = m.get(subpath)
-        if anns is None:
-            anns = {}
-            m[subpath] = anns
-        if anns.has_key(name) and not force:
-            if anns[name] != data:
-                raise FSWriteError(
-                    'Conflicting data storage at %s (%s)' %
-                    (subpath, name))
+    def _moveContents(self, src, dest):
+        """Move a directory's contents, but not the directory.
+
+        Also leaves behind any file whose name starts with an underscore.
+        """
+        ops = self.ops
+        ops.makedirs(dest)
+        for fn in ops.listdir(src):
+            if not fn.startswith('_'):
+                ops.rename(ops.join(src, fn), ops.join(dest, fn))
+
+    def _moveItem(self, src, dest):
+        """Moves a file or directory.
+
+        For files, also moves annotations next to the file.
+        """
+        ops.makedirs(ops.dirname(dest))
+        if not ops.isdir(src):
+            # Move the annotation files also.
+            extra_src = self.afs.getAnnotationPaths(src)
+            extra_dest = self.afs.getAnnotationPaths(dest)
+            for s, d in zip(extra_src, extra_dest):
+                if ops.exists(s):
+                    ops.rename(s, d)
+        ops.rename(src, dest)
+
+    def _do_moveToTemp(self, oid):
+        """Script command: move an object to the temporary directory.
+        """
+        ops = self.ops
+        path = self.getPath(oid)
+        if path == self.basepath:
+            # Move the application root by moving most of the contents
+            # instead of the actual directory.
+            dest_sub = ('_tmp', 'app', 'data')
+            dest = ops.join(self.basepath, *dest_sub)
+            self._moveContents(src, dest)
         else:
-            anns[name] = data
+            # Move an object.
+            dest_sub = ('_tmp', 'oid.%s' % oid, 'data')
+            dest = ops.join(self.basepath, *dest_sub)
+            self._moveItem(src, dest)
+        self._tmp_subpaths[oid] = dest_sub
+        parents = self.table.getParents(oid)
+        for parent_oid, filename in parents:
+            self.table.remove(parent_oid, filename)
+
+    def _restore(self, oid):
+        """Moves an object in the temp directory into the object system.
+        """
+        ops = self.ops
+        dest = self.getPath(oid)
+        src_sub = self._tmp_subpaths[oid]
+        src = ops.join(self.basepath, src_sub)
+        if dest == self.basepath:
+            self._moveContents(src, dest)
+        else:
+            self._moveItem(src, dest)
+        del self._tmp_subpaths[oid]
+
+    def _do_delete(self, oid):
+        """Script command: delete an object.
+        """
+        ops = self.ops
+        path = self.getPath(oid)
+        if path == self.basepath:
+            # Delete the application root.
+            for fn in ops.listdir(path):
+                if not fn.startswith('_'):
+                    ops.rmtree(ops.join(self.basepath, fn))
+        else:
+            # Delete an object.
+            if not ops.isdir(path):
+                # Delete the annotation files also.
+                extra = self.afs.getAnnotationPaths(path)
+                for s in extra:
+                    if ops.exists(s):
+                        ops.rmtree(s)
+            ops.rmtree(path)
+        if self._tmp_subpaths.has_key(oid):
+            del self._tmp_subpaths[oid]
+        parents = self.table.getParents(oid)
+        for parent_oid, filename in parents:
+            self.table.remove(parent_oid, filename)
+            # XXX Need to garbage collect descendants in the OID table.
+
 
+    def _do_writeAll(self, container_changes):
+        """Script command: write all objects.
+
+        Uses multiple passes.
+
+        container_changes: {oid: {filename: child_oid}}
+        """
+        ops = self.ops
+        while self._pending:
+            written = 0
+            for oid, anns in self._pending.items():
+                p = self.getSubpath(oid)
+                if p is None:
+                    # Not linked into the object system yet.
+                    # Try again on the next pass.
+                    continue
+                path = ops.join(self.basepath, *p)
+                t = anns[node_type_ann]
+                if not ops.exists(path):
+                    if t == 'd':
+                        ops.mkdir(path)
+                to_write = {}
+                for name, value in anns.items():
+                    if (name == node_type_ann
+                        or name == suggested_extension_ann):
+                        # Doesn't need to be written.
+                        continue
+                    elif name == data_ann:
+                        data, as_text = value
+                        ops.writefile(path, as_text, data)
+                    elif name == file_list_ann:
+                        # Prepare the object_names annotation.
+                        object_names = []
+                        for objname, child_oid in value:
+                            object_names.append(objname)
+                        to_write[object_names_ann] = '\n'.join(object_names)
+                        # Move objects from the temporary directory.
+                        fn_oid = container_changes.get(oid)
+                        if fn_oid:
+                            for filename, child_oid in fn_oid.items():
+                                self.table.add(oid, filename, child_oid)
+                                if self._tmp_subpaths.has_key(child_oid):
+                                    self._restore(child_oid)
+                        self.afs.invalidate(path)
+                    else:
+                        to_write[name] = value
+                self.afs.writeAnnotations(path, to_write)
+                self.afs.invalidate(self.ops.dirname(path))
+                # This object has been written.
+                written += 1
+                del self._pending[oid]
+
+            if DEBUG and not written:
+                # Nothing was written in this pass.  This means that
+                # the rest of the queued objects are not actually
+                # linked into the object system.  Toss them.
+                tossing = self._pending.keys()
+                tossing.sort()
+                print "tossing: %s" % ', '.join(tossing)
+                break
+
+    def _do_fixupRoot(self, fn_oid):
+        """Script command: fix up the root after the root was written.
+        """
+        path = self.ops.join(self.basepath, '_root', 'Application')
+        if fn_oid.has_key('Application'):
+            # The root has an Application.  Represent it with a directory.
+            if not self.ops.exists(path):
+                self.ops.mkdir(path)
+        else:
+            # The root does not have an Application.  Remove it.
+            if self.ops.exists(path):
+                self.ops.rmtree(path)
 
     #
     # ITPCConnection implementation
@@ -374,26 +644,18 @@
 
         This is done while the transaction can still be vetoed safely.
         """
-        items = self._pending.items()
-        items.sort()  # Ensure that base directories come first.
-        self._beforeWrite(items)
+        self._prepare()
         self._final = 1
 
-    def reset(self):
-        self._final = 0
-        self._pending.clear()
-        self.afs.clearCache()
-
     def abort(self):
         self.reset()
 
     def finish(self):
         if self._final:
             try:
-                items = self._pending.items()
-                items.sort()  # Ensure that base directories come first.
-                for subpath, anns in items:
-                    self._writeFinal(subpath, anns)
+                for code in self._script:
+                    m = getattr(self, '_do_%s' % code[0])
+                    m(*code[1:])
             finally:
                 self.reset()
 


=== Products/Ape/lib/apelib/fs/fileops.py 1.1 => 1.1.8.1 ===


=== Products/Ape/lib/apelib/fs/interfaces.py 1.4 => 1.4.2.1 ===
--- Products/Ape/lib/apelib/fs/interfaces.py:1.4	Fri Feb 20 22:30:39 2004
+++ Products/Ape/lib/apelib/fs/interfaces.py	Wed Feb 25 11:03:28 2004
@@ -19,6 +19,9 @@
 from Interface import Interface
 
 
+class FSReadError (Exception):
+    """Unable to read data"""
+
 class FSWriteError (Exception):
     """Unable to write data"""
 
@@ -27,18 +30,18 @@
     """Filesystem reader that supports annotations.
     """
 
-    def getPath(subpath):
-        """Returns the filesystem path for a subpath.
+    def getPath(oid):
+        """Returns the filesystem path for an oid.
 
         May automatically append an extension if the file already
         exists.
         """
 
-    def readNodeType(subpath):
+    def readNodeType(oid):
         """Reads the node type of a filesystem node.
         """
 
-    def readData(subpath, allow_missing=0, as_text=0):
+    def readData(oid, allow_missing=0, as_text=0):
         """Reads the main data stream from a file.
 
         If the allow_missing flag is specified, this method returns
@@ -46,28 +49,45 @@
         is read in text mode.
         """
 
-    def readDirectory(subpath, allow_missing=0):
+    def readDirectory(oid, allow_missing=0):
         """Reads the contents of a directory.
 
-        Returns a list of object names.  If the allow_missing flag is
-        specified, this method returns None if no such directory is
-        found.
+        Returns a list of (object_name, child_oid).  The child_oid is
+        None for objects not seen before.  The application should
+        assign unique OIDs to the newly found children, then tell this
+        object about the assignments through the assignNew() method.
+
+        If the allow_missing flag is specified, this method returns
+        None if no such directory is found.
         """
 
-    def readAnnotation(subpath, name, default=None):
+    def readAnnotation(oid, name, default=None):
         """Reads a text-based annotation for a file.
         """
 
-    def getExtension(subpath):
-        """Returns the filename extension for a subpath.
+    def readObjectName(oid):
+        """Gets the canonical name for an object.
+
+        Note that this only makes sense when objects can have only one
+        parent.
+        """
+
+    def assignNew(oid, children):
+        """Assigns OIDs to newly found objects.
+
+        See readDirectory().
+        """
+
+    def getExtension(oid):
+        """Returns the filename extension for a file.
         """
 
-    def getModTime(subpath, default=0):
+    def getModTime(oid, default=0):
         """Returns the last-modified time of a file.
         """
 
-    def getPollSources(subpath):
-        """Returns source information for a subpath.
+    def getPollSources(oid):
+        """Returns source information for an oid.
 
         The source information is a mapping that maps
         (source_repository, path) to a state object.  The contents of
@@ -81,29 +101,29 @@
     """Filesystem writer that supports annotations.
     """
 
-    def writeNodeType(subpath, data):
+    def writeNodeType(oid, data):
         """Writes the node type for a filesystem node.
 
         'd' (directory) and 'f' (file) are supported.
         """
 
-    def writeData(subpath, data, as_text=0):
+    def writeData(oid, data, as_text=0):
         """Writes string data to a filesystem node.
 
         If 'as_text' is true, the file is written in text mode.
         """
 
-    def writeDirectory(subpath, names):
+    def writeDirectory(oid, data):
         """Writes data to a directory.
 
-        'names' is a sequence of object names used for determining filenames..
+        'data' is a sequence of (object_name, child_oid).
         """
 
-    def writeAnnotation(subpath, name, data):
+    def writeAnnotation(oid, name, data):
         """Writes a text-based annotation for a filesystem node.
         """
 
-    def suggestExtension(subpath, ext):
+    def suggestExtension(oid, ext):
         """Suggests a filename extension for a filesystem node.
 
         The IFSConnection may use this information to store the file


=== Products/Ape/lib/apelib/fs/params.py 1.1 => 1.1.10.1 ===


=== Products/Ape/lib/apelib/fs/properties.py 1.4 => 1.4.2.1 ===
--- Products/Ape/lib/apelib/fs/properties.py:1.4	Mon Feb  2 10:07:20 2004
+++ Products/Ape/lib/apelib/fs/properties.py	Wed Feb 25 11:03:28 2004
@@ -73,9 +73,8 @@
         FSGatewayBase.__init__(self, conn_name)
 
     def load(self, event):
-        p = event.oid
         fs_conn = self.getConnection(event)
-        text = fs_conn.readAnnotation(p, self.annotation, '')
+        text = fs_conn.readAnnotation(event.oid, self.annotation, '')
         res = []
         if text:
             lines = text.split('\n')
@@ -96,9 +95,8 @@
             lines.append('%s:%s=%s' % (k, t, escape_string(v)))
         lines.sort()
         text = '\n'.join(lines)
-        p = event.oid
         fs_conn = self.getConnection(event)
-        fs_conn.writeAnnotation(p, self.annotation, text)
+        fs_conn.writeAnnotation(event.oid, self.annotation, text)
         state = list(state)
         state.sort()
         return tuple(state)
@@ -117,8 +115,7 @@
 
     def load(self, event):
         fs_conn = self.getConnection(event)
-        p = event.oid
-        state = fs_conn.readAnnotation(p, self.annotation, '').strip()
+        state = fs_conn.readAnnotation(event.oid, self.annotation, '').strip()
         return state, state
 
     def store(self, event, state):
@@ -126,8 +123,6 @@
             raise ValueError('Not a string: %s' % repr(state))
         state = state.strip()
         if state:
-            p = event.oid
             fs_conn = self.getConnection(event)
-            fs_conn.writeAnnotation(p, self.annotation, state)
+            fs_conn.writeAnnotation(event.oid, self.annotation, state)
         return state
-


=== Products/Ape/lib/apelib/fs/security.py 1.3 => 1.3.2.1 ===
--- Products/Ape/lib/apelib/fs/security.py:1.3	Mon Feb  2 10:07:20 2004
+++ Products/Ape/lib/apelib/fs/security.py	Wed Feb 25 11:03:28 2004
@@ -108,9 +108,8 @@
 
     def load(self, event):
         c = self.getConnection(event)
-        p = event.oid
-        assert c.readNodeType(p) == 'f'
-        text = c.readData(p)
+        assert c.readNodeType(event.oid) == 'f'
+        text = c.readData(event.oid)
         res = []
         for line in text.split('\n'):
             L = line.strip()
@@ -156,10 +155,11 @@
             domainlist = self._joinList(domains)
             to_write = '%s:%s:%s:%s' % (id, password, rolelist, domainlist)
             replace_lines[id] = to_write
-        p = event.oid
+        oid = event.oid
         fs_conn = self.getConnection(event)
-        fs_conn.writeNodeType(p, 'f')
-        text = fs_conn.readData(p, allow_missing=1)
+        fs_conn.writeNodeType(oid, 'f')
+        # Read the existing text only to maintain the current order.
+        text = fs_conn.readData(oid, allow_missing=1)
         if text is None:
             text = ''
         new_lines = []
@@ -181,8 +181,7 @@
             new_lines.append(line)
         # Write it
         text = '\n'.join(new_lines)
-        fs_conn.writeData(p, text)
+        fs_conn.writeData(oid, text)
         serial = list(state)
         serial.sort()
         return text
-


=== Products/Ape/lib/apelib/fs/structure.py 1.7 => 1.7.2.1 ===
--- Products/Ape/lib/apelib/fs/structure.py:1.7	Thu Feb 19 01:44:03 2004
+++ Products/Ape/lib/apelib/fs/structure.py	Wed Feb 25 11:03:28 2004
@@ -42,18 +42,16 @@
 
     def load(self, event):
         c = self.getConnection(event)
-        p = event.oid
-        assert c.readNodeType(p) == 'f'
-        state = c.readData(p, as_text=self.text)
+        assert c.readNodeType(event.oid) == 'f'
+        state = c.readData(event.oid, as_text=self.text)
         return state, state
 
     def store(self, event, state):
         if not isinstance(state, StringType):
             raise ValueError('Not a string: %s' % repr(state))
         c = self.getConnection(event)
-        p = event.oid
-        c.writeNodeType(p, 'f')
-        c.writeData(p, state, as_text=self.text)
+        c.writeNodeType(event.oid, 'f')
+        c.writeData(event.oid, state, as_text=self.text)
         return state
 
 
@@ -64,24 +62,13 @@
 
     schema = FieldSchema('id', 'string')
 
-    def getIdFrom(self, event):
-        p = event.oid
-        pos = p.rfind('/')
-        if pos >= 0:
-            return p[pos + 1:]
-        else:
-            return p
-
     def load(self, event):
-        id = self.getIdFrom(event)
+        id = self.getConnection(event).readObjectName(event.oid)
         return id, id
 
     def store(self, event, state):
-        id = self.getIdFrom(event)
-        if state != id:
-            raise ValueError('Mismatched object name: %s != %s' %
-                             (state, id))
-        return id
+        # Ignore.
+        return state
 
     def getPollSources(self, event):
         fs_conn = self.getConnection(event)
@@ -99,34 +86,34 @@
     schema.addField('classification', 'classification')
 
     def load(self, event):
-        p = event.oid
         c = self.getConnection(event)
-        assert c.readNodeType(p) == 'd'
-        names = c.readDirectory(p)
-        names.sort()
+        if c.readNodeType(event.oid) != 'd':
+            raise LoadError("Not a directory")
+        data = list(c.readDirectory(event.oid))
+        data.sort()
         res = []
-        for name in names:
-            oid = event.conf.oid_gen.new_oid(event, name, False)
-            classification = event.classify(oid)
+        assigned = []
+        for objname, child_oid in data:
+            if child_oid is None:
+                child_oid = event.conf.oid_gen.new_oid(event, objname, True)
+                assigned.append((objname, child_oid))
+            classification = event.classify(child_oid)
             # Return info about each subobject.
-            res.append((name, oid, classification))
-        return res, tuple(names)
+            res.append((objname, child_oid, classification))
+        if assigned:
+            # Saw new objects.  Tell the connection what their OIDs are.
+            c.assignNew(event.oid, assigned)
+        return res, tuple(data)
 
     def store(self, event, state):
-        p = event.oid
         c = self.getConnection(event)
-        c.writeNodeType(p, 'd')
-        state = list(state)
-        state.sort()
-        if __debug__:
-            for name, oid, classification in state:
-                expect = event.conf.oid_gen.new_oid(event, name, False)
-                assert expect == oid, (
-                    "Child of %s named %s must use OID %s, but used %s" %
-                    (event.oid, name, expect, oid))
-        names = [row[0] for row in state]
-        c.writeDirectory(p, names)
-        return tuple(names)
+        c.writeNodeType(event.oid, 'd')
+        data = []
+        for objname, child_oid, classification in state:
+            data.append((objname, child_oid))
+        data.sort()
+        c.writeDirectory(event.oid, data)
+        return tuple(data)
 
 
 class FSModTime (FSGatewayBase):
@@ -137,9 +124,8 @@
     schema = FieldSchema('mtime', 'int')
 
     def load(self, event):
-        p = event.oid
         fs_conn = self.getConnection(event)
-        state = long(fs_conn.getModTime(p))
+        state = long(fs_conn.getModTime(event.oid))
         return state, None  # Use None as the hash (see store())
 
     def store(self, event, state):
@@ -148,66 +134,12 @@
         return None
 
 
-class RootDirectoryItems (FSGatewayBase):
-    """Read/write the root object.
-
-    The root object is stored as a normal directory with one special feature:
-    the name 'Application' is always present and points to the OID '/'.  This
-    allows the root object to be stored inside the application object.
-    """
-
-    __implements__ = IGateway
-
-    schema = FSDirectoryItems.schema
-
-    def load(self, event):
-        p = event.oid
-        c = self.getConnection(event)
-        try:
-            t = c.readNodeType(p)
-        except LoadError:
-            # The root object doesn't exist, but it's reasonable
-            # to infer a state anyway.
-            names = []
-        else:
-            assert t == 'd', 'The root object must be a directory'
-            names = c.readDirectory(p)
-            names.sort()
-        res = [('Application', '/', None)]
-        for name in names:
-            if name != 'Application':
-                oid = event.conf.oid_gen.new_oid(event, name, False)
-                classification = event.classify(oid)
-                res.append((name, oid, classification))
-        return res, tuple(names)
-
-    def store(self, event, state):
-        p = event.oid
-        c = self.getConnection(event)
-        c.writeNodeType(p, 'd')
-        state = list(state)
-        state.sort()
-        names = []
-        for name, oid, classification in state:
-            if name == 'Application':
-                expect = '/'
-            else:
-                expect = event.conf.oid_gen.new_oid(event, name, False)
-                names.append(name)
-            assert expect == oid, (
-                "Child of %s named %s must use OID %s, but used %s" %
-                (event.oid, name, expect, oid))
-        c.writeDirectory(p, names)
-        names.sort()
-        return tuple(names)
-
-
 def root_mapping():
     """Returns a gateway suitable for storing the root persistent mapping.
     """
     from apelib.core.gateways import CompositeGateway
     from properties import FSAnnotationData
     g = CompositeGateway()
-    g.add('references', RootDirectoryItems())
+    g.add('references', FSDirectoryItems())
     g.add('others', FSAnnotationData('others'))
     return g




More information about the Zope-CVS mailing list