[Zope3-checkins] CVS: Zope3/src/zodb/code - patch.py:1.5 class_.py:1.8

Jeremy Hylton jeremy@zope.com
Fri, 24 Jan 2003 18:21:30 -0500


Update of /cvs-repository/Zope3/src/zodb/code
In directory cvs.zope.org:/tmp/cvs-serv31712/zodb/code

Modified Files:
	patch.py class_.py 
Log Message:
Merge new-pickle-branch to trunk.  Yee ha!


=== Zope3/src/zodb/code/patch.py 1.4 => 1.5 ===
--- Zope3/src/zodb/code/patch.py:1.4	Mon Dec 30 19:15:58 2002
+++ Zope3/src/zodb/code/patch.py	Fri Jan 24 18:20:56 2003
@@ -41,6 +41,14 @@
 possible to use the copy module, because it isn't possible to extend
 the copy module in a safe way.  The copy module depends on module globals.
 
+The pickler uses a Wrapper object that creates the appropriate new
+object or updates an old one when it is unpickled.  The wrapper also
+causes parts of the wrapped object's state to be traversed by the
+pickler, for example the func_defaults of a function object.  This
+traversal is necessary because references to convertable objects could
+be contained in the state and must be updated to refer to the new
+objects.
+
 What semantics do we want for update-in-place in the presence of aliases?
 
 Semantics based on per-namespace updates don't work in the presence of
@@ -143,25 +151,40 @@
 
         return PersistentClassMetaClass(self._obj.__name__, newbases, dict)
 
+def registerWrapper(atype, wrapper, unwrap_thunk):
+    """Register a patch wrapper for an external object type."""
+    Pickler.dispatch[atype] = Pickler.save_external
+    Pickler.external[atype] = wrapper, unwrap_thunk
+
+marker = object()
+
 class Pickler(pickle.Pickler):
 
     dispatch = {}
     dispatch.update(pickle.Pickler.dispatch)
 
     def __init__(self, file, module, memo, replacements):
+        # The pickler must be created in binary mode, because
+        # it pickles instances using the OBJ code.  The text-mode
+        # pickler uses a different strategy that explicitly
+        # stores the name of the instance's class which defeats
+        # the desire to replace references to classes with
+        # persistent classes.
         pickle.Pickler.__init__(self, file, bin=True)
+        
         self._pmemo = memo
+        self._wrapped = {} # set of objects already wrapped
         self._module = module
         self._repl = replacements
         self._builtins = module.__builtins__
 
-    def wrap(self, wrapperclass, object):
-        return wrapperclass(object, self._module, self._repl.get(id(object)))
+    def wrap(self, wrapperclass, obj):
+        return wrapperclass(obj, self._module, self._repl.get(id(obj)))
 
-    def persistent_id(self, object, force=False):
-        if isinstance(object, Wrapper) or object is self._builtins or force:
-            oid = id(object)
-            self._pmemo[oid] = object
+    def persistent_id(self, obj, force=False):
+        if isinstance(obj, Wrapper) or obj is self._builtins or force:
+            oid = id(obj)
+            self._pmemo[oid] = obj
             return oid
         else:
             # If the object is a real persistent object, patch it by
@@ -172,34 +195,57 @@
             # doesn't use sys.modules.
             
             # XXX Is this safe in all cases?
-            oid = getattr(object, "_p_oid", None)
-            if oid is None:
+            oid = getattr(obj, "_p_oid", marker)
+            if oid is marker:
                 return None
-            self._pmemo[oid] = object
+            elif oid is None:
+                # It's a persistent object, but it's newly created.
+                oid = object()
+            self._pmemo[oid] = obj
             return oid
 
     def save_type(self, atype):
         if atype.__module__ == "__builtin__":
             self.save_global(atype)
         else:
+            d = id(atype)
             self.save_reduce(self.wrap(TypeWrapper, atype),
                              (atype.__bases__, atype.__dict__))
+            memo_len = len(self.memo)
+            self.write(self.put(memo_len))
+            self.memo[d] = memo_len, None
 
     dispatch[TypeType] = save_type
     dispatch[ClassType] = save_type
 
     def save_function(self, func):
+        d = id(func)
         self.save_reduce(self.wrap(FunctionWrapper, func),
                          (func.func_defaults, func.func_dict))
+        memo_len = len(self.memo)
+        self.write(self.put(memo_len))
+        self.memo[d] = memo_len, None
 
     dispatch[FunctionType] = save_function
 
+    external = {}
+
+    def save_external(self, obj):
+        # Save an external type registered through registerWrapper
+        objtype = type(obj)
+        wrapper, unwrap_thunk = self.external[objtype]
+        d = id(obj)
+        self.save_reduce(self.wrap(wrapper, obj), unwrap_thunk(obj))
+        memo_len = len(self.memo)
+        self.write(self.put(memo_len))
+        self.memo[d] = memo_len, None
+
     # New-style classes don't have real dicts.  They have dictproxies.
     # There's no official way to spell the dictproxy type, so we have
     # to get it by using type() on an example.
     dispatch[type(Wrapper.__dict__)] = pickle.Pickler.save_dict
 
-    def save(self, object, ignore=None):
+    def save(self, obj, ignore=None):
         # Override the save() implementation from pickle.py, because
         # we don't ever want to invoke __reduce__() on builtin types
         # that aren't picklable.  Instead, we'd like to pickle all of
@@ -209,18 +255,18 @@
 
         # The ignored parameter is for compatible with Python 2.2,
         # which has the old inst_persistent_id feature.
-        pid = self.persistent_id(object)
+        pid = self.persistent_id(obj)
         if pid is not None:
             self.save_pers(pid)
             return
 
-        d = id(object)
-        t = type(object)
-        if (t is TupleType) and (len(object) == 0):
+        d = id(obj)
+        t = type(obj)
+        if (t is TupleType) and (len(obj) == 0):
             if self.bin:
-                self.save_empty_tuple(object)
+                self.save_empty_tuple(obj)
             else:
-                self.save_tuple(object)
+                self.save_tuple(obj)
             return
 
         if d in self.memo:
@@ -235,19 +281,19 @@
             except TypeError: # t is not a class
                 issc = 0
             if issc:
-                self.save_global(object)
+                self.save_global(obj)
                 return
 
             try:
                 reduce = dispatch_table[t]
             except KeyError:
-                self.save_pers(self.persistent_id(object, True))
+                self.save_pers(self.persistent_id(obj, True))
                 return
             else:
-                tup = reduce(object)
+                tup = reduce(obj)
 
             if type(tup) is StringType:
-                self.save_global(object, tup)
+                self.save_global(obj, tup)
                 return
             if type(tup) is not TupleType:
                 raise pickle.PicklingError("Value returned by %s must be a "
@@ -274,10 +320,10 @@
             self.save_reduce(callable, arg_tup, state)
             memo_len = len(self.memo)
             self.write(self.put(memo_len))
-            self.memo[d] = (memo_len, object)
+            self.memo[d] = (memo_len, obj)
             return
 
-        f(self, object)
+        f(self, obj)
 
 class Unpickler(pickle.Unpickler):
 
@@ -286,6 +332,7 @@
         self._pmemo = pmemo
 
     def persistent_load(self, oid):
+##        return self._pmemo[int(oid)]
         return self._pmemo[oid]
 
 class NameFinder:
@@ -360,7 +407,7 @@
     p = Pickler(f, module, memo, replacements)
     moddict = module.__dict__
     p.dump(moddict)
-    f.reset()
+    f.seek(0)
     u = Unpickler(f, memo)
     newdict = u.load()
     module.__dict__.clear()


=== Zope3/src/zodb/code/class_.py 1.7 => 1.8 ===
--- Zope3/src/zodb/code/class_.py:1.7	Tue Jan 21 15:15:48 2003
+++ Zope3/src/zodb/code/class_.py	Fri Jan 24 18:20:56 2003
@@ -28,6 +28,20 @@
 # separate sets of attributes.  This code should be documented, as it
 # it quite delicate, and it should be move to a separate module.
 
+class SimpleDescriptor(object):
+
+    def __init__(self, value):
+        self._value = value
+
+    def __get__(self, obj, cls):
+        return self._value
+
+    def __set__(self, obj, value):
+        self._value = value
+
+    def __delete__(self, obj):
+        del self._value
+
 class ExtClassDescr:
     """Maintains seperate class and instance descriptors for an attribute.
 
@@ -90,6 +104,10 @@
 class DataMixin:
 
     def __init__(self, name, descr, val):
+        if not hasattr(descr, "__get__"):
+            # If the object defined in the metaclass is not a descriptor,
+            # create one for it.
+            descr = SimpleDescriptor(descr)
         super(DataMixin, self).__init__(name, descr)
         self.val = val
 
@@ -102,42 +120,23 @@
     def clsdelete(self):
         del self.val
 
-class ExtClassObject:
-
-    _missing = object()
-
-    def __init__(self, name, instdescr):
-        self.name = name
-        self.instdescr = instdescr
-
-    def __get__(self, obj, cls):
-        if obj is None:
-            return self.clsget(cls)
-        else:
-            return self.instdescr.__get__(obj, cls)
-
-    def __set__(self, obj, cls):
-        if obj is None:
-            return self.clsset(cls)
-        else:
-            if self.instdescr is None:
-                raise AttributeError, self.name
-            return self.instdescr.__set__(obj, cls)
-
-    def __delete__(self, obj, cls):
-        if obj is None:
-            return self.clsdelete(cls)
-        else:
-            if self.instdescr is None:
-                raise AttributeError, self.name
-            return self.instdescr.__delete__(obj, cls)
-
 class ExtClassMethodDescr(MethodMixin, ExtClassDescr):
     pass
 
 class ExtClassDataDescr(DataMixin, ExtClassDescr):
     pass
 
+class ExtClassHookDataDescr(ExtClassDataDescr):
+    # Calls a hook when clsset() is called.
+
+    def __init__(self, name, descr, val, hook):
+        super(ExtClassHookDataDescr, self).__init__(name, descr, val)
+        self.hook = hook
+        
+    def clsset(self, val):
+        self.val = val
+        self.hook()
+
 # The next three classes conspire to make a PersistentFunction
 # behave like a method when found in a class's __dict__.
 
@@ -150,12 +149,10 @@
 
     def __repr__(self):
         if self.im_self is None:
-            kind = "unbound"
+            fmt = "<persistent unbound method %s.%s>"
         else:
-            kind = "bound"
-        return ("<persistent %s method %s.%s of %s>"
-                % (kind, self.im_class.__name__, self.im_func.__name__,
-                   self.im_self))
+            fmt = "<persistent bound method %%s.%%s of %s>" % (self.im_self,)
+        return fmt % (self.im_class.__name__, self.im_func.__name__)
 
     def __call__(self, *args, **kwargs):
         if self.im_self is None:
@@ -208,13 +205,24 @@
 
 class PersistentClassMetaClass(PersistentMetaClass):
 
-    # an attempt to make persistent classes look just like other
+    # An attempt to make persistent classes look just like other
     # persistent objects by providing class attributes and methods
     # that behave like the persistence machinery.
 
-    # the chief limitation of this approach is that class.attr won't
+    # The chief limitation of this approach is that class.attr won't
     # always behave the way it does for normal classes
 
+    # A persistent class can never be a ghost, because there are too
+    # many places where Python will attempt to inspect the class
+    # without using getattr().  As a result, it would be impossible to
+    # guarantee that the class would be unghostified at the right
+    # time.  It's really difficult to guarantee this property without
+    # help from the connection, because a ghost can't be unghosted
+    # until after the connection sets its _p_jar.
+
+    # The hack solution is to have a hook for _p_jar that activates
+    # the object the first time it is set.
+
     __implements__ = IPersistent
 
     _pc_init = False
@@ -226,6 +234,7 @@
     def __new__(meta, name, bases, dict, state=UPTODATE):
         cls = super(PersistentClassMetaClass, meta).__new__(
             meta, name, bases, dict)
+
         # helper functions
         def extend_attr(attr, v):
             prev = findattr(cls, attr, None)
@@ -236,19 +245,32 @@
             setattr(cls, attr, ExtClassMethodDescr(attr, prev, m))
 
         extend_attr("_p_oid", None)
-        extend_attr("_p_jar", None)
         extend_attr("_p_atime", time.time() % 86400)
         extend_attr("_p_state", state)
+        # XXX A persistent class needs a proprety for _p_changed
+        # so that it can be used to register with the transaction
+        # manager.
+        extend_attr("_p_changed", None)
         extend_meth("_p_activate", meta._p_activate)
         extend_meth("_p_deactivate", meta._p_activate)
-        extend_meth("__getstate__", meta.__getstate__)
-        extend_meth("__setstate__", meta.__setstate__)
-        extend_attr("__implements__", meta.__implements__)
+
+        # Create a descriptor that calls _p_activate() when _p_jar is set.
+        inst_jar_descr = findattr(cls, "_p_jar", None)
+        setattr(cls, "_p_jar",
+                ExtClassHookDataDescr("_p_jar", inst_jar_descr, None,
+                                      getattr(cls, "_p_activate")))
 
         for k, v in dict.items():
             if isinstance(v, PersistentFunction):
                 setattr(cls, k, PersistentDescriptor(cls, v))
 
+        # A class could define any of these attributes, thus we
+        # need to create extended descriptors so that the class
+        # and its instances have separate versions.
+        extend_meth("__getstate__", meta.__getstate__)
+        extend_meth("__setstate__", meta.__setstate__)
+        extend_attr("__implements__", meta.__implements__)
+
         cls._pc_init = True
         return cls
 
@@ -319,20 +341,36 @@
         for k, v in dict.items():
             setattr(cls, k, PersistentDescriptor(cls, v))
 
+    # XXX Should the object get marked as a ghost when it is, in fact,
+    # not a ghost?  The most obvious answer is no.  But if we don't
+    # then we need some other attribute that can be used to handle
+    # invalidations of classes and make _p_activate() work as expected.
+    # Need to decide on a good answer.
+
     def _p_deactivate(cls):
         # do nothing but mark the state change for now
         cls._p_state = GHOST
 
     def _p_activate(cls):
-        if cls._p_state == GHOST:
+        # The logic here is:
+        # If the class hasn't finished executing __new__(), don't
+        # try to load its state.
+        # If the class has a jar but no oid, it's a new object
+        # and doesn't have state in the database.
+
+        # XXX Why would an object be marked a ghost, have a jar, and
+        # not have an oid?
+        if cls._p_state == GHOST and cls._pc_init:
             dm = cls._p_jar
-            if dm is not None:
+            if dm is not None and cls._p_oid:
                 cls._p_state = CHANGED
                 try:
                     dm.setstate(cls)
                 finally:
                     # XXX Should really put in special inconsistent state
                     cls._p_state = UPTODATE
+            else:
+                pass # XXX should log here
 
     # Methods below here are not wrapped to be class-only attributes.
     # They are available as methods of classes using this metaclass.