[Checkins] SVN: zodbupdate/trunk/ Fix pickling error when there is missing factories.

Sylvain Viollon sylvain at infrae.com
Wed Oct 6 13:38:43 EDT 2010


Log message for revision 117308:
  Fix pickling error when there is missing factories.
  

Changed:
  U   zodbupdate/trunk/CHANGES.txt
  U   zodbupdate/trunk/src/zodbupdate/serialize.py
  U   zodbupdate/trunk/src/zodbupdate/tests.py
  U   zodbupdate/trunk/src/zodbupdate/update.py

-=-
Modified: zodbupdate/trunk/CHANGES.txt
===================================================================
--- zodbupdate/trunk/CHANGES.txt	2010-10-06 15:12:11 UTC (rev 117307)
+++ zodbupdate/trunk/CHANGES.txt	2010-10-06 17:38:43 UTC (rev 117308)
@@ -5,8 +5,19 @@
 ----------------
 
 - More debug logging shows now the currently processed OID
-  (that is helpful to determine which object misses the factory)
+  (that is helpful to determine which object misses the factory).
 
+- Support for missing factories have been improved: an error used to
+  occur if a pickle needed an update and contained a reference to a
+  missing class (not instance of this class).
+
+  This case is now fixed.
+
+- Python 2.4 is no longer supported. Please stick to version 0.3 if
+  you need Python .4 support.
+
+
+
 0.4 (2010-07-14)
 ----------------
 

Modified: zodbupdate/trunk/src/zodbupdate/serialize.py
===================================================================
--- zodbupdate/trunk/src/zodbupdate/serialize.py	2010-10-06 15:12:11 UTC (rev 117307)
+++ zodbupdate/trunk/src/zodbupdate/serialize.py	2010-10-06 17:38:43 UTC (rev 117308)
@@ -16,17 +16,77 @@
 import cStringIO
 import logging
 import types
+import sys
 
 from ZODB.broken import find_global, Broken, rebuild
 from zodbupdate import utils
 
 logger = logging.getLogger('zodbupdate')
+known_broken_modules = {}
 
 
-def isbroken(symb):
+def is_broken(symb):
+    """Return true if the given symbol is broken.
+    """
     return isinstance(symb, types.TypeType) and Broken in symb.__mro__
 
 
+def create_broken_module_for(symb):
+    """If your pickle refer a broken class (not an instance of it, a
+       reference to the class symbol itself) you have no choice than
+       having this module available in the same module and with the
+       same name, otherwise repickling doesn't work (as both pickle
+       and cPikle __import__ the module, and verify the class symbol
+       is the same).
+    """
+    parts = symb.__module__.split('.')
+    previous_module = None
+    previous_name = None
+    for fullname, name in reversed(
+        [('.'.join(parts[0:p+1]), parts[p]) for p in range(1, len(parts))]):
+        if fullname not in sys.modules:
+            if fullname not in known_broken_modules:
+                module = types.ModuleType(fullname)
+                module.__name__ = name
+                module.__file__ = '<broken module to pickle class reference>'
+                module.__path__ = []
+                known_broken_modules[fullname] = module
+            else:
+                module = known_broken_modules[fullname]
+            if previous_module and previous_name:
+                setattr(module, previous_name, previous_module)
+            previous_module = module
+            previous_name = name
+        else:
+            if previous_module and previous_name:
+                setattr(sys.modules[fullname], previous_name, previous_module)
+                break
+    if symb.__module__ in known_broken_modules:
+        setattr(known_broken_modules[symb.__module__], symb.__name__, symb)
+    else:
+        setattr(sys.modules[symb.__module__], symb.__name__, symb)
+
+
+class BrokenModuleFinder(object):
+    """This broken module finder works with create_broken_module_for.
+    """
+
+    def load_module(self, fullname):
+        module = known_broken_modules[fullname]
+        if fullname not in sys.modules:
+            sys.modules[fullname] = module
+        module.__loader__ = self
+        return module
+
+    def find_module(self, fullname, path=None):
+        if fullname in known_broken_modules:
+            return self
+        return None
+
+
+sys.meta_path.append(BrokenModuleFinder())
+
+
 class NullIterator(object):
     """An empty iterator that doesn't gives any result.
     """
@@ -102,9 +162,10 @@
             return self.__changes[symb_info]
         else:
             symb = find_global(*symb_info, Broken=ZODBBroken)
-            if isbroken(symb):
-                logger.warning(u'Warning: Missing factory for %s' %
-                               u' '.join(symb_info))
+            if is_broken(symb):
+                logger.warning(
+                    u'Warning: Missing factory for %s' % u' '.join(symb_info))
+                create_broken_module_for(symb)
             elif hasattr(symb, '__name__') and hasattr(symb, '__module__'):
                 new_symb_info = (symb.__module__, symb.__name__)
                 if new_symb_info != symb_info:
@@ -174,10 +235,10 @@
         """
         if isinstance(class_meta, tuple):
             symb, args = class_meta
-            if isbroken(symb):
+            if is_broken(symb):
                 symb_info = (symb.__module__, symb.__name__)
-                logger.warning(u'Warning: Missing factory for %s' %
-                               u' '.join(symb_info))
+                logger.warning(
+                    u'Warning: Missing factory for %s' % u' '.join(symb_info))
                 return (symb_info, args)
             elif isinstance(symb, tuple):
                 return self.__update_symb(symb), args
@@ -207,9 +268,9 @@
         try:
             pickler.dump(class_meta)
             pickler.dump(data)
-        except cPickle.PicklingError:
-            # Could not pickle that record, likely due to a broken
-            # class ignore it.
+        except cPickle.PicklingError, error:
+            logger.error('Error while pickling modified record: %s' % error)
+            # Could not pickle that record, skip it.
             return None
 
         output_file.truncate()

Modified: zodbupdate/trunk/src/zodbupdate/tests.py
===================================================================
--- zodbupdate/trunk/src/zodbupdate/tests.py	2010-10-06 15:12:11 UTC (rev 117307)
+++ zodbupdate/trunk/src/zodbupdate/tests.py	2010-10-06 17:38:43 UTC (rev 117308)
@@ -15,10 +15,8 @@
 import ZODB
 import ZODB.broken
 import ZODB.FileStorage
-import logging
 import os
 import persistent
-import pickle
 import sys
 import tempfile
 import transaction

Modified: zodbupdate/trunk/src/zodbupdate/update.py
===================================================================
--- zodbupdate/trunk/src/zodbupdate/update.py	2010-10-06 15:12:11 UTC (rev 117307)
+++ zodbupdate/trunk/src/zodbupdate/update.py	2010-10-06 17:38:43 UTC (rev 117308)
@@ -12,7 +12,6 @@
 #
 ##############################################################################
 
-from ZODB.DB import DB
 from ZODB.FileStorage import FileStorage
 from struct import pack, unpack
 import ZODB.POSException
@@ -20,9 +19,6 @@
 import ZODB.utils
 import cStringIO
 import logging
-import pickle
-import pickletools
-import sys
 import transaction
 import zodbupdate.serialize
 



More information about the checkins mailing list