[Checkins] SVN: zodbupdate/trunk/src/zodbupdate/ Add support to unpickle with python, and an hack for really old records created with ExtensionClass, as the previously existing hack for it have been removed from Python itself.
Sylvain Viollon
sylvain at infrae.com
Wed Jun 9 11:08:05 EDT 2010
Log message for revision 113301:
Add support to unpickle with python, and an hack for really old records created with ExtensionClass, as the previously existing hack for it have been removed from Python itself.
Changed:
U zodbupdate/trunk/src/zodbupdate/main.py
U zodbupdate/trunk/src/zodbupdate/serialize.py
U zodbupdate/trunk/src/zodbupdate/update.py
A zodbupdate/trunk/src/zodbupdate/utils.py
-=-
Modified: zodbupdate/trunk/src/zodbupdate/main.py
===================================================================
--- zodbupdate/trunk/src/zodbupdate/main.py 2010-06-09 08:36:49 UTC (rev 113300)
+++ zodbupdate/trunk/src/zodbupdate/main.py 2010-06-09 15:08:04 UTC (rev 113301)
@@ -20,26 +20,38 @@
import pprint
import sys
import zodbupdate.update
+import zodbupdate.utils
parser = optparse.OptionParser(
description="Updates all references to classes to their canonical location.")
-parser.add_option("-f", "--file",
- help="load FileStorage")
-parser.add_option("-c", "--config",
- help="load storage from config file")
-parser.add_option("-n", "--dry-run", action="store_true",
- help="perform a trial run with no changes made")
-parser.add_option("-s", "--save-renames",
- help="save automatically determined rename rules to file")
-parser.add_option("-q", "--quiet", action="store_true",
- help="suppress non-error messages")
-parser.add_option("-v", "--verbose", action="store_true",
- help="more verbose output")
-parser.add_option("-o", "--oid",
- help="start with provided oid in hex format, ex: 0xaa1203")
-parser.add_option("-d", "--debug", action="store_true",
- help="post mortem pdb on failure")
+parser.add_option(
+ "-f", "--file",
+ help="load FileStorage")
+parser.add_option(
+ "-c", "--config",
+ help="load storage from config file")
+parser.add_option(
+ "-n", "--dry-run", action="store_true",
+ help="perform a trial run with no changes made")
+parser.add_option(
+ "-s", "--save-renames",
+ help="save automatically determined rename rules to file")
+parser.add_option(
+ "-q", "--quiet", action="store_true",
+ help="suppress non-error messages")
+parser.add_option(
+ "-v", "--verbose", action="store_true",
+ help="more verbose output")
+parser.add_option(
+ "-o", "--oid",
+ help="start with provided oid in hex format, ex: 0xaa1203")
+parser.add_option(
+ "-d", "--debug", action="store_true",
+ help="post mortem pdb on failure")
+parser.add_option(
+ "-p", "--pickler", default="C",
+ help="chooser unpickler implementation C or Python (default C)")
class DuplicateFilter(object):
@@ -59,6 +71,10 @@
def main():
options, args = parser.parse_args()
+ if options.pickler not in zodbupdate.utils.UNPICKLERS:
+ raise SystemExit(u'Invalid pickler chosen. Available picklers: %s' %
+ ', '.join(zodbupdate.utils.UNPICKLERS))
+
if options.quiet:
level = logging.ERROR
elif options.verbose:
@@ -98,7 +114,8 @@
dry=options.dry_run,
renames=rename_rules,
start_at=start_at,
- debug=options.debug)
+ debug=options.debug,
+ pickler_name=options.pickler)
try:
updater()
Modified: zodbupdate/trunk/src/zodbupdate/serialize.py
===================================================================
--- zodbupdate/trunk/src/zodbupdate/serialize.py 2010-06-09 08:36:49 UTC (rev 113300)
+++ zodbupdate/trunk/src/zodbupdate/serialize.py 2010-06-09 15:08:04 UTC (rev 113301)
@@ -12,12 +12,14 @@
#
##############################################################################
-from ZODB.broken import find_global, Broken, rebuild
import cPickle
import cStringIO
import logging
import types
+from ZODB.broken import find_global, Broken, rebuild
+from zodbupdate import utils
+
logger = logging.getLogger('zodbupdate')
@@ -25,7 +27,7 @@
return isinstance(symb, types.TypeType) and Broken in symb.__mro__
-class NullIterator:
+class NullIterator(object):
"""An empty iterator that doesn't gives any result.
"""
@@ -62,7 +64,7 @@
self.__Broken_state__)
-class ZODBReference:
+class ZODBReference(object):
"""Class to remenber reference we don't want to touch.
"""
@@ -70,7 +72,7 @@
self.ref = ref
-class ObjectRenamer:
+class ObjectRenamer(object):
"""This load and save a ZODB record, modifying all references to
renamed class according the given renaming rules:
@@ -81,12 +83,13 @@
- in class information (first pickle of the record).
"""
- def __init__(self, changes):
+ def __init__(self, changes, pickler_name='C'):
self.__added = dict()
self.__changes = dict()
for old, new in changes.iteritems():
self.__changes[tuple(old.split(' '))] = tuple(new.split(' '))
self.__changed = False
+ self.__pickler_name = pickler_name
def __update_symb(self, symb_info):
"""This method look in a klass or symbol have been renamed or
@@ -142,14 +145,12 @@
return ZODBReference(['m', (database_name, oid, klass_info)])
return ZODBReference(reference)
- def __unpickler(self, pickle):
+ def __unpickler(self, input_file):
"""Create an unpickler with our custom global symbol loader
and reference resolver.
"""
- unpickler = cPickle.Unpickler(pickle)
- unpickler.persistent_load = self.__persistent_load
- unpickler.find_global = self.__find_global
- return unpickler
+ return utils.UNPICKLERS[self.__pickler_name](
+ input_file, self.__persistent_load, self.__find_global)
def __persistent_id(self, obj):
"""Save the given object as a reference only if it was a
@@ -159,11 +160,11 @@
return None
return obj.ref
- def __pickler(self, output):
+ def __pickler(self, output_file):
"""Create a pickler able to save to the given file, objects we
loaded while paying attention to any reference we loaded.
"""
- pickler = cPickle.Pickler(output, 1)
+ pickler = cPickle.Pickler(output_file, 1)
pickler.persistent_id = self.__persistent_id
return pickler
@@ -196,7 +197,9 @@
class_meta = self.__update_class_meta(class_meta)
- if not self.__changed:
+ if not (self.__changed or
+ (hasattr(unpickler, 'need_repickle') and
+ unpickler.need_repickle())):
return None
output_file = cStringIO.StringIO()
Modified: zodbupdate/trunk/src/zodbupdate/update.py
===================================================================
--- zodbupdate/trunk/src/zodbupdate/update.py 2010-06-09 08:36:49 UTC (rev 113300)
+++ zodbupdate/trunk/src/zodbupdate/update.py 2010-06-09 15:08:04 UTC (rev 113301)
@@ -35,10 +35,11 @@
"""Update class references for all current objects in a storage."""
def __init__(self, storage, dry=False, renames=None,
- start_at='0x00', debug=False):
+ start_at='0x00', debug=False, pickler_name='C'):
self.dry = dry
self.storage = storage
- self.processor = zodbupdate.serialize.ObjectRenamer(renames or {})
+ self.processor = zodbupdate.serialize.ObjectRenamer(
+ renames or {}, pickler_name)
self.start_at = start_at
self.debug = debug
Added: zodbupdate/trunk/src/zodbupdate/utils.py
===================================================================
--- zodbupdate/trunk/src/zodbupdate/utils.py (rev 0)
+++ zodbupdate/trunk/src/zodbupdate/utils.py 2010-06-09 15:08:04 UTC (rev 113301)
@@ -0,0 +1,71 @@
+##############################################################################
+#
+# Copyright (c) 2009-2010 Zope Corporation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
+
+import cPickle
+import pickle
+import logging
+
+logger = logging.getLogger('zodbupdate')
+
+
+class PythonUnpickler(pickle.Unpickler):
+ """Use Python unpickler.
+ """
+ dispatch = pickle.Unpickler.dispatch.copy()
+
+ def __init__(self, input_file, persistent_load, find_global):
+ pickle.Unpickler.__init__(self, input_file)
+ self.__repickle = False
+ self.persistent_load = persistent_load
+ self.find_class = find_global
+
+ def load_reduce(self):
+ stack = self.stack
+ args = stack.pop()
+ func = stack[-1]
+ if args is None:
+ # Hack for old ExtensionClass that was removed from Python
+ # in 2.6. We set repickle to True to trigger a repickling
+ # of this pickle later on to get ride of those records.
+ value = func.__new__(func)
+ self.__repickle = True
+ logger.warning(
+ u'Warning: Pickle contains ExtensionClass hack for %s' % func)
+ else:
+ value = func(*args)
+ stack[-1] = value
+
+ def need_repickle(self):
+ """Tell the user if it is necessary to repickle the pickle to
+ update it.
+ """
+ return self.__repickle
+
+ dispatch[pickle.REDUCE] = load_reduce
+
+
+
+def CUnpickler(input_file, persistent_load, find_global):
+ """Use C unpickler.
+ """
+ unpickler = cPickle.Unpickler(input_file)
+ unpickler.persistent_load = persistent_load
+ unpickler.find_global = find_global
+ return unpickler
+
+
+UNPICKLERS = {
+ 'Python': PythonUnpickler,
+ 'C': CUnpickler}
+
Property changes on: zodbupdate/trunk/src/zodbupdate/utils.py
___________________________________________________________________
Added: svn:keywords
+ Author Date Id Revision
More information about the checkins
mailing list