[Checkins] SVN: zodbupgrade/trunk/src/zodbupgrade/ Finish refactoring, complete and update tests:
Christian Theune
ct at gocept.com
Sun Jun 14 09:19:30 EDT 2009
Log message for revision 100948:
Finish refactoring, complete and update tests:
- The updater is now a class and thus will be able to hold state about which
factories were translated.
Changed:
U zodbupgrade/trunk/src/zodbupgrade/analyze.py
U zodbupgrade/trunk/src/zodbupgrade/picklefilter.py
U zodbupgrade/trunk/src/zodbupgrade/tests.py
-=-
Modified: zodbupgrade/trunk/src/zodbupgrade/analyze.py
===================================================================
--- zodbupgrade/trunk/src/zodbupgrade/analyze.py 2009-06-14 12:43:13 UTC (rev 100947)
+++ zodbupgrade/trunk/src/zodbupgrade/analyze.py 2009-06-14 13:19:30 UTC (rev 100948)
@@ -16,11 +16,12 @@
import StringIO
import ZODB.broken
import ZODB.utils
+import logging
import pickle
import pickletools
import sys
import transaction
-import logging
+import zodbupgrade.picklefilter
logger = logging.getLogger('zodbupgrade')
@@ -29,94 +30,88 @@
pass
-def update_factory_references(op, arg):
- """Check a pickle operation for moved or missing factory references.
- Returns an updated (op, arg) tuple using the canonical reference for the
- factory as would be created if the pickle was unpickled and re-pickled.
+class Updater(object):
+ """Update class references for all current objects in a storage."""
- """
- if op.code not in 'ci':
- return
+ def __init__(self, storage, dry=False, ignore_missing=False):
+ self.ignore_missing = ignore_missing
+ self.dry = dry
+ self.storage = storage
- factory_module, factory_name = arg.split(' ')
- module = __import__(factory_module, globals(), {}, [factory_name])
- factory = getattr(module, factory_name)
- # XXX Handle missing factories
+ def __call__(self):
+ t = transaction.Transaction()
+ self.storage.tpc_begin(t)
+ t.note('Updated factory references using `zodbupgrade`.')
- if not hasattr(factory, '__name__'):
- logger.warn(
- "factory %r does not have __name__: "
- "can't check canonical location" % factory)
- return
- if not hasattr(factory, '__module__'):
- # TODO: This case isn't covered with a test. I just
- # couldn't provoke a factory to not have a __module__ but
- # users reported this issue to me.
- logger.warn(
- "factory %r does not have __module__: "
- "can't check canonical location" % factory)
- return
+ for oid, serial, current in self.records:
+ new = self.update_record(current)
+ if new == current:
+ continue
+ self.storage.store(oid, serial, new, '', t)
- # XXX Log for later reuse
- new_arg = '%s %s' % (factory.__module__, factory.__name__)
- return op, new_arg
+ if self.dry:
+ self.storage.tpc_abort(t)
+ else:
+ self.storage.tpc_vote(t)
+ self.storage.tpc_finish(t)
+ @property
+ def records(self):
+ next = None
+ while True:
+ oid, tid, data, next = self.storage.record_iternext(next)
+ yield oid, tid, StringIO.StringIO(data)
+ if next is None:
+ break
-def each_record(storage):
- next = None
- while True:
- oid, tid, data, next = storage.record_iternext(next)
- yield StringIO.StringIO(data)
- if next is None:
- break
+ def update_record(self, old):
+ new = ''
+ for i in range(2):
+ # ZODB data records consist of two concatenated pickles, so the
+ # following needs to be done twice:
+ new += zodbupgrade.picklefilter.filter(
+ self.update_operation, old)
+ return new
+ def update_operation(self, code, arg):
+ """Check a pickle operation for moved or missing factory references.
-def update_storage(storage):
- """Update
- and updaAnalyzes class references of current records of a storage.
+ Returns an updated (code, arg) tuple using the canonical reference for the
+ factory as would be created if the pickle was unpickled and re-pickled.
- Look for missing or moved classes and return a list of OIDs that need
- updating, a list of classes that are missing, and a list of rewrites.
+ """
+ if code not in 'ci':
+ return
- """
- logger.info('Analyzing database ...')
- for count, data in enumerate(each_record(storage)):
- if not count % 5000:
- logger.info(' %s objects' % count)
+ # XXX Handle missing factories
+ factory_module, factory_name = arg.split(' ')
+ module = __import__(factory_module, globals(), {}, [factory_name])
+ try:
+ factory = getattr(module, factory_name)
+ except AttributeError:
+ raise ValueError()
- # ZODB records consist of two concatenated pickles, so the following
- # needs to be done twice:
- for i in range(2):
- zodbupgrade.picklefilter.filter(
- update_factory_references, pickle_data)
+ if not hasattr(factory, '__name__'):
+ logger.warn(
+ "factory %r does not have __name__: "
+ "can't check canonical location" % factory)
+ return
+ if not hasattr(factory, '__module__'):
+ # TODO: This case isn't covered with a test. I just
+ # couldn't provoke a factory to not have a __module__ but
+ # users reported this issue to me.
+ logger.warn(
+ "factory %r does not have __module__: "
+ "can't check canonical location" % factory)
+ return
- logger.info(' Analyzation completed.')
+ # XXX Log for later reuse
+ new_arg = '%s %s' % (factory.__module__, factory.__name__)
+ return code, new_arg
-def update_storage(storage, ignore_missing=False, dry=False):
- missing_classes, rewrites_found, oids = analyze_storage(storage)
- if missing_classes and not ignore_missing:
- raise MissingClasses(missing_classes)
- if rewrites_found:
- logger.info("Found moved classes:")
- for (old_mod, old_name), (new_mod, new_name) in rewrites_found.items():
- logger.info("%s.%s -> %s.%s" % (old_mod, old_name, new_mod, new_name))
- logger.info("%i objects need updating" % len(oids))
-
- if dry:
- logger.info('Dry run selected, aborting.')
- return
-
- logger.info('Starting database update')
- db = DB(storage)
- connection = db.open()
- for oid in oids:
- obj = connection.get(oid)
- obj._p_changed = True
- t = transaction.get()
- t.note('Class references updated by `zodbupgrade`')
- transaction.commit()
- db.close()
- logger.info('Database update completed')
+def main(storage, **kw):
+ updater = Updater(storage, **kw)
+ updater()
Modified: zodbupgrade/trunk/src/zodbupgrade/picklefilter.py
===================================================================
--- zodbupgrade/trunk/src/zodbupgrade/picklefilter.py 2009-06-14 12:43:13 UTC (rev 100947)
+++ zodbupgrade/trunk/src/zodbupgrade/picklefilter.py 2009-06-14 13:19:30 UTC (rev 100948)
@@ -101,7 +101,7 @@
generated = generators[opcode](arg)
chunk += generated
else:
- raise ValueError('Unknown opcode: %s')
+ raise ValueError('Unknown opcode: %s' % (opcode,))
return chunk
@@ -116,8 +116,10 @@
"""
new = StringIO.StringIO()
- for op, arg, pos in pickletools.genops(p):
+ for op, arg, pos in pickletools.genops(pickle_data):
+ op = op.code
result = f(op, arg)
- op, arg = result if result is not None else op, arg
+ if result is not None:
+ op, arg = result
new.write(to_pickle_chunk(op, arg))
return new.getvalue()
Modified: zodbupgrade/trunk/src/zodbupgrade/tests.py
===================================================================
--- zodbupgrade/trunk/src/zodbupgrade/tests.py 2009-06-14 12:43:13 UTC (rev 100947)
+++ zodbupgrade/trunk/src/zodbupgrade/tests.py 2009-06-14 13:19:30 UTC (rev 100948)
@@ -49,6 +49,11 @@
self.db = None
self.reopen_db()
+ def update(self):
+ updater = zodbupgrade.analyze.Updater(self.storage)
+ updater()
+ self.storage.close()
+
def tearDown(self):
zodbupgrade.analyze.logger.removeFilter(ignore)
del sys.modules['module1']
@@ -78,8 +83,7 @@
self.db.close()
self.reopen_storage()
- self.assertRaises(ValueError,
- zodbupgrade.analyze.update_storage, self.storage)
+ self.assertRaises(ValueError, self.update)
def test_factory_renamed(self):
# Create a ZODB with an object referencing a factory, then
@@ -95,8 +99,9 @@
self.db.close()
self.reopen_storage()
- zodbupgrade.analyze.update_storage(self.storage)
+ self.update()
+
del sys.modules['module1'].Factory
self.reopen_db()
@@ -126,7 +131,7 @@
transaction.commit()
self.db.close()
self.reopen_storage()
- zodbupgrade.analyze.update_storage(self.storage)
+ self.update()
self.assertEquals('module1', self.root['test'].__class__.__module__)
self.assertEquals('AnonymousFactory', self.root['test'].__class__.__name__)
More information about the Checkins
mailing list