[ZODB-Dev] Support for graceful ZODB Class renaming
Shane Hathaway
shane@zope.com
Thu, 16 Jan 2003 15:56:37 -0500
This is a multi-part message in MIME format.
--------------080301000109070609020804
Content-Type: text/plain; charset=us-ascii; format=flowed
Content-Transfer-Encoding: 7bit
Jim Fulton wrote:
> 2. Another approach would be to write a data conversion utility for the
> database. This would require a conversion file much like the alias
> file
> described above.
>
> You might have to shut down the database while you do the
> conversion, resulting in down time, however, if you combined the
> aliasing approach with conversion, you could avoid the down time.
>
> Suppose, for example, that you had an alias table mapping old to
> new dotted object names. We can use the database without
> modifications if we provide a "global" loader that uses this alias
> file (or if we have a utility that manipulates sys.modules on
> start up).
>
> We can write a utility for file storage, similar to a
> pack, that makes a live copy of the storage file, containing
> converted records and that switches to the new file when the
> conversion is complete. For many other storages, we could perform
> the fix ups in-place, which is even more attractive.
A while back I wrote a utility that did something a lot like this. It's
a Zope external method that accepts three arguments: an old
package/module name, a new package/module name, and a root object. It
traverses all objects reachable from the root object, loading objects of
old classes using new classes, and storing objects that need to be
changed. It works with any storage and any ZODB objects and does not
require any database shutdown.
It monkey-patches ZODB to do it, though. :-|
It's on SourceForge, but SourceForge's ViewCVS seems to be dead at the
moment, so I attached my code. It doesn't have enough comments, but the
neat part is the class translation function. It accepts a class module
and name and returns a class module and name. The ZODB traverser calls
this function for every class it finds. If the translation function
returns something other than what it was given, the ZODB traverser knows
it needs to change the relevant pickle. It separates concerns.
Shane
--------------080301000109070609020804
Content-Type: text/plain;
name="change_modules.py"
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
filename="change_modules.py"
def changeClassModules(self, old_package_name, new_package_name,
root_object=None):
get_transaction().commit(1) # Make sure the current state is stored
jar = self._p_jar
if root_object is None:
root_object = jar.root()
root_oid = root_object._p_oid
stats = [-1, -1]
def translation_func(class_spec,
# Bound to the function:
old_package_name=old_package_name,
new_package_name=new_package_name,
opd=old_package_name + '.',
opdlen=len(old_package_name) + 1,
TupleType=type(())):
"""Translates a class specifications.
If the class_spec takes the form (module, name) and the
module starts with the specified old package name, the spec
is turned into a reference to the new package name.
Returns either the original spec or the translated spec.
"""
if isinstance(class_spec, TupleType) and len(class_spec) == 2:
# class is specified as module, name
module, name = class_spec
if module == old_package_name:
module = new_package_name
class_spec = (module, name)
elif module[:opdlen] == opd:
module = '%s.%s' % (new_package_name, module[opdlen:])
class_spec = (module, name)
return class_spec
jar.onCommitAction('_changeClassDuringCommit',
translation_func, root_oid, stats)
get_transaction().commit(1)
return ('Done. Visited %d objects and changed %d. '
'See the event log for more details.' % tuple(stats))
if 1:
from cStringIO import StringIO
from cPickle import Pickler, Unpickler
from ZODB.ExportImport import Ghost, persistent_id
from zLOG import LOG, INFO, WARNING
def _changeClassDuringCommit(self, transaction, translation_func,
root_oid, stats=None):
'''Recursively changes the class of persistent objects.
Invoked as a method of Connection by the transaction manager
mid subtransaction commit.
'''
to_visit = [root_oid]
seen = {root_oid:1}
storage = self._storage
trigger = [0]
change_count = 0
def persistent_load(oid,
# Bound to the function:
to_visit=to_visit, seen=seen, Ghost=Ghost,
trigger=trigger,
translation_func=translation_func,
TupleType=type(())):
"Adds to the list of oids to visit."
if isinstance(oid, TupleType):
simple_oid, class_spec = oid
new_class_spec = translation_func(class_spec)
if new_class_spec != class_spec:
# Change this reference.
trigger[0] = 1
oid = (simple_oid, new_class_spec)
else:
simple_oid = oid
if not seen.has_key(simple_oid):
seen[simple_oid] = 1
to_visit.append(simple_oid)
g = Ghost()
g.oid = oid
return g
def find_global(module, name,
# Bound to the function:
translation_func=translation_func, trigger=trigger,
_silly=('__doc__',), _globals={}):
class_spec = (module, name)
new_class_spec = translation_func(class_spec)
if new_class_spec != class_spec:
# Change this reference.
trigger[0] = 1
module, name = new_class_spec
m = __import__(module, _globals, _globals, _silly)
return getattr(m, name)
version = self._version
while to_visit:
oid = to_visit.pop()
p, serial = storage.load(oid, version)
convert_ok = 1
trigger[0] = 0
pfile = StringIO(p)
unpickler = Unpickler(pfile)
unpickler.persistent_load = persistent_load
unpickler.find_global = find_global
class_spec, args = unpickler.load()
try:
# Get persistent_load() and find_global() called.
state = unpickler.load()
except (ImportError, AttributeError):
# There are problems with this object.
LOG('changeClasses', WARNING, 'Could not load state for OID '
'%s, class %s' % (repr(oid), repr(class_spec)))
convert_ok = 0
if convert_ok:
# Figure out the new class.
new_class_spec = translation_func(class_spec)
if new_class_spec != class_spec:
# Change this reference.
trigger[0] = 1
if trigger[0]:
# Store the object with changes to classes.
newp = StringIO()
pickler = Pickler(newp, 1)
pickler.persistent_id = persistent_id
pickler.dump((new_class_spec, args))
pickler.dump(state)
data = newp.getvalue()
self.invalidate(oid)
self._invalidating.append(oid)
if self._cache.has_key(oid):
# Ghostify.
ob = self._cache[oid]
if not ob._p_changed:
ob._p_changed = None
del self._cache[oid]
storage.store(oid, serial, data, version, transaction)
change_count = change_count + 1
if class_spec != new_class_spec:
LOG('changeClass', INFO,
'Changed OID %s from class %s to class %s' % (
repr(oid), repr(class_spec), repr(new_class_spec)))
else:
LOG('changeClass', INFO,
'Rewrote OID %s with new references' % repr(oid))
if stats is not None:
stats[:] = [len(seen), change_count]
from ZODB.Connection import Connection
Connection._changeClassDuringCommit = _changeClassDuringCommit
--------------080301000109070609020804--