[Checkins] SVN: Sandbox/ctheune/zodbupgrade/ Restructure, fix copyright, add entry point for creating executable.
Christian Theune
ct at gocept.com
Thu May 28 03:08:42 EDT 2009
Log message for revision 100497:
Restructure, fix copyright, add entry point for creating executable.
Changed:
_U Sandbox/ctheune/zodbupgrade/
D Sandbox/ctheune/zodbupgrade/analyze.py
A Sandbox/ctheune/zodbupgrade/bootstrap.py
U Sandbox/ctheune/zodbupgrade/buildout.cfg
D Sandbox/ctheune/zodbupgrade/create.py
D Sandbox/ctheune/zodbupgrade/debug.py
U Sandbox/ctheune/zodbupgrade/setup.py
D Sandbox/ctheune/zodbupgrade/src/klass2.py
D Sandbox/ctheune/zodbupgrade/src/klass3.py
A Sandbox/ctheune/zodbupgrade/src/zodbupgrade/
A Sandbox/ctheune/zodbupgrade/src/zodbupgrade/__init__.py
A Sandbox/ctheune/zodbupgrade/src/zodbupgrade/analyze.py
A Sandbox/ctheune/zodbupgrade/src/zodbupgrade/main.py
-=-
Property changes on: Sandbox/ctheune/zodbupgrade
___________________________________________________________________
Added: svn:ignore
+ develop-eggs
parts
.installed.cfg
bin
Deleted: Sandbox/ctheune/zodbupgrade/analyze.py
===================================================================
--- Sandbox/ctheune/zodbupgrade/analyze.py 2009-05-28 06:54:27 UTC (rev 100496)
+++ Sandbox/ctheune/zodbupgrade/analyze.py 2009-05-28 07:08:42 UTC (rev 100497)
@@ -1,71 +0,0 @@
-# vim:fileencoding=utf-8
-# Copyright (c) 2008 gocept gmbh & co. kg
-# See also LICENSE.txt
-
-import sys
-import StringIO
-from ZODB.FileStorage import FileStorage
-from ZODB.DB import DB
-import ZODB.broken
-import transaction
-import pickle
-import pickletools
-import ZODB.utils
-
-SAFE_OPS = 'IJKML\x8a\x8bSTUN\x88\x89VXFG]ael)t\x85\x86\x87}dsu02(1ghjpqrRbo\x81\x80.PQ'
-KNOWN_HARD = 'ci'
-
-s = FileStorage('Data.fs')
-d = DB(s)
-c = d.open()
-r = c.root()
-
-missing_classes = set()
-oids_to_rewrite = set()
-rewrites_found = set()
-
-def find_missing_classes(oid, data):
- # First part of the pickle: the object factory
- for op, arg, pos in pickletools.genops(data):
- if op.code in SAFE_OPS:
- continue
- elif op.code in KNOWN_HARD:
- module_name, symbol = arg.split(' ')
- try:
- module = __import__(module_name, globals(), {}, [symbol])
- factory = getattr(module, symbol)
- except (ImportError, AttributeError):
- missing_classes.add('%s.%s' % (module_name, symbol))
- else:
- if ((factory.__module__, factory.__name__) !=
- (module_name, symbol)):
- # The factory is reachable but it's not the
- # canonical location. Mark object for updating.
- rewrites_found.add(((module_name, symbol),
- (factory.__module__, factory.__name__)))
- oids_to_rewrite.add(oid)
- else:
- raise ValueError('Unknown pickle opcode %r' % op.code)
-
-next = None
-while True:
- oid, tid, data, next = s.record_iternext(next)
- pickle_data = StringIO.StringIO(data)
- find_missing_classes(oid, pickle_data)
- find_missing_classes(oid, pickle_data)
- if next is None:
- break
-
-if missing_classes:
- print "The following classes are missing:"
- for class_ in sorted(missing_classes):
- print class_
-else:
- print "All classes found."
- print "%s moved classes detected, will update %s objects" % (len(rewrites_found), len(oids_to_rewrite))
- for (old_mod, old_name), (new_mod, new_name) in rewrites_found:
- print "%s.%s -> %s.%s" % (old_mod, old_name, new_mod, new_name)
- for oid in oids_to_rewrite:
- obj = c.get(oid)
- obj._p_changed = True
- transaction.commit()
Added: Sandbox/ctheune/zodbupgrade/bootstrap.py
===================================================================
--- Sandbox/ctheune/zodbupgrade/bootstrap.py (rev 0)
+++ Sandbox/ctheune/zodbupgrade/bootstrap.py 2009-05-28 07:08:42 UTC (rev 100497)
@@ -0,0 +1,77 @@
+##############################################################################
+#
+# Copyright (c) 2006 Zope Corporation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
+"""Bootstrap a buildout-based project
+
+Simply run this script in a directory containing a buildout.cfg.
+The script accepts buildout command-line options, so you can
+use the -c option to specify an alternate configuration file.
+
+$Id: bootstrap.py 90478 2008-08-27 22:44:46Z georgyberdyshev $
+"""
+
+import os, shutil, sys, tempfile, urllib2
+
+tmpeggs = tempfile.mkdtemp()
+
+is_jython = sys.platform.startswith('java')
+
+try:
+ import pkg_resources
+except ImportError:
+ ez = {}
+ exec urllib2.urlopen('http://peak.telecommunity.com/dist/ez_setup.py'
+ ).read() in ez
+ ez['use_setuptools'](to_dir=tmpeggs, download_delay=0)
+
+ import pkg_resources
+
+if sys.platform == 'win32':
+ def quote(c):
+ if ' ' in c:
+ return '"%s"' % c # work around spawn lamosity on windows
+ else:
+ return c
+else:
+ def quote (c):
+ return c
+
+cmd = 'from setuptools.command.easy_install import main; main()'
+ws = pkg_resources.working_set
+
+if is_jython:
+ import subprocess
+
+ assert subprocess.Popen([sys.executable] + ['-c', quote(cmd), '-mqNxd',
+ quote(tmpeggs), 'zc.buildout'],
+ env=dict(os.environ,
+ PYTHONPATH=
+ ws.find(pkg_resources.Requirement.parse('setuptools')).location
+ ),
+ ).wait() == 0
+
+else:
+ assert os.spawnle(
+ os.P_WAIT, sys.executable, quote (sys.executable),
+ '-c', quote (cmd), '-mqNxd', quote (tmpeggs), 'zc.buildout',
+ dict(os.environ,
+ PYTHONPATH=
+ ws.find(pkg_resources.Requirement.parse('setuptools')).location
+ ),
+ ) == 0
+
+ws.add_entry(tmpeggs)
+ws.require('zc.buildout')
+import zc.buildout.buildout
+zc.buildout.buildout.main(sys.argv[1:] + ['bootstrap'])
+shutil.rmtree(tmpeggs)
Property changes on: Sandbox/ctheune/zodbupgrade/bootstrap.py
___________________________________________________________________
Added: svn:eol-style
+ native
Modified: Sandbox/ctheune/zodbupgrade/buildout.cfg
===================================================================
--- Sandbox/ctheune/zodbupgrade/buildout.cfg 2009-05-28 06:54:27 UTC (rev 100496)
+++ Sandbox/ctheune/zodbupgrade/buildout.cfg 2009-05-28 07:08:42 UTC (rev 100497)
@@ -1,17 +1,8 @@
[buildout]
-develop = .
-parts = py py2 tags
+develop = . ../kita
+parts = zodbupgrade
-[tags]
-recipe = z3c.recipe.tag:tags
-eggs = zodbupgrade
-
-[py2]
+[zodbupgrade]
recipe = zc.recipe.egg
-eggs = ZODB3==3.8.1
-interpreter = py2
-
-[py]
-recipe = zc.recipe.egg
eggs = zodbupgrade
-interpreter = py
+ kita
Deleted: Sandbox/ctheune/zodbupgrade/create.py
===================================================================
--- Sandbox/ctheune/zodbupgrade/create.py 2009-05-28 06:54:27 UTC (rev 100496)
+++ Sandbox/ctheune/zodbupgrade/create.py 2009-05-28 07:08:42 UTC (rev 100497)
@@ -1,20 +0,0 @@
-# vim:fileencoding=utf-8
-# Copyright (c) 2008 gocept gmbh & co. kg
-# See also LICENSE.txt
-
-from ZODB.FileStorage import FileStorage
-from ZODB.DB import DB
-
-s = FileStorage('Data.fs')
-d = DB(s)
-c = d.open()
-r = c.root()
-
-import klass
-
-r['foo'] = klass.P1()
-r['foo'].x = klass.P2()
-
-import transaction
-transaction.commit()
-
Deleted: Sandbox/ctheune/zodbupgrade/debug.py
===================================================================
--- Sandbox/ctheune/zodbupgrade/debug.py 2009-05-28 06:54:27 UTC (rev 100496)
+++ Sandbox/ctheune/zodbupgrade/debug.py 2009-05-28 07:08:42 UTC (rev 100497)
@@ -1,12 +0,0 @@
-# vim:fileencoding=utf-8
-# Copyright (c) 2008 gocept gmbh & co. kg
-# See also LICENSE.txt
-
-from ZODB.FileStorage import FileStorage
-from ZODB.DB import DB
-
-s = FileStorage('Data.fs')
-d = DB(s)
-c = d.open()
-r = c.root()
-
Modified: Sandbox/ctheune/zodbupgrade/setup.py
===================================================================
--- Sandbox/ctheune/zodbupgrade/setup.py 2009-05-28 06:54:27 UTC (rev 100496)
+++ Sandbox/ctheune/zodbupgrade/setup.py 2009-05-28 07:08:42 UTC (rev 100497)
@@ -1,17 +1,34 @@
-# vim:fileencoding=utf-8
-# Copyright (c) 2008 gocept gmbh & co. kg
-# See also LICENSE.txt
+##############################################################################
+#
+# Copyright (c) 2009 Zope Corporation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
from setuptools import setup, find_packages
setup(name='zodbupgrade',
- version='0',
+ author='Zope Developers',
+ author_email='zodb-dev at zope.org',
+ description=
+ 'Transparently update ZODB class references to their '
+ 'canonical locations',
+ version='0.1',
package_dir={'': 'src'},
packages=find_packages('src'),
include_package_data=True,
install_requires=[
- 'ZODB3==3.8.1',
+ 'ZODB3',
'setuptools'
],
- )
+ entry_points = dict(
+ console_scripts =
+ ['zodbupgrade = zodbupgrade.main:main']))
Deleted: Sandbox/ctheune/zodbupgrade/src/klass2.py
===================================================================
--- Sandbox/ctheune/zodbupgrade/src/klass2.py 2009-05-28 06:54:27 UTC (rev 100496)
+++ Sandbox/ctheune/zodbupgrade/src/klass2.py 2009-05-28 07:08:42 UTC (rev 100497)
@@ -1,10 +0,0 @@
-# vim:fileencoding=utf-8
-# Copyright (c) 2008 gocept gmbh & co. kg
-# See also LICENSE.txt
-
-import persistent
-from klass3 import P1Replacement as P1
-
-
-class P2(object):
- pass
Deleted: Sandbox/ctheune/zodbupgrade/src/klass3.py
===================================================================
--- Sandbox/ctheune/zodbupgrade/src/klass3.py 2009-05-28 06:54:27 UTC (rev 100496)
+++ Sandbox/ctheune/zodbupgrade/src/klass3.py 2009-05-28 07:08:42 UTC (rev 100497)
@@ -1,8 +0,0 @@
-# vim:fileencoding=utf-8
-# Copyright (c) 2008 gocept gmbh & co. kg
-# See also LICENSE.txt
-
-import persistent
-
-class P1Replacement(persistent.Persistent):
- pass
Added: Sandbox/ctheune/zodbupgrade/src/zodbupgrade/__init__.py
===================================================================
--- Sandbox/ctheune/zodbupgrade/src/zodbupgrade/__init__.py (rev 0)
+++ Sandbox/ctheune/zodbupgrade/src/zodbupgrade/__init__.py 2009-05-28 07:08:42 UTC (rev 100497)
@@ -0,0 +1 @@
+# Make this a Python package
Property changes on: Sandbox/ctheune/zodbupgrade/src/zodbupgrade/__init__.py
___________________________________________________________________
Added: svn:eol-style
+ native
Added: Sandbox/ctheune/zodbupgrade/src/zodbupgrade/analyze.py
===================================================================
--- Sandbox/ctheune/zodbupgrade/src/zodbupgrade/analyze.py (rev 0)
+++ Sandbox/ctheune/zodbupgrade/src/zodbupgrade/analyze.py 2009-05-28 07:08:42 UTC (rev 100497)
@@ -0,0 +1,123 @@
+##############################################################################
+#
+# Copyright (c) 2009 Zope Corporation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
+
+from ZODB.DB import DB
+import StringIO
+import ZODB.broken
+import ZODB.utils
+import pickle
+import pickletools
+import sys
+import transaction
+import logging
+
+logger = logging.getLogger('zodbupgrade')
+
+
+SAFE_OPS = 'IJKML\x8a\x8bSTUN\x88\x89VXFG]ael)t\x85\x86\x87}dsu02(1ghjpqrRbo\x81\x80.PQ'
+KNOWN_HARD = 'ci'
+
+
+def find_factory_references(pickle):
+ """Analyze a pickle for moved or missing factory references.
+
+ Returns:
+
+ - factories whose dotted name could be imported but stem from an
+ indirect import (this is a dictionary)
+
+ - factories whose dotted name could not be imported (an iterable)
+
+ """
+ missing_classes = set()
+ rewrites_found = dict()
+ for op, arg, pos in pickletools.genops(pickle):
+ if op.code in SAFE_OPS:
+ continue
+ elif op.code in KNOWN_HARD:
+ module_name, symbol = arg.split(' ')
+ try:
+ module = __import__(module_name, globals(), {}, [symbol])
+ factory = getattr(module, symbol)
+ except (ImportError, AttributeError):
+ missing_classes.add('%s.%s' % (module_name, symbol))
+ else:
+ # XXX Special case broken objects. They don't have __module__
+ if ((factory.__module__, factory.__name__) !=
+ (module_name, symbol)):
+ # The factory is reachable but it's not the
+ # canonical location. Mark object for updating.
+ rewrites_found[(module_name, symbol)] = (
+ factory.__module__, factory.__name__)
+ else:
+ raise ValueError('Unknown pickle opcode %r' % op.code)
+ return rewrites_found, missing_classes
+
+
+def analyze_storage(storage):
+ """Analyzes class references of current records of a storage.
+
+ Look for missing or moved classes and return a list of OIDs that need
+ updating, a list of classes that are missing, and a list of rewrites.
+
+ """
+ missing_classes = set()
+ rewrites_found = set()
+ oids_rewrite = set()
+
+ count = 0
+ next = None
+ while True:
+ oid, tid, data, next = storage.record_iternext(next)
+ count += 1
+ pickle_data = StringIO.StringIO(data)
+
+ if not count % 1000:
+ logger.info(
+ 'Analyzed %i objects. Found %i moved classes and %i missing '
+ 'classes so far.' % (count, len(rewrites_found), len(missing_classes)))
+
+ # ZODB records consist of two concatenated pickles, so the following
+ # needs to be done twice:
+ for i in range(2):
+ r, m = find_factory_references(pickle_data)
+ if r:
+ oids_rewrite.add(oid)
+ rewrites_found.update(r)
+ missing_classes.update(m)
+
+ if next is None:
+ break
+ return missing_classes, rewrites_found, oids_rewrite
+
+
+def update_storage(storage):
+ missing_classes, rewrites_found, oids = analyze_storage(storage)
+ if missing_classes:
+ raise ValueError(missing_classes)
+
+ print "Rewriting database with mapping:"
+ for (old_mod, old_name), (new_mod, new_name) in rewrites_found:
+ print "%s.%s -> %s.%s" % (old_mod, old_name, new_mod, new_name)
+
+ print "%i objects need rewriting" % len(oids)
+
+ db = ZODB.DB.DB(storage)
+ connection = db.open()
+ for oid in oids:
+ obj = connection.get(oid)
+ obj._p_changed = True
+ t = transaction.get()
+ t.note('Class references updated by `zodbupgrade`')
+ transaction.commit()
Property changes on: Sandbox/ctheune/zodbupgrade/src/zodbupgrade/analyze.py
___________________________________________________________________
Added: svn:eol-style
+ native
Added: Sandbox/ctheune/zodbupgrade/src/zodbupgrade/main.py
===================================================================
--- Sandbox/ctheune/zodbupgrade/src/zodbupgrade/main.py (rev 0)
+++ Sandbox/ctheune/zodbupgrade/src/zodbupgrade/main.py 2009-05-28 07:08:42 UTC (rev 100497)
@@ -0,0 +1,28 @@
+##############################################################################
+#
+# Copyright (c) 2009 Zope Corporation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
+
+import sys
+import zodbupgrade.analyze
+import ZODB.FileStorage
+import logging
+
+
+logging.getLogger().addHandler(logging.StreamHandler())
+logging.getLogger().setLevel(0)
+
+
+def main():
+ db = sys.argv[1]
+ storage = ZODB.FileStorage.FileStorage('Data.fs')
+ zodbupgrade.analyze.update_storage(storage)
Property changes on: Sandbox/ctheune/zodbupgrade/src/zodbupgrade/main.py
___________________________________________________________________
Added: svn:eol-style
+ native
More information about the Checkins
mailing list