[Checkins] SVN: zc.dict/branches/tlotze-blist/ added a database generation to turn dict orders into BLists

Thomas Lotze tl at gocept.com
Tue Dec 23 13:58:25 EST 2008 

Log message for revision 94294:
  added a database generation to turn dict orders into BLists

Changed:
  U   zc.dict/branches/tlotze-blist/buildout.cfg
  U   zc.dict/branches/tlotze-blist/setup.py
  A   zc.dict/branches/tlotze-blist/src/zc/dict/configure.zcml
  A   zc.dict/branches/tlotze-blist/src/zc/dict/generations/
  A   zc.dict/branches/tlotze-blist/src/zc/dict/generations/__init__.py
  A   zc.dict/branches/tlotze-blist/src/zc/dict/generations/configure.zcml
  A   zc.dict/branches/tlotze-blist/src/zc/dict/generations/evolve1.py
  A   zc.dict/branches/tlotze-blist/src/zc/dict/generations/evolve1.txt
  U   zc.dict/branches/tlotze-blist/src/zc/dict/tests.py

-=-
Modified: zc.dict/branches/tlotze-blist/buildout.cfg
===================================================================
--- zc.dict/branches/tlotze-blist/buildout.cfg	2008-12-23 17:55:00 UTC (rev 94293)
+++ zc.dict/branches/tlotze-blist/buildout.cfg	2008-12-23 18:58:25 UTC (rev 94294)
@@ -4,7 +4,7 @@
 
 [test]
 recipe = zc.recipe.testrunner
-eggs = zc.dict
+eggs = zc.dict [generations]
 defaults = ["-v", "-c"]
 
 [py]

Modified: zc.dict/branches/tlotze-blist/setup.py
===================================================================
--- zc.dict/branches/tlotze-blist/setup.py	2008-12-23 17:55:00 UTC (rev 94293)
+++ zc.dict/branches/tlotze-blist/setup.py	2008-12-23 18:58:25 UTC (rev 94294)
@@ -15,6 +15,7 @@
     package_dir={"": "src"},
     include_package_data=True,
     install_requires=["setuptools", "zope.interface", "ZODB3", "zc.blist"],
+    extras_require={"generations": ["zope.app.generations"]},
     tests_require=["zope.testing"],
     description=open('README.txt').read(),
     long_description=long_description,

Added: zc.dict/branches/tlotze-blist/src/zc/dict/configure.zcml
===================================================================
--- zc.dict/branches/tlotze-blist/src/zc/dict/configure.zcml	                        (rev 0)
+++ zc.dict/branches/tlotze-blist/src/zc/dict/configure.zcml	2008-12-23 18:58:25 UTC (rev 94294)
@@ -0,0 +1,5 @@
+<configure xmlns="http://namespaces.zope.org/zope">
+
+  <include package=".generations" />
+
+</configure>


Property changes on: zc.dict/branches/tlotze-blist/src/zc/dict/configure.zcml
___________________________________________________________________
Added: svn:keywords
   + Id Rev Date
Added: svn:eol-style
   + native

Added: zc.dict/branches/tlotze-blist/src/zc/dict/generations/__init__.py
===================================================================
--- zc.dict/branches/tlotze-blist/src/zc/dict/generations/__init__.py	                        (rev 0)
+++ zc.dict/branches/tlotze-blist/src/zc/dict/generations/__init__.py	2008-12-23 18:58:25 UTC (rev 94294)
@@ -0,0 +1,25 @@
+##############################################################################
+#
+# Copyright (c) 2007 Zope Corporation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
+"""Database generations for zc.dict -- A BTree based persistent mapping
+
+$Id$
+"""
+
+from zope.app.generations.generations import SchemaManager
+
+
+schema_manager = SchemaManager(
+    minimum_generation=1,
+    generation=1,
+    package_name=__name__)


Property changes on: zc.dict/branches/tlotze-blist/src/zc/dict/generations/__init__.py
___________________________________________________________________
Added: svn:keywords
   + Id Rev Date
Added: svn:eol-style
   + native

Added: zc.dict/branches/tlotze-blist/src/zc/dict/generations/configure.zcml
===================================================================
--- zc.dict/branches/tlotze-blist/src/zc/dict/generations/configure.zcml	                        (rev 0)
+++ zc.dict/branches/tlotze-blist/src/zc/dict/generations/configure.zcml	2008-12-23 18:58:25 UTC (rev 94294)
@@ -0,0 +1,11 @@
+<configure xmlns="http://namespaces.zope.org/zope">
+
+  <!-- The name of this utility is used as a persistent reference. Do not
+  change. -->
+  <utility
+    name="zc.dict"
+    provides="zope.app.generations.interfaces.ISchemaManager"
+    component=".schema_manager"
+    />
+
+</configure>


Property changes on: zc.dict/branches/tlotze-blist/src/zc/dict/generations/configure.zcml
___________________________________________________________________
Added: svn:keywords
   + Id Rev Date
Added: svn:eol-style
   + native

Added: zc.dict/branches/tlotze-blist/src/zc/dict/generations/evolve1.py
===================================================================
--- zc.dict/branches/tlotze-blist/src/zc/dict/generations/evolve1.py	                        (rev 0)
+++ zc.dict/branches/tlotze-blist/src/zc/dict/generations/evolve1.py	2008-12-23 18:58:25 UTC (rev 94294)
@@ -0,0 +1,29 @@
+##############################################################################
+#
+# Copyright (c) 2007 Zope Corporation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
+
+import zope.app.generations.utility
+
+import zc.blist
+import zc.dict
+
+
+generation = 1
+
+
+def evolve(context):
+    root = context.connection.root()
+    for obj in zope.app.generations.utility.findObjectsMatching(
+        root, lambda obj: isinstance(obj, zc.dict.OrderedDict)):
+        if type(obj._order) is not zc.blist.BList:
+            obj._order = zc.blist.BList(obj._order)


Property changes on: zc.dict/branches/tlotze-blist/src/zc/dict/generations/evolve1.py
___________________________________________________________________
Added: svn:keywords
   + Id Rev Date
Added: svn:eol-style
   + native

Added: zc.dict/branches/tlotze-blist/src/zc/dict/generations/evolve1.txt
===================================================================
--- zc.dict/branches/tlotze-blist/src/zc/dict/generations/evolve1.txt	                        (rev 0)
+++ zc.dict/branches/tlotze-blist/src/zc/dict/generations/evolve1.txt	2008-12-23 18:58:25 UTC (rev 94294)
@@ -0,0 +1,82 @@
+===========================================
+Evolve generation 1: OrderedDict and BLists
+===========================================
+
+
+Originally, ordered dicts stored the order of the keys in plain persistent
+lists, which have bad behaviour for large amounts of data. This was changed
+into using BLists. The evolution of database generation 1 handles the
+conversion of these structures.
+
+Let's create some ordered dicts [testdb]_:
+
+>>> from zc.dict import OrderedDict
+>>> from persistent.list import PersistentList
+>>> d1 = OrderedDict()
+>>> d1[1] = 2
+>>> d1[3] = 4
+>>> d2 = OrderedDict()
+>>> d2[7] = 8
+>>> d2[5] = 6
+>>> d = root['d'] = OrderedDict()
+>>> d['d2'] = d2
+>>> d['d1'] = d1
+
+Now we convert the order storages to persistent lists in order to create the
+old data structures:
+
+>>> d1._order = PersistentList(d1._order)
+>>> d1._order
+[1, 3]
+>>> type(d1._order)
+<class 'persistent.list.PersistentList'>
+
+>>> d2._order = PersistentList(d2._order)
+>>> d2._order
+[7, 5]
+>>> type(d2._order)
+<class 'persistent.list.PersistentList'>
+
+>>> d._order = PersistentList(d._order)
+>>> d._order
+['d2', 'd1']
+>>> type(d._order)
+<class 'persistent.list.PersistentList'>
+
+After running the evolution, all order storages should be BLists with the same
+contents as the PersistentLists:
+
+>>> from zc.dict.generations.evolve1 import evolve
+>>> evolve(context)
+
+>>> d1._order
+<zc.blist.BList object at 0x...>
+>>> list(d1._order)
+[1, 3]
+
+>>> d2._order
+<zc.blist.BList object at 0x...>
+>>> list(d2._order)
+[7, 5]
+
+>>> d._order
+<zc.blist.BList object at 0x...>
+>>> list(d._order)
+['d2', 'd1']
+
+
+
+.. [testdb] **Setting up a test database**
+
+    >>> from ZODB.tests.util import DB
+    >>> db = DB()
+    >>> conn = db.open()
+    >>> class Context(object): pass
+    >>> context = Context()
+    >>> context.connection = conn
+    >>> root = conn.root()
+
+
+.. Local Variables:
+.. mode: rst
+.. End:


Property changes on: zc.dict/branches/tlotze-blist/src/zc/dict/generations/evolve1.txt
___________________________________________________________________
Added: svn:keywords
   + Id Rev Date
Added: svn:eol-style
   + native

Modified: zc.dict/branches/tlotze-blist/src/zc/dict/tests.py
===================================================================
--- zc.dict/branches/tlotze-blist/src/zc/dict/tests.py	2008-12-23 17:55:00 UTC (rev 94293)
+++ zc.dict/branches/tlotze-blist/src/zc/dict/tests.py	2008-12-23 18:58:25 UTC (rev 94294)
@@ -21,6 +21,7 @@
 def test_suite():
     return unittest.TestSuite([
         doctest.DocFileSuite('dict.txt', 'ordered.txt',
+                             'generations/evolve1.txt',
                              optionflags=doctest.INTERPRET_FOOTNOTES
                              |doctest.REPORT_NDIFF|doctest.ELLIPSIS),
         ])

More information about the Checkins mailing list