[Zodb-checkins] SVN: ZODB/trunk/src/ New Feature:

Jim Fulton jim at zope.com
Fri Feb 12 17:48:06 EST 2010


Log message for revision 108958:
  New Feature:
  
  When transactions are aborted, new object ids allocated during the
    transaction are saved and used in subsequent transactions. This can
    help in situations where object ids are used as BTree keys and the
    sequential allocation of object ids leads to conflict errors.
  

Changed:
  U   ZODB/trunk/src/CHANGES.txt
  U   ZODB/trunk/src/ZODB/Connection.py
  U   ZODB/trunk/src/ZODB/DB.py
  A   ZODB/trunk/src/ZODB/tests/new_oids_get_reused_on_abort.test
  U   ZODB/trunk/src/ZODB/tests/testConnection.py
  U   ZODB/trunk/src/ZODB/tests/testZODB.py
  U   ZODB/trunk/src/ZODB/tests/util.py

-=-
Modified: ZODB/trunk/src/CHANGES.txt
===================================================================
--- ZODB/trunk/src/CHANGES.txt	2010-02-12 17:47:27 UTC (rev 108957)
+++ ZODB/trunk/src/CHANGES.txt	2010-02-12 22:48:05 UTC (rev 108958)
@@ -2,6 +2,17 @@
  Change History
 ================
 
+3.10.0a2 (2010-??-??)
+=====================
+
+New Features
+------------
+
+- When transactions are aborted, new object ids allocated during the
+  transaction are saved and used in subsequent transactions. This can
+  help in situations where object ids are used as BTree keys and the
+  sequential allocation of object ids leads to conflict errors.
+
 3.10.0a1 (2010-02-08)
 =====================
 

Modified: ZODB/trunk/src/ZODB/Connection.py
===================================================================
--- ZODB/trunk/src/ZODB/Connection.py	2010-02-12 17:47:27 UTC (rev 108957)
+++ ZODB/trunk/src/ZODB/Connection.py	2010-02-12 22:48:05 UTC (rev 108958)
@@ -104,7 +104,7 @@
             self._mvcc_storage = False
 
         self._normal_storage = self._storage = storage
-        self.new_oid = storage.new_oid
+        self.new_oid = db.new_oid
         self._savepoint_storage = None
 
         # Do we need to join a txn manager?
@@ -214,7 +214,7 @@
                             " added to a Connection.", obj)
         elif obj._p_jar is None:
             assert obj._p_oid is None
-            oid = obj._p_oid = self._storage.new_oid()
+            oid = obj._p_oid = self.new_oid()
             obj._p_jar = self
             if self._added_during_commit is not None:
                 self._added_during_commit.append(obj)
@@ -426,6 +426,7 @@
         if self._savepoint_storage is not None:
             self._abort_savepoint()
 
+        self._invalidate_creating()
         self._tpc_cleanup()
 
     def _abort(self):
@@ -438,6 +439,7 @@
                 del self._added[oid]
                 del obj._p_jar
                 del obj._p_oid
+                self._db.save_oid(oid)
             else:
 
                 # Note: If we invalidate a non-ghostifiable object
@@ -723,6 +725,7 @@
             self._creating = {}
 
         for oid in creating:
+            self._db.save_oid(oid)
             o = self._cache.get(oid)
             if o is not None:
                 del self._cache[oid]

Modified: ZODB/trunk/src/ZODB/DB.py
===================================================================
--- ZODB/trunk/src/ZODB/DB.py	2010-02-12 17:47:27 UTC (rev 108957)
+++ ZODB/trunk/src/ZODB/DB.py	2010-02-12 22:48:05 UTC (rev 108958)
@@ -384,7 +384,9 @@
                  historical_timeout=300,
                  database_name='unnamed',
                  databases=None,
-                 xrefs=True):
+                 xrefs=True,
+                 max_saved_oids=999,
+                 ):
         """Create an object database.
 
         :Parameters:
@@ -480,6 +482,9 @@
         self._setupUndoMethods()
         self.history = storage.history
 
+        self._saved_oids = []
+        self._max_saved_oids = max_saved_oids
+
     def _setupUndoMethods(self):
         storage = self.storage
         try:
@@ -942,6 +947,19 @@
         return ContextManager(self)
 
 
+    def save_oid(self, oid):
+        if len(self._saved_oids) < self._max_saved_oids:
+            self._saved_oids.append(oid)
+
+    def new_oid(self):
+        if self._saved_oids:
+            try:
+                return self._saved_oids.pop()
+            except IndexError:
+                pass # Hm, threads
+        return self.storage.new_oid()
+
+
 class ContextManager:
     """PEP 343 context manager
     """

Added: ZODB/trunk/src/ZODB/tests/new_oids_get_reused_on_abort.test
===================================================================
--- ZODB/trunk/src/ZODB/tests/new_oids_get_reused_on_abort.test	                        (rev 0)
+++ ZODB/trunk/src/ZODB/tests/new_oids_get_reused_on_abort.test	2010-02-12 22:48:05 UTC (rev 108958)
@@ -0,0 +1,64 @@
+New OIDs get reused if a transaction aborts
+===========================================
+
+Historical note:
+
+  An OID is a terrible thing to waste.
+
+  Seriously: sequential allocation of OIDs could cause problems when
+  OIDs are used as (or as the basis of) BTree keys.  This happened
+  with Zope 3's intid utility where the object->id mapping uses an
+  object key based on the OID.  We got frequent conflict errors
+  because, in a site with many users, many objects are added at the
+  same time and conficts happened when conflicting changes caused
+  bucket splits.
+
+  Reusing an earlier allocated, but discarded OID will allow retries
+  of transactions to work because they'll use earlier OIDs which won't
+  tend to conflict with newly allocated ones.
+
+If a transaction is aborted, new OIDs assigned in the transaction are
+saved and made available for later transactions.
+
+    >>> import ZODB.tests.util, transaction
+    >>> db = ZODB.tests.util.DB()
+    >>> tm1 = transaction.TransactionManager()
+    >>> conn1 = db.open(tm1)
+    >>> conn1.root.x = ZODB.tests.util.P()
+    >>> tm1.commit()
+    >>> conn1.root.x.x = ZODB.tests.util.P()
+    >>> conn1.root.y = 1
+    >>> tm2 = transaction.TransactionManager()
+    >>> conn2 = db.open(tm2)
+    >>> conn2.root.y = ZODB.tests.util.P()
+    >>> tm2.commit()
+
+We get a conflict when we try to commit the change to the first connection:
+
+    >>> tm1.commit() # doctest: +ELLIPSIS
+    Traceback (most recent call last):
+    ...
+    ConflictError: ...
+
+    >>> tm1.abort()
+
+When we try, we get the same oid we would have gotten on the first transaction:
+
+    >>> conn1.root.x.x = ZODB.tests.util.P()
+    >>> tm1.commit()
+    >>> conn1.root.x.x._p_oid
+    '\x00\x00\x00\x00\x00\x00\x00\x03'
+
+We see this more clearly when we use add to assign the oid before the commit:
+
+    >>> conn1.root.z = ZODB.tests.util.P()
+    >>> conn1.add(conn1.root.z)
+    >>> conn1.root.z._p_oid
+    '\x00\x00\x00\x00\x00\x00\x00\x04'
+
+    >>> tm1.abort()
+
+    >>> conn2.root.a = ZODB.tests.util.P()
+    >>> conn2.add(conn2.root.a)
+    >>> conn2.root.a._p_oid
+    '\x00\x00\x00\x00\x00\x00\x00\x04'


Property changes on: ZODB/trunk/src/ZODB/tests/new_oids_get_reused_on_abort.test
___________________________________________________________________
Added: svn:eol-style
   + native

Modified: ZODB/trunk/src/ZODB/tests/testConnection.py
===================================================================
--- ZODB/trunk/src/ZODB/tests/testConnection.py	2010-02-12 17:47:27 UTC (rev 108957)
+++ ZODB/trunk/src/ZODB/tests/testConnection.py	2010-02-12 22:48:05 UTC (rev 108958)
@@ -815,6 +815,7 @@
 
     def __init__(self):
         self.storage = StubStorage()
+        self.new_oid = self.storage.new_oid
 
     classFactory = None
     database_name = 'stubdatabase'
@@ -823,6 +824,8 @@
     def invalidate(self, transaction, dict_with_oid_keys, connection):
         pass
 
+    save_oid = lambda self, oid: None
+
 def test_suite():
     s = unittest.makeSuite(ConnectionDotAdd, 'check')
     s.addTest(doctest.DocTestSuite())

Modified: ZODB/trunk/src/ZODB/tests/testZODB.py
===================================================================
--- ZODB/trunk/src/ZODB/tests/testZODB.py	2010-02-12 17:47:27 UTC (rev 108957)
+++ ZODB/trunk/src/ZODB/tests/testZODB.py	2010-02-12 22:48:05 UTC (rev 108958)
@@ -11,21 +11,22 @@
 # FOR A PARTICULAR PURPOSE.
 #
 ##############################################################################
+
+from persistent import Persistent
+from persistent.mapping import PersistentMapping
+from ZODB.POSException import ReadConflictError, ConflictError
+from ZODB.POSException import TransactionFailedError
+from ZODB.tests.warnhook import WarningsHook
+
+import doctest
+import transaction
 import unittest
 import warnings
-
 import ZODB
 import ZODB.FileStorage
 import ZODB.MappingStorage
-from ZODB.POSException import ReadConflictError, ConflictError
-from ZODB.POSException import TransactionFailedError
-from ZODB.tests.warnhook import WarningsHook
 import ZODB.tests.util
 
-from persistent import Persistent
-from persistent.mapping import PersistentMapping
-import transaction
-
 class P(Persistent):
     pass
 
@@ -631,7 +632,9 @@
         self._p_jar = poisonedjar
 
 def test_suite():
-    return unittest.makeSuite(ZODBTests, 'check')
+    suite = unittest.makeSuite(ZODBTests, 'check')
+    suite.addTest(doctest.DocFileSuite('new_oids_get_reused_on_abort.test'))
+    return suite
 
 if __name__ == "__main__":
     unittest.main(defaultTest="test_suite")

Modified: ZODB/trunk/src/ZODB/tests/util.py
===================================================================
--- ZODB/trunk/src/ZODB/tests/util.py	2010-02-12 17:47:27 UTC (rev 108957)
+++ ZODB/trunk/src/ZODB/tests/util.py	2010-02-12 22:48:05 UTC (rev 108958)
@@ -57,7 +57,7 @@
 
 class P(persistent.Persistent):
 
-    def __init__(self, name):
+    def __init__(self, name=None):
         self.name = name
 
     def __repr__(self):



More information about the Zodb-checkins mailing list