[Checkins] SVN: relstorage/trunk/relstorage/ Allocate batches of OIDs, to reduce round trips to the server.

Shane Hathaway shane at hathawaymix.org
Sun Oct 11 06:17:16 EDT 2009


Log message for revision 105007:
  Allocate batches of OIDs, to reduce round trips to the server.
  

Changed:
  U   relstorage/trunk/relstorage/adapters/interfaces.py
  U   relstorage/trunk/relstorage/adapters/oidallocator.py
  U   relstorage/trunk/relstorage/storage.py

-=-
Modified: relstorage/trunk/relstorage/adapters/interfaces.py
===================================================================
--- relstorage/trunk/relstorage/adapters/interfaces.py	2009-10-10 14:11:27 UTC (rev 105006)
+++ relstorage/trunk/relstorage/adapters/interfaces.py	2009-10-11 10:17:15 UTC (rev 105007)
@@ -262,8 +262,8 @@
 class IOIDAllocator(Interface):
     """Allocate OIDs and control future allocation"""
 
-    def new_oid(cursor):
-        """Return a new, unused OID."""
+    def new_oids(cursor):
+        """Return a sequence of new, unused OIDs."""
 
     def set_min_oid(cursor, oid):
         """Ensure the next OID is at least the given OID."""

Modified: relstorage/trunk/relstorage/adapters/oidallocator.py
===================================================================
--- relstorage/trunk/relstorage/adapters/oidallocator.py	2009-10-10 14:11:27 UTC (rev 105006)
+++ relstorage/trunk/relstorage/adapters/oidallocator.py	2009-10-11 10:17:15 UTC (rev 105007)
@@ -11,8 +11,13 @@
 # FOR A PARTICULAR PURPOSE.
 #
 ##############################################################################
-"""IOIDAllocator implementations"""
+"""IOIDAllocator implementations.
+"""
 
+# All of these allocators allocate 16 OIDs at a time.  In the sequence
+# or table, value 1 represents OID block 1-16, 2 represents OID block 17-32,
+# and so on.
+
 from relstorage.adapters.interfaces import IOIDAllocator
 from zope.interface import implements
 
@@ -21,18 +26,20 @@
 
     def set_min_oid(self, cursor, oid):
         """Ensure the next OID is at least the given OID."""
+        n = (oid + 15) / 16
         cursor.execute("""
         SELECT CASE WHEN %s > nextval('zoid_seq')
             THEN setval('zoid_seq', %s)
             ELSE 0
             END
-        """, (oid, oid))
+        """, (n, n))
 
-    def new_oid(self, cursor):
-        """Return a new, unused OID."""
+    def new_oids(self, cursor):
+        """Return a sequence of new, unused OIDs."""
         stmt = "SELECT NEXTVAL('zoid_seq')"
         cursor.execute(stmt)
-        return cursor.fetchone()[0]
+        n = cursor.fetchone()[0]
+        return range(n * 16 - 15, n * 16 + 1)
 
 
 class MySQLOIDAllocator(object):
@@ -40,18 +47,19 @@
 
     def set_min_oid(self, cursor, oid):
         """Ensure the next OID is at least the given OID."""
-        cursor.execute("REPLACE INTO new_oid VALUES(%s)", (oid,))
+        n = (oid + 15) / 16
+        cursor.execute("REPLACE INTO new_oid VALUES(%s)", (n,))
 
-    def new_oid(self, cursor):
-        """Return a new, unused OID."""
+    def new_oids(self, cursor):
+        """Return a sequence of new, unused OIDs."""
         stmt = "INSERT INTO new_oid VALUES ()"
         cursor.execute(stmt)
-        oid = cursor.connection.insert_id()
-        if oid % 100 == 0:
+        n = cursor.connection.insert_id()
+        if n % 100 == 0:
             # Clean out previously generated OIDs.
             stmt = "DELETE FROM new_oid WHERE zoid < %s"
-            cursor.execute(stmt, (oid,))
-        return oid
+            cursor.execute(stmt, (n,))
+        return range(n * 16 - 15, n * 16 + 1)
 
 
 class OracleOIDAllocator(object):
@@ -62,8 +70,11 @@
 
     def set_min_oid(self, cursor, oid):
         """Ensure the next OID is at least the given OID."""
-        next_oid = self.new_oid(cursor)
-        if next_oid < oid:
+        n = (oid + 15) / 16
+        stmt = "SELECT zoid_seq.nextval FROM DUAL"
+        cursor.execute(stmt)
+        next_n = cursor.fetchone()[0]
+        if next_n < n:
             # Oracle provides no way modify the sequence value
             # except through alter sequence or drop/create sequence,
             # but either statement kills the current transaction.
@@ -73,7 +84,7 @@
             try:
                 # Change the sequence by altering the increment.
                 # (this is safer than dropping and re-creating the sequence)
-                diff = oid - next_oid
+                diff = n - next_n
                 cursor2.execute(
                     "ALTER SEQUENCE zoid_seq INCREMENT BY %d" % diff)
                 cursor2.execute("SELECT zoid_seq.nextval FROM DUAL")
@@ -82,9 +93,10 @@
             finally:
                 self.connmanager.close(conn2, cursor2)
 
-    def new_oid(self, cursor):
-        """Return a new, unused OID."""
+    def new_oids(self, cursor):
+        """Return a sequence of new, unused OIDs."""
         stmt = "SELECT zoid_seq.nextval FROM DUAL"
         cursor.execute(stmt)
-        return cursor.fetchone()[0]
+        n = cursor.fetchone()[0]
+        return range(n * 16 - 15, n * 16 + 1)
 

Modified: relstorage/trunk/relstorage/storage.py
===================================================================
--- relstorage/trunk/relstorage/storage.py	2009-10-10 14:11:27 UTC (rev 105006)
+++ relstorage/trunk/relstorage/storage.py	2009-10-11 10:17:15 UTC (rev 105007)
@@ -133,6 +133,10 @@
         # _max_new_oid is the highest OID provided by new_oid()
         self._max_new_oid = 0
 
+        # _preallocated_oids contains OIDs provided by the database
+        # but not yet used.
+        self._preallocated_oids = []
+
         # set _cache_client
         if options.cache_servers:
             module_name = options.cache_module_name
@@ -820,11 +824,18 @@
             raise POSException.ReadOnlyError()
         self._lock_acquire()
         try:
-            cursor = self._store_cursor
-            if cursor is None:
-                self._open_store_connection()
+            if self._preallocated_oids:
+                oid_int = self._preallocated_oids.pop()
+            else:
                 cursor = self._store_cursor
-            oid_int = self._adapter.oidallocator.new_oid(cursor)
+                if cursor is None:
+                    self._open_store_connection()
+                    cursor = self._store_cursor
+                preallocated = list(
+                    self._adapter.oidallocator.new_oids(cursor))
+                preallocated.sort(reverse=True)
+                oid_int = preallocated.pop()
+                self._preallocated_oids = preallocated
             self._max_new_oid = max(self._max_new_oid, oid_int)
             return p64(oid_int)
         finally:



More information about the checkins mailing list