[Zodb-checkins] SVN: ZODB/branches/shane-poll-invalidations/src/ZODB/ Replaced IStoragePollable with IMVCCStorage.

Shane Hathaway shane at hathawaymix.org
Sat Apr 25 19:00:46 EDT 2009


Log message for revision 99494:
  Replaced IStoragePollable with IMVCCStorage.
  
  IMVCCStorage better describes the contract of an MVCC-providing
  storage implementation.  Changes:
  
  - bind_connection(zodb_conn) became new_instance().  Now we
    don't need a connection to get an instance.
  
  - The propagate_invalidations flag went away.  Now we check
    IMVCCStorage.providedBy(storage).
  
  - connection_closing() became sync(force=False).
  
  - added the release() method, which releases the backend
    database session without closing the storage.
  
  These changes were prompted by Jim, who suggested a better way to
  integrate ZODB with RelStorage.
  
  

Changed:
  U   ZODB/branches/shane-poll-invalidations/src/ZODB/Connection.py
  U   ZODB/branches/shane-poll-invalidations/src/ZODB/DB.py
  U   ZODB/branches/shane-poll-invalidations/src/ZODB/interfaces.py

-=-
Modified: ZODB/branches/shane-poll-invalidations/src/ZODB/Connection.py
===================================================================
--- ZODB/branches/shane-poll-invalidations/src/ZODB/Connection.py	2009-04-25 22:49:53 UTC (rev 99493)
+++ ZODB/branches/shane-poll-invalidations/src/ZODB/Connection.py	2009-04-25 23:00:46 UTC (rev 99494)
@@ -30,6 +30,7 @@
 from persistent.interfaces import IPersistentDataManager
 from ZODB.interfaces import IConnection
 from ZODB.interfaces import IBlobStorage
+from ZODB.interfaces import IMVCCStorage
 from ZODB.blob import Blob, rename_or_copy_blob
 from transaction.interfaces import ISavepointDataManager
 from transaction.interfaces import IDataManagerSavepoint
@@ -95,10 +96,13 @@
         self.connections = {self._db.database_name: self}
 
         storage = db.storage
-        m = getattr(storage, 'bind_connection', None)
-        if m is not None:
-            # Use a storage instance bound to this connection.
-            storage = m(self)
+        if IMVCCStorage.providedBy(storage):
+            # Use a connection-specific storage instance.
+            self._mvcc_storage = True
+            storage = storage.new_instance()
+        else:
+            self._mvcc_storage = False
+
         self._normal_storage = self._storage = storage
         self.new_oid = storage.new_oid
         self._savepoint_storage = None
@@ -153,13 +157,6 @@
         # in the cache on abort and in other connections on finish.
         self._modified = []
 
-        # Allow the storage to decide whether invalidations should
-        # propagate between connections.  If the storage provides MVCC
-        # semantics, it is better to not propagate invalidations between
-        # connections.
-        self._propagate_invalidations = getattr(
-            self._storage, 'propagate_invalidations', True)
-
         # _invalidated queues invalidate messages delivered from the DB
         # _inv_lock prevents one thread from modifying the set while
         # another is processing invalidations.  All the invalidations
@@ -190,10 +187,10 @@
         # _conflicts).
         self._conflicts = {}
 
-        # If MVCC is enabled, then _mvcc is True and _txn_time stores
-        # the upper bound on transactions visible to this connection.
-        # That is, all object revisions must be written before _txn_time.
-        # If it is None, then the current revisions are acceptable.
+        # _txn_time stores the upper bound on transactions visible to
+        # this connection. That is, all object revisions must be
+        # written before _txn_time. If it is None, then the current
+        # revisions are acceptable.
         self._txn_time = None
 
         # To support importFile(), implemented in the ExportImport base
@@ -306,10 +303,8 @@
         if self._opened:
             self.transaction_manager.unregisterSynch(self)
 
-        # If the storage wants to know, tell it this connection is closing.
-        m = getattr(self._storage, 'connection_closing', None)
-        if m is not None:
-            m()
+        if self._mvcc_storage:
+            self._storage.sync(force=False)
 
         if primary:
             for connection in self.connections.values():
@@ -339,8 +334,9 @@
 
     def invalidate(self, tid, oids):
         """Notify the Connection that transaction 'tid' invalidated oids."""
-        if not self._propagate_invalidations:
-            # The storage disabled inter-connection invalidation.
+        if self._mvcc_storage:
+            # Inter-connection invalidation is not needed when the
+            # storage provides MVCC.
             return
         if self.before is not None:
             # this is an historical connection.  Invalidations are irrelevant.
@@ -479,13 +475,11 @@
         self._registered_objects = []
         self._creating.clear()
 
-    def _poll_invalidations(self):
-        """Poll and process object invalidations provided by the storage.
-        """
-        m = getattr(self._storage, 'poll_invalidations', None)
-        if m is not None:
+    # Process pending invalidations.
+    def _flush_invalidations(self):
+        if self._mvcc_storage:
             # Poll the storage for invalidations.
-            invalidated = m()
+            invalidated = self._storage.poll_invalidations()
             if invalidated is None:
                 # special value: the transaction is so old that
                 # we need to flush the whole cache.
@@ -493,9 +487,6 @@
             elif invalidated:
                 self._cache.invalidate(invalidated)
 
-    # Process pending invalidations.
-    def _flush_invalidations(self):
-        self._poll_invalidations()
         self._inv_lock.acquire()
         try:
             # Non-ghostifiable objects may need to read when they are

Modified: ZODB/branches/shane-poll-invalidations/src/ZODB/DB.py
===================================================================
--- ZODB/branches/shane-poll-invalidations/src/ZODB/DB.py	2009-04-25 22:49:53 UTC (rev 99493)
+++ ZODB/branches/shane-poll-invalidations/src/ZODB/DB.py	2009-04-25 23:00:46 UTC (rev 99494)
@@ -35,6 +35,7 @@
 
 from zope.interface import implements
 from ZODB.interfaces import IDatabase
+from ZODB.interfaces import IMVCCStorage
 
 import BTrees.OOBTree
 import transaction
@@ -425,28 +426,32 @@
                 DeprecationWarning, 2)
             storage.tpc_vote = lambda *args: None
 
+        if IMVCCStorage.providedBy(storage):
+            temp_storage = storage.new_instance()
+        else:
+            temp_storage = storage
         try:
-            storage.load(z64, '')
-        except KeyError:
-            # Create the database's root in the storage if it doesn't exist
-            from persistent.mapping import PersistentMapping
-            root = PersistentMapping()
-            # Manually create a pickle for the root to put in the storage.
-            # The pickle must be in the special ZODB format.
-            file = cStringIO.StringIO()
-            p = cPickle.Pickler(file, 1)
-            p.dump((root.__class__, None))
-            p.dump(root.__getstate__())
-            t = transaction.Transaction()
-            t.description = 'initial database creation'
-            storage.tpc_begin(t)
-            storage.store(z64, None, file.getvalue(), '', t)
-            storage.tpc_vote(t)
-            storage.tpc_finish(t)
-        if hasattr(storage, 'connection_closing'):
-            # Let the storage release whatever resources it used for loading
-            # the root object.
-            storage.connection_closing()
+            try:
+                temp_storage.load(z64, '')
+            except KeyError:
+                # Create the database's root in the storage if it doesn't exist
+                from persistent.mapping import PersistentMapping
+                root = PersistentMapping()
+                # Manually create a pickle for the root to put in the storage.
+                # The pickle must be in the special ZODB format.
+                file = cStringIO.StringIO()
+                p = cPickle.Pickler(file, 1)
+                p.dump((root.__class__, None))
+                p.dump(root.__getstate__())
+                t = transaction.Transaction()
+                t.description = 'initial database creation'
+                temp_storage.tpc_begin(t)
+                temp_storage.store(z64, None, file.getvalue(), '', t)
+                temp_storage.tpc_vote(t)
+                temp_storage.tpc_finish(t)
+        finally:
+            if IMVCCStorage.providedBy(temp_storage):
+                temp_storage.release()
 
         # Multi-database setup.
         if databases is None:

Modified: ZODB/branches/shane-poll-invalidations/src/ZODB/interfaces.py
===================================================================
--- ZODB/branches/shane-poll-invalidations/src/ZODB/interfaces.py	2009-04-25 22:49:53 UTC (rev 99493)
+++ ZODB/branches/shane-poll-invalidations/src/ZODB/interfaces.py	2009-04-25 23:00:46 UTC (rev 99494)
@@ -953,46 +953,93 @@
         # DB pass-through
 
 
-class IStoragePollable(Interface):
-    """A storage that can be polled for changes."""
+class IMVCCStorage(IStorage):
+    """A storage that provides MVCC semantics internally.
 
-    def bind_connection(connection):
-        """Returns a storage instance to be used by the given Connection.
+    MVCC (multi-version concurrency control) means each user of a
+    database has a snapshot view of the database. The snapshot view
+    does not change, even if concurrent connections commit
+    transactions, until a transaction boundary. Relational databases
+    that support serializable transaction isolation provide MVCC.
 
-        This method is optional.  By implementing this method, a storage
-        instance can maintain Connection-specific state.
+    Storages that implement IMVCCStorage, such as RelStorage, provide
+    MVCC semantics at the ZODB storage layer. When ZODB.Connection uses
+    a storage that implements IMVCCStorage, each connection uses a
+    connection-specific storage instance, and that storage instance
+    provides a snapshot of the database.
 
-        If this method is not provided, all connections to the same database
-        use the same storage instance (even across threads).
-        """
+    By contrast, storages that do not implement IMVCCStorage, such as
+    FileStorage, rely on ZODB.Connection to provide MVCC semantics, so
+    in that case, one storage instance is shared by many
+    ZODB.Connections. Applications that use ZODB.Connection always have
+    a snapshot view of the database; IMVCCStorage only modifies which
+    layer of ZODB provides MVCC.
 
-    propagate_invalidations = Attribute(
-        """A boolean value indicating whether invalidations should propagate.
+    Furthermore, IMVCCStorage changes the way object invalidation
+    works. An essential feature of ZODB is the propagation of object
+    invalidation messages to keep in-memory caches up to date. Storages
+    like FileStorage and ZEO.ClientStorage send invalidation messages
+    to all other Connection instances at transaction commit time.
+    Storages that implement IMVCCStorage, on the other hand, expect the
+    ZODB.Connection to poll for a list of invalidated objects.
 
-        ZODB normally sends invalidation notifications between
-        Connection objects within a Python process.  If this
-        attribute is false, no such invalidations will be sent.
-        Cross-connection invalidation should normally be enabled, but
-        it adds unnecessary complexity to storages that expect the connection
-        to poll for invalidations instead.
+    Certain methods of IMVCCStorage implementations open persistent
+    back end database sessions and retain the sessions even after the
+    method call finishes::
 
-        If this attribute is not present, it is assumed to be true.
-        """)
+        load
+        loadEx
+        loadSerial
+        loadBefore
+        store
+        restore
+        new_oid
+        history
+        tpc_begin
+        tpc_vote
+        tpc_abort
+        tpc_finish
 
-    def connection_closing():
-        """Notifies the storage that a connection is closing.
+    If you know that the storage instance will no longer be used after
+    calling any of these methods, you should call the release method to
+    release the persistent sessions. The persistent sessions will be
+    reopened as necessary if you call one of those methods again.
 
-        This method is optional.  This method is useful when
-        bind_connection() provides Connection-specific storage instances.
-        It lets the storage release resources.
+    Other storage methods open short lived back end sessions and close
+    the back end sessions before returning. These include::
+
+        __len__
+        getSize
+        undoLog
+        undo
+        pack
+        iterator
+
+    These methods do not provide MVCC semantics, so these methods
+    operate on the most current view of the database, rather than the
+    snapshot view that the other methods use.
+    """
+
+    def new_instance():
+        """Creates and returns another storage instance.
+
+        The returned instance provides IMVCCStorage and connects to the
+        same back-end database. The database state visible by the
+        instance will be a snapshot that varies independently of other
+        storage instances.
         """
 
+    def release():
+        """Release all persistent sessions used by this storage instance.
+
+        After this call, the storage instance can still be used;
+        calling methods that use persistent sessions will cause the
+        persistent sessions to be reopened.
+        """
+
     def poll_invalidations():
         """Poll the storage for external changes.
 
-        This method is optional.  This method is useful when
-        bind_connection() provides Connection-specific storage instances.
-
         Returns either a sequence of OIDs that have changed, or None.  When a
         sequence is returned, the corresponding objects should be removed
         from the ZODB in-memory cache.  When None is returned, the storage is
@@ -1002,7 +1049,15 @@
         In that case, the ZODB in-memory cache should be cleared.
         """
 
+    def sync(force=True):
+        """Updates the internal snapshot to the current state of the database.
 
+        If the force parameter is False, the storage may choose to
+        ignore this call. By ignoring this call, a storage can reduce
+        the frequency of database polls, thus reducing database load.
+        """
+
+
 class IStorageCurrentRecordIteration(IStorage):
 
     def record_iternext(next=None):



More information about the Zodb-checkins mailing list