[Zope-CVS] CVS: Products/Ape/lib/apelib/zodb3 - db.py:1.3.4.2 scanner.py:1.1.2.3 storage.py:1.6.4.3

Shane Hathaway shane@zope.com
Thu, 24 Jul 2003 21:58:52 -0400


Update of /cvs-repository/Products/Ape/lib/apelib/zodb3
In directory cvs.zope.org:/tmp/cvs-serv10991/lib/apelib/zodb3

Modified Files:
      Tag: ape-scan-branch
	db.py scanner.py storage.py 
Log Message:
Closed a hole in the scanner.

Until now, if in some way a connection started using an OID without calling
storage.load(), the scanner would assume that the OID depends on no sources.
This could happen in a ZEO environment and it would result in objects not
getting scanned.  The solution was to give scanners the ability to get the
sources for an OID directly if necessary.  Also enhanced tests.

Periodic scanning now seems to be in good shape.


=== Products/Ape/lib/apelib/zodb3/db.py 1.3.4.1 => 1.3.4.2 ===
--- Products/Ape/lib/apelib/zodb3/db.py:1.3.4.1	Wed Jul 23 00:12:52 2003
+++ Products/Ape/lib/apelib/zodb3/db.py	Thu Jul 24 21:58:46 2003
@@ -53,7 +53,7 @@
                  mapper_resource=None,
                  factory=None,
                  oid_encoder=None,
-                 scan=1,
+                 scan_interval=10,
                  pool_size=7,
                  cache_size=400,
                  cache_deactivate_after=60,
@@ -113,10 +113,11 @@
             assert IOIDEncoder.isImplementedBy(oid_encoder)
         self._oid_encoder = oid_encoder
         self._mapper_resource = mapper_resource
-        if scan:
+        if scan_interval > 0:
             from scanner import ScanControl
-            ctl = ScanControl(self)
+            ctl = ScanControl(db=self, scan_interval=scan_interval)
             self._scan_ctl = ctl
+            ctl.scanner.setStorage(storage)
             storage.setScanner(ctl.scanner)
         else:
             self._scan_ctl = None


=== Products/Ape/lib/apelib/zodb3/scanner.py 1.1.2.2 => 1.1.2.3 ===
--- Products/Ape/lib/apelib/zodb3/scanner.py:1.1.2.2	Thu Jul 24 08:15:40 2003
+++ Products/Ape/lib/apelib/zodb3/scanner.py	Thu Jul 24 21:58:46 2003
@@ -1,5 +1,6 @@
 ##############################################################################
-# Copyright (c) 2002 Zope Corporation and Contributors.
+#
+# Copyright (c) 2003 Zope Corporation and Contributors.
 # All Rights Reserved.
 #
 # This software is subject to the provisions of the Zope Public License,
@@ -22,25 +23,24 @@
 
 from BTrees.OOBTree import OOBTree, OOSet, difference
 from BTrees.IOBTree import IOBTree
+from zLOG import LOG, DEBUG
 
 # FUTURE_TIMEOUT defines how long to keep source information regarding
 # OIDs that might be used soon.
 FUTURE_TIMEOUT = 10 * 60
 
-CONNECTION_UPDATE_INTERVAL = 10
-SCAN_INTERVAL = 10
-
 
 class ScanControl:
 
-    def __init__(self, db=None):
+    def __init__(self, db=None, scan_interval=10):
         self.db = db
         self.next_conn_id = 1
         self.conn_oids = IOBTree()   # IOBTree({ conn_id -> OOSet([oid]) } })
         self.oids = OOSet()          # OOSet([oid])
         self.scanner = Scanner()
         self.lock = allocate_lock()
-        self.next_scan = time() + SCAN_INTERVAL
+        self.scan_interval = scan_interval
+        self.next_scan = time() + scan_interval
 
 
     def newConnection(self):
@@ -78,20 +78,20 @@
     def mayScan(self):
         now = time()
         if now >= self.next_scan:
-            self.next_scan = now + SCAN_INTERVAL
-            print 'Scanning %d objects' % len(self.oids)
+            self.next_scan = now + self.scan_interval
+            LOG('Ape', DEBUG, 'Scanning %d objects.' % len(self.oids))
             inv = self.scanner.scan()
             self.scanner.pruneFuture()
-            print 'Finished scanning'
+            LOG('Ape', DEBUG,
+                'Finished scanning. %d objects changed.' % len(inv))
             if inv:
-                print 'Invalidating', inv
                 d = {}
                 for oid in inv:
                     d[oid] = 1
                 if self.db is not None:
                     self.db.invalidate(d)
                 else:
-                    print 'No database set!'
+                    LOG('Ape', DEBUG, "No database set, so can't invalidate!")
 
 
 class ConnectionScanControl:
@@ -104,7 +104,7 @@
     def ready(self):
         now = time()
         if now >= self.next_update:
-            self.next_update = now + CONNECTION_UPDATE_INTERVAL
+            self.next_update = now + self.ctl.scan_interval
             return 1
         return 0
 
@@ -119,9 +119,14 @@
         self.future = {}          # { oid -> ({source->state}, atime) }
         self.uncommitted = {}     # { tid -> {oid->{source->state}} }
         self.lock = allocate_lock()
+        self.storage = None
 
+    def setStorage(self, s):
+        # This is needed for calling storage.getSources().
+        self.storage = s
 
     def setOIDs(self, oids):
+        new_sources = {}  # { oid -> sourcedict }
         self.lock.acquire()
         try:
             removed = difference(self.current, oids)
@@ -133,11 +138,35 @@
                     # Source info for this OID was provided earlier.
                     sources, atime = self.future[oid]
                     del self.future[oid]
+                    self.current[oid] = sources
                 else:
-                    sources = {}
-                self.current[oid] = sources
+                    new_sources[oid] = None
         finally:
             self.lock.release()
+        if new_sources:
+            # Load source info the slow way.
+            if self.storage is not None:
+                LOG('Ape', DEBUG, 'Getting sources for %d oids.'
+                    % len(new_sources))
+                for oid in new_sources.keys():
+                    new_sources[oid] = self.storage.getSources(oid)
+            else:
+                LOG('Ape', DEBUG, "Can't get sources for %d oids. "
+                    "Assuming no sources!" % len(new_sources))
+                # This will cause the scanner to miss changes, but
+                # since no storage is known, there is little we can
+                # do.
+                for oid in new_sources.keys():
+                    new_sources[oid] = {}
+            self.lock.acquire()
+            try:
+                for oid, sources in new_sources.items():
+                    if not self.current.has_key(oid):
+                        self.current[oid] = sources
+                    # else something else added the source info
+                    # while self.lock was released.
+            finally:
+                self.lock.release()
 
 
     def setSources(self, oid, sources):
@@ -206,6 +235,8 @@
                         del self.future[oid]
             finally:
                 self.lock.release()
+            LOG('Ape', DEBUG,
+                'Future sources cache size: %d objects.' % len(self.future))
 
 
     def afterCommit(self, tid):


=== Products/Ape/lib/apelib/zodb3/storage.py 1.6.4.2 => 1.6.4.3 ===
--- Products/Ape/lib/apelib/zodb3/storage.py:1.6.4.2	Thu Jul 24 08:15:40 2003
+++ Products/Ape/lib/apelib/zodb3/storage.py	Thu Jul 24 21:58:46 2003
@@ -173,6 +173,14 @@
             print 'stored', `oid`, `h64`, `new_h64`
         return new_h64
 
+    def getSources(self, oid):
+        keychain = self._oid_encoder.decode(oid)
+        self._lock_acquire()
+        try:
+            return self._gwio.getSources(keychain)
+        finally:
+            self._lock_release()
+
     def new_oid(self):
         keychain = self._gwio.newKeychain()
         return self._oid_encoder.encode(keychain)