[Checkins] SVN: zc.FileStorage/dev/src/zc/FileStorage/__init__.py During gc, pop entries from the old index to the new index in an

[Jim Fulton]

Log message for revision 82280:
  During gc, pop entries from the old index to the new index in an
  effort to reduce memory consumption.
  
  Use posix_fadvise, when available to try to prevent trashing the file
  cache.
  

Changed:
  U   zc.FileStorage/dev/src/zc/FileStorage/__init__.py

-=-
Modified: zc.FileStorage/dev/src/zc/FileStorage/__init__.py
===================================================================
--- zc.FileStorage/dev/src/zc/FileStorage/__init__.py	2007-12-13 16:11:42 UTC (rev 82279)
+++ zc.FileStorage/dev/src/zc/FileStorage/__init__.py	2007-12-13 17:18:31 UTC (rev 82280)
@@ -27,6 +27,25 @@
 import ZODB.FileStorage.fspack
 import ZODB.fsIndex
 
+
+# fsIndex lacks a pop method. Let's add one. :/
+
+_fsIndexPop_marker = object()
+def fsIndexPop(self, key, default=_fsIndexPop_marker):
+    tree = self._data.get(key[:6], default)
+    if tree is default:
+        if default is _fsIndexPop_marker:
+            raise KeyError(key)
+        return default
+    v = tree.pop(key[6:], default)
+    if v is default:
+        if default is _fsIndexPop_marker:
+            raise KeyError(key)
+        return default
+    return ZODB.fsIndex.str2num(v)
+
+ZODB.fsIndex.fsIndex.pop = fsIndexPop
+
 class OptionalSeekFile(file):
     """File that doesn't seek to current position.
 
@@ -49,6 +68,7 @@
 
         # We set the buffer quite high (32MB) to try to reduce seeks
         # when the storage is disk is doing other io
+
         self._file = OptionalSeekFile(path, "rb")
 
         self._stop = stop
@@ -66,6 +86,7 @@
         self.ltid = z64
 
     def pack(self):
+        
         script = self._name+'.packscript'
         open(script, 'w').write(pack_script_template % dict(
             path = self._name,
@@ -273,6 +294,8 @@
 
         # We set the buffer quite high (32MB) to try to reduce seeks
         # when the storage is disk is doing other io
+
+        
         self._file = OptionalSeekFile(path, "rb")
 
         self._name = path
@@ -282,6 +305,30 @@
 
         self.ltid = z64
 
+        try:
+            import _zc_FileStorage_posix_fadvise
+        except ImportError:
+            def _free(end):
+                pass
+        else:
+            fd = self._file.fileno()
+            last = [0]
+            def _free(pos):
+                if pos == 4:
+                    last[0] = 4
+                elif (pos - last[0]) < 50000000:
+                    return
+                    
+                last[0] = pos
+                _zc_FileStorage_posix_fadvise.fadvise(
+                    fd, 0, last[0]-10000,
+                    _zc_FileStorage_posix_fadvise.POSIX_FADV_DONTNEED)
+
+        self._free = _free
+
+    def _read_txn_header(self, pos, tid=None):
+        return FileStoragePacker._read_txn_header(self, pos, tid)
+
     def pack(self):
         packed, index, references, packpos = self.buildPackIndex(
             self._stop, self.file_end)
@@ -442,14 +489,13 @@
             if oid in reachable:
                 continue
 
-            try:
-                reachable[oid] = index[oid]
-            except KeyError:
-                # Note that the references include references made
-                # after the pack time.  These include references to
-                # objects created after the pack time, which won't be
-                # in the index.
-                pass
+            # Note that the references include references made
+            # after the pack time.  These include references to
+            # objects created after the pack time, which won't be
+            # in the index.
+            pos = index.pop(oid, 0)
+            if pos:
+                reachable[oid] = pos
 
             ioid1, ioid2 = divmod(ioid, 2147483648L)