[Zope-Checkins] CVS: Zope/lib/python/ZODB - coptimizations.c:1.17.60.5 FileStorage.py:1.98.2.4 BaseStorage.py:1.21.2.3

Jeremy Hylton jeremy@zope.com
Wed, 18 Dec 2002 16:59:20 -0500


Update of /cvs-repository/Zope/lib/python/ZODB
In directory cvs.zope.org:/tmp/cvs-serv13244/lib/python/ZODB

Modified Files:
      Tag: Zope-2_6-branch
	coptimizations.c FileStorage.py BaseStorage.py 
Log Message:
Merge latest round of changes from ZODB3-3_1-branch


=== Zope/lib/python/ZODB/coptimizations.c 1.17.60.4 => 1.17.60.5 ===
--- Zope/lib/python/ZODB/coptimizations.c:1.17.60.4	Fri Dec 13 16:40:37 2002
+++ Zope/lib/python/ZODB/coptimizations.c	Wed Dec 18 16:59:19 2002
@@ -156,6 +156,22 @@
     return NULL;
 }
 
+/* persistent_id_call()
+
+   Returns a reference to a persistent object, appending it to the the
+   persistent_id's list of objects.  If a non-persistent object is
+   found, return None.
+
+   The returned reference can be either class info, oid pair or a
+   plain old oid.  If it is a pair, the class info is the module and
+   the name of the class.  The class info can be used to create a
+   ghost without loading the class.
+
+   For unusual objects, e.g. ZClasses, return just the oid.  An object
+   is unusual if it isn't an ExtensionClass, because that means it
+   doesn't inherit from Persistence, or if it has __getinitargs__().
+*/
+
 static PyObject *
 persistent_id_call(persistent_id *self, PyObject *args, PyObject *kwargs)
 {


=== Zope/lib/python/ZODB/FileStorage.py 1.98.2.3 => 1.98.2.4 ===
--- Zope/lib/python/ZODB/FileStorage.py:1.98.2.3	Wed Dec 11 11:30:45 2002
+++ Zope/lib/python/ZODB/FileStorage.py	Wed Dec 18 16:59:19 2002
@@ -202,6 +202,8 @@
     # default pack time is 0
     _packt = z64
 
+    _records_before_save = 10000
+
     def __init__(self, file_name, create=0, read_only=0, stop=None,
                  quota=None):
 
@@ -269,7 +271,10 @@
 
         r = self._restore_index()
         if r is not None:
+            self._used_index = 1 # Marker for testing
+                                            
             index, vindex, start, maxoid, ltid = r
+
             self._initIndex(index, vindex, tindex, tvindex)
             self._pos, self._oid, tid = read_index(
                 self._file, file_name, index, vindex, tindex, stop,
@@ -277,10 +282,16 @@
                 read_only=read_only,
                 )
         else:
+            self._used_index = 0 # Marker for testing
             self._pos, self._oid, tid = read_index(
                 self._file, file_name, index, vindex, tindex, stop,
                 read_only=read_only,
                 )
+            self._save_index()
+
+        self._records_before_save = max(self._records_before_save,
+                                        len(self._index))
+
         self._ltid = tid
 
         self._ts = tid = TimeStamp(tid)
@@ -309,6 +320,7 @@
         # hook to use something other than builtin dict
         return {}, {}, {}, {}
 
+    _saved = 0
     def _save_index(self):
         """Write the database index to a file to support quick startup
         """
@@ -325,6 +337,7 @@
         p.dump(info)
         f.flush()
         f.close()
+
         try:
             try:
                 os.remove(index_name)
@@ -333,6 +346,8 @@
             os.rename(tmp_name, index_name)
         except: pass
 
+        self._saved += 1
+
     def _clear_index(self):
         index_name=self.__name__+'.index'
         if os.path.exists(index_name):
@@ -350,22 +365,29 @@
         object positions cause zero to be returned.
         """
 
-        if pos < 100: return 0
-        file=self._file
-        seek=file.seek
-        read=file.read
+        if pos < 100:
+            return 0 # insane
+        file = self._file
+        seek = file.seek
+        read = file.read
         seek(0,2)
-        if file.tell() < pos: return 0
-        ltid=None
+        if file.tell() < pos:
+            return 0 # insane
+        ltid = None
+
+        max_checked = 5
+        checked = 0
 
-        while 1:
+        while checked < max_checked:
             seek(pos-8)
-            rstl=read(8)
-            tl=U64(rstl)
-            pos=pos-tl-8
-            if pos < 4: return 0
+            rstl = read(8)
+            tl = U64(rstl)
+            pos = pos-tl-8
+            if pos < 4:
+                return 0 # insane
             seek(pos)
             s = read(TRANS_HDR_LEN)
+<<<<<<< FileStorage.py
             tid, stl, status, ul, dl, el = unpack(TRANS_HDR, s)
             if not ltid: ltid=tid
             if stl != rstl: return 0 # inconsistent lengths
@@ -375,23 +397,52 @@
             tend=pos+tl
             opos=pos+(TRANS_HDR_LEN + ul + dl + el)
             if opos==tend: continue # empty trans
+=======
+            tid, stl, status, ul, dl, el = unpack(TRANS_HDR, s)
+            if not ltid:
+                ltid = tid
+            if stl != rstl:
+                return 0 # inconsistent lengths
+            if status == 'u':
+                continue # undone trans, search back
+            if status not in ' p':
+                return 0 # insane
+            if tl < (TRANS_HDR_LEN + ul + dl + el):
+                return 0 # insane
+            tend = pos+tl
+            opos = pos+(TRANS_HDR_LEN + ul + dl + el)
+            if opos == tend:
+                continue # empty trans
+>>>>>>> 1.105.2.11
 
-            while opos < tend:
+            while opos < tend and checked < max_checked:
                 # Read the data records for this transaction
                 seek(opos)
+<<<<<<< FileStorage.py
                 h=read(DATA_HDR_LEN)
                 oid,serial,sprev,stloc,vlen,splen = unpack(DATA_HDR, h)
                 tloc=U64(stloc)
                 plen=U64(splen)
+=======
+                h = read(DATA_HDR_LEN)
+                oid, serial, sprev, stloc, vlen, splen = unpack(DATA_HDR, h)
+                tloc = U64(stloc)
+                plen = U64(splen)
+
+                dlen = DATA_HDR_LEN+(plen or 8)
+                if vlen:
+                    dlen = dlen+(16+vlen)
+>>>>>>> 1.105.2.11
 
-                dlen=DATA_HDR_LEN+(plen or 8)
-                if vlen: dlen=dlen+(16+vlen)
+                if opos+dlen > tend or tloc != pos:
+                    return 0 # insane
 
-                if opos+dlen > tend or tloc != pos: return 0
+                if index.get(oid, 0) != opos:
+                    return 0 # insane
 
-                if index.get(oid, 0) != opos: return 0
+                checked += 1
 
-                opos=opos+dlen
+                opos = opos+dlen
 
             return ltid
 
@@ -420,7 +471,6 @@
         if index is None or pos is None or oid is None or vindex is None:
             return None
         pos = long(pos)
-
         tid=self._sane(index, pos)
         if not tid: return None
 
@@ -946,6 +996,9 @@
         finally:
             self._lock_release()
 
+    # Keep track of the number of records that we've written
+    _records_written = 0
+
     def _finish(self, tid, u, d, e):
         nextpos=self._nextpos
         if nextpos:
@@ -962,6 +1015,16 @@
 
             self._index.update(self._tindex)
             self._vindex.update(self._tvindex)
+
+            # Update the number of records that we've written
+            # +1 for the transaction record
+            self._records_written += len(self._tindex) + 1 
+            if self._records_written >= self._records_before_save:
+                self._save_index()
+                self._records_written = 0
+                self._records_before_save = max(self._records_before_save,
+                                                len(self._index))
+                
         self._ltid = tid
 
     def _abort(self):


=== Zope/lib/python/ZODB/BaseStorage.py 1.21.2.2 => 1.21.2.3 ===
--- Zope/lib/python/ZODB/BaseStorage.py:1.21.2.2	Tue Nov 12 16:13:58 2002
+++ Zope/lib/python/ZODB/BaseStorage.py	Wed Dec 18 16:59:19 2002
@@ -265,8 +265,8 @@
             restoring = 1
         else:
             restoring = 0
-        for transaction in other.iterator():
-
+        fiter = other.iterator()
+        for transaction in fiter:
             tid=transaction.tid
             if _ts is None:
                 _ts=TimeStamp(tid)
@@ -299,6 +299,8 @@
 
             self.tpc_vote(transaction)
             self.tpc_finish(transaction)
+
+        fiter.close()
 
 class TransactionRecord:
     """Abstract base class for iterator protocol"""