[Zope-Checkins] CVS: ZODB3/ZODB - FileStorage.py:1.105.2.3

Jeremy Hylton jeremy@zope.com
Fri, 18 Oct 2002 15:39:59 -0400


Update of /cvs-repository/ZODB3/ZODB
In directory cvs.zope.org:/tmp/cvs-serv8403/ZODB

Modified Files:
      Tag: ZODB3-3_1-branch
	FileStorage.py 
Log Message:
Backport for trunk: data_txn attribute for iterator() and cleanups.



=== ZODB3/ZODB/FileStorage.py 1.105.2.2 => 1.105.2.3 ===
--- ZODB3/ZODB/FileStorage.py:1.105.2.2	Wed Oct 16 15:47:44 2002
+++ ZODB3/ZODB/FileStorage.py	Fri Oct 18 15:39:58 2002
@@ -148,11 +148,17 @@
 register_subsystem('ZODB FS')
 
 z64='\0'*8
+# the struct formats for the headers
+TRANS_HDR = ">8s8scHHH"
+DATA_HDR = ">8s8s8s8sH8s"
 # constants to support various header sizes
 TRANS_HDR_LEN = 23
 DATA_HDR_LEN = 42
 DATA_VERSION_HDR_LEN = 58
 
+assert struct.calcsize(TRANS_HDR) == TRANS_HDR_LEN
+assert struct.calcsize(DATA_HDR) == DATA_HDR_LEN
+
 def warn(message, *data):
     LOG('ZODB FS', WARNING, "%s  warn: %s\n" % (packed_version,
                                                 (message % data)))
@@ -175,21 +181,19 @@
 class FileStorageFormatError(FileStorageError):
     """Invalid file format
 
-    The format of the given file is not valid
+    The format of the given file is not valid.
     """
 
 class CorruptedFileStorageError(FileStorageError,
                                 POSException.StorageSystemError):
-    """Corrupted file storage
-    """
+    """Corrupted file storage."""
 
 class CorruptedTransactionError(CorruptedFileStorageError): pass
 class CorruptedDataError(CorruptedFileStorageError): pass
 
 class FileStorageQuotaError(FileStorageError,
                             POSException.StorageSystemError):
-    """File storage quota exceeded
-    """
+    """File storage quota exceeded."""
 
 packed_version='FS21'
 
@@ -365,7 +369,7 @@
             if pos < 4: return 0
             seek(pos)
             s = read(TRANS_HDR_LEN)
-            tid, stl, status, ul, dl, el = unpack(">8s8scHHH", s)
+            tid, stl, status, ul, dl, el = unpack(TRANS_HDR, s)
             if not ltid: ltid=tid
             if stl != rstl: return 0 # inconsistent lengths
             if status == 'u': continue # undone trans, search back
@@ -379,7 +383,7 @@
                 # Read the data records for this transaction
                 seek(opos)
                 h=read(DATA_HDR_LEN)
-                oid,serial,sprev,stloc,vlen,splen = unpack(">8s8s8s8sH8s", h)
+                oid,serial,sprev,stloc,vlen,splen = unpack(DATA_HDR, h)
                 tloc=U64(stloc)
                 plen=U64(splen)
 
@@ -549,7 +553,7 @@
         file.seek(pos)
         read=file.read
         h=read(DATA_HDR_LEN)
-        doid,serial,prev,tloc,vlen,plen = unpack(">8s8s8s8sH8s", h)
+        doid,serial,prev,tloc,vlen,plen = unpack(DATA_HDR, h)
         if vlen:
             nv = read(8) != z64
             file.seek(8,1) # Skip previous version record pointer
@@ -594,7 +598,7 @@
         file.seek(pos)
         read = file.read
         h = read(DATA_HDR_LEN)
-        doid, serial, prev, tloc, vlen, plen = unpack(">8s8s8s8sH8s", h)
+        doid, serial, prev, tloc, vlen, plen = unpack(DATA_HDR, h)
         if doid != oid:
             raise CorruptedDataError, h
         if vlen:
@@ -633,7 +637,7 @@
             while 1:
                 seek(pos)
                 h=read(DATA_HDR_LEN)
-                doid,dserial,prev,tloc,vlen,plen = unpack(">8s8s8s8sH8s", h)
+                doid,dserial,prev,tloc,vlen,plen = unpack(DATA_HDR, h)
                 if doid != oid: raise CorruptedDataError, h
                 if dserial == serial: break # Yeee ha!
                 # Keep looking for serial
@@ -662,13 +666,12 @@
             except KeyError:
                 raise POSKeyError(oid)
             file=self._file
-            seek=file.seek
-            seek(pos)
+            file.seek(pos)
             doid,serial,prev,tloc,vlen = unpack(">8s8s8s8sH", file.read(34))
             if doid != oid:
                 raise CorruptedDataError, pos
             if vlen:
-                seek(24,1) # skip plen, pnv, and pv
+                file.read(24) # skip plen, pnv, and pv
                 return file.read(vlen)
             return ''
         finally: self._lock_release()
@@ -686,7 +689,7 @@
             if old:
                 self._file.seek(old)
                 h=self._file.read(DATA_HDR_LEN)
-                doid,oserial,sprev,stloc,vlen,splen = unpack(">8s8s8s8sH8s", h)
+                doid,oserial,sprev,stloc,vlen,splen = unpack(DATA_HDR, h)
                 if doid != oid: raise CorruptedDataError, h
                 if vlen:
                     pnv=self._file.read(8) # non-version data pointer
@@ -710,7 +713,7 @@
             here=pos+(tfile.tell()+self._thl)
             self._tindex[oid]=here
             newserial=self._serial
-            write(pack(">8s8s8s8sH8s",
+            write(pack(DATA_HDR,
                        oid, newserial, p64(old), p64(pos),
                        len(version), p64(len(data))
                        )
@@ -784,7 +787,7 @@
                 if old:
                     self._file.seek(old)
                     h = self._file.read(42)
-                    doid, x, y, z, vlen, w = unpack(">8s8s8s8sH8s", h)
+                    doid, x, y, z, vlen, w = unpack(DATA_HDR, h)
                     if doid != oid:
                         raise CorruptedDataError, h
                     # XXX assert versions match?
@@ -940,7 +943,7 @@
                 # Read the data records for this transaction
                 seek(pos)
                 h=read(DATA_HDR_LEN)
-                oid,serial,sprev,stloc,vlen,splen = unpack(">8s8s8s8sH8s", h)
+                oid,serial,sprev,stloc,vlen,splen = unpack(DATA_HDR, h)
                 plen=U64(splen)
                 prev=U64(sprev)
                 dlen=DATA_HDR_LEN+(plen or 8)
@@ -973,7 +976,7 @@
         read=file.read
         file.seek(pos)
         h=read(DATA_HDR_LEN)
-        roid,serial,sprev,stloc,vlen,splen = unpack(">8s8s8s8sH8s", h)
+        roid,serial,sprev,stloc,vlen,splen = unpack(DATA_HDR, h)
         if roid != oid: raise UndoError('Invalid undo transaction id')
         if vlen:
             read(16) # skip nv pointer and version previous pointer
@@ -996,7 +999,7 @@
         self._file.seek(pos)
         read=self._file.read
         h=read(DATA_HDR_LEN)
-        doid,serial,sprev,stloc,vlen,splen = unpack(">8s8s8s8sH8s", h)
+        doid,serial,sprev,stloc,vlen,splen = unpack(DATA_HDR, h)
         if vlen:
             h=read(16)
             return read(vlen), h[:8]
@@ -1171,7 +1174,7 @@
             self._file.seek(pos)
             h = self._file.read(DATA_HDR_LEN)
             oid, serial, sprev, stloc, vlen, splen = \
-                 struct.unpack(">8s8s8s8sH8s", h)
+                 struct.unpack(DATA_HDR, h)
             if failed(oid):
                 del failures[oid] # second chance!
             plen = U64(splen)
@@ -1192,7 +1195,7 @@
                 failures[oid] = v
             else:
                 plen = len(p)
-                self._tfile.write(pack(">8s8s8s8sH8s",
+                self._tfile.write(pack(DATA_HDR,
                                        oid, self._serial, p64(ipos),
                                        ostloc, len(v), p64(plen)))
                 if v:
@@ -1283,7 +1286,7 @@
                 if len(r) >= size: return r
                 seek(pos)
                 h=read(DATA_HDR_LEN)
-                doid,serial,prev,tloc,vlen,plen = unpack(">8s8s8s8sH8s", h)
+                doid,serial,prev,tloc,vlen,plen = unpack(DATA_HDR, h)
                 prev=U64(prev)
 
                 if vlen:
@@ -1302,7 +1305,7 @@
 
                 seek(U64(tloc))
                 h=read(TRANS_HDR_LEN)
-                tid, stl, status, ul, dl, el = unpack(">8s8scHHH",h)
+                tid, stl, status, ul, dl, el = unpack(TRANS_HDR,h)
                 user_name=read(ul)
                 description=read(dl)
                 if el: d=loads(read(el))
@@ -1467,7 +1470,7 @@
                 seek(pos)
                 h=read(TRANS_HDR_LEN)
                 if len(h) < TRANS_HDR_LEN: break
-                tid, stl, status, ul, dl, el = unpack(">8s8scHHH",h)
+                tid, stl, status, ul, dl, el = unpack(TRANS_HDR,h)
                 if status=='c':
                     # Oops. we found a checkpoint flag.
                     break
@@ -1510,7 +1513,7 @@
                     seek(pos)
                     h=read(DATA_HDR_LEN)
                     oid,serial,sprev,stloc,vlen,splen = unpack(
-                        ">8s8s8s8sH8s", h)
+                        DATA_HDR, h)
                     plen=U64(splen)
                     dlen=DATA_HDR_LEN+(plen or 8)
 
@@ -1545,7 +1548,7 @@
                                 seek(ppos)
                                 ph=read(DATA_HDR_LEN)
                                 pdoid,ps,pp,pt,pvlen,pplen = unpack(
-                                    ">8s8s8s8sH8s", ph)
+                                    DATA_HDR, ph)
                                 if not pvlen:
                                     # The most current record is committed, so
                                     # we can toss this one
@@ -1614,7 +1617,7 @@
                             p=p64(p)
 
                     sprev=p64(index_get(oid, 0))
-                    write(pack(">8s8s8s8sH8s",
+                    write(pack(DATA_HDR,
                                oid,serial,sprev,p64(otpos),vlen,splen))
                     if vlen:
                         if not pnv:
@@ -1780,7 +1783,7 @@
         seek(pos)
         h=read(TRANS_HDR_LEN)
         if len(h) < TRANS_HDR_LEN: break
-        tid, stl, status, ul, dl, el = unpack(">8s8scHHH",h)
+        tid, stl, status, ul, dl, el = unpack(TRANS_HDR,h)
         if status=='c': break # Oops. we found a checkpoint flag.
         tl=U64(stl)
         tpos=pos
@@ -1805,7 +1808,7 @@
             # Read the data records for this transaction
             seek(pos)
             h=read(DATA_HDR_LEN)
-            oid,serial,sprev,stloc,vlen,splen = unpack(">8s8s8s8sH8s", h)
+            oid,serial,sprev,stloc,vlen,splen = unpack(DATA_HDR, h)
             plen=U64(splen)
             dlen=DATA_HDR_LEN+(plen or 8)
 
@@ -1834,7 +1837,7 @@
             # WRITE
             seek(opos)
             sprev=p64(index_get(oid, 0))
-            write(pack(">8s8s8s8sH8s",
+            write(pack(DATA_HDR,
                        oid,serial,sprev,p64(otpos),vlen,splen))
             if vlen:
                 if not pnv: write(z64)
@@ -1960,7 +1963,7 @@
                 file.truncate()
             break
 
-        tid, stl, status, ul, dl, el = unpack(">8s8scHHH",h)
+        tid, stl, status, ul, dl, el = unpack(TRANS_HDR,h)
         if el < 0: el=t32-el
 
         if tid <= ltid:
@@ -2027,7 +2030,7 @@
 
             seek(pos)
             h=read(DATA_HDR_LEN)
-            oid,serial,sprev,stloc,vlen,splen = unpack(">8s8s8s8sH8s", h)
+            oid,serial,sprev,stloc,vlen,splen = unpack(DATA_HDR, h)
             prev=U64(sprev)
             tloc=U64(stloc)
             plen=U64(splen)
@@ -2082,37 +2085,39 @@
     return pos, maxoid, ltid
 
 
-def _loadBack(file, oid, back):
+def _loadBack_impl(file, oid, back):
+    # shared implementation used by various _loadBack methods
     while 1:
         old = U64(back)
         if not old:
             raise POSKeyError(oid)
         file.seek(old)
-        h=file.read(DATA_HDR_LEN)
-        doid, serial, prev, tloc, vlen, plen = unpack(">8s8s8s8sH8s", h)
+        h = file.read(DATA_HDR_LEN)
+        doid, serial, prev, tloc, vlen, plen = unpack(DATA_HDR, h)
 
         if vlen:
             file.seek(vlen + 16, 1)
         if plen != z64:
-            return file.read(U64(plen)), serial
+            return file.read(U64(plen)), serial, old, tloc
         back = file.read(8) # We got a back pointer!
 
-def _loadBackPOS(file, oid, back):
-    """Return the position of the record containing the data used by
-    the record at the given position (back)."""
-    seek=file.seek
-    read=file.read
+def _loadBack(file, oid, back):
+    data, serial, old, tloc = _loadBack_impl(file, oid, back)
+    return data, serial
 
-    while 1:
-        old=U64(back)
-        if not old:
-            raise POSKeyError(oid)
-        seek(old)
-        h=read(DATA_HDR_LEN)
-        doid,serial,prev,tloc,vlen,plen = unpack(">8s8s8s8sH8s", h)
-        if vlen: seek(vlen+16,1)
-        if plen != z64: return old
-        back=read(8) # We got a back pointer!
+def _loadBackPOS(file, oid, back):
+    """Return position of data record for backpointer."""
+    data, serial, old, tloc = _loadBack_impl(file, oid, back)
+    return old
+
+def _loadBackTxn(file, oid, back):
+    """Return data, serial, and txn id for backpointer."""
+    data, serial, old, stloc = _loadBack_impl(file, oid, back)
+    tloc = U64(stloc)
+    file.seek(tloc)
+    h = file.read(TRANS_HDR_LEN)
+    tid = h[:8]
+    return data, serial, tid
 
 def _truncate(file, name, pos):
     seek=file.seek
@@ -2224,7 +2229,7 @@
             h=read(TRANS_HDR_LEN)
             if len(h) < TRANS_HDR_LEN: break
 
-            tid, stl, status, ul, dl, el = unpack(">8s8scHHH",h)
+            tid, stl, status, ul, dl, el = unpack(TRANS_HDR,h)
             if el < 0: el=t32-el
 
             if tid <= self._ltid:
@@ -2292,23 +2297,18 @@
                 except: e={}
             else: e={}
 
-            result=RecordIterator(
-                tid, status, user, description, e,
-                pos, (tend, file, seek, read,
-                      tpos,
-                      )
-                )
-
-            pos=tend
+            result = RecordIterator(tid, status, user, description, e, pos,
+                                    tend, file, tpos)
+            pos = tend
 
             # Read the (intentionally redundant) transaction length
             seek(pos)
-            h=read(8)
+            h = read(8)
             if h != stl:
                 warn("%s redundant transaction length check failed at %s",
                      self._file.name, pos)
                 break
-            self._pos=pos+8
+            self._pos = pos + 8
 
             return result
 
@@ -2317,48 +2317,49 @@
 class RecordIterator(Iterator, BaseStorage.TransactionRecord):
     """Iterate over the transactions in a FileStorage file.
     """
-    def __init__(self, tid, status, user, desc, ext, pos, stuff):
-        self.tid=tid
-        self.status=status
-        self.user=user
-        self.description=desc
-        self._extension=ext
-        self._pos=pos
-        self._stuff = stuff
+    def __init__(self, tid, status, user, desc, ext, pos, tend, file, tpos):
+        self.tid = tid
+        self.status = status
+        self.user = user
+        self.description = desc
+        self._extension = ext
+        self._pos = pos
+        self._tend = tend
+        self._file = file
+        self._tpos = tpos
 
     def next(self, index=0):
-        name=''
         pos = self._pos
-        tend, file, seek, read, tpos = self._stuff
-        while pos < tend:
+        while pos < self._tend:
             # Read the data records for this transaction
-            seek(pos)
-            h=read(DATA_HDR_LEN)
-            oid,serial,sprev,stloc,vlen,splen = unpack(">8s8s8s8sH8s", h)
-            prev=U64(sprev)
-            tloc=U64(stloc)
-            plen=U64(splen)
+            self._file.seek(pos)
+            h = self._file.read(DATA_HDR_LEN)
+            oid, serial, sprev, stloc, vlen, splen = unpack(DATA_HDR, h)
+            prev = U64(sprev)
+            tloc = U64(stloc)
+            plen = U64(splen)
 
-            dlen=DATA_HDR_LEN+(plen or 8)
+            dlen = DATA_HDR_LEN + (plen or 8)
 
             if vlen:
-                dlen=dlen+(16+vlen)
-                seek(8,1)
-                pv=U64(read(8))
-                version=read(vlen)
+                dlen += (16 + vlen)
+                tmp = self._file.read(16)
+                pv = U64(tmp[8:16])
+                version = self._file.read(vlen)
             else:
-                version=''
+                version = ''
 
-            if pos+dlen > tend or tloc != tpos:
+            if pos + dlen > self._tend or tloc != self._tpos:
                 warn("%s data record exceeds transaction record at %s",
-                     name, pos)
+                     file.name, pos)
                 break
 
-            self._pos=pos+dlen
+            self._pos = pos + dlen
+            tid = None
             if plen:
-                p = read(plen)
+                p = self._file.read(plen)
             else:
-                p = read(8)
+                p = self._file.read(8)
                 if p == z64:
                     # If the backpointer is 0 (encoded as z64), then
                     # this transaction undoes the object creation.  It
@@ -2368,19 +2369,18 @@
                     # this.
                     p = None
                 else:
-                    p = _loadBack(file, oid, p)[0]
+                    p, _s, tid = _loadBackTxn(self._file, oid, p)
 
-            r = Record(oid, serial, version, p)
+            r = Record(oid, serial, version, p, tid)
 
             return r
 
         raise IndexError, index
 
 class Record(BaseStorage.DataRecord):
-    """An abstract database record
-    """
+    """An abstract database record."""
     def __init__(self, *args):
-        self.oid, self.serial, self.version, self.data = args
+        self.oid, self.serial, self.version, self.data, self.data_txn = args
 
 class UndoSearch:
 
@@ -2414,7 +2414,7 @@
         self.pos -= U64(self.file.read(8)) + 8
         self.file.seek(self.pos)
         h = self.file.read(TRANS_HDR_LEN)
-        tid, tl, status, ul, dl, el = struct.unpack(">8s8scHHH", h)
+        tid, tl, status, ul, dl, el = struct.unpack(TRANS_HDR, h)
         if tid < self.packt or status == 'p':
             self.stop = 1
             return None