[Zodb-checkins] CVS: Zope/lib/python/ZODB - fsdump.py:1.3.70.2 __init__.py:1.14.2.2 FileStorage.py:1.98.2.2

Jeremy Hylton jeremy@zope.com
Fri, 15 Nov 2002 12:45:04 -0500


Update of /cvs-repository/Zope/lib/python/ZODB
In directory cvs.zope.org:/tmp/cvs-serv5028/lib/python/ZODB

Modified Files:
      Tag: Zope-2_6-branch
	fsdump.py __init__.py FileStorage.py 
Log Message:
Fix two problems with restore() and backpointers.

Merge ZODB3-restore-debug-branch and misc changes from
ZODB3-3_1-branch.


=== Zope/lib/python/ZODB/fsdump.py 1.3.70.1 => 1.3.70.2 ===
--- Zope/lib/python/ZODB/fsdump.py:1.3.70.1	Tue Nov 12 16:13:58 2002
+++ Zope/lib/python/ZODB/fsdump.py	Fri Nov 15 12:45:03 2002
@@ -76,3 +76,80 @@
         print >> file
         i += 1
     iter.close()
+
+import struct
+from ZODB.FileStorage import TRANS_HDR, TRANS_HDR_LEN
+from ZODB.FileStorage import DATA_HDR, DATA_HDR_LEN
+
+def fmt(p64):
+    # Return a nicely formatted string for a packaged 64-bit value
+    return "%016x" % U64(p64)
+
+class Dumper:
+    """A very verbose dumper for debuggin FileStorage problems."""
+
+    def __init__(self, path, dest=None):
+        self.file = open(path, "rb")
+        self.dest = dest
+
+    def dump(self):
+        fid = self.file.read(4)
+        print >> self.dest, "*" * 60
+        print >> self.dest, "file identifier: %r" % fid
+        while self.dump_txn():
+            pass
+
+    def dump_txn(self):
+        pos = self.file.tell()
+        h = self.file.read(TRANS_HDR_LEN)
+        if not h:
+            return False
+        tid, stlen, status, ul, dl, el = struct.unpack(TRANS_HDR, h)
+        end = pos + U64(stlen)
+        print >> self.dest, "=" * 60
+        print >> self.dest, "offset: %d" % pos
+        print >> self.dest, "end pos: %d" % end
+        print >> self.dest, "transaction id: %s" % fmt(tid)
+        print >> self.dest, "trec len: %d" % U64(stlen)
+        print >> self.dest, "status: %r" % status
+        user = descr = extra = ""
+        if ul:
+            user = self.file.read(ul)
+        if dl:
+            descr = self.file.read(dl)
+        if el:
+            extra = self.file.read(el)
+        print >> self.dest, "user: %r" % user
+        print >> self.dest, "description: %r" % descr
+        print >> self.dest, "len(extra): %d" % el
+        while self.file.tell() < end:
+            self.dump_data(pos)
+        stlen2 = self.file.read(8)
+        print >> self.dest, "redundant trec len: %d" % U64(stlen2)
+        return True
+
+    def dump_data(self, tloc):
+        pos = self.file.tell()
+        h = self.file.read(DATA_HDR_LEN)
+        assert len(h) == DATA_HDR_LEN
+        oid, revid, sprev, stloc, vlen, sdlen = struct.unpack(DATA_HDR, h)
+        dlen = U64(sdlen)
+        print >> self.dest, "-" * 60
+        print >> self.dest, "offset: %d" % pos
+        print >> self.dest, "oid: %s" % fmt(oid)
+        print >> self.dest, "revid: %s" % fmt(revid)
+        print >> self.dest, "previous record offset: %d" % U64(sprev)
+        print >> self.dest, "transaction offset: %d" % U64(stloc)
+        if vlen:
+            pnv = self.file.read(8)
+            sprevdata = self.file.read(8)
+            version = self.file.read(vlen)
+            print >> self.dest, "version: %r" % version
+            print >> self.dest, "non-version data offset: %d" % U64(pnv)
+            print >> self.dest, \
+                  "previous version data offset: %d" % U64(sprevdata)
+        print >> self.dest, "len(data): %d" % dlen
+        self.file.read(dlen)
+        if not dlen:
+            sbp = self.file.read(8)
+            print >> self.dest, "backpointer: %d" % U64(sbp)


=== Zope/lib/python/ZODB/__init__.py 1.14.2.1 => 1.14.2.2 ===
--- Zope/lib/python/ZODB/__init__.py:1.14.2.1	Tue Nov 12 16:13:58 2002
+++ Zope/lib/python/ZODB/__init__.py	Fri Nov 15 12:45:03 2002
@@ -12,7 +12,7 @@
 #
 ##############################################################################
 
-__version__ = '3.1'
+__version__ = '3.1.1'
 
 import sys
 import cPersistence, Persistence


=== Zope/lib/python/ZODB/FileStorage.py 1.98.2.1 => 1.98.2.2 ===
--- Zope/lib/python/ZODB/FileStorage.py:1.98.2.1	Tue Nov  5 16:23:49 2002
+++ Zope/lib/python/ZODB/FileStorage.py	Fri Nov 15 12:45:03 2002
@@ -309,9 +309,6 @@
         # hook to use something other than builtin dict
         return {}, {}, {}, {}
 
-    def abortVersion(self, src, transaction):
-        return self.commitVersion(src, '', transaction, abort=1)
-
     def _save_index(self):
         """Write the database index to a file to support quick startup
         """
@@ -441,6 +438,9 @@
             # XXX should log the error, though
             pass # We don't care if this fails.
 
+    def abortVersion(self, src, transaction):
+        return self.commitVersion(src, '', transaction, abort=1)
+
     def commitVersion(self, src, dest, transaction, abort=None):
         # We are going to commit by simply storing back pointers.
         if self._is_read_only:
@@ -521,6 +521,9 @@
                 here += heredelta
 
                 current_oids[oid] = 1
+                # Once we've found the data we are looking for,
+                # we can stop chasing backpointers.
+                break
 
             else:
                 # Hm.  This is a non-current record.  Is there a
@@ -763,9 +766,13 @@
                 if vl:
                     self._file.read(vl + 16)
                 # Make sure this looks like the right data record
+                if dl == 0:
+                    # This is also a backpointer.  Gotta trust it.
+                    return pos
                 if dl != len(data):
-                    # XXX what if this data record also has a backpointer?
-                    # I don't think that's possible, but I'm not sure.
+                    # The expected data doesn't match what's in the
+                    # backpointer.  Something is wrong.
+                    error("Mismatch between data and backpointer at %d", pos)
                     return 0
                 _data = self._file.read(dl)
                 if data != _data:
@@ -823,20 +830,7 @@
             # We need to write some version information if this revision is
             # happening in a version.
             if version:
-                pnv = None
-                # We need to write the position of the non-version data.
-                # If the previous revision of the object was in a version,
-                # then it will contain a pnv record.  Otherwise, the
-                # previous record is the non-version data.
-                if old:
-                    self._file.seek(old)
-                    h = self._file.read(42)
-                    doid, x, y, z, vlen, w = unpack(DATA_HDR, h)
-                    if doid != oid:
-                        raise CorruptedDataError, h
-                    # XXX assert versions match?
-                    if vlen > 0:
-                        pnv = self._file.read(8)
+                pnv = self._restore_pnv(oid, old, version, prev_pos)
                 if pnv:
                     self._tfile.write(pnv)
                 else:
@@ -848,20 +842,62 @@
                 self._tfile.write(p64(pv))
                 self._tvindex[version] = here
                 self._tfile.write(version)
-            # And finally, write the data
+            # And finally, write the data or a backpointer
             if data is None:
                 if prev_pos:
                     self._tfile.write(p64(prev_pos))
                 else:
                     # Write a zero backpointer, which indicates an
                     # un-creation transaction.
-                    # write a backpointer instead of data
                     self._tfile.write(z64)
             else:
                 self._tfile.write(data)
         finally:
             self._lock_release()
 
+    def _restore_pnv(self, oid, prev, version, bp):
+        # Find a valid pnv (previous non-version) pointer for this version.
+
+        # If there is no previous record, there can't be a pnv.
+        if not prev:
+            return None
+        
+        pnv = None
+
+        # Load the record pointed to be prev
+        self._file.seek(prev)
+        h = self._file.read(DATA_HDR_LEN)
+        doid, x, y, z, vlen, w = unpack(DATA_HDR, h)
+        if doid != oid:
+            raise CorruptedDataError, h
+        # If the previous record is for a version, it must have
+        # a valid pnv.
+        if vlen > 0:
+            pnv = self._file.read(8)
+            pv = self._file.read(8)
+            v = self._file.read(vlen)
+        elif bp:
+            # XXX Not sure the following is always true:
+            # The previous record is not for this version, yet we
+            # have a backpointer to it.  The current record must
+            # be an undo of an abort or commit, so the backpointer
+            # must be to a version record with a pnv.
+            self._file.seek(bp)
+            h2 = self._file.read(DATA_HDR_LEN)
+            doid2, x, y, z, vlen2, sdl = unpack(DATA_HDR, h2)
+            dl = U64(sdl)
+            if oid != doid2:
+                raise CorruptedDataError, h2
+            if vlen2 > 0:
+                pnv = self._file.read(8)
+                pv = self._file.read(8)
+                v = self._file.read(8)
+            else:
+                warn("restore could not find previous non-version data "
+                     "at %d or %d" % (prev, bp))
+            
+        return pnv
+
     def supportsUndo(self):
         return 1
 
@@ -2155,7 +2191,8 @@
         doid, serial, prev, tloc, vlen, plen = unpack(DATA_HDR, h)
 
         if vlen:
-            file.seek(vlen + 16, 1)
+            file.read(16)
+            version = file.read(vlen)
         if plen != z64:
             return file.read(U64(plen)), serial, old, tloc
         back = file.read(8) # We got a back pointer!
@@ -2178,6 +2215,17 @@
     tid = h[:8]
     return data, serial, tid
 
+def getTxnFromData(file, oid, back):
+    """Return transaction id for data at back."""
+    file.seek(U64(back))
+    h = file.read(DATA_HDR_LEN)
+    doid, serial, prev, stloc, vlen, plen = unpack(DATA_HDR, h)
+    assert oid == doid
+    tloc = U64(stloc)
+    file.seek(tloc)
+    # seek to transaction header, where tid is first 8 bytes
+    return file.read(8)
+
 def _truncate(file, name, pos):
     seek=file.seek
     seek(0,2)
@@ -2407,29 +2455,35 @@
             else:
                 version = ''
 
+            datapos = pos + DATA_HDR_LEN
+            if vlen:
+                datapos += 16 + vlen
+            assert self._file.tell() == datapos, (self._file.tell(), datapos)
+
             if pos + dlen > self._tend or tloc != self._tpos:
                 warn("%s data record exceeds transaction record at %s",
                      file.name, pos)
                 break
 
             self._pos = pos + dlen
-            tid = None
+            prev_txn = None
             if plen:
-                p = self._file.read(plen)
+                data = self._file.read(plen)
             else:
-                p = self._file.read(8)
-                if p == z64:
+                bp = self._file.read(8)
+                if bp == z64:
                     # If the backpointer is 0 (encoded as z64), then
                     # this transaction undoes the object creation.  It
                     # either aborts the version that created the
                     # object or undid the transaction that created it.
                     # Return None instead of a pickle to indicate
                     # this.
-                    p = None
+                    data = None
                 else:
-                    p, _s, tid = _loadBackTxn(self._file, oid, p)
+                    data, _s, tid = _loadBackTxn(self._file, oid, bp)
+                    prev_txn = getTxnFromData(self._file, oid, bp)
 
-            r = Record(oid, serial, version, p, tid)
+            r = Record(oid, serial, version, data, prev_txn)
 
             return r