[Zope-Checkins] CVS: ZODB3/ZODB - fsdump.py:1.3.72.1.4.1 FileStorage.py:1.105.2.6.2.1

Jeremy Hylton jeremy@zope.com
Fri, 15 Nov 2002 10:44:13 -0500


Update of /cvs-repository/ZODB3/ZODB
In directory cvs.zope.org:/tmp/cvs-serv23852/ZODB

Modified Files:
      Tag: ZODB3-restore-debug-branch
	fsdump.py FileStorage.py 
Log Message:
Add new test checkRecoverUndoInVersion(), which fails,
and a bunch of new code for debugging that problem.


=== ZODB3/ZODB/fsdump.py 1.3.72.1 => 1.3.72.1.4.1 ===
--- ZODB3/ZODB/fsdump.py:1.3.72.1	Mon Oct 21 11:15:55 2002
+++ ZODB3/ZODB/fsdump.py	Fri Nov 15 10:44:13 2002
@@ -76,3 +76,80 @@
         print >> file
         i += 1
     iter.close()
+
+import struct
+from ZODB.FileStorage import TRANS_HDR, TRANS_HDR_LEN
+from ZODB.FileStorage import DATA_HDR, DATA_HDR_LEN
+
+def fmt(p64):
+    # Return a nicely formatted string for a packaged 64-bit value
+    return "%016x" % U64(p64)
+
+class Dumper:
+    """A very verbose dumper for debuggin FileStorage problems."""
+
+    def __init__(self, path, dest=None):
+        self.file = open(path, "rb")
+        self.dest = dest
+
+    def dump(self):
+        fid = self.file.read(4)
+        print >> self.dest, "*" * 60
+        print >> self.dest, "file identifier: %r" % fid
+        while self.dump_txn():
+            pass
+
+    def dump_txn(self):
+        pos = self.file.tell()
+        h = self.file.read(TRANS_HDR_LEN)
+        if not h:
+            return False
+        tid, stlen, status, ul, dl, el = struct.unpack(TRANS_HDR, h)
+        end = pos + U64(stlen)
+        print >> self.dest, "=" * 60
+        print >> self.dest, "offset: %d" % pos
+        print >> self.dest, "end pos: %d" % end
+        print >> self.dest, "transaction id: %s" % fmt(tid)
+        print >> self.dest, "trec len: %d" % U64(stlen)
+        print >> self.dest, "status: %r" % status
+        user = descr = extra = ""
+        if ul:
+            user = self.file.read(ul)
+        if dl:
+            descr = self.file.read(dl)
+        if el:
+            extra = self.file.read(el)
+        print >> self.dest, "user: %r" % user
+        print >> self.dest, "description: %r" % descr
+        print >> self.dest, "len(extra): %d" % el
+        while self.file.tell() < end:
+            self.dump_data(pos)
+        stlen2 = self.file.read(8)
+        print >> self.dest, "redundant trec len: %d" % U64(stlen2)
+        return True
+
+    def dump_data(self, tloc):
+        pos = self.file.tell()
+        h = self.file.read(DATA_HDR_LEN)
+        assert len(h) == DATA_HDR_LEN
+        oid, revid, sprev, stloc, vlen, sdlen = struct.unpack(DATA_HDR, h)
+        dlen = U64(sdlen)
+        print >> self.dest, "-" * 60
+        print >> self.dest, "offset: %d" % pos
+        print >> self.dest, "oid: %s" % fmt(oid)
+        print >> self.dest, "revid: %s" % fmt(revid)
+        print >> self.dest, "previous record offset: %d" % U64(sprev)
+        print >> self.dest, "transaction offset: %d" % U64(stloc)
+        if vlen:
+            pnv = self.file.read(8)
+            sprevdata = self.file.read(8)
+            version = self.file.read(vlen)
+            print >> self.dest, "version: %r" % version
+            print >> self.dest, "non-version data offset: %d" % U64(pnv)
+            print >> self.dest, \
+                  "previous version data offset: %d" % U64(sprevdata)
+        print >> self.dest, "len(data): %d" % dlen
+        self.file.read(dlen)
+        if not dlen:
+            sbp = self.file.read(8)
+            print >> self.dest, "backpointer: %d" % U64(sbp)


=== ZODB3/ZODB/FileStorage.py 1.105.2.6 => 1.105.2.6.2.1 ===
--- ZODB3/ZODB/FileStorage.py:1.105.2.6	Thu Nov 14 12:01:45 2002
+++ ZODB3/ZODB/FileStorage.py	Fri Nov 15 10:44:13 2002
@@ -826,20 +826,7 @@
             # We need to write some version information if this revision is
             # happening in a version.
             if version:
-                pnv = None
-                # We need to write the position of the non-version data.
-                # If the previous revision of the object was in a version,
-                # then it will contain a pnv record.  Otherwise, the
-                # previous record is the non-version data.
-                if old:
-                    self._file.seek(old)
-                    h = self._file.read(42)
-                    doid, x, y, z, vlen, w = unpack(DATA_HDR, h)
-                    if doid != oid:
-                        raise CorruptedDataError, h
-                    # XXX assert versions match?
-                    if vlen > 0:
-                        pnv = self._file.read(8)
+                pnv = self._restore_pnv(oid, old, version, prev_pos)
                 if pnv:
                     self._tfile.write(pnv)
                 else:
@@ -851,19 +838,61 @@
                 self._tfile.write(p64(pv))
                 self._tvindex[version] = here
                 self._tfile.write(version)
-            # And finally, write the data
+            # And finally, write the data or a backpointer
             if data is None:
                 if prev_pos:
                     self._tfile.write(p64(prev_pos))
                 else:
                     # Write a zero backpointer, which indicates an
                     # un-creation transaction.
-                    # write a backpointer instead of data
                     self._tfile.write(z64)
             else:
                 self._tfile.write(data)
         finally:
             self._lock_release()
+
+    def _restore_pnv(self, oid, prev, version, bp):
+        # Find a valid pnv (previous non-version) pointer for this version.
+
+        # If there is no previous record, there can't be a pnv.
+        if not prev:
+            return None
+        
+        pnv = None
+
+        # Load the record pointed to be prev
+        self._file.seek(prev)
+        h = self._file.read(DATA_HDR_LEN)
+        doid, x, y, z, vlen, w = unpack(DATA_HDR, h)
+        if doid != oid:
+            raise CorruptedDataError, h
+        # If the previous record is for a version, it must have
+        # a valid pnv.
+        if vlen > 0:
+            pnv = self._file.read(8)
+            pv = self._file.read(8)
+            v = self._file.read(vlen)
+        elif bp:
+            # XXX Not sure the following is always true:
+            # The previous record is not for this version, yet we
+            # have a backpointer to it.  The current record must
+            # be an undo of an abort or commit, so the backpointer
+            # must be to a version record with a pnv.
+            self._file.seek(bp)
+            h2 = self._file.read(DATA_HDR_LEN)
+            doid2, x, y, z, vlen2, sdl = unpack(DATA_HDR, h2)
+            dl = U64(sdl)
+            if oid != doid2:
+                raise CorruptedDataError, h2
+            if vlen2 > 0:
+                pnv = self._file.read(8)
+                pv = self._file.read(8)
+                v = self._file.read(8)
+            else:
+                warn("restore could not find previous non-version data "
+                     "at %d or %d" % (prev, bp))
+            
+        return pnv
 
     def supportsUndo(self):
         return 1