[Zodb-checkins] CVS: ZODB3/ZODB - fsdump.py:1.3.72.2 __init__.py:1.17.2.4 FileStorage.py:1.105.2.7

Jeremy Hylton jeremy@zope.com
Fri, 15 Nov 2002 12:19:28 -0500


Update of /cvs-repository/ZODB3/ZODB
In directory cvs.zope.org:/tmp/cvs-serv1715/ZODB

Modified Files:
      Tag: ZODB3-3_1-branch
	fsdump.py __init__.py FileStorage.py 
Log Message:
Fix two restore() bugs.

Merge changes from short-lived ZODB3-restore-debug-branch.
Add entry to NEWS file about problem.
Bump version numbers to 3.1.1 and 2.0.1.


=== ZODB3/ZODB/fsdump.py 1.3.72.1 => 1.3.72.2 ===
--- ZODB3/ZODB/fsdump.py:1.3.72.1	Mon Oct 21 11:15:55 2002
+++ ZODB3/ZODB/fsdump.py	Fri Nov 15 12:18:55 2002
@@ -76,3 +76,80 @@
         print >> file
         i += 1
     iter.close()
+
+import struct
+from ZODB.FileStorage import TRANS_HDR, TRANS_HDR_LEN
+from ZODB.FileStorage import DATA_HDR, DATA_HDR_LEN
+
+def fmt(p64):
+    # Return a nicely formatted string for a packaged 64-bit value
+    return "%016x" % U64(p64)
+
+class Dumper:
+    """A very verbose dumper for debuggin FileStorage problems."""
+
+    def __init__(self, path, dest=None):
+        self.file = open(path, "rb")
+        self.dest = dest
+
+    def dump(self):
+        fid = self.file.read(4)
+        print >> self.dest, "*" * 60
+        print >> self.dest, "file identifier: %r" % fid
+        while self.dump_txn():
+            pass
+
+    def dump_txn(self):
+        pos = self.file.tell()
+        h = self.file.read(TRANS_HDR_LEN)
+        if not h:
+            return False
+        tid, stlen, status, ul, dl, el = struct.unpack(TRANS_HDR, h)
+        end = pos + U64(stlen)
+        print >> self.dest, "=" * 60
+        print >> self.dest, "offset: %d" % pos
+        print >> self.dest, "end pos: %d" % end
+        print >> self.dest, "transaction id: %s" % fmt(tid)
+        print >> self.dest, "trec len: %d" % U64(stlen)
+        print >> self.dest, "status: %r" % status
+        user = descr = extra = ""
+        if ul:
+            user = self.file.read(ul)
+        if dl:
+            descr = self.file.read(dl)
+        if el:
+            extra = self.file.read(el)
+        print >> self.dest, "user: %r" % user
+        print >> self.dest, "description: %r" % descr
+        print >> self.dest, "len(extra): %d" % el
+        while self.file.tell() < end:
+            self.dump_data(pos)
+        stlen2 = self.file.read(8)
+        print >> self.dest, "redundant trec len: %d" % U64(stlen2)
+        return True
+
+    def dump_data(self, tloc):
+        pos = self.file.tell()
+        h = self.file.read(DATA_HDR_LEN)
+        assert len(h) == DATA_HDR_LEN
+        oid, revid, sprev, stloc, vlen, sdlen = struct.unpack(DATA_HDR, h)
+        dlen = U64(sdlen)
+        print >> self.dest, "-" * 60
+        print >> self.dest, "offset: %d" % pos
+        print >> self.dest, "oid: %s" % fmt(oid)
+        print >> self.dest, "revid: %s" % fmt(revid)
+        print >> self.dest, "previous record offset: %d" % U64(sprev)
+        print >> self.dest, "transaction offset: %d" % U64(stloc)
+        if vlen:
+            pnv = self.file.read(8)
+            sprevdata = self.file.read(8)
+            version = self.file.read(vlen)
+            print >> self.dest, "version: %r" % version
+            print >> self.dest, "non-version data offset: %d" % U64(pnv)
+            print >> self.dest, \
+                  "previous version data offset: %d" % U64(sprevdata)
+        print >> self.dest, "len(data): %d" % dlen
+        self.file.read(dlen)
+        if not dlen:
+            sbp = self.file.read(8)
+            print >> self.dest, "backpointer: %d" % U64(sbp)


=== ZODB3/ZODB/__init__.py 1.17.2.3 => 1.17.2.4 ===
--- ZODB3/ZODB/__init__.py:1.17.2.3	Wed Oct 23 15:46:52 2002
+++ ZODB3/ZODB/__init__.py	Fri Nov 15 12:18:55 2002
@@ -12,7 +12,7 @@
 #
 ##############################################################################
 
-__version__ = '3.1'
+__version__ = '3.1.1'
 
 import sys
 import cPersistence, Persistence


=== ZODB3/ZODB/FileStorage.py 1.105.2.6 => 1.105.2.7 ===
--- ZODB3/ZODB/FileStorage.py:1.105.2.6	Thu Nov 14 12:01:45 2002
+++ ZODB3/ZODB/FileStorage.py	Fri Nov 15 12:18:55 2002
@@ -766,9 +766,13 @@
                 if vl:
                     self._file.read(vl + 16)
                 # Make sure this looks like the right data record
+                if dl == 0:
+                    # This is also a backpointer.  Gotta trust it.
+                    return pos
                 if dl != len(data):
-                    # XXX what if this data record also has a backpointer?
-                    # I don't think that's possible, but I'm not sure.
+                    # The expected data doesn't match what's in the
+                    # backpointer.  Something is wrong.
+                    error("Mismatch between data and backpointer at %d", pos)
                     return 0
                 _data = self._file.read(dl)
                 if data != _data:
@@ -826,20 +830,7 @@
             # We need to write some version information if this revision is
             # happening in a version.
             if version:
-                pnv = None
-                # We need to write the position of the non-version data.
-                # If the previous revision of the object was in a version,
-                # then it will contain a pnv record.  Otherwise, the
-                # previous record is the non-version data.
-                if old:
-                    self._file.seek(old)
-                    h = self._file.read(42)
-                    doid, x, y, z, vlen, w = unpack(DATA_HDR, h)
-                    if doid != oid:
-                        raise CorruptedDataError, h
-                    # XXX assert versions match?
-                    if vlen > 0:
-                        pnv = self._file.read(8)
+                pnv = self._restore_pnv(oid, old, version, prev_pos)
                 if pnv:
                     self._tfile.write(pnv)
                 else:
@@ -851,20 +842,62 @@
                 self._tfile.write(p64(pv))
                 self._tvindex[version] = here
                 self._tfile.write(version)
-            # And finally, write the data
+            # And finally, write the data or a backpointer
             if data is None:
                 if prev_pos:
                     self._tfile.write(p64(prev_pos))
                 else:
                     # Write a zero backpointer, which indicates an
                     # un-creation transaction.
-                    # write a backpointer instead of data
                     self._tfile.write(z64)
             else:
                 self._tfile.write(data)
         finally:
             self._lock_release()
 
+    def _restore_pnv(self, oid, prev, version, bp):
+        # Find a valid pnv (previous non-version) pointer for this version.
+
+        # If there is no previous record, there can't be a pnv.
+        if not prev:
+            return None
+        
+        pnv = None
+
+        # Load the record pointed to be prev
+        self._file.seek(prev)
+        h = self._file.read(DATA_HDR_LEN)
+        doid, x, y, z, vlen, w = unpack(DATA_HDR, h)
+        if doid != oid:
+            raise CorruptedDataError, h
+        # If the previous record is for a version, it must have
+        # a valid pnv.
+        if vlen > 0:
+            pnv = self._file.read(8)
+            pv = self._file.read(8)
+            v = self._file.read(vlen)
+        elif bp:
+            # XXX Not sure the following is always true:
+            # The previous record is not for this version, yet we
+            # have a backpointer to it.  The current record must
+            # be an undo of an abort or commit, so the backpointer
+            # must be to a version record with a pnv.
+            self._file.seek(bp)
+            h2 = self._file.read(DATA_HDR_LEN)
+            doid2, x, y, z, vlen2, sdl = unpack(DATA_HDR, h2)
+            dl = U64(sdl)
+            if oid != doid2:
+                raise CorruptedDataError, h2
+            if vlen2 > 0:
+                pnv = self._file.read(8)
+                pv = self._file.read(8)
+                v = self._file.read(8)
+            else:
+                warn("restore could not find previous non-version data "
+                     "at %d or %d" % (prev, bp))
+            
+        return pnv
+
     def supportsUndo(self):
         return 1
 
@@ -2182,6 +2215,17 @@
     tid = h[:8]
     return data, serial, tid
 
+def getTxnFromData(file, oid, back):
+    """Return transaction id for data at back."""
+    file.seek(U64(back))
+    h = file.read(DATA_HDR_LEN)
+    doid, serial, prev, stloc, vlen, plen = unpack(DATA_HDR, h)
+    assert oid == doid
+    tloc = U64(stloc)
+    file.seek(tloc)
+    # seek to transaction header, where tid is first 8 bytes
+    return file.read(8)
+
 def _truncate(file, name, pos):
     seek=file.seek
     seek(0,2)
@@ -2422,7 +2466,7 @@
                 break
 
             self._pos = pos + dlen
-            tid = None
+            prev_txn = None
             if plen:
                 data = self._file.read(plen)
             else:
@@ -2437,8 +2481,9 @@
                     data = None
                 else:
                     data, _s, tid = _loadBackTxn(self._file, oid, bp)
+                    prev_txn = getTxnFromData(self._file, oid, bp)
 
-            r = Record(oid, serial, version, data, tid)
+            r = Record(oid, serial, version, data, prev_txn)
 
             return r