[Zope3-checkins] CVS: ZODB4/src/zodb/storage/tests - test_recover.py:1.2 undo.py:1.17 synchronization.py:1.7 basic.py:1.10 test_config.py:NONE

Jeremy Hylton jeremy@zope.com
Thu, 19 Jun 2003 17:41:40 -0400


Update of /cvs-repository/ZODB4/src/zodb/storage/tests
In directory cvs.zope.org:/tmp/cvs-serv15960/src/zodb/storage/tests

Modified Files:
	undo.py synchronization.py basic.py 
Added Files:
	test_recover.py 
Removed Files:
	test_config.py 
Log Message:
Merge ZODB3-2-merge branch to the head.

This completes the porting of bug fixes and random improvements from
ZODB 3.2 to ZODB 4.


=== ZODB4/src/zodb/storage/tests/test_recover.py 1.1 => 1.2 ===
--- /dev/null	Thu Jun 19 17:41:40 2003
+++ ZODB4/src/zodb/storage/tests/test_recover.py	Thu Jun 19 17:41:09 2003
@@ -0,0 +1,154 @@
+##############################################################################
+#
+# Copyright (c) 2003 Zope Corporation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.0 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
+"""Tests of the file storage recovery script."""
+
+import base64
+import os
+import random
+import sys
+import tempfile
+import unittest
+import StringIO
+
+from zodb.db import DB
+from zodb.storage.file import FileStorage
+from zodb.storage.file.recover import recover
+from zodb.storage.file.dump import Dumper
+
+from persistence.dict import PersistentDict
+from transaction import get_transaction
+
+class RecoverTest(unittest.TestCase):
+
+    level = 2
+
+    path = None
+
+    def setUp(self):
+        self.path = tempfile.mktemp(suffix=".fs")
+        self.storage = FileStorage(self.path)
+        self.populate()
+        self.dest = tempfile.mktemp(suffix=".fs")
+        self.recovered = None
+
+    def tearDown(self):
+        self.storage.close()
+        if self.recovered is not None:
+            self.recovered.close()
+        self.storage.cleanup()
+##        removefs(self.dest)
+
+    def populate(self):
+        db = DB(self.storage)
+        cn = db.open()
+        rt = cn.root()
+
+        # create a whole bunch of objects,
+        # looks like a Data.fs > 1MB
+        for i in range(50):
+            d = rt[i] = PersistentDict()
+            get_transaction().commit()
+            for j in range(50):
+                d[j] = "a" * j
+            get_transaction().commit()
+
+    def damage(self, num, size):
+        self.storage.close()
+        # Drop size null bytes into num random spots.
+        for i in range(num):
+            offset = random.randint(0, self.storage._pos - size)
+            f = open(self.path, "a+b")
+            f.seek(offset)
+            f.write("\0" * size)
+            f.close()
+
+    ITERATIONS = 5
+
+    def recover(self, source, dest):
+        orig = sys.stdout
+        try:
+            sys.stdout = StringIO.StringIO()
+            try:
+                recover(self.path, self.dest,
+                        verbose=0, partial=1, force=0, pack=1)
+            except SystemExit:
+                raise RuntimeError, "recover tried to exit"
+        finally:
+            sys.stdout = orig
+
+    def testOneBlock(self):
+        for i in range(self.ITERATIONS):
+            self.damage(1, 1024)
+            self.recover(self.path, self.dest)
+            self.recovered = FileStorage(self.dest)
+            self.recovered.close()
+            os.remove(self.path)
+            os.rename(self.dest, self.path)
+
+    def testFourBlocks(self):
+        for i in range(self.ITERATIONS):
+            self.damage(4, 512)
+            self.recover(self.path, self.dest)
+            self.recovered = FileStorage(self.dest)
+            self.recovered.close()
+            os.remove(self.path)
+            os.rename(self.dest, self.path)
+
+    def testBigBlock(self):
+        for i in range(self.ITERATIONS):
+            self.damage(1, 32 * 1024)
+            self.recover(self.path, self.dest)
+            self.recovered = FileStorage(self.dest)
+            self.recovered.close()
+            os.remove(self.path)
+            os.rename(self.dest, self.path)
+
+    def testBadTransaction(self):
+        # Find transaction headers and blast them.
+
+        L = self.storage.undoLog()
+        r = L[3]
+        tid = base64.decodestring(r["id"] + "\n")
+        pos1 = self.storage._txn_find(tid, 0)
+
+        r = L[8]
+        tid = base64.decodestring(r["id"] + "\n")
+        pos2 = self.storage._txn_find(tid, 0)
+
+        self.storage.close()
+
+        # Overwrite the entire header.
+        f = open(self.path, "a+b")
+        f.seek(pos1 - 50)
+        f.write("\0" * 100)
+        f.close()
+        self.recover(self.path, self.dest)
+        self.recovered = FileStorage(self.dest)
+        self.recovered.close()
+        os.remove(self.path)
+        os.rename(self.dest, self.path)
+
+        # Overwrite part of the header.
+        f = open(self.path, "a+b")
+        f.seek(pos2 + 10)
+        f.write("\0" * 100)
+        f.close()
+        self.recover(self.path, self.dest)
+        self.recovered = FileStorage(self.dest)
+        self.recovered.close()
+
+def test_suite():
+    return unittest.makeSuite(RecoverTest)
+
+


=== ZODB4/src/zodb/storage/tests/undo.py 1.16 => 1.17 ===
--- ZODB4/src/zodb/storage/tests/undo.py:1.16	Fri May 16 17:36:57 2003
+++ ZODB4/src/zodb/storage/tests/undo.py	Thu Jun 19 17:41:09 2003
@@ -156,6 +156,27 @@
         eq(zodb_unpickle(data), MinPO(23))
         self._iterate()
 
+##    def testCreationUndoneGetSerial(self):
+##        # XXX Do we really want to change FileStorage to make this test
+##        # pass?  getSerial() is nice and fast right now.
+        
+##        # create an object
+##        oid = self._storage.newObjectId()
+##        revid = self._dostore(oid, data=MinPO(23))
+##        print repr(revid)
+##        # undo its creation
+##        info = self._storage.undoInfo()
+##        tid = info[0]['id']
+##        t = Transaction()
+##        t.note('undo1')
+##        self._storage.tpcBegin(t)
+##        oids = self._storage.undo(tid, t)
+##        self._storage.tpcVote(t)
+##        self._storage.tpcFinish(t)
+##        # Check that calling getSerial on an uncreated object raises a KeyError
+##        # The current version of FileStorage fails this test
+##        self.assertRaises(KeyError, self._storage.getSerial, oid)
+
     def testUndoCreationBranch1(self):
         eq = self.assertEqual
         oid = self._storage.newObjectId()
@@ -766,3 +787,76 @@
                 break
         else:
             self.fail('transaction not found')
+
+    def testPackUndoLog(self):
+        self._initroot()
+        eq = self.assertEqual
+        raises = self.assertRaises
+        # Create a `persistent' object
+        obj = self._newobj()
+        obj.value = 1
+        oid = obj._p_oid
+        # Commit two different revisions
+        revid1 = self._dostore(oid, data=obj)
+        obj.value = 2
+        snooze()
+        packtime = time.time()
+        snooze()
+        revid2 = self._dostore(oid, revid=revid1, data=obj)
+        # Now pack the first transaction
+        self.assertEqual(3, len(self._storage.undoLog()))
+        self._storage.pack(packtime)
+        # The undo log contains only the most resent transaction
+        self.assertEqual(1, len(self._storage.undoLog()))
+
+    def dont_testPackUndoLogUndoable(self):
+        # XXX This test was copied from ZODB3, but no effort was made
+        # to convert the code to make it work in ZODB4.
+        
+        # A disabled test. I wanted to test that the content of the
+        # undo log was consistent, but every storage appears to
+        # include something slightly different. If the result of this
+        # method is only used to fill a GUI then this difference
+        # doesnt matter.  Perhaps re-enable this test once we agree
+        # what should be asserted.
+
+        self._initroot()
+        # Create two `persistent' object
+        obj1 = self._newobj()
+        oid1 = obj1.getoid()
+        obj1.value = 1
+        obj2 = self._newobj()
+        oid2 = obj2.getoid()
+        obj2.value = 2
+        
+        # Commit the first revision of each of them
+        revid11 = self._dostoreNP(oid1, data=pickle.dumps(obj1),
+                                  description="1-1")
+        revid22 = self._dostoreNP(oid2, data=pickle.dumps(obj2),
+                                  description="2-2")
+        
+        # remember the time. everything above here will be packed away
+        snooze()
+        packtime = time.time()
+        snooze()
+        # Commit two revisions of the first object
+        obj1.value = 3
+        revid13 = self._dostoreNP(oid1, revid=revid11,
+                                  data=pickle.dumps(obj1), description="1-3")
+        obj1.value = 4
+        revid14 = self._dostoreNP(oid1, revid=revid13,
+                                  data=pickle.dumps(obj1), description="1-4")
+        # Commit one revision of the second object
+        obj2.value = 5
+        revid25 = self._dostoreNP(oid2, revid=revid22,
+                                  data=pickle.dumps(obj2), description="2-5")
+        # Now pack
+        self.assertEqual(6,len(self._storage.undoLog()))
+        print '\ninitial undoLog was'
+        for r in self._storage.undoLog(): print r
+        self._storage.pack(packtime, referencesf)
+        # The undo log contains only two undoable transaction.
+        print '\nafter packing undoLog was'
+        for r in self._storage.undoLog(): print r
+        # what can we assert about that?
+


=== ZODB4/src/zodb/storage/tests/synchronization.py 1.6 => 1.7 ===
--- ZODB4/src/zodb/storage/tests/synchronization.py:1.6	Thu Mar 20 18:01:41 2003
+++ ZODB4/src/zodb/storage/tests/synchronization.py	Thu Jun 19 17:41:09 2003
@@ -76,7 +76,7 @@
 
     def verifyNotCommitting(self, callable, *args):
         args = (StorageTransactionError, callable) + args
-        apply(self.assertRaises, args)
+        self.assertRaises(*args)
 
     def verifyWrongTrans(self, callable, *args):
         t = Transaction()


=== ZODB4/src/zodb/storage/tests/basic.py 1.9 => 1.10 ===
--- ZODB4/src/zodb/storage/tests/basic.py:1.9	Thu Mar 20 18:01:41 2003
+++ ZODB4/src/zodb/storage/tests/basic.py	Thu Jun 19 17:41:09 2003
@@ -197,3 +197,20 @@
         self._storage.store(oid, None, data, refs, '', t)
         self._storage.tpcVote(t)
         self._storage.tpcFinish(t)
+
+    def testNote(self):
+        oid = self._storage.newObjectId()
+        t = Transaction()
+        self._storage.tpcBegin(t)
+        t.note('this is a test')
+        data, refs = zodb_pickle(MinPO(5))
+        self._storage.store(oid, ZERO, data, refs, '', t)
+        self._storage.tpcVote(t)
+        self._storage.tpcFinish(t)
+
+    def testGetExtensionMethods(self):
+        m = self._storage.getExtensionMethods()
+        self.assertEqual(type(m),type({}))
+        for k,v in m.items():
+            self.assertEqual(v,None)
+            self.assert_(callable(getattr(self._storage,k)))

=== Removed File ZODB4/src/zodb/storage/tests/test_config.py ===