[Zodb-checkins] CVS: ZODB3/ZODB/tests - testRecover.py:1.2.6.1
HistoryStorage.py:1.8.26.1 MTStorage.py:1.7.26.1
PackableStorage.py:1.15.26.1 PersistentStorage.py:1.4.26.1
ReadOnlyStorage.py:1.5.26.1 StorageTestBase.py:1.25.8.1
TransactionalUndoStorage.py:1.28.2.1
VersionStorage.py:1.22.8.1 testFileStorage.py:1.31.2.1
testZODB.py:1.14.2.1 testStorageConfig.py:NONE
Jeremy Hylton
jeremy at zope.com
Thu May 29 13:30:53 EDT 2003
Update of /cvs-repository/ZODB3/ZODB/tests
In directory cvs.zope.org:/tmp/cvs-serv13391/ZODB/tests
Modified Files:
Tag: ZODB3-auth-branch
HistoryStorage.py MTStorage.py PackableStorage.py
PersistentStorage.py ReadOnlyStorage.py StorageTestBase.py
TransactionalUndoStorage.py VersionStorage.py
testFileStorage.py testZODB.py
Added Files:
Tag: ZODB3-auth-branch
testRecover.py
Removed Files:
Tag: ZODB3-auth-branch
testStorageConfig.py
Log Message:
Merge trunk to auth branch.
=== Added File ZODB3/ZODB/tests/testRecover.py ===
##############################################################################
#
# Copyright (c) 2003 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
"""Tests of the file storage recovery script."""
import base64
import os
import random
import sys
import tempfile
import unittest
import StringIO
import ZODB
from ZODB.FileStorage import FileStorage
from ZODB.PersistentMapping import PersistentMapping
from ZODB.fsrecover import recover
from ZODB.tests.StorageTestBase import removefs
from ZODB.fsdump import Dumper
class RecoverTest(unittest.TestCase):
level = 2
path = None
def setUp(self):
self.path = tempfile.mktemp(suffix=".fs")
self.storage = FileStorage(self.path)
self.populate()
self.dest = tempfile.mktemp(suffix=".fs")
self.recovered = None
def tearDown(self):
self.storage.close()
if self.recovered is not None:
self.recovered.close()
removefs(self.path)
removefs(self.dest)
def populate(self):
db = ZODB.DB(self.storage)
cn = db.open()
rt = cn.root()
# create a whole bunch of objects,
# looks like a Data.fs > 1MB
for i in range(50):
d = rt[i] = PersistentMapping()
get_transaction().commit()
for j in range(50):
d[j] = "a" * j
get_transaction().commit()
def damage(self, num, size):
self.storage.close()
# Drop size null bytes into num random spots.
for i in range(num):
offset = random.randint(0, self.storage._pos - size)
f = open(self.path, "a+b")
f.seek(offset)
f.write("\0" * size)
f.close()
ITERATIONS = 5
def recover(self, source, dest):
orig = sys.stdout
try:
sys.stdout = StringIO.StringIO()
try:
recover(self.path, self.dest,
verbose=0, partial=1, force=0, pack=1)
except SystemExit:
raise RuntimeError, "recover tried to exit"
finally:
sys.stdout = orig
def testOneBlock(self):
for i in range(self.ITERATIONS):
self.damage(1, 1024)
self.recover(self.path, self.dest)
self.recovered = FileStorage(self.dest)
self.recovered.close()
os.remove(self.path)
os.rename(self.dest, self.path)
def testFourBlocks(self):
for i in range(self.ITERATIONS):
self.damage(4, 512)
self.recover(self.path, self.dest)
self.recovered = FileStorage(self.dest)
self.recovered.close()
os.remove(self.path)
os.rename(self.dest, self.path)
def testBigBlock(self):
for i in range(self.ITERATIONS):
self.damage(1, 32 * 1024)
self.recover(self.path, self.dest)
self.recovered = FileStorage(self.dest)
self.recovered.close()
os.remove(self.path)
os.rename(self.dest, self.path)
def testBadTransaction(self):
# Find transaction headers and blast them.
L = self.storage.undoLog()
r = L[3]
tid = base64.decodestring(r["id"] + "\n")
pos1 = self.storage._txn_find(tid, False)
r = L[8]
tid = base64.decodestring(r["id"] + "\n")
pos2 = self.storage._txn_find(tid, False)
self.storage.close()
# Overwrite the entire header.
f = open(self.path, "a+b")
f.seek(pos1 - 50)
f.write("\0" * 100)
f.close()
self.recover(self.path, self.dest)
self.recovered = FileStorage(self.dest)
self.recovered.close()
os.remove(self.path)
os.rename(self.dest, self.path)
# Overwrite part of the header.
f = open(self.path, "a+b")
f.seek(pos2 + 10)
f.write("\0" * 100)
f.close()
self.recover(self.path, self.dest)
self.recovered = FileStorage(self.dest)
self.recovered.close()
def test_suite():
return unittest.makeSuite(RecoverTest)
=== ZODB3/ZODB/tests/HistoryStorage.py 1.8 => 1.8.26.1 ===
--- ZODB3/ZODB/tests/HistoryStorage.py:1.8 Thu Dec 5 19:00:53 2002
+++ ZODB3/ZODB/tests/HistoryStorage.py Thu May 29 12:30:52 2003
@@ -72,6 +72,8 @@
eq(d['version'], '')
def checkVersionHistory(self):
+ if not self._storage.supportsVersions():
+ return
eq = self.assertEqual
# Store a couple of non-version revisions
oid = self._storage.new_oid()
@@ -110,6 +112,8 @@
eq(d['version'], '')
def checkHistoryAfterVersionCommit(self):
+ if not self._storage.supportsVersions():
+ return
eq = self.assertEqual
# Store a couple of non-version revisions
oid = self._storage.new_oid()
@@ -168,6 +172,8 @@
eq(d['version'], '')
def checkHistoryAfterVersionAbort(self):
+ if not self._storage.supportsVersions():
+ return
eq = self.assertEqual
# Store a couple of non-version revisions
oid = self._storage.new_oid()
=== ZODB3/ZODB/tests/MTStorage.py 1.7 => 1.7.26.1 ===
--- ZODB3/ZODB/tests/MTStorage.py:1.7 Thu Dec 5 19:00:53 2002
+++ ZODB3/ZODB/tests/MTStorage.py Thu May 29 12:30:52 2003
@@ -1,17 +1,5 @@
-##############################################################################
-#
-# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
-# All Rights Reserved.
-#
-# This software is subject to the provisions of the Zope Public License,
-# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
-# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
-# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
-# FOR A PARTICULAR PURPOSE.
-#
-##############################################################################
import random
+import sys
import threading
import time
@@ -31,19 +19,50 @@
l.sort()
return l
-class ZODBClientThread(threading.Thread):
+class TestThread(threading.Thread):
+ """Base class for defining threads that run from unittest.
- __super_init = threading.Thread.__init__
+ If the thread exits with an uncaught exception, catch it and
+ re-raise it when the thread is joined. The re-raise will cause
+ the test to fail.
+
+ The subclass should define a runtest() method instead of a run()
+ method.
+ """
+
+ def __init__(self, test):
+ threading.Thread.__init__(self)
+ self.test = test
+ self._fail = None
+ self._exc_info = None
+
+ def run(self):
+ try:
+ self.runtest()
+ except:
+ self._exc_info = sys.exc_info()
+
+ def fail(self, msg=""):
+ self._test.fail(msg)
+
+ def join(self, timeout=None):
+ threading.Thread.join(self, timeout)
+ if self._exc_info:
+ raise self._exc_info[0], self._exc_info[1], self._exc_info[2]
+
+class ZODBClientThread(TestThread):
+
+ __super_init = TestThread.__init__
def __init__(self, db, test, commits=10, delay=SHORT_DELAY):
- self.__super_init()
+ self.__super_init(test)
self.setDaemon(1)
self.db = db
self.test = test
self.commits = commits
self.delay = delay
- def run(self):
+ def runtest(self):
conn = self.db.open()
root = conn.root()
d = self.get_thread_dict(root)
@@ -69,27 +88,28 @@
root[name] = m
get_transaction().commit()
break
- except ConflictError:
+ except ConflictError, err:
get_transaction().abort()
+ root._p_jar.sync()
for i in range(10):
try:
return root.get(name)
except ConflictError:
get_transaction().abort()
-class StorageClientThread(threading.Thread):
+class StorageClientThread(TestThread):
- __super_init = threading.Thread.__init__
+ __super_init = TestThread.__init__
def __init__(self, storage, test, commits=10, delay=SHORT_DELAY):
- self.__super_init()
+ self.__super_init(test)
self.storage = storage
self.test = test
self.commits = commits
self.delay = delay
self.oids = {}
- def run(self):
+ def runtest(self):
for i in range(self.commits):
self.dostore(i)
self.check()
@@ -133,7 +153,7 @@
class ExtStorageClientThread(StorageClientThread):
- def run(self):
+ def runtest(self):
# pick some other storage ops to execute
ops = [getattr(self, meth) for meth in dir(ExtStorageClientThread)
if meth.startswith('do_')]
=== ZODB3/ZODB/tests/PackableStorage.py 1.15 => 1.15.26.1 ===
--- ZODB3/ZODB/tests/PackableStorage.py:1.15 Thu Dec 5 19:00:53 2002
+++ ZODB3/ZODB/tests/PackableStorage.py Thu May 29 12:30:52 2003
@@ -25,11 +25,15 @@
except ImportError:
from StringIO import StringIO
+import threading
import time
+
from ZODB import DB
from Persistence import Persistent
from ZODB.referencesf import referencesf
-
+from ZODB.tests.MinPO import MinPO
+from ZODB.tests.StorageTestBase import snooze
+from ZODB.POSException import ConflictError
ZERO = '\0'*8
@@ -376,3 +380,58 @@
conn.sync()
eq(root['obj'].value, 7)
+
+ def checkPackWhileWriting(self):
+ # A storage should allow some reading and writing during
+ # a pack. This test attempts to exercise locking code
+ # in the storage to test that it is safe. It generates
+ # a lot of revisions, so that pack takes a long time.
+
+ db = DB(self._storage)
+ conn = db.open()
+ root = conn.root()
+
+ for i in range(10):
+ root[i] = MinPO(i)
+ get_transaction().commit()
+
+ snooze()
+ packt = time.time()
+
+ for j in range(10):
+ for i in range(10):
+ root[i].value = MinPO(i)
+ get_transaction().commit()
+
+ threads = [ClientThread(db) for i in range(4)]
+ for t in threads:
+ t.start()
+ db.pack(packt)
+ for t in threads:
+ t.join(30)
+ for t in threads:
+ t.join(1)
+ self.assert_(not t.isAlive())
+
+ # iterator over the storage to make sure it's sane
+ if not hasattr(self._storage, "iterator"):
+ return
+ iter = self._storage.iterator()
+ for txn in iter:
+ for data in txn:
+ pass
+ iter.close()
+
+class ClientThread(threading.Thread):
+
+ def __init__(self, db):
+ threading.Thread.__init__(self)
+ self.root = db.open().root()
+
+ def run(self):
+ for j in range(50):
+ try:
+ self.root[j % 10].value = MinPO(j)
+ get_transaction().commit()
+ except ConflictError:
+ get_transaction().abort()
=== ZODB3/ZODB/tests/PersistentStorage.py 1.4 => 1.4.26.1 ===
--- ZODB3/ZODB/tests/PersistentStorage.py:1.4 Fri Dec 6 14:01:40 2002
+++ ZODB3/ZODB/tests/PersistentStorage.py Thu May 29 12:30:52 2003
@@ -28,7 +28,8 @@
self._dostore()
oid = self._storage.new_oid()
revid = self._dostore(oid)
- self._dostore(oid, revid, data=8, version='b')
+ if self._storage.supportsVersions():
+ self._dostore(oid, revid, data=8, version='b')
oid = self._storage.new_oid()
revid = self._dostore(oid, data=1)
revid = self._dostore(oid, revid, data=2)
=== ZODB3/ZODB/tests/ReadOnlyStorage.py 1.5 => 1.5.26.1 ===
--- ZODB3/ZODB/tests/ReadOnlyStorage.py:1.5 Thu Dec 5 19:00:53 2002
+++ ZODB3/ZODB/tests/ReadOnlyStorage.py Thu May 29 12:30:52 2003
@@ -46,10 +46,12 @@
t = Transaction()
self.assertRaises(ReadOnlyError, self._storage.tpc_begin, t)
- self.assertRaises(ReadOnlyError, self._storage.abortVersion,
- '', t)
- self.assertRaises(ReadOnlyError, self._storage.commitVersion,
- '', '', t)
+ if self._storage.supportsVersions():
+ self.assertRaises(ReadOnlyError, self._storage.abortVersion,
+ '', t)
+ self.assertRaises(ReadOnlyError, self._storage.commitVersion,
+ '', '', t)
+
self.assertRaises(ReadOnlyError, self._storage.store,
'\000' * 8, None, '', '', t)
=== ZODB3/ZODB/tests/StorageTestBase.py 1.25 => 1.25.8.1 ===
--- ZODB3/ZODB/tests/StorageTestBase.py:1.25 Mon Mar 17 12:52:34 2003
+++ ZODB3/ZODB/tests/StorageTestBase.py Thu May 29 12:30:52 2003
@@ -31,6 +31,7 @@
from cStringIO import StringIO
from ZODB.Transaction import Transaction
+from ZODB.utils import u64
from ZODB.tests.MinPO import MinPO
@@ -64,10 +65,15 @@
p.dump(state)
return f.getvalue(1)
+def persistent_load(pid):
+ # helper for zodb_unpickle
+ return "ref to %s.%s oid=%s" % (pid[1][0], pid[1][1], u64(pid[0]))
+
def zodb_unpickle(data):
"""Unpickle an object stored using the format expected by ZODB."""
f = StringIO(data)
u = Unpickler(f)
+ u.persistent_load = persistent_load
klass_info = u.load()
if isinstance(klass_info, types.TupleType):
if isinstance(klass_info[0], types.TupleType):
=== ZODB3/ZODB/tests/TransactionalUndoStorage.py 1.28 => 1.28.2.1 ===
--- ZODB3/ZODB/tests/TransactionalUndoStorage.py:1.28 Tue Apr 22 14:02:41 2003
+++ ZODB3/ZODB/tests/TransactionalUndoStorage.py Thu May 29 12:30:52 2003
@@ -483,14 +483,8 @@
# Add a few object revisions
oid = self._storage.new_oid()
revid1 = self._dostore(oid, data=MinPO(51))
- # For the pack(), we need a timestamp greater than revid1's timestamp.
- # The semantics of pack()'s `t' argument is that all non-current
- # revisions with an earlier timestamp will be packed away. If they
- # were equal (because the Windows clock resolution is too coarse),
- # then we won't pack away the first revision.
- now = packtime = time.time()
- while packtime <= now:
- packtime = time.time()
+ packtime = time.time()
+ snooze() # time.time() now distinct from packtime
revid2 = self._dostore(oid, revid=revid1, data=MinPO(52))
revid3 = self._dostore(oid, revid=revid2, data=MinPO(53))
# Now get the undo log
@@ -625,6 +619,7 @@
obj = root["key%d" % i]
self.assertEqual(obj.value, i)
root.items()
+ self._inter_pack_pause()
def checkPackAfterUndoManyTimes(self):
db = DB(self._storage)
@@ -664,95 +659,12 @@
# never change that.
self.assertEqual(rt["test"].value, 3)
self.assertEqual(rt["test2"].value, 2)
+ self._inter_pack_pause()
- def testTransactionalUndoIterator(self):
- # check that data_txn set in iterator makes sense
- if not hasattr(self._storage, "iterator"):
- return
-
- s = self._storage
-
- BATCHES = 4
- OBJECTS = 4
-
- orig = []
- for i in range(BATCHES):
- t = Transaction()
- tid = p64(i + 1)
- s.tpcBegin(t, tid)
- for j in range(OBJECTS):
- oid = s.newObjectId()
- obj = MinPO(i * OBJECTS + j)
- data, refs = zodb_pickle(obj)
- revid = s.store(oid, None, data, refs, '', t)
- orig.append((tid, oid, revid))
- s.tpcVote(t)
- s.tpcFinish(t)
-
- i = 0
- for tid, oid, revid in orig:
- self._dostore(oid, revid=revid, data=MinPO(revid),
- description="update %s" % i)
-
- # Undo the OBJECTS transactions that modified objects created
- # in the ith original transaction.
-
- def undo(i):
- info = s.undoInfo()
- t = Transaction()
- s.tpcBegin(t)
- base = i * OBJECTS + i
- for j in range(OBJECTS):
- tid = info[base + j]['id']
- s.undo(tid, t)
- s.tpcVote(t)
- s.tpcFinish(t)
-
- for i in range(BATCHES):
- undo(i)
-
- # There are now (2 + OBJECTS) * BATCHES transactions:
- # BATCHES original transactions, followed by
- # OBJECTS * BATCHES modifications, followed by
- # BATCHES undos
-
- fsiter = iter(s.iterator())
- offset = 0
-
- eq = self.assertEqual
-
- for i in range(BATCHES):
- txn = fsiter.next()
- offset += 1
-
- tid = p64(i + 1)
- eq(txn.tid, tid)
-
- L1 = [(rec.oid, rec.serial, rec.data_txn) for rec in txn]
- L2 = [(oid, revid, None) for _tid, oid, revid in orig
- if _tid == tid]
-
- eq(L1, L2)
-
- for i in range(BATCHES * OBJECTS):
- txn = fsiter.next()
- offset += 1
- eq(len([rec for rec in txn if rec.data_txn is None]), 1)
-
- for i in range(BATCHES):
- txn = fsiter.next()
- offset += 1
-
- # The undos are performed in reverse order.
- otid = p64(BATCHES - i)
- L1 = [(rec.oid, rec.data_txn) for rec in txn]
- L2 = [(oid, otid) for _tid, oid, revid in orig
- if _tid == otid]
- L1.sort()
- L2.sort()
- eq(L1, L2)
-
- self.assertRaises(StopIteration, fsiter.next)
+ def _inter_pack_pause(self):
+ # DirectoryStorage needs a pause between packs,
+ # most other storages dont.
+ pass
def checkTransactionalUndoIterator(self):
# check that data_txn set in iterator makes sense
=== ZODB3/ZODB/tests/VersionStorage.py 1.22 => 1.22.8.1 ===
--- ZODB3/ZODB/tests/VersionStorage.py:1.22 Tue Apr 1 10:05:34 2003
+++ ZODB3/ZODB/tests/VersionStorage.py Thu May 29 12:30:52 2003
@@ -466,3 +466,65 @@
txn.commit()
self._storage.pack(time.time(), referencesf)
+
+ def checkPackVersionReachable(self):
+ db = DB(self._storage)
+ cn = db.open()
+ root = cn.root()
+
+ names = "a", "b", "c"
+
+ for name in names:
+ root[name] = MinPO(name)
+ get_transaction().commit()
+
+ for name in names:
+ cn2 = db.open(version=name)
+ rt2 = cn2.root()
+ obj = rt2[name]
+ obj.value = MinPO("version")
+ get_transaction().commit()
+ cn2.close()
+
+ root["d"] = MinPO("d")
+ get_transaction().commit()
+
+ self._storage.pack(time.time(), referencesf)
+ cn.sync()
+ cn._cache.clear()
+
+ # make sure all the non-version data is there
+ for name, obj in root.items():
+ self.assertEqual(name, obj.value)
+
+ # make sure all the version-data is there,
+ # and create a new revision in the version
+ for name in names:
+ cn2 = db.open(version=name)
+ rt2 = cn2.root()
+ obj = rt2[name].value
+ self.assertEqual(obj.value, "version")
+ obj.value = "still version"
+ get_transaction().commit()
+ cn2.close()
+
+ db.abortVersion("b")
+ txn = get_transaction()
+ txn.note("abort version b")
+ txn.commit()
+
+ t = time.time()
+ snooze()
+
+ L = db.undoInfo()
+ db.undo(L[0]["id"])
+ txn = get_transaction()
+ txn.note("undo abort")
+ txn.commit()
+
+ self._storage.pack(t, referencesf)
+
+ cn2 = db.open(version="b")
+ rt2 = cn2.root()
+ self.assertEqual(rt2["b"].value.value, "still version")
+
=== ZODB3/ZODB/tests/testFileStorage.py 1.31 => 1.31.2.1 ===
--- ZODB3/ZODB/tests/testFileStorage.py:1.31 Tue Apr 22 14:03:50 2003
+++ ZODB3/ZODB/tests/testFileStorage.py Thu May 29 12:30:52 2003
@@ -170,25 +170,6 @@
self.failUnless(self._storage._records_before_save > 20)
- def checkfsrecover(self):
- # an absolutely minimal test of fsrecover
- # Verify that calling recover on a small, correct storage
- # produces a duplicate of the original.
- for i in range(5):
- oid = self._storage.new_oid()
- revid = None
- for j in range(5):
- revid = self._dostore(oid, revid=revid)
-
- temp = sys.stdout
- sys.stdout = StringIO.StringIO()
- try:
- recover(["", "FileStorageTests.fs", "fsrecover.fs"])
- finally:
- sys.stdout = temp
- self.assert_(filecmp.cmp("FileStorageTests.fs", "fsrecover.fs"))
- StorageTestBase.removefs("fsrecover.fs")
-
# There are a bunch of tests that the current pack() implementation
# does not past. We need to fix pack(), but don't want tests to
# fail until then.
=== ZODB3/ZODB/tests/testZODB.py 1.14 => 1.14.2.1 ===
--- ZODB3/ZODB/tests/testZODB.py:1.14 Wed Apr 23 16:33:48 2003
+++ ZODB3/ZODB/tests/testZODB.py Thu May 29 12:30:52 2003
@@ -18,7 +18,7 @@
import ZODB
import ZODB.FileStorage
from ZODB.PersistentMapping import PersistentMapping
-from ZODB.POSException import ReadConflictError
+from ZODB.POSException import ReadConflictError, ConflictError
from ZODB.tests.StorageTestBase import removefs
from Persistence import Persistent
@@ -229,17 +229,17 @@
obj.child1
cn2.getTransaction().abort()
- def testReadConflictIgnored(self):
+ def checkReadConflictIgnored(self):
# Test that an application that catches a read conflict and
# continues can not commit the transaction later.
root = self._db.open().root()
- root["real_data"] = real_data = PersistentDict()
- root["index"] = index = PersistentDict()
+ root["real_data"] = real_data = PersistentMapping()
+ root["index"] = index = PersistentMapping()
- real_data["a"] = PersistentDict({"indexed_value": 0})
- real_data["b"] = PersistentDict({"indexed_value": 1})
- index[1] = PersistentDict({"b": 1})
- index[0] = PersistentDict({"a": 1})
+ real_data["a"] = PersistentMapping({"indexed_value": 0})
+ real_data["b"] = PersistentMapping({"indexed_value": 1})
+ index[1] = PersistentMapping({"b": 1})
+ index[0] = PersistentMapping({"a": 1})
get_transaction().commit()
# load some objects from one connection
@@ -252,7 +252,7 @@
real_data["b"]["indexed_value"] = 0
del index[1]["b"]
index[0]["b"] = 1
- cn2.getTransaction().commit()
+ get_transaction().commit()
del real_data2["a"]
try:
@@ -271,7 +271,7 @@
self.assert_(not index2[0]._p_changed)
self.assert_(not index2[1]._p_changed)
- self.assertRaises(ConflictError, get_transaction().commit)
+ self.assertRaises(ConflictError, cn2.getTransaction().commit)
get_transaction().abort()
def checkIndependent(self):
=== Removed File ZODB3/ZODB/tests/testStorageConfig.py ===
More information about the Zodb-checkins
mailing list