[Zope-Checkins] CVS: Packages/ZODB/tests -
testFileStorage.py:1.32.8.6
Tim Peters
tim.one at comcast.net
Fri Feb 25 15:31:04 EST 2005
Update of /cvs-repository/Packages/ZODB/tests
In directory cvs.zope.org:/tmp/cvs-serv24962/ZODB/tests
Modified Files:
Tag: Zope-2_7-branch
testFileStorage.py
Log Message:
Stop believing the maximum oid cached in a FileStorage's .index file.
This is a critical bugfix, although the problems it addresses are
(a) rare; and, (b) not entirely fixed yet (more checkins to come).
The true max oid is found efficiently now by exploiting the recently-added
fsIndex.maxKey() method (which was, of course, added for this purpose).
Also fix that the .index file could get updated on disk when the
FileStorage was opened in read-only mode. The code was trying to prevent
this, but missed the most obvious rewrite path.
Incidentally improved many obsolete and/or incorrect comments.
=== Packages/ZODB/tests/testFileStorage.py 1.32.8.5 => 1.32.8.6 ===
--- Packages/ZODB/tests/testFileStorage.py:1.32.8.5 Tue Nov 16 16:39:04 2004
+++ Packages/ZODB/tests/testFileStorage.py Fri Feb 25 15:31:04 2005
@@ -79,66 +79,65 @@
self.assertEqual(self._storage._index.__class__, fsIndex)
- # XXX We could really use some tests for sanity checking
-
- def check_conversion_to_fsIndex_not_if_readonly(self):
-
- self.tearDown()
-
- class OldFileStorage(ZODB.FileStorage.FileStorage):
- def _newIndexes(self):
- return {}, {}, {}, {}, {}, {}, {}
-
+ # A helper for checking that when an .index contains a dict for the
+ # index, it's converted to an fsIndex when the file is opened.
+ def convert_index_to_dict(self):
+ # Convert the index in the current .index file to a Python dict.
+ # Return the index originally found.
+ import cPickle as pickle
+
+ f = open('FileStorageTests.fs.index', 'r+b')
+ p = pickle.Unpickler(f)
+ data = p.load()
+ index = data['index']
+
+ newindex = dict(index)
+ data['index'] = newindex
+
+ f.seek(0)
+ f.truncate()
+ p = pickle.Pickler(f, 1)
+ p.dump(data)
+ f.close()
+ return index
+ def check_conversion_to_fsIndex(self, read_only=False):
from ZODB.fsIndex import fsIndex
- # Hack FileStorage to create dictionary indexes
- self._storage = OldFileStorage('FileStorageTests.fs')
-
- self.assertEqual(type(self._storage._index), type({}))
+ # Create some data, and remember the index.
for i in range(10):
self._dostore()
+ oldindex_as_dict = dict(self._storage._index)
- # Should save the index
+ # Save the index.
self._storage.close()
- self._storage = ZODB.FileStorage.FileStorage(
- 'FileStorageTests.fs', read_only=1)
- self.assertEqual(type(self._storage._index), type({}))
-
- def check_conversion_to_fsIndex(self):
-
- self.tearDown()
-
- class OldFileStorage(ZODB.FileStorage.FileStorage):
- def _newIndexes(self):
- return {}, {}, {}, {}, {}, {}, {}
-
-
- from ZODB.fsIndex import fsIndex
+ # Convert it to a dict.
+ old_index = self.convert_index_to_dict()
+ self.assert_(isinstance(old_index, fsIndex))
+ new_index = self.convert_index_to_dict()
+ self.assert_(isinstance(new_index, dict))
+
+ # Verify it's converted to fsIndex in memory upon open.
+ self.open(read_only=read_only)
+ self.assert_(isinstance(self._storage._index, fsIndex))
+
+ # Verify it has the right content.
+ newindex_as_dict = dict(self._storage._index)
+ self.assertEqual(oldindex_as_dict, newindex_as_dict)
- # Hack FileStorage to create dictionary indexes
- self._storage = OldFileStorage('FileStorageTests.fs')
-
- self.assertEqual(type(self._storage._index), type({}))
- for i in range(10):
- self._dostore()
-
- oldindex = self._storage._index.copy()
-
- # Should save the index
+ # Check that the type on disk has changed iff read_only is False.
self._storage.close()
-
- self._storage = ZODB.FileStorage.FileStorage('FileStorageTests.fs')
- self.assertEqual(self._storage._index.__class__, fsIndex)
- self.failUnless(self._storage._used_index)
-
- index = {}
- for k, v in self._storage._index.items():
- index[k] = v
-
- self.assertEqual(index, oldindex)
-
+ current_index = self.convert_index_to_dict()
+ if read_only:
+ self.assert_(isinstance(current_index, dict))
+ else:
+ self.assert_(isinstance(current_index, fsIndex))
+
+ def check_conversion_to_fsIndex_readonly(self):
+ # Same thing, but the disk .index should continue to hold a
+ # Python dict.
+ self.check_conversion_to_fsIndex(read_only=True)
def check_save_after_load_with_no_index(self):
for i in range(10):
@@ -179,6 +178,45 @@
def checkPackAfterUndoDeletion(self):
pass
+ def check_index_oid_ignored(self):
+ # Prior to ZODB 3.2.6, the 'oid' value stored in the .index file
+ # was believed. But there were cases where adding larger oids
+ # didn't update the FileStorage ._oid attribute -- the restore()
+ # method in particular didn't update it, and that's about the only
+ # method copyTransactionsFrom() uses. A database copy created that
+ # way then stored an 'oid' of z64 in the .index file. This created
+ # torturous problems, as when that file was opened, "new" oids got
+ # generated starting over from 0 again.
+ # Now the cached 'oid' value is ignored: verify that this is so.
+ import cPickle as pickle
+ from ZODB.utils import z64
+ from ZODB.DB import DB
+
+ # Create some data.
+ db = DB(self._storage)
+ conn = db.open()
+ conn.root()['xyz'] = 1
+ get_transaction().commit()
+ true_max_oid = self._storage._oid
+
+ # Save away the index, and poke in a bad 'oid' value by hand.
+ db.close()
+ f = open('FileStorageTests.fs.index', 'r+b')
+ p = pickle.Unpickler(f)
+ data = p.load()
+ saved_oid = data['oid']
+ self.assertEqual(true_max_oid, saved_oid)
+ data['oid'] = z64
+ f.seek(0)
+ f.truncate()
+ p = pickle.Pickler(f, 1)
+ p.dump(data)
+ f.close()
+
+ # Verify that we get the correct oid again when we reopen, despite
+ # that we stored nonsense in the .index file's 'oid'.
+ self.open()
+ self.assertEqual(self._storage._oid, true_max_oid)
def checkTimeTravelOnOpen(self):
from ZODB.DB import DB
More information about the Zope-Checkins
mailing list