[Zodb-checkins] SVN: ZODB/branches/zcZODB-3.8/ Merged revs 91211-91439 from 3.8 branch.
Jim Fulton
jim at zope.com
Fri Sep 26 10:47:37 EDT 2008
Log message for revision 91522:
Merged revs 91211-91439 from 3.8 branch.
Changed:
U ZODB/branches/zcZODB-3.8/NEWS.txt
U ZODB/branches/zcZODB-3.8/src/ZEO/ClientStorage.py
U ZODB/branches/zcZODB-3.8/src/ZEO/tests/ConnectionTests.py
U ZODB/branches/zcZODB-3.8/src/ZEO/tests/invalidations_while_connecting.test
U ZODB/branches/zcZODB-3.8/src/ZEO/tests/testZEO.py
U ZODB/branches/zcZODB-3.8/src/ZEO/tests/test_cache.py
U ZODB/branches/zcZODB-3.8/src/ZODB/FileStorage/FileStorage.py
U ZODB/branches/zcZODB-3.8/src/ZODB/blob.py
U ZODB/branches/zcZODB-3.8/src/ZODB/scripts/migrateblobs.py
U ZODB/branches/zcZODB-3.8/src/ZODB/tests/blob_layout.txt
U ZODB/branches/zcZODB-3.8/src/ZODB/tests/blob_packing.txt
U ZODB/branches/zcZODB-3.8/src/ZODB/tests/blob_transaction.txt
U ZODB/branches/zcZODB-3.8/src/ZODB/tests/testFileStorage.py
U ZODB/branches/zcZODB-3.8/src/ZODB/tests/testblob.py
-=-
Modified: ZODB/branches/zcZODB-3.8/NEWS.txt
===================================================================
--- ZODB/branches/zcZODB-3.8/NEWS.txt 2008-09-26 13:43:57 UTC (rev 91521)
+++ ZODB/branches/zcZODB-3.8/NEWS.txt 2008-09-26 14:47:36 UTC (rev 91522)
@@ -4,6 +4,26 @@
Bugs Fixed:
+- (beta 9) Fixed a bug to allow opening of deep-copied blobs.
+
+- (beta 9) Fixed bug #189542 by prepending the module to an undefined name.
+
+- (beta 8) If there is a failure while FileStorage is finalizing a transaction,
+ the file storage is closed because it's internal meta data may be
+ invalid.
+
+- (beta 8) FileStorages previously saved indexes after a certain
+ number of writes. This was done during the last phase of two-phase
+ commit, which made this critical phase more subject to errors than
+ it should have been. Also, for large databases, saves were done so
+ infrequently as to be useless. The feature was removed to reduce
+ the chance for errors during the last phase of two-phase commit.
+
+- (beta 8) File storages previously kept an internal object id to
+ transaction id mapping as an optimization. This mapping caused
+ excessive memory usage and failures during the last phase of
+ two-phase commit. This optimization has been removed.
+
- (beta 8) Fixed a bug that caused deep copying of blobs to fail.
- (beta 8) Refactored handling of invalidations on ZEO clients to fix
Modified: ZODB/branches/zcZODB-3.8/src/ZEO/ClientStorage.py
===================================================================
--- ZODB/branches/zcZODB-3.8/src/ZEO/ClientStorage.py 2008-09-26 13:43:57 UTC (rev 91521)
+++ ZODB/branches/zcZODB-3.8/src/ZEO/ClientStorage.py 2008-09-26 14:47:36 UTC (rev 91522)
@@ -985,7 +985,7 @@
if self._have_blob(blob_filename, oid, serial):
return blob_filename
- raise POSKeyError("No blob file", oid, serial)
+ raise POSException.POSKeyError("No blob file", oid, serial)
finally:
lock.close()
@@ -1084,19 +1084,27 @@
# tpc_cond condition variable prevents more than one
# thread from calling tpc_finish() at a time.
tid = self._server.tpc_finish(id(txn))
- self._lock.acquire() # for atomic processing of invalidations
+
try:
- self._update_cache(tid)
- if f is not None:
- f(tid)
- finally:
- self._lock.release()
+ self._lock.acquire() # for atomic processing of invalidations
+ try:
+ self._update_cache(tid)
+ if f is not None:
+ f(tid)
+ finally:
+ self._lock.release()
- r = self._check_serials()
- assert r is None or len(r) == 0, "unhandled serialnos: %s" % r
+ r = self._check_serials()
+ assert r is None or len(r) == 0, "unhandled serialnos: %s" % r
+ except:
+ # The server successfully committed. If we get a failure
+ # here, our own state will be in question, so reconnect.
+ self._connection.close()
+ raise
+
+ self.end_transaction()
finally:
self._load_lock.release()
- self.end_transaction()
def _update_cache(self, tid):
"""Internal helper to handle objects modified by a transaction.
Modified: ZODB/branches/zcZODB-3.8/src/ZEO/tests/ConnectionTests.py
===================================================================
--- ZODB/branches/zcZODB-3.8/src/ZEO/tests/ConnectionTests.py 2008-09-26 13:43:57 UTC (rev 91521)
+++ ZODB/branches/zcZODB-3.8/src/ZEO/tests/ConnectionTests.py 2008-09-26 14:47:36 UTC (rev 91522)
@@ -1066,6 +1066,8 @@
self.assert_(storage.is_connected())
# We expect finish to fail.
self.assertRaises(ClientDisconnected, storage.tpc_finish, t)
+ storage.tpc_abort(t)
+
# Now we think we've committed the second transaction, but we really
# haven't. A third one should produce a POSKeyError on the server,
# which manifests as a ConflictError on the client.
Modified: ZODB/branches/zcZODB-3.8/src/ZEO/tests/invalidations_while_connecting.test
===================================================================
--- ZODB/branches/zcZODB-3.8/src/ZEO/tests/invalidations_while_connecting.test 2008-09-26 13:43:57 UTC (rev 91521)
+++ ZODB/branches/zcZODB-3.8/src/ZEO/tests/invalidations_while_connecting.test 2008-09-26 14:47:36 UTC (rev 91522)
@@ -38,7 +38,7 @@
- starting a second client that writes objects more or less
constantly,
- >>> import random, threading
+ >>> import random, threading, time
>>> stop = False
>>> db2 = ZODB.DB(ZEO.ClientStorage.ClientStorage(addr))
>>> tm = transaction.TransactionManager()
@@ -67,7 +67,6 @@
>>> handler = zope.testing.loggingsupport.InstalledHandler(
... 'ZEO', level=logging.ERROR)
- >>> import time
>>> for c in range(10):
... time.sleep(.1)
... db = ZODB.DB(ZEO.ClientStorage.ClientStorage(addr, client='x'))
Modified: ZODB/branches/zcZODB-3.8/src/ZEO/tests/testZEO.py
===================================================================
--- ZODB/branches/zcZODB-3.8/src/ZEO/tests/testZEO.py 2008-09-26 13:43:57 UTC (rev 91521)
+++ ZODB/branches/zcZODB-3.8/src/ZEO/tests/testZEO.py 2008-09-26 14:47:36 UTC (rev 91522)
@@ -856,7 +856,101 @@
"""
+def tpc_finish_error():
+ """Server errors in tpc_finish weren't handled properly.
+ >>> import ZEO.ClientStorage
+
+ >>> class Connection:
+ ... def __init__(self, client):
+ ... self.client = client
+ ... def get_addr(self):
+ ... return 'server'
+ ... def is_async(self):
+ ... return True
+ ... def register_object(self, ob):
+ ... pass
+ ... def close(self):
+ ... print 'connection closed'
+
+ >>> class ConnectionManager:
+ ... def __init__(self, addr, client, tmin, tmax):
+ ... self.client = client
+ ... def connect(self, sync=1):
+ ... self.client.notifyConnected(Connection(self.client))
+
+ >>> class StorageServer:
+ ... should_fail = True
+ ... def __init__(self, conn):
+ ... self.conn = conn
+ ... self.t = None
+ ... def get_info(self):
+ ... return {}
+ ... def endZeoVerify(self):
+ ... self.conn.client.endVerify()
+ ... def tpc_begin(self, t, *args):
+ ... if self.t is not None:
+ ... raise TypeError('already trans')
+ ... self.t = t
+ ... print 'begin', args
+ ... def vote(self, t):
+ ... if self.t != t:
+ ... raise TypeError('bad trans')
+ ... print 'vote'
+ ... def tpc_finish(self, *args):
+ ... if self.should_fail:
+ ... raise TypeError()
+ ... print 'finish'
+ ... def tpc_abort(self, t):
+ ... if self.t != t:
+ ... raise TypeError('bad trans')
+ ... self.t = None
+ ... print 'abort'
+
+ >>> class ClientStorage(ZEO.ClientStorage.ClientStorage):
+ ... ConnectionManagerClass = ConnectionManager
+ ... StorageServerStubClass = StorageServer
+
+ >>> class Transaction:
+ ... user = 'test'
+ ... description = ''
+ ... _extension = {}
+
+ >>> cs = ClientStorage(('', ''))
+ >>> t1 = Transaction()
+ >>> cs.tpc_begin(t1)
+ begin ('test', '', {}, None, ' ')
+
+ >>> cs.tpc_vote(t1)
+ vote
+
+ >>> cs.tpc_finish(t1)
+ Traceback (most recent call last):
+ ...
+ TypeError
+
+ >>> cs.tpc_abort(t1)
+ abort
+
+ >>> t2 = Transaction()
+ >>> cs.tpc_begin(t2)
+ begin ('test', '', {}, None, ' ')
+ >>> cs.tpc_vote(t2)
+ vote
+
+ If client storage has an internal error after the storage finish
+ succeeeds, it will close the connection, which will force a
+ restart and reverification.
+
+ >>> StorageServer.should_fail = False
+ >>> cs._update_cache = lambda : None
+ >>> try: cs.tpc_finish(t2)
+ ... except: pass
+ ... else: print "Should have failed"
+ finish
+ connection closed
+ """
+
test_classes = [FileStorageTests, MappingStorageTests, DemoStorageTests,
BlobAdaptedFileStorageTests, BlobWritableCacheTests]
Modified: ZODB/branches/zcZODB-3.8/src/ZEO/tests/test_cache.py
===================================================================
--- ZODB/branches/zcZODB-3.8/src/ZEO/tests/test_cache.py 2008-09-26 13:43:57 UTC (rev 91521)
+++ ZODB/branches/zcZODB-3.8/src/ZEO/tests/test_cache.py 2008-09-26 14:47:36 UTC (rev 91522)
@@ -331,6 +331,8 @@
>>> logger.setLevel(logging.NOTSET)
>>> logger.removeHandler(handler)
+
+ >>> cache.close()
"""
)
Modified: ZODB/branches/zcZODB-3.8/src/ZODB/FileStorage/FileStorage.py
===================================================================
--- ZODB/branches/zcZODB-3.8/src/ZODB/FileStorage/FileStorage.py 2008-09-26 13:43:57 UTC (rev 91521)
+++ ZODB/branches/zcZODB-3.8/src/ZODB/FileStorage/FileStorage.py 2008-09-26 14:47:36 UTC (rev 91522)
@@ -43,8 +43,6 @@
from ZODB.loglevels import BLATHER
from ZODB.fsIndex import fsIndex
-import BTrees.OOBTree
-
packed_version = "FS21"
logger = logging.getLogger('ZODB.FileStorage')
@@ -94,8 +92,6 @@
# Set True while a pack is in progress; undo is blocked for the duration.
_pack_is_in_progress = False
- _records_before_save = 10000
-
def __init__(self, file_name, create=False, read_only=False, stop=None,
quota=None):
@@ -122,10 +118,8 @@
BaseStorage.BaseStorage.__init__(self, file_name)
- (index, vindex, tindex, tvindex,
- oid2tid, toid2tid, toid2tid_delete) = self._newIndexes()
- self._initIndex(index, vindex, tindex, tvindex,
- oid2tid, toid2tid, toid2tid_delete)
+ index, vindex, tindex, tvindex = self._newIndexes()
+ self._initIndex(index, vindex, tindex, tvindex)
# Now open the file
@@ -159,8 +153,7 @@
self._used_index = 1 # Marker for testing
index, vindex, start, ltid = r
- self._initIndex(index, vindex, tindex, tvindex,
- oid2tid, toid2tid, toid2tid_delete)
+ self._initIndex(index, vindex, tindex, tvindex)
self._pos, self._oid, tid = read_index(
self._file, file_name, index, vindex, tindex, stop,
ltid=ltid, start=start, read_only=read_only,
@@ -173,8 +166,6 @@
)
self._save_index()
- self._records_before_save = max(self._records_before_save,
- len(self._index))
self._ltid = tid
# self._pos should always point just past the last
@@ -194,11 +185,7 @@
self._quota = quota
- # tid cache statistics.
- self._oid2tid_nlookups = self._oid2tid_nhits = 0
-
- def _initIndex(self, index, vindex, tindex, tvindex,
- oid2tid, toid2tid, toid2tid_delete):
+ def _initIndex(self, index, vindex, tindex, tvindex):
self._index=index
self._vindex=vindex
self._tindex=tindex
@@ -206,32 +193,12 @@
self._index_get=index.get
self._vindex_get=vindex.get
- # .store() needs to compare the passed-in serial to the
- # current tid in the database. _oid2tid caches the oid ->
- # current tid mapping for non-version data (if the current
- # record for oid is version data, the oid is not a key in
- # _oid2tid). The point is that otherwise seeking into the
- # storage is needed to extract the current tid, and that's
- # an expensive operation. For example, if a transaction
- # stores 4000 objects, and each random seek + read takes 7ms
- # (that was approximately true on Linux and Windows tests in
- # mid-2003), that's 28 seconds just to find the old tids.
- # TODO: Probably better to junk this and redefine _index as mapping
- # oid to (offset, tid) pair, via a new memory-efficient BTree type.
- self._oid2tid = oid2tid
- # oid->tid map to transactionally add to _oid2tid.
- self._toid2tid = toid2tid
- # Set of oids to transactionally delete from _oid2tid (e.g.,
- # oids reverted by undo, or for which the most recent record
- # becomes version data).
- self._toid2tid_delete = toid2tid_delete
-
def __len__(self):
return len(self._index)
def _newIndexes(self):
# hook to use something other than builtin dict
- return fsIndex(), {}, {}, {}, BTrees.OOBTree.OOBTree(), {}, {}
+ return fsIndex(), {}, {}, {}
_saved = 0
def _save_index(self):
@@ -409,27 +376,6 @@
# Log the error and continue
logger.error("Error saving index on close()", exc_info=True)
- # Return tid of most recent record for oid if that's in the
- # _oid2tid cache. Else return None. It's important to use this
- # instead of indexing _oid2tid directly so that cache statistics
- # can be logged.
- def _get_cached_tid(self, oid):
- self._oid2tid_nlookups += 1
- result = self._oid2tid.get(oid)
- if result is not None:
- self._oid2tid_nhits += 1
-
- # Log a msg every ~8000 tries.
- if self._oid2tid_nlookups & 0x1fff == 0:
- logger.log(BLATHER,
- "_oid2tid size %s lookups %s hits %s rate %.1f%%",
- len(self._oid2tid),
- self._oid2tid_nlookups,
- self._oid2tid_nhits,
- 100.0 * self._oid2tid_nhits / self._oid2tid_nlookups)
-
- return result
-
def abortVersion(self, src, transaction):
return self.commitVersion(src, '', transaction, abort=True)
@@ -504,7 +450,6 @@
srcpos = h.vprev
spos = p64(srcpos)
- self._toid2tid_delete.update(current_oids)
return self._tid, oids
def getSize(self):
@@ -616,24 +561,23 @@
if oid > self._oid:
self.set_max_oid(oid)
old = self._index_get(oid, 0)
- cached_tid = None
+ committed_tid = None
pnv = None
if old:
- cached_tid = self._get_cached_tid(oid)
- if cached_tid is None:
- h = self._read_data_header(old, oid)
- if h.version:
- if h.version != version:
- raise VersionLockError(oid, h.version)
- pnv = h.pnv
- cached_tid = h.tid
+ h = self._read_data_header(old, oid)
+ if h.version:
+ if h.version != version:
+ raise VersionLockError(oid, h.version)
+ pnv = h.pnv
+ committed_tid = h.tid
- if oldserial != cached_tid:
- rdata = self.tryToResolveConflict(oid, cached_tid,
+ if oldserial != committed_tid:
+ rdata = self.tryToResolveConflict(oid, committed_tid,
oldserial, data)
if rdata is None:
raise POSException.ConflictError(
- oid=oid, serials=(cached_tid, oldserial), data=data)
+ oid=oid, serials=(committed_tid, oldserial),
+ data=data)
else:
data = rdata
@@ -651,9 +595,6 @@
pnv = old
new.setVersion(version, pnv, pv)
self._tvindex[version] = here
- self._toid2tid_delete[oid] = 1
- else:
- self._toid2tid[oid] = self._tid
self._tfile.write(new.asString())
self._tfile.write(data)
@@ -663,7 +604,7 @@
raise FileStorageQuotaError(
"The storage quota has been exceeded.")
- if old and oldserial != cached_tid:
+ if old and oldserial != committed_tid:
return ConflictResolution.ResolvedSerial
else:
return self._tid
@@ -771,9 +712,6 @@
vprev = self._vindex.get(version, 0)
new.setVersion(version, pnv, vprev)
self._tvindex[version] = here
- self._toid2tid_delete[oid] = 1
- else:
- self._toid2tid[oid] = serial
self._tfile.write(new.asString())
@@ -822,8 +760,6 @@
def _clear_temp(self):
self._tindex.clear()
self._tvindex.clear()
- self._toid2tid.clear()
- self._toid2tid_delete.clear()
if self._tfile is not None:
self._tfile.seek(0)
@@ -875,41 +811,35 @@
finally:
self._lock_release()
- # Keep track of the number of records that we've written
- _records_written = 0
-
def _finish(self, tid, u, d, e):
- nextpos=self._nextpos
- if nextpos:
- file=self._file
-
+ # If self._nextpos is 0, then the transaction didn't write any
+ # data, so we don't bother writing anything to the file.
+ if self._nextpos:
# Clear the checkpoint flag
- file.seek(self._pos+16)
- file.write(self._tstatus)
- file.flush()
+ self._file.seek(self._pos+16)
+ self._file.write(self._tstatus)
+ try:
+ # At this point, we may have committed the data to disk.
+ # If we fail from here, we're in bad shape.
+ self._finish_finish(tid)
+ except:
+ # Ouch. This is bad. Let's try to get back to where we were
+ # and then roll over and die
+ logger.critical("Failure in _finish. Closing.", exc_info=True)
+ self.close()
+ raise
- if fsync is not None: fsync(file.fileno())
+ def _finish_finish(self, tid):
+ # This is a separate method to allow tests to replace it with
+ # something broken. :)
+
+ self._file.flush()
+ if fsync is not None:
+ fsync(self._file.fileno())
- self._pos = nextpos
-
- self._index.update(self._tindex)
- self._vindex.update(self._tvindex)
- self._oid2tid.update(self._toid2tid)
- for oid in self._toid2tid_delete.keys():
- try:
- del self._oid2tid[oid]
- except KeyError:
- pass
-
- # Update the number of records that we've written
- # +1 for the transaction record
- self._records_written += len(self._tindex) + 1
- if self._records_written >= self._records_before_save:
- self._save_index()
- self._records_written = 0
- self._records_before_save = max(self._records_before_save,
- len(self._index))
-
+ self._pos = self._nextpos
+ self._index.update(self._tindex)
+ self._vindex.update(self._tvindex)
self._ltid = tid
def _abort(self):
@@ -945,17 +875,12 @@
def getTid(self, oid):
self._lock_acquire()
try:
- result = self._get_cached_tid(oid)
- if result is None:
- pos = self._lookup_pos(oid)
- h = self._read_data_header(pos, oid)
- if h.plen == 0 and h.back == 0:
- # Undone creation
- raise POSKeyError(oid)
- else:
- result = h.tid
- self._oid2tid[oid] = result
- return result
+ pos = self._lookup_pos(oid)
+ h = self._read_data_header(pos, oid)
+ if h.plen == 0 and h.back == 0:
+ # Undone creation
+ raise POSKeyError(oid)
+ return h.tid
finally:
self._lock_release()
@@ -1103,10 +1028,6 @@
tpos = self._txn_find(tid, 1)
tindex = self._txn_undo_write(tpos)
self._tindex.update(tindex)
- # Arrange to clear the affected oids from the oid2tid cache.
- # It's too painful to try to update them to correct current
- # values instead.
- self._toid2tid_delete.update(tindex)
return self._tid, tindex.keys()
def _txn_find(self, tid, stop_at_pack):
@@ -1343,9 +1264,7 @@
# OK, we're beyond the point of no return
os.rename(self._file_name + '.pack', self._file_name)
self._file = open(self._file_name, 'r+b')
- self._initIndex(p.index, p.vindex, p.tindex, p.tvindex,
- p.oid2tid, p.toid2tid,
- p.toid2tid_delete)
+ self._initIndex(p.index, p.vindex, p.tindex, p.tvindex)
self._pos = opos
self._save_index()
finally:
Modified: ZODB/branches/zcZODB-3.8/src/ZODB/blob.py
===================================================================
--- ZODB/branches/zcZODB-3.8/src/ZODB/blob.py 2008-09-26 13:43:57 UTC (rev 91521)
+++ ZODB/branches/zcZODB-3.8/src/ZODB/blob.py 2008-09-26 14:47:36 UTC (rev 91522)
@@ -125,6 +125,9 @@
if self.writers:
raise BlobError("Already opened for writing.")
+ if self.readers is None:
+ self.readers = []
+
if mode == 'r':
if self._current_filename() is None:
self._create_uncommitted_file()
@@ -298,7 +301,7 @@
# want to perform blob storage differently.
def __init__(self, base_dir, layout_name='automatic'):
- self.base_dir = os.path.normpath(base_dir) + '/'
+ self.base_dir = os.path.normpath(base_dir) + os.path.sep
self.temp_dir = os.path.join(base_dir, 'tmp')
if layout_name == 'automatic':
@@ -325,9 +328,8 @@
os.path.join(self.base_dir, LAYOUT_MARKER), 'wb')
layout_marker.write(self.layout_name)
else:
- layout_marker = open(
- os.path.join(self.base_dir, LAYOUT_MARKER), 'rb')
- layout = layout_marker.read().strip()
+ layout = open(os.path.join(self.base_dir, LAYOUT_MARKER), 'rb'
+ ).read().strip()
if layout != self.layout_name:
raise ValueError(
"Directory layout `%s` selected for blob directory %s, but "
@@ -488,8 +490,8 @@
"""
- blob_path_pattern = r'^' + (r'0x[0-9a-f]{1,2}/*'*8) + r'$'
- blob_path_pattern = re.compile(blob_path_pattern)
+ blob_path_pattern = re.compile(
+ r'(0x[0-9a-f]{1,2}\%s){7,7}0x[0-9a-f]{1,2}$' % os.path.sep)
def oid_to_path(self, oid):
directories = []
@@ -497,12 +499,12 @@
# first
for byte in str(oid):
directories.append('0x%s' % binascii.hexlify(byte))
- return '/'.join(directories)
+ return os.path.sep.join(directories)
def path_to_oid(self, path):
if self.blob_path_pattern.match(path) is None:
raise ValueError("Not a valid OID path: `%s`" % path)
- path = path.split('/')
+ path = path.split(os.path.sep)
# Each path segment stores a byte in hex representation. Turn it into
# an int and then get the character for our byte string.
oid = ''.join(binascii.unhexlify(byte[2:]) for byte in path)
Modified: ZODB/branches/zcZODB-3.8/src/ZODB/scripts/migrateblobs.py
===================================================================
--- ZODB/branches/zcZODB-3.8/src/ZODB/scripts/migrateblobs.py 2008-09-26 13:43:57 UTC (rev 91521)
+++ ZODB/branches/zcZODB-3.8/src/ZODB/scripts/migrateblobs.py 2008-09-26 14:47:36 UTC (rev 91522)
@@ -17,6 +17,7 @@
import logging
import optparse
import os
+import shutil
from ZODB.blob import FilesystemHelper, rename_or_copy_blob
from ZODB.utils import cp, oid_repr
@@ -28,6 +29,12 @@
except OSError:
shutil.copy(f1, f2)
+# Check if we actually have link
+try:
+ os.link
+except AttributeError:
+ link_or_copy = shutil.copy
+
def migrate(source, dest, layout):
source_fsh = FilesystemHelper(source)
Modified: ZODB/branches/zcZODB-3.8/src/ZODB/tests/blob_layout.txt
===================================================================
--- ZODB/branches/zcZODB-3.8/src/ZODB/tests/blob_layout.txt 2008-09-26 13:43:57 UTC (rev 91521)
+++ ZODB/branches/zcZODB-3.8/src/ZODB/tests/blob_layout.txt 2008-09-26 14:47:36 UTC (rev 91522)
@@ -29,9 +29,12 @@
>>> bushy.oid_to_path('\x00\x00\x00\x00\x00\x00\x00\x01')
'0x00/0x00/0x00/0x00/0x00/0x00/0x00/0x01'
->>> bushy.path_to_oid('0x01/0x00/0x00/0x00/0x00/0x00/0x00/0x00')
+>>> import os
+>>> bushy.path_to_oid(os.path.join(
+... '0x01', '0x00', '0x00', '0x00', '0x00', '0x00', '0x00', '0x00'))
'\x01\x00\x00\x00\x00\x00\x00\x00'
->>> bushy.path_to_oid('0xff/0x00/0x00/0x00/0x00/0x00/0x00/0x00')
+>>> bushy.path_to_oid(os.path.join(
+... '0xff', '0x00', '0x00', '0x00', '0x00', '0x00', '0x00', '0x00'))
'\xff\x00\x00\x00\x00\x00\x00\x00'
Paths that do not represent an OID will cause a ValueError:
@@ -142,8 +145,8 @@
'lawn'
>>> fsh.create() # doctest: +ELLIPSIS
Traceback (most recent call last):
-ValueError: Directory layout `lawn` selected for blob directory /.../blobs/, but marker found for layout `bushy`
->>> shutil.rmtree(blobs)
+ValueError: Directory layout `lawn` selected for blob directory .../blobs/, but marker found for layout `bushy`
+>>> rmtree(blobs)
This function interacts with the automatic detection in the way, that an
unmarked directory will be marked the first time when it is auto-guessed and
@@ -163,10 +166,11 @@
'lawn'
>>> blob_storage = BlobStorage(blobs, base_storage, layout='bushy') # doctest: +ELLIPSIS
Traceback (most recent call last):
-ValueError: Directory layout `bushy` selected for blob directory /.../blobs/, but marker found for layout `lawn`
+ValueError: Directory layout `bushy` selected for blob directory .../blobs/, but marker found for layout `lawn`
->>> shutil.rmtree(d)
+>>> base_storage.close()
+>>> rmtree(d)
Migrating between directory layouts
@@ -206,7 +210,7 @@
>>> bushy = os.path.join(d, 'bushy')
>>> migrate(old, bushy, 'bushy') # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE
-Migrating blob data from `/.../old` (lawn) to `/.../bushy` (bushy)
+Migrating blob data from `.../old` (lawn) to `.../bushy` (bushy)
OID: 0x0a - 2 files
OID: 0x1b7a - 2 files
OID: 0x1b7f - 2 files
@@ -248,7 +252,7 @@
>>> lawn = os.path.join(d, 'lawn')
>>> migrate(bushy, lawn, 'lawn')
-Migrating blob data from `/.../bushy` (bushy) to `/.../lawn` (lawn)
+Migrating blob data from `.../bushy` (bushy) to `.../lawn` (lawn)
OID: 0x0a - 2 files
OID: 0x1b7a - 2 files
OID: 0x1b7f - 2 files
@@ -278,4 +282,4 @@
bushy/0x00/0x00/0x00/0x00/0x00/0x00/0x1b/0x7a/foo5 --> lawn/0x1b7a/foo5
bushy/0x00/0x00/0x00/0x00/0x00/0x00/0x1b/0x7a/foo6 --> lawn/0x1b7a/foo6
->>> shutil.rmtree(d)
+>>> rmtree(d)
Modified: ZODB/branches/zcZODB-3.8/src/ZODB/tests/blob_packing.txt
===================================================================
--- ZODB/branches/zcZODB-3.8/src/ZODB/tests/blob_packing.txt 2008-09-26 13:43:57 UTC (rev 91521)
+++ ZODB/branches/zcZODB-3.8/src/ZODB/tests/blob_packing.txt 2008-09-26 14:47:36 UTC (rev 91522)
@@ -146,7 +146,7 @@
Clean up our blob directory and database:
- >>> shutil.rmtree(blob_dir)
+ >>> rmtree(blob_dir)
>>> base_storage.close()
>>> os.unlink(storagefile)
>>> os.unlink(storagefile+".index")
@@ -273,4 +273,4 @@
Clean up our blob directory:
- >>> shutil.rmtree(blob_dir)
+ >>> rmtree(blob_dir)
Modified: ZODB/branches/zcZODB-3.8/src/ZODB/tests/blob_transaction.txt
===================================================================
--- ZODB/branches/zcZODB-3.8/src/ZODB/tests/blob_transaction.txt 2008-09-26 13:43:57 UTC (rev 91521)
+++ ZODB/branches/zcZODB-3.8/src/ZODB/tests/blob_transaction.txt 2008-09-26 14:47:36 UTC (rev 91522)
@@ -381,6 +381,5 @@
>>> tm1.abort()
>>> tm2.abort()
>>> database.close()
- >>> import shutil
- >>> shutil.rmtree(blob_dir)
- >>> shutil.rmtree(blob_dir2)
+ >>> rmtree(blob_dir)
+ >>> rmtree(blob_dir2)
Modified: ZODB/branches/zcZODB-3.8/src/ZODB/tests/testFileStorage.py
===================================================================
--- ZODB/branches/zcZODB-3.8/src/ZODB/tests/testFileStorage.py 2008-09-26 13:43:57 UTC (rev 91521)
+++ ZODB/branches/zcZODB-3.8/src/ZODB/tests/testFileStorage.py 2008-09-26 14:47:36 UTC (rev 91522)
@@ -221,26 +221,6 @@
self.open()
self.assertEqual(self._storage._oid, true_max_oid)
- # This would make the unit tests too slow
- # check_save_after_load_that_worked_hard(self)
-
- def check_periodic_save_index(self):
-
- # Check the basic algorithm
- oldsaved = self._storage._saved
- self._storage._records_before_save = 10
- for i in range(4):
- self._dostore()
- self.assertEqual(self._storage._saved, oldsaved)
- self._dostore()
- self.assertEqual(self._storage._saved, oldsaved+1)
-
- # Now make sure the parameter changes as we get bigger
- for i in range(20):
- self._dostore()
-
- self.failUnless(self._storage._records_before_save > 20)
-
def checkStoreBumpsOid(self):
# If .store() is handed an oid bigger than the storage knows
# about already, it's crucial that the storage bump its notion
@@ -529,6 +509,60 @@
"""
+def deal_with_finish_failures():
+ r"""
+
+ It's really bad to get errors in FileStorage's _finish method, as
+ that can cause the file storage to be in an inconsistent
+ state. The data file will be fine, but the internal data
+ structures might be hosed. For this reason, FileStorage will close
+ if there is an error after it has finished writing transaction
+ data. It bothers to do very little after writing this data, so
+ this should rarely, if ever, happen.
+
+ >>> fs = ZODB.FileStorage.FileStorage('data.fs')
+ >>> db = DB(fs)
+ >>> conn = db.open()
+ >>> conn.root()[1] = 1
+ >>> transaction.commit()
+
+ Now, we'll indentially break the file storage. It provides a hook
+ for this purpose. :)
+
+ >>> fs._finish_finish = lambda : None
+ >>> conn.root()[1] = 1
+
+ >>> import zope.testing.loggingsupport
+ >>> handler = zope.testing.loggingsupport.InstalledHandler(
+ ... 'ZODB.FileStorage')
+ >>> transaction.commit()
+ Traceback (most recent call last):
+ ...
+ TypeError: <lambda>() takes no arguments (1 given)
+
+
+ >>> print handler
+ ZODB.FileStorage CRITICAL
+ Failure in _finish. Closing.
+
+ >>> handler.uninstall()
+
+ >>> fs.load('\0'*8, '')
+ Traceback (most recent call last):
+ ...
+ ValueError: I/O operation on closed file
+
+ >>> db.close()
+ >>> fs = ZODB.FileStorage.FileStorage('data.fs')
+ >>> db = DB(fs)
+ >>> conn = db.open()
+ >>> conn.root()
+ {1: 1}
+
+ >>> transaction.abort()
+ >>> db.close()
+ """
+
def test_suite():
from zope.testing import doctest
Modified: ZODB/branches/zcZODB-3.8/src/ZODB/tests/testblob.py
===================================================================
--- ZODB/branches/zcZODB-3.8/src/ZODB/tests/testblob.py 2008-09-26 13:43:57 UTC (rev 91521)
+++ ZODB/branches/zcZODB-3.8/src/ZODB/tests/testblob.py 2008-09-26 14:47:36 UTC (rev 91522)
@@ -12,7 +12,7 @@
#
##############################################################################
-import base64, os, re, shutil, tempfile, unittest
+import base64, os, re, shutil, stat, sys, tempfile, unittest
import time
from zope.testing import doctest, renormalizing
import ZODB.tests.util
@@ -140,7 +140,11 @@
clone = u.load()
clone._p_invalidate()
+ # it should also be possible to open the cloned blob
+ # (even though it won't contain the original data)
+ clone.open()
+
class BlobUndoTests(BlobTests):
def testUndoWithoutPreviousVersion(self):
@@ -473,6 +477,11 @@
"""
+# On windows, we can't create secure blob directories, at least not
+# with APIs in the standard library, so there's no point in testing
+# this.
+if sys.platform == 'win32':
+ del secure_blob_directory
def loadblob_tmpstore():
"""
@@ -520,14 +529,28 @@
>>> database.close()
>>> import shutil
- >>> shutil.rmtree(blob_dir)
+ >>> rmtree(blob_dir)
>>> os.unlink(storagefile)
>>> os.unlink(storagefile+".index")
>>> os.unlink(storagefile+".tmp")
"""
+def setUp(test):
+ ZODB.tests.util.setUp(test)
+ def rmtree(path):
+ for path, dirs, files in os.walk(path, False):
+ for fname in files:
+ fname = os.path.join(path, fname)
+ os.chmod(fname, stat.S_IWUSR)
+ os.remove(fname)
+ for dname in dirs:
+ dname = os.path.join(path, dname)
+ os.rmdir(dname)
+ os.rmdir(path)
+ test.globs['rmtree'] = rmtree
+
def test_suite():
suite = unittest.TestSuite()
suite.addTest(unittest.makeSuite(ZODBBlobConfigTest))
@@ -536,23 +559,29 @@
"blob_packing.txt", "blob_importexport.txt", "blob_consume.txt",
"blob_tempdir.txt",
optionflags=doctest.ELLIPSIS,
- setUp=ZODB.tests.util.setUp,
+ setUp=setUp,
tearDown=ZODB.tests.util.tearDown,
))
suite.addTest(doctest.DocFileSuite(
"blob_layout.txt",
optionflags=doctest.ELLIPSIS|doctest.NORMALIZE_WHITESPACE,
- setUp=ZODB.tests.util.setUp,
+ setUp=setUp,
tearDown=ZODB.tests.util.tearDown,
checker = renormalizing.RENormalizing([
- (re.compile(r'[%(sep)s]' % dict(sep=os.path.sep)), '/'),
+ (re.compile(r'\%(sep)s\%(sep)s' % dict(sep=os.path.sep)), '/'),
+ (re.compile(r'\%(sep)s' % dict(sep=os.path.sep)), '/'),
(re.compile(r'\S+/((old|bushy|lawn)/\S+/foo[23456]?)'), r'\1'),
]),
))
suite.addTest(doctest.DocTestSuite(
- setUp=ZODB.tests.util.setUp,
+ setUp=setUp,
tearDown=ZODB.tests.util.tearDown,
+ checker = renormalizing.RENormalizing([
+ (re.compile(r'\%(sep)s\%(sep)s' % dict(sep=os.path.sep)), '/'),
+ (re.compile(r'\%(sep)s' % dict(sep=os.path.sep)), '/'),
+ ]),
))
+ suite.addTest(unittest.makeSuite(BlobCloneTests))
suite.addTest(unittest.makeSuite(BlobUndoTests))
return suite
More information about the Zodb-checkins
mailing list