[Zodb-checkins] SVN: ZODB/branches/jim-transform-wrapping/src/ZODB/ checkpoint
Jim Fulton
jim at zope.com
Fri May 14 16:46:38 EDT 2010
Log message for revision 112318:
checkpoint
Changed:
U ZODB/branches/jim-transform-wrapping/src/ZODB/ConflictResolution.py
U ZODB/branches/jim-transform-wrapping/src/ZODB/FileStorage/FileStorage.py
U ZODB/branches/jim-transform-wrapping/src/ZODB/FileStorage/fspack.py
U ZODB/branches/jim-transform-wrapping/src/ZODB/blob.py
U ZODB/branches/jim-transform-wrapping/src/ZODB/tests/BasicStorage.py
U ZODB/branches/jim-transform-wrapping/src/ZODB/tests/testFileStorage.py
U ZODB/branches/jim-transform-wrapping/src/ZODB/tests/testblob.py
-=-
Modified: ZODB/branches/jim-transform-wrapping/src/ZODB/ConflictResolution.py
===================================================================
--- ZODB/branches/jim-transform-wrapping/src/ZODB/ConflictResolution.py 2010-05-14 20:45:25 UTC (rev 112317)
+++ ZODB/branches/jim-transform-wrapping/src/ZODB/ConflictResolution.py 2010-05-14 20:46:38 UTC (rev 112318)
@@ -53,7 +53,7 @@
def state(self, oid, serial, prfactory, p=''):
p = p or self.loadSerial(oid, serial)
- p = self._crs_wrapper.untransform_record_data(p)
+ p = self._crs_untransform_record_data(p)
file = StringIO(p)
unpickler = Unpickler(file)
unpickler.find_global = find_global
@@ -180,7 +180,7 @@
# class_tuple, old, committed, newstate = ('',''), 0, 0, 0
try:
prfactory = PersistentReferenceFactory()
- newpickle = self._crs_wrapper.untransform_record_data(newpickle)
+ newpickle = self._crs_untransform_record_data(newpickle)
file = StringIO(newpickle)
unpickler = Unpickler(file)
unpickler.find_global = find_global
@@ -196,7 +196,7 @@
newargs = ()
if klass in _unresolvable:
- return None
+ raise ConflictError
newstate = unpickler.load()
inst = klass.__new__(klass, *newargs)
@@ -205,7 +205,7 @@
resolve = inst._p_resolveConflict
except AttributeError:
_unresolvable[klass] = 1
- return None
+ raise ConflictError
old = state(self, oid, oldSerial, prfactory)
committed = state(self, oid, committedSerial, prfactory, committedData)
@@ -217,9 +217,9 @@
pickler.inst_persistent_id = persistent_id
pickler.dump(meta)
pickler.dump(resolved)
- return self._crs_wrapper.transform_record_data(file.getvalue(1))
+ return self._crs_transform_record_data(file.getvalue(1))
except (ConflictError, BadClassName):
- return None
+ pass
except:
# If anything else went wrong, catch it here and avoid passing an
# arbitrary exception back to the client. The error here will mask
@@ -227,13 +227,19 @@
# ConflictError, but not necessarily from other errors. But log
# the error so that any problems can be fixed.
logger.error("Unexpected error", exc_info=True)
- return None
+ raise ConflictError(oid=oid, serials=(committedSerial, oldSerial),
+ data=newpickle)
+
class ConflictResolvingStorage(object):
"Mix-in class that provides conflict resolution handling for storages"
tryToResolveConflict = tryToResolveConflict
+ _crs_transform_record_data = _crs_untransform_record_data = (
+ lambda self, o: o)
+
def registerDB(self, wrapper):
- self._crs_wrapper = wrapper
+ self._crs_untransform_record_data = wrapper.untransform_record_data
+ self._crs_transform_record_data = wrapper.transform_record_data
super(ConflictResolvingStorage, self).registerDB(wrapper)
Modified: ZODB/branches/jim-transform-wrapping/src/ZODB/FileStorage/FileStorage.py
===================================================================
--- ZODB/branches/jim-transform-wrapping/src/ZODB/FileStorage/FileStorage.py 2010-05-14 20:45:25 UTC (rev 112317)
+++ ZODB/branches/jim-transform-wrapping/src/ZODB/FileStorage/FileStorage.py 2010-05-14 20:46:38 UTC (rev 112318)
@@ -476,14 +476,8 @@
committed_tid = h.tid
if oldserial != committed_tid:
- rdata = self.tryToResolveConflict(oid, committed_tid,
+ data = self.tryToResolveConflict(oid, committed_tid,
oldserial, data)
- if rdata is None:
- raise POSException.ConflictError(
- oid=oid, serials=(committed_tid, oldserial),
- data=data)
- else:
- data = rdata
pos = self._pos
here = pos + self._tfile.tell() + self._thl
@@ -843,10 +837,11 @@
except KeyError:
# couldn't find oid; what's the real explanation for this?
raise UndoError("_loadBack() failed for %s", oid)
- data = self.tryToResolveConflict(oid, ctid, tid, bdata, cdata)
-
- if data:
+ try:
+ data = self.tryToResolveConflict(oid, ctid, tid, bdata, cdata)
return data, 0, ipos
+ except POSException.ConflictError:
+ pass
raise UndoError("Some data were modified by a later transaction", oid)
@@ -961,7 +956,7 @@
except ZODB.POSException.POSKeyError:
pass # It was removed, so no need to copy data
else:
- if ZODB.blob.is_blob_record(up):
+ if self.is_blob_record(up):
# We're undoing a blob modification operation.
# We have to copy the blob data
tmp = ZODB.utils.mktemp(dir=self.fshelper.temp_dir)
Modified: ZODB/branches/jim-transform-wrapping/src/ZODB/FileStorage/fspack.py
===================================================================
--- ZODB/branches/jim-transform-wrapping/src/ZODB/FileStorage/fspack.py 2010-05-14 20:45:25 UTC (rev 112317)
+++ ZODB/branches/jim-transform-wrapping/src/ZODB/FileStorage/fspack.py 2010-05-14 20:46:38 UTC (rev 112318)
@@ -30,7 +30,6 @@
import logging
import os
-import ZODB.blob
import ZODB.fsIndex
import ZODB.POSException
@@ -502,7 +501,7 @@
data = self._file.read(h.plen)
else:
data = self.fetchDataViaBackpointer(h.oid, h.back)
- if data and ZODB.blob.is_blob_record(data):
+ if data and self._storage.is_blob_record(data):
# We need to remove the blob record. Maybe we
# need to remove oid:
Modified: ZODB/branches/jim-transform-wrapping/src/ZODB/blob.py
===================================================================
--- ZODB/branches/jim-transform-wrapping/src/ZODB/blob.py 2010-05-14 20:45:25 UTC (rev 112317)
+++ ZODB/branches/jim-transform-wrapping/src/ZODB/blob.py 2010-05-14 20:46:38 UTC (rev 112318)
@@ -617,30 +617,25 @@
"""
self.dirty_oids = []
+ def registerDB(self, db):
+ self.__untransform_record_data = db.untransform_record_data
+ try:
+ m = super(BlobStorageMixin, self).registerDB
+ except AttributeError:
+ pass
+ else:
+ m(db)
+
+ def __untransform_record_data(self, record):
+ return record
+
+ def is_blob_record(self, record):
+ if record:
+ return is_blob_record(self.__untransform_record_data(record))
+
def copyTransactionsFrom(self, other):
- for trans in other.iterator():
- self.tpc_begin(trans, trans.tid, trans.status)
- for record in trans:
- blobfilename = None
- if is_blob_record(record.data):
- try:
- blobfilename = other.loadBlob(record.oid, record.tid)
- except POSKeyError:
- pass
- if blobfilename is not None:
- fd, name = tempfile.mkstemp(
- suffix='.tmp', dir=self.fshelper.temp_dir)
- os.close(fd)
- utils.cp(open(blobfilename, 'rb'), open(name, 'wb'))
- self.restoreBlob(record.oid, record.tid, record.data,
- name, record.data_txn, trans)
- else:
- self.restore(record.oid, record.tid, record.data,
- '', record.data_txn, trans)
+ copyTransactionsFromTo(other, self)
- self.tpc_vote(trans)
- self.tpc_finish(trans)
-
def loadBlob(self, oid, serial):
"""Return the filename where the blob file can be found.
"""
@@ -943,3 +938,27 @@
pass
return False
+
+def copyTransactionsFromTo(source, destination):
+ for trans in source.iterator():
+ destination.tpc_begin(trans, trans.tid, trans.status)
+ for record in trans:
+ blobfilename = None
+ if is_blob_record(record.data):
+ try:
+ blobfilename = source.loadBlob(record.oid, record.tid)
+ except POSKeyError:
+ pass
+ if blobfilename is not None:
+ fd, name = tempfile.mkstemp(
+ suffix='.tmp', dir=destination.fshelper.temp_dir)
+ os.close(fd)
+ utils.cp(open(blobfilename, 'rb'), open(name, 'wb'))
+ destination.restoreBlob(record.oid, record.tid, record.data,
+ name, record.data_txn, trans)
+ else:
+ destination.restore(record.oid, record.tid, record.data,
+ '', record.data_txn, trans)
+
+ destination.tpc_vote(trans)
+ destination.tpc_finish(trans)
Modified: ZODB/branches/jim-transform-wrapping/src/ZODB/tests/BasicStorage.py
===================================================================
--- ZODB/branches/jim-transform-wrapping/src/ZODB/tests/BasicStorage.py 2010-05-14 20:45:25 UTC (rev 112317)
+++ ZODB/branches/jim-transform-wrapping/src/ZODB/tests/BasicStorage.py 2010-05-14 20:46:38 UTC (rev 112318)
@@ -49,7 +49,7 @@
self.assertRaises(
POSException.StorageTransactionError,
self._storage.store,
- ZERO, 1, 2, '', transaction.Transaction())
+ ZERO, 1, '2', '', transaction.Transaction())
self._storage.tpc_abort(t)
def checkSerialIsNoneForInitialRevision(self):
Modified: ZODB/branches/jim-transform-wrapping/src/ZODB/tests/testFileStorage.py
===================================================================
--- ZODB/branches/jim-transform-wrapping/src/ZODB/tests/testFileStorage.py 2010-05-14 20:45:25 UTC (rev 112317)
+++ ZODB/branches/jim-transform-wrapping/src/ZODB/tests/testFileStorage.py 2010-05-14 20:46:38 UTC (rev 112318)
@@ -15,6 +15,7 @@
import os, unittest
import transaction
import ZODB.FileStorage
+import ZODB.tests.hexstorage
import ZODB.tests.testblob
import ZODB.tests.util
import zope.testing.setupstack
@@ -279,14 +280,30 @@
else:
self.assertNotEqual(next_oid, None)
+class FileStorageHexTests(FileStorageTests):
+
+ def open(self, **kwargs):
+ self._storage = ZODB.tests.hexstorage.HexStorage(
+ ZODB.FileStorage.FileStorage('FileStorageTests.fs',**kwargs))
+
+
class FileStorageTestsWithBlobsEnabled(FileStorageTests):
def open(self, **kwargs):
if 'blob_dir' not in kwargs:
kwargs = kwargs.copy()
kwargs['blob_dir'] = 'blobs'
- return FileStorageTests.open(self, **kwargs)
+ FileStorageTests.open(self, **kwargs)
+class FileStorageHexTestsWithBlobsEnabled(FileStorageTests):
+
+ def open(self, **kwargs):
+ if 'blob_dir' not in kwargs:
+ kwargs = kwargs.copy()
+ kwargs['blob_dir'] = 'blobs'
+ FileStorageTests.open(self, **kwargs)
+ self._storage = ZODB.tests.hexstorage.HexStorage(self._storage)
+
class FileStorageRecoveryTest(
StorageTestBase.StorageTestBase,
RecoveryStorage.RecoveryStorage,
@@ -304,7 +321,16 @@
def new_dest(self):
return ZODB.FileStorage.FileStorage('Dest.fs')
+class FileStorageHexRecoveryTest(FileStorageRecoveryTest):
+ def setUp(self):
+ StorageTestBase.StorageTestBase.setUp(self)
+ self._storage = ZODB.tests.hexstorage.HexStorage(
+ ZODB.FileStorage.FileStorage("Source.fs", create=True))
+ self._dst = ZODB.tests.hexstorage.HexStorage(
+ ZODB.FileStorage.FileStorage("Dest.fs", create=True))
+
+
class FileStorageNoRestore(ZODB.FileStorage.FileStorage):
@property
@@ -634,10 +660,14 @@
from zope.testing import doctest
suite = unittest.TestSuite()
- for klass in [FileStorageTests, Corruption.FileStorageCorruptTests,
- FileStorageRecoveryTest, FileStorageNoRestoreRecoveryTest,
- FileStorageTestsWithBlobsEnabled, AnalyzeDotPyTest,
- ]:
+ for klass in [
+ FileStorageTests, FileStorageHexTests,
+ Corruption.FileStorageCorruptTests,
+ FileStorageRecoveryTest, FileStorageHexRecoveryTest,
+ FileStorageNoRestoreRecoveryTest,
+ FileStorageTestsWithBlobsEnabled, FileStorageHexTestsWithBlobsEnabled,
+ AnalyzeDotPyTest,
+ ]:
suite.addTest(unittest.makeSuite(klass, "check"))
suite.addTest(doctest.DocTestSuite(
setUp=zope.testing.setupstack.setUpDirectory,
@@ -649,6 +679,14 @@
test_blob_storage_recovery=True,
test_packing=True,
))
+ suite.addTest(ZODB.tests.testblob.storage_reusable_suite(
+ 'BlobFileHexStorage',
+ lambda name, blob_dir:
+ ZODB.tests.hexstorage.HexStorage(
+ ZODB.FileStorage.FileStorage('%s.fs' % name, blob_dir=blob_dir)),
+ test_blob_storage_recovery=True,
+ test_packing=True,
+ ))
suite.addTest(PackableStorage.IExternalGC_suite(
lambda : ZODB.FileStorage.FileStorage(
'data.fs', blob_dir='blobs', pack_gc=False)))
Modified: ZODB/branches/jim-transform-wrapping/src/ZODB/tests/testblob.py
===================================================================
--- ZODB/branches/jim-transform-wrapping/src/ZODB/tests/testblob.py 2010-05-14 20:45:25 UTC (rev 112317)
+++ ZODB/branches/jim-transform-wrapping/src/ZODB/tests/testblob.py 2010-05-14 20:46:38 UTC (rev 112318)
@@ -145,7 +145,7 @@
# the blob footprint object should exist no longer
self.assertRaises(KeyError, root.__getitem__, 'blob')
database.close()
-
+
def testUndo(self):
database = DB(self._storage)
connection = database.open()
@@ -178,7 +178,7 @@
blob.consumeFile('consume1')
root['blob'] = blob
transaction.commit()
-
+
transaction.begin()
blob = root['blob']
open('consume2', 'w').write('this is state 2')
@@ -279,8 +279,8 @@
transaction.commit()
self._dst.copyTransactionsFrom(self._storage)
self.compare(self._storage, self._dst)
-
+
def gc_blob_removes_uncommitted_data():
"""
>>> blob = Blob()
@@ -335,7 +335,7 @@
>>> transaction.commit() # doctest: +ELLIPSIS
Copied blob file ...
-
+
>>> root['blob2'].open().read()
'test2'
@@ -625,7 +625,7 @@
return factory(name, blob_dir)
test.globs['create_storage'] = create_storage
-
+
suite = unittest.TestSuite()
suite.addTest(doctest.DocFileSuite(
"blob_connection.txt", "blob_importexport.txt",
More information about the Zodb-checkins
mailing list