[Checkins] SVN: zc.bsddbstorage/branches/dev/src/zc/bsddbstorage/ checkpoint
Jim Fulton
jim at zope.com
Mon Nov 16 15:54:58 EST 2009
Log message for revision 105732:
checkpoint
Changed:
U zc.bsddbstorage/branches/dev/src/zc/bsddbstorage/__init__.py
A zc.bsddbstorage/branches/dev/src/zc/bsddbstorage/tests.py
-=-
Modified: zc.bsddbstorage/branches/dev/src/zc/bsddbstorage/__init__.py
===================================================================
--- zc.bsddbstorage/branches/dev/src/zc/bsddbstorage/__init__.py 2009-11-16 20:52:06 UTC (rev 105731)
+++ zc.bsddbstorage/branches/dev/src/zc/bsddbstorage/__init__.py 2009-11-16 20:54:57 UTC (rev 105732)
@@ -32,6 +32,9 @@
def n64(tid):
return p64(868082074056920076L-u64(tid))
+# XXX Still need checkpoint and deadlock detection strategies.
+# Maybe initial config file when creating env.
+
class BSDDBStorage(
ZODB.blob.BlobStorageMixin,
ZODB.ConflictResolution.ConflictResolvingStorage,
@@ -41,7 +44,7 @@
ZODB.interfaces.IStorage,
ZODB.interfaces.IStorageRestoreable,
ZODB.interfaces.IStorageIteration,
-# ZODB.interfaces.IStorageCurrentRecordIteration,
+# XXX? ZODB.interfaces.IStorageCurrentRecordIteration,
ZODB.interfaces.IExternalGC,
)
@@ -67,13 +70,12 @@
if blob_dir:
blob_dir = os.path.abspath(blob_dir)
self._blob_init(blob_dir)
- zope.interface.alsoProvides(self,
- ZODB.interfaces.IBlobStorageRestoreable)
+ zope.interface.alsoProvides(
+ self, ZODB.interfaces.IBlobStorageRestoreable)
else:
- self.blob_dir = None
self._blob_init_no_blobs()
+ self.blob_dir = blob_dir
-
self.env = db.DBEnv()
self.env.log_set_config(db.DB_LOG_AUTO_REMOVE, 1) # XXX should be optional
flags = (db.DB_INIT_LOCK | db.DB_INIT_LOG | db.DB_INIT_MPOOL |
@@ -119,12 +121,18 @@
)
t = time.time()
- t = self._ts = ZODB.TimeStamp.TimeStamp(*(time.gmtime(t)[:5] + (t%60,)))
+ t = self._ts = ZODB.TimeStamp.TimeStamp(
+ *(time.gmtime(t)[:5] + (t%60,)))
self._tid = repr(t)
self._transaction = None
self._commit_lock = threading.Lock()
+ _lock = threading.Lock()
+ # BlobStorageMixin requires these. I'm getting annoyed. :)
+ self._lock_acquire = _lock.acquire
+ self._lock_release = _lock.release
+
# The current lock is used to make sure we consistently order
# information about current data for objects. In particular,
# we want to avoid the following scenario:
@@ -198,7 +206,11 @@
return cursor.get(db.DB_LAST)[0]
def __len__(self):
- return self.data.stat(db.DB_FAST_STAT)['nkeys']
+ # XXX this is probably very expensive, but we need this for the
+ # tests. :( Need to check this out with a big db to decode what the
+ # cost is. Usually, accuracy isn't that important.
+ return self.data.stat()['nkeys']
+ #return self.data.stat(db.DB_FAST_STAT)['nkeys']
def load(self, oid, version=''):
with self._current_lock.read():
@@ -214,24 +226,28 @@
raise ZODB.POSException.POSKeyError(oid)
def loadBefore(self, oid, tid):
+ ntid = p64(868082074056920076L-(u64(tid)-1))
with self.txn(db.DB_TXN_SNAPSHOT) as txn:
with self.cursor(self.data, txn) as cursor:
- kr = cursor.get(oid, db.DB_SET)
+ # Step 1, find the record
+ kr = cursor.get(oid, ntid, db.DB_GET_BOTH_RANGE)
if kr is None:
- raise ZODB.POSException.POSKeyError(oid)
+ kr = cursor.get(oid, db.DB_SET)
+ if kr is None:
+ raise ZODB.POSException.POSKeyError(oid)
record = kr[1]
- if kr[0] != oid or len(record) == 8:
+ rtid = n64(record[:8])
+ if kr[0] != oid or rtid >= tid or len(record) == 8:
raise ZODB.POSException.POSKeyError(oid)
- nexttid = None
- rtid = n64(record[:8])
- while rtid >= tid:
- krecord = cursor.get(oid, flags=db.DB_NEXT_DUP)
- if krecord is None:
- return None
- nexttid = rtid
- record = krecord[1]
- rtid = n64(record[:8])
+ # Now, get the next tid:
+ kr = cursor.get(oid, ntid, db.DB_PREV_DUP, dlen=8, doff=0)
+ if kr is None:
+ nexttid = None
+ else:
+ assert kr[0] == oid
+ nexttid = n64(kr[1])
+
return record[8:], rtid, nexttid
def loadSerial(self, oid, serial):
@@ -392,7 +408,7 @@
# fact that oids are allocates sequentially
paths = filter(os.path.isdir,
[os.path.join(dir, name)
- for name in sorted(os.listdor(dir))
+ for name in sorted(os.listdir(dir))
if name.lower().startswith('0x')])
if not paths:
return
@@ -450,14 +466,15 @@
if transaction is not self._transaction:
raise ZODB.POSException.StorageTransactionError(self, transaction)
committed_tid = self.data.get(oid, dlen=8, doff=0)
- if committed_tid is not None and committed_tid != oldserial:
+ if committed_tid is not None and n64(committed_tid) != oldserial:
raise ZODB.POSException.ConflictError(
- oid=oid, serials=(committed_tid, oldserial))
+ oid=oid, serials=(n64(committed_tid), oldserial))
marshal.dump((oid, n64(self._tid)), self._log_file)
def tpc_abort(self, transaction):
- self._txn.abort()
+ if self._txn is not None:
+ self._txn.abort()
self._txn = self._transaction = None
self._blob_tpc_abort()
self._commit_lock.release()
Added: zc.bsddbstorage/branches/dev/src/zc/bsddbstorage/tests.py
===================================================================
--- zc.bsddbstorage/branches/dev/src/zc/bsddbstorage/tests.py (rev 0)
+++ zc.bsddbstorage/branches/dev/src/zc/bsddbstorage/tests.py 2009-11-16 20:54:57 UTC (rev 105732)
@@ -0,0 +1,70 @@
+##############################################################################
+#
+# Copyright (c) Zope Foundation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
+from zope.testing import doctest
+import unittest
+import zc.bsddbstorage
+import ZODB.tests.BasicStorage
+import ZODB.tests.ConflictResolution
+import ZODB.tests.HistoryStorage
+import ZODB.tests.IteratorStorage
+import ZODB.tests.IteratorStorage
+import ZODB.tests.MTStorage
+import ZODB.tests.PackableStorage
+import ZODB.tests.PersistentStorage
+import ZODB.tests.ReadOnlyStorage
+import ZODB.tests.RevisionStorage
+import ZODB.tests.StorageTestBase
+import ZODB.tests.Synchronization
+import ZODB.tests.testblob
+
+class BSDDBStorageTests(
+ ZODB.tests.StorageTestBase.StorageTestBase,
+ ZODB.tests.BasicStorage.BasicStorage,
+ ZODB.tests.RevisionStorage.RevisionStorage,
+ ZODB.tests.PackableStorage.PackableStorageWithOptionalGC,
+ ZODB.tests.Synchronization.SynchronizedStorage,
+ ZODB.tests.ConflictResolution.ConflictResolvingStorage,
+ ZODB.tests.HistoryStorage.HistoryStorage,
+ ZODB.tests.IteratorStorage.IteratorStorage,
+ ZODB.tests.IteratorStorage.ExtendedIteratorStorage,
+ ZODB.tests.PersistentStorage.PersistentStorage,
+ ZODB.tests.MTStorage.MTStorage,
+ ZODB.tests.ReadOnlyStorage.ReadOnlyStorage
+ ):
+
+ def open(self, **kwargs):
+ self._storage = zc.bsddbstorage.BSDDBStorage(
+ 'storage', **kwargs)
+
+ def setUp(self):
+ StorageTestBase.StorageTestBase.setUp(self)
+ self.open(create=1)
+
+def test_suite():
+ suite = unittest.TestSuite()
+ for klass in [
+ BSDDBStorageTests,
+ ]:
+ suite.addTest(unittest.makeSuite(klass, "check"))
+ suite.addTest(ZODB.tests.testblob.storage_reusable_suite(
+ 'BlobBSDDBStorage',
+ lambda name, blob_dir:
+ zc.bsddbstorage.BSDDBStorage(name, blob_dir=blob_dir),
+ test_blob_storage_recovery=True,
+ test_packing=True,
+ ))
+ suite.addTest(ZODB.tests.PackableStorage.IExternalGC_suite(
+ lambda : zc.bsddbstorage.BSDDBStorage(
+ 'data', blob_dir='blobs')))
+ return suite
Property changes on: zc.bsddbstorage/branches/dev/src/zc/bsddbstorage/tests.py
___________________________________________________________________
Added: svn:keywords
+ Id
Added: svn:eol-style
+ native
More information about the checkins
mailing list