[Checkins] SVN: zc.bsddbstorage/branches/dev/src/zc/bsddbstorage/ Added a truncation utility -- mainly for use when benchmarking.
Jim Fulton
jim at zope.com
Tue Dec 22 14:22:08 EST 2009
Log message for revision 106917:
Added a truncation utility -- mainly for use when benchmarking.
Changed:
U zc.bsddbstorage/branches/dev/src/zc/bsddbstorage/__init__.py
U zc.bsddbstorage/branches/dev/src/zc/bsddbstorage/tests.py
-=-
Modified: zc.bsddbstorage/branches/dev/src/zc/bsddbstorage/__init__.py
===================================================================
--- zc.bsddbstorage/branches/dev/src/zc/bsddbstorage/__init__.py 2009-12-22 19:21:29 UTC (rev 106916)
+++ zc.bsddbstorage/branches/dev/src/zc/bsddbstorage/__init__.py 2009-12-22 19:22:07 UTC (rev 106917)
@@ -645,6 +645,69 @@
#
##############################################################
+
+ @retry_on_deadlock
+ def _kill1(self, truncate_tid):
+ # Pack one transaction. Get the next transaction we haven't yet
+ # packed and stop if it is > pack_tid.
+ # This is done as a transaction.
+ removed_blobs = []
+ removed_oids = 0
+ with self.txn(db.DB_TXN_SNAPSHOT) as txn:
+ with self.cursor(self.transactions, txn) as transactions:
+ kv = transactions.get(flags=db.DB_LAST)
+ if kv is None:
+ return None
+ tid = kv[0]
+ if tid <= truncate_tid:
+ return None
+
+ ntid = n64(tid)
+ oids = cPickle.loads(kv[1])[-1]
+ with self.cursor(self.data, txn) as data:
+ for oid in oids:
+ kr = data.get(oid, ntid, flags=db.DB_GET_BOTH_RANGE)
+ if not kr:
+ kr = data.get(oid, flags=db.DB_SET)
+
+ doid, record = kr
+ assert doid == oid
+ assert record[:8] == ntid
+ data.delete()
+ deleted_oid = data.get(oid, flags=db.DB_SET) is None
+ if deleted_oid:
+ removed_oids += 1
+ if (self.blob_dir and
+ ZODB.blob.is_blob_record(record[8:])
+ ):
+ if deleted_oid:
+ if ((not removed_blobs) or
+ (removed_blobs[-1] != oid)):
+ removed_blobs.append(oid)
+ else:
+ removed_blobs.append(oid+tid)
+
+ transactions.delete()
+
+ self._inc_len(-removed_oids)
+
+ if removed_blobs:
+ self._remove_blob_files_tagged_for_removal_during_pack(
+ removed_blobs)
+
+ return tid
+
+def truncate(tid, *args, **kw):
+ if isinstance(tid, str):
+ u64(tid) # rough sanity check
+ else:
+ tid = timetime2tid(tid)
+ kw['checkpoint'] = 0
+ s = Storage(*args, **kw)
+ while s._kill1(tid):
+ pass
+ s.close()
+
Storage = BSDDBStorage # easier to type alias :)
class TransactionContext(object):
Modified: zc.bsddbstorage/branches/dev/src/zc/bsddbstorage/tests.py
===================================================================
--- zc.bsddbstorage/branches/dev/src/zc/bsddbstorage/tests.py 2009-12-22 19:21:29 UTC (rev 106916)
+++ zc.bsddbstorage/branches/dev/src/zc/bsddbstorage/tests.py 2009-12-22 19:22:07 UTC (rev 106917)
@@ -14,10 +14,12 @@
from zope.testing import doctest, setupstack
import cPickle
import cStringIO
+import os
import time
import unittest
import zc.bsddbstorage
import ZEO.tests.testZEO
+import ZODB.blob
import ZODB.tests.BasicStorage
import ZODB.tests.ConflictResolution
import ZODB.tests.HistoryStorage
@@ -31,6 +33,7 @@
import ZODB.tests.StorageTestBase
import ZODB.tests.Synchronization
import ZODB.tests.testblob
+import ZODB.tests.util
def DISABLED(self):
"disabled"
@@ -166,6 +169,87 @@
# don't know if that is a bad thing.
checkIteratorGCSpanTransactions = DISABLED
+def truncate():
+ """Database truncation
+
+Sometimes, it's useful to be able to truncate a database at a
+particular time/tid. You might do this to undo a bunch of trailing
+activity or when doing benchmark to prepare to replay transactions.
+
+ >>> import transaction
+ >>> db = zc.bsddbstorage.DB('test', 'blobs')
+ >>> conn = db.open()
+ >>> conn.root.x = 0
+ >>> conn.root.blob = ZODB.blob.Blob()
+ >>> conn.root.blobs = conn.root().__class__()
+ >>> for i in range(10):
+ ... conn.root.x += 1
+ ... conn.root.blob.open('w').write(str(conn.root.x))
+ ... conn.root.blobs[conn.root.x] = ZODB.blob.Blob('data')
+ ... transaction.commit()
+ >>> time.sleep(.01)
+ >>> tt10 = time.time()
+ >>> time.sleep(.01)
+ >>> for i in range(10):
+ ... conn.root.x += 1
+ ... conn.root.blob.open('w').write(str(conn.root.x))
+ ... conn.root.blobs[conn.root.x] = ZODB.blob.Blob('data')
+ ... transaction.commit()
+ >>> tid20 = db.storage.lastTransaction()
+ >>> for i in range(10):
+ ... conn.root.x += 1
+ ... conn.root.blob.open('w').write(str(conn.root.x))
+ ... conn.root.blobs[conn.root.x] = ZODB.blob.Blob('data')
+ ... transaction.commit()
+ >>> conn.root.x, conn.root.blob.open().read(), len(conn.root.blobs)
+ (30, '30', 30)
+
+ >>> def count_blob_files(dir):
+ ... n = 0
+ ... for base, dirs, files in os.walk(os.path.join(dir, '0x00')):
+ ... for file in files:
+ ... if file.endswith('.blob'):
+ ... n += 1
+ ... return n
+
+ >>> count_blob_files('blobs')
+ 60
+
+We can truncate using either a tid or a time.time. We can't truncate a
+storage while it's open:
+
+ >>> zc.bsddbstorage.truncate(tid20, 'test', 'blobs') # doctest: +ELLIPSIS
+ Traceback (most recent call last):
+ ...
+ LockError: Couldn't lock ...
+
+ >>> db.close()
+
+First, we'll truncate using a tid:
+
+ >>> zc.bsddbstorage.truncate(tid20, 'test', 'blobs')
+ >>> db = zc.bsddbstorage.DB('test', 'blobs')
+ >>> conn = db.open()
+ >>> conn.root.x, conn.root.blob.open().read(), len(conn.root.blobs)
+ (20, '20', 20)
+ >>> count_blob_files('blobs')
+ 40
+ >>> db.close()
+
+
+We can also use a time.time:
+
+ >>> zc.bsddbstorage.truncate(tt10, 'test', 'blobs')
+ >>> db = zc.bsddbstorage.DB('test', 'blobs')
+ >>> conn = db.open()
+ >>> conn.root.x, conn.root.blob.open().read(), len(conn.root.blobs)
+ (10, '10', 10)
+ >>> count_blob_files('blobs')
+ 20
+ >>> db.close()
+ """
+
+
def test_suite():
suite = unittest.TestSuite()
for klass in [
@@ -187,4 +271,8 @@
suite.addTest(ZODB.tests.PackableStorage.IExternalGC_suite(
lambda : zc.bsddbstorage.BSDDBStorage(
'data', blob_dir='blobs')))
+ suite.addTest(
+ doctest.DocTestSuite(
+ setUp=ZODB.tests.util.setUp, tearDown=ZODB.tests.util.tearDown)
+ )
return suite
More information about the checkins
mailing list