[Zodb-checkins] SVN: ZODB/trunk/src/ A new storage interface, IExternalGC, to support external garbage
Jim Fulton
jim at zope.com
Thu Dec 18 14:08:11 EST 2008
Log message for revision 94175:
A new storage interface, IExternalGC, to support external garbage
collection, http://wiki.zope.org/ZODB/ExternalGC, has been defined
and implemented for FileStorage.
Changed:
U ZODB/trunk/src/CHANGES.txt
U ZODB/trunk/src/ZODB/FileStorage/FileStorage.py
U ZODB/trunk/src/ZODB/interfaces.py
A ZODB/trunk/src/ZODB/tests/IExternalGC.test
U ZODB/trunk/src/ZODB/tests/PackableStorage.py
U ZODB/trunk/src/ZODB/tests/testFileStorage.py
-=-
Modified: ZODB/trunk/src/CHANGES.txt
===================================================================
--- ZODB/trunk/src/CHANGES.txt 2008-12-18 19:08:08 UTC (rev 94174)
+++ ZODB/trunk/src/CHANGES.txt 2008-12-18 19:08:10 UTC (rev 94175)
@@ -35,6 +35,13 @@
- POSKeyErrors are no longer logged by ZEO servers, because they are
really client errors.
+- A new storage interface, IExternalGC, to support external garbage
+ collection, http://wiki.zope.org/ZODB/ExternalGC, has been defined
+ and implemented for FileStorage.
+
+- As a small convenience (mainly for tests), you can now specify
+ initial data as a string argument to the Blob constructor.
+
3.9.0a8 (2008-12-15)
====================
Modified: ZODB/trunk/src/ZODB/FileStorage/FileStorage.py
===================================================================
--- ZODB/trunk/src/ZODB/FileStorage/FileStorage.py 2008-12-18 19:08:08 UTC (rev 94174)
+++ ZODB/trunk/src/ZODB/FileStorage/FileStorage.py 2008-12-18 19:08:10 UTC (rev 94175)
@@ -425,11 +425,13 @@
if h.plen:
data = self._file.read(h.plen)
return data, h.tid
- else:
+ elif h.back:
# Get the data from the backpointer, but tid from
# current txn.
data = self._loadBack_impl(oid, h.back)[0]
return data, h.tid
+ else:
+ raise POSKeyError(oid)
finally:
self._lock_release()
@@ -524,6 +526,41 @@
finally:
self._lock_release()
+ def deleteObject(self, oid, oldserial, transaction):
+ if self._is_read_only:
+ raise POSException.ReadOnlyError()
+ if transaction is not self._transaction:
+ raise POSException.StorageTransactionError(self, transaction)
+
+ self._lock_acquire()
+ try:
+ old = self._index_get(oid, 0)
+ if not old:
+ raise POSException.POSKeyError(oid)
+ h = self._read_data_header(old, oid)
+ committed_tid = h.tid
+
+ if oldserial != committed_tid:
+ raise POSException.ConflictError(
+ oid=oid, serials=(committed_tid, oldserial))
+
+ pos = self._pos
+ here = pos + self._tfile.tell() + self._thl
+ self._tindex[oid] = here
+ new = DataHeader(oid, self._tid, old, pos, 0, 0)
+ self._tfile.write(new.asString())
+ self._tfile.write(z64)
+
+ # Check quota
+ if self._quota is not None and here > self._quota:
+ raise FileStorageQuotaError(
+ "The storage quota has been exceeded.")
+
+ return self._tid
+
+ finally:
+ self._lock_release()
+
def _data_find(self, tpos, oid, data):
# Return backpointer for oid. Must call with the lock held.
# This is a file offset to oid's data record if found, else 0.
Modified: ZODB/trunk/src/ZODB/interfaces.py
===================================================================
--- ZODB/trunk/src/ZODB/interfaces.py 2008-12-18 19:08:08 UTC (rev 94174)
+++ ZODB/trunk/src/ZODB/interfaces.py 2008-12-18 19:08:10 UTC (rev 94175)
@@ -969,7 +969,28 @@
"""
+class IExternalGC(IStorage):
+ def deleteObject(oid, serial, transaction):
+ """Mark an object as deleted
+
+ This method marks an object as deleted via a new object
+ revision. Subsequent attempts to load current data for the
+ object will fail with a POSKeyError, but loads for
+ non-current data will suceed if there are previous
+ non-delete records. The object will be removed from the
+ storage when all not-delete records are removed.
+
+ The the storage's transaction id for the current transaction is
+ returned.
+
+ The serial argument must match the most recently committed
+ serial for the object. This is a seat belt.
+
+ This method can only be called in the first phase of 2-phase
+ commit.
+ """
+
class IBlob(Interface):
"""A BLOB supports efficient handling of large data within ZODB."""
Added: ZODB/trunk/src/ZODB/tests/IExternalGC.test
===================================================================
--- ZODB/trunk/src/ZODB/tests/IExternalGC.test (rev 0)
+++ ZODB/trunk/src/ZODB/tests/IExternalGC.test 2008-12-18 19:08:10 UTC (rev 94175)
@@ -0,0 +1,111 @@
+Storage Support for external GC
+===============================
+
+A storage that provides IExternalGC supports external garbage
+collectors by providing a deleteObject method that transactionally
+deletes an object.
+
+A create_storage function is provided that creates a storage.
+
+ >>> storage = create_storage()
+ >>> import ZODB.blob, transaction
+ >>> db = ZODB.DB(storage)
+ >>> conn = db.open()
+ >>> conn.root()[0] = conn.root().__class__()
+ >>> conn.root()[1] = ZODB.blob.Blob('some data')
+ >>> transaction.commit()
+ >>> oid0 = conn.root()[0]._p_oid
+ >>> oid1 = conn.root()[1]._p_oid
+ >>> del conn.root()[0]
+ >>> del conn.root()[1]
+ >>> transaction.commit()
+
+At this point, object 0 and 1 is garbage, but it's still in the storage:
+
+ >>> p0, s0 = storage.load(oid0, '')
+ >>> p1, s1 = storage.load(oid1, '')
+
+Now we'll use the new deleteObject API to delete the objects. We can't
+go through the database to do this, so we'll have to manage the
+transaction ourselves.
+
+ >>> txn = transaction.begin()
+ >>> storage.tpc_begin(txn)
+ >>> tid = storage.deleteObject(oid0, s0, txn)
+ >>> tid = storage.deleteObject(oid1, s1, txn)
+ >>> storage.tpc_vote(txn)
+ >>> storage.tpc_finish(txn)
+
+Now if we try to load data for the objects, we get a POSKeyError:
+
+
+ >>> storage.load(oid0, '') # doctest: +ELLIPSIS
+ Traceback (most recent call last):
+ ...
+ POSKeyError: ...
+
+ >>> storage.load(oid1, '') # doctest: +ELLIPSIS
+ Traceback (most recent call last):
+ ...
+ POSKeyError: ...
+
+We can still get the data if we load before the time we deleted.
+
+ >>> storage.loadBefore(oid0, conn.root()._p_serial) == (p0, s0, tid)
+ True
+ >>> storage.loadBefore(oid1, conn.root()._p_serial) == (p1, s1, tid)
+ True
+ >>> open(storage.loadBlob(oid1, s1)).read()
+ 'some data'
+
+If we pack, however, the old data will be removed and the data will be
+gone:
+
+ >>> import time
+ >>> db.pack(time.time()+1)
+
+ >>> storage.load(oid0, '') # doctest: +ELLIPSIS
+ Traceback (most recent call last):
+ ...
+ POSKeyError: ...
+
+ >>> storage.load(oid1, '') # doctest: +ELLIPSIS
+ Traceback (most recent call last):
+ ...
+ POSKeyError: ...
+
+ >>> storage.loadBefore(oid0, conn.root()._p_serial) # doctest: +ELLIPSIS
+ Traceback (most recent call last):
+ ...
+ POSKeyError: ...
+
+ >>> storage.loadBefore(oid1, conn.root()._p_serial) # doctest: +ELLIPSIS
+ Traceback (most recent call last):
+ ...
+ POSKeyError: ...
+
+ >>> storage.loadBlob(oid1, s1) # doctest: +ELLIPSIS
+ Traceback (most recent call last):
+ ...
+ POSKeyError: ...
+
+A conflict error is raised if the serial we provide to deleteObject
+isn't current:
+
+ >>> conn.root()[0] = conn.root().__class__()
+ >>> transaction.commit()
+ >>> oid = conn.root()[0]._p_oid
+ >>> bad_serial = conn.root()[0]._p_serial
+ >>> conn.root()[0].x = 1
+ >>> transaction.commit()
+
+ >>> txn = transaction.begin()
+ >>> storage.tpc_begin(txn)
+ >>> storage.deleteObject(oid, bad_serial, txn) # doctest: +ELLIPSIS
+ Traceback (most recent call last):
+ ...
+ ConflictError: database conflict error ...
+
+ >>> storage.tpc_abort(txn)
+
+ >>> storage.close()
Property changes on: ZODB/trunk/src/ZODB/tests/IExternalGC.test
___________________________________________________________________
Added: svn:eol-style
+ native
Modified: ZODB/trunk/src/ZODB/tests/PackableStorage.py
===================================================================
--- ZODB/trunk/src/ZODB/tests/PackableStorage.py 2008-12-18 19:08:08 UTC (rev 94174)
+++ ZODB/trunk/src/ZODB/tests/PackableStorage.py 2008-12-18 19:08:10 UTC (rev 94175)
@@ -13,24 +13,22 @@
##############################################################################
"""Run some tests relevant for storages that support pack()."""
-import cPickle
from cStringIO import StringIO
-
-import time
-
from persistent import Persistent
from persistent.mapping import PersistentMapping
-import transaction
-import ZODB.interfaces
from ZODB import DB
+from ZODB.POSException import ConflictError, StorageError
from ZODB.serialize import referencesf
from ZODB.tests.MinPO import MinPO
+from ZODB.tests.MTStorage import TestThread
from ZODB.tests.StorageTestBase import snooze
-from ZODB.POSException import ConflictError, StorageError
-
-from ZODB.tests.MTStorage import TestThread
-
+from zope.testing import doctest
+import cPickle
+import time
+import transaction
+import ZODB.interfaces
import ZODB.tests.util
+import zope.testing.setupstack
ZERO = '\0'*8
@@ -750,3 +748,17 @@
def elapsed_millis(self):
return int((time.time() - self.start_time) * 1000)
+
+def IExternalGC_suite(factory):
+ """Return a test suite for a generic .
+
+ Pass a factory taking a name and a blob directory name.
+ """
+
+ def setup(test):
+ ZODB.tests.util.setUp(test)
+ test.globs['create_storage'] = factory
+
+ return doctest.DocFileSuite(
+ 'IExternalGC.test',
+ setUp=setup, tearDown=zope.testing.setupstack.tearDown)
Modified: ZODB/trunk/src/ZODB/tests/testFileStorage.py
===================================================================
--- ZODB/trunk/src/ZODB/tests/testFileStorage.py 2008-12-18 19:08:08 UTC (rev 94174)
+++ ZODB/trunk/src/ZODB/tests/testFileStorage.py 2008-12-18 19:08:10 UTC (rev 94175)
@@ -580,7 +580,6 @@
>>> db.close()
"""
-
def test_suite():
from zope.testing import doctest
@@ -600,6 +599,8 @@
test_blob_storage_recovery=True,
test_packing=True,
))
+ suite.addTest(PackableStorage.IExternalGC_suite(
+ lambda : ZODB.FileStorage.FileStorage('data.fs', blob_dir='blobs')))
return suite
if __name__=='__main__':
More information about the Zodb-checkins
mailing list