[Checkins] SVN: zc.zlibstorage/branches/dev/src/zc/zlibstorage/ checkpoint
Jim Fulton
jim at zope.com
Mon May 17 16:26:59 EDT 2010
Log message for revision 112429:
checkpoint
Changed:
U zc.zlibstorage/branches/dev/src/zc/zlibstorage/__init__.py
U zc.zlibstorage/branches/dev/src/zc/zlibstorage/component.xml
U zc.zlibstorage/branches/dev/src/zc/zlibstorage/tests.py
-=-
Modified: zc.zlibstorage/branches/dev/src/zc/zlibstorage/__init__.py
===================================================================
--- zc.zlibstorage/branches/dev/src/zc/zlibstorage/__init__.py 2010-05-17 20:08:32 UTC (rev 112428)
+++ zc.zlibstorage/branches/dev/src/zc/zlibstorage/__init__.py 2010-05-17 20:26:58 UTC (rev 112429)
@@ -15,39 +15,32 @@
import ZODB.interfaces
import zope.interface
-class Storage(object):
+class ZlibStorage(object):
zope.interface.implements(ZODB.interfaces.IStorageWrapper)
- def __init__(self, base, compress=True):
- self.base = base
- self.compress = compress
-
- for name in (
+ copied_methods = (
'close', 'getName', 'getSize', 'history', 'isReadOnly',
'lastTransaction', 'new_oid', 'sortKey',
'tpc_abort', 'tpc_begin', 'tpc_finish', 'tpc_vote',
'loadBlob', 'openCommittedBlobFile', 'temporaryDirectory',
'supportsUndo', 'undo', 'undoLog', 'undoInfo',
- ):
+ )
+
+ def __init__(self, base, compress=True):
+ self.base = base
+ self.compress = compress
+
+ for name in self.copied_methods:
v = getattr(base, name, None)
if v is not None:
setattr(self, name, v)
zope.interface.directlyProvides(self, zope.interface.providedBy(base))
- def _transform(self, data):
- if self.compress:
- compressed = '.z'+zlib.compress(data)
- if len(compressed) < len(data):
- return compressed
- return data
+ def __getattr__(self, name):
+ return getattr(self.base, name)
- def _untransform(self, data):
- if data[:2] == '.z':
- return zlib.decompress(data[2:])
- return data
-
def __len__(self):
return len(self.base)
@@ -66,14 +59,22 @@
def loadSerial(self, oid, serial):
return self._untransform(self.base.loadSerial(oid, serial))
- def pack(self, pack_time, referencesf):
+ def pack(self, pack_time, referencesf, gc=None):
_untransform = self._untransform
def refs(p, oids=None):
return referencesf(_untransform(p), oids)
+ if gc is not None:
+ return self.base.pack(pack_time, refs, gc)
+ else:
+ return self.base.pack(pack_time, refs)
def registerDB(self, db):
self.db = db
+ self._db_transform = db.transform_record_data
+ self._db_untransform = db.untransform_record_data
+ _db_transform = _db_untransform = lambda self, data: data
+
def store(self, oid, serial, data, version, transaction):
if self.compress:
data = self._transform(data)
@@ -86,8 +87,8 @@
oid, serial, data, version, prev_txn, transaction)
def iterator(self, start=None, stop=None):
- for t in self.base.iterator(start, end):
- yield Transaction(t)
+ for t in self.base.iterator(start, stop):
+ yield Transaction(self, t)
def storeBlob(self, oid, oldserial, data, blobfilename, version,
transaction):
@@ -118,7 +119,41 @@
def untransform_record_data(self, data):
return self.db.untransform_record_data(self._untransform(data))
+ def record_iternext(self, next=None):
+ oid, tid, data, next = self.base.record_iternext(next)
+ return oid, tid, self._untransform(data), next
+ def copyTransactionsFrom(self, other):
+ ZODB.blob.copyTransactionsFromTo(other, self)
+
+ def _transform(self, data):
+ if data and self.compress and len(data) > 20:
+ compressed = '.z'+zlib.compress(data)
+ if len(compressed) < len(data):
+ return compressed
+ return data
+
+ def _untransform(self, data):
+ if data[:2] == '.z':
+ return zlib.decompress(data[2:])
+ return data
+
+ def copyTransactionsFrom(self, other):
+ ZODB.blob.copyTransactionsFromTo(other, self)
+
+
+class ServerZlibStorage(ZlibStorage):
+ """Use on ZEO storage server when ZlibStorage is used on client
+
+ Don't do conversion as part of load/store, but provide
+ pickle decoding.
+ """
+
+ copied_methods = ZlibStorage.copied_methods + (
+ 'load', 'loadBefore', 'loadSerial', 'store', 'restore',
+ 'iterator', 'storeBlob', 'restoreBlob', 'record_iternext',
+ )
+
class Transaction(object):
def __init__(self, store, trans):
@@ -132,10 +167,13 @@
yield r
def __getattr__(self, name):
- return getattr(self.__trans)
+ return getattr(self.__trans, name)
+
class ZConfig:
+ _factory = ZlibStorage
+
def __init__(self, config):
self.config = config
self.name = config.getSectionName()
@@ -145,4 +183,8 @@
compress = self.config.compress
if compress is None:
compress = True
- return Storage(base, compress)
+ return self._factory(base, compress)
+
+class ZConfigServer(ZConfig):
+
+ _factory = ServerZlibStorage
Modified: zc.zlibstorage/branches/dev/src/zc/zlibstorage/component.xml
===================================================================
--- zc.zlibstorage/branches/dev/src/zc/zlibstorage/component.xml 2010-05-17 20:08:32 UTC (rev 112428)
+++ zc.zlibstorage/branches/dev/src/zc/zlibstorage/component.xml 2010-05-17 20:26:58 UTC (rev 112429)
@@ -4,4 +4,9 @@
<section type="ZODB.storage" name="*" attribute="base" required="yes" />
<key name="compress" datatype="boolean" required="no" />
</sectiontype>
+ <sectiontype name="serverzlibstorage" datatype="zc.zlibstorage.ZConfigServer"
+ implements="ZODB.storage">
+ <section type="ZODB.storage" name="*" attribute="base" required="yes" />
+ <key name="compress" datatype="boolean" required="no" />
+ </sectiontype>
</component>
Modified: zc.zlibstorage/branches/dev/src/zc/zlibstorage/tests.py
===================================================================
--- zc.zlibstorage/branches/dev/src/zc/zlibstorage/tests.py 2010-05-17 20:08:32 UTC (rev 112428)
+++ zc.zlibstorage/branches/dev/src/zc/zlibstorage/tests.py 2010-05-17 20:26:58 UTC (rev 112429)
@@ -17,11 +17,14 @@
import transaction
import unittest
import zc.zlibstorage
+import ZEO.tests.testZEO
import zlib
import ZODB.config
import ZODB.FileStorage
import ZODB.interfaces
import ZODB.MappingStorage
+import ZODB.tests.StorageTestBase
+import ZODB.tests.testFileStorage
import ZODB.utils
import zope.interface.verify
@@ -119,62 +122,97 @@
>>> db = ZODB.DB(ZODB.FileStorage.FileStorage('data.fs', blob_dir='blobs'))
>>> conn = db.open()
- >>> conn.root()['a'] = 1
+ >>> conn.root.a = 1
>>> transaction.commit()
- >>> conn.root()['b'] = ZODB.blob.Blob('Hi\nworld.\n')
+ >>> conn.root.b = ZODB.blob.Blob('Hi\nworld.\n')
>>> transaction.commit()
- >>> conn.root()['c'] = conn.root().__class__()
- >>> conn.root()['c']['a'] = conn.root().__class__()
+ >>> conn.root.c = conn.root().__class__((i,i) for i in range(100))
>>> transaction.commit()
>>> db.close()
Now let's open the database compressed:
- >>> db = ZODB.DB(zc.zlibstorage.Storage(
+ >>> db = ZODB.DB(zc.zlibstorage.ZlibStorage(
... ZODB.FileStorage.FileStorage('data.fs', blob_dir='blobs')))
>>> conn = db.open()
>>> conn.root()['a']
1
>>> conn.root()['b'].open().read()
'Hi\nworld.\n'
- >>> del conn.root()['b']
+ >>> conn.root()['b'] = ZODB.blob.Blob('Hello\nworld.\n')
>>> transaction.commit()
>>> db.close()
Having updated the root, it is now compressed. To see this, we'll
open it as a file storage and inspect the record for object 0:
- >>> s = ZODB.FileStorage.FileStorage('data.fs')
- >>> data, _ = s.load('\0'*8)
+ >>> storage = ZODB.FileStorage.FileStorage('data.fs')
+ >>> data, _ = storage.load('\0'*8)
>>> data[:2] == '.z'
True
>>> zlib.decompress(data[2:])[:50]
'cpersistent.mapping\nPersistentMapping\nq\x01.}q\x02U\x04data'
-The blob record is still uncompressed:
+The new blob record is uncompressed because it is too small:
- >>> s.load('\0'*7+'\1')
+ >>> storage.load('\0'*7+'\3')[0]
+ 'cZODB.blob\nBlob\nq\x01.N.'
- >>> s.close()
+Records that we didn't modify remain uncompressed
-Let's try packing the file 2 ways:
+ >>> storage.load('\0'*7+'\2')[0] # doctest: +ELLIPSIS
+ 'cpersistent.mapping\nPersistentMapping...
+
+ >>> storage.close()
+
+Let's try packing the file 4 ways:
+
- using the compressed storage:
>>> open('data.fs.save', 'wb').write(open('data.fs', 'rb').read())
- >>> db = ZODB.DB(zc.zlibstorage.Storage(
+ >>> db = ZODB.DB(zc.zlibstorage.ZlibStorage(
... ZODB.FileStorage.FileStorage('data.fs', blob_dir='blobs')))
>>> db.pack()
>>> sorted(ZODB.utils.u64(i[0]) for i in record_iter(db.storage))
+ [0, 2, 3]
+ >>> db.close()
-- and using the storage in non-compress mode:
+- using the storage in non-compress mode:
- >>> open('data.fs.save', 'wb').write(open('data.fs', 'rb').read())
- >>> db = ZODB.DB(zc.zlibstorage(
+ >>> open('data.fs', 'wb').write(open('data.fs.save', 'rb').read())
+ >>> db = ZODB.DB(zc.zlibstorage.ZlibStorage(
... ZODB.FileStorage.FileStorage('data.fs', blob_dir='blobs'),
... compress=False))
+
>>> db.pack()
>>> sorted(ZODB.utils.u64(i[0]) for i in record_iter(db.storage))
+ [0, 2, 3]
+ >>> db.close()
+
+- using the server storage:
+
+ >>> open('data.fs', 'wb').write(open('data.fs.save', 'rb').read())
+ >>> db = ZODB.DB(zc.zlibstorage.ServerZlibStorage(
+ ... ZODB.FileStorage.FileStorage('data.fs', blob_dir='blobs'),
+ ... compress=False))
+
+ >>> db.pack()
+ >>> sorted(ZODB.utils.u64(i[0]) for i in record_iter(db.storage))
+ [0, 2, 3]
+ >>> db.close()
+
+- using the server storage in non-compress mode:
+
+ >>> open('data.fs', 'wb').write(open('data.fs.save', 'rb').read())
+ >>> db = ZODB.DB(zc.zlibstorage.ServerZlibStorage(
+ ... ZODB.FileStorage.FileStorage('data.fs', blob_dir='blobs'),
+ ... compress=False))
+
+ >>> db.pack()
+ >>> sorted(ZODB.utils.u64(i[0]) for i in record_iter(db.storage))
+ [0, 2, 3]
+ >>> db.close()
"""
class Dummy:
@@ -202,7 +240,7 @@
"""
Make sure the wrapping methods do what's expected.
- >>> s = zc.zlibstorage.Storage(ZODB.MappingStorage.MappingStorage())
+ >>> s = zc.zlibstorage.ZlibStorage(ZODB.MappingStorage.MappingStorage())
>>> zope.interface.verify.verifyObject(ZODB.interfaces.IStorageWrapper, s)
True
@@ -260,14 +298,101 @@
def record_iter(store):
next = None
while 1:
- oid, tid, data, next = storage.record_iternext(next)
+ oid, tid, data, next = store.record_iternext(next)
yield oid, tid, data
if next is None:
break
+class FileStorageZlibTests(ZODB.tests.testFileStorage.FileStorageTests):
+
+ def open(self, **kwargs):
+ self._storage = zc.zlibstorage.ZlibStorage(
+ ZODB.FileStorage.FileStorage('FileStorageTests.fs',**kwargs))
+
+class FileStorageZlibTestsWithBlobsEnabled(
+ ZODB.tests.testFileStorage.FileStorageTests):
+
+ def open(self, **kwargs):
+ if 'blob_dir' not in kwargs:
+ kwargs = kwargs.copy()
+ kwargs['blob_dir'] = 'blobs'
+ ZODB.tests.testFileStorage.FileStorageTests.open(self, **kwargs)
+ self._storage = zc.zlibstorage.ZlibStorage(self._storage)
+
+class FileStorageZlibRecoveryTest(
+ ZODB.tests.testFileStorage.FileStorageRecoveryTest):
+
+ def setUp(self):
+ ZODB.tests.StorageTestBase.StorageTestBase.setUp(self)
+ self._storage = zc.zlibstorage.ZlibStorage(
+ ZODB.FileStorage.FileStorage("Source.fs", create=True))
+ self._dst = zc.zlibstorage.ZlibStorage(
+ ZODB.FileStorage.FileStorage("Dest.fs", create=True))
+
+
+
+class FileStorageZEOZlibTests(ZEO.tests.testZEO.FileStorageTests):
+ _expected_interfaces = (
+ ('ZODB.interfaces', 'IStorageRestoreable'),
+ ('ZODB.interfaces', 'IStorageIteration'),
+ ('ZODB.interfaces', 'IStorageUndoable'),
+ ('ZODB.interfaces', 'IStorageCurrentRecordIteration'),
+ ('ZODB.interfaces', 'IExternalGC'),
+ ('ZODB.interfaces', 'IStorage'),
+ ('ZODB.interfaces', 'IStorageWrapper'),
+ ('zope.interface', 'Interface'),
+ )
+
+ def getConfig(self):
+ return """\
+ %import zc.zlibstorage
+ <zlibstorage>
+ <filestorage 1>
+ path Data.fs
+ </filestorage>
+ </zlibstorage>
+ """
+
+class FileStorageClientZlibTests(FileStorageZlibTests):
+
+ def getConfig(self):
+ return """\
+ %import zc.zlibstorage
+ <serverzlibstorage>
+ <filestorage 1>
+ path Data.fs
+ </filestorage>
+ </serverzlibstorage>
+ """
+
+ def _wrap_client(self, client):
+ return zc.zlibstorage.ZlibStorage(client)
+
+
+
+
+
+
def test_suite():
- return unittest.TestSuite((
+ suite = unittest.TestSuite()
+ for class_ in (
+ FileStorageZlibTests,
+ FileStorageZlibTestsWithBlobsEnabled,
+ FileStorageZlibRecoveryTest,
+ FileStorageZlibTests,
+ FileStorageClientZlibTests,
+ ):
+ s = unittest.makeSuite(class_, "check")
+ s.layer = ZODB.tests.util.MininalTestLayer(
+ 'zlibstoragetests.%s' % class_.__name__)
+ suite.addTest(s)
+
+
+ suite.addTest(
doctest.DocTestSuite(
- setUp=setupstack.setUpDirectory, tearDown=setupstack.tearDown),
- ))
+ setUp=setupstack.setUpDirectory, tearDown=setupstack.tearDown
+ )
+ )
+ return suite
+
More information about the checkins
mailing list