[Checkins] SVN: zc.zlibstorage/branches/dev/ checkpoint
Jim Fulton
jim at zope.com
Fri May 14 17:02:00 EDT 2010
Log message for revision 112321:
checkpoint
Changed:
U zc.zlibstorage/branches/dev/README.txt
U zc.zlibstorage/branches/dev/buildout.cfg
U zc.zlibstorage/branches/dev/setup.py
A zc.zlibstorage/branches/dev/src/zc/zlibstorage/
A zc.zlibstorage/branches/dev/src/zc/zlibstorage/__init__.py
A zc.zlibstorage/branches/dev/src/zc/zlibstorage/component.xml
A zc.zlibstorage/branches/dev/src/zc/zlibstorage/tests.py
-=-
Modified: zc.zlibstorage/branches/dev/README.txt
===================================================================
--- zc.zlibstorage/branches/dev/README.txt 2010-05-14 20:52:53 UTC (rev 112320)
+++ zc.zlibstorage/branches/dev/README.txt 2010-05-14 21:02:00 UTC (rev 112321)
@@ -1,10 +1,9 @@
-Title Here
-**********
+zlib Storage
+************
+A ZODB wrapper storage that compresses data records using zlib.
-To learn more, see
-
Changes
*******
Modified: zc.zlibstorage/branches/dev/buildout.cfg
===================================================================
--- zc.zlibstorage/branches/dev/buildout.cfg 2010-05-14 20:52:53 UTC (rev 112320)
+++ zc.zlibstorage/branches/dev/buildout.cfg 2010-05-14 21:02:00 UTC (rev 112321)
@@ -1,10 +1,10 @@
[buildout]
-develop = .
+develop = . trunk
parts = test py
[test]
recipe = zc.recipe.testrunner
-eggs =
+eggs = zc.zlibstorage
[py]
recipe = zc.recipe.egg
Modified: zc.zlibstorage/branches/dev/setup.py
===================================================================
--- zc.zlibstorage/branches/dev/setup.py 2010-05-14 20:52:53 UTC (rev 112320)
+++ zc.zlibstorage/branches/dev/setup.py 2010-05-14 21:02:00 UTC (rev 112321)
@@ -11,9 +11,9 @@
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
-name, version = 'zc.', '0'
+name, version = 'zc.zlibstorage', '0'
-install_requires = ['setuptools']
+install_requires = ['ZODB3', 'setuptools']
extras_require = dict(test=['zope.testing'])
entry_points = """
Added: zc.zlibstorage/branches/dev/src/zc/zlibstorage/__init__.py
===================================================================
--- zc.zlibstorage/branches/dev/src/zc/zlibstorage/__init__.py (rev 0)
+++ zc.zlibstorage/branches/dev/src/zc/zlibstorage/__init__.py 2010-05-14 21:02:00 UTC (rev 112321)
@@ -0,0 +1,148 @@
+##############################################################################
+#
+# Copyright (c) 2010 Zope Foundation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
+import zlib
+import ZODB.interfaces
+import zope.interface
+
+class Storage(object):
+
+ zope.interface.implements(ZODB.interfaces.IStorageWrapper)
+
+ def __init__(self, base, compress=True):
+ self.base = base
+ self.compress = compress
+
+ for name in (
+ 'close', 'getName', 'getSize', 'history', 'isReadOnly',
+ 'lastTransaction', 'new_oid', 'sortKey',
+ 'tpc_abort', 'tpc_begin', 'tpc_finish', 'tpc_vote',
+ 'loadBlob', 'openCommittedBlobFile', 'temporaryDirectory',
+ 'supportsUndo', 'undo', 'undoLog', 'undoInfo',
+ ):
+ v = getattr(base, name, None)
+ if v is not None:
+ setattr(self, name, v)
+
+ zope.interface.directlyProvides(self, zope.interface.providedBy(base))
+
+ def _transform(self, data):
+ if self.compress:
+ compressed = '.z'+zlib.compress(data)
+ if len(compressed) < len(data):
+ return compressed
+ return data
+
+ def _untransform(self, data):
+ if data[:2] == '.z':
+ return zlib.decompress(data[2:])
+ return data
+
+ def __len__(self):
+ return len(self.base)
+
+ def load(self, oid, version=''):
+ data, serial = self.base.load(oid, version)
+ return self._untransform(data), serial
+
+ def loadBefore(self, oid, tid):
+ r = self.base.loadBefore(oid, tid)
+ if r is not None:
+ data, serial, after = r
+ return self._untransform(data), serial, after
+ else:
+ return r
+
+ def loadSerial(self, oid, serial):
+ return self._untransform(self.base.loadSerial(oid, serial))
+
+ def pack(self, pack_time, referencesf):
+ _untransform = self._untransform
+ def refs(p, oids=None):
+ return referencesf(_untransform(p), oids)
+
+ def registerDB(self, db):
+ self.db = db
+
+ def store(self, oid, serial, data, version, transaction):
+ if self.compress:
+ data = self._transform(data)
+ return self.base.store(oid, serial, data, version, transaction)
+
+ def restore(self, oid, serial, data, version, prev_txn, transaction):
+ if self.compress:
+ data = self._transform(data)
+ return self.base.restore(
+ oid, serial, data, version, prev_txn, transaction)
+
+ def iterator(self, start=None, stop=None):
+ for t in self.base.iterator(start, end):
+ yield Transaction(t)
+
+ def storeBlob(self, oid, oldserial, data, blobfilename, version,
+ transaction):
+ if self.compress:
+ data = self._transform(data)
+ return self.base.storeBlob(oid, oldserial, data, blobfilename, version,
+ transaction)
+
+ def restoreBlob(self, oid, serial, data, blobfilename, prev_txn,
+ transaction):
+ if self.compress:
+ data = self._transform(data)
+ return self.base.restoreBlob(oid, serial, data, blobfilename, prev_txn,
+ transaction)
+
+ def invalidateCache(self):
+ return self.db.invalidateCache()
+
+ def invalidate(self, transaction_id, oids, version=''):
+ return self.db.invalidate(transaction_id, oids, version)
+
+ def references(self, record, oids=None):
+ return self.db.references(self._untransform(record), oids)
+
+ def transform_record_data(self, data):
+ return self._transform(self.db.transform_record_data(data))
+
+ def untransform_record_data(self, data):
+ return self.db.untransform_record_data(self._untransform(data))
+
+
+class Transaction(object):
+
+ def __init__(self, store, trans):
+ self.__store = store
+ self.__trans = trans
+
+ def __iter__(self):
+ for r in self.__trans:
+ if r.data:
+ r.data = self.__store._untransform(r.data)
+ yield r
+
+ def __getattr__(self, name):
+ return getattr(self.__trans)
+
+class ZConfig:
+
+ def __init__(self, config):
+ self.config = config
+ self.name = config.getSectionName()
+
+ def open(self):
+ base = self.config.base.open()
+ compress = self.config.compress
+ if compress is None:
+ compress = True
+ return Storage(base, compress)
Property changes on: zc.zlibstorage/branches/dev/src/zc/zlibstorage/__init__.py
___________________________________________________________________
Added: svn:keywords
+ Id
Added: svn:eol-style
+ native
Added: zc.zlibstorage/branches/dev/src/zc/zlibstorage/component.xml
===================================================================
--- zc.zlibstorage/branches/dev/src/zc/zlibstorage/component.xml (rev 0)
+++ zc.zlibstorage/branches/dev/src/zc/zlibstorage/component.xml 2010-05-14 21:02:00 UTC (rev 112321)
@@ -0,0 +1,7 @@
+<component>
+ <sectiontype name="zlibstorage" datatype="zc.zlibstorage.ZConfig"
+ implements="ZODB.storage">
+ <section type="ZODB.storage" name="*" attribute="base" required="yes" />
+ <key name="compress" datatype="boolean" required="no" />
+ </sectiontype>
+</component>
Property changes on: zc.zlibstorage/branches/dev/src/zc/zlibstorage/component.xml
___________________________________________________________________
Added: svn:eol-style
+ native
Added: zc.zlibstorage/branches/dev/src/zc/zlibstorage/tests.py
===================================================================
--- zc.zlibstorage/branches/dev/src/zc/zlibstorage/tests.py (rev 0)
+++ zc.zlibstorage/branches/dev/src/zc/zlibstorage/tests.py 2010-05-14 21:02:00 UTC (rev 112321)
@@ -0,0 +1,273 @@
+##############################################################################
+#
+# Copyright (c) 2010 Zope Foundation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
+from zope.testing import setupstack
+
+import doctest
+import transaction
+import unittest
+import zc.zlibstorage
+import zlib
+import ZODB.config
+import ZODB.FileStorage
+import ZODB.interfaces
+import ZODB.MappingStorage
+import ZODB.utils
+import zope.interface.verify
+
+
+def test_config():
+ r"""
+
+To configure a zlibstorage, import zc.zlibstorage and use the
+zlibstorage tag:
+
+ >>> config = '''
+ ... %import zc.zlibstorage
+ ... <zodb>
+ ... <zlibstorage>
+ ... <filestorage>
+ ... path data.fs
+ ... blob-dir blobs
+ ... </filestorage>
+ ... </zlibstorage>
+ ... </zodb>
+ ... '''
+ >>> db = ZODB.config.databaseFromString(config)
+
+ >>> conn = db.open()
+ >>> conn.root()['a'] = 1
+ >>> transaction.commit()
+ >>> conn.root()['b'] = ZODB.blob.Blob('Hi\nworld.\n')
+ >>> transaction.commit()
+
+ >>> db.close()
+
+ >>> db = ZODB.config.databaseFromString(config)
+ >>> conn = db.open()
+ >>> conn.root()['a']
+ 1
+ >>> conn.root()['b'].open().read()
+ 'Hi\nworld.\n'
+ >>> db.close()
+
+After putting some data in, the records will be compressed, unless
+doing so would make them bigger:
+
+ >>> for t in ZODB.FileStorage.FileIterator('data.fs'):
+ ... for r in t:
+ ... data = r.data
+ ... if r.data[:2] != '.z':
+ ... if len(zlib.compress(data))+2 < len(data):
+ ... print 'oops', `r.oid`
+ ... else: _ = zlib.decompress(data[2:])
+ """
+
+def test_config_no_compress():
+ r"""
+
+You can disable compression.
+
+ >>> config = '''
+ ... %import zc.zlibstorage
+ ... <zodb>
+ ... <zlibstorage>
+ ... compress no
+ ... <filestorage>
+ ... path data.fs
+ ... blob-dir blobs
+ ... </filestorage>
+ ... </zlibstorage>
+ ... </zodb>
+ ... '''
+ >>> db = ZODB.config.databaseFromString(config)
+
+ >>> conn = db.open()
+ >>> conn.root()['a'] = 1
+ >>> transaction.commit()
+ >>> conn.root()['b'] = ZODB.blob.Blob('Hi\nworld.\n')
+ >>> transaction.commit()
+
+ >>> db.close()
+
+Since we didn't compress, we can open the storage using a plain file storage:
+
+ >>> db = ZODB.DB(ZODB.FileStorage.FileStorage('data.fs', blob_dir='blobs'))
+ >>> conn = db.open()
+ >>> conn.root()['a']
+ 1
+ >>> conn.root()['b'].open().read()
+ 'Hi\nworld.\n'
+ >>> db.close()
+ """
+
+def test_mixed_compressed_and_uncompressed_and_packing():
+ r"""
+We can deal with a mixture of compressed and uncompressed data.
+
+First, we'll create an existing file storage:
+
+ >>> db = ZODB.DB(ZODB.FileStorage.FileStorage('data.fs', blob_dir='blobs'))
+ >>> conn = db.open()
+ >>> conn.root()['a'] = 1
+ >>> transaction.commit()
+ >>> conn.root()['b'] = ZODB.blob.Blob('Hi\nworld.\n')
+ >>> transaction.commit()
+ >>> conn.root()['c'] = conn.root().__class__()
+ >>> conn.root()['c']['a'] = conn.root().__class__()
+ >>> transaction.commit()
+ >>> db.close()
+
+Now let's open the database compressed:
+
+ >>> db = ZODB.DB(zc.zlibstorage.Storage(
+ ... ZODB.FileStorage.FileStorage('data.fs', blob_dir='blobs')))
+ >>> conn = db.open()
+ >>> conn.root()['a']
+ 1
+ >>> conn.root()['b'].open().read()
+ 'Hi\nworld.\n'
+ >>> del conn.root()['b']
+ >>> transaction.commit()
+ >>> db.close()
+
+Having updated the root, it is now compressed. To see this, we'll
+open it as a file storage and inspect the record for object 0:
+
+ >>> s = ZODB.FileStorage.FileStorage('data.fs')
+ >>> data, _ = s.load('\0'*8)
+ >>> data[:2] == '.z'
+ True
+ >>> zlib.decompress(data[2:])[:50]
+ 'cpersistent.mapping\nPersistentMapping\nq\x01.}q\x02U\x04data'
+
+The blob record is still uncompressed:
+
+ >>> s.load('\0'*7+'\1')
+
+ >>> s.close()
+
+Let's try packing the file 2 ways:
+
+- using the compressed storage:
+
+ >>> open('data.fs.save', 'wb').write(open('data.fs', 'rb').read())
+ >>> db = ZODB.DB(zc.zlibstorage.Storage(
+ ... ZODB.FileStorage.FileStorage('data.fs', blob_dir='blobs')))
+ >>> db.pack()
+ >>> sorted(ZODB.utils.u64(i[0]) for i in record_iter(db.storage))
+
+- and using the storage in non-compress mode:
+
+ >>> open('data.fs.save', 'wb').write(open('data.fs', 'rb').read())
+ >>> db = ZODB.DB(zc.zlibstorage(
+ ... ZODB.FileStorage.FileStorage('data.fs', blob_dir='blobs'),
+ ... compress=False))
+ >>> db.pack()
+ >>> sorted(ZODB.utils.u64(i[0]) for i in record_iter(db.storage))
+ """
+
+class Dummy:
+
+ def invalidateCache(self):
+ print 'invalidateCache called'
+
+ def invalidate(self, *args):
+ print 'invalidate', args
+
+ def references(self, record, oids=None):
+ if oids is None:
+ oids = []
+ oids.extend(record.decode('hex').split())
+ return oids
+
+ def transform_record_data(self, data):
+ return data.encode('hex')
+
+ def untransform_record_data(self, data):
+ return data.decode('hex')
+
+
+def test_wrapping():
+ """
+Make sure the wrapping methods do what's expected.
+
+ >>> s = zc.zlibstorage.Storage(ZODB.MappingStorage.MappingStorage())
+ >>> zope.interface.verify.verifyObject(ZODB.interfaces.IStorageWrapper, s)
+ True
+
+ >>> s.registerDB(Dummy())
+ >>> s.invalidateCache()
+ invalidateCache called
+
+ >>> s.invalidate('1', range(3), '')
+ invalidate ('1', [0, 1, 2], '')
+
+ >>> data = ' '.join(map(str, range(9)))
+ >>> transformed = s.transform_record_data(data)
+ >>> transformed == '.z'+zlib.compress(data.encode('hex'))
+ True
+
+ >>> s.untransform_record_data(transformed) == data
+ True
+
+ >>> s.references(transformed)
+ ['0', '1', '2', '3', '4', '5', '6', '7', '8']
+
+ >>> l = range(3)
+ >>> s.references(transformed, l)
+ [0, 1, 2, '0', '1', '2', '3', '4', '5', '6', '7', '8']
+
+ >>> l
+ [0, 1, 2, '0', '1', '2', '3', '4', '5', '6', '7', '8']
+
+If the data are small or otherwise not compressable, it is left as is:
+
+ >>> data = ' '.join(map(str, range(2)))
+ >>> transformed = s.transform_record_data(data)
+ >>> transformed == '.z'+zlib.compress(data.encode('hex'))
+ False
+
+ >>> transformed == data.encode('hex')
+ True
+
+ >>> s.untransform_record_data(transformed) == data
+ True
+
+ >>> s.references(transformed)
+ ['0', '1']
+
+ >>> l = range(3)
+ >>> s.references(transformed, l)
+ [0, 1, 2, '0', '1']
+
+ >>> l
+ [0, 1, 2, '0', '1']
+
+
+ """
+
+def record_iter(store):
+ next = None
+ while 1:
+ oid, tid, data, next = storage.record_iternext(next)
+ yield oid, tid, data
+ if next is None:
+ break
+
+
+def test_suite():
+ return unittest.TestSuite((
+ doctest.DocTestSuite(
+ setUp=setupstack.setUpDirectory, tearDown=setupstack.tearDown),
+ ))
Property changes on: zc.zlibstorage/branches/dev/src/zc/zlibstorage/tests.py
___________________________________________________________________
Added: svn:keywords
+ Id
Added: svn:eol-style
+ native
More information about the checkins
mailing list