[Checkins] SVN: relstorage/trunk/relstorage/ Added blob cache tests by adapting the tests from ZEO. Also, use the ZEO implementation of BlobCacheLayout when it exists.
Shane Hathaway
shane at hathawaymix.org
Tue Mar 1 16:51:05 EST 2011
Log message for revision 120654:
Added blob cache tests by adapting the tests from ZEO. Also, use the ZEO implementation of BlobCacheLayout when it exists.
Changed:
U relstorage/trunk/relstorage/blobhelper.py
A relstorage/trunk/relstorage/tests/blob/blob_cache.test
U relstorage/trunk/relstorage/tests/blob/testblob.py
U relstorage/trunk/relstorage/tests/reltestbase.py
U relstorage/trunk/relstorage/tests/test_blobhelper.py
U relstorage/trunk/relstorage/tests/testmysql.py
U relstorage/trunk/relstorage/tests/testoracle.py
U relstorage/trunk/relstorage/tests/testpostgresql.py
A relstorage/trunk/relstorage/tests/util.py
-=-
Modified: relstorage/trunk/relstorage/blobhelper.py
===================================================================
--- relstorage/trunk/relstorage/blobhelper.py 2011-03-01 21:16:38 UTC (rev 120653)
+++ relstorage/trunk/relstorage/blobhelper.py 2011-03-01 21:51:04 UTC (rev 120654)
@@ -71,6 +71,25 @@
return False
+try:
+ from ZEO.ClientStorage import BlobCacheLayout
+except ImportError:
+
+ class BlobCacheLayout(object):
+
+ size = 997
+
+ def oid_to_path(self, oid):
+ return str(utils.u64(oid) % self.size)
+
+ def getBlobFilePath(self, oid, tid):
+ base, rem = divmod(utils.u64(oid), self.size)
+ return os.path.join(
+ str(rem),
+ "%s.%s%s" % (base, tid.encode('hex'), ZODB.blob.BLOB_SUFFIX)
+ )
+
+
class BlobHelper(object):
"""Blob support for RelStorage.
@@ -404,21 +423,6 @@
# and 3.7 for a few years.
-class BlobCacheLayout(object):
-
- size = 997
-
- def oid_to_path(self, oid):
- return str(utils.u64(oid) % self.size)
-
- def getBlobFilePath(self, oid, tid):
- base, rem = divmod(utils.u64(oid), self.size)
- return os.path.join(
- str(rem),
- "%s.%s%s" % (base, tid.encode('hex'), ZODB.blob.BLOB_SUFFIX)
- )
-
-
def _accessed(filename):
try:
os.utime(filename, (time.time(), os.stat(filename).st_mtime))
Added: relstorage/trunk/relstorage/tests/blob/blob_cache.test
===================================================================
--- relstorage/trunk/relstorage/tests/blob/blob_cache.test (rev 0)
+++ relstorage/trunk/relstorage/tests/blob/blob_cache.test 2011-03-01 21:51:04 UTC (rev 120654)
@@ -0,0 +1,146 @@
+Caching of blob data
+====================
+
+(This test is adapted from ZEO/tests/zeo_blob_cache.test in ZODB3.)
+
+We support 2 modes for providing clients access to blob data:
+
+shared
+ Blob data are shared via a network file system. The client shares
+ a common blob directory with the server.
+
+non-shared
+ Blob data are loaded from the database and cached locally.
+ A maximum size for the blob data can be set and data are removed
+ when the size is exceeded.
+
+In this test, we'll demonstrate that blobs data are removed from a blob
+cache when the amount of data stored exceeds a given limit.
+
+Let's start by setting up some data:
+
+ >>> blob_storage = create_storage(blob_dir='blobs',
+ ... blob_cache_size=3000, blob_cache_size_check=10)
+ >>> from ZODB.DB import DB
+ >>> db = DB(blob_storage)
+
+Here, we passed a blob_cache_size parameter, which specifies a target
+blob cache size. This is not a hard limit, but rather a target. It
+defaults to no limit. We also passed a blob_cache_size_check
+option. The blob_cache_size_check option specifies the number of
+bytes, as a percent of the target that can be written or downloaded
+from the server before the cache size is checked. The
+blob_cache_size_check option defaults to 100. We passed 10, to check
+after writing 10% of the target size.
+
+.. We're going to wait for any threads we started to finish, so...
+
+ >>> import threading
+ >>> old_threads = list(threading.enumerate())
+
+We want to check for name collisions in the blob cache dir. We'll try
+to provoke name collisions by reducing the number of cache directory
+subdirectories.
+
+ >>> import relstorage.blobhelper
+ >>> orig_blob_cache_layout_size = relstorage.blobhelper.BlobCacheLayout.size
+ >>> relstorage.blobhelper.BlobCacheLayout.size = 11
+
+Now, let's write some data:
+
+ >>> import ZODB.blob, transaction, time
+ >>> conn = db.open()
+ >>> for i in range(1, 101):
+ ... conn.root()[i] = ZODB.blob.Blob()
+ ... conn.root()[i].open('w').write(chr(i)*100)
+ >>> transaction.commit()
+
+We've committed 10000 bytes of data, but our target size is 3000. We
+expect to have not much more than the target size in the cache blob
+directory.
+
+ >>> import os
+ >>> def cache_size(d):
+ ... size = 0
+ ... for base, dirs, files in os.walk(d):
+ ... for f in files:
+ ... if f.endswith('.blob'):
+ ... try:
+ ... size += os.stat(os.path.join(base, f)).st_size
+ ... except OSError:
+ ... if os.path.exists(os.path.join(base, f)):
+ ... raise
+ ... return size
+
+ >>> cache_size('blobs') > 2000
+ True
+ >>> def check():
+ ... return cache_size('blobs') < 5000
+ >>> def onfail():
+ ... return cache_size('blobs')
+
+ >>> from relstorage.tests.util import wait_until
+ >>> wait_until("size is reduced", check, 99, onfail)
+
+If we read all of the blobs, data will be downloaded again, as
+necessary, but the cache size will remain not much bigger than the
+target:
+
+ >>> for i in range(1, 101):
+ ... data = conn.root()[i].open().read()
+ ... if data != chr(i)*100:
+ ... print 'bad data', `chr(i)`, `data`
+
+ >>> wait_until("size is reduced", check, 99, onfail)
+
+ >>> for i in range(1, 101):
+ ... data = conn.root()[i].open().read()
+ ... if data != chr(i)*100:
+ ... print 'bad data', `chr(i)`, `data`
+
+ >>> for i in range(1, 101):
+ ... data = conn.root()[i].open('c').read()
+ ... if data != chr(i)*100:
+ ... print 'bad data', `chr(i)`, `data`
+
+ >>> wait_until("size is reduced", check, 99, onfail)
+
+Now let see if we can stress things a bit. We'll create many clients
+and get them to pound on the blobs all at once to see if we can
+provoke problems:
+
+ >>> import threading, random
+ >>> def run():
+ ... conn = db.open()
+ ... for i in range(300):
+ ... time.sleep(0)
+ ... i = random.randint(1, 100)
+ ... data = conn.root()[i].open().read()
+ ... if data != chr(i)*100:
+ ... print 'bad data', `chr(i)`, `data`
+ ... i = random.randint(1, 100)
+ ... data = conn.root()[i].open('c').read()
+ ... if data != chr(i)*100:
+ ... print 'bad data', `chr(i)`, `data`
+ ... conn.close()
+
+ >>> threads = [threading.Thread(target=run) for i in range(10)]
+ >>> for thread in threads:
+ ... thread.setDaemon(True)
+ >>> for thread in threads:
+ ... thread.start()
+ >>> for thread in threads:
+ ... thread.join(99)
+ ... if thread.isAlive():
+ ... print "Can't join thread."
+
+ >>> wait_until("size is reduced", check, 99, onfail)
+
+.. cleanup
+
+ >>> for thread in threading.enumerate():
+ ... if thread not in old_threads:
+ ... thread.join(33)
+
+ >>> db.close()
+ >>> relstorage.blobhelper.BlobCacheLayout.size = orig_blob_cache_layout_size
Modified: relstorage/trunk/relstorage/tests/blob/testblob.py
===================================================================
--- relstorage/trunk/relstorage/tests/blob/testblob.py 2011-03-01 21:16:38 UTC (rev 120653)
+++ relstorage/trunk/relstorage/tests/blob/testblob.py 2011-03-01 21:51:04 UTC (rev 120654)
@@ -494,6 +494,7 @@
test_undo=True,
keep_history=True,
pack_test_name='blob_packing.txt',
+ test_blob_cache=False,
):
"""Return a test suite for a generic IBlobStorage.
@@ -502,10 +503,10 @@
def setup(test):
setUp(test)
- def create_storage(name='data', blob_dir=None):
+ def create_storage(name='data', blob_dir=None, **kw):
if blob_dir is None:
blob_dir = '%s.bobs' % name
- return factory(name, blob_dir)
+ return factory(name, blob_dir, **kw)
test.globs['create_storage'] = create_storage
@@ -521,6 +522,11 @@
pack_test_name,
setUp=setup, tearDown=tearDown,
))
+ if test_blob_cache:
+ suite.addTest(doctest.DocFileSuite(
+ "blob_cache.test",
+ setUp=setup, tearDown=tearDown,
+ ))
suite.addTest(doctest.DocTestSuite(
setUp=setup, tearDown=tearDown,
checker = zope.testing.renormalizing.RENormalizing([
@@ -529,10 +535,10 @@
]),
))
- def create_storage(self, name='data', blob_dir=None):
+ def create_storage(self, name='data', blob_dir=None, **kw):
if blob_dir is None:
blob_dir = '%s.bobs' % name
- return factory(name, blob_dir)
+ return factory(name, blob_dir, **kw)
def add_test_based_on_test_class(class_):
new_class = class_.__class__(
Modified: relstorage/trunk/relstorage/tests/reltestbase.py
===================================================================
--- relstorage/trunk/relstorage/tests/reltestbase.py 2011-03-01 21:16:38 UTC (rev 120653)
+++ relstorage/trunk/relstorage/tests/reltestbase.py 2011-03-01 21:51:04 UTC (rev 120654)
@@ -694,6 +694,7 @@
finally:
db.close()
+
class DoubleCommitter(Persistent):
"""A crazy persistent class that changes self in __getstate__"""
def __getstate__(self):
Modified: relstorage/trunk/relstorage/tests/test_blobhelper.py
===================================================================
--- relstorage/trunk/relstorage/tests/test_blobhelper.py 2011-03-01 21:16:38 UTC (rev 120653)
+++ relstorage/trunk/relstorage/tests/test_blobhelper.py 2011-03-01 21:51:04 UTC (rev 120654)
@@ -371,54 +371,6 @@
self.assertFalse(os.path.exists(fn2))
-class BlobCacheCheckerTest(unittest.TestCase):
-
- def _class(self):
- from relstorage.blobhelper import BlobCacheChecker
- return BlobCacheChecker
-
- def _make(self, *args, **kw):
- return self._class()(*args, **kw)
-
-
-class BlobCacheLayoutTest(unittest.TestCase):
-
- def _class(self):
- from relstorage.blobhelper import BlobCacheLayout
- return BlobCacheLayout
-
- def _make(self, *args, **kw):
- return self._class()(*args, **kw)
-
-
-class AccessedTest(unittest.TestCase):
-
- def _call(self, *args, **kw):
- from relstorage.blobhelper import _accessed
- return _accessed(*args, **kw)
-
-
-class CheckBlobCacheSizeTest(unittest.TestCase):
-
- def _call(self, *args, **kw):
- from relstorage.blobhelper import _check_blob_cache_size
- return _check_blob_cache_size(*args, **kw)
-
-
-class LockBlobTest(unittest.TestCase):
-
- def _call(self, *args, **kw):
- from relstorage.blobhelper import _lock_blob
- return _lock_blob(*args, **kw)
-
-
-class HasFilesTest(unittest.TestCase):
-
- def _call(self, *args, **kw):
- from relstorage.blobhelper import _has_files
- return _has_files(*args, **kw)
-
-
def test_suite():
try:
import ZODB.blob
@@ -430,12 +382,6 @@
for klass in [
IsBlobRecordTest,
BlobHelperTest,
- BlobCacheCheckerTest,
- BlobCacheLayoutTest,
- AccessedTest,
- CheckBlobCacheSizeTest,
- LockBlobTest,
- HasFilesTest,
]:
suite.addTest(unittest.makeSuite(klass, "test"))
Modified: relstorage/trunk/relstorage/tests/testmysql.py
===================================================================
--- relstorage/trunk/relstorage/tests/testmysql.py 2011-03-01 21:16:38 UTC (rev 120653)
+++ relstorage/trunk/relstorage/tests/testmysql.py 2011-03-01 21:51:04 UTC (rev 120654)
@@ -166,7 +166,7 @@
for keep_history in (False, True):
def create_storage(name, blob_dir,
shared_blob_dir=shared_blob_dir,
- keep_history=keep_history):
+ keep_history=keep_history, **kw):
from relstorage.storage import RelStorage
from relstorage.adapters.mysql import MySQLAdapter
db = db_names[name]
@@ -176,7 +176,7 @@
keep_history=keep_history,
shared_blob_dir=shared_blob_dir,
blob_dir=os.path.abspath(blob_dir),
- )
+ **kw)
adapter = MySQLAdapter(
options=options,
db=db,
@@ -207,7 +207,8 @@
test_packing=test_packing,
test_undo=keep_history,
pack_test_name=pack_test_name,
- ))
+ test_blob_cache=(not shared_blob_dir),
+ ))
return suite
Modified: relstorage/trunk/relstorage/tests/testoracle.py
===================================================================
--- relstorage/trunk/relstorage/tests/testoracle.py 2011-03-01 21:16:38 UTC (rev 120653)
+++ relstorage/trunk/relstorage/tests/testoracle.py 2011-03-01 21:51:04 UTC (rev 120654)
@@ -175,7 +175,7 @@
for keep_history in (False, True):
def create_storage(name, blob_dir,
shared_blob_dir=shared_blob_dir,
- keep_history=keep_history):
+ keep_history=keep_history, **kw):
from relstorage.storage import RelStorage
from relstorage.adapters.oracle import OracleAdapter
db = db_names[name]
@@ -185,7 +185,7 @@
keep_history=keep_history,
shared_blob_dir=shared_blob_dir,
blob_dir=os.path.abspath(blob_dir),
- )
+ **kw)
adapter = OracleAdapter(
user=db,
password='relstoragetest',
@@ -216,7 +216,8 @@
test_packing=test_packing,
test_undo=keep_history,
pack_test_name=pack_test_name,
- ))
+ test_blob_cache=(not shared_blob_dir),
+ ))
return suite
Modified: relstorage/trunk/relstorage/tests/testpostgresql.py
===================================================================
--- relstorage/trunk/relstorage/tests/testpostgresql.py 2011-03-01 21:16:38 UTC (rev 120653)
+++ relstorage/trunk/relstorage/tests/testpostgresql.py 2011-03-01 21:51:04 UTC (rev 120654)
@@ -163,7 +163,7 @@
for keep_history in (False, True):
def create_storage(name, blob_dir,
shared_blob_dir=shared_blob_dir,
- keep_history=keep_history):
+ keep_history=keep_history, **kw):
from relstorage.storage import RelStorage
from relstorage.adapters.postgresql import PostgreSQLAdapter
db = db_names[name]
@@ -175,7 +175,7 @@
keep_history=keep_history,
shared_blob_dir=shared_blob_dir,
blob_dir=os.path.abspath(blob_dir),
- )
+ **kw)
adapter = PostgreSQLAdapter(dsn=dsn, options=options)
storage = RelStorage(adapter, name=name, options=options)
storage.zap_all()
@@ -201,7 +201,8 @@
test_packing=test_packing,
test_undo=keep_history,
pack_test_name=pack_test_name,
- ))
+ test_blob_cache=(not shared_blob_dir),
+ ))
return suite
Added: relstorage/trunk/relstorage/tests/util.py
===================================================================
--- relstorage/trunk/relstorage/tests/util.py (rev 0)
+++ relstorage/trunk/relstorage/tests/util.py 2011-03-01 21:51:04 UTC (rev 120654)
@@ -0,0 +1,26 @@
+
+import time
+
+def wait_until(label=None, func=None, timeout=30, onfail=None):
+ """Copied from ZEO.tests.forker, because it does not exist in ZODB 3.8"""
+ if label is None:
+ if func is not None:
+ label = func.__name__
+ elif not isinstance(label, basestring) and func is None:
+ func = label
+ label = func.__name__
+
+ if func is None:
+ def wait_decorator(f):
+ wait_until(label, f, timeout, onfail)
+
+ return wait_decorator
+
+ giveup = time.time() + timeout
+ while not func():
+ if time.time() > giveup:
+ if onfail is None:
+ raise AssertionError("Timed out waiting for: ", label)
+ else:
+ return onfail()
+ time.sleep(0.01)
More information about the checkins
mailing list