[Checkins] SVN: relstorage/trunk/ Support for "shared-blob-dir false" now requires ZODB 3.9 or better.
Shane Hathaway
shane at hathawaymix.org
Tue Mar 1 18:50:21 EST 2011
Log message for revision 120655:
Support for "shared-blob-dir false" now requires ZODB 3.9 or better.
The code in the ZODB 3.8 version of ZODB.blob is not compatible with
BlobCacheLayout, leading to blob filename collisions.
Changed:
U relstorage/trunk/CHANGES.txt
U relstorage/trunk/README.txt
U relstorage/trunk/buildout.cfg
U relstorage/trunk/relstorage/blobhelper.py
U relstorage/trunk/relstorage/tests/blob/testblob.py
U relstorage/trunk/relstorage/tests/testmysql.py
U relstorage/trunk/relstorage/tests/testoracle.py
U relstorage/trunk/relstorage/tests/testpostgresql.py
-=-
Modified: relstorage/trunk/CHANGES.txt
===================================================================
--- relstorage/trunk/CHANGES.txt 2011-03-01 21:51:04 UTC (rev 120654)
+++ relstorage/trunk/CHANGES.txt 2011-03-01 23:50:20 UTC (rev 120655)
@@ -26,7 +26,7 @@
rows from the object reference tables; these tables are
pack-specific and regular ZODB commits never touch these.
-- Add an option to control schema creation / updating on startup.
+- Added an option to control schema creation / updating on startup.
Setting the ``create-schema`` option to false will let you use
RelStorage without a schema update.
@@ -39,6 +39,10 @@
for a thread to read a partially downloaded blob. Fixed. Thanks for
the report from Maurits van Rees.
+- Support for "shared-blob-dir false" now requires ZODB 3.9 or better.
+ The code in the ZODB 3.8 version of ZODB.blob is not compatible with
+ BlobCacheLayout, leading to blob filename collisions.
+
1.5.0b1 (2011-02-05)
--------------------
Modified: relstorage/trunk/README.txt
===================================================================
--- relstorage/trunk/README.txt 2011-03-01 21:51:04 UTC (rev 120654)
+++ relstorage/trunk/README.txt 2011-03-01 23:50:20 UTC (rev 120655)
@@ -393,24 +393,28 @@
blob directory holds a cache of blobs. When this option is
false, the blob directory should not be shared among clients.
+ This option must be true when using ZODB 3.8, because ZODB 3.8
+ is not compatible with the file layout required for a blob
+ cache. Use ZODB 3.9 or later if you want to store blobs in
+ the relational database.
+
``blob-cache-size``
Maximum size of the blob cache, in bytes. If empty (the
default), the cache size isn't checked and the blob directory
will grow without bounds. This should be either empty or
significantly larger than the largest blob you store. At least
1 gigabyte is recommended for typical databases. More is
- recommended if you store videos, CD/DVD images, or virtual
- machine images. If this option is set too small, blobs could
- be evicted from the cache before application code can get them,
- resulting in POSExceptions.
+ recommended if you store large files such as videos, CD/DVD
+ images, or virtual machine images.
This option allows suffixes such as "mb" or "gb".
This option is ignored if shared-blob-dir is true.
``blob-cache-size-check``
- Blob cache check size as percent of blob-cache-size. The blob
- cache size will be checked when this many bytes have been
- loaded into the cache. Defaults to 10% of the blob cache size.
+ Blob cache check size as percent of blob-cache-size: "10" means
+ "10%". The blob cache size will be checked when this many bytes
+ have been loaded into the cache. Defaults to 10% of the blob
+ cache size.
This option is ignored if shared-blob-dir is true.
Modified: relstorage/trunk/buildout.cfg
===================================================================
--- relstorage/trunk/buildout.cfg 2011-03-01 21:51:04 UTC (rev 120654)
+++ relstorage/trunk/buildout.cfg 2011-03-01 23:50:20 UTC (rev 120655)
@@ -27,5 +27,5 @@
eggs =
${buildout:eggs}
z3c.coverage
-scripts = coverage=coverage-report
+scripts = coveragereport
arguments = ('coverage', 'coverage/report')
Modified: relstorage/trunk/relstorage/blobhelper.py
===================================================================
--- relstorage/trunk/relstorage/blobhelper.py 2011-03-01 21:51:04 UTC (rev 120654)
+++ relstorage/trunk/relstorage/blobhelper.py 2011-03-01 23:50:20 UTC (rev 120655)
@@ -72,24 +72,13 @@
try:
- from ZEO.ClientStorage import BlobCacheLayout
+ from ZEO.ClientStorage import BlobCacheLayout, _check_blob_cache_size
except ImportError:
+ # ZODB 3.8 or older
+ BlobCacheLayout = None
+ _check_blob_cache_size = None
- class BlobCacheLayout(object):
- size = 997
-
- def oid_to_path(self, oid):
- return str(utils.u64(oid) % self.size)
-
- def getBlobFilePath(self, oid, tid):
- base, rem = divmod(utils.u64(oid), self.size)
- return os.path.join(
- str(rem),
- "%s.%s%s" % (base, tid.encode('hex'), ZODB.blob.BLOB_SUFFIX)
- )
-
-
class BlobHelper(object):
"""Blob support for RelStorage.
@@ -113,6 +102,13 @@
if self.shared_blob_dir:
# Share files over NFS or similar
fshelper = ZODB.blob.FilesystemHelper(self.blob_dir)
+ elif BlobCacheLayout is None:
+ # BlobCacheLayout is required for 'shared_blob_dir=False',
+ # but BlobCacheLayout is incompatible with ZODB 3.8 because
+ # ZODB 3.8 constructs blob filenames in an inflexible way.
+ raise ValueError(
+ "The shared_blob_dir option must be true when "
+ "RelStorage is used with ZODB 3.8.")
else:
# The blob directory is a cache of the blobs
if 'zeocache' not in ZODB.blob.LAYOUTS:
@@ -414,7 +410,7 @@
# Note: the following code is copied directly from ZEO.ClientStorage.
# It is copied for two reasons:
#
-# 1. Most of the symbols are not public (the function names start
+# 1. The symbols are not public (the function names start
# with an underscore), indicating their signature could change
# at any time.
#
@@ -430,107 +426,6 @@
pass # We tried. :)
return filename
-cache_file_name = re.compile(r'\d+$').match
-def _check_blob_cache_size(blob_dir, target):
-
- logger = logging.getLogger(__name__+'.check_blob_cache')
-
- layout = open(os.path.join(blob_dir, ZODB.blob.LAYOUT_MARKER)
- ).read().strip()
- if not layout == 'zeocache':
- logger.critical("Invalid blob directory layout %s", layout)
- raise ValueError("Invalid blob directory layout", layout)
-
- attempt_path = os.path.join(blob_dir, 'check_size.attempt')
-
- try:
- check_lock = zc.lockfile.LockFile(
- os.path.join(blob_dir, 'check_size.lock'))
- except zc.lockfile.LockError:
- try:
- time.sleep(1)
- check_lock = zc.lockfile.LockFile(
- os.path.join(blob_dir, 'check_size.lock'))
- except zc.lockfile.LockError:
- # Someone is already cleaning up, so don't bother
- logger.debug("%s Another thread is checking the blob cache size.",
- thread.get_ident())
- open(attempt_path, 'w').close() # Mark that we tried
- return
-
- logger.debug("%s Checking blob cache size. (target: %s)",
- thread.get_ident(), target)
-
- try:
- while 1:
- size = 0
- blob_suffix = ZODB.blob.BLOB_SUFFIX
- files_by_atime = BTrees.OOBTree.BTree()
-
- for dirname in os.listdir(blob_dir):
- if not cache_file_name(dirname):
- continue
- base = os.path.join(blob_dir, dirname)
- if not os.path.isdir(base):
- continue
- for file_name in os.listdir(base):
- if not file_name.endswith(blob_suffix):
- continue
- file_path = os.path.join(base, file_name)
- if not os.path.isfile(file_path):
- continue
- stat = os.stat(file_path)
- size += stat.st_size
- t = stat.st_atime
- if t not in files_by_atime:
- files_by_atime[t] = []
- files_by_atime[t].append(os.path.join(dirname, file_name))
-
- logger.debug("%s blob cache size: %s", thread.get_ident(), size)
-
- if size <= target:
- if os.path.isfile(attempt_path):
- try:
- os.remove(attempt_path)
- except OSError:
- pass # Sigh, windows
- continue
- logger.debug("%s -->", thread.get_ident())
- break
-
- while size > target and files_by_atime:
- for file_name in files_by_atime.pop(files_by_atime.minKey()):
- file_name = os.path.join(blob_dir, file_name)
- lockfilename = os.path.join(os.path.dirname(file_name),
- '.lock')
- try:
- lock = zc.lockfile.LockFile(lockfilename)
- except zc.lockfile.LockError:
- logger.debug("%s Skipping locked %s",
- thread.get_ident(),
- os.path.basename(file_name))
- continue # In use, skip
-
- try:
- fsize = os.stat(file_name).st_size
- try:
- ZODB.blob.remove_committed(file_name)
- except OSError, v:
- pass # probably open on windows
- else:
- size -= fsize
- finally:
- lock.close()
-
- if size <= target:
- break
-
- logger.debug("%s reduced blob cache size: %s",
- thread.get_ident(), size)
-
- finally:
- check_lock.close()
-
def _lock_blob(path):
lockfilename = os.path.join(os.path.dirname(path), '.lock')
n = 0
Modified: relstorage/trunk/relstorage/tests/blob/testblob.py
===================================================================
--- relstorage/trunk/relstorage/tests/blob/testblob.py 2011-03-01 21:51:04 UTC (rev 120654)
+++ relstorage/trunk/relstorage/tests/blob/testblob.py 2011-03-01 23:50:20 UTC (rev 120655)
@@ -500,7 +500,6 @@
Pass a factory taking a name and a blob directory name.
"""
-
def setup(test):
setUp(test)
def create_storage(name='data', blob_dir=None, **kw):
@@ -555,3 +554,13 @@
suite.layer = MinimalTestLayer(prefix+'BlobTests')
return suite
+
+
+try:
+ from ZEO.ClientStorage import BlobCacheLayout
+except ImportError:
+ # ZODB 3.8. The blob directory must be shared.
+ shared_blob_dir_choices = (True,)
+else:
+ # ZODB >= 3.9. The blob directory can be a private cache.
+ shared_blob_dir_choices = (False, True)
Modified: relstorage/trunk/relstorage/tests/testmysql.py
===================================================================
--- relstorage/trunk/relstorage/tests/testmysql.py 2011-03-01 21:51:04 UTC (rev 120654)
+++ relstorage/trunk/relstorage/tests/testmysql.py 2011-03-01 23:50:20 UTC (rev 120655)
@@ -162,7 +162,8 @@
pass
else:
from relstorage.tests.blob.testblob import storage_reusable_suite
- for shared_blob_dir in (False, True):
+ from relstorage.tests.blob.testblob import shared_blob_dir_choices
+ for shared_blob_dir in shared_blob_dir_choices:
for keep_history in (False, True):
def create_storage(name, blob_dir,
shared_blob_dir=shared_blob_dir,
Modified: relstorage/trunk/relstorage/tests/testoracle.py
===================================================================
--- relstorage/trunk/relstorage/tests/testoracle.py 2011-03-01 21:51:04 UTC (rev 120654)
+++ relstorage/trunk/relstorage/tests/testoracle.py 2011-03-01 23:50:20 UTC (rev 120655)
@@ -171,7 +171,8 @@
else:
from relstorage.tests.blob.testblob import storage_reusable_suite
dsn = os.environ.get('ORACLE_TEST_DSN', 'XE')
- for shared_blob_dir in (False, True):
+ from relstorage.tests.blob.testblob import shared_blob_dir_choices
+ for shared_blob_dir in shared_blob_dir_choices:
for keep_history in (False, True):
def create_storage(name, blob_dir,
shared_blob_dir=shared_blob_dir,
Modified: relstorage/trunk/relstorage/tests/testpostgresql.py
===================================================================
--- relstorage/trunk/relstorage/tests/testpostgresql.py 2011-03-01 21:51:04 UTC (rev 120654)
+++ relstorage/trunk/relstorage/tests/testpostgresql.py 2011-03-01 23:50:20 UTC (rev 120655)
@@ -159,7 +159,8 @@
pass
else:
from relstorage.tests.blob.testblob import storage_reusable_suite
- for shared_blob_dir in (False, True):
+ from relstorage.tests.blob.testblob import shared_blob_dir_choices
+ for shared_blob_dir in shared_blob_dir_choices:
for keep_history in (False, True):
def create_storage(name, blob_dir,
shared_blob_dir=shared_blob_dir,
More information about the checkins
mailing list