[Zodb-checkins] SVN: ZODB/trunk/src/Z Collapsed the Blob package
into a single module (and a single test
Jim Fulton
jim at zope.com
Sun Jun 3 04:26:49 EDT 2007
Log message for revision 76192:
Collapsed the Blob package into a single module (and a single test
module).
Changed:
U ZODB/trunk/src/ZEO/ClientStorage.py
U ZODB/trunk/src/ZEO/tests/testZEO.py
D ZODB/trunk/src/ZODB/Blobs/Blob.py
D ZODB/trunk/src/ZODB/Blobs/BlobStorage.py
D ZODB/trunk/src/ZODB/Blobs/TODO.txt
D ZODB/trunk/src/ZODB/Blobs/__init__.py
D ZODB/trunk/src/ZODB/Blobs/concept.txt
D ZODB/trunk/src/ZODB/Blobs/exceptions.py
D ZODB/trunk/src/ZODB/Blobs/interfaces.py
D ZODB/trunk/src/ZODB/Blobs/tests/__init__.py
D ZODB/trunk/src/ZODB/Blobs/tests/basic.txt
D ZODB/trunk/src/ZODB/Blobs/tests/connection.txt
D ZODB/trunk/src/ZODB/Blobs/tests/consume.txt
D ZODB/trunk/src/ZODB/Blobs/tests/importexport.txt
D ZODB/trunk/src/ZODB/Blobs/tests/packing.txt
D ZODB/trunk/src/ZODB/Blobs/tests/test_config.py
D ZODB/trunk/src/ZODB/Blobs/tests/test_doctests.py
D ZODB/trunk/src/ZODB/Blobs/tests/test_undo.py
D ZODB/trunk/src/ZODB/Blobs/tests/transaction.txt
U ZODB/trunk/src/ZODB/Connection.py
U ZODB/trunk/src/ZODB/ExportImport.py
A ZODB/trunk/src/ZODB/blob.py
U ZODB/trunk/src/ZODB/config.py
U ZODB/trunk/src/ZODB/interfaces.py
A ZODB/trunk/src/ZODB/tests/blob_basic.txt
A ZODB/trunk/src/ZODB/tests/blob_connection.txt
A ZODB/trunk/src/ZODB/tests/blob_consume.txt
A ZODB/trunk/src/ZODB/tests/blob_importexport.txt
A ZODB/trunk/src/ZODB/tests/blob_packing.txt
A ZODB/trunk/src/ZODB/tests/blob_transaction.txt
A ZODB/trunk/src/ZODB/tests/testblob.py
-=-
Modified: ZODB/trunk/src/ZEO/ClientStorage.py
===================================================================
--- ZODB/trunk/src/ZEO/ClientStorage.py 2007-06-03 05:57:35 UTC (rev 76191)
+++ ZODB/trunk/src/ZEO/ClientStorage.py 2007-06-03 08:26:47 UTC (rev 76192)
@@ -40,7 +40,7 @@
from ZODB import POSException
from ZODB import utils
from ZODB.loglevels import BLATHER
-from ZODB.Blobs.interfaces import IBlobStorage
+from ZODB.interfaces import IBlobStorage
from persistent.TimeStamp import TimeStamp
logger = logging.getLogger('ZEO.ClientStorage')
@@ -324,8 +324,8 @@
if blob_dir is not None:
# Avoid doing this import unless we need it, as it
# currently requires pywin32 on Windows.
- import ZODB.Blobs.Blob
- self.fshelper = ZODB.Blobs.Blob.FilesystemHelper(blob_dir)
+ import ZODB.blob
+ self.fshelper = ZODB.blob.FilesystemHelper(blob_dir)
self.fshelper.create()
self.fshelper.checkSecure()
else:
Modified: ZODB/trunk/src/ZEO/tests/testZEO.py
===================================================================
--- ZODB/trunk/src/ZEO/tests/testZEO.py 2007-06-03 05:57:35 UTC (rev 76191)
+++ ZODB/trunk/src/ZEO/tests/testZEO.py 2007-06-03 08:26:47 UTC (rev 76192)
@@ -460,8 +460,7 @@
def checkStoreBlob(self):
from ZODB.utils import oid_repr, tid_repr
- from ZODB.Blobs.Blob import Blob
- from ZODB.Blobs.BlobStorage import BLOB_SUFFIX
+ from ZODB.blob import Blob, BLOB_SUFFIX
from ZODB.tests.StorageTestBase import zodb_pickle, ZERO, \
handle_serials
import transaction
@@ -494,7 +493,7 @@
self.assertEqual(somedata, open(filename).read())
def checkLoadBlob(self):
- from ZODB.Blobs.Blob import Blob
+ from ZODB.blob import Blob
from ZODB.tests.StorageTestBase import zodb_pickle, ZERO, \
handle_serials
import transaction
@@ -534,8 +533,7 @@
def checkStoreAndLoadBlob(self):
from ZODB.utils import oid_repr, tid_repr
- from ZODB.Blobs.Blob import Blob
- from ZODB.Blobs.BlobStorage import BLOB_SUFFIX
+ from ZODB.blob import Blob, BLOB_SUFFIX
from ZODB.tests.StorageTestBase import zodb_pickle, ZERO, \
handle_serials
import transaction
Deleted: ZODB/trunk/src/ZODB/Blobs/Blob.py
===================================================================
--- ZODB/trunk/src/ZODB/Blobs/Blob.py 2007-06-03 05:57:35 UTC (rev 76191)
+++ ZODB/trunk/src/ZODB/Blobs/Blob.py 2007-06-03 08:26:47 UTC (rev 76192)
@@ -1,473 +0,0 @@
-##############################################################################
-#
-# Copyright (c) 2005-2006 Zope Corporation and Contributors.
-# All Rights Reserved.
-#
-# This software is subject to the provisions of the Zope Public License,
-# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
-# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
-# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
-# FOR A PARTICULAR PURPOSE
-#
-##############################################################################
-"""The blob class and related utilities.
-
-"""
-__docformat__ = "reStructuredText"
-
-import os
-import sys
-import time
-import tempfile
-import logging
-
-import zope.interface
-
-from ZODB.Blobs.interfaces import IBlob
-from ZODB.Blobs.exceptions import BlobError
-from ZODB import utils
-import transaction
-import transaction.interfaces
-from persistent import Persistent
-
-BLOB_SUFFIX = ".blob"
-
-valid_modes = 'r', 'w', 'r+', 'a'
-
-class Blob(Persistent):
- """A BLOB supports efficient handling of large data within ZODB."""
-
- zope.interface.implements(IBlob)
-
- _os_link = os.rename
-
- _p_blob_readers = 0
- _p_blob_writers = 0
- _p_blob_uncommitted = None # Filename of the uncommitted (dirty) data
- _p_blob_data = None # Filename of the committed data
-
- # All persistent object store a reference to their data manager, a database
- # connection in the _p_jar attribute. So we are going to do the same with
- # blobs here.
- _p_blob_manager = None
-
- # Blobs need to participate in transactions even when not connected to
- # a database yet. If you want to use a non-default transaction manager,
- # you can override it via _p_blob_transaction. This is currently
- # required for unit testing.
- _p_blob_transaction = None
-
- def open(self, mode="r"):
- """Returns a file(-like) object representing blob data."""
- result = None
-
- if mode not in valid_modes:
- raise ValueError("invalid mode", mode)
-
- if mode == 'r':
- if self._current_filename() is None:
- raise BlobError("Blob does not exist.")
-
- if self._p_blob_writers != 0:
- raise BlobError("Already opened for writing.")
-
- self._p_blob_readers += 1
- result = BlobFile(self._current_filename(), mode, self)
-
- elif mode == 'w':
- if self._p_blob_readers != 0:
- raise BlobError("Already opened for reading.")
-
- self._p_blob_writers += 1
- if self._p_blob_uncommitted is None:
- self._create_uncommitted_file()
- result = BlobFile(self._p_blob_uncommitted, mode, self)
-
- elif mode in ('a', 'r+'):
- if self._p_blob_readers != 0:
- raise BlobError("Already opened for reading.")
-
- if self._p_blob_uncommitted is None:
- # Create a new working copy
- uncommitted = BlobFile(self._create_uncommitted_file(),
- mode, self)
- # NOTE: _p_blob data appears by virtue of Connection._setstate
- utils.cp(file(self._p_blob_data), uncommitted)
- uncommitted.seek(0)
- else:
- # Re-use existing working copy
- uncommitted = BlobFile(self._p_blob_uncommitted, mode, self)
-
- self._p_blob_writers += 1
- result = uncommitted
-
- else:
- raise IOError('invalid mode: %s ' % mode)
-
- if result is not None:
- self._setup_transaction_manager(result)
- return result
-
- def openDetached(self, class_=file):
- """Returns a file(-like) object in read mode that can be used
- outside of transaction boundaries.
-
- """
- if self._current_filename() is None:
- raise BlobError("Blob does not exist.")
- if self._p_blob_writers != 0:
- raise BlobError("Already opened for writing.")
- # XXX this should increase the reader number and have a test !?!
- return class_(self._current_filename(), "rb")
-
- def consumeFile(self, filename):
- """Will replace the current data of the blob with the file given under
- filename.
- """
- if self._p_blob_writers != 0:
- raise BlobError("Already opened for writing.")
- if self._p_blob_readers != 0:
- raise BlobError("Already opened for reading.")
-
- previous_uncommitted = bool(self._p_blob_uncommitted)
- if previous_uncommitted:
- # If we have uncommitted data, we move it aside for now
- # in case the consumption doesn't work.
- target = self._p_blob_uncommitted
- target_aside = target+".aside"
- os.rename(target, target_aside)
- else:
- target = self._create_uncommitted_file()
- # We need to unlink the freshly created target again
- # to allow link() to do its job
- os.unlink(target)
-
- try:
- self._os_link(filename, target)
- except:
- # Recover from the failed consumption: First remove the file, it
- # might exist and mark the pointer to the uncommitted file.
- self._p_blob_uncommitted = None
- if os.path.exists(target):
- os.unlink(target)
-
- # If there was a file moved aside, bring it back including the
- # pointer to the uncommitted file.
- if previous_uncommitted:
- os.rename(target_aside, target)
- self._p_blob_uncommitted = target
-
- # Re-raise the exception to make the application aware of it.
- raise
- else:
- if previous_uncommitted:
- # The relinking worked so we can remove the data that we had
- # set aside.
- os.unlink(target_aside)
-
- # We changed the blob state and have to make sure we join the
- # transaction.
- self._change()
-
- # utility methods
-
- def _current_filename(self):
- # NOTE: _p_blob_data and _p_blob_uncommitted appear by virtue of
- # Connection._setstate
- return self._p_blob_uncommitted or self._p_blob_data
-
- def _create_uncommitted_file(self):
- assert self._p_blob_uncommitted is None, (
- "Uncommitted file already exists.")
- tempdir = os.environ.get('ZODB_BLOB_TEMPDIR', tempfile.gettempdir())
- self._p_blob_uncommitted = utils.mktemp(dir=tempdir)
- return self._p_blob_uncommitted
-
- def _change(self):
- self._p_changed = 1
-
- def _setup_transaction_manager(self, result):
- # We join the transaction with our own data manager in order to be
- # notified of commit/vote/abort events. We do this because at
- # transaction boundaries, we need to fix up _p_ reference counts
- # that keep track of open readers and writers and close any
- # writable filehandles we've opened.
- if self._p_blob_manager is None:
- # Blobs need to always participate in transactions.
- if self._p_jar is not None:
- # If we are connected to a database, then we use the
- # transaction manager that belongs to this connection
- tm = self._p_jar.transaction_manager
- else:
- # If we are not connected to a database, we check whether
- # we have been given an explicit transaction manager
- if self._p_blob_transaction:
- tm = self._p_blob_transaction
- else:
- # Otherwise we use the default
- # transaction manager as an educated guess.
- tm = transaction.manager
- # Create our datamanager and join he current transaction.
- dm = BlobDataManager(self, result, tm)
- tm.get().join(dm)
- elif result:
- # Each blob data manager should manage only the one blob
- # assigned to it. Assert that this is the case and it is the
- # correct blob
- assert self._p_blob_manager.blob is self
- self._p_blob_manager.register_fh(result)
-
- # utility methods which should not cause the object's state to be
- # loaded if they are called while the object is a ghost. Thus,
- # they are named with the _p_ convention and only operate against
- # other _p_ instance attributes. We conventionally name these methods
- # and attributes with a _p_blob prefix.
-
- def _p_blob_clear(self):
- self._p_blob_readers = 0
- self._p_blob_writers = 0
-
- def _p_blob_decref(self, mode):
- if mode == 'r':
- self._p_blob_readers = max(0, self._p_blob_readers - 1)
- else:
- assert mode in valid_modes, "Invalid mode %r" % mode
- self._p_blob_writers = max(0, self._p_blob_writers - 1)
-
- def _p_blob_refcounts(self):
- # used by unit tests
- return self._p_blob_readers, self._p_blob_writers
-
-
-class BlobDataManager:
- """Special data manager to handle transaction boundaries for blobs.
-
- Blobs need some special care-taking on transaction boundaries. As
-
- a) the ghost objects might get reused, the _p_reader and _p_writer
- refcount attributes must be set to a consistent state
- b) the file objects might get passed out of the thread/transaction
- and must deny any relationship to the original blob.
- c) writable blob filehandles must be closed at the end of a txn so
- as to not allow reuse between two transactions.
-
- """
-
- zope.interface.implements(transaction.interfaces.IDataManager)
-
- def __init__(self, blob, filehandle, tm):
- self.blob = blob
- self.transaction = tm.get()
- # we keep a weakref to the file handle because we don't want to
- # keep it alive if all other references to it die (e.g. in the
- # case it's opened without assigning it to a name).
- self.fhrefs = utils.WeakSet()
- self.register_fh(filehandle)
- self.sortkey = time.time()
- self.prepared = False
-
- # Blob specific methods
-
- def register_fh(self, filehandle):
- self.fhrefs.add(filehandle)
-
- def _remove_uncommitted_data(self):
- self.blob._p_blob_clear()
- self.fhrefs.map(lambda fhref: fhref.close())
- if (self.blob._p_blob_uncommitted is not None and
- os.path.exists(self.blob._p_blob_uncommitted)):
- os.unlink(self.blob._p_blob_uncommitted)
- self.blob._p_blob_uncommitted = None
-
- # IDataManager
-
- def tpc_begin(self, transaction):
- if self.prepared:
- raise TypeError('Already prepared')
- self._checkTransaction(transaction)
- self.prepared = True
- self.transaction = transaction
- self.fhrefs.map(lambda fhref: fhref.close())
-
- def commit(self, transaction):
- if not self.prepared:
- raise TypeError('Not prepared to commit')
- self._checkTransaction(transaction)
- self.transaction = None
- self.prepared = False
-
- self.blob._p_blob_clear()
-
- def abort(self, transaction):
- self.tpc_abort(transaction)
-
- def tpc_abort(self, transaction):
- self._checkTransaction(transaction)
- if self.transaction is not None:
- self.transaction = None
- self.prepared = False
-
- self._remove_uncommitted_data()
-
- def tpc_finish(self, transaction):
- pass
-
- def tpc_vote(self, transaction):
- pass
-
- def sortKey(self):
- return self.sortkey
-
- def _checkTransaction(self, transaction):
- if (self.transaction is not None and
- self.transaction is not transaction):
- raise TypeError("Transaction missmatch",
- transaction, self.transaction)
-
-
-class BlobFile(file):
- """A BlobFile that holds a file handle to actual blob data.
-
- It is a file that can be used within a transaction boundary; a BlobFile is
- just a Python file object, we only override methods which cause a change to
- blob data in order to call methods on our 'parent' persistent blob object
- signifying that the change happened.
-
- """
-
- # XXX these files should be created in the same partition as
- # the storage later puts them to avoid copying them ...
-
- def __init__(self, name, mode, blob):
- super(BlobFile, self).__init__(name, mode+'b')
- self.blob = blob
- self.close_called = False
-
- def write(self, data):
- super(BlobFile, self).write(data)
- self.blob._change()
-
- def writelines(self, lines):
- super(BlobFile, self).writelines(lines)
- self.blob._change()
-
- def truncate(self, size=0):
- super(BlobFile, self).truncate(size)
- self.blob._change()
-
- def close(self):
- # we don't want to decref twice
- if not self.close_called:
- self.blob._p_blob_decref(self.mode[:-1])
- self.close_called = True
- super(BlobFile, self).close()
-
- def __del__(self):
- # XXX we need to ensure that the file is closed at object
- # expiration or our blob's refcount won't be decremented.
- # This probably needs some work; I don't know if the names
- # 'BlobFile' or 'super' will be available at program exit, but
- # we'll assume they will be for now in the name of not
- # muddying the code needlessly.
- self.close()
-
-
-logger = logging.getLogger('ZODB.Blobs')
-_pid = str(os.getpid())
-
-
-def log(msg, level=logging.INFO, subsys=_pid, exc_info=False):
- message = "(%s) %s" % (subsys, msg)
- logger.log(level, message, exc_info=exc_info)
-
-
-class FilesystemHelper:
- # Storages that implement IBlobStorage can choose to use this
- # helper class to generate and parse blob filenames. This is not
- # a set-in-stone interface for all filesystem operations dealing
- # with blobs and storages needn't indirect through this if they
- # want to perform blob storage differently.
-
- def __init__(self, base_dir):
- self.base_dir = base_dir
-
- def create(self):
- if not os.path.exists(self.base_dir):
- os.makedirs(self.base_dir, 0700)
- log("Blob cache directory '%s' does not exist. "
- "Created new directory." % self.base_dir,
- level=logging.INFO)
-
- def isSecure(self, path):
- """Ensure that (POSIX) path mode bits are 0700."""
- return (os.stat(path).st_mode & 077) != 0
-
- def checkSecure(self):
- if not self.isSecure(self.base_dir):
- log('Blob dir %s has insecure mode setting' % self.base_dir,
- level=logging.WARNING)
-
- def getPathForOID(self, oid):
- """Given an OID, return the path on the filesystem where
- the blob data relating to that OID is stored.
-
- """
- return os.path.join(self.base_dir, utils.oid_repr(oid))
-
- def getBlobFilename(self, oid, tid):
- """Given an oid and a tid, return the full filename of the
- 'committed' blob file related to that oid and tid.
-
- """
- oid_path = self.getPathForOID(oid)
- filename = "%s%s" % (utils.tid_repr(tid), BLOB_SUFFIX)
- return os.path.join(oid_path, filename)
-
- def blob_mkstemp(self, oid, tid):
- """Given an oid and a tid, return a temporary file descriptor
- and a related filename.
-
- The file is guaranteed to exist on the same partition as committed
- data, which is important for being able to rename the file without a
- copy operation. The directory in which the file will be placed, which
- is the return value of self.getPathForOID(oid), must exist before this
- method may be called successfully.
-
- """
- oidpath = self.getPathForOID(oid)
- fd, name = tempfile.mkstemp(suffix='.tmp', prefix=utils.tid_repr(tid),
- dir=oidpath)
- return fd, name
-
- def splitBlobFilename(self, filename):
- """Returns the oid and tid for a given blob filename.
-
- If the filename cannot be recognized as a blob filename, (None, None)
- is returned.
-
- """
- if not filename.endswith(BLOB_SUFFIX):
- return None, None
- path, filename = os.path.split(filename)
- oid = os.path.split(path)[1]
-
- serial = filename[:-len(BLOB_SUFFIX)]
- oid = utils.repr_to_oid(oid)
- serial = utils.repr_to_oid(serial)
- return oid, serial
-
- def getOIDsForSerial(self, search_serial):
- """Return all oids related to a particular tid that exist in
- blob data.
-
- """
- oids = []
- base_dir = self.base_dir
- for oidpath in os.listdir(base_dir):
- for filename in os.listdir(os.path.join(base_dir, oidpath)):
- blob_path = os.path.join(base_dir, oidpath, filename)
- oid, serial = self.splitBlobFilename(blob_path)
- if search_serial == serial:
- oids.append(oid)
- return oids
Deleted: ZODB/trunk/src/ZODB/Blobs/BlobStorage.py
===================================================================
--- ZODB/trunk/src/ZODB/Blobs/BlobStorage.py 2007-06-03 05:57:35 UTC (rev 76191)
+++ ZODB/trunk/src/ZODB/Blobs/BlobStorage.py 2007-06-03 08:26:47 UTC (rev 76192)
@@ -1,272 +0,0 @@
-##############################################################################
-#
-# Copyright (c) 2005-2006 Zope Corporation and Contributors.
-# All Rights Reserved.
-#
-# This software is subject to the provisions of the Zope Public License,
-# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
-# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
-# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
-# FOR A PARTICULAR PURPOSE
-#
-##############################################################################
-"""A ZODB storage that provides blob capabilities.
-
-"""
-__docformat__ = "reStructuredText"
-
-import os
-import shutil
-import base64
-import logging
-
-from zope.interface import implements
-from zope.proxy import getProxiedObject, non_overridable
-from zope.proxy.decorator import SpecificationDecoratorBase
-
-from ZODB import utils
-from ZODB.Blobs.interfaces import IBlobStorage, IBlob
-from ZODB.POSException import POSKeyError
-from ZODB.Blobs.Blob import BLOB_SUFFIX
-from ZODB.Blobs.Blob import FilesystemHelper
-
-logger = logging.getLogger('ZODB.BlobStorage')
-
-
-class BlobStorage(SpecificationDecoratorBase):
- """A storage to support blobs."""
-
- implements(IBlobStorage)
-
- # Proxies can't have a __dict__ so specifying __slots__ here allows
- # us to have instance attributes explicitly on the proxy.
- __slots__ = ('fshelper', 'dirty_oids', '_BlobStorage__supportsUndo')
-
- def __new__(self, base_directory, storage):
- return SpecificationDecoratorBase.__new__(self, storage)
-
- def __init__(self, base_directory, storage):
- # XXX Log warning if storage is ClientStorage
- SpecificationDecoratorBase.__init__(self, storage)
- self.fshelper = FilesystemHelper(base_directory)
- self.fshelper.create()
- self.fshelper.checkSecure()
- self.dirty_oids = []
- try:
- supportsUndo = storage.supportsUndo
- except AttributeError:
- supportsUndo = False
- else:
- supportsUndo = supportsUndo()
- self.__supportsUndo = supportsUndo
-
- @non_overridable
- def temporaryDirectory(self):
- return self.fshelper.base_dir
-
-
- @non_overridable
- def __repr__(self):
- normal_storage = getProxiedObject(self)
- return '<BlobStorage proxy for %r at %s>' % (normal_storage,
- hex(id(self)))
- @non_overridable
- def storeBlob(self, oid, oldserial, data, blobfilename, version,
- transaction):
- """Stores data that has a BLOB attached."""
- serial = self.store(oid, oldserial, data, version, transaction)
- assert isinstance(serial, str) # XXX in theory serials could be
- # something else
-
- # the user may not have called "open" on the blob object,
- # in which case, the blob will not have a filename.
- if blobfilename is not None:
- self._lock_acquire()
- try:
- targetpath = self.fshelper.getPathForOID(oid)
- if not os.path.exists(targetpath):
- os.makedirs(targetpath, 0700)
-
- targetname = self.fshelper.getBlobFilename(oid, serial)
- os.rename(blobfilename, targetname)
-
- # XXX if oid already in there, something is really hosed.
- # The underlying storage should have complained anyway
- self.dirty_oids.append((oid, serial))
- finally:
- self._lock_release()
- return self._tid
-
- @non_overridable
- def tpc_finish(self, *arg, **kw):
- # We need to override the base storage's tpc_finish instead of
- # providing a _finish method because methods found on the proxied
- # object aren't rebound to the proxy
- getProxiedObject(self).tpc_finish(*arg, **kw)
- self.dirty_oids = []
-
- @non_overridable
- def tpc_abort(self, *arg, **kw):
- # We need to override the base storage's abort instead of
- # providing an _abort method because methods found on the proxied object
- # aren't rebound to the proxy
- getProxiedObject(self).tpc_abort(*arg, **kw)
- while self.dirty_oids:
- oid, serial = self.dirty_oids.pop()
- clean = self.fshelper.getBlobFilename(oid, serial)
- if os.exists(clean):
- os.unlink(clean)
-
- @non_overridable
- def loadBlob(self, oid, serial):
- """Return the filename where the blob file can be found.
- """
- filename = self.fshelper.getBlobFilename(oid, serial)
- if not os.path.exists(filename):
- return None
- return filename
-
- @non_overridable
- def _packUndoing(self, packtime, referencesf):
- # Walk over all existing revisions of all blob files and check
- # if they are still needed by attempting to load the revision
- # of that object from the database. This is maybe the slowest
- # possible way to do this, but it's safe.
-
- # XXX we should be tolerant of "garbage" directories/files in
- # the base_directory here.
-
- base_dir = self.fshelper.base_dir
- for oid_repr in os.listdir(base_dir):
- oid = utils.repr_to_oid(oid_repr)
- oid_path = os.path.join(base_dir, oid_repr)
- files = os.listdir(oid_path)
- files.sort()
-
- for filename in files:
- filepath = os.path.join(oid_path, filename)
- whatever, serial = self.fshelper.splitBlobFilename(filepath)
- try:
- fn = self.fshelper.getBlobFilename(oid, serial)
- self.loadSerial(oid, serial)
- except POSKeyError:
- os.unlink(filepath)
-
- if not os.listdir(oid_path):
- shutil.rmtree(oid_path)
-
- @non_overridable
- def _packNonUndoing(self, packtime, referencesf):
- base_dir = self.fshelper.base_dir
- for oid_repr in os.listdir(base_dir):
- oid = utils.repr_to_oid(oid_repr)
- oid_path = os.path.join(base_dir, oid_repr)
- exists = True
-
- try:
- self.load(oid, None) # no version support
- except (POSKeyError, KeyError):
- exists = False
-
- if exists:
- files = os.listdir(oid_path)
- files.sort()
- latest = files[-1] # depends on ever-increasing tids
- files.remove(latest)
- for file in files:
- os.unlink(os.path.join(oid_path, file))
- else:
- shutil.rmtree(oid_path)
- continue
-
- if not os.listdir(oid_path):
- shutil.rmtree(oid_path)
-
- @non_overridable
- def pack(self, packtime, referencesf):
- """Remove all unused oid/tid combinations."""
- unproxied = getProxiedObject(self)
-
- # pack the underlying storage, which will allow us to determine
- # which serials are current.
- result = unproxied.pack(packtime, referencesf)
-
- # perform a pack on blob data
- self._lock_acquire()
- try:
- if self.__supportsUndo:
- self._packUndoing(packtime, referencesf)
- else:
- self._packNonUndoing(packtime, referencesf)
- finally:
- self._lock_release()
-
- return result
-
- @non_overridable
- def getSize(self):
- """Return the size of the database in bytes."""
- orig_size = getProxiedObject(self).getSize()
-
- blob_size = 0
- base_dir = self.fshelper.base_dir
- for oid in os.listdir(base_dir):
- for serial in os.listdir(os.path.join(base_dir, oid)):
- if not serial.endswith(BLOB_SUFFIX):
- continue
- file_path = os.path.join(base_dir, oid, serial)
- blob_size += os.stat(file_path).st_size
-
- return orig_size + blob_size
-
- @non_overridable
- def undo(self, serial_id, transaction):
- undo_serial, keys = getProxiedObject(self).undo(serial_id, transaction)
- # serial_id is the transaction id of the txn that we wish to undo.
- # "undo_serial" is the transaction id of txn in which the undo is
- # performed. "keys" is the list of oids that are involved in the
- # undo transaction.
-
- # The serial_id is assumed to be given to us base-64 encoded
- # (belying the web UI legacy of the ZODB code :-()
- serial_id = base64.decodestring(serial_id+'\n')
-
- self._lock_acquire()
-
- try:
- # we get all the blob oids on the filesystem related to the
- # transaction we want to undo.
- for oid in self.fshelper.getOIDsForSerial(serial_id):
-
- # we want to find the serial id of the previous revision
- # of this blob object.
- load_result = self.loadBefore(oid, serial_id)
-
- if load_result is None:
- # There was no previous revision of this blob
- # object. The blob was created in the transaction
- # represented by serial_id. We copy the blob data
- # to a new file that references the undo
- # transaction in case a user wishes to undo this
- # undo.
- orig_fn = self.fshelper.getBlobFilename(oid, serial_id)
- new_fn = self.fshelper.getBlobFilename(oid, undo_serial)
- else:
- # A previous revision of this blob existed before the
- # transaction implied by "serial_id". We copy the blob
- # data to a new file that references the undo transaction
- # in case a user wishes to undo this undo.
- data, serial_before, serial_after = load_result
- orig_fn = self.fshelper.getBlobFilename(oid, serial_before)
- new_fn = self.fshelper.getBlobFilename(oid, undo_serial)
- orig = open(orig_fn, "r")
- new = open(new_fn, "wb")
- utils.cp(orig, new)
- orig.close()
- new.close()
- self.dirty_oids.append((oid, undo_serial))
-
- finally:
- self._lock_release()
- return undo_serial, keys
Deleted: ZODB/trunk/src/ZODB/Blobs/TODO.txt
===================================================================
--- ZODB/trunk/src/ZODB/Blobs/TODO.txt 2007-06-03 05:57:35 UTC (rev 76191)
+++ ZODB/trunk/src/ZODB/Blobs/TODO.txt 2007-06-03 08:26:47 UTC (rev 76192)
@@ -1,32 +0,0 @@
-Production
-
- - Ensure we detect and replay a failed txn involving blobs forward or
- backward at startup.
-
-Far future
-
- More options for blob directory structures (e.g. dirstorages
- bushy/chunky/lawn/flat).
-
- Make the ClientStorage support minimizing the blob cache. (Idea: LRU
- principle via mstat access time and a size-based threshold) currently).
-
- Make blobs able to efficiently consume existing files from the filesystem
-
-Savepoint support
-=================
-
- - A savepoint represents the whole state of the data at a certain point in
- time
-
- - Need special storage for blob savepointing (in the spirit of tmpstorage)
-
- - What belongs to the state of the data?
-
- - Data contained in files at that point in time
-
- - File handles are complex because they might be referred to from various
- places. We would have to introduce an abstraction layer to allow
- switching them around...
-
- Simpler solution: :
Deleted: ZODB/trunk/src/ZODB/Blobs/__init__.py
===================================================================
--- ZODB/trunk/src/ZODB/Blobs/__init__.py 2007-06-03 05:57:35 UTC (rev 76191)
+++ ZODB/trunk/src/ZODB/Blobs/__init__.py 2007-06-03 08:26:47 UTC (rev 76192)
@@ -1 +0,0 @@
-"""The ZODB Blob package."""
Deleted: ZODB/trunk/src/ZODB/Blobs/concept.txt
===================================================================
--- ZODB/trunk/src/ZODB/Blobs/concept.txt 2007-06-03 05:57:35 UTC (rev 76191)
+++ ZODB/trunk/src/ZODB/Blobs/concept.txt 2007-06-03 08:26:47 UTC (rev 76192)
@@ -1,79 +0,0 @@
-
-Goal: Handle storage and retrieval of binary large objects efficiently,
- transactionally, and transparently.
-
-Measure:
-
- - Don't block ZServer on uploads and downloads
-
- - Don't hold BLOBS in memory or cache if not necessary (LRU caches tend
- to break if we split BLOBs in lot of small objects. Size-based caches
- tend to break on single large objects)
-
- - Transparent for other systems, support normal ZODB operations.
-
-Comments:
-
- - Cache: BLOBs could be cached in a seperate "BLOB" space, e.g. in
- single files
-
- - Be storage independent?
-
- - Memory efficiency: Storge.load() currently holds all data of an
- object in a string.
-
-Steps:
-
- - simple aspects:
-
- - blobs should be known by zodb
-
- - storages, esp. clientstorage must be able to recognize blobs
-
- - to avoid putting blob data into the client cache.
-
- - blob data mustn't end up in the object cache
-
- - blob object and blob data need to be handled separately
-
- - blob data on client is stored in temporary files
-
- - complicated aspects
-
- - temporary files holding blob data could server as a
- separated cache for blob data
-
- - storage / zodb api change
-
-Restrictions:
-
- - a particular BLOB instance can't be open for read _and_ write at
- the same time
-
- - Allowed: N readers, no writers; 1 writer, no readers
-
- - Reason:
-
- - a writable filehandle opened via a BLOB's 'open' method has a
- lifetime tied to the transaction in which the 'open' method was
- called. We do this in order to prevent changes to blob data
- from "bleeding over" between transactions.
-
-- Data has been committed? -> File(name) for commited data available
-
-- .open("r") on fresh loaded blob returns committed data
-
-- first .open("w") -> new empty file for uncommitted data
-
-- .open("a") or .open("r+"), we copy existing data into file for
- uncommitted data
-
-- if uncommitted data exists, subsequent .open("*") will use the
- uncommitted data
-
-- if opened for writing, the object is marked as changed
- (optimiziation possible)
-
-- connections want to recognize blobs on transaction boundaries
-
-
Deleted: ZODB/trunk/src/ZODB/Blobs/exceptions.py
===================================================================
--- ZODB/trunk/src/ZODB/Blobs/exceptions.py 2007-06-03 05:57:35 UTC (rev 76191)
+++ ZODB/trunk/src/ZODB/Blobs/exceptions.py 2007-06-03 08:26:47 UTC (rev 76192)
@@ -1,3 +0,0 @@
-
-class BlobError(Exception):
- pass
Deleted: ZODB/trunk/src/ZODB/Blobs/interfaces.py
===================================================================
--- ZODB/trunk/src/ZODB/Blobs/interfaces.py 2007-06-03 05:57:35 UTC (rev 76191)
+++ ZODB/trunk/src/ZODB/Blobs/interfaces.py 2007-06-03 08:26:47 UTC (rev 76192)
@@ -1,75 +0,0 @@
-##############################################################################
-#
-# Copyright (c) 2005-2007 Zope Corporation and Contributors.
-# All Rights Reserved.
-#
-# This software is subject to the provisions of the Zope Public License,
-# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
-# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
-# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
-# FOR A PARTICULAR PURPOSE
-#
-##############################################################################
-"""Blob-related interfaces
-
-"""
-
-from zope.interface import Interface
-
-
-class IBlob(Interface):
- """A BLOB supports efficient handling of large data within ZODB."""
-
- def open(mode):
- """Returns a file(-like) object for handling the blob data.
-
- mode: Mode to open the file with. Possible values: r,w,r+,a
- """
-
- def openDetached(class_=file):
- """Returns a file(-like) object in read mode that can be used
- outside of transaction boundaries.
-
- The file handle returned by this method is read-only and at the
- beginning of the file.
-
- The handle is not attached to the blob and can be used outside of a
- transaction.
-
- Optionally the class that should be used to open the file can be
- specified. This can be used to e.g. use Zope's FileStreamIterator.
- """
-
- def consumeFile(filename):
- """Will replace the current data of the blob with the file given under
- filename.
-
- This method uses link-like semantics internally and has the requirement
- that the file that is to be consumed lives on the same volume (or
- mount/share) as the blob directory.
-
- The blob must not be opened for reading or writing when consuming a
- file.
- """
-
-
-class IBlobStorage(Interface):
- """A storage supporting BLOBs."""
-
- def storeBlob(oid, oldserial, data, blob, version, transaction):
- """Stores data that has a BLOB attached."""
-
- def loadBlob(oid, serial):
- """Return the filename of the Blob data for this OID and serial.
-
- Returns a filename or None if no Blob data is connected with this OID.
-
- Raises POSKeyError if the blobfile cannot be found.
- """
-
- def temporaryDirectory():
- """Return a directory that should be used for uncommitted blob data.
-
- If Blobs use this, then commits can be performed with a simple rename.
- """
Deleted: ZODB/trunk/src/ZODB/Blobs/tests/__init__.py
===================================================================
--- ZODB/trunk/src/ZODB/Blobs/tests/__init__.py 2007-06-03 05:57:35 UTC (rev 76191)
+++ ZODB/trunk/src/ZODB/Blobs/tests/__init__.py 2007-06-03 08:26:47 UTC (rev 76192)
@@ -1 +0,0 @@
-# python package
Deleted: ZODB/trunk/src/ZODB/Blobs/tests/basic.txt
===================================================================
--- ZODB/trunk/src/ZODB/Blobs/tests/basic.txt 2007-06-03 05:57:35 UTC (rev 76191)
+++ ZODB/trunk/src/ZODB/Blobs/tests/basic.txt 2007-06-03 08:26:47 UTC (rev 76192)
@@ -1,167 +0,0 @@
-##############################################################################
-#
-# Copyright (c) 2005 Zope Corporation and Contributors.
-# All Rights Reserved.
-#
-# This software is subject to the provisions of the Zope Public License,
-# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
-# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
-# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
-# FOR A PARTICULAR PURPOSE.
-#
-##############################################################################
-
-ZODB Blob support
-=================
-
-You create a blob like this:
-
- >>> from ZODB.Blobs.Blob import Blob
- >>> myblob = Blob()
-
-A blob implements the IBlob interface:
-
- >>> from ZODB.Blobs.interfaces import IBlob
- >>> IBlob.providedBy(myblob)
- True
-
-Opening a new Blob for reading fails:
-
- >>> myblob.open("r")
- Traceback (most recent call last):
- ...
- BlobError: Blob does not exist.
-
-But we can write data to a new Blob by opening it for writing:
-
- >>> f = myblob.open("w")
- >>> f.write("Hi, Blob!")
-
-If we try to open a Blob again while it is open for writing, we get an error:
-
- >>> myblob.open("r")
- Traceback (most recent call last):
- ...
- BlobError: Already opened for writing.
-
-We can close the file:
-
- >>> f.close()
-
-Now we can open it for reading:
-
- >>> f2 = myblob.open("r")
-
-And we get the data back:
-
- >>> f2.read()
- 'Hi, Blob!'
-
-If we want to, we can open it again:
-
- >>> f3 = myblob.open("r")
- >>> f3.read()
- 'Hi, Blob!'
-
-But we can't open it for writing, while it is opened for reading:
-
- >>> myblob.open("a")
- Traceback (most recent call last):
- ...
- BlobError: Already opened for reading.
-
-Before we can write, we have to close the readers:
-
- >>> f2.close()
- >>> f3.close()
-
-Now we can open it for writing again and e.g. append data:
-
- >>> f4 = myblob.open("a")
- >>> f4.write("\nBlob is fine.")
- >>> f4.close()
-
-Now we can read it:
-
- >>> f4a = myblob.open("r")
- >>> f4a.read()
- 'Hi, Blob!\nBlob is fine.'
- >>> f4a.close()
-
-You shouldn't need to explicitly close a blob unless you hold a reference
-to it via a name. If the first line in the following test kept a reference
-around via a name, the second call to open it in a writable mode would fail
-with a BlobError, but it doesn't.
-
- >>> myblob.open("r+").read()
- 'Hi, Blob!\nBlob is fine.'
- >>> f4b = myblob.open("a")
- >>> f4b.close()
-
-We can read lines out of the blob too:
-
- >>> f5 = myblob.open("r")
- >>> f5.readline()
- 'Hi, Blob!\n'
- >>> f5.readline()
- 'Blob is fine.'
- >>> f5.close()
-
-We can seek to certain positions in a blob and read portions of it:
-
- >>> f6 = myblob.open('r')
- >>> f6.seek(4)
- >>> int(f6.tell())
- 4
- >>> f6.read(5)
- 'Blob!'
- >>> f6.close()
-
-We can use the object returned by a blob open call as an iterable:
-
- >>> f7 = myblob.open('r')
- >>> for line in f7:
- ... print line
- Hi, Blob!
- <BLANKLINE>
- Blob is fine.
- >>> f7.close()
-
-We can truncate a blob:
-
- >>> f8 = myblob.open('a')
- >>> f8.truncate(0)
- >>> f8.close()
- >>> f8 = myblob.open('r')
- >>> f8.read()
- ''
- >>> f8.close()
-
-Blobs are always opened in binary mode:
-
- >>> f9 = myblob.open("r")
- >>> f9.mode
- 'rb'
- >>> f9.close()
-
-We can specify the tempdir that blobs use to keep uncommitted data by
-modifying the ZODB_BLOB_TEMPDIR environment variable:
-
- >>> import os, tempfile, shutil
- >>> tempdir = tempfile.mkdtemp()
- >>> os.environ['ZODB_BLOB_TEMPDIR'] = tempdir
- >>> myblob = Blob()
- >>> len(os.listdir(tempdir))
- 0
- >>> f = myblob.open('w')
- >>> len(os.listdir(tempdir))
- 1
- >>> f.close()
- >>> shutil.rmtree(tempdir)
- >>> del os.environ['ZODB_BLOB_TEMPDIR']
-
-Some cleanup in this test is needed:
-
- >>> import transaction
- >>> transaction.get().abort()
Deleted: ZODB/trunk/src/ZODB/Blobs/tests/connection.txt
===================================================================
--- ZODB/trunk/src/ZODB/Blobs/tests/connection.txt 2007-06-03 05:57:35 UTC (rev 76191)
+++ ZODB/trunk/src/ZODB/Blobs/tests/connection.txt 2007-06-03 08:26:47 UTC (rev 76192)
@@ -1,83 +0,0 @@
-##############################################################################
-#
-# Copyright (c) 2005 Zope Corporation and Contributors.
-# All Rights Reserved.
-#
-# This software is subject to the provisions of the Zope Public License,
-# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
-# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
-# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
-# FOR A PARTICULAR PURPOSE.
-#
-##############################################################################
-
-Connection support for Blobs tests
-==================================
-
-Connections handle Blobs specially. To demonstrate that, we first need a Blob with some data:
-
- >>> from ZODB.Blobs.interfaces import IBlob
- >>> from ZODB.Blobs.Blob import Blob
- >>> import transaction
- >>> blob = Blob()
- >>> data = blob.open("w")
- >>> data.write("I'm a happy Blob.")
- >>> data.close()
-
-We also need a database with a blob supporting storage:
-
- >>> from ZODB.MappingStorage import MappingStorage
- >>> from ZODB.Blobs.BlobStorage import BlobStorage
- >>> from ZODB.DB import DB
- >>> from tempfile import mkdtemp
- >>> base_storage = MappingStorage("test")
- >>> blob_dir = mkdtemp()
- >>> blob_storage = BlobStorage(blob_dir, base_storage)
- >>> database = DB(blob_storage)
-
-Putting a Blob into a Connection works like every other object:
-
- >>> connection = database.open()
- >>> root = connection.root()
- >>> root['myblob'] = blob
- >>> transaction.commit()
-
-We can also commit a transaction that seats a blob into place without
-calling the blob's open method (this currently fails):
-
- >>> nothing = transaction.begin()
- >>> anotherblob = Blob()
- >>> root['anotherblob'] = anotherblob
- >>> nothing = transaction.commit()
-
-Getting stuff out of there works similar:
-
- >>> connection2 = database.open()
- >>> root = connection2.root()
- >>> blob2 = root['myblob']
- >>> IBlob.providedBy(blob2)
- True
- >>> blob2.open("r").read()
- "I'm a happy Blob."
-
-You can't put blobs into a database that has uses a Non-Blob-Storage, though:
-
- >>> no_blob_storage = MappingStorage()
- >>> database2 = DB(no_blob_storage)
- >>> connection3 = database2.open()
- >>> root = connection3.root()
- >>> root['myblob'] = Blob()
- >>> transaction.commit() # doctest: +ELLIPSIS
- Traceback (most recent call last):
- ...
- Unsupported: Storing Blobs in <ZODB.MappingStorage.MappingStorage instance at ...> is not supported.
-
-While we are testing this, we don't need the storage directory and
-databases anymore:
-
- >>> import shutil
- >>> shutil.rmtree(blob_dir)
- >>> transaction.abort()
- >>> database.close()
- >>> database2.close()
Deleted: ZODB/trunk/src/ZODB/Blobs/tests/consume.txt
===================================================================
--- ZODB/trunk/src/ZODB/Blobs/tests/consume.txt 2007-06-03 05:57:35 UTC (rev 76191)
+++ ZODB/trunk/src/ZODB/Blobs/tests/consume.txt 2007-06-03 08:26:47 UTC (rev 76192)
@@ -1,109 +0,0 @@
-Consuming existing files
-========================
-
-The ZODB Blob implementation allows to import existing files as Blobs within
-an O(1) operation we call `consume`::
-
-Let's create a file::
-
- >>> to_import = open('to_import', 'wb')
- >>> to_import.write("I'm a Blob and I feel fine.")
-
-The file *must* be closed before giving it to consumeFile:
-
- >>> to_import.close()
-
-Now, let's consume this file in a blob by specifying it's name::
-
- >>> from ZODB.Blobs.Blob import Blob
- >>> blob = Blob()
- >>> blob.consumeFile('to_import')
-
-After the consumeFile operation, the original file has been removed:
-
- >>> import os
- >>> os.path.exists('to_import')
- False
-
-We now can call open on the blob and read and write the data::
-
- >>> blob_read = blob.open('r')
- >>> blob_read.read()
- "I'm a Blob and I feel fine."
- >>> blob_read.close()
- >>> blob_write = blob.open('w')
- >>> blob_write.write('I was changed.')
- >>> blob_write.close()
-
-We can not consume a file when there is a reader or writer around for a blob
-already::
-
- >>> open('to_import', 'wb').write('I am another blob.')
- >>> blob_read = blob.open('r')
- >>> blob.consumeFile('to_import')
- Traceback (most recent call last):
- BlobError: Already opened for reading.
- >>> blob_read.close()
- >>> blob_write = blob.open('w')
- >>> blob.consumeFile('to_import')
- Traceback (most recent call last):
- BlobError: Already opened for writing.
- >>> blob_write.close()
-
-Now, after closing all readers and writers we can consume files again::
-
- >>> blob.consumeFile('to_import')
- >>> blob_read = blob.open('r')
- >>> blob_read.read()
- 'I am another blob.'
-
-
-Edge cases
-==========
-
-There are some edge cases what happens when the link() operation
-fails. We simulate this in different states:
-
-Case 1: We don't have uncommitted data, but the link operation fails. The
-exception will be re-raised and the target file will not exist::
-
- >>> open('to_import', 'wb').write('Some data.')
-
- >>> def failing_link(self, filename):
- ... raise Exception("I can't link.")
-
- >>> blob = Blob()
- >>> blob.open('r')
- Traceback (most recent call last):
- BlobError: Blob does not exist.
-
- >>> blob._os_link = failing_link
- >>> blob.consumeFile('to_import')
- Traceback (most recent call last):
- Exception: I can't link.
-
-The blob did not exist before, so it shouldn't exist now::
-
- >>> blob.open('r')
- Traceback (most recent call last):
- BlobError: Blob does not exist.
-
-Case 2: We thave uncommitted data, but the link operation fails. The
-exception will be re-raised and the target file will exist with the previous
-uncomitted data::
-
- >>> blob = Blob()
- >>> blob_writing = blob.open('w')
- >>> blob_writing.write('Uncommitted data')
- >>> blob_writing.close()
-
- >>> blob._os_link = failing_link
- >>> blob.consumeFile('to_import')
- Traceback (most recent call last):
- Exception: I can't link.
-
-The blob did existed before and had uncommitted data, this shouldn't have
-changed::
-
- >>> blob.open('r').read()
- 'Uncommitted data'
Deleted: ZODB/trunk/src/ZODB/Blobs/tests/importexport.txt
===================================================================
--- ZODB/trunk/src/ZODB/Blobs/tests/importexport.txt 2007-06-03 05:57:35 UTC (rev 76191)
+++ ZODB/trunk/src/ZODB/Blobs/tests/importexport.txt 2007-06-03 08:26:47 UTC (rev 76192)
@@ -1,102 +0,0 @@
-##############################################################################
-#
-# Copyright (c) 2005 Zope Corporation and Contributors.
-# All Rights Reserved.
-#
-# This software is subject to the provisions of the Zope Public License,
-# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
-# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
-# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
-# FOR A PARTICULAR PURPOSE.
-#
-##############################################################################
-
-Import/export support for blob data
-===================================
-
-Set up:
-
- >>> from ZODB.FileStorage import FileStorage
- >>> from ZODB.Blobs.BlobStorage import BlobStorage
- >>> from ZODB.Blobs.Blob import Blob
- >>> from ZODB.DB import DB
- >>> from persistent.mapping import PersistentMapping
- >>> import shutil
- >>> import transaction
- >>> from tempfile import mkdtemp, mktemp
- >>> storagefile1 = mktemp()
- >>> blob_dir1 = mkdtemp()
- >>> storagefile2 = mktemp()
- >>> blob_dir2 = mkdtemp()
-
-We need an database with an undoing blob supporting storage:
-
- >>> base_storage1 = FileStorage(storagefile1)
- >>> blob_storage1 = BlobStorage(blob_dir1, base_storage1)
- >>> base_storage2 = FileStorage(storagefile2)
- >>> blob_storage2 = BlobStorage(blob_dir2, base_storage2)
- >>> database1 = DB(blob_storage1)
- >>> database2 = DB(blob_storage2)
-
-Create our root object for database1:
-
- >>> connection1 = database1.open()
- >>> root1 = connection1.root()
-
-Put a couple blob objects in our database1 and on the filesystem:
-
- >>> import time, os
- >>> nothing = transaction.begin()
- >>> tid = blob_storage1._tid
- >>> data1 = 'x'*100000
- >>> blob1 = Blob()
- >>> blob1.open('w').write(data1)
- >>> data2 = 'y'*100000
- >>> blob2 = Blob()
- >>> blob2.open('w').write(data2)
- >>> d = PersistentMapping({'blob1':blob1, 'blob2':blob2})
- >>> root1['blobdata'] = d
- >>> transaction.commit()
-
-Export our blobs from a database1 connection:
-
- >>> conn = root1['blobdata']._p_jar
- >>> oid = root1['blobdata']._p_oid
- >>> exportfile = mktemp()
- >>> nothing = connection1.exportFile(oid, exportfile)
-
-Import our exported data into database2:
-
- >>> connection2 = database2.open()
- >>> root2 = connection2.root()
- >>> nothing = transaction.begin()
- >>> data = root2._p_jar.importFile(exportfile)
- >>> root2['blobdata'] = data
- >>> transaction.commit()
-
-Make sure our data exists:
-
- >>> items1 = root1['blobdata']
- >>> items2 = root2['blobdata']
- >>> bool(items1.keys() == items2.keys())
- True
- >>> items1['blob1'].open().read() == items2['blob1'].open().read()
- True
- >>> items1['blob2'].open().read() == items2['blob2'].open().read()
- True
- >>> transaction.get().abort()
-
-Clean up our blob directory:
-
- >>> base_storage1.close()
- >>> base_storage2.close()
- >>> shutil.rmtree(blob_dir1)
- >>> shutil.rmtree(blob_dir2)
- >>> os.unlink(exportfile)
- >>> os.unlink(storagefile1)
- >>> os.unlink(storagefile1+".index")
- >>> os.unlink(storagefile1+".tmp")
- >>> os.unlink(storagefile2)
- >>> os.unlink(storagefile2+".index")
- >>> os.unlink(storagefile2+".tmp")
Deleted: ZODB/trunk/src/ZODB/Blobs/tests/packing.txt
===================================================================
--- ZODB/trunk/src/ZODB/Blobs/tests/packing.txt 2007-06-03 05:57:35 UTC (rev 76191)
+++ ZODB/trunk/src/ZODB/Blobs/tests/packing.txt 2007-06-03 08:26:47 UTC (rev 76192)
@@ -1,255 +0,0 @@
-##############################################################################
-#
-# Copyright (c) 2005 Zope Corporation and Contributors.
-# All Rights Reserved.
-#
-# This software is subject to the provisions of the Zope Public License,
-# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
-# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
-# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
-# FOR A PARTICULAR PURPOSE.
-#
-##############################################################################
-
-Packing support for blob data
-=============================
-
-Set up:
-
- >>> from ZODB.FileStorage import FileStorage
- >>> from ZODB.MappingStorage import MappingStorage
- >>> from ZODB.serialize import referencesf
- >>> from ZODB.Blobs.BlobStorage import BlobStorage
- >>> from ZODB.Blobs.Blob import Blob
- >>> from ZODB import utils
- >>> from ZODB.DB import DB
- >>> import shutil
- >>> import transaction
- >>> from tempfile import mkdtemp, mktemp
- >>> storagefile = mktemp()
- >>> blob_dir = mkdtemp()
-
-A helper method to assure a unique timestamp across multiple platforms. This
-method also makes sure that after retrieving a timestamp that was *before* a
-transaction was committed, that at least one second passes so the packing time
-actually is before the commit time.
-
- >>> import time
- >>> def new_time():
- ... now = new_time = time.time()
- ... while new_time <= now:
- ... new_time = time.time()
- ... time.sleep(1)
- ... return new_time
-
-UNDOING
-=======
-
-We need a database with an undoing blob supporting storage:
-
- >>> base_storage = FileStorage(storagefile)
- >>> blob_storage = BlobStorage(blob_dir, base_storage)
- >>> database = DB(blob_storage)
-
-Create our root object:
-
- >>> connection1 = database.open()
- >>> root = connection1.root()
-
-Put some revisions of a blob object in our database and on the filesystem:
-
- >>> import os
- >>> tids = []
- >>> times = []
- >>> nothing = transaction.begin()
- >>> times.append(new_time())
- >>> blob = Blob()
- >>> blob.open('w').write('this is blob data 0')
- >>> root['blob'] = blob
- >>> transaction.commit()
- >>> tids.append(blob_storage._tid)
-
- >>> nothing = transaction.begin()
- >>> times.append(new_time())
- >>> root['blob'].open('w').write('this is blob data 1')
- >>> transaction.commit()
- >>> tids.append(blob_storage._tid)
-
- >>> nothing = transaction.begin()
- >>> times.append(new_time())
- >>> root['blob'].open('w').write('this is blob data 2')
- >>> transaction.commit()
- >>> tids.append(blob_storage._tid)
-
- >>> nothing = transaction.begin()
- >>> times.append(new_time())
- >>> root['blob'].open('w').write('this is blob data 3')
- >>> transaction.commit()
- >>> tids.append(blob_storage._tid)
-
- >>> nothing = transaction.begin()
- >>> times.append(new_time())
- >>> root['blob'].open('w').write('this is blob data 4')
- >>> transaction.commit()
- >>> tids.append(blob_storage._tid)
-
- >>> oid = root['blob']._p_oid
- >>> fns = [ blob_storage.fshelper.getBlobFilename(oid, x) for x in tids ]
- >>> [ os.path.exists(x) for x in fns ]
- [True, True, True, True, True]
-
-Do a pack to the slightly before the first revision was written:
-
- >>> packtime = times[0]
- >>> blob_storage.pack(packtime, referencesf)
- >>> [ os.path.exists(x) for x in fns ]
- [True, True, True, True, True]
-
-Do a pack to the slightly before the second revision was written:
-
- >>> packtime = times[1]
- >>> blob_storage.pack(packtime, referencesf)
- >>> [ os.path.exists(x) for x in fns ]
- [True, True, True, True, True]
-
-Do a pack to the slightly before the third revision was written:
-
- >>> packtime = times[2]
- >>> blob_storage.pack(packtime, referencesf)
- >>> [ os.path.exists(x) for x in fns ]
- [False, True, True, True, True]
-
-Do a pack to the slightly before the fourth revision was written:
-
- >>> packtime = times[3]
- >>> blob_storage.pack(packtime, referencesf)
- >>> [ os.path.exists(x) for x in fns ]
- [False, False, True, True, True]
-
-Do a pack to the slightly before the fifth revision was written:
-
- >>> packtime = times[4]
- >>> blob_storage.pack(packtime, referencesf)
- >>> [ os.path.exists(x) for x in fns ]
- [False, False, False, True, True]
-
-Do a pack to now:
-
- >>> packtime = new_time()
- >>> blob_storage.pack(packtime, referencesf)
- >>> [ os.path.exists(x) for x in fns ]
- [False, False, False, False, True]
-
-Delete the object and do a pack, it should get rid of the most current
-revision as well as the entire directory:
-
- >>> nothing = transaction.begin()
- >>> del root['blob']
- >>> transaction.commit()
- >>> packtime = new_time()
- >>> blob_storage.pack(packtime, referencesf)
- >>> [ os.path.exists(x) for x in fns ]
- [False, False, False, False, False]
- >>> os.path.exists(os.path.split(fns[0])[0])
- False
-
-Clean up our blob directory and database:
-
- >>> shutil.rmtree(blob_dir)
- >>> base_storage.close()
- >>> os.unlink(storagefile)
- >>> os.unlink(storagefile+".index")
- >>> os.unlink(storagefile+".tmp")
- >>> os.unlink(storagefile+".old")
-
-NON-UNDOING
-===========
-
-We need an database with a NON-undoing blob supporting storage:
-
- >>> base_storage = MappingStorage('storage')
- >>> blob_storage = BlobStorage(blob_dir, base_storage)
- >>> database = DB(blob_storage)
-
-Create our root object:
-
- >>> connection1 = database.open()
- >>> root = connection1.root()
-
-Put some revisions of a blob object in our database and on the filesystem:
-
- >>> import time, os
- >>> tids = []
- >>> times = []
- >>> nothing = transaction.begin()
- >>> times.append(new_time())
- >>> blob = Blob()
- >>> blob.open('w').write('this is blob data 0')
- >>> root['blob'] = blob
- >>> transaction.commit()
- >>> tids.append(blob_storage._tid)
-
- >>> nothing = transaction.begin()
- >>> times.append(new_time())
- >>> root['blob'].open('w').write('this is blob data 1')
- >>> transaction.commit()
- >>> tids.append(blob_storage._tid)
-
- >>> nothing = transaction.begin()
- >>> times.append(new_time())
- >>> root['blob'].open('w').write('this is blob data 2')
- >>> transaction.commit()
- >>> tids.append(blob_storage._tid)
-
- >>> nothing = transaction.begin()
- >>> times.append(new_time())
- >>> root['blob'].open('w').write('this is blob data 3')
- >>> transaction.commit()
- >>> tids.append(blob_storage._tid)
-
- >>> nothing = transaction.begin()
- >>> times.append(new_time())
- >>> root['blob'].open('w').write('this is blob data 4')
- >>> transaction.commit()
- >>> tids.append(blob_storage._tid)
-
- >>> oid = root['blob']._p_oid
- >>> fns = [ blob_storage.fshelper.getBlobFilename(oid, x) for x in tids ]
- >>> [ os.path.exists(x) for x in fns ]
- [True, True, True, True, True]
-
-Get our blob filenames for this oid.
-
- >>> fns = [ blob_storage.fshelper.getBlobFilename(oid, x) for x in tids ]
-
-Do a pack to the slightly before the first revision was written:
-
- >>> packtime = times[0]
- >>> blob_storage.pack(packtime, referencesf)
- >>> [ os.path.exists(x) for x in fns ]
- [False, False, False, False, True]
-
-Do a pack to now:
-
- >>> packtime = new_time()
- >>> blob_storage.pack(packtime, referencesf)
- >>> [ os.path.exists(x) for x in fns ]
- [False, False, False, False, True]
-
-Delete the object and do a pack, it should get rid of the most current
-revision as well as the entire directory:
-
- >>> nothing = transaction.begin()
- >>> del root['blob']
- >>> transaction.commit()
- >>> packtime = new_time()
- >>> blob_storage.pack(packtime, referencesf)
- >>> [ os.path.exists(x) for x in fns ]
- [False, False, False, False, False]
- >>> os.path.exists(os.path.split(fns[0])[0])
- False
-
-Clean up our blob directory:
-
- >>> shutil.rmtree(blob_dir)
Deleted: ZODB/trunk/src/ZODB/Blobs/tests/test_config.py
===================================================================
--- ZODB/trunk/src/ZODB/Blobs/tests/test_config.py 2007-06-03 05:57:35 UTC (rev 76191)
+++ ZODB/trunk/src/ZODB/Blobs/tests/test_config.py 2007-06-03 08:26:47 UTC (rev 76192)
@@ -1,83 +0,0 @@
-##############################################################################
-#
-# Copyright (c) 2004-2006 Zope Corporation and Contributors.
-# All Rights Reserved.
-#
-# This software is subject to the provisions of the Zope Public License,
-# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
-# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
-# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
-# FOR A PARTICULAR PURPOSE.
-#
-##############################################################################
-import tempfile, shutil, unittest
-import os
-
-from ZODB.tests.testConfig import ConfigTestBase
-from ZConfig import ConfigurationSyntaxError
-
-
-class BlobConfigTestBase(ConfigTestBase):
-
- def setUp(self):
- super(BlobConfigTestBase, self).setUp()
-
- self.blob_dir = tempfile.mkdtemp()
-
- def tearDown(self):
- super(BlobConfigTestBase, self).tearDown()
-
- shutil.rmtree(self.blob_dir)
-
-
-class ZODBBlobConfigTest(BlobConfigTestBase):
-
- def test_map_config1(self):
- self._test(
- """
- <zodb>
- <blobstorage>
- blob-dir %s
- <mappingstorage/>
- </blobstorage>
- </zodb>
- """ % self.blob_dir)
-
- def test_file_config1(self):
- path = tempfile.mktemp()
- self._test(
- """
- <zodb>
- <blobstorage>
- blob-dir %s
- <filestorage>
- path %s
- </filestorage>
- </blobstorage>
- </zodb>
- """ %(self.blob_dir, path))
- os.unlink(path)
- os.unlink(path+".index")
- os.unlink(path+".tmp")
-
- def test_blob_dir_needed(self):
- self.assertRaises(ConfigurationSyntaxError,
- self._test,
- """
- <zodb>
- <blobstorage>
- <mappingstorage/>
- </blobstorage>
- </zodb>
- """)
-
-
-def test_suite():
- suite = unittest.TestSuite()
- suite.addTest(unittest.makeSuite(ZODBBlobConfigTest))
-
- return suite
-
-if __name__ == '__main__':
- unittest.main(defaultTest = 'test_suite')
Deleted: ZODB/trunk/src/ZODB/Blobs/tests/test_doctests.py
===================================================================
--- ZODB/trunk/src/ZODB/Blobs/tests/test_doctests.py 2007-06-03 05:57:35 UTC (rev 76191)
+++ ZODB/trunk/src/ZODB/Blobs/tests/test_doctests.py 2007-06-03 08:26:47 UTC (rev 76192)
@@ -1,24 +0,0 @@
-##############################################################################
-#
-# Copyright (c) 2004 Zope Corporation and Contributors.
-# All Rights Reserved.
-#
-# This software is subject to the provisions of the Zope Public License,
-# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
-# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
-# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
-# FOR A PARTICULAR PURPOSE.
-#
-##############################################################################
-
-from zope.testing import doctest
-import ZODB.tests.util
-
-def test_suite():
- return doctest.DocFileSuite(
- "basic.txt", "connection.txt", "transaction.txt",
- "packing.txt", "importexport.txt", "consume.txt",
- setUp=ZODB.tests.util.setUp,
- tearDown=ZODB.tests.util.tearDown,
- )
Deleted: ZODB/trunk/src/ZODB/Blobs/tests/test_undo.py
===================================================================
--- ZODB/trunk/src/ZODB/Blobs/tests/test_undo.py 2007-06-03 05:57:35 UTC (rev 76191)
+++ ZODB/trunk/src/ZODB/Blobs/tests/test_undo.py 2007-06-03 08:26:47 UTC (rev 76192)
@@ -1,220 +0,0 @@
-##############################################################################
-#
-# Copyright (c) 2004 Zope Corporation and Contributors.
-# All Rights Reserved.
-#
-# This software is subject to the provisions of the Zope Public License,
-# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
-# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
-# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
-# FOR A PARTICULAR PURPOSE.
-#
-##############################################################################
-import unittest
-import tempfile
-import os
-import shutil
-import base64
-
-from ZODB.FileStorage import FileStorage
-from ZODB.Blobs.BlobStorage import BlobStorage
-from ZODB.Blobs.Blob import Blob
-from ZODB.DB import DB
-import transaction
-from ZODB.Blobs.Blob import Blob
-from ZODB import utils
-
-class BlobUndoTests(unittest.TestCase):
-
- def setUp(self):
- self.test_dir = tempfile.mkdtemp()
- self.here = os.getcwd()
- os.chdir(self.test_dir)
- self.storagefile = 'Data.fs'
- os.mkdir('blobs')
- self.blob_dir = 'blobs'
-
- def tearDown(self):
- os.chdir(self.here)
- shutil.rmtree(self.test_dir)
-
- def testUndoWithoutPreviousVersion(self):
- base_storage = FileStorage(self.storagefile)
- blob_storage = BlobStorage(self.blob_dir, base_storage)
- database = DB(blob_storage)
- connection = database.open()
- root = connection.root()
- transaction.begin()
- root['blob'] = Blob()
- transaction.commit()
-
- serial = base64.encodestring(blob_storage._tid)
-
- # undo the creation of the previously added blob
- transaction.begin()
- database.undo(serial, blob_storage._transaction)
- transaction.commit()
-
- connection.close()
- connection = database.open()
- root = connection.root()
- # the blob footprint object should exist no longer
- self.assertRaises(KeyError, root.__getitem__, 'blob')
- database.close()
-
- def testUndo(self):
- base_storage = FileStorage(self.storagefile)
- blob_storage = BlobStorage(self.blob_dir, base_storage)
- database = DB(blob_storage)
- connection = database.open()
- root = connection.root()
- transaction.begin()
- blob = Blob()
- blob.open('w').write('this is state 1')
- root['blob'] = blob
- transaction.commit()
-
- transaction.begin()
- blob = root['blob']
- blob.open('w').write('this is state 2')
- transaction.commit()
-
- transaction.begin()
- blob = root['blob']
- self.assertEqual(blob.open('r').read(), 'this is state 2')
- transaction.abort()
-
- serial = base64.encodestring(blob_storage._tid)
-
- transaction.begin()
- blob_storage.undo(serial, blob_storage._transaction)
- transaction.commit()
-
- transaction.begin()
- blob = root['blob']
- self.assertEqual(blob.open('r').read(), 'this is state 1')
- transaction.abort()
- database.close()
-
- def testUndoAfterConsumption(self):
- base_storage = FileStorage(self.storagefile)
- blob_storage = BlobStorage(self.blob_dir, base_storage)
- database = DB(blob_storage)
- connection = database.open()
- root = connection.root()
- transaction.begin()
- open('consume1', 'w').write('this is state 1')
- blob = Blob()
- blob.consumeFile('consume1')
- root['blob'] = blob
- transaction.commit()
-
- transaction.begin()
- blob = root['blob']
- open('consume2', 'w').write('this is state 2')
- blob.consumeFile('consume2')
- transaction.commit()
-
- transaction.begin()
- blob = root['blob']
- self.assertEqual(blob.open('r').read(), 'this is state 2')
- transaction.abort()
-
- serial = base64.encodestring(blob_storage._tid)
-
- transaction.begin()
- blob_storage.undo(serial, blob_storage._transaction)
- transaction.commit()
-
- transaction.begin()
- blob = root['blob']
- self.assertEqual(blob.open('r').read(), 'this is state 1')
- transaction.abort()
-
- database.close()
-
- def testRedo(self):
- base_storage = FileStorage(self.storagefile)
- blob_storage = BlobStorage(self.blob_dir, base_storage)
- database = DB(blob_storage)
- connection = database.open()
- root = connection.root()
- blob = Blob()
-
- transaction.begin()
- blob.open('w').write('this is state 1')
- root['blob'] = blob
- transaction.commit()
-
- transaction.begin()
- blob = root['blob']
- blob.open('w').write('this is state 2')
- transaction.commit()
-
- serial = base64.encodestring(blob_storage._tid)
-
- transaction.begin()
- database.undo(serial)
- transaction.commit()
-
- transaction.begin()
- blob = root['blob']
- self.assertEqual(blob.open('r').read(), 'this is state 1')
- transaction.abort()
-
- serial = base64.encodestring(blob_storage._tid)
-
- transaction.begin()
- database.undo(serial)
- transaction.commit()
-
- transaction.begin()
- blob = root['blob']
- self.assertEqual(blob.open('r').read(), 'this is state 2')
- transaction.abort()
-
- database.close()
-
- def testRedoOfCreation(self):
- base_storage = FileStorage(self.storagefile)
- blob_storage = BlobStorage(self.blob_dir, base_storage)
- database = DB(blob_storage)
- connection = database.open()
- root = connection.root()
- blob = Blob()
-
- transaction.begin()
- blob.open('w').write('this is state 1')
- root['blob'] = blob
- transaction.commit()
-
- serial = base64.encodestring(blob_storage._tid)
-
- transaction.begin()
- database.undo(serial)
- transaction.commit()
-
- self.assertRaises(KeyError, root.__getitem__, 'blob')
-
- serial = base64.encodestring(blob_storage._tid)
-
- transaction.begin()
- database.undo(serial)
- transaction.commit()
-
- transaction.begin()
- blob = root['blob']
- self.assertEqual(blob.open('r').read(), 'this is state 1')
- transaction.abort()
-
- database.close()
-
-def test_suite():
- suite = unittest.TestSuite()
- suite.addTest(unittest.makeSuite(BlobUndoTests))
-
- return suite
-
-if __name__ == '__main__':
- unittest.main(defaultTest = 'test_suite')
Deleted: ZODB/trunk/src/ZODB/Blobs/tests/transaction.txt
===================================================================
--- ZODB/trunk/src/ZODB/Blobs/tests/transaction.txt 2007-06-03 05:57:35 UTC (rev 76191)
+++ ZODB/trunk/src/ZODB/Blobs/tests/transaction.txt 2007-06-03 08:26:47 UTC (rev 76192)
@@ -1,316 +0,0 @@
-##############################################################################
-#
-# Copyright (c) 2005-2007 Zope Corporation and Contributors.
-# All Rights Reserved.
-#
-# This software is subject to the provisions of the Zope Public License,
-# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
-# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
-# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
-# FOR A PARTICULAR PURPOSE.
-#
-##############################################################################
-
-Transaction support for Blobs
-=============================
-
-We need a database with a blob supporting storage::
-
- >>> from ZODB.MappingStorage import MappingStorage
- >>> from ZODB.Blobs.BlobStorage import BlobStorage
- >>> from ZODB.DB import DB
- >>> import transaction
- >>> import tempfile
- >>> from tempfile import mkdtemp
- >>> base_storage = MappingStorage("test")
- >>> blob_dir = mkdtemp()
- >>> blob_storage = BlobStorage(blob_dir, base_storage)
- >>> database = DB(blob_storage)
- >>> connection1 = database.open()
- >>> root1 = connection1.root()
- >>> from ZODB.Blobs.Blob import Blob
-
-Putting a Blob into a Connection works like any other Persistent object::
-
- >>> blob1 = Blob()
- >>> blob1.open('w').write('this is blob 1')
- >>> root1['blob1'] = blob1
- >>> transaction.commit()
-
-Aborting a transaction involving a blob write cleans up uncommitted
-file data::
-
- >>> dead_blob = Blob()
- >>> dead_blob.open('w').write('this is a dead blob')
- >>> root1['dead_blob'] = dead_blob
- >>> fname = dead_blob._p_blob_uncommitted
- >>> import os
- >>> os.path.exists(fname)
- True
- >>> transaction.abort()
- >>> os.path.exists(fname)
- False
-
-Opening a blob gives us a filehandle. Getting data out of the
-resulting filehandle is accomplished via the filehandle's read method::
-
- >>> connection2 = database.open()
- >>> root2 = connection2.root()
- >>> blob1a = root2['blob1']
- >>> blob1a._p_blob_refcounts()
- (0, 0)
- >>>
- >>> blob1afh1 = blob1a.open("r")
- >>> blob1afh1.read()
- 'this is blob 1'
- >>> # The filehandle keeps a reference to its blob object
- >>> blob1afh1.blob._p_blob_refcounts()
- (1, 0)
-
-Let's make another filehandle for read only to blob1a, this should bump
-up its refcount by one, and each file handle has a reference to the
-(same) underlying blob::
-
- >>> blob1afh2 = blob1a.open("r")
- >>> blob1afh2.blob._p_blob_refcounts()
- (2, 0)
- >>> blob1afh1.blob._p_blob_refcounts()
- (2, 0)
- >>> blob1afh2.blob is blob1afh1.blob
- True
-
-Let's close the first filehandle we got from the blob, this should decrease
-its refcount by one::
-
- >>> blob1afh1.close()
- >>> blob1a._p_blob_refcounts()
- (1, 0)
-
-Let's abort this transaction, and ensure that the filehandles that we
-opened are now closed and that the filehandle refcounts on the blob
-object are cleared::
-
- >>> transaction.abort()
- >>> blob1afh1.blob._p_blob_refcounts()
- (0, 0)
- >>> blob1afh2.blob._p_blob_refcounts()
- (0, 0)
- >>> blob1a._p_blob_refcounts()
- (0, 0)
- >>> blob1afh2.read()
- Traceback (most recent call last):
- ...
- ValueError: I/O operation on closed file
-
-If we open a blob for append, its write refcount should be nonzero.
-Additionally, writing any number of bytes to the blobfile should
-result in the blob being marked "dirty" in the connection (we just
-aborted above, so the object should be "clean" when we start)::
-
- >>> bool(blob1a._p_changed)
- False
- >>> blob1a.open('r').read()
- 'this is blob 1'
- >>> blob1afh3 = blob1a.open('a')
- >>> blob1afh3.write('woot!')
- >>> blob1a._p_blob_refcounts()
- (0, 1)
- >>> bool(blob1a._p_changed)
- True
-
-We can open more than one blob object during the course of a single
-transaction::
-
- >>> blob2 = Blob()
- >>> blob2.open('w').write('this is blob 3')
- >>> root2['blob2'] = blob2
- >>> transaction.commit()
- >>> blob2._p_blob_refcounts()
- (0, 0)
- >>> blob1._p_blob_refcounts()
- (0, 0)
-
-Since we committed the current transaction above, the aggregate
-changes we've made to blob, blob1a (these refer to the same object) and
-blob2 (a different object) should be evident::
-
- >>> blob1.open('r').read()
- 'this is blob 1woot!'
- >>> blob1a.open('r').read()
- 'this is blob 1woot!'
- >>> blob2.open('r').read()
- 'this is blob 3'
-
-We shouldn't be able to persist a blob filehandle at commit time
-(although the exception which is raised when an object cannot be
-pickled appears to be particulary unhelpful for casual users at the
-moment)::
-
- >>> root1['wontwork'] = blob1.open('r')
- >>> transaction.commit()
- Traceback (most recent call last):
- ...
- TypeError: coercing to Unicode: need string or buffer, BlobFile found
-
-Abort for good measure::
-
- >>> transaction.abort()
-
-Attempting to change a blob simultaneously from two different
-connections should result in a write conflict error::
-
- >>> tm1 = transaction.TransactionManager()
- >>> tm2 = transaction.TransactionManager()
- >>> root3 = database.open(transaction_manager=tm1).root()
- >>> root4 = database.open(transaction_manager=tm2).root()
- >>> blob1c3 = root3['blob1']
- >>> blob1c4 = root4['blob1']
- >>> blob1c3fh1 = blob1c3.open('a')
- >>> blob1c4fh1 = blob1c4.open('a')
- >>> blob1c3fh1.write('this is from connection 3')
- >>> blob1c4fh1.write('this is from connection 4')
- >>> tm1.get().commit()
- >>> root3['blob1'].open('r').read()
- 'this is blob 1woot!this is from connection 3'
- >>> tm2.get().commit()
- Traceback (most recent call last):
- ...
- ConflictError: database conflict error (oid 0x01, class ZODB.Blobs.Blob.Blob)
-
-After the conflict, the winning transaction's result is visible on both
-connections::
-
- >>> root3['blob1'].open('r').read()
- 'this is blob 1woot!this is from connection 3'
- >>> tm2.get().abort()
- >>> root4['blob1'].open('r').read()
- 'this is blob 1woot!this is from connection 3'
-
-BlobStorages implementation of getSize() includes the blob data and adds it to
-the underlying storages result of getSize(). (We need to ensure the last
-number to be an int, otherwise it will be a long on 32-bit platforms and an
-int on 64-bit)::
-
- >>> underlying_size = base_storage.getSize()
- >>> blob_size = blob_storage.getSize()
- >>> int(blob_size - underlying_size)
- 91
-
-
-Savepoints and Blobs
---------------------
-
-We do support optimistic savepoints ::
-
- >>> connection5 = database.open()
- >>> root5 = connection5.root()
- >>> blob = Blob()
- >>> blob_fh = blob.open("w")
- >>> blob_fh.write("I'm a happy blob.")
- >>> blob_fh.close()
- >>> root5['blob'] = blob
- >>> transaction.commit()
- >>> root5['blob'].open("r").read()
- "I'm a happy blob."
- >>> blob_fh = root5['blob'].open("a")
- >>> blob_fh.write(" And I'm singing.")
- >>> blob_fh.close()
- >>> root5['blob'].open("r").read()
- "I'm a happy blob. And I'm singing."
- >>> savepoint = transaction.savepoint(optimistic=True)
- >>> root5['blob'].open("r").read()
- "I'm a happy blob. And I'm singing."
- >>> transaction.get().commit()
-
-We do not support non-optimistic savepoints::
-
- >>> blob_fh = root5['blob'].open("a")
- >>> blob_fh.write(" And the weather is beautiful.")
- >>> blob_fh.close()
- >>> root5['blob'].open("r").read()
- "I'm a happy blob. And I'm singing. And the weather is beautiful."
- >>> savepoint = transaction.savepoint() # doctest: +ELLIPSIS
- Traceback (most recent call last):
- ...
- TypeError: ('Savepoints unsupported', <ZODB.Blobs.Blob.BlobDataManager instance at 0x...>)
- >>> transaction.abort()
-
-Reading Blobs outside of a transaction
---------------------------------------
-
-If you want to read from a Blob outside of transaction boundaries (e.g. to
-stream a file to the browser), you can use the openDetached() method::
-
- >>> connection6 = database.open()
- >>> root6 = connection6.root()
- >>> blob = Blob()
- >>> blob_fh = blob.open("w")
- >>> blob_fh.write("I'm a happy blob.")
- >>> blob_fh.close()
- >>> root6['blob'] = blob
- >>> transaction.commit()
- >>> blob.openDetached().read()
- "I'm a happy blob."
-
-Of course, that doesn't work for empty blobs::
-
- >>> blob = Blob()
- >>> blob.openDetached()
- Traceback (most recent call last):
- ...
- BlobError: Blob does not exist.
-
-nor when the Blob is already opened for writing::
-
- >>> blob = Blob()
- >>> blob_fh = blob.open("w")
- >>> blob.openDetached()
- Traceback (most recent call last):
- ...
- BlobError: Already opened for writing.
-
-You can also pass a factory to the openDetached method that will be used to
-instantiate the file. This is used for e.g. creating filestream iterators::
-
- >>> class customfile(file):
- ... pass
- >>> blob_fh.write('Something')
- >>> blob_fh.close()
- >>> fh = blob.openDetached(customfile)
- >>> fh # doctest: +ELLIPSIS
- <open file '...', mode 'rb' at 0x...>
- >>> isinstance(fh, customfile)
- True
-
-
-Note: Nasty people could use a factory that opens the file for writing. This
-would be evil.
-
-It does work when the transaction was aborted, though::
-
- >>> blob = Blob()
- >>> blob_fh = blob.open("w")
- >>> blob_fh.write("I'm a happy blob.")
- >>> blob_fh.close()
- >>> root6['blob'] = blob
- >>> transaction.commit()
-
- >>> blob_fh = blob.open("w")
- >>> blob_fh.write("And I'm singing.")
- >>> blob_fh.close()
- >>> transaction.abort()
- >>> blob.openDetached().read()
- "I'm a happy blob."
-
-
-Teardown
---------
-
-We don't need the storage directory and databases anymore::
-
- >>> import shutil
- >>> shutil.rmtree(blob_dir)
- >>> tm1.get().abort()
- >>> tm2.get().abort()
- >>> database.close()
Modified: ZODB/trunk/src/ZODB/Connection.py
===================================================================
--- ZODB/trunk/src/ZODB/Connection.py 2007-06-03 05:57:35 UTC (rev 76191)
+++ ZODB/trunk/src/ZODB/Connection.py 2007-06-03 08:26:47 UTC (rev 76192)
@@ -29,7 +29,7 @@
# interfaces
from persistent.interfaces import IPersistentDataManager
from ZODB.interfaces import IConnection
-from ZODB.Blobs.interfaces import IBlob, IBlobStorage
+from ZODB.interfaces import IBlob, IBlobStorage
from transaction.interfaces import ISavepointDataManager
from transaction.interfaces import IDataManagerSavepoint
from transaction.interfaces import ISynchronizer
Modified: ZODB/trunk/src/ZODB/ExportImport.py
===================================================================
--- ZODB/trunk/src/ZODB/ExportImport.py 2007-06-03 05:57:35 UTC (rev 76191)
+++ ZODB/trunk/src/ZODB/ExportImport.py 2007-06-03 08:26:47 UTC (rev 76192)
@@ -22,7 +22,7 @@
from ZODB.POSException import ExportError, POSKeyError
from ZODB.utils import p64, u64, cp, mktemp
-from ZODB.Blobs.interfaces import IBlobStorage
+from ZODB.interfaces import IBlobStorage
from ZODB.serialize import referencesf
logger = logging.getLogger('ZODB.ExportImport')
Copied: ZODB/trunk/src/ZODB/blob.py (from rev 76139, ZODB/trunk/src/ZODB/Blobs/Blob.py)
===================================================================
--- ZODB/trunk/src/ZODB/blob.py (rev 0)
+++ ZODB/trunk/src/ZODB/blob.py 2007-06-03 08:26:47 UTC (rev 76192)
@@ -0,0 +1,753 @@
+##############################################################################
+#
+# Copyright (c) 2005-2006 Zope Corporation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE
+#
+##############################################################################
+"""Blobs
+"""
+
+import base64
+import logging
+import os
+import shutil
+import sys
+import time
+import tempfile
+import logging
+
+import zope.interface
+
+import ZODB.interfaces
+from ZODB.interfaces import BlobError
+from ZODB import utils
+from ZODB.POSException import POSKeyError
+import transaction
+import transaction.interfaces
+import persistent
+
+from zope.proxy import getProxiedObject, non_overridable
+from zope.proxy.decorator import SpecificationDecoratorBase
+
+logger = logging.getLogger('ZODB.blob')
+
+BLOB_SUFFIX = ".blob"
+
+valid_modes = 'r', 'w', 'r+', 'a'
+
+class Blob(persistent.Persistent):
+ """A BLOB supports efficient handling of large data within ZODB."""
+
+ zope.interface.implements(ZODB.interfaces.IBlob)
+
+ _os_link = os.rename
+
+ _p_blob_readers = 0
+ _p_blob_writers = 0
+ _p_blob_uncommitted = None # Filename of the uncommitted (dirty) data
+ _p_blob_data = None # Filename of the committed data
+
+ # All persistent object store a reference to their data manager, a database
+ # connection in the _p_jar attribute. So we are going to do the same with
+ # blobs here.
+ _p_blob_manager = None
+
+ # Blobs need to participate in transactions even when not connected to
+ # a database yet. If you want to use a non-default transaction manager,
+ # you can override it via _p_blob_transaction. This is currently
+ # required for unit testing.
+ _p_blob_transaction = None
+
+ def open(self, mode="r"):
+ """Returns a file(-like) object representing blob data."""
+ result = None
+
+ if mode not in valid_modes:
+ raise ValueError("invalid mode", mode)
+
+ if mode == 'r':
+ if self._current_filename() is None:
+ raise BlobError("Blob does not exist.")
+
+ if self._p_blob_writers != 0:
+ raise BlobError("Already opened for writing.")
+
+ self._p_blob_readers += 1
+ result = BlobFile(self._current_filename(), mode, self)
+
+ elif mode == 'w':
+ if self._p_blob_readers != 0:
+ raise BlobError("Already opened for reading.")
+
+ self._p_blob_writers += 1
+ if self._p_blob_uncommitted is None:
+ self._create_uncommitted_file()
+ result = BlobFile(self._p_blob_uncommitted, mode, self)
+
+ elif mode in ('a', 'r+'):
+ if self._p_blob_readers != 0:
+ raise BlobError("Already opened for reading.")
+
+ if self._p_blob_uncommitted is None:
+ # Create a new working copy
+ uncommitted = BlobFile(self._create_uncommitted_file(),
+ mode, self)
+ # NOTE: _p_blob data appears by virtue of Connection._setstate
+ utils.cp(file(self._p_blob_data), uncommitted)
+ uncommitted.seek(0)
+ else:
+ # Re-use existing working copy
+ uncommitted = BlobFile(self._p_blob_uncommitted, mode, self)
+
+ self._p_blob_writers += 1
+ result = uncommitted
+
+ else:
+ raise IOError('invalid mode: %s ' % mode)
+
+ if result is not None:
+ self._setup_transaction_manager(result)
+ return result
+
+ def openDetached(self, class_=file):
+ """Returns a file(-like) object in read mode that can be used
+ outside of transaction boundaries.
+
+ """
+ if self._current_filename() is None:
+ raise BlobError("Blob does not exist.")
+ if self._p_blob_writers != 0:
+ raise BlobError("Already opened for writing.")
+ # XXX this should increase the reader number and have a test !?!
+ return class_(self._current_filename(), "rb")
+
+ def consumeFile(self, filename):
+ """Will replace the current data of the blob with the file given under
+ filename.
+ """
+ if self._p_blob_writers != 0:
+ raise BlobError("Already opened for writing.")
+ if self._p_blob_readers != 0:
+ raise BlobError("Already opened for reading.")
+
+ previous_uncommitted = bool(self._p_blob_uncommitted)
+ if previous_uncommitted:
+ # If we have uncommitted data, we move it aside for now
+ # in case the consumption doesn't work.
+ target = self._p_blob_uncommitted
+ target_aside = target+".aside"
+ os.rename(target, target_aside)
+ else:
+ target = self._create_uncommitted_file()
+ # We need to unlink the freshly created target again
+ # to allow link() to do its job
+ os.unlink(target)
+
+ try:
+ self._os_link(filename, target)
+ except:
+ # Recover from the failed consumption: First remove the file, it
+ # might exist and mark the pointer to the uncommitted file.
+ self._p_blob_uncommitted = None
+ if os.path.exists(target):
+ os.unlink(target)
+
+ # If there was a file moved aside, bring it back including the
+ # pointer to the uncommitted file.
+ if previous_uncommitted:
+ os.rename(target_aside, target)
+ self._p_blob_uncommitted = target
+
+ # Re-raise the exception to make the application aware of it.
+ raise
+ else:
+ if previous_uncommitted:
+ # The relinking worked so we can remove the data that we had
+ # set aside.
+ os.unlink(target_aside)
+
+ # We changed the blob state and have to make sure we join the
+ # transaction.
+ self._change()
+
+ # utility methods
+
+ def _current_filename(self):
+ # NOTE: _p_blob_data and _p_blob_uncommitted appear by virtue of
+ # Connection._setstate
+ return self._p_blob_uncommitted or self._p_blob_data
+
+ def _create_uncommitted_file(self):
+ assert self._p_blob_uncommitted is None, (
+ "Uncommitted file already exists.")
+ tempdir = os.environ.get('ZODB_BLOB_TEMPDIR', tempfile.gettempdir())
+ self._p_blob_uncommitted = utils.mktemp(dir=tempdir)
+ return self._p_blob_uncommitted
+
+ def _change(self):
+ self._p_changed = 1
+
+ def _setup_transaction_manager(self, result):
+ # We join the transaction with our own data manager in order to be
+ # notified of commit/vote/abort events. We do this because at
+ # transaction boundaries, we need to fix up _p_ reference counts
+ # that keep track of open readers and writers and close any
+ # writable filehandles we've opened.
+ if self._p_blob_manager is None:
+ # Blobs need to always participate in transactions.
+ if self._p_jar is not None:
+ # If we are connected to a database, then we use the
+ # transaction manager that belongs to this connection
+ tm = self._p_jar.transaction_manager
+ else:
+ # If we are not connected to a database, we check whether
+ # we have been given an explicit transaction manager
+ if self._p_blob_transaction:
+ tm = self._p_blob_transaction
+ else:
+ # Otherwise we use the default
+ # transaction manager as an educated guess.
+ tm = transaction.manager
+ # Create our datamanager and join he current transaction.
+ dm = BlobDataManager(self, result, tm)
+ tm.get().join(dm)
+ elif result:
+ # Each blob data manager should manage only the one blob
+ # assigned to it. Assert that this is the case and it is the
+ # correct blob
+ assert self._p_blob_manager.blob is self
+ self._p_blob_manager.register_fh(result)
+
+ # utility methods which should not cause the object's state to be
+ # loaded if they are called while the object is a ghost. Thus,
+ # they are named with the _p_ convention and only operate against
+ # other _p_ instance attributes. We conventionally name these methods
+ # and attributes with a _p_blob prefix.
+
+ def _p_blob_clear(self):
+ self._p_blob_readers = 0
+ self._p_blob_writers = 0
+
+ def _p_blob_decref(self, mode):
+ if mode == 'r':
+ self._p_blob_readers = max(0, self._p_blob_readers - 1)
+ else:
+ assert mode in valid_modes, "Invalid mode %r" % mode
+ self._p_blob_writers = max(0, self._p_blob_writers - 1)
+
+ def _p_blob_refcounts(self):
+ # used by unit tests
+ return self._p_blob_readers, self._p_blob_writers
+
+
+class BlobDataManager:
+ """Special data manager to handle transaction boundaries for blobs.
+
+ Blobs need some special care-taking on transaction boundaries. As
+
+ a) the ghost objects might get reused, the _p_reader and _p_writer
+ refcount attributes must be set to a consistent state
+ b) the file objects might get passed out of the thread/transaction
+ and must deny any relationship to the original blob.
+ c) writable blob filehandles must be closed at the end of a txn so
+ as to not allow reuse between two transactions.
+
+ """
+
+ zope.interface.implements(transaction.interfaces.IDataManager)
+
+ def __init__(self, blob, filehandle, tm):
+ self.blob = blob
+ self.transaction = tm.get()
+ # we keep a weakref to the file handle because we don't want to
+ # keep it alive if all other references to it die (e.g. in the
+ # case it's opened without assigning it to a name).
+ self.fhrefs = utils.WeakSet()
+ self.register_fh(filehandle)
+ self.sortkey = time.time()
+ self.prepared = False
+
+ # Blob specific methods
+
+ def register_fh(self, filehandle):
+ self.fhrefs.add(filehandle)
+
+ def _remove_uncommitted_data(self):
+ self.blob._p_blob_clear()
+ self.fhrefs.map(lambda fhref: fhref.close())
+ if (self.blob._p_blob_uncommitted is not None and
+ os.path.exists(self.blob._p_blob_uncommitted)):
+ os.unlink(self.blob._p_blob_uncommitted)
+ self.blob._p_blob_uncommitted = None
+
+ # IDataManager
+
+ def tpc_begin(self, transaction):
+ if self.prepared:
+ raise TypeError('Already prepared')
+ self._checkTransaction(transaction)
+ self.prepared = True
+ self.transaction = transaction
+ self.fhrefs.map(lambda fhref: fhref.close())
+
+ def commit(self, transaction):
+ if not self.prepared:
+ raise TypeError('Not prepared to commit')
+ self._checkTransaction(transaction)
+ self.transaction = None
+ self.prepared = False
+
+ self.blob._p_blob_clear()
+
+ def abort(self, transaction):
+ self.tpc_abort(transaction)
+
+ def tpc_abort(self, transaction):
+ self._checkTransaction(transaction)
+ if self.transaction is not None:
+ self.transaction = None
+ self.prepared = False
+
+ self._remove_uncommitted_data()
+
+ def tpc_finish(self, transaction):
+ pass
+
+ def tpc_vote(self, transaction):
+ pass
+
+ def sortKey(self):
+ return self.sortkey
+
+ def _checkTransaction(self, transaction):
+ if (self.transaction is not None and
+ self.transaction is not transaction):
+ raise TypeError("Transaction missmatch",
+ transaction, self.transaction)
+
+
+class BlobFile(file):
+ """A BlobFile that holds a file handle to actual blob data.
+
+ It is a file that can be used within a transaction boundary; a BlobFile is
+ just a Python file object, we only override methods which cause a change to
+ blob data in order to call methods on our 'parent' persistent blob object
+ signifying that the change happened.
+
+ """
+
+ # XXX these files should be created in the same partition as
+ # the storage later puts them to avoid copying them ...
+
+ def __init__(self, name, mode, blob):
+ super(BlobFile, self).__init__(name, mode+'b')
+ self.blob = blob
+ self.close_called = False
+
+ def write(self, data):
+ super(BlobFile, self).write(data)
+ self.blob._change()
+
+ def writelines(self, lines):
+ super(BlobFile, self).writelines(lines)
+ self.blob._change()
+
+ def truncate(self, size=0):
+ super(BlobFile, self).truncate(size)
+ self.blob._change()
+
+ def close(self):
+ # we don't want to decref twice
+ if not self.close_called:
+ self.blob._p_blob_decref(self.mode[:-1])
+ self.close_called = True
+ super(BlobFile, self).close()
+
+ def __del__(self):
+ # XXX we need to ensure that the file is closed at object
+ # expiration or our blob's refcount won't be decremented.
+ # This probably needs some work; I don't know if the names
+ # 'BlobFile' or 'super' will be available at program exit, but
+ # we'll assume they will be for now in the name of not
+ # muddying the code needlessly.
+ self.close()
+
+_pid = str(os.getpid())
+
+def log(msg, level=logging.INFO, subsys=_pid, exc_info=False):
+ message = "(%s) %s" % (subsys, msg)
+ logger.log(level, message, exc_info=exc_info)
+
+
+class FilesystemHelper:
+ # Storages that implement IBlobStorage can choose to use this
+ # helper class to generate and parse blob filenames. This is not
+ # a set-in-stone interface for all filesystem operations dealing
+ # with blobs and storages needn't indirect through this if they
+ # want to perform blob storage differently.
+
+ def __init__(self, base_dir):
+ self.base_dir = base_dir
+
+ def create(self):
+ if not os.path.exists(self.base_dir):
+ os.makedirs(self.base_dir, 0700)
+ log("Blob cache directory '%s' does not exist. "
+ "Created new directory." % self.base_dir,
+ level=logging.INFO)
+
+ def isSecure(self, path):
+ """Ensure that (POSIX) path mode bits are 0700."""
+ return (os.stat(path).st_mode & 077) != 0
+
+ def checkSecure(self):
+ if not self.isSecure(self.base_dir):
+ log('Blob dir %s has insecure mode setting' % self.base_dir,
+ level=logging.WARNING)
+
+ def getPathForOID(self, oid):
+ """Given an OID, return the path on the filesystem where
+ the blob data relating to that OID is stored.
+
+ """
+ return os.path.join(self.base_dir, utils.oid_repr(oid))
+
+ def getBlobFilename(self, oid, tid):
+ """Given an oid and a tid, return the full filename of the
+ 'committed' blob file related to that oid and tid.
+
+ """
+ oid_path = self.getPathForOID(oid)
+ filename = "%s%s" % (utils.tid_repr(tid), BLOB_SUFFIX)
+ return os.path.join(oid_path, filename)
+
+ def blob_mkstemp(self, oid, tid):
+ """Given an oid and a tid, return a temporary file descriptor
+ and a related filename.
+
+ The file is guaranteed to exist on the same partition as committed
+ data, which is important for being able to rename the file without a
+ copy operation. The directory in which the file will be placed, which
+ is the return value of self.getPathForOID(oid), must exist before this
+ method may be called successfully.
+
+ """
+ oidpath = self.getPathForOID(oid)
+ fd, name = tempfile.mkstemp(suffix='.tmp', prefix=utils.tid_repr(tid),
+ dir=oidpath)
+ return fd, name
+
+ def splitBlobFilename(self, filename):
+ """Returns the oid and tid for a given blob filename.
+
+ If the filename cannot be recognized as a blob filename, (None, None)
+ is returned.
+
+ """
+ if not filename.endswith(BLOB_SUFFIX):
+ return None, None
+ path, filename = os.path.split(filename)
+ oid = os.path.split(path)[1]
+
+ serial = filename[:-len(BLOB_SUFFIX)]
+ oid = utils.repr_to_oid(oid)
+ serial = utils.repr_to_oid(serial)
+ return oid, serial
+
+ def getOIDsForSerial(self, search_serial):
+ """Return all oids related to a particular tid that exist in
+ blob data.
+
+ """
+ oids = []
+ base_dir = self.base_dir
+ for oidpath in os.listdir(base_dir):
+ for filename in os.listdir(os.path.join(base_dir, oidpath)):
+ blob_path = os.path.join(base_dir, oidpath, filename)
+ oid, serial = self.splitBlobFilename(blob_path)
+ if search_serial == serial:
+ oids.append(oid)
+ return oids
+
+class BlobStorage(SpecificationDecoratorBase):
+ """A storage to support blobs."""
+
+ zope.interface.implements(ZODB.interfaces.IBlobStorage)
+
+ # Proxies can't have a __dict__ so specifying __slots__ here allows
+ # us to have instance attributes explicitly on the proxy.
+ __slots__ = ('fshelper', 'dirty_oids', '_BlobStorage__supportsUndo')
+
+ def __new__(self, base_directory, storage):
+ return SpecificationDecoratorBase.__new__(self, storage)
+
+ def __init__(self, base_directory, storage):
+ # XXX Log warning if storage is ClientStorage
+ SpecificationDecoratorBase.__init__(self, storage)
+ self.fshelper = FilesystemHelper(base_directory)
+ self.fshelper.create()
+ self.fshelper.checkSecure()
+ self.dirty_oids = []
+ try:
+ supportsUndo = storage.supportsUndo
+ except AttributeError:
+ supportsUndo = False
+ else:
+ supportsUndo = supportsUndo()
+ self.__supportsUndo = supportsUndo
+
+ @non_overridable
+ def temporaryDirectory(self):
+ return self.fshelper.base_dir
+
+
+ @non_overridable
+ def __repr__(self):
+ normal_storage = getProxiedObject(self)
+ return '<BlobStorage proxy for %r at %s>' % (normal_storage,
+ hex(id(self)))
+ @non_overridable
+ def storeBlob(self, oid, oldserial, data, blobfilename, version,
+ transaction):
+ """Stores data that has a BLOB attached."""
+ serial = self.store(oid, oldserial, data, version, transaction)
+ assert isinstance(serial, str) # XXX in theory serials could be
+ # something else
+
+ # the user may not have called "open" on the blob object,
+ # in which case, the blob will not have a filename.
+ if blobfilename is not None:
+ self._lock_acquire()
+ try:
+ targetpath = self.fshelper.getPathForOID(oid)
+ if not os.path.exists(targetpath):
+ os.makedirs(targetpath, 0700)
+
+ targetname = self.fshelper.getBlobFilename(oid, serial)
+ os.rename(blobfilename, targetname)
+
+ # XXX if oid already in there, something is really hosed.
+ # The underlying storage should have complained anyway
+ self.dirty_oids.append((oid, serial))
+ finally:
+ self._lock_release()
+ return self._tid
+
+ @non_overridable
+ def tpc_finish(self, *arg, **kw):
+ # We need to override the base storage's tpc_finish instead of
+ # providing a _finish method because methods found on the proxied
+ # object aren't rebound to the proxy
+ getProxiedObject(self).tpc_finish(*arg, **kw)
+ self.dirty_oids = []
+
+ @non_overridable
+ def tpc_abort(self, *arg, **kw):
+ # We need to override the base storage's abort instead of
+ # providing an _abort method because methods found on the proxied object
+ # aren't rebound to the proxy
+ getProxiedObject(self).tpc_abort(*arg, **kw)
+ while self.dirty_oids:
+ oid, serial = self.dirty_oids.pop()
+ clean = self.fshelper.getBlobFilename(oid, serial)
+ if os.exists(clean):
+ os.unlink(clean)
+
+ @non_overridable
+ def loadBlob(self, oid, serial):
+ """Return the filename where the blob file can be found.
+ """
+ filename = self.fshelper.getBlobFilename(oid, serial)
+ if not os.path.exists(filename):
+ return None
+ return filename
+
+ @non_overridable
+ def _packUndoing(self, packtime, referencesf):
+ # Walk over all existing revisions of all blob files and check
+ # if they are still needed by attempting to load the revision
+ # of that object from the database. This is maybe the slowest
+ # possible way to do this, but it's safe.
+
+ # XXX we should be tolerant of "garbage" directories/files in
+ # the base_directory here.
+
+ base_dir = self.fshelper.base_dir
+ for oid_repr in os.listdir(base_dir):
+ oid = utils.repr_to_oid(oid_repr)
+ oid_path = os.path.join(base_dir, oid_repr)
+ files = os.listdir(oid_path)
+ files.sort()
+
+ for filename in files:
+ filepath = os.path.join(oid_path, filename)
+ whatever, serial = self.fshelper.splitBlobFilename(filepath)
+ try:
+ fn = self.fshelper.getBlobFilename(oid, serial)
+ self.loadSerial(oid, serial)
+ except POSKeyError:
+ os.unlink(filepath)
+
+ if not os.listdir(oid_path):
+ shutil.rmtree(oid_path)
+
+ @non_overridable
+ def _packNonUndoing(self, packtime, referencesf):
+ base_dir = self.fshelper.base_dir
+ for oid_repr in os.listdir(base_dir):
+ oid = utils.repr_to_oid(oid_repr)
+ oid_path = os.path.join(base_dir, oid_repr)
+ exists = True
+
+ try:
+ self.load(oid, None) # no version support
+ except (POSKeyError, KeyError):
+ exists = False
+
+ if exists:
+ files = os.listdir(oid_path)
+ files.sort()
+ latest = files[-1] # depends on ever-increasing tids
+ files.remove(latest)
+ for file in files:
+ os.unlink(os.path.join(oid_path, file))
+ else:
+ shutil.rmtree(oid_path)
+ continue
+
+ if not os.listdir(oid_path):
+ shutil.rmtree(oid_path)
+
+ @non_overridable
+ def pack(self, packtime, referencesf):
+ """Remove all unused oid/tid combinations."""
+ unproxied = getProxiedObject(self)
+
+ # pack the underlying storage, which will allow us to determine
+ # which serials are current.
+ result = unproxied.pack(packtime, referencesf)
+
+ # perform a pack on blob data
+ self._lock_acquire()
+ try:
+ if self.__supportsUndo:
+ self._packUndoing(packtime, referencesf)
+ else:
+ self._packNonUndoing(packtime, referencesf)
+ finally:
+ self._lock_release()
+
+ return result
+
+ @non_overridable
+ def getSize(self):
+ """Return the size of the database in bytes."""
+ orig_size = getProxiedObject(self).getSize()
+
+ blob_size = 0
+ base_dir = self.fshelper.base_dir
+ for oid in os.listdir(base_dir):
+ for serial in os.listdir(os.path.join(base_dir, oid)):
+ if not serial.endswith(BLOB_SUFFIX):
+ continue
+ file_path = os.path.join(base_dir, oid, serial)
+ blob_size += os.stat(file_path).st_size
+
+ return orig_size + blob_size
+
+ @non_overridable
+ def undo(self, serial_id, transaction):
+ undo_serial, keys = getProxiedObject(self).undo(serial_id, transaction)
+ # serial_id is the transaction id of the txn that we wish to undo.
+ # "undo_serial" is the transaction id of txn in which the undo is
+ # performed. "keys" is the list of oids that are involved in the
+ # undo transaction.
+
+ # The serial_id is assumed to be given to us base-64 encoded
+ # (belying the web UI legacy of the ZODB code :-()
+ serial_id = base64.decodestring(serial_id+'\n')
+
+ self._lock_acquire()
+
+ try:
+ # we get all the blob oids on the filesystem related to the
+ # transaction we want to undo.
+ for oid in self.fshelper.getOIDsForSerial(serial_id):
+
+ # we want to find the serial id of the previous revision
+ # of this blob object.
+ load_result = self.loadBefore(oid, serial_id)
+
+ if load_result is None:
+ # There was no previous revision of this blob
+ # object. The blob was created in the transaction
+ # represented by serial_id. We copy the blob data
+ # to a new file that references the undo
+ # transaction in case a user wishes to undo this
+ # undo.
+ orig_fn = self.fshelper.getBlobFilename(oid, serial_id)
+ new_fn = self.fshelper.getBlobFilename(oid, undo_serial)
+ else:
+ # A previous revision of this blob existed before the
+ # transaction implied by "serial_id". We copy the blob
+ # data to a new file that references the undo transaction
+ # in case a user wishes to undo this undo.
+ data, serial_before, serial_after = load_result
+ orig_fn = self.fshelper.getBlobFilename(oid, serial_before)
+ new_fn = self.fshelper.getBlobFilename(oid, undo_serial)
+ orig = open(orig_fn, "r")
+ new = open(new_fn, "wb")
+ utils.cp(orig, new)
+ orig.close()
+ new.close()
+ self.dirty_oids.append((oid, undo_serial))
+
+ finally:
+ self._lock_release()
+ return undo_serial, keys
+
+# To do:
+#
+# Production
+#
+# - Ensure we detect and replay a failed txn involving blobs forward or
+# backward at startup.
+#
+# Jim: What does this mean?
+#
+# Far future
+#
+# More options for blob directory structures (e.g. dirstorages
+# bushy/chunky/lawn/flat).
+#
+# Make the ClientStorage support minimizing the blob
+# cache. (Idea: LRU principle via mstat access time and a
+# size-based threshold) currently).
+#
+# Make blobs able to efficiently consume existing files from the
+# filesystem
+#
+# Savepoint support
+# =================
+#
+# - A savepoint represents the whole state of the data at a certain point in
+# time
+#
+# - Need special storage for blob savepointing (in the spirit of tmpstorage)
+#
+# - What belongs to the state of the data?
+#
+# - Data contained in files at that point in time
+#
+# - File handles are complex because they might be referred to from various
+# places. We would have to introduce an abstraction layer to allow
+# switching them around...
+#
+# Simpler solution: :
Modified: ZODB/trunk/src/ZODB/config.py
===================================================================
--- ZODB/trunk/src/ZODB/config.py 2007-06-03 05:57:35 UTC (rev 76191)
+++ ZODB/trunk/src/ZODB/config.py 2007-06-03 08:26:47 UTC (rev 76192)
@@ -137,7 +137,7 @@
class BlobStorage(BaseConfig):
def open(self):
- from ZODB.Blobs.BlobStorage import BlobStorage
+ from ZODB.blob import BlobStorage
base = self.config.base.open()
return BlobStorage(self.config.blob_dir, base)
Modified: ZODB/trunk/src/ZODB/interfaces.py
===================================================================
--- ZODB/trunk/src/ZODB/interfaces.py 2007-06-03 05:57:35 UTC (rev 76191)
+++ ZODB/trunk/src/ZODB/interfaces.py 2007-06-03 08:26:47 UTC (rev 76192)
@@ -897,3 +897,62 @@
... break
"""
+
+class IBlob(Interface):
+ """A BLOB supports efficient handling of large data within ZODB."""
+
+ def open(mode):
+ """Returns a file(-like) object for handling the blob data.
+
+ mode: Mode to open the file with. Possible values: r,w,r+,a
+ """
+
+ def openDetached(class_=file):
+ """Returns a file(-like) object in read mode that can be used
+ outside of transaction boundaries.
+
+ The file handle returned by this method is read-only and at the
+ beginning of the file.
+
+ The handle is not attached to the blob and can be used outside of a
+ transaction.
+
+ Optionally the class that should be used to open the file can be
+ specified. This can be used to e.g. use Zope's FileStreamIterator.
+ """
+
+ def consumeFile(filename):
+ """Will replace the current data of the blob with the file given under
+ filename.
+
+ This method uses link-like semantics internally and has the requirement
+ that the file that is to be consumed lives on the same volume (or
+ mount/share) as the blob directory.
+
+ The blob must not be opened for reading or writing when consuming a
+ file.
+ """
+
+
+class IBlobStorage(Interface):
+ """A storage supporting BLOBs."""
+
+ def storeBlob(oid, oldserial, data, blob, version, transaction):
+ """Stores data that has a BLOB attached."""
+
+ def loadBlob(oid, serial):
+ """Return the filename of the Blob data for this OID and serial.
+
+ Returns a filename or None if no Blob data is connected with this OID.
+
+ Raises POSKeyError if the blobfile cannot be found.
+ """
+
+ def temporaryDirectory():
+ """Return a directory that should be used for uncommitted blob data.
+
+ If Blobs use this, then commits can be performed with a simple rename.
+ """
+
+class BlobError(Exception):
+ pass
Copied: ZODB/trunk/src/ZODB/tests/blob_basic.txt (from rev 76139, ZODB/trunk/src/ZODB/Blobs/tests/basic.txt)
===================================================================
--- ZODB/trunk/src/ZODB/tests/blob_basic.txt (rev 0)
+++ ZODB/trunk/src/ZODB/tests/blob_basic.txt 2007-06-03 08:26:47 UTC (rev 76192)
@@ -0,0 +1,167 @@
+##############################################################################
+#
+# Copyright (c) 2005 Zope Corporation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
+
+ZODB Blob support
+=================
+
+You create a blob like this:
+
+ >>> from ZODB.blob import Blob
+ >>> myblob = Blob()
+
+A blob implements the IBlob interface:
+
+ >>> from ZODB.interfaces import IBlob
+ >>> IBlob.providedBy(myblob)
+ True
+
+Opening a new Blob for reading fails:
+
+ >>> myblob.open("r")
+ Traceback (most recent call last):
+ ...
+ BlobError: Blob does not exist.
+
+But we can write data to a new Blob by opening it for writing:
+
+ >>> f = myblob.open("w")
+ >>> f.write("Hi, Blob!")
+
+If we try to open a Blob again while it is open for writing, we get an error:
+
+ >>> myblob.open("r")
+ Traceback (most recent call last):
+ ...
+ BlobError: Already opened for writing.
+
+We can close the file:
+
+ >>> f.close()
+
+Now we can open it for reading:
+
+ >>> f2 = myblob.open("r")
+
+And we get the data back:
+
+ >>> f2.read()
+ 'Hi, Blob!'
+
+If we want to, we can open it again:
+
+ >>> f3 = myblob.open("r")
+ >>> f3.read()
+ 'Hi, Blob!'
+
+But we can't open it for writing, while it is opened for reading:
+
+ >>> myblob.open("a")
+ Traceback (most recent call last):
+ ...
+ BlobError: Already opened for reading.
+
+Before we can write, we have to close the readers:
+
+ >>> f2.close()
+ >>> f3.close()
+
+Now we can open it for writing again and e.g. append data:
+
+ >>> f4 = myblob.open("a")
+ >>> f4.write("\nBlob is fine.")
+ >>> f4.close()
+
+Now we can read it:
+
+ >>> f4a = myblob.open("r")
+ >>> f4a.read()
+ 'Hi, Blob!\nBlob is fine.'
+ >>> f4a.close()
+
+You shouldn't need to explicitly close a blob unless you hold a reference
+to it via a name. If the first line in the following test kept a reference
+around via a name, the second call to open it in a writable mode would fail
+with a BlobError, but it doesn't.
+
+ >>> myblob.open("r+").read()
+ 'Hi, Blob!\nBlob is fine.'
+ >>> f4b = myblob.open("a")
+ >>> f4b.close()
+
+We can read lines out of the blob too:
+
+ >>> f5 = myblob.open("r")
+ >>> f5.readline()
+ 'Hi, Blob!\n'
+ >>> f5.readline()
+ 'Blob is fine.'
+ >>> f5.close()
+
+We can seek to certain positions in a blob and read portions of it:
+
+ >>> f6 = myblob.open('r')
+ >>> f6.seek(4)
+ >>> int(f6.tell())
+ 4
+ >>> f6.read(5)
+ 'Blob!'
+ >>> f6.close()
+
+We can use the object returned by a blob open call as an iterable:
+
+ >>> f7 = myblob.open('r')
+ >>> for line in f7:
+ ... print line
+ Hi, Blob!
+ <BLANKLINE>
+ Blob is fine.
+ >>> f7.close()
+
+We can truncate a blob:
+
+ >>> f8 = myblob.open('a')
+ >>> f8.truncate(0)
+ >>> f8.close()
+ >>> f8 = myblob.open('r')
+ >>> f8.read()
+ ''
+ >>> f8.close()
+
+Blobs are always opened in binary mode:
+
+ >>> f9 = myblob.open("r")
+ >>> f9.mode
+ 'rb'
+ >>> f9.close()
+
+We can specify the tempdir that blobs use to keep uncommitted data by
+modifying the ZODB_BLOB_TEMPDIR environment variable:
+
+ >>> import os, tempfile, shutil
+ >>> tempdir = tempfile.mkdtemp()
+ >>> os.environ['ZODB_BLOB_TEMPDIR'] = tempdir
+ >>> myblob = Blob()
+ >>> len(os.listdir(tempdir))
+ 0
+ >>> f = myblob.open('w')
+ >>> len(os.listdir(tempdir))
+ 1
+ >>> f.close()
+ >>> shutil.rmtree(tempdir)
+ >>> del os.environ['ZODB_BLOB_TEMPDIR']
+
+Some cleanup in this test is needed:
+
+ >>> import transaction
+ >>> transaction.get().abort()
Copied: ZODB/trunk/src/ZODB/tests/blob_connection.txt (from rev 76139, ZODB/trunk/src/ZODB/Blobs/tests/connection.txt)
===================================================================
--- ZODB/trunk/src/ZODB/tests/blob_connection.txt (rev 0)
+++ ZODB/trunk/src/ZODB/tests/blob_connection.txt 2007-06-03 08:26:47 UTC (rev 76192)
@@ -0,0 +1,83 @@
+##############################################################################
+#
+# Copyright (c) 2005 Zope Corporation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
+
+Connection support for Blobs tests
+==================================
+
+Connections handle Blobs specially. To demonstrate that, we first need a Blob with some data:
+
+ >>> from ZODB.interfaces import IBlob
+ >>> from ZODB.blob import Blob
+ >>> import transaction
+ >>> blob = Blob()
+ >>> data = blob.open("w")
+ >>> data.write("I'm a happy Blob.")
+ >>> data.close()
+
+We also need a database with a blob supporting storage:
+
+ >>> from ZODB.MappingStorage import MappingStorage
+ >>> from ZODB.blob import BlobStorage
+ >>> from ZODB.DB import DB
+ >>> from tempfile import mkdtemp
+ >>> base_storage = MappingStorage("test")
+ >>> blob_dir = mkdtemp()
+ >>> blob_storage = BlobStorage(blob_dir, base_storage)
+ >>> database = DB(blob_storage)
+
+Putting a Blob into a Connection works like every other object:
+
+ >>> connection = database.open()
+ >>> root = connection.root()
+ >>> root['myblob'] = blob
+ >>> transaction.commit()
+
+We can also commit a transaction that seats a blob into place without
+calling the blob's open method (this currently fails):
+
+ >>> nothing = transaction.begin()
+ >>> anotherblob = Blob()
+ >>> root['anotherblob'] = anotherblob
+ >>> nothing = transaction.commit()
+
+Getting stuff out of there works similar:
+
+ >>> connection2 = database.open()
+ >>> root = connection2.root()
+ >>> blob2 = root['myblob']
+ >>> IBlob.providedBy(blob2)
+ True
+ >>> blob2.open("r").read()
+ "I'm a happy Blob."
+
+You can't put blobs into a database that has uses a Non-Blob-Storage, though:
+
+ >>> no_blob_storage = MappingStorage()
+ >>> database2 = DB(no_blob_storage)
+ >>> connection3 = database2.open()
+ >>> root = connection3.root()
+ >>> root['myblob'] = Blob()
+ >>> transaction.commit() # doctest: +ELLIPSIS
+ Traceback (most recent call last):
+ ...
+ Unsupported: Storing Blobs in <ZODB.MappingStorage.MappingStorage instance at ...> is not supported.
+
+While we are testing this, we don't need the storage directory and
+databases anymore:
+
+ >>> import shutil
+ >>> shutil.rmtree(blob_dir)
+ >>> transaction.abort()
+ >>> database.close()
+ >>> database2.close()
Copied: ZODB/trunk/src/ZODB/tests/blob_consume.txt (from rev 76139, ZODB/trunk/src/ZODB/Blobs/tests/consume.txt)
===================================================================
--- ZODB/trunk/src/ZODB/tests/blob_consume.txt (rev 0)
+++ ZODB/trunk/src/ZODB/tests/blob_consume.txt 2007-06-03 08:26:47 UTC (rev 76192)
@@ -0,0 +1,109 @@
+Consuming existing files
+========================
+
+The ZODB Blob implementation allows to import existing files as Blobs within
+an O(1) operation we call `consume`::
+
+Let's create a file::
+
+ >>> to_import = open('to_import', 'wb')
+ >>> to_import.write("I'm a Blob and I feel fine.")
+
+The file *must* be closed before giving it to consumeFile:
+
+ >>> to_import.close()
+
+Now, let's consume this file in a blob by specifying it's name::
+
+ >>> from ZODB.blob import Blob
+ >>> blob = Blob()
+ >>> blob.consumeFile('to_import')
+
+After the consumeFile operation, the original file has been removed:
+
+ >>> import os
+ >>> os.path.exists('to_import')
+ False
+
+We now can call open on the blob and read and write the data::
+
+ >>> blob_read = blob.open('r')
+ >>> blob_read.read()
+ "I'm a Blob and I feel fine."
+ >>> blob_read.close()
+ >>> blob_write = blob.open('w')
+ >>> blob_write.write('I was changed.')
+ >>> blob_write.close()
+
+We can not consume a file when there is a reader or writer around for a blob
+already::
+
+ >>> open('to_import', 'wb').write('I am another blob.')
+ >>> blob_read = blob.open('r')
+ >>> blob.consumeFile('to_import')
+ Traceback (most recent call last):
+ BlobError: Already opened for reading.
+ >>> blob_read.close()
+ >>> blob_write = blob.open('w')
+ >>> blob.consumeFile('to_import')
+ Traceback (most recent call last):
+ BlobError: Already opened for writing.
+ >>> blob_write.close()
+
+Now, after closing all readers and writers we can consume files again::
+
+ >>> blob.consumeFile('to_import')
+ >>> blob_read = blob.open('r')
+ >>> blob_read.read()
+ 'I am another blob.'
+
+
+Edge cases
+==========
+
+There are some edge cases what happens when the link() operation
+fails. We simulate this in different states:
+
+Case 1: We don't have uncommitted data, but the link operation fails. The
+exception will be re-raised and the target file will not exist::
+
+ >>> open('to_import', 'wb').write('Some data.')
+
+ >>> def failing_link(self, filename):
+ ... raise Exception("I can't link.")
+
+ >>> blob = Blob()
+ >>> blob.open('r')
+ Traceback (most recent call last):
+ BlobError: Blob does not exist.
+
+ >>> blob._os_link = failing_link
+ >>> blob.consumeFile('to_import')
+ Traceback (most recent call last):
+ Exception: I can't link.
+
+The blob did not exist before, so it shouldn't exist now::
+
+ >>> blob.open('r')
+ Traceback (most recent call last):
+ BlobError: Blob does not exist.
+
+Case 2: We thave uncommitted data, but the link operation fails. The
+exception will be re-raised and the target file will exist with the previous
+uncomitted data::
+
+ >>> blob = Blob()
+ >>> blob_writing = blob.open('w')
+ >>> blob_writing.write('Uncommitted data')
+ >>> blob_writing.close()
+
+ >>> blob._os_link = failing_link
+ >>> blob.consumeFile('to_import')
+ Traceback (most recent call last):
+ Exception: I can't link.
+
+The blob did existed before and had uncommitted data, this shouldn't have
+changed::
+
+ >>> blob.open('r').read()
+ 'Uncommitted data'
Copied: ZODB/trunk/src/ZODB/tests/blob_importexport.txt (from rev 76139, ZODB/trunk/src/ZODB/Blobs/tests/importexport.txt)
===================================================================
--- ZODB/trunk/src/ZODB/tests/blob_importexport.txt (rev 0)
+++ ZODB/trunk/src/ZODB/tests/blob_importexport.txt 2007-06-03 08:26:47 UTC (rev 76192)
@@ -0,0 +1,101 @@
+##############################################################################
+#
+# Copyright (c) 2005 Zope Corporation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
+
+Import/export support for blob data
+===================================
+
+Set up:
+
+ >>> from ZODB.FileStorage import FileStorage
+ >>> from ZODB.blob import Blob, BlobStorage
+ >>> from ZODB.DB import DB
+ >>> from persistent.mapping import PersistentMapping
+ >>> import shutil
+ >>> import transaction
+ >>> from tempfile import mkdtemp, mktemp
+ >>> storagefile1 = mktemp()
+ >>> blob_dir1 = mkdtemp()
+ >>> storagefile2 = mktemp()
+ >>> blob_dir2 = mkdtemp()
+
+We need an database with an undoing blob supporting storage:
+
+ >>> base_storage1 = FileStorage(storagefile1)
+ >>> blob_storage1 = BlobStorage(blob_dir1, base_storage1)
+ >>> base_storage2 = FileStorage(storagefile2)
+ >>> blob_storage2 = BlobStorage(blob_dir2, base_storage2)
+ >>> database1 = DB(blob_storage1)
+ >>> database2 = DB(blob_storage2)
+
+Create our root object for database1:
+
+ >>> connection1 = database1.open()
+ >>> root1 = connection1.root()
+
+Put a couple blob objects in our database1 and on the filesystem:
+
+ >>> import time, os
+ >>> nothing = transaction.begin()
+ >>> tid = blob_storage1._tid
+ >>> data1 = 'x'*100000
+ >>> blob1 = Blob()
+ >>> blob1.open('w').write(data1)
+ >>> data2 = 'y'*100000
+ >>> blob2 = Blob()
+ >>> blob2.open('w').write(data2)
+ >>> d = PersistentMapping({'blob1':blob1, 'blob2':blob2})
+ >>> root1['blobdata'] = d
+ >>> transaction.commit()
+
+Export our blobs from a database1 connection:
+
+ >>> conn = root1['blobdata']._p_jar
+ >>> oid = root1['blobdata']._p_oid
+ >>> exportfile = mktemp()
+ >>> nothing = connection1.exportFile(oid, exportfile)
+
+Import our exported data into database2:
+
+ >>> connection2 = database2.open()
+ >>> root2 = connection2.root()
+ >>> nothing = transaction.begin()
+ >>> data = root2._p_jar.importFile(exportfile)
+ >>> root2['blobdata'] = data
+ >>> transaction.commit()
+
+Make sure our data exists:
+
+ >>> items1 = root1['blobdata']
+ >>> items2 = root2['blobdata']
+ >>> bool(items1.keys() == items2.keys())
+ True
+ >>> items1['blob1'].open().read() == items2['blob1'].open().read()
+ True
+ >>> items1['blob2'].open().read() == items2['blob2'].open().read()
+ True
+ >>> transaction.get().abort()
+
+Clean up our blob directory:
+
+ >>> base_storage1.close()
+ >>> base_storage2.close()
+ >>> shutil.rmtree(blob_dir1)
+ >>> shutil.rmtree(blob_dir2)
+ >>> os.unlink(exportfile)
+ >>> os.unlink(storagefile1)
+ >>> os.unlink(storagefile1+".index")
+ >>> os.unlink(storagefile1+".tmp")
+ >>> os.unlink(storagefile2)
+ >>> os.unlink(storagefile2+".index")
+ >>> os.unlink(storagefile2+".tmp")
Copied: ZODB/trunk/src/ZODB/tests/blob_packing.txt (from rev 76139, ZODB/trunk/src/ZODB/Blobs/tests/packing.txt)
===================================================================
--- ZODB/trunk/src/ZODB/tests/blob_packing.txt (rev 0)
+++ ZODB/trunk/src/ZODB/tests/blob_packing.txt 2007-06-03 08:26:47 UTC (rev 76192)
@@ -0,0 +1,254 @@
+##############################################################################
+#
+# Copyright (c) 2005 Zope Corporation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
+
+Packing support for blob data
+=============================
+
+Set up:
+
+ >>> from ZODB.FileStorage import FileStorage
+ >>> from ZODB.MappingStorage import MappingStorage
+ >>> from ZODB.serialize import referencesf
+ >>> from ZODB.blob import Blob, BlobStorage
+ >>> from ZODB import utils
+ >>> from ZODB.DB import DB
+ >>> import shutil
+ >>> import transaction
+ >>> from tempfile import mkdtemp, mktemp
+ >>> storagefile = mktemp()
+ >>> blob_dir = mkdtemp()
+
+A helper method to assure a unique timestamp across multiple platforms. This
+method also makes sure that after retrieving a timestamp that was *before* a
+transaction was committed, that at least one second passes so the packing time
+actually is before the commit time.
+
+ >>> import time
+ >>> def new_time():
+ ... now = new_time = time.time()
+ ... while new_time <= now:
+ ... new_time = time.time()
+ ... time.sleep(1)
+ ... return new_time
+
+UNDOING
+=======
+
+We need a database with an undoing blob supporting storage:
+
+ >>> base_storage = FileStorage(storagefile)
+ >>> blob_storage = BlobStorage(blob_dir, base_storage)
+ >>> database = DB(blob_storage)
+
+Create our root object:
+
+ >>> connection1 = database.open()
+ >>> root = connection1.root()
+
+Put some revisions of a blob object in our database and on the filesystem:
+
+ >>> import os
+ >>> tids = []
+ >>> times = []
+ >>> nothing = transaction.begin()
+ >>> times.append(new_time())
+ >>> blob = Blob()
+ >>> blob.open('w').write('this is blob data 0')
+ >>> root['blob'] = blob
+ >>> transaction.commit()
+ >>> tids.append(blob_storage._tid)
+
+ >>> nothing = transaction.begin()
+ >>> times.append(new_time())
+ >>> root['blob'].open('w').write('this is blob data 1')
+ >>> transaction.commit()
+ >>> tids.append(blob_storage._tid)
+
+ >>> nothing = transaction.begin()
+ >>> times.append(new_time())
+ >>> root['blob'].open('w').write('this is blob data 2')
+ >>> transaction.commit()
+ >>> tids.append(blob_storage._tid)
+
+ >>> nothing = transaction.begin()
+ >>> times.append(new_time())
+ >>> root['blob'].open('w').write('this is blob data 3')
+ >>> transaction.commit()
+ >>> tids.append(blob_storage._tid)
+
+ >>> nothing = transaction.begin()
+ >>> times.append(new_time())
+ >>> root['blob'].open('w').write('this is blob data 4')
+ >>> transaction.commit()
+ >>> tids.append(blob_storage._tid)
+
+ >>> oid = root['blob']._p_oid
+ >>> fns = [ blob_storage.fshelper.getBlobFilename(oid, x) for x in tids ]
+ >>> [ os.path.exists(x) for x in fns ]
+ [True, True, True, True, True]
+
+Do a pack to the slightly before the first revision was written:
+
+ >>> packtime = times[0]
+ >>> blob_storage.pack(packtime, referencesf)
+ >>> [ os.path.exists(x) for x in fns ]
+ [True, True, True, True, True]
+
+Do a pack to the slightly before the second revision was written:
+
+ >>> packtime = times[1]
+ >>> blob_storage.pack(packtime, referencesf)
+ >>> [ os.path.exists(x) for x in fns ]
+ [True, True, True, True, True]
+
+Do a pack to the slightly before the third revision was written:
+
+ >>> packtime = times[2]
+ >>> blob_storage.pack(packtime, referencesf)
+ >>> [ os.path.exists(x) for x in fns ]
+ [False, True, True, True, True]
+
+Do a pack to the slightly before the fourth revision was written:
+
+ >>> packtime = times[3]
+ >>> blob_storage.pack(packtime, referencesf)
+ >>> [ os.path.exists(x) for x in fns ]
+ [False, False, True, True, True]
+
+Do a pack to the slightly before the fifth revision was written:
+
+ >>> packtime = times[4]
+ >>> blob_storage.pack(packtime, referencesf)
+ >>> [ os.path.exists(x) for x in fns ]
+ [False, False, False, True, True]
+
+Do a pack to now:
+
+ >>> packtime = new_time()
+ >>> blob_storage.pack(packtime, referencesf)
+ >>> [ os.path.exists(x) for x in fns ]
+ [False, False, False, False, True]
+
+Delete the object and do a pack, it should get rid of the most current
+revision as well as the entire directory:
+
+ >>> nothing = transaction.begin()
+ >>> del root['blob']
+ >>> transaction.commit()
+ >>> packtime = new_time()
+ >>> blob_storage.pack(packtime, referencesf)
+ >>> [ os.path.exists(x) for x in fns ]
+ [False, False, False, False, False]
+ >>> os.path.exists(os.path.split(fns[0])[0])
+ False
+
+Clean up our blob directory and database:
+
+ >>> shutil.rmtree(blob_dir)
+ >>> base_storage.close()
+ >>> os.unlink(storagefile)
+ >>> os.unlink(storagefile+".index")
+ >>> os.unlink(storagefile+".tmp")
+ >>> os.unlink(storagefile+".old")
+
+NON-UNDOING
+===========
+
+We need an database with a NON-undoing blob supporting storage:
+
+ >>> base_storage = MappingStorage('storage')
+ >>> blob_storage = BlobStorage(blob_dir, base_storage)
+ >>> database = DB(blob_storage)
+
+Create our root object:
+
+ >>> connection1 = database.open()
+ >>> root = connection1.root()
+
+Put some revisions of a blob object in our database and on the filesystem:
+
+ >>> import time, os
+ >>> tids = []
+ >>> times = []
+ >>> nothing = transaction.begin()
+ >>> times.append(new_time())
+ >>> blob = Blob()
+ >>> blob.open('w').write('this is blob data 0')
+ >>> root['blob'] = blob
+ >>> transaction.commit()
+ >>> tids.append(blob_storage._tid)
+
+ >>> nothing = transaction.begin()
+ >>> times.append(new_time())
+ >>> root['blob'].open('w').write('this is blob data 1')
+ >>> transaction.commit()
+ >>> tids.append(blob_storage._tid)
+
+ >>> nothing = transaction.begin()
+ >>> times.append(new_time())
+ >>> root['blob'].open('w').write('this is blob data 2')
+ >>> transaction.commit()
+ >>> tids.append(blob_storage._tid)
+
+ >>> nothing = transaction.begin()
+ >>> times.append(new_time())
+ >>> root['blob'].open('w').write('this is blob data 3')
+ >>> transaction.commit()
+ >>> tids.append(blob_storage._tid)
+
+ >>> nothing = transaction.begin()
+ >>> times.append(new_time())
+ >>> root['blob'].open('w').write('this is blob data 4')
+ >>> transaction.commit()
+ >>> tids.append(blob_storage._tid)
+
+ >>> oid = root['blob']._p_oid
+ >>> fns = [ blob_storage.fshelper.getBlobFilename(oid, x) for x in tids ]
+ >>> [ os.path.exists(x) for x in fns ]
+ [True, True, True, True, True]
+
+Get our blob filenames for this oid.
+
+ >>> fns = [ blob_storage.fshelper.getBlobFilename(oid, x) for x in tids ]
+
+Do a pack to the slightly before the first revision was written:
+
+ >>> packtime = times[0]
+ >>> blob_storage.pack(packtime, referencesf)
+ >>> [ os.path.exists(x) for x in fns ]
+ [False, False, False, False, True]
+
+Do a pack to now:
+
+ >>> packtime = new_time()
+ >>> blob_storage.pack(packtime, referencesf)
+ >>> [ os.path.exists(x) for x in fns ]
+ [False, False, False, False, True]
+
+Delete the object and do a pack, it should get rid of the most current
+revision as well as the entire directory:
+
+ >>> nothing = transaction.begin()
+ >>> del root['blob']
+ >>> transaction.commit()
+ >>> packtime = new_time()
+ >>> blob_storage.pack(packtime, referencesf)
+ >>> [ os.path.exists(x) for x in fns ]
+ [False, False, False, False, False]
+ >>> os.path.exists(os.path.split(fns[0])[0])
+ False
+
+Clean up our blob directory:
+
+ >>> shutil.rmtree(blob_dir)
Copied: ZODB/trunk/src/ZODB/tests/blob_transaction.txt (from rev 76139, ZODB/trunk/src/ZODB/Blobs/tests/transaction.txt)
===================================================================
--- ZODB/trunk/src/ZODB/tests/blob_transaction.txt (rev 0)
+++ ZODB/trunk/src/ZODB/tests/blob_transaction.txt 2007-06-03 08:26:47 UTC (rev 76192)
@@ -0,0 +1,315 @@
+##############################################################################
+#
+# Copyright (c) 2005-2007 Zope Corporation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
+
+Transaction support for Blobs
+=============================
+
+We need a database with a blob supporting storage::
+
+ >>> from ZODB.MappingStorage import MappingStorage
+ >>> from ZODB.blob import Blob, BlobStorage
+ >>> from ZODB.DB import DB
+ >>> import transaction
+ >>> import tempfile
+ >>> from tempfile import mkdtemp
+ >>> base_storage = MappingStorage("test")
+ >>> blob_dir = mkdtemp()
+ >>> blob_storage = BlobStorage(blob_dir, base_storage)
+ >>> database = DB(blob_storage)
+ >>> connection1 = database.open()
+ >>> root1 = connection1.root()
+
+Putting a Blob into a Connection works like any other Persistent object::
+
+ >>> blob1 = Blob()
+ >>> blob1.open('w').write('this is blob 1')
+ >>> root1['blob1'] = blob1
+ >>> transaction.commit()
+
+Aborting a transaction involving a blob write cleans up uncommitted
+file data::
+
+ >>> dead_blob = Blob()
+ >>> dead_blob.open('w').write('this is a dead blob')
+ >>> root1['dead_blob'] = dead_blob
+ >>> fname = dead_blob._p_blob_uncommitted
+ >>> import os
+ >>> os.path.exists(fname)
+ True
+ >>> transaction.abort()
+ >>> os.path.exists(fname)
+ False
+
+Opening a blob gives us a filehandle. Getting data out of the
+resulting filehandle is accomplished via the filehandle's read method::
+
+ >>> connection2 = database.open()
+ >>> root2 = connection2.root()
+ >>> blob1a = root2['blob1']
+ >>> blob1a._p_blob_refcounts()
+ (0, 0)
+ >>>
+ >>> blob1afh1 = blob1a.open("r")
+ >>> blob1afh1.read()
+ 'this is blob 1'
+ >>> # The filehandle keeps a reference to its blob object
+ >>> blob1afh1.blob._p_blob_refcounts()
+ (1, 0)
+
+Let's make another filehandle for read only to blob1a, this should bump
+up its refcount by one, and each file handle has a reference to the
+(same) underlying blob::
+
+ >>> blob1afh2 = blob1a.open("r")
+ >>> blob1afh2.blob._p_blob_refcounts()
+ (2, 0)
+ >>> blob1afh1.blob._p_blob_refcounts()
+ (2, 0)
+ >>> blob1afh2.blob is blob1afh1.blob
+ True
+
+Let's close the first filehandle we got from the blob, this should decrease
+its refcount by one::
+
+ >>> blob1afh1.close()
+ >>> blob1a._p_blob_refcounts()
+ (1, 0)
+
+Let's abort this transaction, and ensure that the filehandles that we
+opened are now closed and that the filehandle refcounts on the blob
+object are cleared::
+
+ >>> transaction.abort()
+ >>> blob1afh1.blob._p_blob_refcounts()
+ (0, 0)
+ >>> blob1afh2.blob._p_blob_refcounts()
+ (0, 0)
+ >>> blob1a._p_blob_refcounts()
+ (0, 0)
+ >>> blob1afh2.read()
+ Traceback (most recent call last):
+ ...
+ ValueError: I/O operation on closed file
+
+If we open a blob for append, its write refcount should be nonzero.
+Additionally, writing any number of bytes to the blobfile should
+result in the blob being marked "dirty" in the connection (we just
+aborted above, so the object should be "clean" when we start)::
+
+ >>> bool(blob1a._p_changed)
+ False
+ >>> blob1a.open('r').read()
+ 'this is blob 1'
+ >>> blob1afh3 = blob1a.open('a')
+ >>> blob1afh3.write('woot!')
+ >>> blob1a._p_blob_refcounts()
+ (0, 1)
+ >>> bool(blob1a._p_changed)
+ True
+
+We can open more than one blob object during the course of a single
+transaction::
+
+ >>> blob2 = Blob()
+ >>> blob2.open('w').write('this is blob 3')
+ >>> root2['blob2'] = blob2
+ >>> transaction.commit()
+ >>> blob2._p_blob_refcounts()
+ (0, 0)
+ >>> blob1._p_blob_refcounts()
+ (0, 0)
+
+Since we committed the current transaction above, the aggregate
+changes we've made to blob, blob1a (these refer to the same object) and
+blob2 (a different object) should be evident::
+
+ >>> blob1.open('r').read()
+ 'this is blob 1woot!'
+ >>> blob1a.open('r').read()
+ 'this is blob 1woot!'
+ >>> blob2.open('r').read()
+ 'this is blob 3'
+
+We shouldn't be able to persist a blob filehandle at commit time
+(although the exception which is raised when an object cannot be
+pickled appears to be particulary unhelpful for casual users at the
+moment)::
+
+ >>> root1['wontwork'] = blob1.open('r')
+ >>> transaction.commit()
+ Traceback (most recent call last):
+ ...
+ TypeError: coercing to Unicode: need string or buffer, BlobFile found
+
+Abort for good measure::
+
+ >>> transaction.abort()
+
+Attempting to change a blob simultaneously from two different
+connections should result in a write conflict error::
+
+ >>> tm1 = transaction.TransactionManager()
+ >>> tm2 = transaction.TransactionManager()
+ >>> root3 = database.open(transaction_manager=tm1).root()
+ >>> root4 = database.open(transaction_manager=tm2).root()
+ >>> blob1c3 = root3['blob1']
+ >>> blob1c4 = root4['blob1']
+ >>> blob1c3fh1 = blob1c3.open('a')
+ >>> blob1c4fh1 = blob1c4.open('a')
+ >>> blob1c3fh1.write('this is from connection 3')
+ >>> blob1c4fh1.write('this is from connection 4')
+ >>> tm1.get().commit()
+ >>> root3['blob1'].open('r').read()
+ 'this is blob 1woot!this is from connection 3'
+ >>> tm2.get().commit()
+ Traceback (most recent call last):
+ ...
+ ConflictError: database conflict error (oid 0x01, class ZODB.blob.Blob)
+
+After the conflict, the winning transaction's result is visible on both
+connections::
+
+ >>> root3['blob1'].open('r').read()
+ 'this is blob 1woot!this is from connection 3'
+ >>> tm2.get().abort()
+ >>> root4['blob1'].open('r').read()
+ 'this is blob 1woot!this is from connection 3'
+
+BlobStorages implementation of getSize() includes the blob data and adds it to
+the underlying storages result of getSize(). (We need to ensure the last
+number to be an int, otherwise it will be a long on 32-bit platforms and an
+int on 64-bit)::
+
+ >>> underlying_size = base_storage.getSize()
+ >>> blob_size = blob_storage.getSize()
+ >>> int(blob_size - underlying_size)
+ 91
+
+
+Savepoints and Blobs
+--------------------
+
+We do support optimistic savepoints ::
+
+ >>> connection5 = database.open()
+ >>> root5 = connection5.root()
+ >>> blob = Blob()
+ >>> blob_fh = blob.open("w")
+ >>> blob_fh.write("I'm a happy blob.")
+ >>> blob_fh.close()
+ >>> root5['blob'] = blob
+ >>> transaction.commit()
+ >>> root5['blob'].open("r").read()
+ "I'm a happy blob."
+ >>> blob_fh = root5['blob'].open("a")
+ >>> blob_fh.write(" And I'm singing.")
+ >>> blob_fh.close()
+ >>> root5['blob'].open("r").read()
+ "I'm a happy blob. And I'm singing."
+ >>> savepoint = transaction.savepoint(optimistic=True)
+ >>> root5['blob'].open("r").read()
+ "I'm a happy blob. And I'm singing."
+ >>> transaction.get().commit()
+
+We do not support non-optimistic savepoints::
+
+ >>> blob_fh = root5['blob'].open("a")
+ >>> blob_fh.write(" And the weather is beautiful.")
+ >>> blob_fh.close()
+ >>> root5['blob'].open("r").read()
+ "I'm a happy blob. And I'm singing. And the weather is beautiful."
+ >>> savepoint = transaction.savepoint() # doctest: +ELLIPSIS
+ Traceback (most recent call last):
+ ...
+ TypeError: ('Savepoints unsupported', <ZODB.blob.BlobDataManager instance at 0x...>)
+ >>> transaction.abort()
+
+Reading Blobs outside of a transaction
+--------------------------------------
+
+If you want to read from a Blob outside of transaction boundaries (e.g. to
+stream a file to the browser), you can use the openDetached() method::
+
+ >>> connection6 = database.open()
+ >>> root6 = connection6.root()
+ >>> blob = Blob()
+ >>> blob_fh = blob.open("w")
+ >>> blob_fh.write("I'm a happy blob.")
+ >>> blob_fh.close()
+ >>> root6['blob'] = blob
+ >>> transaction.commit()
+ >>> blob.openDetached().read()
+ "I'm a happy blob."
+
+Of course, that doesn't work for empty blobs::
+
+ >>> blob = Blob()
+ >>> blob.openDetached()
+ Traceback (most recent call last):
+ ...
+ BlobError: Blob does not exist.
+
+nor when the Blob is already opened for writing::
+
+ >>> blob = Blob()
+ >>> blob_fh = blob.open("w")
+ >>> blob.openDetached()
+ Traceback (most recent call last):
+ ...
+ BlobError: Already opened for writing.
+
+You can also pass a factory to the openDetached method that will be used to
+instantiate the file. This is used for e.g. creating filestream iterators::
+
+ >>> class customfile(file):
+ ... pass
+ >>> blob_fh.write('Something')
+ >>> blob_fh.close()
+ >>> fh = blob.openDetached(customfile)
+ >>> fh # doctest: +ELLIPSIS
+ <open file '...', mode 'rb' at 0x...>
+ >>> isinstance(fh, customfile)
+ True
+
+
+Note: Nasty people could use a factory that opens the file for writing. This
+would be evil.
+
+It does work when the transaction was aborted, though::
+
+ >>> blob = Blob()
+ >>> blob_fh = blob.open("w")
+ >>> blob_fh.write("I'm a happy blob.")
+ >>> blob_fh.close()
+ >>> root6['blob'] = blob
+ >>> transaction.commit()
+
+ >>> blob_fh = blob.open("w")
+ >>> blob_fh.write("And I'm singing.")
+ >>> blob_fh.close()
+ >>> transaction.abort()
+ >>> blob.openDetached().read()
+ "I'm a happy blob."
+
+
+Teardown
+--------
+
+We don't need the storage directory and databases anymore::
+
+ >>> import shutil
+ >>> shutil.rmtree(blob_dir)
+ >>> tm1.get().abort()
+ >>> tm2.get().abort()
+ >>> database.close()
Copied: ZODB/trunk/src/ZODB/tests/testblob.py (from rev 76139, ZODB/trunk/src/ZODB/Blobs/tests/test_doctests.py)
===================================================================
--- ZODB/trunk/src/ZODB/tests/testblob.py (rev 0)
+++ ZODB/trunk/src/ZODB/tests/testblob.py 2007-06-03 08:26:47 UTC (rev 76192)
@@ -0,0 +1,285 @@
+##############################################################################
+#
+# Copyright (c) 2004 Zope Corporation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
+
+import base64, os, shutil, tempfile, unittest
+from zope.testing import doctest
+import ZODB.tests.util
+
+from ZODB import utils
+from ZODB.FileStorage import FileStorage
+from ZODB.blob import Blob, BlobStorage
+from ZODB.DB import DB
+import transaction
+
+from ZODB.tests.testConfig import ConfigTestBase
+from ZConfig import ConfigurationSyntaxError
+
+class BlobConfigTestBase(ConfigTestBase):
+
+ def setUp(self):
+ super(BlobConfigTestBase, self).setUp()
+
+ self.blob_dir = tempfile.mkdtemp()
+
+ def tearDown(self):
+ super(BlobConfigTestBase, self).tearDown()
+
+ shutil.rmtree(self.blob_dir)
+
+
+class ZODBBlobConfigTest(BlobConfigTestBase):
+
+ def test_map_config1(self):
+ self._test(
+ """
+ <zodb>
+ <blobstorage>
+ blob-dir %s
+ <mappingstorage/>
+ </blobstorage>
+ </zodb>
+ """ % self.blob_dir)
+
+ def test_file_config1(self):
+ path = tempfile.mktemp()
+ self._test(
+ """
+ <zodb>
+ <blobstorage>
+ blob-dir %s
+ <filestorage>
+ path %s
+ </filestorage>
+ </blobstorage>
+ </zodb>
+ """ %(self.blob_dir, path))
+ os.unlink(path)
+ os.unlink(path+".index")
+ os.unlink(path+".tmp")
+
+ def test_blob_dir_needed(self):
+ self.assertRaises(ConfigurationSyntaxError,
+ self._test,
+ """
+ <zodb>
+ <blobstorage>
+ <mappingstorage/>
+ </blobstorage>
+ </zodb>
+ """)
+
+
+class BlobUndoTests(unittest.TestCase):
+
+ def setUp(self):
+ self.test_dir = tempfile.mkdtemp()
+ self.here = os.getcwd()
+ os.chdir(self.test_dir)
+ self.storagefile = 'Data.fs'
+ os.mkdir('blobs')
+ self.blob_dir = 'blobs'
+
+ def tearDown(self):
+ os.chdir(self.here)
+ shutil.rmtree(self.test_dir)
+
+ def testUndoWithoutPreviousVersion(self):
+ base_storage = FileStorage(self.storagefile)
+ blob_storage = BlobStorage(self.blob_dir, base_storage)
+ database = DB(blob_storage)
+ connection = database.open()
+ root = connection.root()
+ transaction.begin()
+ root['blob'] = Blob()
+ transaction.commit()
+
+ serial = base64.encodestring(blob_storage._tid)
+
+ # undo the creation of the previously added blob
+ transaction.begin()
+ database.undo(serial, blob_storage._transaction)
+ transaction.commit()
+
+ connection.close()
+ connection = database.open()
+ root = connection.root()
+ # the blob footprint object should exist no longer
+ self.assertRaises(KeyError, root.__getitem__, 'blob')
+ database.close()
+
+ def testUndo(self):
+ base_storage = FileStorage(self.storagefile)
+ blob_storage = BlobStorage(self.blob_dir, base_storage)
+ database = DB(blob_storage)
+ connection = database.open()
+ root = connection.root()
+ transaction.begin()
+ blob = Blob()
+ blob.open('w').write('this is state 1')
+ root['blob'] = blob
+ transaction.commit()
+
+ transaction.begin()
+ blob = root['blob']
+ blob.open('w').write('this is state 2')
+ transaction.commit()
+
+ transaction.begin()
+ blob = root['blob']
+ self.assertEqual(blob.open('r').read(), 'this is state 2')
+ transaction.abort()
+
+ serial = base64.encodestring(blob_storage._tid)
+
+ transaction.begin()
+ blob_storage.undo(serial, blob_storage._transaction)
+ transaction.commit()
+
+ transaction.begin()
+ blob = root['blob']
+ self.assertEqual(blob.open('r').read(), 'this is state 1')
+ transaction.abort()
+ database.close()
+
+ def testUndoAfterConsumption(self):
+ base_storage = FileStorage(self.storagefile)
+ blob_storage = BlobStorage(self.blob_dir, base_storage)
+ database = DB(blob_storage)
+ connection = database.open()
+ root = connection.root()
+ transaction.begin()
+ open('consume1', 'w').write('this is state 1')
+ blob = Blob()
+ blob.consumeFile('consume1')
+ root['blob'] = blob
+ transaction.commit()
+
+ transaction.begin()
+ blob = root['blob']
+ open('consume2', 'w').write('this is state 2')
+ blob.consumeFile('consume2')
+ transaction.commit()
+
+ transaction.begin()
+ blob = root['blob']
+ self.assertEqual(blob.open('r').read(), 'this is state 2')
+ transaction.abort()
+
+ serial = base64.encodestring(blob_storage._tid)
+
+ transaction.begin()
+ blob_storage.undo(serial, blob_storage._transaction)
+ transaction.commit()
+
+ transaction.begin()
+ blob = root['blob']
+ self.assertEqual(blob.open('r').read(), 'this is state 1')
+ transaction.abort()
+
+ database.close()
+
+ def testRedo(self):
+ base_storage = FileStorage(self.storagefile)
+ blob_storage = BlobStorage(self.blob_dir, base_storage)
+ database = DB(blob_storage)
+ connection = database.open()
+ root = connection.root()
+ blob = Blob()
+
+ transaction.begin()
+ blob.open('w').write('this is state 1')
+ root['blob'] = blob
+ transaction.commit()
+
+ transaction.begin()
+ blob = root['blob']
+ blob.open('w').write('this is state 2')
+ transaction.commit()
+
+ serial = base64.encodestring(blob_storage._tid)
+
+ transaction.begin()
+ database.undo(serial)
+ transaction.commit()
+
+ transaction.begin()
+ blob = root['blob']
+ self.assertEqual(blob.open('r').read(), 'this is state 1')
+ transaction.abort()
+
+ serial = base64.encodestring(blob_storage._tid)
+
+ transaction.begin()
+ database.undo(serial)
+ transaction.commit()
+
+ transaction.begin()
+ blob = root['blob']
+ self.assertEqual(blob.open('r').read(), 'this is state 2')
+ transaction.abort()
+
+ database.close()
+
+ def testRedoOfCreation(self):
+ base_storage = FileStorage(self.storagefile)
+ blob_storage = BlobStorage(self.blob_dir, base_storage)
+ database = DB(blob_storage)
+ connection = database.open()
+ root = connection.root()
+ blob = Blob()
+
+ transaction.begin()
+ blob.open('w').write('this is state 1')
+ root['blob'] = blob
+ transaction.commit()
+
+ serial = base64.encodestring(blob_storage._tid)
+
+ transaction.begin()
+ database.undo(serial)
+ transaction.commit()
+
+ self.assertRaises(KeyError, root.__getitem__, 'blob')
+
+ serial = base64.encodestring(blob_storage._tid)
+
+ transaction.begin()
+ database.undo(serial)
+ transaction.commit()
+
+ transaction.begin()
+ blob = root['blob']
+ self.assertEqual(blob.open('r').read(), 'this is state 1')
+ transaction.abort()
+
+ database.close()
+
+def test_suite():
+ suite = unittest.TestSuite()
+ suite.addTest(unittest.makeSuite(ZODBBlobConfigTest))
+ suite.addTest(doctest.DocFileSuite(
+ "blob_basic.txt", "blob_connection.txt", "blob_transaction.txt",
+ "blob_packing.txt", "blob_importexport.txt", "blob_consume.txt",
+ setUp=ZODB.tests.util.setUp,
+ tearDown=ZODB.tests.util.tearDown,
+ ))
+ suite.addTest(unittest.makeSuite(BlobUndoTests))
+
+ return suite
+
+if __name__ == '__main__':
+ unittest.main(defaultTest = 'test_suite')
+
+
+
More information about the Zodb-checkins
mailing list