[Zodb-checkins]
SVN: ZODB/branches/ctheune-blobszerocopy/src/ZODB/Blobs/
- Added `consumeFile` method to provide an O(1) import for blobs
Christian Theune
ct at gocept.com
Wed Mar 7 16:43:32 EST 2007
Log message for revision 73040:
- Added `consumeFile` method to provide an O(1) import for blobs
Changed:
U ZODB/branches/ctheune-blobszerocopy/src/ZODB/Blobs/Blob.py
U ZODB/branches/ctheune-blobszerocopy/src/ZODB/Blobs/interfaces.py
A ZODB/branches/ctheune-blobszerocopy/src/ZODB/Blobs/tests/consume.txt
U ZODB/branches/ctheune-blobszerocopy/src/ZODB/Blobs/tests/test_doctests.py
-=-
Modified: ZODB/branches/ctheune-blobszerocopy/src/ZODB/Blobs/Blob.py
===================================================================
--- ZODB/branches/ctheune-blobszerocopy/src/ZODB/Blobs/Blob.py 2007-03-07 20:59:22 UTC (rev 73039)
+++ ZODB/branches/ctheune-blobszerocopy/src/ZODB/Blobs/Blob.py 2007-03-07 21:43:31 UTC (rev 73040)
@@ -63,44 +63,40 @@
if (mode.startswith("r") or mode=="U"):
if self._current_filename() is None:
- raise BlobError, "Blob does not exist."
+ raise BlobError("Blob does not exist.")
if self._p_blob_writers != 0:
- raise BlobError, "Already opened for writing."
+ raise BlobError("Already opened for writing.")
self._p_blob_readers += 1
result = BlobFile(self._current_filename(), mode, self)
elif mode.startswith("w"):
if self._p_blob_readers != 0:
- raise BlobError, "Already opened for reading."
+ raise BlobError("Already opened for reading.")
- if self._p_blob_uncommitted is None:
- self._p_blob_uncommitted = utils.mktemp(dir=tempdir)
-
self._p_blob_writers += 1
- result = BlobFile(self._p_blob_uncommitted, mode, self)
+ result = BlobFile(self._get_uncommitted_filename(), mode, self)
elif mode.startswith("a"):
if self._p_blob_readers != 0:
- raise BlobError, "Already opened for reading."
+ raise BlobError("Already opened for reading.")
if self._p_blob_uncommitted is None:
# Create a new working copy
- self._p_blob_uncommitted = utils.mktemp(dir=tempdir)
- uncommitted = BlobFile(self._p_blob_uncommitted, mode, self)
+ uncommitted = BlobFile(self._get_uncommitted_filename(), mode, self)
# NOTE: _p_blob data appears by virtue of Connection._setstate
utils.cp(file(self._p_blob_data), uncommitted)
uncommitted.seek(0)
else:
# Re-use existing working copy
- uncommitted = BlobFile(self._p_blob_uncommitted, mode, self)
+ uncommitted = BlobFile(self._get_uncommitted_filename(), mode, self)
self._p_blob_writers += 1
result = uncommitted
else:
- raise IOError, 'invalid mode: %s ' % mode
+ raise IOError('invalid mode: %s ' % mode)
if result is not None:
# We join the transaction with our own data manager in order to be
@@ -140,11 +136,28 @@
"""
if self._current_filename() is None:
- raise BlobError, "Blob does not exist."
+ raise BlobError("Blob does not exist.")
if self._p_blob_writers != 0:
- raise BlobError, "Already opened for writing."
+ raise BlobError("Already opened for writing.")
+ # XXX this should increase the reader number and have a test !?!
return file(self._current_filename(), "rb")
+ def consumeFile(self, filename):
+ """Will replace the current data of the blob with the file given under
+ filename.
+ """
+ if self._p_blob_writers != 0:
+ raise BlobError("Already opened for writing.")
+ if self._p_blob_readers != 0:
+ raise BlobError("Already opened for reading.")
+ target = self._get_uncommitted_filename()
+ # XXX What if link fails and the target was removed? We should do a rename and
+ # maybe name it back if link gives an exception.
+ if os.path.exists(target):
+ os.unlink(target)
+ # XXX what if link() fails
+ os.link(filename, target)
+
# utility methods
def _current_filename(self):
@@ -152,6 +165,16 @@
# Connection._setstate
return self._p_blob_uncommitted or self._p_blob_data
+ def _get_uncommitted_filename(self):
+ """Return the filename for existing uncommitted data
+ or generate a new filename and set it as the current filename
+ for uncomitted data.
+ """
+ if self._p_blob_uncommitted is None:
+ tempdir = os.environ.get('ZODB_BLOB_TEMPDIR', tempfile.gettempdir())
+ self._p_blob_uncommitted = utils.mktemp(dir=tempdir)
+ return self._p_blob_uncommitted
+
def _change(self):
self._p_changed = 1
@@ -171,7 +194,7 @@
elif mode.startswith('w') or mode.startswith('a'):
self._p_blob_writers = max(0, self._p_blob_writers - 1)
else:
- raise AssertionError, 'Unknown mode %s' % mode
+ raise AssertionError('Unknown mode %s' % mode)
def _p_blob_refcounts(self):
# used by unit tests
Modified: ZODB/branches/ctheune-blobszerocopy/src/ZODB/Blobs/interfaces.py
===================================================================
--- ZODB/branches/ctheune-blobszerocopy/src/ZODB/Blobs/interfaces.py 2007-03-07 20:59:22 UTC (rev 73039)
+++ ZODB/branches/ctheune-blobszerocopy/src/ZODB/Blobs/interfaces.py 2007-03-07 21:43:31 UTC (rev 73040)
@@ -1,6 +1,23 @@
+##############################################################################
+#
+# Copyright (c) 2005-2007 Zope Corporation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE
+#
+##############################################################################
+"""Blob-related interfaces
+"""
+
from zope.interface import Interface
+
class IBlob(Interface):
"""A BLOB supports efficient handling of large data within ZODB."""
@@ -21,10 +38,18 @@
transaction.
"""
- # XXX need a method to initialize the blob from the storage
- # this means a) setting the _p_blob_data filename and b) putting
- # the current data in that file
+ def consumeFile(filename):
+ """Will replace the current data of the blob with the file given under
+ filename.
+ This method uses link() internally and has the same requirements (UNIX
+ only and must live on the same partition as the original file).
+
+ The blob must not be opened for reading or writing when consuming a
+ file.
+ """
+
+
class IBlobStorage(Interface):
"""A storage supporting BLOBs."""
@@ -39,4 +64,3 @@
Raises POSKeyError if the blobfile cannot be found.
"""
-
Added: ZODB/branches/ctheune-blobszerocopy/src/ZODB/Blobs/tests/consume.txt
===================================================================
--- ZODB/branches/ctheune-blobszerocopy/src/ZODB/Blobs/tests/consume.txt 2007-03-07 20:59:22 UTC (rev 73039)
+++ ZODB/branches/ctheune-blobszerocopy/src/ZODB/Blobs/tests/consume.txt 2007-03-07 21:43:31 UTC (rev 73040)
@@ -0,0 +1,67 @@
+Consuming existing files
+========================
+
+The ZODB Blob implementation allows to import existing files as Blobs within
+an O(1) operation we call `consume`::
+
+Let's create a file::
+
+ >>> import tempfile
+ >>> to_import = tempfile.NamedTemporaryFile()
+ >>> to_import.write("I'm a Blob and I feel fine.")
+ >>> to_import.flush()
+
+Now, let's consume this file in a blob by specifying it's name::
+
+ >>> from ZODB.Blobs.Blob import Blob
+ >>> blob = Blob()
+ >>> blob.consumeFile(to_import.name)
+
+We now can call open on the blob and read and write the data::
+
+ >>> blob_read = blob.open('rb')
+ >>> blob_read.read()
+ "I'm a Blob and I feel fine."
+ >>> blob_read.close()
+ >>> blob_write = blob.open('w')
+ >>> blob_write.write('I was changed.')
+ >>> blob_write.close()
+
+Please note that the interface for the `consume` method specifies a hard-link
+as a part of the contract so your existing file and the blob file will be the
+same. If one gets changed the other will reflect those changes as well. This
+is especially a known side-effect when consuming a file and then opening the
+blob for writing before committing in between::
+
+ >>> to_import.seek(0)
+ >>> to_import.read()
+ 'I was changed.'
+
+(Applications are expected that files for consumption are typically copies of
+existing data and that the imported link to the file will be removed after a
+successfull import. This can be achieved (as in this test) by using a
+NamedTempFile.)
+
+We can not consume a file when there is a reader or writer around for a blob
+already::
+
+ >>> to_import2 = tempfile.NamedTemporaryFile()
+ >>> to_import2.write('I am another blob.')
+ >>> to_import2.flush()
+ >>> blob_read = blob.open('r')
+ >>> blob.consumeFile(to_import2.name)
+ Traceback (most recent call last):
+ BlobError: Already opened for reading.
+ >>> blob_read.close()
+ >>> blob_write = blob.open('w')
+ >>> blob.consumeFile(to_import2.name)
+ Traceback (most recent call last):
+ BlobError: Already opened for writing.
+ >>> blob_write.close()
+
+Now, after closing all readers and writers we can consume files again::
+
+ >>> blob.consumeFile(to_import2.name)
+ >>> blob_read = blob.open('r')
+ >>> blob_read.read()
+ 'I am another blob.'
Property changes on: ZODB/branches/ctheune-blobszerocopy/src/ZODB/Blobs/tests/consume.txt
___________________________________________________________________
Name: svn:keywords
+ Id Rev Date
Name: svn:eol-style
+ native
Modified: ZODB/branches/ctheune-blobszerocopy/src/ZODB/Blobs/tests/test_doctests.py
===================================================================
--- ZODB/branches/ctheune-blobszerocopy/src/ZODB/Blobs/tests/test_doctests.py 2007-03-07 20:59:22 UTC (rev 73039)
+++ ZODB/branches/ctheune-blobszerocopy/src/ZODB/Blobs/tests/test_doctests.py 2007-03-07 21:43:31 UTC (rev 73040)
@@ -16,4 +16,4 @@
def test_suite():
return DocFileSuite("basic.txt", "connection.txt", "transaction.txt",
- "packing.txt", "importexport.txt")
+ "packing.txt", "importexport.txt", "consume.txt")
More information about the Zodb-checkins
mailing list