[Zodb-checkins] SVN: ZODB/branches/ctheune-blobsupport/src/Z -
Added Import/Export support. Actually this is even backwards
compatible.
Christian Theune
ct at gocept.com
Thu Mar 24 01:16:35 EST 2005
Log message for revision 29658:
- Added Import/Export support. Actually this is even backwards compatible.
Changed:
U ZODB/branches/ctheune-blobsupport/src/ZEO/ClientStorage.py
U ZODB/branches/ctheune-blobsupport/src/ZODB/Blobs/BlobStorage.py
U ZODB/branches/ctheune-blobsupport/src/ZODB/Blobs/TODO.txt
U ZODB/branches/ctheune-blobsupport/src/ZODB/Blobs/interfaces.py
U ZODB/branches/ctheune-blobsupport/src/ZODB/ExportImport.py
U ZODB/branches/ctheune-blobsupport/src/ZODB/TmpStore.py
-=-
Modified: ZODB/branches/ctheune-blobsupport/src/ZEO/ClientStorage.py
===================================================================
--- ZODB/branches/ctheune-blobsupport/src/ZEO/ClientStorage.py 2005-03-24 03:06:57 UTC (rev 29657)
+++ ZODB/branches/ctheune-blobsupport/src/ZEO/ClientStorage.py 2005-03-24 06:16:33 UTC (rev 29658)
@@ -901,6 +901,7 @@
self._server.storeBlobEnd(oid, serial, data, version, id(txn))
break
self._server.storeBlob(oid, serial, chunk, version, id(txn))
+ os.unlink(blobfilename)
return serials
def _getDirtyFilename(self, oid, serial):
@@ -917,6 +918,7 @@
utils.tid_repr(tid),
BLOB_SUFFIX,)
)
+
def loadBlob(self, oid, serial, version):
blob_filename = self._getCleanFilename(oid, serial)
if os.path.exists(blob_filename): # XXX see race condition below
Modified: ZODB/branches/ctheune-blobsupport/src/ZODB/Blobs/BlobStorage.py
===================================================================
--- ZODB/branches/ctheune-blobsupport/src/ZODB/Blobs/BlobStorage.py 2005-03-24 03:06:57 UTC (rev 29657)
+++ ZODB/branches/ctheune-blobsupport/src/ZODB/Blobs/BlobStorage.py 2005-03-24 06:16:33 UTC (rev 29658)
@@ -109,7 +109,10 @@
def loadBlob(self, oid, serial, version):
"""Return the filename where the blob file can be found.
"""
- return self._getCleanFilename(oid, serial)
+ filename = self._getCleanFilename(oid, serial)
+ if not os.path.exists(filename):
+ raise POSKeyError, "Not an existing blob."
+ return filename
def _getNewestBlobSerial(self, oid):
blob_path = self._getBlobPath(oid)
Modified: ZODB/branches/ctheune-blobsupport/src/ZODB/Blobs/TODO.txt
===================================================================
--- ZODB/branches/ctheune-blobsupport/src/ZODB/Blobs/TODO.txt 2005-03-24 03:06:57 UTC (rev 29657)
+++ ZODB/branches/ctheune-blobsupport/src/ZODB/Blobs/TODO.txt 2005-03-24 06:16:33 UTC (rev 29658)
@@ -1,12 +1,11 @@
- Support database import/export
-- Support ZEO
+- Support selection of text/binary mode for opening blobs
-- Support selection of text/binary mode for blobs
+- Generic wrapper configuration for BlobStorage
- - implement loadBlob and storeBlob
+Tests
+-----
- - loadBlob needs to handle the BLOB_CACHE_DIRECTORY
-
- - storeBlob needs to hand the actual file data off to the server
+- Importing backward compatible ZEXP files (no \0BLOBSTART) used
Modified: ZODB/branches/ctheune-blobsupport/src/ZODB/Blobs/interfaces.py
===================================================================
--- ZODB/branches/ctheune-blobsupport/src/ZODB/Blobs/interfaces.py 2005-03-24 03:06:57 UTC (rev 29657)
+++ ZODB/branches/ctheune-blobsupport/src/ZODB/Blobs/interfaces.py 2005-03-24 06:16:33 UTC (rev 29658)
@@ -25,6 +25,8 @@
serial.
Returns a filename or None if no Blob data is connected with this OID.
+
+ Raises POSKeyError if the blobfile cannot be found.
"""
def getBlobDirectory():
Modified: ZODB/branches/ctheune-blobsupport/src/ZODB/ExportImport.py
===================================================================
--- ZODB/branches/ctheune-blobsupport/src/ZODB/ExportImport.py 2005-03-24 03:06:57 UTC (rev 29657)
+++ ZODB/branches/ctheune-blobsupport/src/ZODB/ExportImport.py 2005-03-24 06:16:33 UTC (rev 29658)
@@ -13,13 +13,16 @@
##############################################################################
"""Support for database export and import."""
+import os
+
from cStringIO import StringIO
from cPickle import Pickler, Unpickler
from tempfile import TemporaryFile
import logging
-from ZODB.POSException import ExportError
-from ZODB.utils import p64, u64
+from ZODB.POSException import ExportError, POSKeyError
+from ZODB.utils import p64, u64, cp, mktemp
+from ZODB.Blobs.interfaces import IBlobStorage
from ZODB.serialize import referencesf
logger = logging.getLogger('ZODB.ExportImport')
@@ -49,6 +52,21 @@
else:
referencesf(p, oids)
f.writelines([oid, p64(len(p)), p])
+ # Blob support
+ if not IBlobStorage.providedBy(self._storage):
+ continue
+ try:
+ blobfilename = self._storage.loadBlob(oid,
+ serial, self._version)
+ except POSKeyError: # Looks like this is not a blob
+ continue
+
+ f.write(blob_begin_marker)
+ f.write(p64(os.stat(blobfilename).st_size))
+ blobdata = open(blobfilename, "rb")
+ cp(blobdata, f)
+ blobdata.close()
+
f.write(export_end_marker)
return f
@@ -109,17 +127,20 @@
version = self._version
while 1:
- h = f.read(16)
- if h == export_end_marker:
+ header = f.read(16)
+ if header == export_end_marker:
break
- if len(h) != 16:
+ if len(header) != 16:
raise ExportError("Truncated export file")
- l = u64(h[8:16])
- p = f.read(l)
- if len(p) != l:
+
+ # Extract header information
+ ooid = header[:8]
+ length = u64(header[8:16])
+ data = f.read(length)
+
+ if len(data) != length:
raise ExportError("Truncated export file")
- ooid = h[:8]
if oids:
oid = oids[ooid]
if isinstance(oid, tuple):
@@ -128,7 +149,21 @@
oids[ooid] = oid = self._storage.new_oid()
return_oid_list.append(oid)
- pfile = StringIO(p)
+ # Blob support
+ blob_begin = f.read(len(blob_begin_marker))
+ if blob_begin == blob_begin_marker:
+ # Copy the blob data to a temporary file
+ # and remember the name
+ blob_len = u64(f.read(8))
+ blob_filename = mktemp()
+ blob_file = open(blob_filename, "wb")
+ cp(f, blob_file, blob_len)
+ blob_file.close()
+ else:
+ f.seek(-len(blob_begin_marker),1)
+ blob_filename = None
+
+ pfile = StringIO(data)
unpickler = Unpickler(pfile)
unpickler.persistent_load = persistent_load
@@ -138,12 +173,17 @@
pickler.dump(unpickler.load())
pickler.dump(unpickler.load())
- p = newp.getvalue()
+ data = newp.getvalue()
- self._storage.store(oid, None, p, version, transaction)
+ if blob_filename is not None:
+ self._storage.storeBlob(oid, None, data, blob_filename,
+ version, transaction)
+ else:
+ self._storage.store(oid, None, data, version, transaction)
export_end_marker = '\377'*16
+blob_begin_marker = '\000BLOBSTART'
class Ghost(object):
__slots__ = ("oid",)
Modified: ZODB/branches/ctheune-blobsupport/src/ZODB/TmpStore.py
===================================================================
--- ZODB/branches/ctheune-blobsupport/src/ZODB/TmpStore.py 2005-03-24 03:06:57 UTC (rev 29657)
+++ ZODB/branches/ctheune-blobsupport/src/ZODB/TmpStore.py 2005-03-24 06:16:33 UTC (rev 29658)
@@ -12,6 +12,8 @@
#
##############################################################################
+import os
+
from zope.interface import implements
from ZODB.Blobs.interfaces import IBlobStorage
@@ -150,6 +152,6 @@
def generateBlobFile(self, oid):
if not self.blob_files.has_key(oid):
handle, name = tempfile.mkstemp()
- handle.close()
+ os.close(handle)
self.blob_files[oid] = name
return self.blob_files[oid]
More information about the Zodb-checkins
mailing list