[Zodb-checkins] SVN: ZODB/branches/blob-merge-branch/src/ First crack at merging ctheune-blobsupport into a recent trunk checkout (tests have not yet been run).

Chris McDonough chrism at plope.com
Fri Sep 23 19:23:15 EDT 2005


Log message for revision 38574:
  First crack at merging ctheune-blobsupport into a recent trunk checkout (tests have not yet been run).
  

Changed:
  U   ZODB/branches/blob-merge-branch/src/ZEO/ClientStorage.py
  U   ZODB/branches/blob-merge-branch/src/ZEO/ServerStub.py
  U   ZODB/branches/blob-merge-branch/src/ZEO/StorageServer.py
  U   ZODB/branches/blob-merge-branch/src/ZEO/tests/testZEO.py
  A   ZODB/branches/blob-merge-branch/src/ZODB/Blobs/
  _U  ZODB/branches/blob-merge-branch/src/ZODB/Blobs/tests/transaction.txt
  U   ZODB/branches/blob-merge-branch/src/ZODB/Connection.py
  U   ZODB/branches/blob-merge-branch/src/ZODB/ExportImport.py
  U   ZODB/branches/blob-merge-branch/src/ZODB/FileStorage/FileStorage.py
  U   ZODB/branches/blob-merge-branch/src/ZODB/component.xml
  U   ZODB/branches/blob-merge-branch/src/ZODB/config.py
  A   ZODB/branches/blob-merge-branch/src/ZODB/tests/loggingsupport.py
  U   ZODB/branches/blob-merge-branch/src/ZODB/utils.py
  A   ZODB/branches/blob-merge-branch/src/zope/proxy/
  _U  ZODB/branches/blob-merge-branch/src/zope/proxy/SETUP.cfg
  _U  ZODB/branches/blob-merge-branch/src/zope/proxy/__init__.py
  _U  ZODB/branches/blob-merge-branch/src/zope/proxy/_zope_proxy_proxy.c
  _U  ZODB/branches/blob-merge-branch/src/zope/proxy/interfaces.py
  _U  ZODB/branches/blob-merge-branch/src/zope/proxy/proxy.h
  _U  ZODB/branches/blob-merge-branch/src/zope/proxy/tests/__init__.py
  _U  ZODB/branches/blob-merge-branch/src/zope/proxy/tests/test_proxy.py

-=-
Modified: ZODB/branches/blob-merge-branch/src/ZEO/ClientStorage.py
===================================================================
--- ZODB/branches/blob-merge-branch/src/ZEO/ClientStorage.py	2005-09-23 21:33:02 UTC (rev 38573)
+++ ZODB/branches/blob-merge-branch/src/ZEO/ClientStorage.py	2005-09-23 23:23:15 UTC (rev 38574)
@@ -27,6 +27,7 @@
 import types
 import logging
 
+from zope.interface import implements
 from ZEO import ServerStub
 from ZEO.cache import ClientCache
 from ZEO.TransactionBuffer import TransactionBuffer
@@ -34,8 +35,11 @@
 from ZEO.auth import get_module
 from ZEO.zrpc.client import ConnectionManager
 
+from ZODB.Blobs.BlobStorage import BLOB_SUFFIX, BLOB_DIRTY
 from ZODB import POSException
+from ZODB import utils
 from ZODB.loglevels import BLATHER
+from ZODB.Blobs.interfaces import IBlobStorage
 from persistent.TimeStamp import TimeStamp
 
 logger = logging.getLogger('ZEO.ClientStorage')
@@ -93,6 +97,7 @@
     tpc_begin().
     """
 
+    implements(IBlobStorage)
     # Classes we instantiate.  A subclass might override.
 
     TransactionBufferClass = TransactionBuffer
@@ -106,7 +111,8 @@
                  wait_for_server_on_startup=None, # deprecated alias for wait
                  wait=None, wait_timeout=None,
                  read_only=0, read_only_fallback=0,
-                 username='', password='', realm=None):
+                 username='', password='', realm=None,
+                 blob_dir=tempfile.gettempdir()):
         """ClientStorage constructor.
 
         This is typically invoked from a custom_zodb.py file.
@@ -177,6 +183,11 @@
         password -- string with plaintext password to be used
             when authenticated.
 
+        realm -- not documented.
+
+        blob_dir -- directory path for blob data.  'blob data' is data that
+            is retrieved via the loadBlob API.
+
         Note that the authentication protocol is defined by the server
         and is detected by the ClientStorage upon connecting (see
         testConnection() and doAuth() for details).
@@ -303,6 +314,8 @@
         # is executing.
         self._lock = threading.Lock()
 
+        self.blob_dir = blob_dir
+
         # Decide whether to use non-temporary files
         if client is not None:
             dir = var or os.getcwd()
@@ -885,6 +898,60 @@
         self._tbuf.store(oid, version, data)
         return self._check_serials()
 
+    def storeBlob(self, oid, serial, data, blobfilename, version, txn):
+        serials = self.store(oid, serial, data, version, txn)
+        blobfile = open(blobfilename, "rb")
+        while True:
+            chunk = blobfile.read(4096)
+            # even if the blobfile is completely empty, we need to call
+            # storeBlob at least once in order to be able to call
+            # storeBlobEnd successfully.
+            self._server.storeBlob(oid, serial, chunk, version, id(txn))
+            if not chunk:
+                self._server.storeBlobEnd(oid, serial, data, version, id(txn))
+                break
+        os.unlink(blobfilename)
+        return serials
+
+    def _getDirtyFilename(self, oid, serial):
+        """Generate an intermediate filename for two-phase commit.
+        """
+        return self._getCleanFilename(oid, serial) + "." + BLOB_DIRTY
+
+    def _getCleanFilename(self, oid, tid):
+        return os.path.join(self.blob_dir,
+                            "%s-%s%s" % (utils.oid_repr(oid),
+                                         utils.tid_repr(tid), 
+                                         BLOB_SUFFIX,)
+                            )
+
+    def loadBlob(self, oid, serial, version):
+        blob_filename = self._getCleanFilename(oid, serial)
+        if os.path.exists(blob_filename):    # XXX see race condition below
+            return blob_filename
+
+        self._load_lock.acquire()
+        try:
+            if self._server is None:
+                raise ClientDisconnected()
+
+            tempfilename = self._getDirtyFilename(oid, serial)
+            tempfile = open(tempfilename, "wb")
+            
+            offset = 0
+            while True:
+                chunk = self._server.loadBlob(oid, serial, version, offset)
+                if not chunk:
+                    break
+                offset += len(chunk)
+                tempfile.write(chunk)
+
+            tempfile.close()
+            utils.best_rename(tempfilename, blob_filename)
+            return blob_filename
+        finally:
+            self._load_lock.release()
+
     def tpc_vote(self, txn):
         """Storage API: vote on a transaction."""
         if txn is not self._transaction:

Modified: ZODB/branches/blob-merge-branch/src/ZEO/ServerStub.py
===================================================================
--- ZODB/branches/blob-merge-branch/src/ZEO/ServerStub.py	2005-09-23 21:33:02 UTC (rev 38573)
+++ ZODB/branches/blob-merge-branch/src/ZEO/ServerStub.py	2005-09-23 23:23:15 UTC (rev 38574)
@@ -216,6 +216,12 @@
     def storea(self, oid, serial, data, version, id):
         self.rpc.callAsync('storea', oid, serial, data, version, id)
 
+    def storeBlobEnd(self, oid, serial, data, version, id):
+        self.rpc.callAsync('storeBlobEnd', oid, serial, data, version, id)
+
+    def storeBlob(self, oid, serial, chunk, version, id):
+        self.rpc.callAsync('storeBlob', oid, serial, chunk, version, id)
+
     ##
     # Start two-phase commit for a transaction
     # @param id id used by client to identify current transaction.  The
@@ -255,6 +261,9 @@
     def load(self, oid, version):
         return self.rpc.call('load', oid, version)
 
+    def loadBlob(self, oid, serial, version, offset):
+        return self.rpc.call('loadBlob', oid, serial, version, offset)
+
     def getSerial(self, oid):
         return self.rpc.call('getSerial', oid)
 

Modified: ZODB/branches/blob-merge-branch/src/ZEO/StorageServer.py
===================================================================
--- ZODB/branches/blob-merge-branch/src/ZEO/StorageServer.py	2005-09-23 21:33:02 UTC (rev 38573)
+++ ZODB/branches/blob-merge-branch/src/ZEO/StorageServer.py	2005-09-23 23:23:15 UTC (rev 38574)
@@ -42,7 +42,7 @@
 from ZODB.POSException import StorageError, StorageTransactionError
 from ZODB.POSException import TransactionError, ReadOnlyError, ConflictError
 from ZODB.serialize import referencesf
-from ZODB.utils import u64, oid_repr
+from ZODB.utils import u64, oid_repr, mktemp
 from ZODB.loglevels import BLATHER
 
 logger = logging.getLogger('ZEO.StorageServer')
@@ -93,6 +93,9 @@
         self.log_label = _label
         self.authenticated = 0
         self.auth_realm = auth_realm
+        self.blob_transfer = {}
+        self.blob_log = []
+        self.blob_loads = {}
         # The authentication protocol may define extra methods.
         self._extensions = {}
         for func in self.extensions:
@@ -454,6 +457,49 @@
         self.stats.stores += 1
         self.txnlog.store(oid, serial, data, version)
 
+    def storeBlobEnd(self, oid, serial, data, version, id):
+        key = (oid, id)
+        if key not in self.blob_transfer:
+            raise Exception, "Can't finish a non-started Blob"
+        tempname, tempfile = self.blob_transfer.pop(key)
+        tempfile.close()
+        self.blob_log.append((oid, serial, data, tempname, version))
+
+    def storeBlob(self, oid, serial, chunk, version, id):
+        # XXX check that underlying storage supports blobs
+        key = (oid, id)
+        if key not in self.blob_transfer:
+            tempname = mktemp()
+            tempfile = open(tempname, "wb")
+            self.blob_transfer[key] = (tempname, tempfile)   # XXX Force close and remove them when Storage closes
+        else:
+            tempname, tempfile = self.blob_transfer[key]
+
+        tempfile.write(chunk)
+ 
+    def loadBlob(self, oid, serial, version, offset):
+        key = (oid, serial)
+        if not key in self.blob_loads:
+            self.blob_loads[key] = \
+                    open(self.storage.loadBlob(oid, serial, version))
+        blobdata = self.blob_loads[key]
+        blobdata.seek(offset)
+        chunk = blobdata.read(4096)
+        if not chunk:
+            del self.blob_loads[key]
+        return chunk
+
+            
+           
+        
+            
+
+
+            
+
+
+        
+
     # The following four methods return values, so they must acquire
     # the storage lock and begin the transaction before returning.
 
@@ -596,6 +642,13 @@
             # load oid, serial, data, version
             if not self._store(*loader.load()):
                 break
+
+        # Blob support
+        while self.blob_log:
+            oid, oldserial, data, blobfilename, version = self.blob_log.pop()
+            self.storage.storeBlob(oid, oldserial, data, blobfilename, 
+                                   version, self.transaction,)
+
         resp = self._thunk()
         if delay is not None:
             delay.reply(resp)

Modified: ZODB/branches/blob-merge-branch/src/ZEO/tests/testZEO.py
===================================================================
--- ZODB/branches/blob-merge-branch/src/ZEO/tests/testZEO.py	2005-09-23 21:33:02 UTC (rev 38573)
+++ ZODB/branches/blob-merge-branch/src/ZEO/tests/testZEO.py	2005-09-23 23:23:15 UTC (rev 38574)
@@ -196,10 +196,66 @@
     def getConfig(self):
         return """<mappingstorage 1/>"""
 
-test_classes = [OneTimeTests,
-                FileStorageTests,
-                MappingStorageTests]
+class BlobAdaptedFileStorageTests(GenericTests):
+    """ZEO backed by a BlobStorage-adapted FileStorage."""
+    def setUp(self):
+        self.blobdir = tempfile.mkdtemp()
+        super(BlobAdaptedFileStorageTests, self).setUp()
+        
+    def tearDown(self):
+        import shutil
+        shutil.rmtree(self.blobdir)
+        super(BlobAdaptedFileStorageTests, self).tearDown()
 
+    def getConfig(self):
+        return """
+        <blobstorage 1>
+          blob-dir %s
+          <filestorage 2>
+            path %s
+          </filestorage>
+        </blobstorage>
+        """ % (self.blobdir, tempfile.mktemp())
+
+    def checkStoreBlob(self):
+        from ZODB.utils import oid_repr, tid_repr
+        from ZODB.Blobs.Blob import Blob
+        from ZODB.Blobs.BlobStorage import BLOB_SUFFIX
+        from ZODB.tests.StorageTestBase import zodb_pickle, ZERO, \
+             handle_serials
+        import transaction
+
+        somedata = 'a' * 10
+
+        blob = Blob()
+        bd_fh = blob.open('w')
+        bd_fh.write(somedata)
+        bd_fh.close()
+        tfname = bd_fh.name
+        oid = self._storage.new_oid()
+        data = zodb_pickle(blob)
+        self.assert_(os.path.exists(tfname))
+
+        t = transaction.Transaction()
+        try:
+            self._storage.tpc_begin(t)
+            r1 = self._storage.storeBlob(oid, ZERO, data, tfname, '', t)
+            r2 = self._storage.tpc_vote(t)
+            revid = handle_serials(oid, r1, r2)
+            self._storage.tpc_finish(t)
+        except:
+            self._storage.tpc_abort(t)
+            raise
+        self.assert_(not os.path.exists(tfname))
+        filename = os.path.join(self.blobdir, oid_repr(oid),
+                                tid_repr(revid) + BLOB_SUFFIX)
+        self.assert_(os.path.exists(filename))
+        self.assertEqual(somedata, open(filename).read())
+        
+
+test_classes = [FileStorageTests, MappingStorageTests,
+                BlobAdaptedFileStorageTests]
+
 def test_suite():
     suite = unittest.TestSuite()
     for klass in test_classes:

Copied: ZODB/branches/blob-merge-branch/src/ZODB/Blobs (from rev 38565, ZODB/branches/ctheune-blobsupport/src/ZODB/Blobs)

Modified: ZODB/branches/blob-merge-branch/src/ZODB/Connection.py
===================================================================
--- ZODB/branches/blob-merge-branch/src/ZODB/Connection.py	2005-09-23 21:33:02 UTC (rev 38573)
+++ ZODB/branches/blob-merge-branch/src/ZODB/Connection.py	2005-09-23 23:23:15 UTC (rev 38574)
@@ -27,6 +27,7 @@
 # interfaces
 from persistent.interfaces import IPersistentDataManager
 from ZODB.interfaces import IConnection
+from ZODB.Blobs.interfaces import IBlob, IBlobStorage
 from transaction.interfaces import ISavepointDataManager
 from transaction.interfaces import IDataManagerSavepoint
 from transaction.interfaces import ISynchronizer
@@ -551,7 +552,23 @@
                     raise ConflictError(object=obj)
                 self._modified.append(oid)
             p = writer.serialize(obj)  # This calls __getstate__ of obj
-            s = self._storage.store(oid, serial, p, self._version, transaction)
+
+            if IBlob.providedBy(obj):
+                if not IBlobStorage.providedBy(self._storage):
+                    raise Unsupported(
+                        "Storing Blobs in %s is not supported." % 
+                        repr(self._storage))
+                s = self._storage.storeBlob(oid, serial, p,
+                                            obj._p_blob_uncommitted,
+                                            self._version, transaction)
+                # we invalidate the object here in order to ensure
+                # that that the next attribute access of its name
+                # unghostify it, which will cause its blob data
+                # to be reattached "cleanly"
+                obj._p_invalidate()
+            else:
+                s = self._storage.store(oid, serial, p, self._version,
+                                        transaction)
             self._store_count += 1
             # Put the object in the cache before handling the
             # response, just in case the response contains the

Modified: ZODB/branches/blob-merge-branch/src/ZODB/ExportImport.py
===================================================================
--- ZODB/branches/blob-merge-branch/src/ZODB/ExportImport.py	2005-09-23 21:33:02 UTC (rev 38573)
+++ ZODB/branches/blob-merge-branch/src/ZODB/ExportImport.py	2005-09-23 23:23:15 UTC (rev 38574)
@@ -13,13 +13,16 @@
 ##############################################################################
 """Support for database export and import."""
 
+import os
+
 from cStringIO import StringIO
 from cPickle import Pickler, Unpickler
 from tempfile import TemporaryFile
 import logging
 
-from ZODB.POSException import ExportError
-from ZODB.utils import p64, u64
+from ZODB.POSException import ExportError, POSKeyError
+from ZODB.utils import p64, u64, cp, mktemp
+from ZODB.Blobs.interfaces import IBlobStorage
 from ZODB.serialize import referencesf
 
 logger = logging.getLogger('ZODB.ExportImport')
@@ -49,6 +52,21 @@
             else:
                 referencesf(p, oids)
                 f.writelines([oid, p64(len(p)), p])
+            # Blob support
+            if not IBlobStorage.providedBy(self._storage):
+                continue
+            try:
+                blobfilename = self._storage.loadBlob(oid, 
+                                                      serial, self._version)
+            except POSKeyError: # Looks like this is not a blob
+                continue
+
+            f.write(blob_begin_marker)
+            f.write(p64(os.stat(blobfilename).st_size))
+            blobdata = open(blobfilename, "rb")
+            cp(blobdata, f)
+            blobdata.close()
+            
         f.write(export_end_marker)
         return f
 
@@ -113,17 +131,20 @@
         version = self._version
 
         while 1:
-            h = f.read(16)
-            if h == export_end_marker:
+            header = f.read(16)
+            if header == export_end_marker:
                 break
-            if len(h) != 16:
+            if len(header) != 16:
                 raise ExportError("Truncated export file")
-            l = u64(h[8:16])
-            p = f.read(l)
-            if len(p) != l:
+
+            # Extract header information
+            ooid = header[:8]
+            length = u64(header[8:16])
+            data = f.read(length)
+
+            if len(data) != length:
                 raise ExportError("Truncated export file")
 
-            ooid = h[:8]
             if oids:
                 oid = oids[ooid]
                 if isinstance(oid, tuple):
@@ -132,7 +153,21 @@
                 oids[ooid] = oid = self._storage.new_oid()
                 return_oid_list.append(oid)
 
-            pfile = StringIO(p)
+            # Blob support
+            blob_begin = f.read(len(blob_begin_marker))
+            if blob_begin == blob_begin_marker:
+                # Copy the blob data to a temporary file
+                # and remember the name
+                blob_len = u64(f.read(8))
+                blob_filename = mktemp()
+                blob_file = open(blob_filename, "wb")
+                cp(f, blob_file, blob_len)
+                blob_file.close()
+            else:
+                f.seek(-len(blob_begin_marker),1)
+                blob_filename = None
+
+            pfile = StringIO(data)
             unpickler = Unpickler(pfile)
             unpickler.persistent_load = persistent_load
 
@@ -142,12 +177,17 @@
 
             pickler.dump(unpickler.load())
             pickler.dump(unpickler.load())
-            p = newp.getvalue()
+            data = newp.getvalue()
 
-            self._storage.store(oid, None, p, version, transaction)
+            if blob_filename is not None:
+                self._storage.storeBlob(oid, None, data, blob_filename, 
+                                        version, transaction)
+            else:
+                self._storage.store(oid, None, data, version, transaction)
 
 
 export_end_marker = '\377'*16
+blob_begin_marker = '\000BLOBSTART'
 
 class Ghost(object):
     __slots__ = ("oid",)

Modified: ZODB/branches/blob-merge-branch/src/ZODB/FileStorage/FileStorage.py
===================================================================
--- ZODB/branches/blob-merge-branch/src/ZODB/FileStorage/FileStorage.py	2005-09-23 21:33:02 UTC (rev 38573)
+++ ZODB/branches/blob-merge-branch/src/ZODB/FileStorage/FileStorage.py	2005-09-23 23:23:15 UTC (rev 38574)
@@ -628,7 +628,7 @@
         finally:
             self._lock_release()
 
-    def store(self, oid, serial, data, version, transaction):
+    def store(self, oid, oldserial, data, version, transaction):
         if self._is_read_only:
             raise POSException.ReadOnlyError()
         if transaction is not self._transaction:
@@ -651,12 +651,12 @@
                         pnv = h.pnv
                     cached_tid = h.tid
 
-                if serial != cached_tid:
+                if oldserial != cached_tid:
                     rdata = self.tryToResolveConflict(oid, cached_tid,
-                                                     serial, data)
+                                                     oldserial, data)
                     if rdata is None:
                         raise POSException.ConflictError(
-                            oid=oid, serials=(cached_tid, serial), data=data)
+                            oid=oid, serials=(cached_tid, oldserial), data=data)
                     else:
                         data = rdata
 
@@ -686,7 +686,7 @@
                 raise FileStorageQuotaError(
                     "The storage quota has been exceeded.")
 
-            if old and serial != cached_tid:
+            if old and oldserial != cached_tid:
                 return ConflictResolution.ResolvedSerial
             else:
                 return self._tid

Modified: ZODB/branches/blob-merge-branch/src/ZODB/component.xml
===================================================================
--- ZODB/branches/blob-merge-branch/src/ZODB/component.xml	2005-09-23 21:33:02 UTC (rev 38573)
+++ ZODB/branches/blob-merge-branch/src/ZODB/component.xml	2005-09-23 23:23:15 UTC (rev 38574)
@@ -65,6 +65,11 @@
   <sectiontype name="zeoclient" datatype=".ZEOClient"
                implements="ZODB.storage">
     <multikey name="server" datatype="socket-connection-address" required="yes"/>
+    <key name="blob-dir" required="no" default="/tmp">
+      <description>
+        Path name to the blob storage directory.
+      </description>
+    </key>
     <key name="storage" default="1">
       <description>
         The name of the storage that the client wants to use.  If the
@@ -158,4 +163,18 @@
     <key name="version-cache-size" datatype="integer" default="100"/>
   </sectiontype>
 
+  <sectiontype name="blobstorage" datatype=".BlobStorage"
+    implements="ZODB.storage">
+    <key name="blob-dir" required="yes">
+      <description>
+        Path name to the blob storage directory.
+      </description>
+    </key>
+    <section type="ZODB.storage" name="*" attribute="base"/>
+  </sectiontype>
+
+
+    
+
+
 </component>

Modified: ZODB/branches/blob-merge-branch/src/ZODB/config.py
===================================================================
--- ZODB/branches/blob-merge-branch/src/ZODB/config.py	2005-09-23 21:33:02 UTC (rev 38573)
+++ ZODB/branches/blob-merge-branch/src/ZODB/config.py	2005-09-23 23:23:15 UTC (rev 38574)
@@ -132,6 +132,14 @@
                            read_only=self.config.read_only,
                            quota=self.config.quota)
 
+class BlobStorage(BaseConfig):
+
+    def open(self):
+        from ZODB.Blobs.BlobStorage import BlobStorage
+        base = self.config.base.open()
+        return BlobStorage(self.config.blob_dir, base)
+
+        
 class ZEOClient(BaseConfig):
 
     def open(self):
@@ -141,6 +149,7 @@
         L = [server.address for server in self.config.server]
         return ClientStorage(
             L,
+            blob_dir=self.config.blob_dir,
             storage=self.config.storage,
             cache_size=self.config.cache_size,
             name=self.config.name,

Copied: ZODB/branches/blob-merge-branch/src/ZODB/tests/loggingsupport.py (from rev 38565, ZODB/branches/ctheune-blobsupport/src/ZODB/tests/loggingsupport.py)

Modified: ZODB/branches/blob-merge-branch/src/ZODB/utils.py
===================================================================
--- ZODB/branches/blob-merge-branch/src/ZODB/utils.py	2005-09-23 21:33:02 UTC (rev 38573)
+++ ZODB/branches/blob-merge-branch/src/ZODB/utils.py	2005-09-23 23:23:15 UTC (rev 38574)
@@ -16,11 +16,13 @@
 import time
 import struct
 from struct import pack, unpack
-from binascii import hexlify
+from binascii import hexlify, unhexlify
 import cPickle as pickle
 from cStringIO import StringIO
 import weakref
 import warnings
+from tempfile import mkstemp
+import os
 
 from persistent.TimeStamp import TimeStamp
 
@@ -90,21 +92,34 @@
 
 U64 = u64
 
-def cp(f1, f2, l):
+def cp(f1, f2, length=None):
+    """Copy all data from one file to another.
+    
+    It copies the data from the current position of the input file (f1)
+    appending it to the current position of the output file (f2). 
+    
+    It copies at most 'length' bytes. If 'length' isn't given, it copies
+    until the end of the input file.
+    """
     read = f1.read
     write = f2.write
     n = 8192
 
-    while l > 0:
-        if n > l:
-            n = l
-        d = read(n)
-        if not d:
+    if length is None:
+        old_pos = f1.tell()
+        f1.seek(0,2)
+        length = f1.tell()
+        f1.seek(old_pos)
+    
+    while length > 0:
+        if n > length:
+            n = length
+        data = read(n)
+        if not data:
             break
-        write(d)
-        l = l - len(d)
+        write(data)
+        length -= len(data)
 
-
 def newTimeStamp(old=None,
                  TimeStamp=TimeStamp,
                  time=time.time, gmtime=time.gmtime):
@@ -128,6 +143,13 @@
     else:
         return repr(oid)
 
+def repr_to_oid(repr):
+    if repr.startswith("0x"):
+        repr = repr[2:]
+    as_bin = unhexlify(repr)
+    as_bin = "\x00"*(8-len(as_bin)) + as_bin
+    return as_bin
+
 serial_repr = oid_repr
 tid_repr = serial_repr
 
@@ -273,3 +295,35 @@
         # We're cheating by breaking into the internals of Python's
         # WeakValueDictionary here (accessing its .data attribute).
         return self.data.data.values()
+
+
+def mktemp():
+    """Create a temp file, known by name, in a semi-secure manner."""
+    handle, filename = mkstemp()
+    os.close(handle)
+    return filename
+
+def best_rename(sourcename, targetname):
+    """ Try to rename via os.rename, but if we can't (for instance, if the
+    source and target are on separate partitions/volumes), fall back to copying
+    the file and unlinking the original. """
+    try:
+        os.rename(sourcename, targetname)
+    except OSError:
+        # XXX CM: I don't think this is a good idea; maybe just fail
+        # here instead of doing a brute force copy?  This is awfully
+        # expensive and people won't know it's happening without
+        # at least a warning.  It also increases the possibility of a race
+        # condition: both the source and target filenames exist at the
+        # same time.
+        source = open(sourcename, "rb")
+        target = open(targetname, "wb")
+        while True:
+            chunk = source.read(1<<16)
+            if not chunk:
+                break
+            target.write(chunk)
+        source.close()
+        target.close()
+        os.unlink(sourcename)
+

Copied: ZODB/branches/blob-merge-branch/src/zope/proxy (from rev 38565, ZODB/branches/ctheune-blobsupport/src/zope/proxy)


Property changes on: ZODB/branches/blob-merge-branch/src/zope/proxy
___________________________________________________________________
Name: svn:ignore
   + *so




More information about the Zodb-checkins mailing list