[Zope3-checkins] CVS: Zope3/lib/python/ZODB - BaseStorage.py:1.22 FileStorage.py:1.102 dbmStorage.py:1.5 utils.py:1.14

Jeremy Hylton jeremy@zope.com
Mon, 25 Nov 2002 14:54:51 -0500


Update of /cvs-repository/Zope3/lib/python/ZODB
In directory cvs.zope.org:/tmp/cvs-serv7770/ZODB

Modified Files:
	BaseStorage.py FileStorage.py dbmStorage.py utils.py 
Log Message:
First step in ZODB3 / ZODB4 integration.

Merge FileStorage and BaseStorage changes and all the other dependent
stuff.

Make copyTransactionsFrom() always use restore().
Replace all uses of U64 with u64.
Use newTimeStamp() from ZODB.TimeStamp.
Remove __len__() and getSize().
Add getExtensionMethods().


=== Zope3/lib/python/ZODB/BaseStorage.py 1.21 => 1.22 ===
--- Zope3/lib/python/ZODB/BaseStorage.py:1.21	Wed Jul 24 19:12:46 2002
+++ Zope3/lib/python/ZODB/BaseStorage.py	Mon Nov 25 14:54:50 2002
@@ -17,36 +17,33 @@
 """
 
 import threading
-import time
-import POSException
-from TimeStamp import TimeStamp
+from ZODB import POSException
+from ZODB.TimeStamp import newTimeStamp, TimeStamp
 z64='\0'*8
 
 class BaseStorage:
-    _transaction=None # Transaction that is being committed
-    _serial=z64       # Transaction serial number
-    _tstatus=' '      # Transaction status, used for copying data
-    _is_read_only = 0
+    _transaction = None # Transaction that is being committed
+    _serial = z64       # Transaction serial number
+    _tstatus = ' '      # Transaction status, used for copying data
+    _is_read_only = False
 
     def __init__(self, name, base=None):
-        
-        self.__name__=name
+        self._name = name
 
         # Allocate locks:
         l=threading.RLock()
-        self._lock_acquire=l.acquire
-        self._lock_release=l.release
+        self._lock_acquire = l.acquire
+        self._lock_release = l.release
         l=threading.Lock()
-        self._commit_lock_acquire=l.acquire
-        self._commit_lock_release=l.release
+        self._commit_lock_acquire = l.acquire
+        self._commit_lock_release = l.release
 
-        t=time.time()
-        t=self._ts=apply(TimeStamp,(time.gmtime(t)[:5]+(t%60,)))
-        self._serial=t.raw()
+        self._ts = newTimeStamp()
+        self._serial = self._ts.raw()
         if base is None:
-            self._oid='\0\0\0\0\0\0\0\0'
+            self._oid = z64
         else:
-            self._oid=base._oid
+            self._oid = base._oid
 
     def abortVersion(self, src, transaction):
         if transaction is not self._transaction:
@@ -61,11 +58,14 @@
     def close(self):
         pass
 
+    def sortKey(self):
+        # An implementation should override this if it can be shared
+        # by multiple processes and / or guarantee that self._name is
+        # uniquely defines the storage across all procsses.
+        return self._name
+
     def getName(self):
-        return self.__name__
-    
-    def getSize(self):
-        return len(self)*300 # WAG!
+        return self._name
     
     def history(self, oid, version, length=1):
         pass
@@ -74,6 +74,7 @@
         return ''
 
     def new_oid(self, last=None):
+         # 'last' is only for internal use, not part of the public API
         if self._is_read_only:
             raise POSException.ReadOnlyError()
         if last is None:
@@ -98,7 +99,7 @@
         return self._is_read_only
     
     def supportsVersions(self):
-        return 0
+        return False
         
     def tpc_abort(self, transaction):
         self._lock_acquire()
@@ -106,15 +107,18 @@
             if transaction is not self._transaction: return
             self._abort()
             self._clear_temp()
-            self._transaction=None
+            self._transaction = None
             self._commit_lock_release()
-        finally: self._lock_release()
+        finally:
+            self._lock_release()
 
     def _abort(self):
-        """Subclasses should redefine this to supply abort actions"""
+        # Subclasses should define this to supply abort actions.
         pass
 
     def tpc_begin(self, transaction, tid=None, status=' '):
+        if self._is_read_only:
+            raise POSException.ReadOnlyError()
         self._lock_acquire()
         try:
             if self._transaction is transaction:
@@ -122,51 +126,41 @@
             self._lock_release()
             self._commit_lock_acquire()
             self._lock_acquire()
-            self._transaction=transaction
+            self._transaction = transaction
             self._clear_temp()
 
-            user=transaction.user
-            desc=transaction.description
-            ext=transaction._extension
-            if ext:
+            if transaction._extension:
                 import cPickle
-                ext = cPickle.dumps(ext, 1)
+                ext = cPickle.dumps(transaction._extension, 1)
             else:
                 ext = ""
-            self._ude=user, desc, ext
-
+            self._ude= transaction.user, transaction.description, ext
             if tid is None:
-                t=time.time()
-                t=apply(TimeStamp,(time.gmtime(t)[:5]+(t%60,)))
-                self._ts=t=t.laterThan(self._ts)
-                self._serial=t.raw()
+                self._ts = newTimeStamp(self._ts)
+                self._serial = self._ts.raw()
             else:
-                self._ts=TimeStamp(tid)
-                self._serial=tid
-
-            self._tstatus=status
-
-            self._begin(self._serial, user, desc, ext)
-            
+                self._ts = TimeStamp(tid)
+                self._serial = tid
+            self._tstatus = status
+            self._begin(self._serial, *self._ude)
         finally:
             self._lock_release()
 
     def _begin(self, tid, u, d, e):
-        """Subclasses should redefine this to supply transaction start actions.
-        """
+        # Subclasses should define this to supply transaction start actions.
         pass
 
     def tpc_vote(self, transaction):
         self._lock_acquire()
         try:
-            if transaction is not self._transaction: return
+            if transaction is not self._transaction:
+                return
             self._vote()
         finally:
             self._lock_release()
 
     def _vote(self):
-        """Subclasses should redefine this to supply transaction vote actions.
-        """
+        # Subclasses should define this to supply transaction vote actions.
         pass
 
     def tpc_finish(self, transaction, f=None):
@@ -177,9 +171,7 @@
             try:
                 if f is not None:
                     f()
-
-                u, d, e = self._ude
-                self._finish(self._serial, u, d, e)
+                self._finish(self._serial, *self._ude)
                 self._clear_temp()
             finally:
                 self._ude = None
@@ -189,10 +181,11 @@
             self._lock_release()
 
     def _finish(self, tid, u, d, e):
-        """Subclasses should redefine this to supply transaction finish actions
-        """
+        # Subclasses should define this to supply transaction finish actions.
         pass
 
+    # XXX One of these two should go
+
     def undoInfo(self, first=0, last=-20, specification=None):
         """Return a list of transaction descriptions for use with undo.
 
@@ -230,7 +223,7 @@
         return ()
 
     def versionEmpty(self, version):
-        return 1
+        return True
 
     def versions(self, max=None):
         return ()
@@ -252,67 +245,56 @@
         raise POSException.Unsupported, (
             "Retrieval of historical revisions is not supported")
 
+    def getExtensionMethods(self):
+        """getExtensionMethods
+ 
+        This returns a dictionary whose keys are names of extra methods
+        provided by this storage. Storage proxies (such as ZEO) should
+        call this method to determine the extra methods that they need
+        to proxy in addition to the standard storage methods.
+        Dictionary values should be None; this will be a handy place
+        for extra marshalling information, should we need it
+        """
+	return {}
+
     def copyTransactionsFrom(self, other, verbose=0):
         """Copy transactions from another storage.
 
-        This is typically used for converting data from one storage to another.
+        This is typically used for converting data from one storage to
+        another.
         """
-        _ts=None
-        ok=1
-        preindex={};
-        preget=preindex.get   # waaaa
-        # restore() is a new storage API method which has an identical
-        # signature to store() except that it does not return anything.
-        # Semantically, restore() is also identical to store() except that it
-        # doesn't do the ConflictError or VersionLockError consistency
-        # checks.  The reason to use restore() over store() in this method is
-        # that store() cannot be used to copy transactions spanning a version
-        # commit or abort, or over transactional undos.
-        #
-        # We'll use restore() if it's available, otherwise we'll fall back to
-        # using store().  However, if we use store, then
-        # copyTransactionsFrom() may fail with VersionLockError or
-        # ConflictError.
-        if hasattr(self, 'restore'):
-            restoring = 1
-        else:
-            restoring = 0
+        _ts = None
+        ok = True
         for transaction in other.iterator():
-            
-            tid=transaction.tid
+            tid = transaction.tid
             if _ts is None:
-                _ts=TimeStamp(tid)
+                _ts = TimeStamp(tid)
             else:
-                t=TimeStamp(tid)
+                t = TimeStamp(tid)
                 if t <= _ts:
-                    if ok: print ('Time stamps out of order %s, %s' % (_ts, t))
-                    ok=0
-                    _ts=t.laterThan(_ts)
-                    tid=_ts.raw()
+                    if ok:
+                        print ('Time stamps out of order %s, %s' % (_ts, t))
+                    ok = False
+                    _ts = t.laterThan(_ts)
+                    tid = _ts.raw()
                 else:
                     _ts = t
                     if not ok:
                         print ('Time stamps back in order %s' % (t))
-                        ok=1
+                        ok = True
 
             if verbose: print _ts
             
             self.tpc_begin(transaction, tid, transaction.status)
             for r in transaction:
-                oid=r.oid
-                if verbose: print `oid`, r.version, len(r.data)
-                if restoring:
-                    self.restore(oid, r.serial, r.data, r.version, transaction)
-                else:
-                    pre=preget(oid, None)
-                    s=self.store(oid, pre, r.data, r.version, transaction)
-                    preindex[oid]=s
-                
+                if verbose: print `r.oid`, r.version, len(r.data)
+                self.restore(r.oid, r.serial, r.data, r.version,
+                             r.data_txn, transaction)
             self.tpc_vote(transaction)
             self.tpc_finish(transaction)
 
 class TransactionRecord:
-    """Abstract base class for iterator protocol"""
+    """Abstract base class for iterator protocol."""
 
 class DataRecord:
-    """Abstract base class for iterator protocol"""
+    """Abstract base class for iterator protocol."""


=== Zope3/lib/python/ZODB/FileStorage.py 1.101 => 1.102 === (1836/1936 lines abridged)
--- Zope3/lib/python/ZODB/FileStorage.py:1.101	Fri Nov 15 16:49:42 2002
+++ Zope3/lib/python/ZODB/FileStorage.py	Mon Nov 25 14:54:50 2002
@@ -2,52 +2,52 @@
 #
 # Copyright (c) 2001, 2002 Zope Corporation and Contributors.
 # All Rights Reserved.
-# 
+#
 # This software is subject to the provisions of the Zope Public License,
 # Version 2.0 (ZPL).  A copy of the ZPL should accompany this distribution.
 # THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
 # WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
 # WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
 # FOR A PARTICULAR PURPOSE.
-# 
+#
 ##############################################################################
 """File-based ZODB storage
- 
+
 Files are arranged as follows.
 
   - The first 4 bytes are a file identifier.
-  
+
   - The rest of the file consists of a sequence of transaction
     "records".
 
 A transaction record consists of:
 
   - 8-byte transaction id, which is also a time stamp.
-  
+
   - 8-byte transaction record length - 8.
-  
+
   - 1-byte status code
-  
+
   - 2-byte length of user name
-  
-  - 2-byte length of description 
-  
-  - 2-byte length of extension attributes 
-  
+
+  - 2-byte length of description
+
+  - 2-byte length of extension attributes
+

[-=- -=- -=- 1836 lines omitted -=- -=- -=-]

+        self.packt = packt
+        self.first = first
+        self.last = last
+        self.filter = filter
+        self.i = 0
+        self.results = []
+        self.stop = 0
+
+    def finished(self):
+        """Return True if UndoSearch has found enough records."""
+        # BAW: Why 39 please?  This makes no sense (see also below).
+        return self.i >= self.last or self.pos < 39 or self.stop
+
+    def search(self):
+        """Search for another record."""
+        dict = self._readnext()
+        if dict is not None and (self.filter is None or self.filter(dict)):
+            if self.i >= self.first:
+                self.results.append(dict)
+            self.i += 1
+
+    def _readnext(self):
+        """Read the next record from the storage."""
+        self.file.seek(self.pos - 8)
+        self.pos -= u64(self.file.read(8)) + 8
+        self.file.seek(self.pos)
+        h = self.file.read(TRANS_HDR_LEN)
+        tid, tl, status, ul, dl, el = struct.unpack(TRANS_HDR, h)
+        if tid < self.packt or status == 'p':
+            self.stop = 1
+            return None
+        if status != ' ':
+            return None
+        d = u = ''
+        if ul:
+            u = self.file.read(ul)
+        if dl:
+            d = self.file.read(dl)
+        e = {}
+        if el:
+            try:
+                e = loads(self.file.read(el))
+            except:
+                pass
+        d = {'id': base64.encodestring(tid).rstrip(),
+             'time': TimeStamp(tid).timeTime(),
+             'user_name': u,
+             'description': d}
+        d.update(e)
+        return d


=== Zope3/lib/python/ZODB/dbmStorage.py 1.4 => 1.5 ===
--- Zope3/lib/python/ZODB/dbmStorage.py:1.4	Mon Jun 10 19:27:44 2002
+++ Zope3/lib/python/ZODB/dbmStorage.py	Mon Nov 25 14:54:50 2002
@@ -43,11 +43,11 @@
         self._lock_acquire()
         try:    
             try:
-                return (os.stat(self.__name__+'.data')[6] +
-                        os.stat(self.__name__+'.dir')[6]
+                return (os.stat(self._name+'.data')[6] +
+                        os.stat(self._name+'.dir')[6]
                         )
             except:
-                try: return os.stat(self.__name__)[6]
+                try: return os.stat(self._name)[6]
                 except: return 0
         finally: self._lock_release()
 
@@ -70,7 +70,7 @@
 
     def getSize(self):
         self._lock_acquire()
-        try: return os.stat(self.__name__)[6]
+        try: return os.stat(self._name)[6]
         finally: self._lock_release()
 
     def pack(self, t, referencesf):


=== Zope3/lib/python/ZODB/utils.py 1.13 => 1.14 ===
--- Zope3/lib/python/ZODB/utils.py:1.13	Thu Aug  1 12:11:04 2002
+++ Zope3/lib/python/ZODB/utils.py	Mon Nov 25 14:54:50 2002
@@ -19,18 +19,10 @@
     """Pack an integer or long into a 8-byte string"""
     return struct.pack(">Q", v)
 
-# Note that the distinction between ints and longs is blurred in
-# Python 2.2.  So make u64() and U64() the same.
-
 def u64(v):
     """Unpack an 8-byte string into a 64-bit long integer."""
     return struct.unpack(">Q", v)[0]
 
-U64 = u64
-
-# clients use this module to get this contant
-t32 = 1L << 32
-
 def cp(f1, f2, l):
     read = f1.read
     write = f2.write
@@ -44,11 +36,3 @@
             break
         write(d)
         l = l - len(d)
-
-
-def newTimeStamp(old=None):
-    t = time.time()
-    ts = TimeStamp(time.gmtime(t)[:5]+(t%60,))
-    if old is not None:
-        return ts.laterThan(old)
-    return ts