[Zope-Checkins] CVS: Zope/lib/python/ZODB - config.py:1.2.2.1 config.xml:1.2.2.1 ActivityMonitor.py:1.3.4.5 BaseStorage.py:1.20.4.11 Connection.py:1.72.4.10 DB.py:1.43.4.4 DemoStorage.py:1.12.4.11 FileStorage.py:1.95.4.10 POSException.py:1.12.4.13 TimeStamp.c:1.15.58.7 cPersistence.c:1.62.8.7 cPickleCache.c:1.68.8.4

Chris McDonough chrism@zope.com
Sat, 4 Jan 2003 13:16:58 -0500


Update of /cvs-repository/Zope/lib/python/ZODB
In directory cvs.zope.org:/tmp/cvs-serv3465

Modified Files:
      Tag: chrism-install-branch
	ActivityMonitor.py BaseStorage.py Connection.py DB.py 
	DemoStorage.py FileStorage.py POSException.py TimeStamp.c 
	cPersistence.c cPickleCache.c 
Added Files:
      Tag: chrism-install-branch
	config.py config.xml 
Log Message:
Merge with HEAD.


=== Added File Zope/lib/python/ZODB/config.py ===
##############################################################################
#
# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL).  A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
"""Open database and storage from a configuration.

$Id: config.py,v 1.2.2.1 2003/01/04 18:16:21 chrism Exp $"""

import os
import StringIO

import ZConfig

import ZODB

schema_path = os.path.join(ZODB.__path__[0], "config.xml")
_schema = None

def getSchema():
    global _schema
    if _schema is None:
        _schema = ZConfig.loadSchema(schema_path)
    return _schema

def databaseFromString(s):
    return databaseFromFile(StringIO.StringIO(s))

def databaseFromFile(f):
    config, handle = ZConfig.loadConfigFile(getSchema(), f)
    return databaseFromConfig(config)

def databaseFromURL(url):
    config, handler = ZConfig.loadConfig(getSchema(), url)
    return databaseFromConfig(config)

def databaseFromConfig(config):
    return ZODB.DB(config.storage.open(),
                   pool_size=config.pool_size,
                   cache_size=config.cache_size,
                   version_pool_size=config.version_pool_size,
                   version_cache_size=config.version_cache_size)

class StorageConfig:

    def __init__(self, config):
        self.config = config

    def open(self):
        raise NotImplementedError

class MappingStorage(StorageConfig):

    def open(self):
        from ZODB.MappingStorage import MappingStorage
        return MappingStorage(self.config.name)

class FileStorage(StorageConfig):

    def open(self):
        from ZODB.FileStorage import FileStorage
        return FileStorage(self.config.path,
                           create=self.config.create,
                           read_only=self.config.read_only,
                           stop=self.config.stop,
                           quota=self.config.quota)

class ZEOClient(StorageConfig):

    def open(self):
        from ZEO.ClientStorage import ClientStorage
        # config.server is a multikey of socket-address values
        # where the value is a socket family, address tuple.
        L = [addr for family, addr in self.config.server]
        return ClientStorage(
            L,
            storage=self.config.storage,
            cache_size=self.config.cache_size,
            name=self.config.name,
            client=self.config.client,
            var=self.config.var,
            min_disconnect_poll=self.config.min_disconnect_poll,
            max_disconnect_poll=self.config.max_disconnect_poll,
            wait=self.config.wait,
            read_only=self.config.read_only,
            read_only_fallback=self.config.read_only_fallback)

class BDBStorage(StorageConfig):
    
    def open(self):
        from BDBStorage.BerkeleyBase import BerkeleyConfig
        from BDBStorage.BDBFullStorage import BDBFullStorage
        from BDBStorage.BDBMinimalStorage import BDBMinimalStorage
        # Figure out which class we want
        sectiontype = self.config.getSectionType()
        storageclass = {'fullstorage': BDBFullStorage,
                        'minimalstorage': BDBMinimalStorage,
                        }[sectiontype]
        bconf = BerkeleyConfig()
        for name in dir(BerkeleyConfig):
            if name.startswith('_'):
                continue
            setattr(bconf, name, getattr(self.config, name))
        return storageclass(self.config.name, config=bconf)


=== Added File Zope/lib/python/ZODB/config.xml ===
<schema type="database">

  <sectiongroup type="storage">

    <sectiontype type="filestorage" datatype="ZODB.config.FileStorage">
      <key name="path" required="yes"/>
      <key name="create" datatype="boolean" default="true"/>
      <key name="read_only" datatype="boolean" default="false"/>
      <key name="stop"/>
      <key name="quota" datatype="integer"/>
    </sectiontype>

    <sectiontype type="mappingstorage" datatype="ZODB.config.MappingStorage">
      <key name="name" default="Mapping Storage"/>
    </sectiontype>

    <sectiontype type="fullstorage" datatype="ZODB.config.BDBStorage">
      <key name="name" required="yes" />
      <key name="interval" datatype="time-interval" default="2m" />
      <key name="kbyte" datatype="integer" default="0" />
      <key name="min" datatype="integer" default="0" />
      <key name="logdir" />
      <key name="cachesize" datatype="byte-size" default="128MB" />
      <key name="frequency" datatype="time-interval" default="0" />
      <key name="packtime" datatype="time-interval" default="4h" />
      <key name="classicpack" datatype="integer" default="0" />
      <key name="read_only" datatype="boolean" default="off"/>
    </sectiontype>

    <!-- XXX Fred promises to make it so minimal storage is just an
         extension of fullstorage -->
    <sectiontype type="minimalstorage" datatype="ZODB.config.BDBStorage">
      <key name="name" required="yes" />
      <key name="interval" datatype="time-interval" default="2m" />
      <key name="kbyte" datatype="integer" default="0" />
      <key name="min" datatype="integer" default="0" />
      <key name="logdir" />
      <key name="cachesize" datatype="byte-size" default="128MB" />
      <key name="frequency" datatype="time-interval" default="0" />
      <key name="packtime" datatype="time-interval" default="4h" />
      <key name="classicpack" datatype="integer" default="0" />
      <key name="read_only" datatype="boolean" default="off"/>
    </sectiontype>

    <sectiontype type="zeoclient" datatype="ZODB.config.ZEOClient">
      <multikey name="server" datatype="socket-address" required="yes"/>
      <key name="storage" default="1"/>
      <key name="cache_size" datatype="integer" default="20000000"/>
      <key name="name" default=""/>
      <key name="client"/>
      <key name="var"/>
      <key name="min_disconnect_poll" datatype="integer" default="5"/>
      <key name="max_disconnect_poll" datatype="integer" default="300"/>
      <key name="wait" datatype="boolean" default="on"/>
      <key name="read_only" datatype="boolean" default="off"/>
      <key name="read_only_fallback" datatype="boolean" default="off"/>
    </sectiontype>

    <sectiontype type="demostorage">
      <!--datatype="ZODB.config.DemoStorage"-->
      <key name="name" default="Demo Storage"/>
      <section type="storage" name="*" attribute="base"/>
      <key name="quota" datatype="integer"/>
    </sectiontype>

  </sectiongroup>

  <!-- the rest is the actual configuration for the database -->
  <section type="storage" name="*" attribute="storage"/>
  <key name="cache_size" datatype="integer" default="5000"/>
  <key name="pool_size" datatype="integer" default="7"/>
  <key name="version_pool_size" datatype="integer" default="3"/>
  <key name="version_cache_size" datatype="integer" default="100"/>

</schema>


=== Zope/lib/python/ZODB/ActivityMonitor.py 1.3.4.4 => 1.3.4.5 ===


=== Zope/lib/python/ZODB/BaseStorage.py 1.20.4.10 => 1.20.4.11 ===
--- Zope/lib/python/ZODB/BaseStorage.py:1.20.4.10	Fri Jan  3 01:37:19 2003
+++ Zope/lib/python/ZODB/BaseStorage.py	Sat Jan  4 13:16:20 2003
@@ -12,11 +12,9 @@
 #
 ##############################################################################
 """Handy standard storage machinery
-"""
-# Do this portably in the face of checking out with -kv
-import string
-__version__ = string.split('$Revision$')[-2:][0]
 
+$Id$
+"""
 import cPickle
 import ThreadLock, bpthread
 import time, UndoLogCompatible
@@ -277,8 +275,8 @@
             restoring = 1
         else:
             restoring = 0
-        for transaction in other.iterator():
-
+        fiter = other.iterator()
+        for transaction in fiter:
             tid=transaction.tid
             if _ts is None:
                 _ts=TimeStamp(tid)
@@ -312,6 +310,8 @@
 
             self.tpc_vote(transaction)
             self.tpc_finish(transaction)
+
+        fiter.close()
 
 class TransactionRecord:
     """Abstract base class for iterator protocol"""


=== Zope/lib/python/ZODB/Connection.py 1.72.4.9 => 1.72.4.10 ===


=== Zope/lib/python/ZODB/DB.py 1.43.4.3 => 1.43.4.4 ===


=== Zope/lib/python/ZODB/DemoStorage.py 1.12.4.10 => 1.12.4.11 ===


=== Zope/lib/python/ZODB/FileStorage.py 1.95.4.9 => 1.95.4.10 ===
--- Zope/lib/python/ZODB/FileStorage.py:1.95.4.9	Fri Jan  3 01:37:19 2003
+++ Zope/lib/python/ZODB/FileStorage.py	Sat Jan  4 13:16:20 2003
@@ -124,7 +124,7 @@
 import struct
 import sys
 import time
-from types import StringType
+from types import StringType, DictType
 from struct import pack, unpack
 
 try:
@@ -137,7 +137,12 @@
 from ZODB.TimeStamp import TimeStamp
 from ZODB.lock_file import lock_file
 from ZODB.utils import p64, u64, cp, z64
-from ZODB.fsIndex import fsIndex
+
+try:
+    from ZODB.fsIndex import fsIndex
+except ImportError:
+    def fsIndex():
+        return {}
 
 from zLOG import LOG, BLATHER, WARNING, ERROR, PANIC
 
@@ -203,6 +208,8 @@
     # default pack time is 0
     _packt = z64
 
+    _records_before_save = 10000
+
     def __init__(self, file_name, create=0, read_only=0, stop=None,
                  quota=None):
 
@@ -270,7 +277,9 @@
 
         r = self._restore_index()
         if r is not None:
+            self._used_index = 1 # Marker for testing
             index, vindex, start, maxoid, ltid = r
+
             self._initIndex(index, vindex, tindex, tvindex)
             self._pos, self._oid, tid = read_index(
                 self._file, file_name, index, vindex, tindex, stop,
@@ -278,10 +287,15 @@
                 read_only=read_only,
                 )
         else:
+            self._used_index = 0 # Marker for testing
             self._pos, self._oid, tid = read_index(
                 self._file, file_name, index, vindex, tindex, stop,
                 read_only=read_only,
                 )
+            self._save_index()
+
+        self._records_before_save = max(self._records_before_save,
+                                        len(self._index))
         self._ltid = tid
         
         # self._pos should always point just past the last
@@ -314,6 +328,7 @@
         # hook to use something other than builtin dict
         return fsIndex(), {}, {}, {}
 
+    _saved = 0
     def _save_index(self):
         """Write the database index to a file to support quick startup."""
 
@@ -329,6 +344,7 @@
         p.dump(info)
         f.flush()
         f.close()
+
         try:
             try:
                 os.remove(index_name)
@@ -337,6 +353,8 @@
             os.rename(tmp_name, index_name)
         except: pass
 
+        self._saved += 1
+
     def _clear_index(self):
         index_name = self.__name__ + '.index'
         if os.path.exists(index_name):
@@ -354,58 +372,77 @@
         object positions cause zero to be returned.
         """
 
-        if pos < 100: return 0
-        file=self._file
-        seek=file.seek
-        read=file.read
+        if pos < 100:
+            return 0 # insane
+        file = self._file
+        seek = file.seek
+        read = file.read
         seek(0,2)
-        if file.tell() < pos: return 0
-        ltid=None
+        if file.tell() < pos:
+            return 0 # insane
+        ltid = None
 
-        while 1:
+        max_checked = 5
+        checked = 0
+
+        while checked < max_checked:
             seek(pos-8)
-            rstl=read(8)
-            tl=u64(rstl)
-            pos=pos-tl-8
-            if pos < 4: return 0
+            rstl = read(8)
+            tl = u64(rstl)
+            pos = pos-tl-8
+            if pos < 4:
+                return 0 # insane
             seek(pos)
             s = read(TRANS_HDR_LEN)
             tid, stl, status, ul, dl, el = unpack(TRANS_HDR, s)
-            if not ltid: ltid=tid
-            if stl != rstl: return 0 # inconsistent lengths
-            if status == 'u': continue # undone trans, search back
-            if status not in ' p': return 0
-            if tl < (TRANS_HDR_LEN + ul + dl + el): return 0
-            tend=pos+tl
-            opos=pos+(TRANS_HDR_LEN + ul + dl + el)
-            if opos==tend: continue # empty trans
+            if not ltid:
+                ltid = tid
+            if stl != rstl:
+                return 0 # inconsistent lengths
+            if status == 'u':
+                continue # undone trans, search back
+            if status not in ' p':
+                return 0 # insane
+            if tl < (TRANS_HDR_LEN + ul + dl + el):
+                return 0 # insane
+            tend = pos+tl
+            opos = pos+(TRANS_HDR_LEN + ul + dl + el)
+            if opos == tend:
+                continue # empty trans
 
-            while opos < tend:
+            while opos < tend and checked < max_checked:
                 # Read the data records for this transaction
                 seek(opos)
-                h=read(DATA_HDR_LEN)
-                oid,serial,sprev,stloc,vlen,splen = unpack(DATA_HDR, h)
-                tloc=u64(stloc)
-                plen=u64(splen)
+                h = read(DATA_HDR_LEN)
+                oid, serial, sprev, stloc, vlen, splen = unpack(DATA_HDR, h)
+                tloc = u64(stloc)
+                plen = u64(splen)
+
+                dlen = DATA_HDR_LEN+(plen or 8)
+                if vlen:
+                    dlen = dlen+(16+vlen)
 
-                dlen=DATA_HDR_LEN+(plen or 8)
-                if vlen: dlen=dlen+(16+vlen)
+                if opos+dlen > tend or tloc != pos:
+                    return 0 # insane
 
-                if opos+dlen > tend or tloc != pos: return 0
+                if index.get(oid, 0) != opos:
+                    return 0 # insane
 
-                if index.get(oid, 0) != opos: return 0
+                checked += 1
 
-                opos=opos+dlen
+                opos = opos+dlen
 
             return ltid
 
     def _restore_index(self):
         """Load database index to support quick startup."""
-        try:
-            f = open("%s.index" % self.__name__, 'rb')
-        except:
-            return None
-        p = Unpickler(f)
+        file_name=self.__name__
+        index_name=file_name+'.index'
+
+        try: f=open(index_name,'rb')
+        except: return None
+
+        p=Unpickler(f)
 
         try:
             info=p.load()
@@ -422,6 +459,23 @@
             return None
         pos = long(pos)
 
+        if isinstance(index, DictType) and not self._is_read_only:
+            # Convert to fsIndex
+            newindex = fsIndex()
+            if type(newindex) is not type(index):
+                # And we have fsIndex
+                newindex.update(index)
+
+                # Now save the index
+                f = open(index_name, 'wb')
+                p = Pickler(f, 1)
+                info['index'] = newindex
+                p.dump(info)
+                f.close()
+
+                # Now call this method again to get the new data
+                return self._restore_index()
+
         tid = self._sane(index, pos)
         if not tid:
             return None
@@ -955,6 +1009,9 @@
         finally:
             self._lock_release()
 
+    # Keep track of the number of records that we've written
+    _records_written = 0
+
     def _finish(self, tid, u, d, e):
         nextpos=self._nextpos
         if nextpos:
@@ -967,10 +1024,20 @@
 
             if fsync is not None: fsync(file.fileno())
 
-            self._pos=nextpos
-
+            self._pos = nextpos
+            
             self._index.update(self._tindex)
             self._vindex.update(self._tvindex)
+            
+            # Update the number of records that we've written
+            # +1 for the transaction record
+            self._records_written += len(self._tindex) + 1 
+            if self._records_written >= self._records_before_save:
+                self._save_index()
+                self._records_written = 0
+                self._records_before_save = max(self._records_before_save,
+                                                len(self._index))
+                
         self._ltid = tid
 
     def _abort(self):
@@ -1210,7 +1277,8 @@
         while pos < tend:
             self._file.seek(pos)
             h = self._file.read(DATA_HDR_LEN)
-            oid, serial, sprev, stloc, vlen, splen = struct.unpack(DATA_HDR, h)
+            oid, serial, sprev, stloc, vlen, splen = \
+                 struct.unpack(DATA_HDR, h)
             if failed(oid):
                 del failures[oid] # second chance!
             plen = u64(splen)
@@ -1966,6 +2034,7 @@
     id in the data.  The transaction id is the tid of the last
     transaction.
     """
+
     read = file.read
     seek = file.seek
     seek(0, 2)
@@ -2001,7 +2070,7 @@
 
         if tid <= ltid:
             warn("%s time-stamp reduction at %s", name, pos)
-        ltid=tid
+        ltid = tid
 
         tl=u64(stl)
 
@@ -2074,7 +2143,12 @@
             if vlen:
                 dlen=dlen+(16+vlen)
                 read(16)
+                pv=u64(read(8))
                 version=read(vlen)
+                # Jim says: "It's just not worth the bother."
+                #if vndexpos(version, 0) != pv:
+                #    panic("%s incorrect previous version pointer at %s",
+                #          name, pos)
                 vindex[version]=pos
 
             if pos+dlen > tend or tloc != tpos:
@@ -2223,8 +2297,11 @@
         self._stop = stop
 
     def __len__(self):
-        # This is a lie.  It's here only for Python 2.1 support for
-        # list()-ifying these objects.
+        # Define a bogus __len__() to make the iterator work
+        # with code like builtin list() and tuple() in Python 2.1.
+        # There's a lot of C code that expects a sequence to have
+        # an __len__() but can cope with any sort of mistake in its
+        # implementation.  So just return 0.
         return 0
 
     def close(self):
@@ -2362,7 +2439,6 @@
 
 class RecordIterator(Iterator, BaseStorage.TransactionRecord):
     """Iterate over the transactions in a FileStorage file."""
-    
     def __init__(self, tid, status, user, desc, ext, pos, tend, file, tpos):
         self.tid = tid
         self.status = status


=== Zope/lib/python/ZODB/POSException.py 1.12.4.12 => 1.12.4.13 ===
--- Zope/lib/python/ZODB/POSException.py:1.12.4.12	Fri Jan  3 01:37:20 2003
+++ Zope/lib/python/ZODB/POSException.py	Sat Jan  4 13:16:20 2003
@@ -219,4 +219,6 @@
     o A reference to a wrapped persistent object.
 
     o A reference to an object in a different database connection.
+
+    XXX The exception ought to have a member that is the invalid object.
     """


=== Zope/lib/python/ZODB/TimeStamp.c 1.15.58.6 => 1.15.58.7 ===


=== Zope/lib/python/ZODB/cPersistence.c 1.62.8.6 => 1.62.8.7 ===


=== Zope/lib/python/ZODB/cPickleCache.c 1.68.8.3 => 1.68.8.4 ===