[Zope3-checkins] CVS: ZODB4/src/zodb - conflict.py:1.9 connection.py:1.17 db.py:1.13 export.py:1.5 interfaces.py:1.15 lockfile.py:1.6 serialize.py:1.15 utils.py:1.4
Barry Warsaw
barry@wooz.org
Thu, 13 Mar 2003 16:32:59 -0500
Update of /cvs-repository/ZODB4/src/zodb
In directory cvs.zope.org:/tmp/cvs-serv27419/src/zodb
Modified Files:
conflict.py connection.py db.py export.py interfaces.py
lockfile.py serialize.py utils.py
Log Message:
> I believe we're ready to merge back to the head.
merging the opaque-pickles-branch back into the head
=== ZODB4/src/zodb/conflict.py 1.8 => 1.9 ===
--- ZODB4/src/zodb/conflict.py:1.8 Fri Mar 7 18:09:51 2003
+++ ZODB4/src/zodb/conflict.py Thu Mar 13 16:32:27 2003
@@ -183,4 +183,4 @@
writer = ResolvedObjectWriter()
obj = ResolvedObjectAdapter(get_self(resolve), resolved)
- return writer.getState(obj)
+ return writer.getState(obj)[0]
=== ZODB4/src/zodb/connection.py 1.16 => 1.17 ===
--- ZODB4/src/zodb/connection.py:1.16 Thu Mar 13 13:48:55 2003
+++ ZODB4/src/zodb/connection.py Thu Mar 13 16:32:27 2003
@@ -37,21 +37,20 @@
$Id$
"""
-import cPickle
-from cStringIO import StringIO
import logging
+import struct
import tempfile
import threading
-import time
from types import StringType, ClassType, TupleType
from zodb import interfaces
from zodb.conflict import ResolvedSerial
from zodb.export import ExportImport
from zodb.interfaces import *
-from zodb.serialize import ConnectionObjectReader, ObjectWriter
+from zodb.serialize import ConnectionObjectReader, ObjectWriter, findrefs
from zodb.storage.interfaces import IStorage
-from zodb.utils import p64, u64, Set, z64
+from zodb.storage.base import splitrefs
+from zodb.utils import p64, u64, Set
from transaction import get_transaction
from transaction.interfaces import IDataManager, IRollback, TransactionError
@@ -130,7 +129,7 @@
# root(), sync(), get()
def root(self):
- return self.get(z64)
+ return self.get(ZERO)
def sync(self):
if self._txn:
@@ -155,7 +154,7 @@
obj._p_serial = serial
self._cache[oid] = obj
- if oid == z64:
+ if oid == ZERO:
# Keep a reference to the root so that the pickle cache
# won't evict it. XXX Not sure if this is necessary. If
# the cache is LRU, it should know best if the root is needed.
@@ -356,10 +355,9 @@
def savepoint(self, txn):
if self._tmp is None:
- tmp = TmpStore(self._version)
+ tmp = TmpStore(self._db, self._storage, self._version)
self._tmp = self._storage
self._storage = tmp
- tmp.registerDB(self._db)
self._modified = Set()
self._created = Set()
self._storage.tpcBegin(txn)
@@ -385,7 +383,7 @@
del self._cache[oid]
def _invalidate_modified(self):
- self._db.invalidate(self._modified, self)
+ self._db.invalidate(self._modified, self, self._version)
def _flush_invalidations(self):
self._inv_lock.acquire()
@@ -472,8 +470,9 @@
self._inv_lock.release()
self._modified.add(oid)
- s = self._storage.store(oid, serial, writer.getState(pobject),
- self._version, transaction)
+ data, refs = writer.getState(pobject)
+ s = self._storage.store(oid, serial, data, refs, self._version,
+ transaction)
# Put the object in the cache before handling the
# response, just in case the response contains the
# serial number for a newly created object
@@ -495,9 +494,11 @@
self._created |= tmp._created
for oid in tmp._index:
- data, serial = tmp.load(oid, tmp._bver)
- s = self._storage.store(oid, serial, data, self._version, txn)
+ data, refs, serial = tmp.loadrefs(oid, tmp._bver)
+ s = self._storage.store(oid, serial, data, refs,
+ self._version, txn)
self._handle_serial(s, oid, change=False)
+ tmp.close()
def _abort_sub(self):
# Abort work done in subtransactions.
@@ -509,6 +510,7 @@
self._cache.invalidateMany(tmp._index)
self._invalidate_created(tmp._created)
+ tmp.close()
class Rollback:
"""Rollback changes associated with savepoint"""
@@ -533,15 +535,41 @@
self._tmp_undo.rollback()
self._conn._cache.invalidateMany(self._conn._modified)
+class UndoInfo:
+ """A helper class for rollback.
+
+ The class stores the state necessary for rolling back to a
+ particular time.
+ """
+
+ def __init__(self, store, pos, index):
+ self._store = store
+ self._pos = pos
+ self._index = index
+
+ def current(self, cur_store):
+ """Return true if the UndoInfo is for cur_store."""
+ return self._store is cur_store
+
+ def rollback(self):
+ self._store.rollback(self._pos, self._index)
+
+
class TmpStore:
"""A storage to support savepoints."""
_bver = ''
- # XXX This storage doesn't implementation the full API yet
- __implements__ = IStorage
+ # The header format is oid, serial, nrefs, len(data). Following
+ # the header are the refs and the data, where the size of refs is
+ # nrefs * 8.
+
+ _fmt = ">8s8sQI"
+ _header_size = 28
- def __init__(self, base_version):
+ def __init__(self, db, storage, base_version):
+ self._db = db
+ self._storage = storage
self._transaction = None
if base_version:
self._bver = base_version
@@ -564,39 +592,36 @@
pos = self._index.get(oid, None)
if pos is None:
return self._storage.load(oid, self._bver)
+ data, refs, serial = self.loadrefs(oid, version)
+ return data, serial
+
+ def loadrefs(self, oid, version):
+ # A version of load the returns data, refs, and serial.
+ pos = self._index.get(oid)
+ # We only call loadrefs() for objects in the TmpStore.
+ assert pos is not None
self._file.seek(pos)
- h = self._file.read(24)
- if h[:8] != oid:
- raise interfaces.StorageSystemError('Bad temporary storage')
- size = u64(h[16:])
- serial = h[8:16]
- return self._file.read(size), serial
-
- # XXX clarify difference between self._storage & self._db._storage
-
- def modifiedInVersion(self, oid):
- if self._index.has_key(oid):
- return self._bver
- return self._storage.modifiedInVersion(oid)
+ buf = self._file.read(self._header_size)
+ oid, serial, nrefs, size = struct.unpack(self._fmt, buf)
+ refs = self._file.read(nrefs * 8)
+ data = self._file.read(size)
+ return data, splitrefs(refs), serial
def newObjectId(self):
return self._storage.newObjectId()
- def registerDB(self, db):
- self._db = db
- self._storage = db._storage
-
- def store(self, oid, serial, data, version, transaction):
+ def store(self, oid, serial, data, refs, version, transaction):
if transaction is not self._transaction:
raise interfaces.StorageTransactionError(self, transaction)
self._file.seek(self._pos)
- l = len(data)
if serial is None:
- serial = z64
- self._file.write(oid + serial + p64(l))
+ serial = ZERO
+ buf = struct.pack(self._fmt, oid, serial, len(refs), len(data))
+ self._file.write(buf)
+ self._file.write("".join(refs))
self._file.write(data)
self._tindex[oid] = self._pos
- self._pos += l + 24
+ self._pos += len(refs) * 8 + len(data) + self._header_size
return serial
def tpcAbort(self, transaction):
@@ -642,22 +667,3 @@
self._tpos = self._pos = pos
self._index = index
self._tindex.clear()
-
-class UndoInfo:
- """A helper class for rollback.
-
- The class stores the state necessary for rolling back to a
- particular time.
- """
-
- def __init__(self, store, pos, index):
- self._store = store
- self._pos = pos
- self._index = index
-
- def current(self, cur_store):
- """Return true if the UndoInfo is for cur_store."""
- return self._store is cur_store
-
- def rollback(self):
- self._store.rollback(self._pos, self._index)
=== ZODB4/src/zodb/db.py 1.12 => 1.13 ===
--- ZODB4/src/zodb/db.py:1.12 Thu Mar 13 13:48:55 2003
+++ ZODB4/src/zodb/db.py Thu Mar 13 16:32:27 2003
@@ -28,7 +28,8 @@
from zodb.connection import Connection
from zodb.serialize import getDBRoot
from zodb.ztransaction import Transaction
-from zodb.utils import z64, Set
+from zodb.interfaces import ZERO
+from zodb.utils import Set
from transaction import get_transaction
from transaction.interfaces import IDataManager
@@ -80,12 +81,15 @@
self._checkVersion()
storage.registerDB(self)
try:
- storage.load(z64, "")
+ storage.load(ZERO, "")
except KeyError:
# Create the database's root in the storage if it doesn't exist
t = Transaction(description="initial database creation")
storage.tpcBegin(t)
- storage.store(z64, None, getDBRoot(), '', t)
+ # Because this is the initial root object, we know it can't have
+ # any references, so include a longer comment then it would take
+ # to unpack getDBRoot()'s return value.
+ storage.store(ZERO, None, getDBRoot()[0], [], '', t)
storage.tpcVote(t)
storage.tpcFinish(t)
=== ZODB4/src/zodb/export.py 1.4 => 1.5 ===
--- ZODB4/src/zodb/export.py:1.4 Wed Feb 5 18:28:34 2003
+++ ZODB4/src/zodb/export.py Thu Mar 13 16:32:27 2003
@@ -103,7 +103,7 @@
"""
copier = ObjectCopier(self, self._storage, self._created)
- while 1:
+ while True:
h = file.read(16)
if h == export_end_marker:
break
@@ -116,7 +116,7 @@
# XXX I think it would be better if copier.copy()
# returned an oid and a new pickle so that this logic
- # wasn't smeared across to modules.
+ # wasn't smeared across two modules.
oid = h[:8]
new_ref = copier.oids.get(oid)
if new_ref is None:
@@ -127,5 +127,6 @@
else:
newObjectId = new_ref[0]
- new = copier.copy(p)
- self._storage.store(newObjectId, None, new, self._version, txn)
+ data, refs = copier.copy(p)
+ self._storage.store(newObjectId, None, data, refs,
+ self._version, txn)
=== ZODB4/src/zodb/interfaces.py 1.14 => 1.15 ===
--- ZODB4/src/zodb/interfaces.py:1.14 Thu Mar 13 13:48:55 2003
+++ ZODB4/src/zodb/interfaces.py Thu Mar 13 16:32:27 2003
@@ -35,25 +35,32 @@
from transaction.interfaces \
import TransactionError, RollbackError, ConflictError as _ConflictError
-__all__ = ['POSError',
- 'POSKeyError',
- 'ConflictError',
- 'ReadConflictError',
- 'DanglingReferenceError',
- 'InvalidObjectReference',
- 'VersionError',
- 'VersionCommitError',
- 'VersionLockError',
- 'UndoError',
- 'MultipleUndoErrors',
- 'ExportError',
- 'Unsupported',
+__all__ = [
+ # Constants
+ 'ZERO', 'MAXTID',
+ # Exceptions
+ 'POSError',
+ 'POSKeyError',
+ 'ConflictError',
+ 'ReadConflictError',
+ 'DanglingReferenceError',
+ 'VersionError',
+ 'VersionCommitError',
+ 'VersionLockError',
+ 'UndoError',
+ 'MultipleUndoErrors',
+ 'ExportError',
+ 'Unsupported',
+ 'InvalidObjectReference',
+ # Interfaces
+ 'IAppConnection',
+ 'IConnection',
+ 'ITransaction',
+ 'ITransactionAttrs',
+ ]
- 'IAppConnection',
- 'IConnection',
- 'ITransaction',
- 'ITransactionAttrs',
- ]
+ZERO = '\0'*8
+MAXTID = '\377'*8
def _fmt_oid(oid):
return "%016x" % zodb.utils.u64(oid)
=== ZODB4/src/zodb/lockfile.py 1.5 => 1.6 ===
--- ZODB4/src/zodb/lockfile.py:1.5 Thu Mar 13 13:48:55 2003
+++ ZODB4/src/zodb/lockfile.py Thu Mar 13 16:32:27 2003
@@ -45,7 +45,7 @@
pass
-
+
# This is a better interface to use than the lockfile.lock_file() interface.
# Creating the instance acquires the lock. The file remains open. Calling
# close both closes and unlocks the lock file.
=== ZODB4/src/zodb/serialize.py 1.14 => 1.15 ===
--- ZODB4/src/zodb/serialize.py:1.14 Mon Mar 10 18:45:57 2003
+++ ZODB4/src/zodb/serialize.py Thu Mar 13 16:32:27 2003
@@ -65,10 +65,12 @@
__metaclass__ = type
-from cStringIO import StringIO
-import cPickle
-from types import StringType, TupleType
import logging
+import cPickle
+from cStringIO import StringIO
+
+from zodb.interfaces import ZERO
+
def getClassMetadata(obj):
if obj._p_state == 3:
@@ -81,7 +83,7 @@
class RootJar:
def newObjectId(self):
- return "\0" * 8
+ return ZERO
def getDBRoot():
"""Return a serialized database root object."""
@@ -123,7 +125,7 @@
# if isinstance(oid, types.MemberDescriptor):
# -- but I can't because the type doesn't have a canonical name.
# Instead, we'll assert that an oid must always be a string
- if not (oid is None or isinstance(oid, StringType)):
+ if not (oid is None or isinstance(oid, str)):
# XXX log a warning
return None
@@ -143,7 +145,9 @@
return NewObjectIterator(self._stack)
def getState(self, obj):
- return self._dump(getClassMetadata(obj), obj.__getstate__())
+ data = self._dump(getClassMetadata(obj), obj.__getstate__())
+ refs = findrefs(data)
+ return data, refs
def _dump(self, classmeta, state):
# To reuse the existing cStringIO object, we must reset
@@ -236,7 +240,7 @@
def _persistent_load(self, oid):
# persistent_load function to pass to ObjectReader
- if isinstance(oid, TupleType):
+ if isinstance(oid, tuple):
# XXX We get here via new_persistent_id()
# Quick instance reference. We know all we need to know
@@ -273,7 +277,7 @@
self._cache = oids
def _persistent_load(self, oid):
- if isinstance(oid, TupleType):
+ if isinstance(oid, tuple):
oid, classmeta = oid
else:
classmeta = None
@@ -307,7 +311,8 @@
def copy(self, pickle):
classmeta, state = self._reader.readPickle(pickle)
- return self._writer._dump(classmeta, state)
+ data = self._writer._dump(classmeta, state)
+ return data, findrefs(data)
def findrefs(p):
f = StringIO(p)
@@ -321,7 +326,7 @@
# Iterator over L and convert persistent references to simple oids.
oids = []
for ref in L:
- if isinstance(ref, TupleType):
+ if isinstance(ref, tuple):
oids.append(ref[0])
else:
oids.append(ref)
=== ZODB4/src/zodb/utils.py 1.3 => 1.4 ===
--- ZODB4/src/zodb/utils.py:1.3 Wed Mar 5 17:14:30 2003
+++ ZODB4/src/zodb/utils.py Thu Mar 13 16:32:27 2003
@@ -15,8 +15,6 @@
import struct
import time
-z64 = "\0" * 8
-
def p64(v):
"""Pack an integer or long into a 8-byte string"""
return struct.pack(">Q", v)