[Checkins] SVN: zc.demostorage2/branches/dev/src/zc/demostorage2/
Checked in code for an old version written for ZODB 3.2.
Jim Fulton
jim at zope.com
Mon Jan 21 14:56:33 EST 2008
Log message for revision 83075:
Checked in code for an old version written for ZODB 3.2.
Lots of changes are necessary. :)
Changed:
A zc.demostorage2/branches/dev/src/zc/demostorage2/
A zc.demostorage2/branches/dev/src/zc/demostorage2/README.txt
A zc.demostorage2/branches/dev/src/zc/demostorage2/__init__.py
A zc.demostorage2/branches/dev/src/zc/demostorage2/component.xml
A zc.demostorage2/branches/dev/src/zc/demostorage2/config.py
A zc.demostorage2/branches/dev/src/zc/demostorage2/storage.py
A zc.demostorage2/branches/dev/src/zc/demostorage2/synchronized.py
A zc.demostorage2/branches/dev/src/zc/demostorage2/synchronized.txt
A zc.demostorage2/branches/dev/src/zc/demostorage2/tests.py
-=-
Added: zc.demostorage2/branches/dev/src/zc/demostorage2/README.txt
===================================================================
--- zc.demostorage2/branches/dev/src/zc/demostorage2/README.txt (rev 0)
+++ zc.demostorage2/branches/dev/src/zc/demostorage2/README.txt 2008-01-21 19:56:31 UTC (rev 83075)
@@ -0,0 +1,141 @@
+Second-generation demo storage
+==============================
+
+The demostorage2 module provides a storage implementation that
+wraps two storages, a base storage and a storage to hold changes.
+The base storage is never written to. All new records are written to
+the changes storage. Both storages are expected to:
+
+- Use packed 64-bit unsigned integers as object ids,
+
+- Allocate object ids sequentially, starting from 0, and
+
+- in the case of the changes storage, accept object ids assigned externally.
+
+In addition, it is assumed that less than 2**63 object ids have been
+allocated in the first storage.
+
+Note that DemoStorage also assumes that it's base storage uses 64-bit
+unsigned integer object ids allocated sequentially.
+
+To see how this works, we'll start by creating a base storage and
+puting an object (in addition to the root object) in it:
+
+ >>> import os, tempfile
+ >>> tempdir = tempfile.mkdtemp()
+ >>> base_path = os.path.join(tempdir, 'base.fs')
+
+ >>> from ZODB.FileStorage import FileStorage
+ >>> base = FileStorage(base_path)
+ >>> from ZODB.DB import DB
+ >>> db = DB(base)
+ >>> from ZODB.PersistentMapping import PersistentMapping
+ >>> conn = db.open()
+ >>> conn.root()['1'] = PersistentMapping({'a': 1, 'b':2})
+ >>> get_transaction().commit()
+ >>> db.close()
+ >>> original_size = os.path.getsize(base_path)
+
+Now, lets reopen the base storage in read-only mode:
+
+ >>> base = FileStorage(base_path, read_only=True)
+
+And open a new storage to store changes:
+
+ >>> changes_path = os.path.join(tempdir, 'changes.fs')
+ >>> changes = FileStorage(changes_path)
+
+and combine the 2 in a demofilestorage:
+
+ >>> from demostorage2 import DemoStorage2
+ >>> storage = DemoStorage2(base, changes)
+
+If there are no transactions, the storage reports the lastTransaction
+of the base database:
+
+ >>> storage.lastTransaction() == base.lastTransaction()
+ True
+
+Let's add some data:
+
+ >>> db = DB(storage)
+ >>> conn = db.open()
+ >>> items = conn.root()['1'].items()
+ >>> items.sort()
+ >>> items
+ [('a', 1), ('b', 2)]
+
+ >>> conn.root()['2'] = PersistentMapping({'a': 3, 'b':4})
+ >>> get_transaction().commit()
+
+ >>> conn.root()['2']['c'] = 5
+ >>> get_transaction().commit()
+
+Here we can see that we haven't modified the base storage:
+
+ >>> original_size == os.path.getsize(base_path)
+ True
+
+But we have modified the changes database:
+
+ >>> len(changes)
+ 2
+
+Our lastTransaction reflects the lastTransaction of the changes:
+
+ >>> storage.lastTransaction() > base.lastTransaction()
+ True
+
+ >>> storage.lastTransaction() == changes.lastTransaction()
+ True
+
+
+Let's walk over some of the methods so ewe can see how we delegate to
+the new oderlying storages:
+
+ >>> from ZODB.utils import p64, u64
+ >>> storage.load(p64(0), '') == changes.load(p64(0), '')
+ True
+ >>> storage.load(p64(0), '') == base.load(p64(0), '')
+ False
+ >>> storage.load(p64(1), '') == base.load(p64(1), '')
+ True
+
+ >>> serial = base.getSerial(p64(0))
+ >>> storage.loadSerial(p64(0), serial) == base.loadSerial(p64(0), serial)
+ True
+
+ >>> serial = changes.getSerial(p64(0))
+ >>> storage.loadSerial(p64(0), serial) == changes.loadSerial(p64(0),
+ ... serial)
+ True
+
+The object id of the new object is quite large:
+
+ >>> u64(conn.root()['2']._p_oid)
+ 9223372036854775809L
+
+Versions aren't supported:
+
+ >>> storage.supportsVersions()
+ False
+ >>> storage.versions()
+ ()
+ >>> storage.versionEmpty(p64(0))
+ True
+ >>> storage.versionEmpty(p64(60))
+ True
+ >>> storage.modifiedInVersion(p64(0))
+ ''
+ >>> storage.modifiedInVersion(p64(60))
+ ''
+
+Many methods are simply copied from the base storage:
+
+ >>> [getattr(storage, name) == getattr(changes, name)
+ ... for name in ('getName', 'sortKey', 'getSize', '__len__',
+ ... 'supportsUndo', 'undo', 'undoLog', 'undoInfo',
+ ... 'supportsTransactionalUndo')
+ ... ]
+ [True, True, True, True, True, True, True, True, True]
+
Property changes on: zc.demostorage2/branches/dev/src/zc/demostorage2/README.txt
___________________________________________________________________
Name: svn:eol-style
+ native
Added: zc.demostorage2/branches/dev/src/zc/demostorage2/__init__.py
===================================================================
--- zc.demostorage2/branches/dev/src/zc/demostorage2/__init__.py (rev 0)
+++ zc.demostorage2/branches/dev/src/zc/demostorage2/__init__.py 2008-01-21 19:56:31 UTC (rev 83075)
@@ -0,0 +1 @@
+from storage import DemoStorage2
Property changes on: zc.demostorage2/branches/dev/src/zc/demostorage2/__init__.py
___________________________________________________________________
Name: svn:keywords
+ Id
Name: svn:eol-style
+ native
Added: zc.demostorage2/branches/dev/src/zc/demostorage2/component.xml
===================================================================
--- zc.demostorage2/branches/dev/src/zc/demostorage2/component.xml (rev 0)
+++ zc.demostorage2/branches/dev/src/zc/demostorage2/component.xml 2008-01-21 19:56:31 UTC (rev 83075)
@@ -0,0 +1,9 @@
+<component prefix="demostorage2.config">
+ <sectiontype name="demostorage2" datatype=".DemoStorage2"
+ implements="ZODB.storage">
+ <section type="ZODB.storage" name="base" attribute="base"/>
+ <section type="ZODB.storage" name="changes" attribute="changes"/>
+ </sectiontype>
+
+
+</component>
Property changes on: zc.demostorage2/branches/dev/src/zc/demostorage2/component.xml
___________________________________________________________________
Name: svn:eol-style
+ native
Added: zc.demostorage2/branches/dev/src/zc/demostorage2/config.py
===================================================================
--- zc.demostorage2/branches/dev/src/zc/demostorage2/config.py (rev 0)
+++ zc.demostorage2/branches/dev/src/zc/demostorage2/config.py 2008-01-21 19:56:31 UTC (rev 83075)
@@ -0,0 +1,10 @@
+
+import demostorage2
+from ZODB.config import BaseConfig
+
+class DemoStorage2(BaseConfig):
+
+ def open(self):
+ base = self.config.base.open()
+ changes = self.config.changes.open()
+ return demostorage2.DemoStorage2(base, changes)
Property changes on: zc.demostorage2/branches/dev/src/zc/demostorage2/config.py
___________________________________________________________________
Name: svn:keywords
+ Id
Name: svn:eol-style
+ native
Added: zc.demostorage2/branches/dev/src/zc/demostorage2/storage.py
===================================================================
--- zc.demostorage2/branches/dev/src/zc/demostorage2/storage.py (rev 0)
+++ zc.demostorage2/branches/dev/src/zc/demostorage2/storage.py 2008-01-21 19:56:31 UTC (rev 83075)
@@ -0,0 +1,159 @@
+##############################################################################
+#
+# Copyright (c) 2006 Zope Corporation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
+"""Demo storage that stores changes in a file storage
+
+$Id$
+"""
+
+import threading
+
+import ZODB.POSException
+from ZODB.utils import p64, u64, z64
+
+from demostorage2.synchronized import synchronized
+
+class DemoStorage2:
+
+ def __init__(self, base, changes):
+ self.changes = changes
+ self.base = base
+
+ self.getName = changes.getName
+ self.sortKey = changes.sortKey
+ self.getSize = changes.getSize
+ self.__len__ = changes.__len__
+ self.supportsUndo = changes.supportsUndo
+ self.supportsTransactionalUndo = changes.supportsTransactionalUndo
+ self.undo = changes.undo
+ self.undoLog = changes.undoLog
+ self.undoInfo = changes.undoInfo
+
+ self._oid = max(u64(changes.new_oid()), 1l << 63)
+ self._lock = threading.RLock()
+ self._commit_lock = threading.Lock()
+
+ self._transaction = None
+
+ def registerDB(self, db, limit):
+ self.base.registerDB(db, limit)
+ self.changes.registerDB(db, limit)
+
+ def close(self):
+ self.base.close()
+ self.changes.close()
+
+ def load(self, oid, version):
+ try:
+ return self.changes.load(oid, version)
+ except ZODB.POSException.POSKeyError:
+ return self.base.load(oid, version)
+ load = synchronized(load)
+
+ def getSerial(self, oid):
+ return self.load(oid, '')[1]
+
+ def loadSerial(self, oid, serial):
+ try:
+ return self.changes.loadSerial(oid, serial)
+ except ZODB.POSException.POSKeyError:
+ return self.base.loadSerial(oid, serial)
+ loadSerial = synchronized(loadSerial)
+
+ def new_oid(self):
+ self._oid += 1
+ return p64(self._oid)
+ new_oid = synchronized(new_oid)
+
+ def tpc_begin(self, transaction, tid=None, status=' '):
+ if self._transaction is transaction:
+ return
+ self._commit_lock.acquire()
+ self._begin(transaction, tid, status)
+
+ def _begin(self, transaction, tid, status):
+ self._transaction = transaction
+ self.changes.tpc_begin(transaction, tid, status)
+ _begin = synchronized(_begin)
+
+ def tpc_abort(self, transaction):
+ if self._transaction is not transaction:
+ return
+ self._transaction = None
+ try:
+ self.changes.tpc_abort(transaction)
+ finally:
+ self._commit_lock.release()
+ tpc_abort = synchronized(tpc_abort)
+
+ def store(self, oid, serial, data, version, transaction):
+ if transaction is not self._transaction:
+ raise ZODB.POSException.StorageTransactionError(self, transaction)
+
+ if version:
+ raise ValueError("Invalid version", version)
+
+ # See if we already have changes for this oid
+ try:
+ old = self.changes.getSerial(oid)
+ except ZODB.POSException.POSKeyError:
+ try:
+ old = self.base.getSerial(oid)
+ except ZODB.POSException.POSKeyError:
+ old = serial
+
+ if old != serial:
+ raise ZODB.POSException.ConflictError(
+ oid=oid, serials=(oserial, serial))
+
+ return self.changes.store(oid, serial, data, '', transaction)
+ store = synchronized(store)
+
+ def supportsVersions(self):
+ return False
+
+ def tpc_vote(self, transaction):
+ if self._transaction is not transaction:
+ return
+ return self.changes.tpc_vote(transaction)
+ tpc_vote = synchronized(tpc_vote)
+
+ def tpc_finish(self, transaction, func = lambda: None):
+ if self._transaction is not transaction:
+ return
+ self._transaction = None
+ self.changes.tpc_finish(transaction)
+ self._commit_lock.release()
+ tpc_finish = synchronized(tpc_finish)
+
+ def history(self, *args, **kw):
+ return self.changes.history(*args, **kw)
+
+ def lastTransaction(self):
+ t = self.changes.lastTransaction()
+ if t == z64:
+ t = self.base.lastTransaction()
+ return t
+ lastTransaction = synchronized(lastTransaction)
+
+ def isReadOnly(self):
+ return False
+
+ def versionEmpty(*args, **kw):
+ return True
+
+ def modifiedInVersion(*args, **kw):
+ return ''
+
+ def versions(*args, **kw):
+ return ()
Property changes on: zc.demostorage2/branches/dev/src/zc/demostorage2/storage.py
___________________________________________________________________
Name: svn:keywords
+ Id
Name: svn:eol-style
+ native
Added: zc.demostorage2/branches/dev/src/zc/demostorage2/synchronized.py
===================================================================
--- zc.demostorage2/branches/dev/src/zc/demostorage2/synchronized.py (rev 0)
+++ zc.demostorage2/branches/dev/src/zc/demostorage2/synchronized.py 2008-01-21 19:56:31 UTC (rev 83075)
@@ -0,0 +1,54 @@
+##############################################################################
+#
+# Copyright (c) 2006 Zope Corporation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
+"""Support for Java-style synchronized methods
+
+Support for synchornized method through a synchronized decorator.
+
+$Id$
+"""
+
+class SynchronizedFunction(object):
+
+ def __init__(self, func, lock_name, inst=None):
+ self._func = func
+ self._lock_name = lock_name
+ self._inst = inst
+
+ def __get__(self, inst, class_):
+ return self.__class__(self._func, self._lock_name, inst)
+
+ def __call__(self, *args, **kw):
+ inst = self._inst
+ if inst is None:
+ inst = args[0]
+ args = args[1:]
+
+ lock = getattr(inst, self._lock_name)
+ lock.acquire()
+ try:
+ return self._func(inst, *args, **kw)
+ finally:
+ lock.release()
+
+def synchronized(func):
+ if isinstance(func, str):
+ lock_name = func
+ def synchronized(func):
+ return SynchronizedFunction(func, lock_name)
+ return synchronized
+ else:
+ return SynchronizedFunction(func, '_lock')
+
+
+
Property changes on: zc.demostorage2/branches/dev/src/zc/demostorage2/synchronized.py
___________________________________________________________________
Name: svn:keywords
+ Id
Name: svn:eol-style
+ native
Added: zc.demostorage2/branches/dev/src/zc/demostorage2/synchronized.txt
===================================================================
--- zc.demostorage2/branches/dev/src/zc/demostorage2/synchronized.txt (rev 0)
+++ zc.demostorage2/branches/dev/src/zc/demostorage2/synchronized.txt 2008-01-21 19:56:31 UTC (rev 83075)
@@ -0,0 +1,86 @@
+Synchronized methods
+====================
+
+A thread-safe class uses locks to mediate access to
+the classes methods in a thread-safe way. For simple applications,
+this involves acquiring and releasing a reentrant lock on entry and
+access to methods that access mutable instance data:
+
+ >>> import threading
+ >>> import time
+ >>> class Counter:
+ ... def __init__(self):
+ ... self._lock = threading.RLock()
+ ... self.value = 0
+ ...
+ ... def inc(self):
+ ... self._lock.acquire()
+ ... try:
+ ... self.value = self.value + 1
+ ... finally:
+ ... self._lock.release()
+ ...
+ ... def dec(self):
+ ... self._lock.acquire()
+ ... try:
+ ... self.value = self.value - 1
+ ... finally:
+ ... self._lock.release()
+
+If there are a lot of methods to protect, the try/finally code can get
+fairly burdonsome. The synchonized module provides a simple
+decorator/descriptor that automates this:
+
+
+ >>> from demostorage2.synchronized import synchronized
+ >>> class Counter:
+ ... def __init__(self):
+ ... self._lock = threading.RLock()
+ ... self.value = 0
+ ...
+ ... def inc(self):
+ ... # looking for trouble
+ ... old = self.value
+ ... time.sleep(0.0001)
+ ... self.value = old + 1
+ ... inc = synchronized(inc)
+ ...
+ ... def dec(self):
+ ... # looking for trouble
+ ... old = self.value
+ ... time.sleep(0.0001)
+ ... self.value = old - 1
+ ... dec = synchronized('_lock')(dec)
+ ...
+ ... def getvalue(self):
+ ... return self.value
+ ... getvalue = synchronized(getvalue)
+
+The decorator can be passed a lock name. If no name is passed, then
+'_lock' is assumed.
+
+With something like this in place, we can safely update instances of
+our class from multiple threads:
+
+ >>> def updatealot(ob, n):
+ ... for i in range(n):
+ ... ob.inc()
+
+ >>> counter = Counter()
+ >>> threads = [threading.Thread(target=updatealot, args=(counter, 10))
+ ... for i in range(3)]
+ >>> _ = [t.start() for t in threads]
+ >>> _ = [t.join() for t in threads]
+ >>> counter.getvalue()
+ 30
+
+ >>> def updatealot(ob, n):
+ ... for i in range(n):
+ ... ob.dec()
+
+ >>> threads = [threading.Thread(target=updatealot, args=(counter, 10))
+ ... for i in range(3)]
+ >>> _ = [t.start() for t in threads]
+ >>> _ = [t.join() for t in threads]
+ >>> counter.getvalue()
+ 0
Property changes on: zc.demostorage2/branches/dev/src/zc/demostorage2/synchronized.txt
___________________________________________________________________
Name: svn:eol-style
+ native
Added: zc.demostorage2/branches/dev/src/zc/demostorage2/tests.py
===================================================================
--- zc.demostorage2/branches/dev/src/zc/demostorage2/tests.py (rev 0)
+++ zc.demostorage2/branches/dev/src/zc/demostorage2/tests.py 2008-01-21 19:56:31 UTC (rev 83075)
@@ -0,0 +1,69 @@
+##############################################################################
+#
+# Copyright (c) 2006 Zope Corporation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
+import unittest
+import shutil
+from ZODB.Transaction import get_transaction
+from zope.testing import doctest
+
+def cleanupReadme(test):
+ get_transaction().abort()
+ test.globs['db'].close()
+ shutil.rmtree(test.globs['tempdir'])
+
+def testSomeDlegation():
+ r"""
+ >>> class S:
+ ... def __init__(self, name):
+ ... self.name = name
+ ... def registerDB(self, db, limit):
+ ... print self.name, db, limit
+ ... def close(self):
+ ... print self.name, 'closed'
+ ... getName = sortKey = getSize = __len__ = None
+ ... supportsUndo = undo = undoLog = undoInfo = None
+ ... supportsTransactionalUndo = None
+ ... def new_oid(self):
+ ... return '\0' * 8
+ ... def tpc_begin(self, t, tid, status):
+ ... print 'begin', tid, status
+ ... def tpc_abort(self, t):
+ ... pass
+
+ >>> from demostorage2 import DemoStorage2
+ >>> storage = DemoStorage2(S(1), S(2))
+
+ >>> storage.registerDB(1, 2)
+ 1 1 2
+ 2 1 2
+
+ >>> storage.close()
+ 1 closed
+ 2 closed
+
+ >>> storage.tpc_begin(1, 2, 3)
+ begin 2 3
+ >>> storage.tpc_abort(1)
+
+ """
+
+def test_suite():
+ return unittest.TestSuite((
+ doctest.DocFileSuite('synchronized.txt'),
+ doctest.DocTestSuite(),
+ doctest.DocFileSuite('README.txt', tearDown=cleanupReadme),
+ ))
+
+if __name__ == '__main__':
+ unittest.main(defaultTest='test_suite')
+
Property changes on: zc.demostorage2/branches/dev/src/zc/demostorage2/tests.py
___________________________________________________________________
Name: svn:keywords
+ Id
Name: svn:eol-style
+ native
More information about the Checkins
mailing list