From shane at cvs.zope.org Mon Jun 10 18:04:52 2002
From: shane at cvs.zope.org (Shane Hathaway)
Date: Sun Aug 10 16:31:21 2008
Subject: [ZEO-Checkins] CVS: StandaloneZODB/ZEO/tests - testZEO.py:1.16.4.4.2.15
Message-ID: <200206102204.g5AM4qu15544@cvs.baymountain.com>
Update of /cvs-repository/StandaloneZODB/ZEO/tests
In directory cvs.zope.org:/tmp/cvs-serv13120/tests
Modified Files:
Tag: ZEO2-branch
testZEO.py
Log Message:
Added a test that ensures ZEO invalidation messages get sent and received.
This currently fails; in a moment I will check in the fix.
Also cleaned up a test and the test framework invocation.
=== StandaloneZODB/ZEO/tests/testZEO.py 1.16.4.4.2.14 => 1.16.4.4.2.15 ===
self._dostore(data=obj)
+ def checkZEOInvalidation(self):
+ addr = self._storage._rpc_mgr.addr[0][1]
+ storage2 = ZEO.ClientStorage.ClientStorage(addr, wait=1,
+ min_disconnect_poll=0.1)
+ try:
+ oid = self._storage.new_oid()
+ ob = MinPO('first')
+ revid1 = self._dostore(oid, data=ob)
+ data, serial = storage2.load(oid, '')
+ self.assertEqual(zodb_unpickle(data), MinPO('first'))
+ self.assertEqual(serial, revid1)
+ revid2 = self._dostore(oid, data=MinPO('second'), revid=revid1)
+ for n in range(3):
+ # Let the server and client talk for a moment.
+ # Is there a better way to do this?
+ asyncore.poll(0.1)
+ data, serial = storage2.load(oid, '')
+ self.assertEqual(zodb_unpickle(data), MinPO('second'),
+ 'Invalidation message was not sent!')
+ self.assertEqual(serial, revid2)
+ finally:
+ storage2.close()
+
+
class ZEOFileStorageTests(GenericTests):
__super_setUp = GenericTests.setUp
@@ -297,8 +321,8 @@
self.shutdownServer()
self._storage = self.openClientStorage('test', 100000, wait=0)
data, revid2 = self._storage.load(oid, '')
- assert zodb_unpickle(data) == MinPO(12)
- assert revid1 == revid2
+ self.assertEqual(zodb_unpickle(data), MinPO(12))
+ self.assertEqual(revid1, revid2)
self._storage.close()
def checkRollover(self):
@@ -413,23 +437,5 @@
suite.addTest(sub)
return suite
-def main():
- import sys, getopt
-
- name_of_test = ''
-
- opts, args = getopt.getopt(sys.argv[1:], 'n:')
- for flag, val in opts:
- if flag == '-n':
- name_of_test = val
-
- if args:
- print "Did not expect arguments. Got %s" % args
- return 0
-
- tests = makeTestSuite(name_of_test)
- runner = unittest.TextTestRunner()
- runner.run(tests)
-
if __name__ == "__main__":
- main()
+ unittest.main(defaultTest='test_suite')
From shane at cvs.zope.org Mon Jun 10 18:07:41 2002
From: shane at cvs.zope.org (Shane Hathaway)
Date: Sun Aug 10 16:31:21 2008
Subject: [ZEO-Checkins] CVS: StandaloneZODB/ZEO - StorageServer.py:1.32.6.3.2.7
Message-ID: <200206102207.g5AM7f122286@cvs.baymountain.com>
Update of /cvs-repository/StandaloneZODB/ZEO
In directory cvs.zope.org:/tmp/cvs-serv16606
Modified Files:
Tag: ZEO2-branch
StorageServer.py
Log Message:
The __invalidated attribute is no longer meaningful, but it was still
being examined at transaction finish. This resulted in no invalidations
being sent at all. Fixed.
=== StandaloneZODB/ZEO/StorageServer.py 1.32.6.3.2.6 => 1.32.6.3.2.7 ===
self.__storage = None
self.__storage_id = "uninitialized"
- self.__invalidated = []
self._transaction = None
def notifyConnected(self, conn):
@@ -304,7 +303,7 @@
invalidated = self.strategy.tpc_finish()
if invalidated:
self.server.invalidate(self, self.__storage_id,
- self.__invalidated, self.get_size_info())
+ invalidated, self.get_size_info())
if not self._handle_waiting():
self._transaction = None
self.strategy = None
From jeremy at zope.com Tue Jun 11 09:43:07 2002
From: jeremy at zope.com (Jeremy Hylton)
Date: Sun Aug 10 16:31:21 2008
Subject: [ZEO-Checkins] CVS: ZEO/docs - ClientStorage.txt:1.2 NonZopeREADME.txt:1.4 ZopeREADME.txt:1.5
Message-ID: <200206111343.g5BDh7k05580@cvs.baymountain.com>
Update of /cvs-repository/ZEO/docs
In directory cvs.zope.org:/tmp/cvs-serv5548/docs
Modified Files:
ClientStorage.txt NonZopeREADME.txt ZopeREADME.txt
Log Message:
Merge ZEO2-branch to trunk.
=== ZEO/docs/ClientStorage.txt 1.1 => 1.2 ===
Creating a ClientStorage
- At a minimum, a client storage requires an argument (named
- connection) giving connection information. This argument should be
- a string, specifying a unix-domain socket file name, or a tuple
- consisting of a host and port. The host should be a string host
- name or IP number. The port should be a numeric port number.
+ The ClientStorage requires at leats one argument, the address or
+ addresses of the server(s) to use. It accepts several other
+ optional keyword arguments.
- The ClientStorage constructor provides a number of additional
- options (arguments). The full list of arguments is:
+ The address argument can be one of:
+
+ - a tuple containing hostname and port number
- connection -- Connection information.
+ - a string specifying the path to a Unix domain socket
- This argument is either a string containing a socket file name
- or a tuple consisting of a string host name or ip number and an
- integer port.
+ - a sequence of the previous two
+
+ If a sequence of addresses is specified, the client will use the
+ first server from the list that it can connect to.
+
+ The ClientStorage constructor provides a number of additional
+ options (arguments). The full list of arguments is:
storage -- The name of the storage to connect to.
@@ -33,7 +36,9 @@
default name for both the server and client is '1'.
cache_size -- The number of bytes to allow for the client cache.
- The default is 20,000,000.
+ The default is 20,000,000. A large cache can significantly
+ increase the performance of a ZEO system. For applications that
+ have a large database, the default size may be too small.
For more information on client caches, see ClientCache.txt.
@@ -54,10 +59,6 @@
For more information on client cache files, see ClientCache.txt.
- debug -- If this is provided, it should be a non-empty string. It
- indicates that client should log tracing and debugging
- information, using zLOG.
-
var -- The directory in which persistent cache files should be
written. If this option is provided, it is unnecessary to
set INSTANCE_HOME in __builtins__.
@@ -82,6 +83,13 @@
The default is 300 seconds.
- wait_for_server_on_starup -- Indicate whether the ClientStorage
- should block waiting for a storage server connection, or whether
- it should proceed, satisfying reads from the client cache.
+ wait -- Indicate whether the ClientStorage should block waiting
+ for a storage server connection, or whether it should proceed,
+ satisfying reads from the client cache.
+
+ read_only -- Open a read-only connection to the server. If the
+ client attempts to commit a transaction, it will get a
+ ReadOnlyError exception.
+
+ Each storage served by a ZEO server can be configured as either
+ read-write or read-only.
=== ZEO/docs/NonZopeREADME.txt 1.3 => 1.4 ===
- ZEO 1.0 requires Python 2.0 when used without Zope. It depends on
- versions of asyncore and cPickle that were first released with
- Python 2.0.
-
- Put the ZEO package in a directory on your Python path. On a Unix
- system, you can use the site-packages directory of your Python lib
- directory. The ZEO package is the directory named ZEO that contains
- an __init__.py file.
+ Installation
- Starting (and configuring) the ZEO Server
+ ZEO 2.0 requires Python 2.1 or higher when used without Zope. If
+ you use Python 2.1, we recommend the latest minor release (2.1.3 as
+ of this writing) because it includes a few bug fixes that affect
+ ZEO.
+
+ ZEO is packaged with distutils. To install it, run this command
+ from the top-level ZEO directory::
+
+ python setup.py install
+
+ The setup script will install the ZEO package in your Python
+ site-packages directory.
+
+ You can test ZEO before installing it with the test script::
+
+ python test.py -v
+
+ Run the script with the -h option for a full list of options. The
+ ZEO 2.0a1 release contains 87 unit tests on Unix.
- To start the storage server, run the start.py script contained in
- the ZEO package. You can run the script from the package
- directory or copy it to a directory on your path.
+ Starting (and configuring) the ZEO Server
- Specify the port number when you run the script::
+ To start the storage server, go to your Zope install directory and
+ run::
- python ZEO/start.py -p port_number
+ python lib/python/ZEO/start.py -p port_number
- Or run start.py without arguments to see options. The options are
- documented in start.txt.
+ This run the storage sever under zdaemon. zdaemon automatically
+ restarts programs that exit unexpectedly.
The server and the client don't have to be on the same machine.
- If the server and client *are* on the same machine, then you can
- use a Unix domain socket::
+ If they are on the same machine, then you can use a Unix domain
+ socket::
+
+ python lib/python/ZEO/start.py -U filename
- python ZEO/start.py -U filename
+ The start script provides a number of options not documented here.
+ See doc/start.txt for more information.
Running a ZEO client
In your application, create a ClientStorage, rather than, say, a
FileStorage:
- import ZODB, ZEO.ClientStorage
- Storage=ZEO.ClientStorage.ClientStorage(('',port_number))
- db=ZODB.DB(Storage)
+ import ZODB
+ from ZEO.ClientStorage import ClientStorage
+ Storage = ClientStorage(('', port_number))
+ db = ZODB.DB(Storage)
You can specify a host name (rather than '') if you want. The port
number is, of course, the port number used to start the storage
@@ -43,38 +57,24 @@
You can also give the name of a Unix domain socket file::
- import ZODB, ZEO.ClientStorage
- Storage=ZEO.ClientStorage.ClientStorage(filename)
- db=ZODB.DB(Storage)
+ import ZODB
+ from ZEO.ClientStorage import ClientStorage
+ Storage = ClientStorage(filename)
+ db = ZODB.DB(Storage)
There are a number of configuration options available for the
ClientStorage. See ClientStorage.txt for details.
If you want a persistent client cache which retains cache contents
across ClientStorage restarts, you need to define the environment
- variable, ZEO_CLIENT, to a unique name for the client. This is
- needed so that unique cache name files can be computed. Otherwise,
- the client cache is stored in temporary files which are removed when
+ variable, ZEO_CLIENT, or set the client keyword argument to the
+ constructor to a unique name for the client. This is needed so
+ that unique cache name files can be computed. Otherwise, the
+ client cache is stored in temporary files which are removed when
the ClientStorage shuts down.
Dependencies on other modules
- - The module ThreadedAsync must be on the Python path.
-
- - The zdaemon module is necessary if you want to run your
- storage server as a daemon that automatically restarts itself
- if there is a fatal error.
-
- - The zLOG module provides a handy logging capability.
-
- If you are using a version of Python before Python 2:
-
- - ZServer should be in the Python path, or you should copy the
- version of asyncore.py from ZServer (from Zope 2.2 or CVS) to
- your Python path, or you should copy a version of a asyncore
- from the medusa CVS tree to your Python path. A recent change
- in asyncore is required.
-
- - The version of cPickle from Zope, or from the python.org CVS
- tree must be used. It has a hook to provide control over which
- "global objects" (e.g. classes) may be pickled.
+ ZEO depends on other modules that are distributed with
+ StandaloneZODB and with Zope. You can download StandaloneZODB
+ from http://www.zope.org/Products/StandaloneZODB.
=== ZEO/docs/ZopeREADME.txt 1.4 => 1.5 ===
Installation
- ZEO 1.0 requires Zope 2.2 or higher.
+ ZEO 2.0 requires Zope 2.4 or higher and Python 2.1 or higher.
+ If you use Python 2.1, we recommend the latest minor release
+ (2.1.3 as of this writing) because it includes a few bug fixes
+ that affect ZEO.
- Put this package (the ZEO directory, without any wrapping directory
+ Put the package (the ZEO directory, without any wrapping directory
included in a distribution) in your Zope lib/python.
- If you are using Python 1.5.2, the lib/python/ZODB directory must
- contain a cPickle.so (Unix) or cPickle.pyd (Windows) file. In
- many cases, the Zope installation process will not place this file
- in the right location. You may need to copy it from lib/python to
- lib/python/ZODB.
+ The setup.py script in the top-level ZEO directory can also be
+ used. Run "python setup.py install --home=ZOPE" where ZOPE is the
+ top-level Zope directory.
+
+ You can test ZEO before installing it with the test script::
+
+ python test.py -v
+
+ Run the script with the -h option for a full list of options. The
+ ZEO 2.0a1 release contains 87 unit tests on Unix.
Starting (and configuring) the ZEO Server
- To start the storage server, go to your Zope install directory and::
+ To start the storage server, go to your Zope install directory and
+ run::
python lib/python/ZEO/start.py -p port_number
- (Run start without arguments to see options.)
+ This run the storage sever under zdaemon. zdaemon automatically
+ restarts programs that exit unexpectedly.
- Of course, the server and the client don't have to be on the same
- machine.
-
- If the server and client *are* on the same machine, then you can use
- a Unix domain socket::
+ The server and the client don't have to be on the same machine.
+ If they are on the same machine, then you can use a Unix domain
+ socket::
python lib/python/ZEO/start.py -U filename
@@ -38,10 +46,8 @@
custom_zodb.py, in your Zope install directory, so that Zope uses a
ClientStorage::
- import ZEO.ClientStorage
- Storage=ZEO.ClientStorage.ClientStorage(('',port_number))
-
- (See the misc/custom_zodb.py for an example.)
+ from ZEO.ClientStorage import ClientStorage
+ Storage = ClientStorage(('', port_number))
You can specify a host name (rather than '') if you want. The port
number is, of course, the port number used to start the storage
@@ -49,19 +55,20 @@
You can also give the name of a Unix domain socket file::
- import ZEO.ClientStorage
- Storage=ZEO.ClientStorage.ClientStorage(filename)
+ from ZEO.ClientStorage import ClientStorage
+ Storage = ClientStorage(filename)
There are a number of configuration options available for the
ClientStorage. See doc/ClientStorage.txt for details.
If you want a persistent client cache which retains cache contents
across ClientStorage restarts, you need to define the environment
- variable, ZEO_CLIENT, to a unique name for the client. This is
- needed so that unique cache name files can be computed. Otherwise,
- the client cache is stored in temporary files which are removed when
+ variable, ZEO_CLIENT, or set the client keyword argument to the
+ constructor to a unique name for the client. This is needed so
+ that unique cache name files can be computed. Otherwise, the
+ client cache is stored in temporary files which are removed when
the ClientStorage shuts down. For example, to start two Zope
- processes with unique caches, use something like:
+ processes with unique caches, use something like::
python z2.py -P8700 ZEO_CLIENT=8700
python z2.py -P8800 ZEO_CLIENT=8800
@@ -74,9 +81,8 @@
different clients have different software installed, the correct
state of the database is ambiguous.
- Starting in Zope 2.2, Zope will not modify the Zope database
- during product installation if the environment variable ZEO_CLIENT
- is set.
+ Zope will not modify the Zope database during product installation
+ if the environment variable ZEO_CLIENT is set.
Normally, Zope ZEO clients should be run with ZEO_CLIENT set so
that product initialization is not performed.
From jeremy at zope.com Tue Jun 11 09:43:08 2002
From: jeremy at zope.com (Jeremy Hylton)
Date: Sun Aug 10 16:31:21 2008
Subject: [ZEO-Checkins] CVS: ZEO/ZEO/tests - CommitLockTests.py:1.2 ThreadTests.py:1.2 testTransactionBuffer.py:1.4 Cache.py:1.8 forker.py:1.16 multi.py:1.8 speed.py:1.7 stress.py:1.6 testZEO.py:1.25
Message-ID: <200206111343.g5BDh8u05597@cvs.baymountain.com>
Update of /cvs-repository/ZEO/ZEO/tests
In directory cvs.zope.org:/tmp/cvs-serv5548/ZEO/tests
Modified Files:
Cache.py forker.py multi.py speed.py stress.py testZEO.py
Added Files:
CommitLockTests.py ThreadTests.py testTransactionBuffer.py
Log Message:
Merge ZEO2-branch to trunk.
=== ZEO/ZEO/tests/CommitLockTests.py 1.1 => 1.2 ===
+#
+# Copyright (c) 2002 Zope Corporation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE
+#
+##############################################################################
+"""Tests of the distributed commit lock."""
+
+import threading
+
+from ZODB.Transaction import Transaction
+from ZODB.tests.StorageTestBase import zodb_pickle, MinPO
+
+import ZEO.ClientStorage
+from ZEO.Exceptions import Disconnected
+
+ZERO = '\0'*8
+
+class DummyDB:
+ def invalidate(self, *args):
+ pass
+
+class WorkerThread(threading.Thread):
+
+ # run the entire test in a thread so that the blocking call for
+ # tpc_vote() doesn't hang the test suite.
+
+ def __init__(self, storage, trans, method="tpc_finish"):
+ self.storage = storage
+ self.trans = trans
+ self.method = method
+ threading.Thread.__init__(self)
+
+ def run(self):
+ try:
+ self.storage.tpc_begin(self.trans)
+ oid = self.storage.new_oid()
+ self.storage.store(oid, ZERO, zodb_pickle(MinPO("c")), '', self.trans)
+ oid = self.storage.new_oid()
+ self.storage.store(oid, ZERO, zodb_pickle(MinPO("c")), '', self.trans)
+ self.storage.tpc_vote(self.trans)
+ if self.method == "tpc_finish":
+ self.storage.tpc_finish(self.trans)
+ else:
+ self.storage.tpc_abort(self.trans)
+ except Disconnected:
+ pass
+
+class CommitLockTests:
+
+ # The commit lock tests verify that the storage successfully
+ # blocks and restarts transactions when there is content for a
+ # single storage. There are a lot of cases to cover.
+
+ # CommitLock1 checks the case where a single transaction delays
+ # other transactions before they actually block. IOW, by the time
+ # the other transactions get to the vote stage, the first
+ # transaction has finished.
+
+ def checkCommitLock1OnCommit(self):
+ self._storages = []
+ try:
+ self._checkCommitLock("tpc_finish", self._dosetup1, self._dowork1)
+ finally:
+ self._cleanup()
+
+ def checkCommitLock1OnAbort(self):
+ self._storages = []
+ try:
+ self._checkCommitLock("tpc_abort", self._dosetup1, self._dowork1)
+ finally:
+ self._cleanup()
+
+ def checkCommitLock2OnCommit(self):
+ self._storages = []
+ try:
+ self._checkCommitLock("tpc_finish", self._dosetup2, self._dowork2)
+ finally:
+ self._cleanup()
+
+ def checkCommitLock2OnAbort(self):
+ self._storages = []
+ try:
+ self._checkCommitLock("tpc_abort", self._dosetup2, self._dowork2)
+ finally:
+ self._cleanup()
+
+ def _cleanup(self):
+ for store, trans in self._storages:
+ store.tpc_abort(trans)
+ store.close()
+ self._storages = []
+
+ def _checkCommitLock(self, method_name, dosetup, dowork):
+ # check the commit lock when a client attemps a transaction,
+ # but fails/exits before finishing the commit.
+
+ # Start on transaction normally.
+ t = Transaction()
+ self._storage.tpc_begin(t)
+
+ # Start a second transaction on a different connection without
+ # blocking the test thread.
+ self._storages = []
+ for i in range(4):
+ storage2 = self._duplicate_client()
+ t2 = Transaction()
+ tid = `ZEO.ClientStorage.get_timestamp()` # XXX why?
+ dosetup(storage2, t2, tid)
+ if i == 0:
+ storage2.close()
+ else:
+ self._storages.append((storage2, t2))
+
+ oid = self._storage.new_oid()
+ self._storage.store(oid, ZERO, zodb_pickle(MinPO(1)), '', t)
+ self._storage.tpc_vote(t)
+ if method_name == "tpc_finish":
+ self._storage.tpc_finish(t)
+ self._storage.load(oid, '')
+ else:
+ self._storage.tpc_abort(t)
+
+ dowork(method_name)
+
+ # Make sure the server is still responsive
+ self._dostore()
+
+ def _dosetup1(self, storage, trans, tid):
+ storage.tpc_begin(trans, tid)
+
+ def _dowork1(self, method_name):
+ for store, trans in self._storages:
+ oid = store.new_oid()
+ store.store(oid, ZERO, zodb_pickle(MinPO("c")), '', trans)
+ store.tpc_vote(trans)
+ if method_name == "tpc_finish":
+ store.tpc_finish(trans)
+ else:
+ store.tpc_abort(trans)
+
+ def _dosetup2(self, storage, trans, tid):
+ self._threads = []
+ t = WorkerThread(storage, trans)
+ self._threads.append(t)
+ t.start()
+
+ def _dowork2(self, method_name):
+ for t in self._threads:
+ t.join()
+
+ def _duplicate_client(self):
+ "Open another ClientStorage to the same server."
+ # XXX argh it's hard to find the actual address
+ # The rpc mgr addr attribute is a list. Each element in the
+ # list is a socket domain (AF_INET, AF_UNIX, etc.) and an
+ # address.
+ addr = self._storage._rpc_mgr.addr[0][1]
+ new = ZEO.ClientStorage.ClientStorage(addr, wait=1)
+ new.registerDB(DummyDB(), None)
+ return new
+
+ def _get_timestamp(self):
+ t = time.time()
+ t = apply(TimeStamp,(time.gmtime(t)[:5]+(t%60,)))
+ return `t`
+
=== ZEO/ZEO/tests/ThreadTests.py 1.1 => 1.2 ===
+#
+# Copyright (c) 2002 Zope Corporation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE
+#
+##############################################################################
+"""Compromising positions involving threads."""
+
+import threading
+
+from ZODB.Transaction import Transaction
+from ZODB.tests.StorageTestBase import zodb_pickle, MinPO
+
+import ZEO.ClientStorage
+from ZEO.Exceptions import Disconnected
+
+ZERO = '\0'*8
+
+class BasicThread(threading.Thread):
+ def __init__(self, storage, doNextEvent, threadStartedEvent):
+ self.storage = storage
+ self.trans = Transaction()
+ self.doNextEvent = doNextEvent
+ self.threadStartedEvent = threadStartedEvent
+ self.gotValueError = 0
+ self.gotDisconnected = 0
+ threading.Thread.__init__(self)
+
+
+class GetsThroughVoteThread(BasicThread):
+ # This thread gets partially through a transaction before it turns
+ # execution over to another thread. We're trying to establish that a
+ # tpc_finish() after a storage has been closed by another thread will get
+ # a ClientStorageError error.
+ #
+ # This class gets does a tpc_begin(), store(), tpc_vote() and is waiting
+ # to do the tpc_finish() when the other thread closes the storage.
+ def run(self):
+ self.storage.tpc_begin(self.trans)
+ oid = self.storage.new_oid()
+ self.storage.store(oid, ZERO, zodb_pickle(MinPO("c")), '', self.trans)
+ self.storage.tpc_vote(self.trans)
+ self.threadStartedEvent.set()
+ self.doNextEvent.wait(10)
+ try:
+ self.storage.tpc_finish(self.trans)
+ except ZEO.ClientStorage.ClientStorageError:
+ self.gotValueError = 1
+ self.storage.tpc_abort(self.trans)
+
+
+class GetsThroughBeginThread(BasicThread):
+ # This class is like the above except that it is intended to be run when
+ # another thread is already in a tpc_begin(). Thus, this thread will
+ # block in the tpc_begin until another thread closes the storage. When
+ # that happens, this one will get disconnected too.
+ def run(self):
+ try:
+ self.storage.tpc_begin(self.trans)
+ except ZEO.ClientStorage.ClientStorageError:
+ self.gotValueError = 1
+
+
+class AbortsAfterBeginFailsThread(BasicThread):
+ # This class is identical to GetsThroughBeginThread except that it
+ # attempts to tpc_abort() after the tpc_begin() fails. That will raise a
+ # ClientDisconnected exception which implies that we don't have the lock,
+ # and that's what we really want to test (but it's difficult given the
+ # threading module's API).
+ def run(self):
+ try:
+ self.storage.tpc_begin(self.trans)
+ except ZEO.ClientStorage.ClientStorageError:
+ self.gotValueError = 1
+ try:
+ self.storage.tpc_abort(self.trans)
+ except Disconnected:
+ self.gotDisconnected = 1
+
+
+class ThreadTests:
+ # Thread 1 should start a transaction, but not get all the way through it.
+ # Main thread should close the connection. Thread 1 should then get
+ # disconnected.
+ def checkDisconnectedOnThread2Close(self):
+ doNextEvent = threading.Event()
+ threadStartedEvent = threading.Event()
+ thread1 = GetsThroughVoteThread(self._storage,
+ doNextEvent, threadStartedEvent)
+ thread1.start()
+ threadStartedEvent.wait(10)
+ self._storage.close()
+ doNextEvent.set()
+ thread1.join()
+ self.assertEqual(thread1.gotValueError, 1)
+
+ # Thread 1 should start a transaction, but not get all the way through
+ # it. While thread 1 is in the middle of the transaction, a second thread
+ # should start a transaction, and it will block in the tcp_begin() --
+ # because thread 1 has acquired the lock in its tpc_begin(). Now the main
+ # thread closes the storage and both sub-threads should get disconnected.
+ def checkSecondBeginFails(self):
+ doNextEvent = threading.Event()
+ threadStartedEvent = threading.Event()
+ thread1 = GetsThroughVoteThread(self._storage,
+ doNextEvent, threadStartedEvent)
+ thread2 = GetsThroughBeginThread(self._storage,
+ doNextEvent, threadStartedEvent)
+ thread1.start()
+ threadStartedEvent.wait(1)
+ thread2.start()
+ self._storage.close()
+ doNextEvent.set()
+ thread1.join()
+ thread2.join()
+ self.assertEqual(thread1.gotValueError, 1)
+ self.assertEqual(thread2.gotValueError, 1)
+
+ def checkThatFailedBeginDoesNotHaveLock(self):
+ doNextEvent = threading.Event()
+ threadStartedEvent = threading.Event()
+ thread1 = GetsThroughVoteThread(self._storage,
+ doNextEvent, threadStartedEvent)
+ thread2 = AbortsAfterBeginFailsThread(self._storage,
+ doNextEvent, threadStartedEvent)
+ thread1.start()
+ threadStartedEvent.wait(1)
+ thread2.start()
+ self._storage.close()
+ doNextEvent.set()
+ thread1.join()
+ thread2.join()
+ self.assertEqual(thread1.gotValueError, 1)
+ self.assertEqual(thread2.gotValueError, 1)
+ self.assertEqual(thread2.gotDisconnected, 1)
=== ZEO/ZEO/tests/testTransactionBuffer.py 1.3 => 1.4 ===
+#
+# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE
+#
+##############################################################################
+import random
+import unittest
+
+from ZEO.TransactionBuffer import TransactionBuffer
+
+def random_string(size):
+ """Return a random string of size size."""
+ l = [chr(random.randrange(256)) for i in range(size)]
+ return "".join(l)
+
+def new_store_data():
+ """Return arbitrary data to use as argument to store() method."""
+ return random_string(8), '', random_string(random.randrange(1000))
+
+def new_invalidate_data():
+ """Return arbitrary data to use as argument to invalidate() method."""
+ return random_string(8), ''
+
+class TransBufTests(unittest.TestCase):
+
+ def checkTypicalUsage(self):
+ tbuf = TransactionBuffer()
+ tbuf.store(*new_store_data())
+ tbuf.invalidate(*new_invalidate_data())
+ tbuf.begin_iterate()
+ while 1:
+ o = tbuf.next()
+ if o is None:
+ break
+ tbuf.clear()
+
+ def doUpdates(self, tbuf):
+ data = []
+ for i in range(10):
+ d = new_store_data()
+ tbuf.store(*d)
+ data.append(d)
+ d = new_invalidate_data()
+ tbuf.invalidate(*d)
+ data.append(d)
+
+ tbuf.begin_iterate()
+ for i in range(len(data)):
+ x = tbuf.next()
+ if x[2] is None:
+ # the tbuf add a dummy None to invalidates
+ x = x[:2]
+ self.assertEqual(x, data[i])
+
+ def checkOrderPreserved(self):
+ tbuf = TransactionBuffer()
+ self.doUpdates(tbuf)
+
+ def checkReusable(self):
+ tbuf = TransactionBuffer()
+ self.doUpdates(tbuf)
+ tbuf.clear()
+ self.doUpdates(tbuf)
+ tbuf.clear()
+ self.doUpdates(tbuf)
+
+def test_suite():
+ return unittest.makeSuite(TransBufTests, 'check')
=== ZEO/ZEO/tests/Cache.py 1.7 => 1.8 ===
# Make sure this doesn't load invalid data into the cache
self._storage.load(oid, '')
-
+
self._storage.tpc_vote(t)
self._storage.tpc_finish(t)
=== ZEO/ZEO/tests/forker.py 1.15 => 1.16 ===
import asyncore
import os
-import profile
import random
import socket
import sys
+import traceback
import types
-import ZEO.ClientStorage, ZEO.StorageServer
+import ZEO.ClientStorage
+# Change value of PROFILE to enable server-side profiling
PROFILE = 0
+if PROFILE:
+ import hotshot
def get_port():
"""Return a port that is not in use.
@@ -47,21 +50,23 @@
if os.name == "nt":
- def start_zeo_server(storage_name, args, port=None):
+ def start_zeo_server(storage_name, args, addr=None):
"""Start a ZEO server in a separate process.
Returns the ZEO port, the test server port, and the pid.
"""
import ZEO.tests.winserver
- if port is None:
+ if addr is None:
port = get_port()
+ else:
+ port = addr[1]
script = ZEO.tests.winserver.__file__
if script.endswith('.pyc'):
script = script[:-1]
args = (sys.executable, script, str(port), storage_name) + args
d = os.environ.copy()
d['PYTHONPATH'] = os.pathsep.join(sys.path)
- pid = os.spawnve(os.P_NOWAIT, sys.executable, args, os.environ)
+ pid = os.spawnve(os.P_NOWAIT, sys.executable, args, d)
return ('localhost', port), ('localhost', port + 1), pid
else:
@@ -79,9 +84,11 @@
buf = self.recv(4)
if buf:
assert buf == "done"
+ server.close_server()
asyncore.socket_map.clear()
def handle_close(self):
+ server.close_server()
asyncore.socket_map.clear()
class ZEOClientExit:
@@ -90,38 +97,56 @@
self.pipe = pipe
def close(self):
- os.write(self.pipe, "done")
- os.close(self.pipe)
+ try:
+ os.write(self.pipe, "done")
+ os.close(self.pipe)
+ except os.error:
+ pass
- def start_zeo_server(storage, addr):
+ def start_zeo_server(storage_name, args, addr):
+ assert isinstance(args, types.TupleType)
rd, wr = os.pipe()
pid = os.fork()
if pid == 0:
- if PROFILE:
- p = profile.Profile()
- p.runctx("run_server(storage, addr, rd, wr)", globals(),
- locals())
- p.dump_stats("stats.s.%d" % os.getpid())
- else:
- run_server(storage, addr, rd, wr)
+ import ZEO.zrpc.log
+ reload(ZEO.zrpc.log)
+ try:
+ if PROFILE:
+ p = hotshot.Profile("stats.s.%d" % os.getpid())
+ p.runctx("run_server(storage, addr, rd, wr)",
+ globals(), locals())
+ p.close()
+ else:
+ run_server(addr, rd, wr, storage_name, args)
+ except:
+ print "Exception in ZEO server process"
+ traceback.print_exc()
os._exit(0)
else:
os.close(rd)
return pid, ZEOClientExit(wr)
- def run_server(storage, addr, rd, wr):
+ def load_storage(name, args):
+ package = __import__("ZODB." + name)
+ mod = getattr(package, name)
+ klass = getattr(mod, name)
+ return klass(*args)
+
+ def run_server(addr, rd, wr, storage_name, args):
# in the child, run the storage server
+ global server
os.close(wr)
ZEOServerExit(rd)
- serv = ZEO.StorageServer.StorageServer(addr, {'1':storage})
- asyncore.loop()
- os.close(rd)
+ import ZEO.StorageServer, ZEO.zrpc.server
+ storage = load_storage(storage_name, args)
+ server = ZEO.StorageServer.StorageServer(addr, {'1':storage})
+ ZEO.zrpc.server.loop()
storage.close()
if isinstance(addr, types.StringType):
os.unlink(addr)
- def start_zeo(storage, cache=None, cleanup=None, domain="AF_INET",
- storage_id="1", cache_size=20000000):
+ def start_zeo(storage_name, args, cache=None, cleanup=None,
+ domain="AF_INET", storage_id="1", cache_size=20000000):
"""Setup ZEO client-server for storage.
Returns a ClientStorage instance and a ZEOClientExit instance.
@@ -137,10 +162,10 @@
else:
raise ValueError, "bad domain: %s" % domain
- pid, exit = start_zeo_server(storage, addr)
+ pid, exit = start_zeo_server(storage_name, args, addr)
s = ZEO.ClientStorage.ClientStorage(addr, storage_id,
- debug=1, client=cache,
+ client=cache,
cache_size=cache_size,
- min_disconnect_poll=0.5)
+ min_disconnect_poll=0.5,
+ wait=1)
return s, exit, pid
-
=== ZEO/ZEO/tests/multi.py 1.7 => 1.8 ===
pid = os.fork()
if pid == 0:
- import ZEO.ClientStorage
- if VERBOSE:
- print "Client process started:", os.getpid()
- cli = ZEO.ClientStorage.ClientStorage(addr, client=CLIENT_CACHE)
- if client_func is None:
- run(cli)
- else:
- client_func(cli)
- cli.close()
- os._exit(0)
+ try:
+ import ZEO.ClientStorage
+ if VERBOSE:
+ print "Client process started:", os.getpid()
+ cli = ZEO.ClientStorage.ClientStorage(addr, client=CLIENT_CACHE)
+ if client_func is None:
+ run(cli)
+ else:
+ client_func(cli)
+ cli.close()
+ finally:
+ os._exit(0)
else:
return pid
=== ZEO/ZEO/tests/speed.py 1.6 => 1.7 ===
"""
-import asyncore
+import asyncore
import sys, os, getopt, string, time
##sys.path.insert(0, os.getcwd())
@@ -81,7 +81,7 @@
for r in 1, 10, 100, 1000:
t = time.time()
conflicts = 0
-
+
jar = db.open()
while 1:
try:
@@ -105,7 +105,7 @@
else:
break
jar.close()
-
+
t = time.time() - t
if detailed:
if threadno is None:
@@ -205,11 +205,11 @@
for v in l:
tot = tot + v
return tot / len(l)
-
+
##def compress(s):
## c = zlib.compressobj()
## o = c.compress(s)
-## return o + c.flush()
+## return o + c.flush()
if __name__=='__main__':
main(sys.argv[1:])
=== ZEO/ZEO/tests/stress.py 1.5 => 1.6 ===
if pid != 0:
return pid
-
- storage = ClientStorage(zaddr, debug=1, min_disconnect_poll=0.5)
+ try:
+ _start_child(zaddr)
+ finally:
+ os._exit(0)
+
+def _start_child(zaddr):
+ storage = ClientStorage(zaddr, debug=1, min_disconnect_poll=0.5, wait=1)
db = ZODB.DB(storage, pool_size=NUM_CONNECTIONS)
setup(db.open())
conns = []
@@ -128,8 +133,6 @@
else:
c.__count += 1
work(c)
-
- os._exit(0)
if __name__ == "__main__":
main()
=== ZEO/ZEO/tests/testZEO.py 1.24 => 1.25 === (612/712 lines abridged)
import os
import random
+import select
import socket
import sys
import tempfile
+import thread
import time
import types
import unittest
@@ -26,22 +28,20 @@
import ZEO.ClientStorage, ZEO.StorageServer
import ThreadedAsync, ZEO.trigger
from ZODB.FileStorage import FileStorage
-from ZODB.TimeStamp import TimeStamp
from ZODB.Transaction import Transaction
-import thread
+from ZODB.tests.StorageTestBase import zodb_pickle, MinPO
+import zLOG
-from ZEO.tests import forker, Cache
+from ZEO.tests import forker, Cache, CommitLockTests, ThreadTests
from ZEO.smac import Disconnected
-# Sorry Jim...
from ZODB.tests import StorageTestBase, BasicStorage, VersionStorage, \
TransactionalUndoStorage, TransactionalUndoVersionStorage, \
- PackableStorage, Synchronization, ConflictResolution
+ PackableStorage, Synchronization, ConflictResolution, RevisionStorage, \
+ MTStorage, ReadOnlyStorage
from ZODB.tests.MinPO import MinPO
from ZODB.tests.StorageTestBase import zodb_unpickle
-ZERO = '\0'*8
-
class DummyDB:
def invalidate(self, *args):
pass
@@ -56,93 +56,22 @@
def pack(self, t, f):
self.storage.pack(t, f, wait=1)
-class ZEOTestBase(StorageTestBase.StorageTestBase):
- """Version of the storage test class that supports ZEO.
-
- For ZEO, we don't always get the serialno/exception for a
- particular store as the return value from the store. But we
- will get no later than the return value from vote.
- """
-
[-=- -=- -=- 612 lines omitted -=- -=- -=-]
- for k, v in klass.__dict__.items():
- if callable(v):
- meth[k] = 1
- return meth.keys()
+ # XXX waitpid() isn't available until Python 2.3
+ time.sleep(0.5)
if os.name == "posix":
test_classes = ZEOFileStorageTests, UnixConnectionTests
@@ -502,36 +430,12 @@
else:
raise RuntimeError, "unsupported os: %s" % os.name
-def makeTestSuite(testname=''):
+def test_suite():
suite = unittest.TestSuite()
- name = 'check' + testname
- lname = len(name)
for klass in test_classes:
- for meth in get_methods(klass):
- if meth[:lname] == name:
- suite.addTest(klass(meth))
+ sub = unittest.makeSuite(klass, 'check')
+ suite.addTest(sub)
return suite
-def test_suite():
- return makeTestSuite()
-
-def main():
- import sys, getopt
-
- name_of_test = ''
-
- opts, args = getopt.getopt(sys.argv[1:], 'n:')
- for flag, val in opts:
- if flag == '-n':
- name_of_test = val
-
- if args:
- print "Did not expect arguments. Got %s" % args
- return 0
-
- tests = makeTestSuite(name_of_test)
- runner = unittest.TextTestRunner()
- runner.run(tests)
-
if __name__ == "__main__":
- main()
+ unittest.main(defaultTest='test_suite')
From jeremy at zope.com Tue Jun 11 09:43:37 2002
From: jeremy at zope.com (Jeremy Hylton)
Date: Sun Aug 10 16:31:21 2008
Subject: [ZEO-Checkins] CVS: ZEO - .cvsignore:1.2 LICENSE.txt:1.2 README.txt:1.2 setup.py:1.2 test.py:1.2 CHANGES.txt:1.29
Message-ID: <200206111343.g5BDhbi05605@cvs.baymountain.com>
Update of /cvs-repository/ZEO
In directory cvs.zope.org:/tmp/cvs-serv5548
Modified Files:
CHANGES.txt
Added Files:
.cvsignore LICENSE.txt README.txt setup.py test.py
Log Message:
Merge ZEO2-branch to trunk.
=== ZEO/.cvsignore 1.1 => 1.2 ===
=== ZEO/LICENSE.txt 1.1 => 1.2 ===
+-----------------------------------------------
+
+This software is Copyright (c) Zope Corporation (tm) and
+Contributors. All rights reserved.
+
+This license has been certified as open source. It has also
+been designated as GPL compatible by the Free Software
+Foundation (FSF).
+
+Redistribution and use in source and binary forms, with or
+without modification, are permitted provided that the
+following conditions are met:
+
+1. Redistributions in source code must retain the above
+ copyright notice, this list of conditions, and the following
+ disclaimer.
+
+2. Redistributions in binary form must reproduce the above
+ copyright notice, this list of conditions, and the following
+ disclaimer in the documentation and/or other materials
+ provided with the distribution.
+
+3. The name Zope Corporation (tm) must not be used to
+ endorse or promote products derived from this software
+ without prior written permission from Zope Corporation.
+
+4. The right to distribute this software or to use it for
+ any purpose does not give you the right to use Servicemarks
+ (sm) or Trademarks (tm) of Zope Corporation. Use of them is
+ covered in a separate agreement (see
+ http://www.zope.com/Marks).
+
+5. If any files are modified, you must cause the modified
+ files to carry prominent notices stating that you changed
+ the files and the date of any change.
+
+Disclaimer
+
+ THIS SOFTWARE IS PROVIDED BY ZOPE CORPORATION ``AS IS''
+ AND ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT
+ NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY
+ AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN
+ NO EVENT SHALL ZOPE CORPORATION OR ITS CONTRIBUTORS BE
+ LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
+ OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ DAMAGE.
+
+
+This software consists of contributions made by Zope
+Corporation and many individuals on behalf of Zope
+Corporation. Specific attributions are listed in the
+accompanying credits file.
=== ZEO/README.txt 1.1 => 1.2 ===
+
+ Zope Enterprise Objects (ZEO) extends the Zope Object Database
+ (ZODB) to multiple processes, machines, and locations. It provides
+ scalability, high availability, and distribution for ZODB. For more
+ information, see the ZEO Web page at http://www.zope.org/Products/ZEO/.
+
+ IMPORTANT: ZEO version 2 is not backwards compatible with ZEO 1.0.
+ A system that uses ZEO must upgrade all clients and servers at the
+ same time.
+
+ The ZEO package is contained in the directory named ZEO.
+
+ If you are using Zope, see doc/ZopeREADME.txt; otherwise, see
+ doc/NonZopeREADME.txt.
=== ZEO/setup.py 1.1 => 1.2 ===
+#
+# Copyright (c) 2002 Zope Corporation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE
+#
+##############################################################################
+from distutils.core import setup
+
+packages = ['ZEO', 'ZEO.zrpc', 'ZEO.tests']
+
+setup(name="ZEO",
+ version="2.0a1",
+ description="Zope Enterprise Objects",
+ maintainer="Zope Corp.",
+ maintainer_email="zodb-dev@zope.org",
+ url = "http://www.zope.org/Products/ZEO",
+
+ packages = packages,
+ )
=== ZEO/test.py 1.1 => 1.2 ===
+#
+# Copyright (c) 2002 Zope Corporation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE
+#
+##############################################################################
+"""Test harness for ZEO
+
+usage: python test.py [options] [modulepath] [testcase]
+
+options:
+ -v -- verbose (can be repeated to increase verbosity)
+ -b -- run "setup.py -q build" before running the tests
+ -d -- run tests in debug mode
+ -L -- run tests in an infinite loop
+ -h -- print this message
+
+The optional modulepath and testcase arguments are regular expressions
+that can be used to limit the number of tests that are run. The
+modulepath regex must be found in the path of the module that contains
+the tests. The testcase regex must be found in the name of the test case.
+
+When it finishes, the test harness prints a report of the tests run
+and how long it took. If errors or failures occured, they will be
+reported along with a traceback. If one -v is specified, a dot will
+be printed as each test is run. If two -v's are specified, the name
+of each test will be printed as it runs.
+"""
+
+import os
+import re
+import sys
+import traceback
+import unittest
+
+from distutils.util import get_platform
+
+class ImmediateTestResult(unittest._TextTestResult):
+
+ def _print_traceback(self, msg, err, test, errlist):
+ if self.showAll or self.dots:
+ self.stream.writeln("\n")
+
+ tb = ''.join(traceback.format_exception(*err))
+ self.stream.writeln(msg)
+ self.stream.writeln(tb)
+ errlist.append((test, tb))
+
+ def addError(self, test, err):
+ self._print_traceback("Error in test %s" % test, err,
+ test, self.errors)
+
+ def addFailure(self, test, err):
+ self._print_traceback("Failure in test %s" % test, err,
+ test, self.failures)
+
+ def printErrorList(self, flavor, errors):
+ for test, err in errors:
+ self.stream.writeln(self.separator1)
+ self.stream.writeln("%s: %s" % (flavor, self.getDescription(test)))
+ self.stream.writeln(self.separator2)
+ self.stream.writeln(err)
+
+
+class ImmediateTestRunner(unittest.TextTestRunner):
+
+ def _makeResult(self):
+ return ImmediateTestResult(self.stream, self.descriptions,
+ self.verbosity)
+
+# setup list of directories to put on the path
+
+PLAT_SPEC = "%s-%s" % (get_platform(), sys.version[0:3])
+
+def setup_path():
+ DIRS = ["lib",
+ "lib.%s" % PLAT_SPEC,
+ ]
+ for d in DIRS:
+ sys.path.insert(0, d)
+
+
+# Find test files.
+# They live under either a lib.PLAT_SPEC or plain "lib" directory.
+_sep = re.escape(os.sep)
+_pat = "%s(%s|lib)%s" % (_sep, re.escape("lib." + PLAT_SPEC), _sep)
+hasgooddir = re.compile(_pat).search
+del _sep, _pat
+
+class TestFileFinder:
+ def __init__(self):
+ self.files = []
+
+ def visit(self, rx, dir, files):
+ if dir[-5:] != "tests":
+ return
+ # ignore tests that aren't in packages
+ if not "__init__.py" in files:
+ print "not a package", dir
+ return
+ for file in files:
+ if file[:4] == "test" and file[-3:] == ".py":
+ path = os.path.join(dir, file)
+ if not hasgooddir(path):
+ # built for a different version
+ continue
+ if rx is not None:
+ if rx.search(path):
+ self.files.append(path)
+ else:
+ self.files.append(path)
+
+def find_tests(filter):
+ if filter is not None:
+ rx = re.compile(filter)
+ else:
+ rx = None
+ finder = TestFileFinder()
+ os.path.walk("build", finder.visit, rx)
+ return finder.files
+
+def package_import(modname):
+ mod = __import__(modname)
+ for part in modname.split(".")[1:]:
+ mod = getattr(mod, part)
+ return mod
+
+def module_from_path(path):
+ """Return the Python package name indiciated by the filesystem path.
+
+ The path starts with build/lib or build /lib.mumble..."""
+
+ assert path[-3:] == '.py'
+ path = path[:-3]
+ dirs = []
+ while path:
+ path, end = os.path.split(path)
+ dirs.insert(0, end)
+ assert dirs[0] == "build"
+ assert dirs[1][:3] == "lib"
+ return ".".join(dirs[2:])
+
+def get_suite(file):
+ assert file[:5] == "build"
+ assert file[-3:] == '.py'
+ modname = module_from_path(file)
+ mod = package_import(modname)
+ try:
+ suite_factory = mod.test_suite
+ except AttributeError, err:
+ return None
+ return suite_factory()
+
+def match(rx, s):
+ if not rx:
+ return 1
+ if rx[0] == '!':
+ return re.search(rx[1:], s) is None
+ else:
+ return re.search(rx, s) is not None
+
+def filter_testcases(s, rx):
+ new = unittest.TestSuite()
+ for test in s._tests:
+ if isinstance(test, unittest.TestCase):
+ name = test.id() # Full test name: package.module.class.method
+ name = name[1 + name.rfind('.'):] # extract method name
+ if match(rx, name):
+ new.addTest(test)
+ else:
+ filtered = filter_testcases(test, rx)
+ if filtered:
+ new.addTest(filtered)
+ return new
+
+def runner(files, test_filter, debug):
+ runner = ImmediateTestRunner(verbosity=VERBOSE)
+ suite = unittest.TestSuite()
+ for file in files:
+ s = get_suite(file)
+ if s is not None:
+ if test_filter is not None:
+ s = filter_testcases(s, test_filter)
+ suite.addTest(s)
+ if debug:
+ suite.debug()
+ return 0
+ r = runner.run(suite)
+ return len(r.errors) + len(r.failures)
+
+def main(module_filter, test_filter):
+ setup_path()
+ files = find_tests(module_filter)
+ files.sort()
+
+ os.chdir("build")
+
+ if LOOP:
+ while 1:
+ runner(files, test_filter, debug)
+ else:
+ runner(files, test_filter, debug)
+
+if __name__ == "__main__":
+ import getopt
+
+ module_filter = None
+ test_filter = None
+ VERBOSE = 0
+ LOOP = 0
+ debug = 0 # Don't collect test results; simply let tests crash
+ build = 0
+
+ try:
+ opts, args = getopt.getopt(sys.argv[1:], 'vdLbh')
+ except getopt.error, msg:
+ print msg
+ print "Try `python %s -h' for more information." % sys.argv[0]
+ sys.exit(2)
+
+ for k, v in opts:
+ if k == '-v':
+ VERBOSE = VERBOSE + 1
+ elif k == '-d':
+ debug = 1
+ elif k == '-L':
+ LOOP = 1
+ elif k == '-b':
+ build = 1
+ elif k == '-h':
+ print __doc__
+ sys.exit(0)
+
+ if build:
+ cmd = sys.executable + " setup.py -q build"
+ if VERBOSE:
+ print cmd
+ sts = os.system(cmd)
+ if sts:
+ print "Build failed", hex(sts)
+ sys.exit(1)
+
+ if args:
+ if len(args) > 1:
+ test_filter = args[1]
+ module_filter = args[0]
+ try:
+ bad = main(module_filter, test_filter)
+ if bad:
+ sys.exit(1)
+ except ImportError, err:
+ print err
+ print sys.path
+ raise
=== ZEO/CHANGES.txt 1.28 => 1.29 ===
+Revision History, Zope Enterprise Objects, version 2
- ZEO 1.0 final
+ ZEO 2.0 alpha 1
- Bugs fixed
-
- - Fixed a bug that prevented a client from reconnecting to a
- server if the server restarted.
-
- - Fixed start.py so that it prints a message on the console when
- it fails in addition to using zLOG.
-
- - Fleshed out installation instructions and version dependencies.
-
- ZEO 1.0 beta 3
-
- Bugs fixed
-
- - The previous beta failed to startup a connection to a server
- running any storage that did not support transactionalUndo.
- The server now checks for supported features in a way that will
- not cause errors.
-
- ZEO 1.0 beta 2
-
- New Features
-
- - Support for transactionalUndo when the underlying storage
- supports it.
-
- - A unit test suite was added. The test suite requires that
- PyUnit be installed; it's part of the standard library in
- Python 2.1. It also requires that the ZODB installation
- defines the ZODB.tests package. If these conditions don't
- hold, the test suite can't be run.
-
- Bugs fixed
-
- - A cache invalidation bug was fixed for commitVersion and
- abortVersion. It was possible for a load the occurred between
- a commit version and a tpc_finish to store invalid data in the
- cache.
-
- - The undoInfo() method defines default values for its arguments.
-
- - The final argument to tpc_begin(), the transaction extended
- metadata, was ignored.
-
- - A theoretical bug in the client caching code for objects
- created in versions was fixed.
-
- ZEO 1.0 beta 1
-
- New Features
-
- - Improved release organization.
-
- - Moved documentation and misc files out of the ZEO package into
- the release directory.
-
- Bugs fixed
-
- - Normal shutdown was reported as a panic.
-
- - The signal exception handler was disabled.
-
- - Errors arising from incompatable versions of cPickle were
- uclear.
-
-
- ZEO 0.5.0
-
- New Features
-
- - The server can be made to reopen it's log file
- by sending it a HUP (on systems supporting signals). Note
- that this requires a change to asyncore to catch interrupted
- system calls on some platforms.
-
- - The shutdown signals have been changed:
-
- o To shutdown, use TERM
-
- o To restart, use INT. (This must be send to the
- child, not the parent.
-
- - Client scripts can now be written to pack a remote storage and
- wait for the pack results. This is handy when packing as part
- of cron jobs.
-
- - It is no longer necessary to symbolically link cPickle or
- ZServer. ZServer is no longer necessary at all.
-
- - A Zope-style INSTANCE_HOME and var directory are no longer
- needed.
-
- - An option, -d, was added to facilitate generation of a
- detailed debug log while running in the background.
-
- - The documentation has been simplified and spread over multiple
- files in the doc subdirectory.
-
- Bugs Fixed
-
- - Application-level conflict resolution, introduced in Zope
- 2.3.1, was not supported. This caused the ZEO cache to be
- written incorrectly.
-
- - A possible (but unobserved) race condition that could
- lead to ZEO cache corruption was corrected.
-
- - ZEO clients could fail to start if they needed data that
- wasn't in their cache and if they couldn't talk to a ZEO
- server right away. For now, on startup, the client storage
- will wait to connect to a storage before returning from
- initialization.
-
- - Restarting the ZEO server shortly after shutting down could
- lead to "address already in use" errors.
-
- - User-level eceptions, like undo, version-lock, and conflict
- errors were logged in the server event log.
-
- - Pack errors weren't logged in the server event log.
-
- - If an attempt was made to commit a transaction with updates
- while the client storage was disconnected from the server,
- no further write transactions would be allowed, even after
- reconnection, and the site would eventually hang.
-
- - A forgotten argument made it unreliable to start a ClientStorage
- after the main loop has started.
-
- - In combination with recent changes in zdeamon, startup errors
- could cause infinite loops.
-
- - The handling of the Python global, __debug__, was not
- compatible with Python 2.1.
-
- - If an exception raised on the server which could not be
- unpickled on the client could cause the client connection to
- fail.
-
- Planned for (future) ZEO releases
-
- New Features
-
- - Provide optional data compression. This should enhance
- performance over slow connections to the storage server and
- reduce the server I/O load.
-
- - Provide optional authentication adapters that allow for
- pluggable authentication and encryption schemes.
-
- This is a feature that is listed on the ZEO fact sheet, but
- that didn't make it into the 1.0 release. Firewall or secure
- tunneling techniques can be used to secure communication
- between clients and the storage for now when the client and
- storage are on different machines. (If they are on the same
- machine, then unix-domain sockets or the loop-back interface
- can be used.)
-
- - Provide an option to start a client process without waiting
- for a connection to the storage server. This was the original
- intent, however, it turns out that it can be extremely
- problemantic to get storage errors resulting from attempts to
- read objects not in the cache during process (e.g. Zope)
- startup. In addition, some smarter cache management can be
- done to decrease the probability of important objects being
- removed from the cache.
-
- - Provide improved client cache management. This will involve
- changes like:
-
- o Increasing the number of cache files to reduce the number of
- objects lost from the cache (or that need to be recovered)
- when the cache "rolls over".
-
- o Use separate indexes for each cache.
-
- o use better cache indexing structures
-
- ZEO 0.4.1
-
- Bugs fixed
-
- - Improperly handled server exeptions could cause clients to
- lock up.
-
- - Misshandling of client transaction meta data could cause
- server errors because transaction ids were mangled.
-
- - The storage server didn't close sockets on shutdown. This
- could sometimes make it necessary to wait before restarting
- the server to avoid "address already in use" messages.
-
- - The storage server did not log shutdown.
-
- ZEO 0.4
-
- Bugs fixed
-
- - The new (in 0.3) logic to switch to an ordinary user when
- started as root was executed too late so that some files were
- incorrectly owned by root. This caused ZEO clients to fail
- when the cache files were rotated.
-
- - There were some unusual error conditions that were not handled
- correctly that could cause clients to fail. This was detected
- only when ZEO was put into production on zope.org.
-
- - The cache files weren't rotated on reads. This could cause the
- caches to grow way beyond their target sizes.
-
- - Exceptions raised in the servers asynchronous store handler
- could cause the client and server to get out of sync.
-
- - Connection and disconnection events weren't logged on the
- server.
-
- Features added
-
- - ClientStorage objects have two new constructor arguments,
- min_disconnect_poll and max_disconnect_poll to set the minimum
- and maximum times to wait, in seconds, before retrying to
- reconnect when disconnected from the ZEO server.
-
- - A call to get database info on startup was eliminated in
- favor of having the server send the information
- automatically. This eliminates a round-trip and, therefore
- speeds up startup a tiny bit.
-
- - Database size info is now sent to all clients (asynchronously)
- after a pack and after a transaction commit, allowing all
- clients to have timely size information.
-
- - Added client logging of connection attempts.
-
- - Added a misc subdirectory with sample storage server start and
- stop scripts and with a sample custom_zodb.py module.
-
- ZEO 0.3.0
-
- Bugs fixed
-
- - Large transactions (e.g. ZCatalog updates) could cause
- spurious conflict errors that could, eventually, make it
- impossible to modify some objects without restarting Zope.
-
- - Temporary non-persistent cache files were not removed at the
- end of a run.
-
- Features added
-
- - On Unix, when the storage server start script is run as root,
- the script will switch to a different user (nobody by
- default). There is a new '-u' option that can be used to
- specify the user.
-
- - On Unix the server will gracefully close served storages when
- the server is killed with a SIGTERM or SIGHUP. If a
- FileStorage is being served, then an index file will be
- written.
-
- ZEO 0.2.3
-
- Bugs fixed
-
- - Versions didn't work. Not even close. :|
-
- - If a client was disconnected from a server during transaction
- commit, then, when the client was reconnected to the server,
- attempts to commit transactions caused the client to hang.
-
- - The server would fail (and successfully automatically restart)
- if an unpickleable exception was raised.
-
- ZEO 0.2.2
-
- Bugs fixed
-
- - The storage server didn't fully implement a new ZODB storage
- protocol. This caused serving of FileStorages to fail in Zope
- 2.2.1, since FileStorages now use this protocol.
-
- - In the start.py start script
-
- o The '-S' option did not allow spaces between the option and it's
- argument.
-
- o The '-S' option did not work with FileStorages.
-
- o The README file didn't mention the '-S' option.
-
- ZEO 0.2.1
-
- Bugs fixed
-
- - ZEO clients didn't work properly (effectively at all) on
- Solaris or Windows NT.
-
- - An error in the handling of the distributed transaction lock
- could cause a client to stop writing and eventually hang if
- two clients tried to commit a transaction at the same time.
-
- - Extra (harmless) messages were sent from the server
- when invalidating objects during a commit.
-
- - New protocols (especially 'loadSerial'), used for looking at
- DTML historical versions, were not implemented.
-
- Features
-
- - The '-S' option was added to the storage server startup script
- to allow selection of one or more storages to serve.
-
- ZEO 0.2
-
- This release is expected to be close to beta quality. Initially, the
- primary goals of this release were to:
-
- - Correct some consistency problems that had been observed in
- 0.1 on starup.
-
- - Allow ZEO clients to detect, survive, and recover from
- disconnection from the ZEO server.
-
- Based on some feedback from some folks who tried 0.1, improving
- write performance was made a priority.
-
- Features
-
- - The ZEO Client now handles server failures gracefully:
-
- o The client with a persistent cache can generally startup
- even if the server is not running, assuming that it has at
- least a minimal number of objects in the cache.
-
- o The client will continue to function even if the server
- connection is interuppted.
-
- o Server availability is detected by the client (which tries
- to connect to the server every few minutes). A disconnected
- client will automatically reconnect to an available server.
-
- o When the client is disconnected, write transactions cannot
- be performed. Reads fail for objects that are not in the
- cache.
-
- - Performance enhancements
-
- The speed of write-intensive operations have been improved
- approximately 70%. When using Unix domain sockets for
- client/server communication, ZEO transactions take roughly 2-3
- times as long as FileStorage transactions to commit.
- (This was based on some tests. Your mileage may vary.)
-
- - Packing support was added. Note that packing is done
- asynchrounously. The client returns immediately from a pack
- call. The server packs in a thread and sends updated
- statistics to the client when packing is completed.
-
- - Support for Unix-domain sockets was added.
-
- - Pickles sent to the server are now checked to make sure that
- they don't contain unapproved instance or global-variable
- (function) pickles.
-
- Bugs fixed
-
- - Data could be badly inconsistent when a persistent cache
- was started, due to a bug in the cache initialization logic.
+ Brief overview of the differences between ZEO 1.0 and 2.0.
- - The application was allowed to begin operation while the cache
- was being verified. This could lead to harmful inconsistencies.
-
- Changes made to Zope to support ZEO
-
- - A number of changes were made to ZODB to support asynchronous
- storage during transaction commit.
-
- - Normally Zope updates the database during startup to reflect
- product changes. This behavior is now suppressed when the
- ZEO_CLIENT environment variable is set. It doesn't make sense
- for many clients to update the database for the same products.
+ - New protocol.
- - The asyncore module was modified to add support for multiple
- asyncore loops. This change was applied to asyncore in the
- Zope and the (official, owned by Sam Rushing) medusa CVS
- trees.
+ ZEO 2 uses a different wire protocol and a different API to
+ make RPC calls. The new protocol was designed to be flexible
+ and simple. It includes an initial handshake to set the
+ version number, which should allow future changes to the
+ protocol while reducing the difficulty of upgrades.
+
+ - Better handling of concurrent commits.
+
+ The ZEO server serializes concurrent commits to guarantee
+ consistency; the mechanism is often called the distributed
+ commit lock. ZEO 2 improves the efficiency of concurrent
+ commits by allowing data to be transferred to the server
+ before entering the commit lock.
- - A new module, ThreadedAsync.py has been added in the Zope
- lib/python directory. This module provides notification to
- async objects (like ZEO clients) to let them know when the
- asyncore main loop has started. This was needed to enable use
- of async code before the main loop starts.
+ - The ZEO client and server can be configured to operate in
+ read-only mode.
- ZEO 0.1 (aka "iteration 1")
+ - A ZEO client can be configured with multiple server addresses.
+ It uses the first server it can connect to.
- This was an initial alpha of ZEO that demonstrated basic
- functionalities. It lacked robustness and has some performance
- problems on writes.
+ - The wait_for_server_on_startup keyword argument to
+ ClientStorage has been renamed wait.
From jeremy at zope.com Tue Jun 11 09:43:37 2002
From: jeremy at zope.com (Jeremy Hylton)
Date: Sun Aug 10 16:31:21 2008
Subject: [ZEO-Checkins] CVS: ZEO/ZEO - ClientStub.py:1.4 CommitLog.py:1.2 Exceptions.py:1.4 ICache.py:1.2 ServerStub.py:1.4 TransactionBuffer.py:1.4 ClientCache.py:1.23 ClientStorage.py:1.41 StorageServer.py:1.37 __init__.py:1.9 smac.py:1.17 start.py:1.33 trigger.py:1.6
Message-ID: <200206111343.g5BDhbo05636@cvs.baymountain.com>
Update of /cvs-repository/ZEO/ZEO
In directory cvs.zope.org:/tmp/cvs-serv5548/ZEO
Modified Files:
ClientCache.py ClientStorage.py StorageServer.py __init__.py
smac.py start.py trigger.py
Added Files:
ClientStub.py CommitLog.py Exceptions.py ICache.py
ServerStub.py TransactionBuffer.py
Log Message:
Merge ZEO2-branch to trunk.
=== ZEO/ZEO/ClientStub.py 1.3 => 1.4 ===
+#
+# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE
+#
+##############################################################################
+"""Stub for interface exported by ClientStorage"""
+
+class ClientStorage:
+ def __init__(self, rpc):
+ self.rpc = rpc
+
+ def beginVerify(self):
+ self.rpc.callAsync('begin')
+
+ # XXX must rename the two invalidate messages. I can never
+ # remember which is which
+
+ def invalidate(self, args):
+ self.rpc.callAsync('invalidate', args)
+
+ def Invalidate(self, args):
+ self.rpc.callAsync('Invalidate', args)
+
+ def endVerify(self):
+ self.rpc.callAsync('end')
+
+ def serialnos(self, arg):
+ self.rpc.callAsync('serialnos', arg)
+
+ def info(self, arg):
+ self.rpc.callAsync('info', arg)
=== ZEO/ZEO/CommitLog.py 1.1 => 1.2 ===
+#
+# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE
+#
+##############################################################################
+"""Log a transaction's commit info during two-phase commit.
+
+A storage server allows multiple clients to commit transactions, but
+must serialize them as the actually execute at the server. The
+concurrent commits are achieved by logging actions up until the
+tpc_vote(). At that point, the entire transaction is committed on the
+real storage.
+"""
+import cPickle
+import tempfile
+
+class CommitLog:
+
+ def __init__(self):
+ self.file = tempfile.TemporaryFile(suffix=".log")
+ self.pickler = cPickle.Pickler(self.file, 1)
+ self.pickler.fast = 1
+ self.stores = 0
+ self.read = 0
+
+ def tpc_begin(self, t, tid, status):
+ self.t = t
+ self.tid = tid
+ self.status = status
+
+ def store(self, oid, serial, data, version):
+ self.pickler.dump((oid, serial, data, version))
+ self.stores += 1
+
+ def get_loader(self):
+ self.read = 1
+ self.file.seek(0)
+ return self.stores, cPickle.Unpickler(self.file)
+
=== ZEO/ZEO/Exceptions.py 1.3 => 1.4 ===
+#
+# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE
+#
+##############################################################################
+"""Exceptions for ZEO."""
+
+class Disconnected(Exception):
+ """Exception raised when a ZEO client is disconnected from the
+ ZEO server."""
=== ZEO/ZEO/ICache.py 1.1 => 1.2 ===
+ from Interface import Base
+except ImportError:
+ class Base:
+ # a dummy interface for use when Zope's is unavailable
+ pass
+
+class ICache(Base):
+ """ZEO client cache.
+
+ __init__(storage, size, client, var)
+
+ All arguments optional.
+
+ storage -- name of storage
+ size -- max size of cache in bytes
+ client -- a string; if specified, cache is persistent.
+ var -- var directory to store cache files in
+ """
+
+ def open():
+ """Returns a sequence of object info tuples.
+
+ An object info tuple is a pair containing an object id and a
+ pair of serialnos, a non-version serialno and a version serialno:
+ oid, (serial, ver_serial)
+
+ This method builds an index of the cache and returns a
+ sequence used for cache validation.
+ """
+
+ def close():
+ """Closes the cache."""
+
+ def verify(func):
+ """Call func on every object in cache.
+
+ func is called with three arguments
+ func(oid, serial, ver_serial)
+ """
+
+ def invalidate(oid, version):
+ """Remove object from cache."""
+
+ def load(oid, version):
+ """Load object from cache.
+
+ Return None if object not in cache.
+ Return data, serialno if object is in cache.
+ """
+
+ def store(oid, p, s, version, pv, sv):
+ """Store a new object in the cache."""
+
+ def update(oid, serial, version, data):
+ """Update an object already in the cache.
+
+ XXX This method is called to update objects that were modified by
+ a transaction. It's likely that it is already in the cache,
+ and it may be possible for the implementation to operate more
+ efficiently.
+ """
+
+ def modifiedInVersion(oid):
+ """Return the version an object is modified in.
+
+ '' signifies the trunk.
+ Returns None if the object is not in the cache.
+ """
+
+ def checkSize(size):
+ """Check if adding size bytes would exceed cache limit.
+
+ This method is often called just before store or update. The
+ size is a hint about the amount of data that is about to be
+ stored. The cache may want to evict some data to make space.
+ """
+
+
+
+
+
+
=== ZEO/ZEO/ServerStub.py 1.3 => 1.4 ===
+#
+# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE
+#
+##############################################################################
+"""Stub for interface exposed by StorageServer"""
+
+class StorageServer:
+
+ def __init__(self, rpc):
+ self.rpc = rpc
+
+ def register(self, storage_name, read_only):
+ self.rpc.call('register', storage_name, read_only)
+
+ def get_info(self):
+ return self.rpc.call('get_info')
+
+ def get_size_info(self):
+ return self.rpc.call('get_size_info')
+
+ def beginZeoVerify(self):
+ self.rpc.callAsync('beginZeoVerify')
+
+ def zeoVerify(self, oid, s, sv):
+ self.rpc.callAsync('zeoVerify', oid, s, sv)
+
+ def endZeoVerify(self):
+ self.rpc.callAsync('endZeoVerify')
+
+ def new_oids(self, n=None):
+ if n is None:
+ return self.rpc.call('new_oids')
+ else:
+ return self.rpc.call('new_oids', n)
+
+ def pack(self, t, wait=None):
+ if wait is None:
+ self.rpc.call('pack', t)
+ else:
+ self.rpc.call('pack', t, wait)
+
+ def zeoLoad(self, oid):
+ return self.rpc.call('zeoLoad', oid)
+
+ def storea(self, oid, serial, data, version, id):
+ self.rpc.callAsync('storea', oid, serial, data, version, id)
+
+ def tpc_begin(self, id, user, descr, ext, tid, status):
+ return self.rpc.call('tpc_begin', id, user, descr, ext, tid, status)
+
+ def vote(self, trans_id):
+ return self.rpc.call('vote', trans_id)
+
+ def tpc_finish(self, id):
+ return self.rpc.call('tpc_finish', id)
+
+ def tpc_abort(self, id):
+ self.rpc.callAsync('tpc_abort', id)
+
+ def abortVersion(self, src, id):
+ return self.rpc.call('abortVersion', src, id)
+
+ def commitVersion(self, src, dest, id):
+ return self.rpc.call('commitVersion', src, dest, id)
+
+ def history(self, oid, version, length=None):
+ if length is not None:
+ return self.rpc.call('history', oid, version)
+ else:
+ return self.rpc.call('history', oid, version, length)
+
+ def load(self, oid, version):
+ return self.rpc.call('load', oid, version)
+
+ def loadSerial(self, oid, serial):
+ return self.rpc.call('loadSerial', oid, serial)
+
+ def modifiedInVersion(self, oid):
+ return self.rpc.call('modifiedInVersion', oid)
+
+ def new_oid(self, last=None):
+ if last is None:
+ return self.rpc.call('new_oid')
+ else:
+ return self.rpc.call('new_oid', last)
+
+ def store(self, oid, serial, data, version, trans):
+ return self.rpc.call('store', oid, serial, data, version, trans)
+
+ def transactionalUndo(self, trans_id, trans):
+ return self.rpc.call('transactionalUndo', trans_id, trans)
+
+ def undo(self, trans_id):
+ return self.rpc.call('undo', trans_id)
+
+ def undoLog(self, first, last):
+ # XXX filter not allowed across RPC
+ return self.rpc.call('undoLog', first, last)
+
+ def undoInfo(self, first, last, spec):
+ return self.rpc.call('undoInfo', first, last, spec)
+
+ def versionEmpty(self, vers):
+ return self.rpc.call('versionEmpty', vers)
+
+ def versions(self, max=None):
+ if max is None:
+ return self.rpc.call('versions')
+ else:
+ return self.rpc.call('versions', max)
=== ZEO/ZEO/TransactionBuffer.py 1.3 => 1.4 ===
+#
+# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE
+#
+##############################################################################
+"""A TransactionBuffer store transaction updates until commit or abort.
+
+A transaction may generate enough data that it is not practical to
+always hold pending updates in memory. Instead, a TransactionBuffer
+is used to store the data until a commit or abort.
+"""
+
+# A faster implementation might store trans data in memory until it
+# reaches a certain size.
+
+import tempfile
+import cPickle
+
+class TransactionBuffer:
+
+ def __init__(self):
+ self.file = tempfile.TemporaryFile(suffix=".tbuf")
+ self.count = 0
+ self.size = 0
+ # It's safe to use a fast pickler because the only objects
+ # stored are builtin types -- strings or None.
+ self.pickler = cPickle.Pickler(self.file, 1)
+ self.pickler.fast = 1
+
+ def close(self):
+ try:
+ self.file.close()
+ except OSError:
+ pass
+
+
+ def store(self, oid, version, data):
+ """Store oid, version, data for later retrieval"""
+ self.pickler.dump((oid, version, data))
+ self.count += 1
+ # Estimate per-record cache size
+ self.size = self.size + len(data) + (27 + 12)
+ if version:
+ self.size = self.size + len(version) + 4
+
+ def invalidate(self, oid, version):
+ self.pickler.dump((oid, version, None))
+ self.count += 1
+
+ def clear(self):
+ """Mark the buffer as empty"""
+ self.file.seek(0)
+ self.count = 0
+ self.size = 0
+
+ # unchecked constraints:
+ # 1. can't call store() after begin_iterate()
+ # 2. must call clear() after iteration finishes
+
+ def begin_iterate(self):
+ """Move the file pointer in advance of iteration"""
+ self.file.flush()
+ self.file.seek(0)
+ self.unpickler = cPickle.Unpickler(self.file)
+
+ def next(self):
+ """Return next tuple of data or None if EOF"""
+ if self.count == 0:
+ del self.unpickler
+ return None
+ oid_ver_data = self.unpickler.load()
+ self.count -= 1
+ return oid_ver_data
+
+ def get_size(self):
+ """Return size of data stored in buffer (just a hint)."""
+
+ return self.size
=== ZEO/ZEO/ClientCache.py 1.22 => 1.23 === (447/547 lines abridged)
##############################################################################
"""Implement a client cache
-
+
The cache is managed as two files, var/c0.zec and var/c1.zec.
Each cache file is a sequence of records of the form:
@@ -75,143 +75,181 @@
__version__ = "$Revision$"[11:-2]
-import os, tempfile
+import os
+import sys
+import tempfile
from struct import pack, unpack
from thread import allocate_lock
-import zLOG
-magic='ZEC0'
+import zLOG
+from ZEO.ICache import ICache
-def LOG(msg, level=zLOG.BLATHER):
+def log(msg, level=zLOG.INFO):
zLOG.LOG("ZEC", level, msg)
+magic='ZEC0'
+
class ClientCache:
+ __implements__ = ICache
+
def __init__(self, storage='', size=20000000, client=None, var=None):
# Allocate locks:
- l=allocate_lock()
- self._acquire=l.acquire
- self._release=l.release
+ L = allocate_lock()
+ self._acquire = L.acquire
+ self._release = L.release
if client:
# Create a persistent cache
if var is None:
- try: var=CLIENT_HOME
+ try:
+ var = CLIENT_HOME
except:
[-=- -=- -=- 447 lines omitted -=- -=- -=-]
- else: vs=None
+ vs = read(8)
+ if read(4) != h[9:13]:
+ break
+ else:
+ vs = None
if h[8] in 'vn':
- if current: index[oid]=-pos
- else: index[oid]=pos
- serial[oid]=h[-8:], vs
+ if current:
+ index[oid] = -pos
+ else:
+ index[oid] = pos
+ serial[oid] = h[-8:], vs
else:
if serial.has_key(oid):
# We have a record for this oid, but it was invalidated!
del serial[oid]
del index[oid]
-
-
- pos=pos+tlen
+
+
+ pos = pos + tlen
f.seek(pos)
- try: f.truncate()
- except: pass
-
- return pos
+ try:
+ f.truncate()
+ except:
+ pass
-def main(files):
- for file in files:
- print file
- index = {}
- serial = {}
- read_index(index, serial, open(file), 0)
- print index.keys()
-
-if __name__ == "__main__":
- import sys
- main(sys.argv[1:])
+ return pos
=== ZEO/ZEO/ClientStorage.py 1.40 => 1.41 === (866/966 lines abridged)
__version__='$Revision$'[11:-2]
-import struct, time, os, socket, string
-import tempfile, thread
-from struct import pack, unpack
-from types import TupleType
+import cPickle
+import os
+import tempfile
+import threading
+import time
+
+from ZEO import ClientCache, ServerStub
+from ZEO.TransactionBuffer import TransactionBuffer
+from ZEO.Exceptions import Disconnected
+from ZEO.zrpc.client import ConnectionManager
-import Invalidator, ExtensionClass
-import ThreadedAsync, Sync, zrpc, ClientCache
-
-from ZODB import POSException, BaseStorage
+from ZODB import POSException
from ZODB.TimeStamp import TimeStamp
+from zLOG import LOG, PROBLEM, INFO, BLATHER
-from ZEO.logger import zLogger
-
-log = zLogger("ZEO Client")
+def log2(type, msg, subsys="ClientStorage %d" % os.getpid()):
+ LOG(subsys, type, msg)
try:
from ZODB.ConflictResolution import ResolvedSerial
-except:
- ResolvedSerial='rs'
+except ImportError:
+ ResolvedSerial = 'rs'
class ClientStorageError(POSException.StorageError):
"""An error occured in the ZEO Client Storage"""
class UnrecognizedResult(ClientStorageError):
- """A server call returned an unrecognized result
- """
+ """A server call returned an unrecognized result"""
-class ClientDisconnected(ClientStorageError):
- """The database storage is disconnected from the storage.
- """
+class ClientDisconnected(ClientStorageError, Disconnected):
[-=- -=- -=- 866 lines omitted -=- -=- -=-]
- _w.append(t)
- return t
+ return self._server.versions(max)
+
+ # below are methods invoked by the StorageServer
+
+ def serialnos(self, args):
+ self._serials.extend(args)
+
+ def info(self, dict):
+ self._info.update(dict)
+
+ def begin(self):
+ self._tfile = tempfile.TemporaryFile(suffix=".inv")
+ self._pickler = cPickle.Pickler(self._tfile, 1)
+ self._pickler.fast = 1 # Don't use the memo
+
+ def invalidate(self, args):
+ # Queue an invalidate for the end the transaction
+ if self._pickler is None:
+ return
+ self._pickler.dump(args)
+
+ def end(self):
+ if self._pickler is None:
+ return
+ self._pickler.dump((0,0))
+ self._tfile.seek(0)
+ unpick = cPickle.Unpickler(self._tfile)
+ f = self._tfile
+ self._tfile = None
+
+ while 1:
+ oid, version = unpick.load()
+ if not oid:
+ break
+ self._cache.invalidate(oid, version=version)
+ self._db.invalidate(oid, version=version)
+ f.close()
+
+ def Invalidate(self, args):
+ for oid, version in args:
+ self._cache.invalidate(oid, version=version)
+ try:
+ self._db.invalidate(oid, version=version)
+ except AttributeError, msg:
+ log2(PROBLEM,
+ "Invalidate(%s, %s) failed for _db: %s" % (repr(oid),
+ repr(version),
+ msg))
=== ZEO/ZEO/StorageServer.py 1.36 => 1.37 === (1035/1135 lines abridged)
#
##############################################################################
+"""Network ZODB storage server
-__version__ = "$Revision$"[11:-2]
+This server acts as a front-end for one or more real storages, like
+file storage or Berkeley storage.
-import asyncore, socket, string, sys, os
-import cPickle
-from cPickle import Unpickler
-from cStringIO import StringIO
-from thread import start_new_thread
-import time
-from types import StringType
+XXX Need some basic access control-- a declaration of the methods
+exported for invocation by the server.
+"""
-from ZODB import POSException
-from ZODB.POSException import TransactionError, UndoError, VersionCommitError
-from ZODB.Transaction import Transaction
+import asyncore
+import cPickle
+import os
+import sys
+import threading
+
+from ZEO import ClientStub
+from ZEO.CommitLog import CommitLog
+from ZEO.zrpc.server import Dispatcher
+from ZEO.zrpc.connection import ManagedServerConnection, Delay
+
+import zLOG
+from ZODB.POSException import StorageError, StorageTransactionError, \
+ TransactionError, ReadOnlyError
from ZODB.referencesf import referencesf
-from ZODB.utils import U64
-
-from ZEO import trigger
-from ZEO import asyncwrap
-from ZEO.smac import Disconnected, SizedMessageAsyncConnection
-from ZEO.logger import zLogger, format_msg
-
-class StorageServerError(POSException.StorageError):
- pass
+from ZODB.Transaction import Transaction
+from ZODB.TmpStore import TmpStore
# We create a special fast pickler! This allows us
[-=- -=- -=- 1035 lines omitted -=- -=- -=-]
+ self.log = CommitLog()
+ self.invalidated = []
+
+ # Store information about the call that blocks
+ self.name = None
+ self.args = None
+
+ def tpc_begin(self, txn, tid, status):
+ self.txn = txn
+ self.tid = tid
+ self.status = status
+
+ def store(self, oid, serial, data, version):
+ self.log.store(oid, serial, data, version)
+
+ def tpc_abort(self):
+ pass # just forget about this strategy
+
+ def tpc_finish(self):
+ raise RuntimeError, "Logic error. This method must not be called."
+
+ def tpc_vote(self):
+ self.name = "tpc_vote"
+ self.args = ()
+ return self.block()
+
+ def commitVersion(self, src, dest):
+ self.name = "commitVersion"
+ self.args = src, dest
+ return self.block()
+
+ def abortVersion(self, src):
+ self.name = "abortVersion"
+ self.args = src,
+ return self.block()
+
+ def transactionalUndo(self, trans_id):
+ self.name = "transactionalUndo"
+ self.args = trans_id,
+ return self.block()
+
+ def restart(self, new_strategy):
+ # called by the storage when the storage is available
+ new_strategy.tpc_begin(self.txn, self.tid, self.status)
+ loads, loader = self.log.get_loader()
+ for i in range(loads):
+ oid, serial, data, version = loader.load()
+ new_strategy.store(oid, serial, data, version)
+ meth = getattr(new_strategy, self.name)
+ return meth(*self.args)
=== ZEO/ZEO/__init__.py 1.8 => 1.9 ===
#
##############################################################################
-
-import fap
=== ZEO/ZEO/smac.py 1.16 => 1.17 ===
__version__ = "$Revision$"[11:-2]
-import asyncore, string, struct, zLOG, sys, Acquisition
+import asyncore, struct
+from Exceptions import Disconnected
+from zLOG import LOG, TRACE, ERROR, INFO, BLATHER
+from types import StringType
+
import socket, errno
-from logger import zLogger
# Use the dictionary to make sure we get the minimum number of errno
# entries. We expect that EWOULDBLOCK == EAGAIN on most systems --
@@ -38,81 +41,103 @@
expected_socket_write_errors = tuple(tmp_dict.keys())
del tmp_dict
-class SizedMessageAsyncConnection(Acquisition.Explicit, asyncore.dispatcher):
+class SizedMessageAsyncConnection(asyncore.dispatcher):
+ __super_init = asyncore.dispatcher.__init__
+ __super_close = asyncore.dispatcher.close
+
+ __closed = 1 # Marker indicating that we're closed
- __append=None # Marker indicating that we're closed
+ socket = None # to outwit Sam's getattr
- socket=None # to outwit Sam's getattr
+ READ_SIZE = 8096
def __init__(self, sock, addr, map=None, debug=None):
- SizedMessageAsyncConnection.inheritedAttribute(
- '__init__')(self, sock, map)
- self.addr=addr
- if debug is None and __debug__:
- self._debug = zLogger("smac")
- else:
+ self.addr = addr
+ if debug is not None:
self._debug = debug
- self.__state=None
- self.__inp=None
- self.__inpl=0
- self.__l=4
- self.__output=output=[]
- self.__append=output.append
- self.__pop=output.pop
-
- def handle_read(self,
- join=string.join, StringType=type(''), _type=type,
- _None=None):
-
+ elif not hasattr(self, '_debug'):
+ self._debug = __debug__ and 'smac'
+ self.__state = None
+ self.__inp = None # None, a single String, or a list
+ self.__input_len = 0
+ self.__msg_size = 4
+ self.__output = []
+ self.__closed = None
+ self.__super_init(sock, map)
+
+ # XXX avoid expensive getattr calls? Can't remember exactly what
+ # this comment was supposed to mean, but it has something to do
+ # with the way asyncore uses getattr and uses if sock:
+ def __nonzero__(self):
+ return 1
+
+ def handle_read(self):
+ # Use a single __inp buffer and integer indexes to make this
+ # fast.
try:
d=self.recv(8096)
except socket.error, err:
if err[0] in expected_socket_read_errors:
return
raise
- if not d: return
+ if not d:
+ return
- inp=self.__inp
- if inp is _None:
- inp=d
- elif _type(inp) is StringType:
- inp=[inp,d]
+ input_len = self.__input_len + len(d)
+ msg_size = self.__msg_size
+ state = self.__state
+
+ inp = self.__inp
+ if msg_size > input_len:
+ if inp is None:
+ self.__inp = d
+ elif type(self.__inp) is StringType:
+ self.__inp = [self.__inp, d]
+ else:
+ self.__inp.append(d)
+ self.__input_len = input_len
+ return # keep waiting for more input
+
+ # load all previous input and d into single string inp
+ if isinstance(inp, StringType):
+ inp = inp + d
+ elif inp is None:
+ inp = d
else:
inp.append(d)
+ inp = "".join(inp)
- inpl=self.__inpl+len(d)
- l=self.__l
-
- while 1:
-
- if l <= inpl:
- # Woo hoo, we have enough data
- if _type(inp) is not StringType: inp=join(inp,'')
- d=inp[:l]
- inp=inp[l:]
- inpl=inpl-l
- if self.__state is _None:
- # waiting for message
- l=struct.unpack(">i",d)[0]
- self.__state=1
- else:
- l=4
- self.__state=_None
- self.message_input(d)
+ offset = 0
+ while (offset + msg_size) <= input_len:
+ msg = inp[offset:offset + msg_size]
+ offset = offset + msg_size
+ if state is None:
+ # waiting for message
+ msg_size = struct.unpack(">i", msg)[0]
+ state = 1
else:
- break # not enough data
-
- self.__l=l
- self.__inp=inp
- self.__inpl=inpl
-
- def readable(self): return 1
- def writable(self): return not not self.__output
+ msg_size = 4
+ state = None
+ self.message_input(msg)
+
+ self.__state = state
+ self.__msg_size = msg_size
+ self.__inp = inp[offset:]
+ self.__input_len = input_len - offset
+
+ def readable(self):
+ return 1
+
+ def writable(self):
+ if len(self.__output) == 0:
+ return 0
+ else:
+ return 1
def handle_write(self):
- output=self.__output
+ output = self.__output
while output:
- v=output[0]
+ v = output[0]
try:
n=self.send(v)
except socket.error, err:
@@ -120,37 +145,33 @@
break # we couldn't write anything
raise
if n < len(v):
- output[0]=v[n:]
+ output[0] = v[n:]
break # we can't write any more
else:
del output[0]
- #break # waaa
-
def handle_close(self):
self.close()
- def message_output(self, message,
- pack=struct.pack, len=len):
- if self._debug is not None:
- if len(message) > 40:
- m = message[:40]+' ...'
- else:
- m = message
- self._debug.trace('message_output %s' % `m`)
+ def message_output(self, message):
+ if __debug__:
+ if self._debug:
+ if len(message) > 40:
+ m = message[:40]+' ...'
+ else:
+ m = message
+ LOG(self._debug, TRACE, 'message_output %s' % `m`)
- append=self.__append
- if append is None:
- raise Disconnected("This action is temporarily unavailable.")
-
- append(pack(">i",len(message))+message)
+ if self.__closed is not None:
+ raise Disconnected, (
+ "This action is temporarily unavailable."
+ "
"
+ )
+ # do two separate appends to avoid copying the message string
+ self.__output.append(struct.pack(">i", len(message)))
+ self.__output.append(message)
def close(self):
- if self.__append is not None:
- self.__append=None
- SizedMessageAsyncConnection.inheritedAttribute('close')(self)
-
-class Disconnected(Exception):
- """The client has become disconnected from the server
- """
-
+ if self.__closed is None:
+ self.__closed = 1
+ self.__super_close()
=== ZEO/ZEO/start.py 1.32 => 1.33 ===
#
##############################################################################
-
"""Start the server storage.
"""
@@ -19,13 +18,16 @@
import sys, os, getopt, string
+import StorageServer
+import asyncore
+
def directory(p, n=1):
d=p
while n:
d=os.path.split(d)[0]
if not d or d=='.': d=os.getcwd()
n=n-1
-
+
return d
def get_storage(m, n, cache={}):
@@ -44,9 +46,11 @@
def main(argv):
me=argv[0]
- sys.path[:]==filter(None, sys.path)
sys.path.insert(0, directory(me, 2))
+ # XXX hack for profiling support
+ global unix, storages, zeo_pid, asyncore
+
args=[]
last=''
for a in argv[1:]:
@@ -77,23 +81,22 @@
fs = os.path.join(var, 'Data.fs')
- usage = """%s [options] [filename]
+ usage="""%s [options] [filename]
where options are:
-D -- Run in debug mode
- -d -- Generate detailed debug logging without running
- in the foreground.
+ -d -- Set STUPD_LOG_SEVERITY to -300
-U -- Unix-domain socket file to listen on
-
+
-u username or uid number
The username to run the ZEO server as. You may want to run
the ZEO server as 'nobody' or some other user with limited
- resouces. The only works under Unix, and if the storage
- server is started by root.
+ resouces. The only works under Unix, and if ZServer is
+ started by root.
-p port -- port to listen on
@@ -116,30 +119,47 @@
attr_name -- This is the name to which the storage object
is assigned in the module.
+ -P file -- Run under profile and dump output to file. Implies the
+ -s flag.
+
if no file name is specified, then %s is used.
""" % (me, fs)
try:
- opts, args = getopt.getopt(args, 'p:Ddh:U:sS:u:')
- except getopt.error, err:
- print err
+ opts, args = getopt.getopt(args, 'p:Dh:U:sS:u:P:d')
+ except getopt.error, msg:
print usage
+ print msg
sys.exit(1)
- port=None
- debug=detailed=0
- host=''
- unix=None
- Z=1
- UID='nobody'
+ port = None
+ debug = 0
+ host = ''
+ unix =None
+ Z = 1
+ UID = 'nobody'
+ prof = None
+ detailed = 0
for o, v in opts:
- if o=='-p': port=string.atoi(v)
- elif o=='-h': host=v
- elif o=='-U': unix=v
- elif o=='-u': UID=v
- elif o=='-D': debug=1
- elif o=='-d': detailed=1
- elif o=='-s': Z=0
+ if o=='-p':
+ port = int(v)
+ elif o=='-h':
+ host = v
+ elif o=='-U':
+ unix = v
+ elif o=='-u':
+ UID = v
+ elif o=='-D':
+ debug = 1
+ elif o=='-d':
+ detailed = 1
+ elif o=='-s':
+ Z = 0
+ elif o=='-P':
+ prof = v
+
+ if prof:
+ Z = 0
if port is None and unix is None:
print usage
@@ -153,14 +173,16 @@
sys.exit(1)
fs=args[0]
- if debug: os.environ['Z_DEBUG_MODE']='1'
-
- if detailed: os.environ['STUPID_LOG_SEVERITY']='-99999'
+ __builtins__.__debug__=debug
+ if debug:
+ os.environ['Z_DEBUG_MODE'] = '1'
+ if detailed:
+ os.environ['STUPID_LOG_SEVERITY'] = '-300'
from zLOG import LOG, INFO, ERROR
# Try to set uid to "-u" -provided uid.
- # Try to set gid to "-u" user's primary group.
+ # Try to set gid to "-u" user's primary group.
# This will only work if this script is run by root.
try:
import pwd
@@ -175,7 +197,7 @@
uid = pwd.getpwuid(UID)[2]
gid = pwd.getpwuid(UID)[3]
else:
- raise KeyError
+ raise KeyError
try:
if gid is not None:
try:
@@ -200,7 +222,7 @@
try:
import ZEO.StorageServer, asyncore
-
+
storages={}
for o, v in opts:
if o=='-S':
@@ -243,15 +265,15 @@
if not unix: unix=host, port
- ZEO.StorageServer.StorageServer(unix, storages)
-
+ StorageServer.StorageServer(unix, storages)
+
try:
ppid, pid = os.getppid(), os.getpid()
except:
pass # getpid not supported
else:
open(zeo_pid,'w').write("%s %s" % (ppid, pid))
-
+
except:
# Log startup exception and tell zdaemon not to restart us.
info = sys.exc_info()
@@ -269,7 +291,6 @@
asyncore.loop()
-
def rotate_logs():
import zLOG
if hasattr(zLOG.log_write, 'reinitialize'):
@@ -292,29 +313,21 @@
# unnecessary, since we now use so_reuseaddr.
for ignored in 1,2:
for socket in asyncore.socket_map.values():
- try:
- socket.close()
- except:
- pass
+ try: socket.close()
+ except: pass
for storage in storages.values():
- try:
- storage.close()
- except:
- pass
+ try: storage.close()
+ finally: pass
try:
from zLOG import LOG, INFO
LOG('ZEO Server', INFO,
"Shutting down (%s)" % (die and "shutdown" or "restart")
)
- except:
- pass
-
- if die:
- sys.exit(0)
- else:
- sys.exit(1)
+ except: pass
+
+ if die: sys.exit(0)
+ else: sys.exit(1)
-if __name__ == '__main__':
- main(sys.argv)
+if __name__=='__main__': main(sys.argv)
=== ZEO/ZEO/trigger.py 1.5 => 1.6 ===
#
##############################################################################
-
-# This module is a simplified version of the select_trigger module
-# from Sam Rushing's Medusa server.
-
import asyncore
-import errno
+
import os
import socket
import string
import thread
-
+
if os.name == 'posix':
- class trigger(asyncore.file_dispatcher):
+ class trigger (asyncore.file_dispatcher):
"Wake up a call to select() running in the main thread"
@@ -56,46 +52,50 @@
# new data onto a channel's outgoing data queue at the same time that
# the main thread is trying to remove some]
- def __init__(self):
+ def __init__ (self):
r, w = self._fds = os.pipe()
self.trigger = w
- asyncore.file_dispatcher.__init__(self, r)
+ asyncore.file_dispatcher.__init__ (self, r)
self.lock = thread.allocate_lock()
self.thunks = []
+ self._closed = None
- def __del__(self):
- os.close(self._fds[0])
- os.close(self._fds[1])
+ # Override the asyncore close() method, because it seems that
+ # it would only close the r file descriptor and not w. The
+ # constructor calls file_dispactcher.__init__ and passes r,
+ # which would get stored in a file_wrapper and get closed by
+ # the default close. But that would leave w open...
+
+ def close(self):
+ if self._closed is None:
+ self._closed = 1
+ self.del_channel()
+ for fd in self._fds:
+ os.close(fd)
- def __repr__(self):
- return '' % id(self)
+ def __repr__ (self):
+ return '' % id(self)
- def readable(self):
+ def readable (self):
return 1
- def writable(self):
+ def writable (self):
return 0
- def handle_connect(self):
+ def handle_connect (self):
pass
- def pull_trigger(self, thunk=None):
- # print 'PULL_TRIGGER: ', len(self.thunks)
+ def pull_trigger (self, thunk=None):
if thunk:
try:
self.lock.acquire()
- self.thunks.append(thunk)
+ self.thunks.append (thunk)
finally:
self.lock.release()
- os.write(self.trigger, 'x')
+ os.write (self.trigger, 'x')
- def handle_read(self):
- try:
- self.recv(8192)
- except os.error, err:
- if err[0] == errno.EAGAIN: # resource temporarily unavailable
- return
- raise
+ def handle_read (self):
+ self.recv (8192)
try:
self.lock.acquire()
for thunk in self.thunks:
@@ -104,7 +104,7 @@
except:
nil, t, v, tbinfo = asyncore.compact_traceback()
print ('exception in trigger thunk:'
- '(%s:%s %s)' % (t, v, tbinfo))
+ ' (%s:%s %s)' % (t, v, tbinfo))
self.thunks = []
finally:
self.lock.release()
@@ -116,13 +116,13 @@
# win32-safe version
- class trigger(asyncore.dispatcher):
+ class trigger (asyncore.dispatcher):
address = ('127.9.9.9', 19999)
- def __init__(self):
- a = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
- w = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+ def __init__ (self):
+ a = socket.socket (socket.AF_INET, socket.SOCK_STREAM)
+ w = socket.socket (socket.AF_INET, socket.SOCK_STREAM)
# set TCP_NODELAY to true to avoid buffering
w.setsockopt(socket.IPPROTO_TCP, 1, 1)
@@ -139,51 +139,46 @@
if port <= 19950:
raise 'Bind Error', 'Cannot bind trigger!'
port=port - 1
-
- a.listen(1)
- w.setblocking(0)
+
+ a.listen (1)
+ w.setblocking (0)
try:
- w.connect(self.address)
+ w.connect (self.address)
except:
pass
r, addr = a.accept()
a.close()
- w.setblocking(1)
+ w.setblocking (1)
self.trigger = w
- asyncore.dispatcher.__init__(self, r)
+ asyncore.dispatcher.__init__ (self, r)
self.lock = thread.allocate_lock()
self.thunks = []
self._trigger_connected = 0
- def __repr__(self):
+ def __repr__ (self):
return '' % id(self)
- def readable(self):
+ def readable (self):
return 1
- def writable(self):
+ def writable (self):
return 0
- def handle_connect(self):
+ def handle_connect (self):
pass
- def pull_trigger(self, thunk=None):
+ def pull_trigger (self, thunk=None):
if thunk:
try:
self.lock.acquire()
- self.thunks.append(thunk)
+ self.thunks.append (thunk)
finally:
self.lock.release()
- self.trigger.send('x')
+ self.trigger.send ('x')
- def handle_read(self):
- try:
- self.recv(8192)
- except os.error, err:
- if err[0] == errno.EAGAIN: # resource temporarily unavailable
- return
- raise
+ def handle_read (self):
+ self.recv (8192)
try:
self.lock.acquire()
for thunk in self.thunks:
From jeremy at zope.com Tue Jun 11 09:47:43 2002
From: jeremy at zope.com (Jeremy Hylton)
Date: Sun Aug 10 16:31:21 2008
Subject: [ZEO-Checkins] CVS: ZEO/ZEO - Invalidator.py:NONE asyncwrap.py:NONE fap.py:NONE zrpc.py:NONE
Message-ID: <200206111347.g5BDlhS06874@cvs.baymountain.com>
Update of /cvs-repository/ZEO/ZEO
In directory cvs.zope.org:/tmp/cvs-serv6865
Removed Files:
Invalidator.py asyncwrap.py fap.py zrpc.py
Log Message:
Merge ZEO2-branch to trunk. (File removal.)
=== Removed File ZEO/ZEO/Invalidator.py ===
=== Removed File ZEO/ZEO/asyncwrap.py ===
=== Removed File ZEO/ZEO/fap.py ===
=== Removed File ZEO/ZEO/zrpc.py ===
From jeremy at zope.com Tue Jun 11 15:22:27 2002
From: jeremy at zope.com (Jeremy Hylton)
Date: Sun Aug 10 16:31:22 2008
Subject: [ZEO-Checkins] CVS: ZEO/ZEO/zrpc - NOTES:1.2 __init__.py:1.2 client.py:1.2 connection.py:1.2 error.py:1.2 log.py:1.2 marshal.py:1.2 server.py:1.2 trigger.py:1.2
Message-ID: <200206111922.g5BJMRd15268@cvs.baymountain.com>
Update of /cvs-repository/ZEO/ZEO/zrpc
In directory cvs.zope.org:/tmp/cvs-serv15241/zrpc
Added Files:
NOTES __init__.py client.py connection.py error.py log.py
marshal.py server.py trigger.py
Log Message:
Merge ZEO2-branch to trunk. (Files added on branch.)
=== ZEO/ZEO/zrpc/NOTES 1.1 => 1.2 ===
+handling for outstanding calls. In particular, it should be possible
+to have multiple calls with return values outstanding.
+
+The mechanism described here is based on the promises mechanism in
+Argus, which was influenced by futures in Multilisp.
+
+ Promises: Linguistic Support for Efficient Asynchronous Procedure
+ Calls in Distributed Systems. Barbara Liskov and Liuba Shrira.
+ Proc. of Conf. on Programming Language Design and Implementation
+ (PLDI), June 1988.
+
+We want to support two different kinds of calls:
+
+ - send : invoke a method that returns no value
+ - call : invoke a method that returns a value
+
+On the client, a call immediately returns a promise. A promise is an
+object that can be used to claim the return value when it becomes
+available.
+
+ - ready(): returns true if the return value is ready or an exception
+ occurred
+ - claim(): returns the call's return value or raises an exception,
+ blocking if necessary
+
+The server side of a zrpc connection can be implemented using
+asyncore. In that case, a method call blocks other RPC activity until
+it returns. If a call needs to return a value, but can't return
+immediately, it returns a delay object (ZEO.zrpc.server.Delay).
+
+When the zrpc connection receives a Delay object, it does not
+immediately return to the caller. Instead, it returns when the
+reply() method is called. A Delay has two methods:
+
+ - set_sender()
+ - reply(obj): returns obj to the sender
+
+-----------------------------------------
+
+Open issues:
+
+Delayed exception
+
+There is currently no mechanism to raise an exception from a delayed
+pcall.
+
+Synchronization
+
+The following item is part of Argus, but the motivation isn't entirely
+clear.
+
+ For any two calls, C1 and C2, C1 always starts on the server
+ first. For the promises, C2 is ready() iff C1 is also ready().
+ The promises can be claimed in any order.
+
+A related notion:
+
+ The connection should also support a synch() method that returns
+ only when all outstanding calls have completed. If any of these
+ calls raised an exception, the synch() call raises an exception.
+
+XXX synch() sounds potentially useful, but it's not clear if it would
+be useful for ZEO. In ZEO a single connection object handles multiple
+threads, each thread is going to make independent calls. When a
+particular tpc_begin() returns and a thread commits its transaction,
+it makes more calls. These calls will before any of the other
+tpc_begin() calls.
+
+I think the Argus approach would be to use separate handlers for each
+thread (not sure Argus had threads), so that a single thread could
+rely on ordering guarantees.
+
+Multithreaded server
+
+There are lots of issues to work out here.
+
+Delays may not be necessary if the connecftion handler runs in a
+different thread than the object the handles the calls.
\ No newline at end of file
=== ZEO/ZEO/zrpc/__init__.py 1.1 => 1.2 ===
+#
+# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE
+#
+##############################################################################
+# zrpc is a package with the following modules
+# error -- exceptions raised by zrpc
+# marshal -- internal, handles basic protocol issues
+# connection -- object dispatcher
+# client -- manages connection creation to remote server
+# server -- manages incoming connections from remote clients
+# trigger -- medusa's trigger
=== ZEO/ZEO/zrpc/client.py 1.1 => 1.2 ===
+#
+# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE
+#
+##############################################################################
+import errno
+import select
+import socket
+import sys
+import threading
+import time
+import types
+
+import ThreadedAsync
+import zLOG
+
+from ZEO.zrpc.log import log
+from ZEO.zrpc.trigger import trigger
+from ZEO.zrpc.connection import ManagedConnection
+
+class ConnectionManager:
+ """Keeps a connection up over time"""
+
+ def __init__(self, addr, client, tmin=1, tmax=180):
+ self.set_addr(addr)
+ self.client = client
+ self.tmin = tmin
+ self.tmax = tmax
+ self.connected = 0
+ self.connection = None
+ self.closed = 0
+ # If _thread is not None, then there is a helper thread
+ # attempting to connect. _thread is protected by _connect_lock.
+ self._thread = None
+ self._connect_lock = threading.Lock()
+ self.trigger = None
+ self.thr_async = 0
+ ThreadedAsync.register_loop_callback(self.set_async)
+
+ def __repr__(self):
+ return "<%s for %s>" % (self.__class__.__name__, self.addr)
+
+ def set_addr(self, addr):
+ "Set one or more addresses to use for server."
+
+ # For backwards compatibility (and simplicity?) the
+ # constructor accepts a single address in the addr argument --
+ # a string for a Unix domain socket or a 2-tuple with a
+ # hostname and port. It can also accept a list of such addresses.
+
+ addr_type = self._guess_type(addr)
+ if addr_type is not None:
+ self.addr = [(addr_type, addr)]
+ else:
+ self.addr = []
+ for a in addr:
+ addr_type = self._guess_type(a)
+ if addr_type is None:
+ raise ValueError, "unknown address in list: %s" % repr(a)
+ self.addr.append((addr_type, a))
+
+ def _guess_type(self, addr):
+ if isinstance(addr, types.StringType):
+ return socket.AF_UNIX
+
+ if (len(addr) == 2
+ and isinstance(addr[0], types.StringType)
+ and isinstance(addr[1], types.IntType)):
+ return socket.AF_INET
+
+ # not anything I know about
+ return None
+
+ def close(self):
+ """Prevent ConnectionManager from opening new connections"""
+ self.closed = 1
+ self._connect_lock.acquire()
+ try:
+ if self._thread is not None:
+ # XXX race on _thread
+ self._thread.stop()
+ self._thread.join()
+ finally:
+ self._connect_lock.release()
+ if self.connection:
+ self.connection.close()
+ if self.trigger is not None:
+ self.trigger.close()
+
+ def set_async(self, map):
+ # This is the callback registered with ThreadedAsync. The
+ # callback might be called multiple times, so it shouldn't
+ # create a trigger every time and should never do anything
+ # after it's closed.
+
+ # It may be that the only case where it is called multiple
+ # times is in the test suite, where ThreadedAsync's loop can
+ # be started in a child process after a fork. Regardless,
+ # it's good to be defensive.
+
+ # XXX need each connection started with async==0 to have a
+ # callback
+ if not self.closed and self.trigger is None:
+ self.trigger = trigger()
+ self.thr_async = 1 # XXX needs to be set on the Connection
+
+ def attempt_connect(self):
+ """Attempt a connection to the server without blocking too long.
+
+ There isn't a crisp definition for too long. When a
+ ClientStorage is created, it attempts to connect to the
+ server. If the server isn't immediately available, it can
+ operate from the cache. This method will start the background
+ connection thread and wait a little while to see if it
+ finishes quickly.
+ """
+
+ # XXX will a single attempt take too long?
+ self.connect()
+ try:
+ event = self._thread.one_attempt
+ except AttributeError:
+ # An AttributeError means that (1) _thread is None and (2)
+ # as a consquence of (1) that the connect thread has
+ # already exited.
+ pass
+ else:
+ event.wait()
+ return self.connected
+
+ def connect(self, sync=0):
+ if self.connected == 1:
+ return
+ self._connect_lock.acquire()
+ try:
+ if self._thread is None:
+ log("starting thread to connect to server")
+ self._thread = ConnectThread(self, self.client, self.addr,
+ self.tmin, self.tmax)
+ self._thread.start()
+ if sync:
+ try:
+ self._thread.join()
+ except AttributeError:
+ # probably means the thread exited quickly
+ pass
+ finally:
+ self._connect_lock.release()
+
+ def connect_done(self, c):
+ log("connect_done()")
+ self.connected = 1
+ self.connection = c
+ self._thread = None
+
+ def notify_closed(self, conn):
+ self.connected = 0
+ self.connection = None
+ self.client.notifyDisconnected()
+ if not self.closed:
+ self.connect()
+
+class Connected(Exception):
+ # helper for non-local exit
+ def __init__(self, sock):
+ self.sock = sock
+
+# When trying to do a connect on a non-blocking socket, some outcomes
+# are expected. Set _CONNECT_IN_PROGRESS to the errno value(s) expected
+# when an initial connect can't complete immediately. Set _CONNECT_OK
+# to the errno value(s) expected if the connect succeeds *or* if it's
+# already connected (our code can attempt redundant connects).
+if hasattr(errno, "WSAEWOULDBLOCK"): # Windows
+ _CONNECT_IN_PROGRESS = (errno.WSAEWOULDBLOCK,)
+ _CONNECT_OK = (0, errno.WSAEISCONN)
+else: # Unix
+ _CONNECT_IN_PROGRESS = (errno.EINPROGRESS,)
+ _CONNECT_OK = (0, errno.EISCONN)
+
+class ConnectThread(threading.Thread):
+ """Thread that tries to connect to server given one or more addresses.
+ The thread is passed a ConnectionManager and the manager's client
+ as arguments. It calls notifyConnected() on the client when a
+ socket connects. If notifyConnected() returns without raising an
+ exception, the thread is done; it calls connect_done() on the
+ manager and exits.
+
+ The thread will continue to run, attempting connections, until a
+ successful notifyConnected() or stop() is called.
+ """
+
+ __super_init = threading.Thread.__init__
+
+ # We don't expect clients to call any methods of this Thread other
+ # than close() and those defined by the Thread API.
+
+ def __init__(self, mgr, client, addrs, tmin, tmax):
+ self.__super_init(name="Connect(%s)" % addrs)
+ self.mgr = mgr
+ self.client = client
+ self.addrs = addrs
+ self.tmin = tmin
+ self.tmax = tmax
+ self.stopped = 0
+ self.one_attempt = threading.Event()
+ # A ConnectThread keeps track of whether it has finished a
+ # call to attempt_connects(). This allows the
+ # ConnectionManager to make an attempt to connect right away,
+ # but not block for too long if the server isn't immediately
+ # available.
+
+ def stop(self):
+ self.stopped = 1
+
+ # Every method from run() to the end is used internally by the Thread.
+
+ def run(self):
+ delay = self.tmin
+ while not self.stopped:
+ success = self.attempt_connects()
+ if not self.one_attempt.isSet():
+ self.one_attempt.set()
+ if success:
+ break
+ time.sleep(delay)
+ delay *= 2
+ if delay > self.tmax:
+ delay = self.tmax
+ log("thread exiting: %s" % self.getName())
+
+ def close_sockets(self):
+ for s in self.sockets.keys():
+ s.close()
+
+ def attempt_connects(self):
+ """Try connecting to all self.addrs addresses.
+
+ If at least one succeeds, pick a success arbitrarily, close all other
+ successes (if any), and return true. If none succeed, return false.
+ """
+
+ self.sockets = {} # {open socket: connection address}
+
+ log("attempting connection on %d sockets" % len(self.addrs))
+ try:
+ for domain, addr in self.addrs:
+ if __debug__:
+ log("attempt connection to %s" % repr(addr),
+ level=zLOG.DEBUG)
+ try:
+ s = socket.socket(domain, socket.SOCK_STREAM)
+ except socket.error, err:
+ log("Failed to create socket with domain=%s: %s" % (
+ domain, err), level=zLOG.ERROR)
+ continue
+ s.setblocking(0)
+ self.sockets[s] = addr
+ # connect() raises Connected iff it succeeds
+ # XXX can still block for a while if addr requires DNS
+ self.connect(s)
+
+ # next wait until they actually connect
+ while self.sockets:
+ if self.stopped:
+ self.close_sockets()
+ return 0
+ try:
+ sockets = self.sockets.keys()
+ r, w, x = select.select([], sockets, sockets, 1.0)
+ except select.error:
+ continue
+ for s in x:
+ del self.sockets[s]
+ s.close()
+ for s in w:
+ # connect() raises Connected iff it succeeds
+ self.connect(s)
+ except Connected, container:
+ s = container.sock
+ del self.sockets[s] # don't close the newly connected socket
+ self.close_sockets()
+ return 1
+ return 0
+
+ def connect(self, s):
+ """Call s.connect_ex(addr); raise Connected iff connection succeeds.
+
+ We have to handle several possible return values from
+ connect_ex(). If the socket is connected and the initial ZEO
+ setup works, we're done. Report success by raising an
+ exception. Yes, the is odd, but we need to bail out of the
+ select() loop in the caller and an exception is a principled
+ way to do the abort.
+
+ If the socket sonnects and the initial ZEO setup
+ (notifyConnected()) fails or the connect_ex() returns an
+ error, we close the socket, remove it from self.sockets, and
+ proceed with the other sockets.
+
+ If connect_ex() returns EINPROGRESS, we need to try again later.
+ """
+ addr = self.sockets[s]
+ try:
+ e = s.connect_ex(addr)
+ except socket.error, msg:
+ log("failed to connect to %s: %s" % (addr, msg),
+ level=zLOG.ERROR)
+ else:
+ log("connect_ex(%s) == %s" % (addr, e))
+ if e in _CONNECT_IN_PROGRESS:
+ return
+ elif e in _CONNECT_OK:
+ # special cases to deal with winsock oddities
+ if sys.platform.startswith("win") and e == 0:
+
+ # It appears that winsock isn't behaving as
+ # expected on Win2k. It's possible for connect()
+ # to return 0, but the connection to have failed.
+ # In particular, in situations where I expect to
+ # get a Connection refused (10061), I'm seeing
+ # connect_ex() return 0. OTOH, it looks like
+ # select() is a more reliable indicator on
+ # Windows.
+
+ r, w, x = select.select([s], [s], [s], 0.1)
+ if not (r or w or x):
+ return
+ if x:
+ # see comment at the end of the function
+ s.close()
+ del self.socket[s]
+ c = self.test_connection(s, addr)
+ if c:
+ log("connected to %s" % repr(addr), level=zLOG.DEBUG)
+ raise Connected(s)
+ else:
+ log("error connecting to %s: %s" % (addr, errno.errorcode[e]),
+ level=zLOG.DEBUG)
+ # Any execution that doesn't raise Connected() or return
+ # because of CONNECT_IN_PROGRESS is an error. Make sure the
+ # socket is closed and remove it from the dict of pending
+ # sockets.
+ s.close()
+ del self.sockets[s]
+
+ def test_connection(self, s, addr):
+ # Establish a connection at the zrpc level and call the
+ # client's notifyConnected(), giving the zrpc application a
+ # chance to do app-level check of whether the connection is
+ # okay.
+ c = ManagedConnection(s, addr, self.client, self.mgr)
+ try:
+ self.client.notifyConnected(c)
+ except:
+ log("error connecting to server: %s" % str(addr),
+ level=zLOG.ERROR, error=sys.exc_info())
+ c.close()
+ # Closing the ZRPC connection will eventually close the
+ # socket, somewhere in asyncore.
+ return 0
+ self.mgr.connect_done(c)
+ return 1
=== ZEO/ZEO/zrpc/connection.py 1.1 => 1.2 ===
+#
+# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE
+#
+##############################################################################
+import asyncore
+import sys
+import threading
+import types
+
+import ThreadedAsync
+from ZEO import smac # XXX put smac in zrpc?
+from ZEO.zrpc.error import ZRPCError, DisconnectedError, DecodingError
+from ZEO.zrpc.log import log, short_repr
+from ZEO.zrpc.marshal import Marshaller
+from ZEO.zrpc.trigger import trigger
+import zLOG
+from ZODB import POSException
+
+REPLY = ".reply" # message name used for replies
+ASYNC = 1
+
+class Delay:
+ """Used to delay response to client for synchronous calls
+
+ When a synchronous call is made and the original handler returns
+ without handling the call, it returns a Delay object that prevents
+ the mainloop from sending a response.
+ """
+
+ def set_sender(self, msgid, send_reply):
+ self.msgid = msgid
+ self.send_reply = send_reply
+
+ def reply(self, obj):
+ self.send_reply(self.msgid, obj)
+
+class Connection(smac.SizedMessageAsyncConnection):
+ """Dispatcher for RPC on object on both sides of socket.
+
+ The connection supports synchronous calls, which expect a return,
+ and asynchronous calls that do not.
+
+ It uses the Marshaller class to handle encoding and decoding of
+ method calls are arguments.
+
+ A Connection is designed for use in a multithreaded application,
+ where a synchronous call must block until a response is ready.
+ The current design only allows a single synchronous call to be
+ outstanding.
+
+ A socket connection between a client and a server allows either
+ side to invoke methods on the other side. The processes on each
+ end of the socket use a Connection object to manage communication.
+ """
+
+ __super_init = smac.SizedMessageAsyncConnection.__init__
+ __super_close = smac.SizedMessageAsyncConnection.close
+ __super_writable = smac.SizedMessageAsyncConnection.writable
+ __super_message_output = smac.SizedMessageAsyncConnection.message_output
+
+ protocol_version = "Z200"
+
+ def __init__(self, sock, addr, obj=None):
+ self.obj = None
+ self.marshal = Marshaller()
+ self.closed = 0
+ self.msgid = 0
+ self.__super_init(sock, addr)
+ # A Connection either uses asyncore directly or relies on an
+ # asyncore mainloop running in a separate thread. If
+ # thr_async is true, then the mainloop is running in a
+ # separate thread. If thr_async is true, then the asyncore
+ # trigger (self.trigger) is used to notify that thread of
+ # activity on the current thread.
+ self.thr_async = 0
+ self.trigger = None
+ self._prepare_async()
+ self._map = {self._fileno: self}
+ self.__call_lock = threading.Lock()
+ # The reply lock is used to block when a synchronous call is
+ # waiting for a response
+ self.__reply_lock = threading.Lock()
+ self.__reply_lock.acquire()
+ self.register_object(obj)
+ self.handshake()
+
+ def __repr__(self):
+ return "<%s %s>" % (self.__class__.__name__, self.addr)
+
+ def close(self):
+ if self.closed:
+ return
+ self.closed = 1
+ self.close_trigger()
+ self.__super_close()
+
+ def close_trigger(self):
+ # overridden by ManagedConnection
+ if self.trigger is not None:
+ self.trigger.close()
+
+ def register_object(self, obj):
+ """Register obj as the true object to invoke methods on"""
+ self.obj = obj
+
+ def handshake(self):
+ # When a connection is created the first message sent is a
+ # 4-byte protocol version. This mechanism should allow the
+ # protocol to evolve over time, and let servers handle clients
+ # using multiple versions of the protocol.
+
+ # The mechanism replace the message_input() method for the
+ # first message received.
+
+ # The client sends the protocol version it is using.
+ self._message_input = self.message_input
+ self.message_input = self.recv_handshake
+ self.message_output(self.protocol_version)
+
+ def recv_handshake(self, message):
+ if message == self.protocol_version:
+ self.message_input = self._message_input
+ # otherwise do something else...
+
+ def message_input(self, message):
+ """Decoding an incoming message and dispatch it"""
+ # XXX Not sure what to do with errors that reach this level.
+ # Need to catch ZRPCErrors in handle_reply() and
+ # handle_request() so that they get back to the client.
+ try:
+ msgid, flags, name, args = self.marshal.decode(message)
+ except DecodingError, msg:
+ return self.return_error(None, None, DecodingError, msg)
+
+ if __debug__:
+ log("recv msg: %s, %s, %s, %s" % (msgid, flags, name,
+ short_repr(args)),
+ level=zLOG.DEBUG)
+ if name == REPLY:
+ self.handle_reply(msgid, flags, args)
+ else:
+ self.handle_request(msgid, flags, name, args)
+
+ def handle_reply(self, msgid, flags, args):
+ if __debug__:
+ log("recv reply: %s, %s, %s" % (msgid, flags, str(args)[:40]),
+ level=zLOG.DEBUG)
+ self.__reply = msgid, flags, args
+ self.__reply_lock.release() # will fail if lock is unlocked
+
+ def handle_request(self, msgid, flags, name, args):
+ if not self.check_method(name):
+ msg = "Invalid method name: %s on %s" % (name, repr(self.obj))
+ raise ZRPCError(msg)
+ if __debug__:
+ log("%s%s" % (name, args), level=zLOG.BLATHER)
+
+ meth = getattr(self.obj, name)
+ try:
+ ret = meth(*args)
+ except Exception, msg:
+ error = sys.exc_info()[:2]
+ log("%s() raised exception: %s" % (name, msg), zLOG.ERROR, error)
+ return self.return_error(msgid, flags, error[0], error[1])
+
+ if flags & ASYNC:
+ if ret is not None:
+ raise ZRPCError("async method %s returned value %s" %
+ (name, repr(ret)))
+ else:
+ if __debug__:
+ log("%s return %s" % (name, short_repr(ret)), zLOG.DEBUG)
+ if isinstance(ret, Delay):
+ ret.set_sender(msgid, self.send_reply)
+ else:
+ self.send_reply(msgid, ret)
+
+ def handle_error(self):
+ self.log_error()
+ self.close()
+
+ def log_error(self, msg="No error message supplied"):
+ log(msg, zLOG.ERROR, error=sys.exc_info())
+
+ def check_method(self, name):
+ # XXX Is this sufficient "security" for now?
+ if name.startswith('_'):
+ return None
+ return hasattr(self.obj, name)
+
+ def send_reply(self, msgid, ret):
+ msg = self.marshal.encode(msgid, 0, REPLY, ret)
+ self.message_output(msg)
+
+ def return_error(self, msgid, flags, err_type, err_value):
+ if flags is None:
+ self.log_error("Exception raised during decoding")
+ return
+ if flags & ASYNC:
+ self.log_error("Asynchronous call raised exception: %s" % self)
+ return
+ if type(err_value) is not types.InstanceType:
+ err_value = err_type, err_value
+
+ try:
+ msg = self.marshal.encode(msgid, 0, REPLY, (err_type, err_value))
+ except self.marshal.errors:
+ err = ZRPCError("Couldn't pickle error %s" % `err_value`)
+ msg = self.marshal.encode(msgid, 0, REPLY, (ZRPCError, err))
+ self.message_output(msg)
+ self._do_async_poll()
+
+ # The next two public methods (call and callAsync) are used by
+ # clients to invoke methods on remote objects
+
+ def call(self, method, *args):
+ self.__call_lock.acquire()
+ try:
+ return self._call(method, args)
+ finally:
+ self.__call_lock.release()
+
+ def _call(self, method, args):
+ if self.closed:
+ raise DisconnectedError("This action is temporarily unavailable")
+ msgid = self.msgid
+ self.msgid = self.msgid + 1
+ if __debug__:
+ log("send msg: %d, 0, %s, ..." % (msgid, method))
+ self.message_output(self.marshal.encode(msgid, 0, method, args))
+
+ # XXX implementation of promises starts here
+
+ self.__reply = None
+ # reply lock is currently held
+ self._do_async_loop()
+ # reply lock is held again...
+ r_msgid, r_flags, r_args = self.__reply
+ self.__reply_lock.acquire()
+ assert r_msgid == msgid, "%s != %s: %s" % (r_msgid, msgid, r_args)
+
+ if type(r_args) == types.TupleType \
+ and type(r_args[0]) == types.ClassType \
+ and issubclass(r_args[0], Exception):
+ raise r_args[1] # error raised by server
+ return r_args
+
+ def callAsync(self, method, *args):
+ self.__call_lock.acquire()
+ try:
+ self._callAsync(method, args)
+ finally:
+ self.__call_lock.release()
+
+ def _callAsync(self, method, args):
+ if self.closed:
+ raise DisconnectedError("This action is temporarily unavailable")
+ msgid = self.msgid
+ self.msgid += 1
+ if __debug__:
+ log("send msg: %d, %d, %s, ..." % (msgid, ASYNC, method))
+ self.message_output(self.marshal.encode(msgid, ASYNC, method, args))
+ # XXX The message won't go out right away in this case. It
+ # will wait for the asyncore loop to get control again. Seems
+ # okay to comment our for now, but need to understand better.
+ self._do_async_poll()
+
+ # handle IO, possibly in async mode
+
+ def _prepare_async(self):
+ self.thr_async = 0
+ ThreadedAsync.register_loop_callback(self.set_async)
+ # XXX If we are not in async mode, this will cause dead
+ # Connections to be leaked.
+
+ def set_async(self, map):
+ self.trigger = trigger()
+ self.thr_async = 1
+
+ def is_async(self):
+ if self.thr_async:
+ return 1
+ else:
+ return 0
+
+ def _do_async_loop(self):
+ "Invoke asyncore mainloop and wait for reply."
+ if __debug__:
+ log("_do_async_loop() async=%d" % self.is_async(),
+ level=zLOG.DEBUG)
+ if self.is_async():
+ self.trigger.pull_trigger()
+ self.__reply_lock.acquire()
+ # wait until reply...
+ else:
+ # Do loop only if lock is already acquired. XXX But can't
+ # we already guarantee that the lock is already acquired?
+ while not self.__reply_lock.acquire(0):
+ asyncore.poll(10.0, self._map)
+ if self.closed:
+ raise DisconnectedError()
+ self.__reply_lock.release()
+
+ def _do_async_poll(self, wait_for_reply=0):
+ "Invoke asyncore mainloop to get pending message out."
+
+ if __debug__:
+ log("_do_async_poll(), async=%d" % self.is_async(),
+ level=zLOG.DEBUG)
+ if self.is_async():
+ self.trigger.pull_trigger()
+ else:
+ asyncore.poll(0.0, self._map)
+
+class ServerConnection(Connection):
+ """Connection on the server side"""
+
+ # The server side does not send a protocol message. Instead, it
+ # adapts to whatever the client sends it.
+
+class ManagedServerConnection(ServerConnection):
+ """A connection that notifies its ConnectionManager of closing"""
+ __super_init = Connection.__init__
+ __super_close = Connection.close
+
+ def __init__(self, sock, addr, obj, mgr):
+ self.__mgr = mgr
+ self.__super_init(sock, addr, obj)
+ obj.notifyConnected(self)
+
+ def close(self):
+ self.__super_close()
+ self.__mgr.close(self)
+
+class ManagedConnection(Connection):
+ """A connection that notifies its ConnectionManager of closing.
+
+ A managed connection also defers the ThreadedAsync work to its
+ manager.
+ """
+ __super_init = Connection.__init__
+ __super_close = Connection.close
+
+ def __init__(self, sock, addr, obj, mgr):
+ self.__mgr = mgr
+ self.__super_init(sock, addr, obj)
+ self.check_mgr_async()
+
+ def close_trigger(self):
+ # the manager should actually close the trigger
+ del self.trigger
+
+ def set_async(self, map):
+ pass
+
+ def _prepare_async(self):
+ # Don't do the register_loop_callback that the superclass does
+ pass
+
+ def check_mgr_async(self):
+ if not self.thr_async and self.__mgr.thr_async:
+ assert self.__mgr.trigger is not None, \
+ "manager (%s) has no trigger" % self.__mgr
+ self.thr_async = 1
+ self.trigger = self.__mgr.trigger
+ return 1
+ return 0
+
+ def is_async(self):
+ if self.thr_async:
+ return 1
+ return self.check_mgr_async()
+
+ def close(self):
+ self.__super_close()
+ self.__mgr.notify_closed(self)
=== ZEO/ZEO/zrpc/error.py 1.1 => 1.2 ===
+#
+# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE
+#
+##############################################################################
+from ZODB import POSException
+from ZEO.Exceptions import Disconnected
+
+class ZRPCError(POSException.StorageError):
+ pass
+
+class DecodingError(ZRPCError):
+ """A ZRPC message could not be decoded."""
+
+class DisconnectedError(ZRPCError, Disconnected):
+ """The database storage is disconnected from the storage server."""
=== ZEO/ZEO/zrpc/log.py 1.1 => 1.2 ===
+#
+# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE
+#
+##############################################################################
+import os
+import types
+import zLOG
+
+_label = "zrpc:%s" % os.getpid()
+
+def new_label():
+ global _label
+ _label = "zrpc:%s" % os.getpid()
+
+def log(message, level=zLOG.BLATHER, label=None, error=None):
+ zLOG.LOG(label or _label, level, message, error=error)
+
+REPR_LIMIT = 40
+
+def short_repr(obj):
+ "Return an object repr limited to REPR_LIMIT bytes."
+ # Some of the objects being repr'd are large strings. It's wastes
+ # a lot of memory to repr them and then truncate, so special case
+ # them in this function.
+ # Also handle short repr of a tuple containing a long string.
+ if isinstance(obj, types.StringType):
+ obj = obj[:REPR_LIMIT]
+ elif isinstance(obj, types.TupleType):
+ elts = []
+ size = 0
+ for elt in obj:
+ r = repr(elt)
+ elts.append(r)
+ size += len(r)
+ if size > REPR_LIMIT:
+ break
+ obj = tuple(elts)
+ return repr(obj)[:REPR_LIMIT]
=== ZEO/ZEO/zrpc/marshal.py 1.1 => 1.2 ===
+#
+# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE
+#
+##############################################################################
+import cPickle
+from cStringIO import StringIO
+import struct
+import types
+
+from ZEO.zrpc.error import ZRPCError
+
+class Marshaller:
+ """Marshal requests and replies to second across network"""
+
+ # It's okay to share a single Pickler as long as it's in fast
+ # mode, which means that it doesn't have a memo.
+
+ pickler = cPickle.Pickler()
+ pickler.fast = 1
+ pickle = pickler.dump
+
+ errors = (cPickle.UnpickleableError,
+ cPickle.UnpicklingError,
+ cPickle.PickleError,
+ cPickle.PicklingError)
+
+ VERSION = 1
+
+ def encode(self, msgid, flags, name, args):
+ """Returns an encoded message"""
+ return self.pickle((msgid, flags, name, args), 1)
+
+ def decode(self, msg):
+ """Decodes msg and returns its parts"""
+ unpickler = cPickle.Unpickler(StringIO(msg))
+ unpickler.find_global = find_global
+
+ try:
+ return unpickler.load() # msgid, flags, name, args
+ except (self.errors, IndexError), err_msg:
+ log("can't decode %s" % repr(msg), level=zLOG.ERROR)
+ raise DecodingError(msg)
+
+_globals = globals()
+_silly = ('__doc__',)
+
+def find_global(module, name):
+ """Helper for message unpickler"""
+ try:
+ m = __import__(module, _globals, _globals, _silly)
+ except ImportError, msg:
+ raise ZRPCError("import error %s: %s" % (module, msg))
+
+ try:
+ r = getattr(m, name)
+ except AttributeError:
+ raise ZRPCError("module %s has no global %s" % (module, name))
+
+ safe = getattr(r, '__no_side_effects__', 0)
+ if safe:
+ return r
+
+ # XXX what's a better way to do this? esp w/ 2.1 & 2.2
+ if type(r) == types.ClassType and issubclass(r, Exception):
+ return r
+
+ raise ZRPCError("Unsafe global: %s.%s" % (module, name))
=== ZEO/ZEO/zrpc/server.py 1.1 => 1.2 ===
+#
+# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE
+#
+##############################################################################
+import asyncore
+import socket
+import types
+
+from ZEO.zrpc.connection import Connection, Delay
+from ZEO.zrpc.log import log
+
+# Export the main asyncore loop
+loop = asyncore.loop
+
+class Dispatcher(asyncore.dispatcher):
+ """A server that accepts incoming RPC connections"""
+ __super_init = asyncore.dispatcher.__init__
+
+ reuse_addr = 1
+
+ def __init__(self, addr, factory=Connection, reuse_addr=None):
+ self.__super_init()
+ self.addr = addr
+ self.factory = factory
+ self.clients = []
+ if reuse_addr is not None:
+ self.reuse_addr = reuse_addr
+ self._open_socket()
+
+ def _open_socket(self):
+ if type(self.addr) == types.TupleType:
+ self.create_socket(socket.AF_INET, socket.SOCK_STREAM)
+ else:
+ self.create_socket(socket.AF_UNIX, socket.SOCK_STREAM)
+ self.set_reuse_addr()
+ log("listening on %s" % str(self.addr))
+ self.bind(self.addr)
+ self.listen(5)
+
+ def writable(self):
+ return 0
+
+ def readable(self):
+ return 1
+
+ def handle_accept(self):
+ try:
+ sock, addr = self.accept()
+ except socket.error, msg:
+ log("accepted failed: %s" % msg)
+ return
+ c = self.factory(sock, addr)
+ log("connect from %s: %s" % (repr(addr), c))
+ self.clients.append(c)
=== ZEO/ZEO/zrpc/trigger.py 1.1 => 1.2 ===
+#
+# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE
+#
+##############################################################################
+# This module is a simplified version of the select_trigger module
+# from Sam Rushing's Medusa server.
+
+import asyncore
+
+import os
+import socket
+import thread
+
+if os.name == 'posix':
+
+ class trigger (asyncore.file_dispatcher):
+
+ "Wake up a call to select() running in the main thread"
+
+ # This is useful in a context where you are using Medusa's I/O
+ # subsystem to deliver data, but the data is generated by another
+ # thread. Normally, if Medusa is in the middle of a call to
+ # select(), new output data generated by another thread will have
+ # to sit until the call to select() either times out or returns.
+ # If the trigger is 'pulled' by another thread, it should immediately
+ # generate a READ event on the trigger object, which will force the
+ # select() invocation to return.
+
+ # A common use for this facility: letting Medusa manage I/O for a
+ # large number of connections; but routing each request through a
+ # thread chosen from a fixed-size thread pool. When a thread is
+ # acquired, a transaction is performed, but output data is
+ # accumulated into buffers that will be emptied more efficiently
+ # by Medusa. [picture a server that can process database queries
+ # rapidly, but doesn't want to tie up threads waiting to send data
+ # to low-bandwidth connections]
+
+ # The other major feature provided by this class is the ability to
+ # move work back into the main thread: if you call pull_trigger()
+ # with a thunk argument, when select() wakes up and receives the
+ # event it will call your thunk from within that thread. The main
+ # purpose of this is to remove the need to wrap thread locks around
+ # Medusa's data structures, which normally do not need them. [To see
+ # why this is true, imagine this scenario: A thread tries to push some
+ # new data onto a channel's outgoing data queue at the same time that
+ # the main thread is trying to remove some]
+
+ def __init__ (self):
+ r, w = os.pipe()
+ self.trigger = w
+ asyncore.file_dispatcher.__init__ (self, r)
+ self.lock = thread.allocate_lock()
+ self.thunks = []
+
+ def close(self):
+ self.del_channel()
+ self.socket.close() # the read side of the pipe
+ os.close(self.trigger) # the write side of the pipe
+
+ def __repr__ (self):
+ return '' % id(self)
+
+ def readable (self):
+ return 1
+
+ def writable (self):
+ return 0
+
+ def handle_connect (self):
+ pass
+
+ def pull_trigger (self, thunk=None):
+ # print 'PULL_TRIGGER: ', len(self.thunks)
+ if thunk:
+ try:
+ self.lock.acquire()
+ self.thunks.append (thunk)
+ finally:
+ self.lock.release()
+ os.write (self.trigger, 'x')
+
+ def handle_read (self):
+ self.recv (8192)
+ try:
+ self.lock.acquire()
+ for thunk in self.thunks:
+ try:
+ thunk()
+ except:
+ (file, fun, line), t, v, tbinfo = asyncore.compact_traceback()
+ print 'exception in trigger thunk: (%s:%s %s)' % (t, v, tbinfo)
+ self.thunks = []
+ finally:
+ self.lock.release()
+
+else:
+
+ # win32-safe version
+
+ class trigger (asyncore.dispatcher):
+
+ address = ('127.9.9.9', 19999)
+
+ def __init__ (self):
+ a = socket.socket (socket.AF_INET, socket.SOCK_STREAM)
+ w = socket.socket (socket.AF_INET, socket.SOCK_STREAM)
+
+ # set TCP_NODELAY to true to avoid buffering
+ w.setsockopt(socket.IPPROTO_TCP, 1, 1)
+
+ # tricky: get a pair of connected sockets
+ host='127.0.0.1'
+ port=19999
+ while 1:
+ try:
+ self.address=(host, port)
+ a.bind(self.address)
+ break
+ except:
+ if port <= 19950:
+ raise 'Bind Error', 'Cannot bind trigger!'
+ port=port - 1
+
+ a.listen (1)
+ w.setblocking (0)
+ try:
+ w.connect (self.address)
+ except:
+ pass
+ r, addr = a.accept()
+ a.close()
+ w.setblocking (1)
+ self.trigger = w
+
+ asyncore.dispatcher.__init__ (self, r)
+ self.lock = thread.allocate_lock()
+ self.thunks = []
+ self._trigger_connected = 0
+
+ def __repr__ (self):
+ return '' % id(self)
+
+ def readable (self):
+ return 1
+
+ def writable (self):
+ return 0
+
+ def handle_connect (self):
+ pass
+
+ def pull_trigger (self, thunk=None):
+ if thunk:
+ try:
+ self.lock.acquire()
+ self.thunks.append (thunk)
+ finally:
+ self.lock.release()
+ self.trigger.send ('x')
+
+ def handle_read (self):
+ self.recv (8192)
+ try:
+ self.lock.acquire()
+ for thunk in self.thunks:
+ try:
+ thunk()
+ except:
+ (file, fun, line), t, v, tbinfo = asyncore.compact_traceback()
+ print 'exception in trigger thunk: (%s:%s %s)' % (t, v, tbinfo)
+ self.thunks = []
+ finally:
+ self.lock.release()
From jeremy at zope.com Fri Jun 14 11:10:19 2002
From: jeremy at zope.com (Jeremy Hylton)
Date: Sun Aug 10 16:31:22 2008
Subject: [ZEO-Checkins] CVS: StandaloneZODB/ZEO - start.py:1.34
Message-ID: <200206141510.g5EFAJN27814@cvs.baymountain.com>
Update of /cvs-repository/StandaloneZODB/ZEO
In directory cvs.zope.org:/tmp/cvs-serv27803
Modified Files:
start.py
Log Message:
Remove global import statements that may not work before path munging.
XXX Should start still do path munging?
=== StandaloneZODB/ZEO/start.py 1.33 => 1.34 ===
import sys, os, getopt, string
-import StorageServer
-import asyncore
-
def directory(p, n=1):
d=p
while n:
@@ -265,7 +262,7 @@
if not unix: unix=host, port
- StorageServer.StorageServer(unix, storages)
+ ZEO.StorageServer.StorageServer(unix, storages)
try:
ppid, pid = os.getppid(), os.getpid()
From shane at cvs.zope.org Fri Jun 21 10:54:24 2002
From: shane at cvs.zope.org (Shane Hathaway)
Date: Sun Aug 10 16:31:22 2008
Subject: [ZEO-Checkins] CVS: ZEO/ZEO - StorageServer.py:1.38
Message-ID: <200206211454.g5LEsOZ08568@cvs.baymountain.com>
Update of /cvs-repository/ZEO/ZEO
In directory cvs.zope.org:/tmp/cvs-serv7636
Modified Files:
StorageServer.py
Log Message:
slog(), called when a routine error happens in store(), had a NameError on
self, which effectively turned the error into a fatal error. Fixed.
=== ZEO/ZEO/StorageServer.py 1.37 => 1.38 ===
name = getattr(storage, '__name__', None)
if name is None:
- name = str(self.storage)
+ name = str(storage)
zLOG.LOG("ZEO Server:%s:%s" % (pid, name), level, msg, error=error)
class StorageServerError(StorageError):
From tdickenson at geminidataloggers.com Thu Jun 27 05:56:54 2002
From: tdickenson at geminidataloggers.com (Toby Dickenson)
Date: Sun Aug 10 16:31:22 2008
Subject: [ZEO-Checkins] CVS: Packages/ZEO/zrpc - connection.py:1.1.2.2.2.5
Message-ID: <200206270956.g5R9usE13327@cvs.baymountain.com>
Update of /cvs-repository/Packages/ZEO/zrpc
In directory cvs.zope.org:/tmp/cvs-serv13313/zrpc
Modified Files:
Tag: ZEO2-branch
connection.py
Log Message:
fix bug that caused exceptions log entries to be mangled
=== Packages/ZEO/zrpc/connection.py 1.1.2.2.2.4 => 1.1.2.2.2.5 ===
except Exception, msg:
error = sys.exc_info()[:2]
- log("%s() raised exception: %s" % (name, msg), zLOG.ERROR, error)
+ log("%s() raised exception: %s" % (name, msg), zLOG.ERROR, error=sys.exc_info())
return self.return_error(msgid, flags, error[0], error[1])
if flags & ASYNC:
From tdickenson at geminidataloggers.com Thu Jun 27 05:58:05 2002
From: tdickenson at geminidataloggers.com (Toby Dickenson)
Date: Sun Aug 10 16:31:22 2008
Subject: [ZEO-Checkins] CVS: Packages/ZEO - ServerStub.py:1.3.2.1.2.2
Message-ID: <200206270958.g5R9w5013738@cvs.baymountain.com>
Update of /cvs-repository/Packages/ZEO
In directory cvs.zope.org:/tmp/cvs-serv13727
Modified Files:
Tag: ZEO2-branch
ServerStub.py
Log Message:
fix bug that prevented history() returning more than a single item
=== Packages/ZEO/ServerStub.py 1.3.2.1.2.1 => 1.3.2.1.2.2 ===
def history(self, oid, version, length=None):
- if length is not None:
+ if length is None:
return self.rpc.call('history', oid, version)
else:
return self.rpc.call('history', oid, version, length)
From tdickenson at geminidataloggers.com Thu Jun 27 11:06:19 2002
From: tdickenson at geminidataloggers.com (Toby Dickenson)
Date: Sun Aug 10 16:31:22 2008
Subject: [ZEO-Checkins] CVS: Packages/ZEO - ServerStub.py:1.5
Message-ID: <200206271506.g5RF6JP17465@cvs.baymountain.com>
Update of /cvs-repository/Packages/ZEO
In directory cvs.zope.org:/tmp/cvs-serv17451
Modified Files:
ServerStub.py
Log Message:
merge change from ZEO2-branch; fix bug that prevented history() returning more than a single item
=== Packages/ZEO/ServerStub.py 1.4 => 1.5 ===
def history(self, oid, version, length=None):
- if length is not None:
+ if length is None:
return self.rpc.call('history', oid, version)
else:
return self.rpc.call('history', oid, version, length)
From tdickenson at geminidataloggers.com Thu Jun 27 11:06:57 2002
From: tdickenson at geminidataloggers.com (Toby Dickenson)
Date: Sun Aug 10 16:31:22 2008
Subject: [ZEO-Checkins] CVS: Packages/ZEO/zrpc - connection.py:1.3
Message-ID: <200206271506.g5RF6vP17521@cvs.baymountain.com>
Update of /cvs-repository/Packages/ZEO/zrpc
In directory cvs.zope.org:/tmp/cvs-serv17510/zrpc
Modified Files:
connection.py
Log Message:
merge change from ZEO2-branch; fix bug that caused exceptions log entries to be mangled
=== Packages/ZEO/zrpc/connection.py 1.2 => 1.3 ===
except Exception, msg:
error = sys.exc_info()[:2]
- log("%s() raised exception: %s" % (name, msg), zLOG.ERROR, error)
+ log("%s() raised exception: %s" % (name, msg), zLOG.ERROR, error=sys.exc_info())
return self.return_error(msgid, flags, error[0], error[1])
if flags & ASYNC:
From tdickenson at geminidataloggers.com Fri Jun 28 05:32:22 2002
From: tdickenson at geminidataloggers.com (Toby Dickenson)
Date: Sun Aug 10 16:31:22 2008
Subject: [ZEO-Checkins] CVS: Packages/ZEO - StorageServer.py:1.38.2.1 ServerStub.py:1.5.2.1 ClientStorage.py:1.41.2.1
Message-ID: <200206280932.g5S9WMk14521@cvs.baymountain.com>
Update of /cvs-repository/Packages/ZEO
In directory cvs.zope.org:/tmp/cvs-serv14506
Modified Files:
Tag: toby-signal-branch
StorageServer.py ServerStub.py ClientStorage.py
Log Message:
added 'signal', a new ZEO-marshalled method for accessing a storages additional features that are not covered by the standard storage API. This change is experimental
=== Packages/ZEO/StorageServer.py 1.38 => 1.38.2.1 ===
return ()
+ def signal(self,signalname,params):
+ fn = getattr(self.__storage,'signal',None)
+ if callable(fn):
+ return fn(signalname,params)
+ else:
+ raise NotImplementedError('signal(%r)'%(signalname,))
+
def tpc_begin(self, id, user, description, ext, tid, status):
if self._transaction is not None:
if self._transaction.id == id:
=== Packages/ZEO/ServerStub.py 1.5 => 1.5.2.1 ===
return self.rpc.call('history', oid, version, length)
+ def signal(self, signalname, params):
+ return self.rpc.call('signal', str(signalname), params)
+
def load(self, oid, version):
return self.rpc.call('load', oid, version)
=== Packages/ZEO/ClientStorage.py 1.41 => 1.41.2.1 ===
return self._server.history(oid, version, length)
+ def signal(self, signalname, params):
+ """Call the signal method of the underlying storage.
+ This is useful if your storage supports features outside
+ the standard storage API, and you want to access them
+ over ZEO. Your storage should raise NotImplementedError
+ if it does not recognise signalname. The StorageServer
+ will do the same if a storage does not provide a signal
+ method.
+ """
+ return self._server.signal(signalname, params)
+
def loadSerial(self, oid, serial):
return self._server.loadSerial(oid, serial)