From jeremy at zope.com Wed May 1 15:12:46 2002 From: jeremy at zope.com (Jeremy Hylton) Date: Sun Aug 10 16:31:20 2008 Subject: [ZEO-Checkins] CVS: ZEO/ZEO/tests - testZEO.py:1.16.4.4.2.3 Message-ID: <200205011912.g41JCkQ19555@cvs.baymountain.com> Update of /cvs-repository/ZEO/ZEO/tests In directory cvs.zope.org:/tmp/cvs-serv19538 Modified Files: Tag: ZEO2-branch testZEO.py Log Message: temporarily disable two broken tests === ZEO/ZEO/tests/testZEO.py 1.16.4.4.2.2 => 1.16.4.4.2.3 === self._dostore(data=obj) - def checkCommitLockOnCommit(self): - self._checkCommitLock("tpc_finish") +## def checkCommitLockOnCommit(self): +## self._checkCommitLock("tpc_finish") - def checkCommitLockOnAbort(self): - self._checkCommitLock("tpc_abort") +## def checkCommitLockOnAbort(self): +## self._checkCommitLock("tpc_abort") def _checkCommitLock(self, method_name): # check the commit lock when a client attemps a transaction, From jeremy at zope.com Mon May 6 14:17:23 2002 From: jeremy at zope.com (Jeremy Hylton) Date: Sun Aug 10 16:31:20 2008 Subject: [ZEO-Checkins] CVS: StandaloneZODB/ZEO - StorageServer.py:1.32.6.3.2.3 Message-ID: <200205061817.g46IHNU04704@cvs.baymountain.com> Update of /cvs-repository/StandaloneZODB/ZEO In directory cvs.zope.org:/tmp/cvs-serv4691 Modified Files: Tag: ZEO2-branch StorageServer.py Log Message: Catch errors when attempting to restart a blocked client. Log a message when a client blocks waiting for the commit lock. === StandaloneZODB/ZEO/StorageServer.py 1.32.6.3.2.2 => 1.32.6.3.2.3 === d = Delay() self.__storage._waiting.append((d, self)) + self._log("Transaction block waiting for storage. " + "%d clients waiting." % len(self.__storage._waiting)) return d def _handle_waiting(self): if self.__storage._waiting: delay, zeo_storage = self.__storage._waiting.pop(0) - self.restart(delay) - zeo_storage.restart(delay) + try: + zeo_storage.restart(delay) + except: + self._log("Unexpected error handling waiting transaction", + level=zLOG.WARNING, error=sys.exc_info()) + zeo_storage.close() def restart(self, delay): old_strategy = self.strategy From jeremy at zope.com Mon May 6 14:17:39 2002 From: jeremy at zope.com (Jeremy Hylton) Date: Sun Aug 10 16:31:20 2008 Subject: [ZEO-Checkins] CVS: StandaloneZODB/ZEO/zrpc - marshal.py:1.1.2.1.2.3 Message-ID: <200205061817.g46IHdH04758@cvs.baymountain.com> Update of /cvs-repository/StandaloneZODB/ZEO/zrpc In directory cvs.zope.org:/tmp/cvs-serv4746/zrpc Modified Files: Tag: ZEO2-branch marshal.py Log Message: Import ZRPCError. === StandaloneZODB/ZEO/zrpc/marshal.py 1.1.2.1.2.2 => 1.1.2.1.2.3 === import types +from ZEO.zrpc.error import ZRPCError + class Marshaller: """Marshal requests and replies to second across network""" From jeremy at zope.com Mon May 6 14:21:29 2002 From: jeremy at zope.com (Jeremy Hylton) Date: Sun Aug 10 16:31:20 2008 Subject: [ZEO-Checkins] CVS: StandaloneZODB/ZEO/tests - testZEO.py:1.16.4.4.2.4 Message-ID: <200205061821.g46ILTu06054@cvs.baymountain.com> Update of /cvs-repository/StandaloneZODB/ZEO/tests In directory cvs.zope.org:/tmp/cvs-serv6035/tests Modified Files: Tag: ZEO2-branch testZEO.py Log Message: Re-enable to commit tests now that they work. === StandaloneZODB/ZEO/tests/testZEO.py 1.16.4.4.2.3 => 1.16.4.4.2.4 === self._dostore(data=obj) -## def checkCommitLockOnCommit(self): -## self._checkCommitLock("tpc_finish") + def checkCommitLockOnCommit(self): + try: + self._checkCommitLock("tpc_finish") + finally: + self._cleanup() + + def checkCommitLockOnAbort(self): + try: + self._checkCommitLock("tpc_abort") + finally: + self._cleanup() -## def checkCommitLockOnAbort(self): -## self._checkCommitLock("tpc_abort") + def _cleanup(self): + for store, trans in self._storages: + store.tpc_abort(trans) + store.close() + self._storages = [] def _checkCommitLock(self, method_name): # check the commit lock when a client attemps a transaction, @@ -123,9 +135,12 @@ for i in range(3): storage2 = self._duplicate_client() t2 = Transaction() - tid = ZEO.ClientStorage.get_timestamp() - storage2._server.tpc_begin(tid, t2.user, t2.description, - t2._extension, None, ' ') + tid = `ZEO.ClientStorage.get_timestamp()` + try: + storage2._server.tpc_begin(tid, t2.user, t2.description, + t2._extension, None, ' ') + except Disconnected: + self.fail("client %d disconnected!" % i) if i == 0: storage2.close() else: @@ -134,12 +149,9 @@ oid = self._storage.new_oid() self._storage.store(oid, None, '', '', t) self._storage.tpc_vote(t) - self._storage.status() self._storage.tpc_finish(t) - for store, trans in self._storages: - store.tpc_abort(trans) - store.close() + self._cleanup() # Make sure the server is still responsive self._dostore() From jeremy at zope.com Mon May 6 17:02:59 2002 From: jeremy at zope.com (Jeremy Hylton) Date: Sun Aug 10 16:31:20 2008 Subject: [ZEO-Checkins] CVS: StandaloneZODB/ZEO/zrpc - connection.py:1.1.2.2.2.4 Message-ID: <200205062102.g46L2xg18400@cvs.baymountain.com> Update of /cvs-repository/StandaloneZODB/ZEO/zrpc In directory cvs.zope.org:/tmp/cvs-serv18385/zrpc Modified Files: Tag: ZEO2-branch connection.py Log Message: Add logging of individual method calls at BLATHER level === StandaloneZODB/ZEO/zrpc/connection.py 1.1.2.2.2.3 => 1.1.2.2.2.4 === msg = "Invalid method name: %s on %s" % (name, repr(self.obj)) raise ZRPCError(msg) + if __debug__: + log("%s%s" % (name, args), level=zLOG.BLATHER) meth = getattr(self.obj, name) try: From jeremy at zope.com Mon May 6 17:08:22 2002 From: jeremy at zope.com (Jeremy Hylton) Date: Sun Aug 10 16:31:20 2008 Subject: [ZEO-Checkins] CVS: StandaloneZODB/ZEO/tests - CommitLockTests.py:1.1.2.1 testZEO.py:1.16.4.4.2.5 Message-ID: <200205062108.g46L8MG20476@cvs.baymountain.com> Update of /cvs-repository/StandaloneZODB/ZEO/tests In directory cvs.zope.org:/tmp/cvs-serv20463/tests Modified Files: Tag: ZEO2-branch testZEO.py Added Files: Tag: ZEO2-branch CommitLockTests.py Log Message: Factor out the commit lock tests into a separate module. === Added File StandaloneZODB/ZEO/tests/CommitLockTests.py === """Tests of the distributed commit lock.""" from ZODB.Transaction import Transaction from ZODB.tests.StorageTestBase import zodb_pickle, MinPO import ZEO.ClientStorage ZERO = '\0'*8 class DummyDB: def invalidate(self, *args): pass class CommitLockTests: def checkCommitLockOnCommit(self): self._storages = [] try: self._checkCommitLock("tpc_finish") finally: self._cleanup() def checkCommitLockOnAbort(self): self._storages = [] try: self._checkCommitLock("tpc_abort") finally: self._cleanup() def _cleanup(self): for store, trans in self._storages: store.tpc_abort(trans) store.close() self._storages = [] def _checkCommitLock(self, method_name): # check the commit lock when a client attemps a transaction, # but fails/exits before finishing the commit. # Start on transaction normally. t = Transaction() self._storage.tpc_begin(t) # Start a second transaction on a different connection without # blocking the test thread. self._storages = [] for i in range(3): storage2 = self._duplicate_client() t2 = Transaction() # ??? tid = `ZEO.ClientStorage.get_timestamp()` storage2.tpc_begin(t2, tid) if i == 0: storage2.close() else: print "storage", i, "opened" self._storages.append((storage2, t2)) print "finishing original commit" oid = self._storage.new_oid() self._storage.store(oid, ZERO, zodb_pickle(MinPO(1)), '', t) self._storage.tpc_vote(t) if method_name == "tpc_finish": self._storage.tpc_finish(t) self._storage.load(oid, '') else: self._storage.tpc_abort(t) print "done" self._dowork() # Make sure the server is still responsive self._dostore() def _dowork(self): for store, trans in self._storages: oid = store.new_oid() store.store(oid, ZERO, zodb_pickle(MinPO("c")), '', trans) store.tpc_vote(trans) store.tpc_abort(trans) def _duplicate_client(self): "Open another ClientStorage to the same server." # XXX argh it's hard to find the actual address # The rpc mgr addr attribute is a list. Each element in the # list is a socket domain (AF_INET, AF_UNIX, etc.) and an # address. addr = self._storage._rpc_mgr.addr[0][1] new = ZEO.ClientStorage.ClientStorage(addr, wait=1) new.registerDB(DummyDB(), None) return new def _get_timestamp(self): t = time.time() t = apply(TimeStamp,(time.gmtime(t)[:5]+(t%60,))) return `t` === StandaloneZODB/ZEO/tests/testZEO.py 1.16.4.4.2.4 => 1.16.4.4.2.5 === import sys import tempfile +import thread import time import types import unittest @@ -28,10 +29,10 @@ import ThreadedAsync, ZEO.trigger from ZODB.FileStorage import FileStorage from ZODB.Transaction import Transaction -import thread +from ZODB.tests.StorageTestBase import zodb_pickle, MinPO import zLOG -from ZEO.tests import forker, Cache +from ZEO.tests import forker, Cache, CommitLockTests from ZEO.smac import Disconnected # Sorry Jim... @@ -42,8 +43,6 @@ from ZODB.tests.MinPO import MinPO from ZODB.tests.StorageTestBase import zodb_unpickle -ZERO = '\0'*8 - class DummyDB: def invalidate(self, *args): pass @@ -72,6 +71,7 @@ Synchronization.SynchronizedStorage, MTStorage.MTStorage, ReadOnlyStorage.ReadOnlyStorage, + CommitLockTests.CommitLockTests, ): """An abstract base class for ZEO tests @@ -102,75 +102,6 @@ def checkLargeUpdate(self): obj = MinPO("X" * (10 * 128 * 1024)) self._dostore(data=obj) - - def checkCommitLockOnCommit(self): - try: - self._checkCommitLock("tpc_finish") - finally: - self._cleanup() - - def checkCommitLockOnAbort(self): - try: - self._checkCommitLock("tpc_abort") - finally: - self._cleanup() - - def _cleanup(self): - for store, trans in self._storages: - store.tpc_abort(trans) - store.close() - self._storages = [] - - def _checkCommitLock(self, method_name): - # check the commit lock when a client attemps a transaction, - # but fails/exits before finishing the commit. - - # Start on transaction normally. - t = Transaction() - self._storage.tpc_begin(t) - - # Start a second transaction on a different connection without - # blocking the test thread. - self._storages = [] - for i in range(3): - storage2 = self._duplicate_client() - t2 = Transaction() - tid = `ZEO.ClientStorage.get_timestamp()` - try: - storage2._server.tpc_begin(tid, t2.user, t2.description, - t2._extension, None, ' ') - except Disconnected: - self.fail("client %d disconnected!" % i) - if i == 0: - storage2.close() - else: - self._storages.append((storage2, t2)) - - oid = self._storage.new_oid() - self._storage.store(oid, None, '', '', t) - self._storage.tpc_vote(t) - self._storage.tpc_finish(t) - - self._cleanup() - - # Make sure the server is still responsive - self._dostore() - - def _duplicate_client(self): - "Open another ClientStorage to the same server." - # XXX argh it's hard to find the actual address - # The rpc mgr addr attribute is a list. Each element in the - # list is a socket domain (AF_INET, AF_UNIX, etc.) and an - # address. - addr = self._storage._rpc_mgr.addr[0][1] - new = ZEO.ClientStorage.ClientStorage(addr, wait=1) - new.registerDB(DummyDB(), None) - return new - - def _get_timestamp(self): - t = time.time() - t = apply(TimeStamp,(time.gmtime(t)[:5]+(t%60,))) - return `t` class ZEOFileStorageTests(GenericTests): __super_setUp = GenericTests.setUp From jeremy at zope.com Mon May 6 17:22:19 2002 From: jeremy at zope.com (Jeremy Hylton) Date: Sun Aug 10 16:31:20 2008 Subject: [ZEO-Checkins] CVS: StandaloneZODB/ZEO - StorageServer.py:1.32.6.3.2.4 Message-ID: <200205062122.g46LMJp25693@cvs.baymountain.com> Update of /cvs-repository/StandaloneZODB/ZEO In directory cvs.zope.org:/tmp/cvs-serv25680 Modified Files: Tag: ZEO2-branch StorageServer.py Log Message: Fix more bugs in the distributed commit lock implementation. If a transaction got into the Delayed strategy and got to the wait() call after the other transaction was finished, it was added to the waiting list even though there was no running transaction to unblock it. The solution is to check whether the commit lock is held when wait() is called. If it isn't, the transaction can transition immediately to the immediate strategy. In _handle_waiting(), continue to pop transactions off the waiting queue until one succeeds in starting. In restart(), make delay argument optional. There is no delay if wait() finds the lock free. In DelayedCommitStrategy's restart() method, loads is an int not a list, so call range() not len(). === StandaloneZODB/ZEO/StorageServer.py 1.32.6.3.2.3 => 1.32.6.3.2.4 === def wait(self): - d = Delay() - self.__storage._waiting.append((d, self)) - self._log("Transaction block waiting for storage. " - "%d clients waiting." % len(self.__storage._waiting)) - return d + if self.__storage._transaction: + d = Delay() + self.__storage._waiting.append((d, self)) + self._log("Transaction block waiting for storage. " + "%d clients waiting." % len(self.__storage._waiting)) + return d + else: + self.restart() def _handle_waiting(self): - if self.__storage._waiting: + while self.__storage._waiting: delay, zeo_storage = self.__storage._waiting.pop(0) - try: - zeo_storage.restart(delay) - except: - self._log("Unexpected error handling waiting transaction", - level=zLOG.WARNING, error=sys.exc_info()) - zeo_storage.close() + if self._restart(delay): + break - def restart(self, delay): + def _restart(self, delay): + # call the restart() method on the appropriate server + try: + zeo_storage.restart(delay) + except: + self._log("Unexpected error handling waiting transaction", + level=zLOG.WARNING, error=sys.exc_info()) + zeo_storage.close() + return 0 + else: + return 1 + + def restart(self, delay=None): old_strategy = self.strategy self.strategy = ImmediateCommitStrategy(self.__storage, self.client) - delay.reply(old_strategy.restart(self.strategy)) + resp = old_strategy.restart(self.strategy) + if delay is not None: + delay.reply(resp) # A ZEOStorage instance can use different strategies to commit a # transaction. The current implementation uses different strategies @@ -538,7 +551,7 @@ # called by the storage when the storage is available new_strategy.tpc_begin(self.txn, self.tid, self.status) loads, loader = self.log.get_loader() - for i in len(loads): + for i in range(loads): oid, serial, data, version = loader.load() new_strategy.store(oid, serial, data, version) meth = getattr(new_strategy, self.name) From jeremy at zope.com Mon May 6 17:24:47 2002 From: jeremy at zope.com (Jeremy Hylton) Date: Sun Aug 10 16:31:20 2008 Subject: [ZEO-Checkins] CVS: StandaloneZODB/ZEO/tests - CommitLockTests.py:1.1.2.2 Message-ID: <200205062124.g46LOl126419@cvs.baymountain.com> Update of /cvs-repository/StandaloneZODB/ZEO/tests In directory cvs.zope.org:/tmp/cvs-serv26408 Modified Files: Tag: ZEO2-branch CommitLockTests.py Log Message: remove debugging prints === StandaloneZODB/ZEO/tests/CommitLockTests.py 1.1.2.1 => 1.1.2.2 === storage2.close() else: - print "storage", i, "opened" self._storages.append((storage2, t2)) - print "finishing original commit" oid = self._storage.new_oid() self._storage.store(oid, ZERO, zodb_pickle(MinPO(1)), '', t) self._storage.tpc_vote(t) @@ -65,7 +63,6 @@ self._storage.load(oid, '') else: self._storage.tpc_abort(t) - print "done" self._dowork() From jeremy at zope.com Tue May 7 00:42:30 2002 From: jeremy at zope.com (Jeremy Hylton) Date: Sun Aug 10 16:31:20 2008 Subject: [ZEO-Checkins] CVS: StandaloneZODB/ZEO - ClientStorage.py:1.35.6.4.2.4 Message-ID: <200205070442.g474gU117630@cvs.baymountain.com> Update of /cvs-repository/StandaloneZODB/ZEO In directory cvs.zope.org:/tmp/cvs-serv17615 Modified Files: Tag: ZEO2-branch ClientStorage.py Log Message: It the tpc_cond condition variable is released, make sure _transaction is None. === StandaloneZODB/ZEO/ClientStorage.py 1.35.6.4.2.3 => 1.35.6.4.2.4 === if self._server is None: self.tpc_cond.release() + self._transaction = None raise ClientDisconnected() if tid is None: @@ -372,6 +373,7 @@ # Client may have disconnected during the tpc_begin(). # Then notifyDisconnected() will have released the lock. if self._server is not disconnected_stub: + self._transaction = None self.tpc_cond.release() raise From jeremy at zope.com Tue May 7 00:42:58 2002 From: jeremy at zope.com (Jeremy Hylton) Date: Sun Aug 10 16:31:20 2008 Subject: [ZEO-Checkins] CVS: StandaloneZODB/ZEO - StorageServer.py:1.32.6.3.2.5 Message-ID: <200205070442.g474gwG17684@cvs.baymountain.com> Update of /cvs-repository/StandaloneZODB/ZEO In directory cvs.zope.org:/tmp/cvs-serv17670 Modified Files: Tag: ZEO2-branch StorageServer.py Log Message: zeo_storage must be passed as an argument to _restart === StandaloneZODB/ZEO/StorageServer.py 1.32.6.3.2.4 => 1.32.6.3.2.5 === while self.__storage._waiting: delay, zeo_storage = self.__storage._waiting.pop(0) - if self._restart(delay): + if self._restart(zeo_storage, delay): break - def _restart(self, delay): + def _restart(self, zeo_storage, delay): # call the restart() method on the appropriate server try: zeo_storage.restart(delay) From jeremy at zope.com Tue May 7 00:43:15 2002 From: jeremy at zope.com (Jeremy Hylton) Date: Sun Aug 10 16:31:20 2008 Subject: [ZEO-Checkins] CVS: StandaloneZODB/ZEO/tests - CommitLockTests.py:1.1.2.3 Message-ID: <200205070443.g474hFC17803@cvs.baymountain.com> Update of /cvs-repository/StandaloneZODB/ZEO/tests In directory cvs.zope.org:/tmp/cvs-serv17792/tests Modified Files: Tag: ZEO2-branch CommitLockTests.py Log Message: Add a multi-threaded test of the commit lock that actually queues clients. === StandaloneZODB/ZEO/tests/CommitLockTests.py 1.1.2.2 => 1.1.2.3 === +import threading + from ZODB.Transaction import Transaction from ZODB.tests.StorageTestBase import zodb_pickle, MinPO import ZEO.ClientStorage +from ZEO.Exceptions import Disconnected ZERO = '\0'*8 @@ -11,19 +14,68 @@ def invalidate(self, *args): pass +class WorkerThread(threading.Thread): + + # run the entire test in a thread so that the blocking call for + # tpc_vote() doesn't hang the test suite. + + def __init__(self, storage, trans, method="tpc_finish"): + self.storage = storage + self.trans = trans + self.method = method + threading.Thread.__init__(self) + + def run(self): + try: + self.storage.tpc_begin(self.trans) + except Disconnected: + return + oid = self.storage.new_oid() + self.storage.store(oid, ZERO, zodb_pickle(MinPO("c")), '', self.trans) + oid = self.storage.new_oid() + self.storage.store(oid, ZERO, zodb_pickle(MinPO("c")), '', self.trans) + self.storage.tpc_vote(self.trans) + if self.method == "tpc_finish": + self.storage.tpc_finish(self.trans) + else: + self.storage.tpc_abort(self.trans) + class CommitLockTests: + + # The commit lock tests verify that the storage successfully + # blocks and restarts transactions when there is content for a + # single storage. There are a lot of cases to cover. + + # CommitLock1 checks the case where a single transaction delays + # other transactions before they actually block. IOW, by the time + # the other transactions get to the vote stage, the first + # transaction has finished. - def checkCommitLockOnCommit(self): + def checkCommitLock1OnCommit(self): + self._storages = [] + try: + self._checkCommitLock("tpc_finish", self._dosetup1, self._dowork1) + finally: + self._cleanup() + + def checkCommitLock1OnAbort(self): self._storages = [] try: - self._checkCommitLock("tpc_finish") + self._checkCommitLock("tpc_abort", self._dosetup1, self._dowork1) finally: self._cleanup() - def checkCommitLockOnAbort(self): + def checkCommitLock2OnCommit(self): self._storages = [] try: - self._checkCommitLock("tpc_abort") + self._checkCommitLock("tpc_finish", self._dosetup2, self._dowork2) + finally: + self._cleanup() + + def checkCommitLock2OnAbort(self): + self._storages = [] + try: + self._checkCommitLock("tpc_abort", self._dosetup2, self._dowork2) finally: self._cleanup() @@ -33,7 +85,7 @@ store.close() self._storages = [] - def _checkCommitLock(self, method_name): + def _checkCommitLock(self, method_name, dosetup, dowork): # check the commit lock when a client attemps a transaction, # but fails/exits before finishing the commit. @@ -44,12 +96,11 @@ # Start a second transaction on a different connection without # blocking the test thread. self._storages = [] - for i in range(3): + for i in range(4): storage2 = self._duplicate_client() t2 = Transaction() - # ??? - tid = `ZEO.ClientStorage.get_timestamp()` - storage2.tpc_begin(t2, tid) + tid = `ZEO.ClientStorage.get_timestamp()` # XXX why? + dosetup(storage2, t2, tid) if i == 0: storage2.close() else: @@ -64,17 +115,33 @@ else: self._storage.tpc_abort(t) - self._dowork() + dowork(method_name) # Make sure the server is still responsive self._dostore() - def _dowork(self): + def _dosetup1(self, storage, trans, tid): + storage.tpc_begin(trans, tid) + + def _dowork1(self, method_name): for store, trans in self._storages: oid = store.new_oid() store.store(oid, ZERO, zodb_pickle(MinPO("c")), '', trans) store.tpc_vote(trans) - store.tpc_abort(trans) + if method_name == "tpc_finish": + store.tpc_finish(trans) + else: + store.tpc_abort(trans) + + def _dosetup2(self, storage, trans, tid): + self._threads = [] + t = WorkerThread(storage, trans) + self._threads.append(t) + t.start() + + def _dowork2(self, method_name): + for t in self._threads: + t.join() def _duplicate_client(self): "Open another ClientStorage to the same server." From barry at wooz.org Wed May 15 01:11:58 2002 From: barry at wooz.org (Barry Warsaw) Date: Sun Aug 10 16:31:21 2008 Subject: [ZEO-Checkins] CVS: ZEO/ZEO/tests - ThreadTests.py:1.1.2.1 testZEO.py:1.16.4.4.2.6 Message-ID: <200205150511.g4F5Bwo05439@cvs.baymountain.com> Update of /cvs-repository/ZEO/ZEO/tests In directory cvs.zope.org:/tmp/cvs-serv5426/ZEO/tests Modified Files: Tag: ZEO2-branch testZEO.py Added Files: Tag: ZEO2-branch ThreadTests.py Log Message: Added a new test class/module ThreadTests which exercises some tricky ZEO threading issues. Specifically added now is a test of a suspected ZEO bug that can occur when one thread closes the storage while another thread is in the middle of a transaction. === Added File ZEO/ZEO/tests/ThreadTests.py === """Compromising positions involving threads.""" import threading from ZODB.Transaction import Transaction from ZODB.tests.StorageTestBase import zodb_pickle, MinPO import ZEO.ClientStorage from ZEO.Exceptions import Disconnected ZERO = '\0'*8 class DummyDB: def invalidate(self, *args): pass class GetsThroughVoteThread(threading.Thread): # run the entire test in a thread so that the blocking call for # tpc_vote() doesn't hang the test suite. def __init__(self, storage, doCloseEvent, threadStartedEvent): self.storage = storage self.trans = Transaction() self.doCloseEvent = doCloseEvent self.threadStartedEvent = threadStartedEvent self.gotDisconnected = 0 threading.Thread.__init__(self) def run(self): self.storage.tpc_begin(self.trans) oid = self.storage.new_oid() self.storage.store(oid, ZERO, zodb_pickle(MinPO("c")), '', self.trans) self.storage.tpc_vote(self.trans) self.threadStartedEvent.set() self.doCloseEvent.wait(10) try: self.storage.tpc_finish(self.trans) except Disconnected: self.gotDisconnected = 1 class ThreadTests: # Thread 1 should start a transaction, but not get all the way through it. # Main thread should close the connection. Thread 1 should then get # disconnected. def checkDisconnectedOnThread2Close(self): doCloseEvent = threading.Event() threadStartedEvent = threading.Event() thread1 = GetsThroughVoteThread(self._storage, doCloseEvent, threadStartedEvent) thread1.start() threadStartedEvent.wait(10) self._storage.close() doCloseEvent.set() thread1.join() self.assertEqual(thread1.gotDisconnected, 1) === ZEO/ZEO/tests/testZEO.py 1.16.4.4.2.5 => 1.16.4.4.2.6 === import zLOG -from ZEO.tests import forker, Cache, CommitLockTests +from ZEO.tests import forker, Cache, CommitLockTests, ThreadTests from ZEO.smac import Disconnected # Sorry Jim... @@ -72,6 +72,7 @@ MTStorage.MTStorage, ReadOnlyStorage.ReadOnlyStorage, CommitLockTests.CommitLockTests, + ThreadTests.ThreadTests, ): """An abstract base class for ZEO tests From barry at wooz.org Wed May 15 01:16:27 2002 From: barry at wooz.org (Barry Warsaw) Date: Sun Aug 10 16:31:21 2008 Subject: [ZEO-Checkins] CVS: ZEO/ZEO - ClientStorage.py:1.35.6.4.2.5 Message-ID: <200205150516.g4F5GRa06732@cvs.baymountain.com> Update of /cvs-repository/ZEO/ZEO In directory cvs.zope.org:/tmp/cvs-serv6721/ZEO Modified Files: Tag: ZEO2-branch ClientStorage.py Log Message: tpc_begin(): Add a comment explaining why the release of the tpc_cond lock is safe. === ZEO/ZEO/ClientStorage.py 1.35.6.4.2.4 => 1.35.6.4.2.5 === while self._transaction is not None: if self._transaction == transaction: + # Our tpc_cond lock is re-entrant. It is allowable for a + # client to call two tpc_begins in a row with the same + # transaction, and the second of these must be ignored. Our + # locking is safe because the acquire() above gives us a + # second lock on tpc_cond, and the following release() brings + # us back to owning just the one tpc_cond lock (acquired + # during the first of two consecutive tpc_begins). self.tpc_cond.release() return self.tpc_cond.wait() From barry at wooz.org Wed May 15 19:44:32 2002 From: barry at wooz.org (Barry Warsaw) Date: Sun Aug 10 16:31:21 2008 Subject: [ZEO-Checkins] CVS: ZEO/ZEO/tests - ThreadTests.py:1.1.2.2 Message-ID: <200205152344.g4FNiWG10383@cvs.baymountain.com> Update of /cvs-repository/ZEO/ZEO/tests In directory cvs.zope.org:/tmp/cvs-serv10372 Modified Files: Tag: ZEO2-branch ThreadTests.py Log Message: Updates and additions to the threading tests. We specifically want to make sure: - that if a storage is closed in one thread, while another thread is in the middle of a transaction, the second thread will get an exception when it tries to tpc_finish(); - that if one thread begins a transaction, and a second thread starts a transaction, the second will block in the tpc_begin() for a while, getting an exception when a third thread closes the storage; - that in the situation above, the second thread, which never successfully got through its tpc_begin() does not get the transaction lock (I think we're only moderately successful in testing this situation); These tests and the accompanying fixes to ClientStorage should knock off the annoying "AssertionError: notify() of un-acquire()d lock" errors in the ZEO2 thread. === ZEO/ZEO/tests/ThreadTests.py 1.1.2.1 => 1.1.2.2 === ZERO = '\0'*8 -class DummyDB: - def invalidate(self, *args): - pass - - -class GetsThroughVoteThread(threading.Thread): - # run the entire test in a thread so that the blocking call for - # tpc_vote() doesn't hang the test suite. - - def __init__(self, storage, doCloseEvent, threadStartedEvent): +class BasicThread(threading.Thread): + def __init__(self, storage, doNextEvent, threadStartedEvent): self.storage = storage self.trans = Transaction() - self.doCloseEvent = doCloseEvent + self.doNextEvent = doNextEvent self.threadStartedEvent = threadStartedEvent + self.gotValueError = 0 self.gotDisconnected = 0 threading.Thread.__init__(self) + +class GetsThroughVoteThread(BasicThread): + # This thread gets partially through a transaction before it turns + # execution over to another thread. We're trying to establish that a + # tpc_finish() after a storage has been closed by another thread will get + # a ClientStorageError error. + # + # This class gets does a tpc_begin(), store(), tpc_vote() and is waiting + # to do the tpc_finish() when the other thread closes the storage. def run(self): self.storage.tpc_begin(self.trans) oid = self.storage.new_oid() self.storage.store(oid, ZERO, zodb_pickle(MinPO("c")), '', self.trans) self.storage.tpc_vote(self.trans) self.threadStartedEvent.set() - self.doCloseEvent.wait(10) + self.doNextEvent.wait(10) try: self.storage.tpc_finish(self.trans) + except ZEO.ClientStorage.ClientStorageError: + self.gotValueError = 1 + self.storage.tpc_abort(self.trans) + + +class GetsThroughBeginThread(BasicThread): + # This class is like the above except that it is intended to be run when + # another thread is already in a tpc_begin(). Thus, this thread will + # block in the tpc_begin until another thread closes the storage. When + # that happens, this one will get disconnected too. + def run(self): + try: + self.storage.tpc_begin(self.trans) + except ZEO.ClientStorage.ClientStorageError: + self.gotValueError = 1 + + +class AbortsAfterBeginFailsThread(BasicThread): + # This class is identical to GetsThroughBeginThread except that it + # attempts to tpc_abort() after the tpc_begin() fails. That will raise a + # ClientDisconnected exception which implies that we don't have the lock, + # and that's what we really want to test (but it's difficult given the + # threading module's API). + def run(self): + try: + self.storage.tpc_begin(self.trans) + except ZEO.ClientStorage.ClientStorageError: + self.gotValueError = 1 + try: + self.storage.tpc_abort(self.trans) except Disconnected: self.gotDisconnected = 1 class ThreadTests: - # Thread 1 should start a transaction, but not get all the way through it. # Main thread should close the connection. Thread 1 should then get # disconnected. - def checkDisconnectedOnThread2Close(self): - doCloseEvent = threading.Event() + doNextEvent = threading.Event() threadStartedEvent = threading.Event() thread1 = GetsThroughVoteThread(self._storage, - doCloseEvent, threadStartedEvent) + doNextEvent, threadStartedEvent) thread1.start() threadStartedEvent.wait(10) self._storage.close() - doCloseEvent.set() + doNextEvent.set() + thread1.join() + self.assertEqual(thread1.gotValueError, 1) + + # Thread 1 should start a transaction, but not get all the way through + # it. While thread 1 is in the middle of the transaction, a second thread + # should start a transaction, and it will block in the tcp_begin() -- + # because thread 1 has acquired the lock in its tpc_begin(). Now the main + # thread closes the storage and both sub-threads should get disconnected. + def checkSecondBeginFails(self): + doNextEvent = threading.Event() + threadStartedEvent = threading.Event() + thread1 = GetsThroughVoteThread(self._storage, + doNextEvent, threadStartedEvent) + thread2 = GetsThroughBeginThread(self._storage, + doNextEvent, threadStartedEvent) + thread1.start() + threadStartedEvent.wait(1) + thread2.start() + self._storage.close() + doNextEvent.set() + thread1.join() + thread2.join() + self.assertEqual(thread1.gotValueError, 1) + self.assertEqual(thread2.gotValueError, 1) + + def checkThatFailedBeginDoesNotHaveLock(self): + doNextEvent = threading.Event() + threadStartedEvent = threading.Event() + thread1 = GetsThroughVoteThread(self._storage, + doNextEvent, threadStartedEvent) + thread2 = AbortsAfterBeginFailsThread(self._storage, + doNextEvent, threadStartedEvent) + thread1.start() + threadStartedEvent.wait(1) + thread2.start() + self._storage.close() + doNextEvent.set() thread1.join() - self.assertEqual(thread1.gotDisconnected, 1) + thread2.join() + self.assertEqual(thread1.gotValueError, 1) + self.assertEqual(thread2.gotValueError, 1) + self.assertEqual(thread2.gotDisconnected, 1) From barry at wooz.org Wed May 15 19:47:49 2002 From: barry at wooz.org (Barry Warsaw) Date: Sun Aug 10 16:31:21 2008 Subject: [ZEO-Checkins] CVS: ZEO/ZEO - ClientStorage.py:1.35.6.4.2.6 Message-ID: <200205152347.g4FNlnf11839@cvs.baymountain.com> Update of /cvs-repository/ZEO/ZEO In directory cvs.zope.org:/tmp/cvs-serv11828 Modified Files: Tag: ZEO2-branch ClientStorage.py Log Message: Updates and additions to the threading tests. We specifically want to make sure: - that if a storage is closed in one thread, while another thread is in the middle of a transaction, the second thread will get an exception when it tries to tpc_finish(); - that if one thread begins a transaction, and a second thread starts a transaction, the second will block in the tpc_begin() for a while, getting an exception when a third thread closes the storage; - that in the situation above, the second thread, which never successfully got through its tpc_begin() does not get the transaction lock (I think we're only moderately successful in testing this situation); These tests and the accompanying fixes to ClientStorage should knock off the annoying "AssertionError: notify() of un-acquire()d lock" errors in the ZEO2 branch. Changes to ClientStorage include: - notifyDisconnected(): Don't do the notifyAll(), release() or setting of self._transaction to None. This is now the responsibility of tpc_finish() and tpc_abort(). - tpc_abort(), tpc_finish(): Wrap the core logic in a try/finally which does the proper cleanup that notifyDisconnected() used to do. - _update_cache(): Catch the ValueError that can happen in the begin_iterate() call if the backing storage has already been closed. Transform this into a ClientStorageError, just like the try/except in the body of the "while 1:" was doing. === ZEO/ZEO/ClientStorage.py 1.35.6.4.2.5 => 1.35.6.4.2.6 === log2(PROBLEM, "Disconnected from storage") self._server = disconnected_stub - if self._transaction: - self._transaction = None - self.tpc_cond.notifyAll() - self.tpc_cond.release() def __len__(self): return self._info['length'] @@ -334,13 +330,15 @@ def tpc_abort(self, transaction): if transaction is not self._transaction: return - self._server.tpc_abort(self._serial) - self._tbuf.clear() - self._seriald.clear() - del self._serials[:] - self._transaction = None - self.tpc_cond.notify() - self.tpc_cond.release() + try: + self._server.tpc_abort(self._serial) + self._tbuf.clear() + self._seriald.clear() + del self._serials[:] + finally: + self._transaction = None + self.tpc_cond.notify() + self.tpc_cond.release() def tpc_begin(self, transaction, tid=None, status=' '): self.tpc_cond.acquire() @@ -391,25 +389,31 @@ def tpc_finish(self, transaction, f=None): if transaction is not self._transaction: return - if f is not None: - f() - - self._server.tpc_finish(self._serial) + try: + if f is not None: + f() - r = self._check_serials() - assert r is None or len(r) == 0, "unhandled serialnos: %s" % r + self._server.tpc_finish(self._serial) - self._update_cache() + r = self._check_serials() + assert r is None or len(r) == 0, "unhandled serialnos: %s" % r - self._transaction = None - self.tpc_cond.notify() - self.tpc_cond.release() + self._update_cache() + finally: + self._transaction = None + self.tpc_cond.notify() + self.tpc_cond.release() def _update_cache(self): # Iterate over the objects in the transaction buffer and # update or invalidate the cache. self._cache.checkSize(self._tbuf.get_size()) - self._tbuf.begin_iterate() + try: + self._tbuf.begin_iterate() + except ValueError, msg: + raise ClientStorageError, ( + "Unexpected error reading temporary file in " + "client storage: %s" % msg) while 1: try: t = self._tbuf.next() From jeremy at zope.com Fri May 17 15:37:44 2002 From: jeremy at zope.com (Jeremy Hylton) Date: Sun Aug 10 16:31:21 2008 Subject: [ZEO-Checkins] CVS: ZEO - README.txt:1.1.4.1 Message-ID: <200205171937.g4HJbi727316@cvs.baymountain.com> Update of /cvs-repository/ZEO In directory cvs.zope.org:/tmp/cvs-serv24622 Added Files: Tag: ZEO2-branch README.txt Log Message: Add preliminary versions of some docs. === Added File ZEO/README.txt === Zope Enterprise Objects, version 2 Zope Enterprise Objects (ZEO) extends the Zope Object Database (ZODB) to multiple processes, machines, and locations. It provides scalability, high availability, and distribution for ZODB. For more information, see the ZEO Web page at http://www.zope.org/Products/ZEO/. ZEO version 2 is not backwards compatible with ZEO 1.0. A system that uses ZEO must upgrade all clients and the same time it upgrades the server. The ZEO package is contained in the directory named ZEO. If you are using Zope, see docs/ZopeREADME.txt; otherwise, see docs/NonZopeREADME.txt. From jeremy at zope.com Fri May 17 15:37:44 2002 From: jeremy at zope.com (Jeremy Hylton) Date: Sun Aug 10 16:31:21 2008 Subject: [ZEO-Checkins] CVS: ZEO/ZEO - version.txt:1.1.2.1 Message-ID: <200205171937.g4HJbi827328@cvs.baymountain.com> Update of /cvs-repository/ZEO/ZEO In directory cvs.zope.org:/tmp/cvs-serv24622/ZEO Added Files: Tag: ZEO2-branch version.txt Log Message: Add preliminary versions of some docs. === Added File ZEO/ZEO/version.txt === 2.0a1 From jeremy at zope.com Fri May 17 15:37:44 2002 From: jeremy at zope.com (Jeremy Hylton) Date: Sun Aug 10 16:31:21 2008 Subject: [ZEO-Checkins] CVS: ZEO/docs - ZopeREADME.txt:1.4.4.1 Message-ID: <200205171937.g4HJbio27324@cvs.baymountain.com> Update of /cvs-repository/ZEO/docs In directory cvs.zope.org:/tmp/cvs-serv24622/docs Added Files: Tag: ZEO2-branch ZopeREADME.txt Log Message: Add preliminary versions of some docs. === Added File ZEO/docs/ZopeREADME.txt === Zope Enterprise Objects Installation ZEO 2.0 requires Zope 2.5 or higher and Python 2.1 or higher. Put the package (the ZEO directory, without any wrapping directory included in a distribution) in your Zope lib/python. The setup.py script in the top-level ZEO directory can also be used. Run "python setup.py install --home=ZOPE" where ZOPE is the top-level Zope directory. Starting (and configuring) the ZEO Server To start the storage server, go to your Zope install directory and:: python lib/python/ZEO/start.py -p port_number (Run start without arguments to see options.) Of course, the server and the client don't have to be on the same machine. If the server and client *are* on the same machine, then you can use a Unix domain socket:: python lib/python/ZEO/start.py -U filename The start script provides a number of options not documented here. See doc/start.txt for more information. Running Zope as a ZEO client To get Zope to use the server, create a custom_zodb module, custom_zodb.py, in your Zope install directory, so that Zope uses a ClientStorage:: import ZEO.ClientStorage Storage=ZEO.ClientStorage.ClientStorage(('',port_number)) (See the misc/custom_zodb.py for an example.) You can specify a host name (rather than '') if you want. The port number is, of course, the port number used to start the storage server. You can also give the name of a Unix domain socket file:: import ZEO.ClientStorage Storage=ZEO.ClientStorage.ClientStorage(filename) There are a number of configuration options available for the ClientStorage. See doc/ClientStorage.txt for details. If you want a persistent client cache which retains cache contents across ClientStorage restarts, you need to define the environment variable, ZEO_CLIENT, to a unique name for the client. This is needed so that unique cache name files can be computed. Otherwise, the client cache is stored in temporary files which are removed when the ClientStorage shuts down. For example, to start two Zope processes with unique caches, use something like: python z2.py -P8700 ZEO_CLIENT=8700 python z2.py -P8800 ZEO_CLIENT=8800 Zope product installation Normally, Zope updates the Zope database during startup to reflect product changes or new products found. It makes no sense for multiple ZEO clients to do the same installation. Further, if different clients have different software installed, the correct state of the database is ambiguous. Starting in Zope 2.2, Zope will not modify the Zope database during product installation if the environment variable ZEO_CLIENT is set. Normally, Zope ZEO clients should be run with ZEO_CLIENT set so that product initialization is not performed. If you do install new Zope products, then you need to take a special step to cause the new products to be properly registered in the database. The easiest way to do this is to start Zope once with the environment variable FORCE_PRODUCT_LOAD set. The interaction between ZEO and Zope product installation is unfortunate. In the future, this interaction will be removed by From jeremy at zope.com Mon May 20 15:36:25 2002 From: jeremy at zope.com (Jeremy Hylton) Date: Sun Aug 10 16:31:21 2008 Subject: [ZEO-Checkins] CVS: ZEO - CHANGES.txt:1.28.2.1 README.txt:1.1.4.2 Message-ID: <200205201936.g4KJaPo31909@cvs.baymountain.com> Update of /cvs-repository/ZEO In directory cvs.zope.org:/tmp/cvs-serv31901 Modified Files: Tag: ZEO2-branch README.txt Added Files: Tag: ZEO2-branch CHANGES.txt Log Message: Emphasize protocol incompatibility in README.txt. Add CHANGES.txt with brief overview of 2.0 and 1.0 differences. === Added File ZEO/CHANGES.txt === Revision History, Zope Enterprise Objects, version 2 ZEO 2.0 alpha 1 Brief overview of the differences between ZEO 1.0 and 2.0. - New protocol. ZEO 2 uses a different wire protocol and a different API to make RPC calls. The new protocol was designed to be flexible and simple. It includes an initial handshake to set the version number, which should allow future changes to the protocol while reducing the difficulty of upgrades. - Better handling of concurrent commits. The ZEO server serializes concurrent commits to guarantee consistency; the mechanism is often called the distributed commit lock. ZEO 2 improves the efficiency of concurrent commits by allowing data to be transferred to the server before entering the commit lock. - The ZEO client and server can be configured to operate in read-only mode. - A ZEO client can be configured with multiple server addresses. It uses the first server it can connect to. - The wait_for_server_on_startup keyword argument to ClientStorage has been renamed wait. === ZEO/README.txt 1.1.4.1 => 1.1.4.2 === information, see the ZEO Web page at http://www.zope.org/Products/ZEO/. - ZEO version 2 is not backwards compatible with ZEO 1.0. A system - that uses ZEO must upgrade all clients and the same time it upgrades - the server. + IMPORTANT: ZEO version 2 is not backwards compatible with ZEO 1.0. + A system that uses ZEO must upgrade all clients and the same time it + upgrades the server. The ZEO package is contained in the directory named ZEO. From jeremy at zope.com Mon May 20 15:38:11 2002 From: jeremy at zope.com (Jeremy Hylton) Date: Sun Aug 10 16:31:21 2008 Subject: [ZEO-Checkins] CVS: ZEO - setup.py:1.1.2.1 test.py:1.1.2.1 Message-ID: <200205201938.g4KJcBT32497@cvs.baymountain.com> Update of /cvs-repository/ZEO In directory cvs.zope.org:/tmp/cvs-serv32055 Added Files: Tag: ZEO2-branch setup.py test.py Log Message: Add a setup.py and test.py based on StandaloneZODB. === Added File ZEO/setup.py === from distutils.core import setup packages = ['ZEO', 'ZEO.zrpc', 'ZEO.tests'] setup(name="ZEO", version="2.0a1", description="Zope Enterprise Objects", maintainer="Zope Corp.", maintainer_email="zodb-dev@zope.org", url = "http://www.zope.org/Products/ZEO", packages = packages, ) === Added File ZEO/test.py === """Test harness for StandaloneZODB""" import os import re import sys import traceback import unittest from distutils.util import get_platform class ImmediateTestResult(unittest._TextTestResult): def _print_traceback(self, msg, err, test, errlist): if self.showAll or self.dots: self.stream.writeln("\n") tb = ''.join(traceback.format_exception(*err)) self.stream.writeln(msg) self.stream.writeln(tb) errlist.append((test, tb)) def addError(self, test, err): self._print_traceback("Error in test %s" % test, err, test, self.errors) def addFailure(self, test, err): self._print_traceback("Failure in test %s" % test, err, test, self.failures) def printErrorList(self, flavor, errors): for test, err in errors: self.stream.writeln(self.separator1) self.stream.writeln("%s: %s" % (flavor, self.getDescription(test))) self.stream.writeln(self.separator2) self.stream.writeln(err) class ImmediateTestRunner(unittest.TextTestRunner): def _makeResult(self): return ImmediateTestResult(self.stream, self.descriptions, self.verbosity) # setup list of directories to put on the path PLAT_SPEC = "%s-%s" % (get_platform(), sys.version[0:3]) def setup_path(): DIRS = ["lib", "lib.%s" % PLAT_SPEC, ] for d in DIRS: sys.path.insert(0, d) # Find test files. # They live under either a lib.PLAT_SPEC or plain "lib" directory. _sep = re.escape(os.sep) _pat = "%s(%s|lib)%s" % (_sep, re.escape("lib." + PLAT_SPEC), _sep) hasgooddir = re.compile(_pat).search del _sep, _pat class TestFileFinder: def __init__(self): self.files = [] def visit(self, rx, dir, files): if dir[-5:] != "tests": return # ignore tests that aren't in packages if not "__init__.py" in files: print "not a package", dir return for file in files: if file[:4] == "test" and file[-3:] == ".py": path = os.path.join(dir, file) if not hasgooddir(path): # built for a different version continue if rx is not None: if rx.search(path): self.files.append(path) else: self.files.append(path) def find_tests(filter): if filter is not None: rx = re.compile(filter) else: rx = None finder = TestFileFinder() os.path.walk("build", finder.visit, rx) return finder.files def package_import(modname): mod = __import__(modname) for part in modname.split(".")[1:]: mod = getattr(mod, part) return mod def module_from_path(path): """Return the Python package name indiciated by the filesystem path. The path starts with build/lib or build /lib.mumble...""" assert path[-3:] == '.py' path = path[:-3] dirs = [] while path: path, end = os.path.split(path) dirs.insert(0, end) assert dirs[0] == "build" assert dirs[1][:3] == "lib" return ".".join(dirs[2:]) def get_suite(file): assert file[:5] == "build" assert file[-3:] == '.py' modname = module_from_path(file) mod = package_import(modname) try: return mod.test_suite() except AttributeError: return None def match(rx, s): if not rx: return 1 if rx[0] == '!': return re.search(rx[1:], s) is None else: return re.search(rx, s) is not None def filter_testcases(s, rx): new = unittest.TestSuite() for test in s._tests: if isinstance(test, unittest.TestCase): name = test.id() # Full test name: package.module.class.method name = name[1 + name.rfind('.'):] # extract method name if match(rx, name): new.addTest(test) else: filtered = filter_testcases(test, rx) if filtered: new.addTest(filtered) return new def runner(files, test_filter, debug): runner = ImmediateTestRunner(verbosity=VERBOSE) suite = unittest.TestSuite() for file in files: s = get_suite(file) if s is not None: if test_filter is not None: s = filter_testcases(s, test_filter) suite.addTest(s) if debug: suite.debug() return 0 r = runner.run(suite) return len(r.errors) + len(r.failures) def main(module_filter, test_filter): setup_path() files = find_tests(module_filter) files.sort() os.chdir("build") if LOOP: while 1: runner(files, test_filter, debug) else: runner(files, test_filter, debug) if __name__ == "__main__": import getopt module_filter = None test_filter = None VERBOSE = 0 LOOP = 0 debug = 0 # Don't collect test results; simply let tests crash build = 0 try: opts, args = getopt.getopt(sys.argv[1:], 'vdLbh') except getopt.error, msg: print msg print "Try `python %s -h' for more information." % sys.argv[0] sys.exit(2) for k, v in opts: if k == '-v': VERBOSE = VERBOSE + 1 elif k == '-d': debug = 1 elif k == '-L': LOOP = 1 elif k == '-b': build = 1 elif k == '-h': print __doc__ sys.exit(0) if build: cmd = sys.executable + " setup.py -q build" if VERBOSE: print cmd sts = os.system(cmd) if sts: print "Build failed", hex(sts) sys.exit(1) if args: if len(args) > 1: test_filter = args[1] module_filter = args[0] try: bad = main(module_filter, test_filter) if bad: sys.exit(1) except ImportError, err: print err print sys.path raise From jeremy at zope.com Tue May 21 01:33:31 2002 From: jeremy at zope.com (Jeremy Hylton) Date: Sun Aug 10 16:31:21 2008 Subject: [ZEO-Checkins] CVS: StandaloneZODB/ZEO - __init__.py:1.7.6.1.2.2 asyncwrap.py:NONE fap.py:NONE Message-ID: <200205210533.g4L5XVl24402@cvs.baymountain.com> Update of /cvs-repository/StandaloneZODB/ZEO In directory cvs.zope.org:/tmp/cvs-serv24391/ZEO Modified Files: Tag: ZEO2-branch __init__.py Removed Files: Tag: ZEO2-branch asyncwrap.py fap.py Log Message: fap and asyncwrap are not needed w/ Python 2.1 === StandaloneZODB/ZEO/__init__.py 1.7.6.1.2.1 => 1.7.6.1.2.2 === # ############################################################################## -import fap === Removed File StandaloneZODB/ZEO/asyncwrap.py === === Removed File StandaloneZODB/ZEO/fap.py === From jeremy at zope.com Tue May 21 01:41:37 2002 From: jeremy at zope.com (Jeremy Hylton) Date: Sun Aug 10 16:31:21 2008 Subject: [ZEO-Checkins] CVS: ZEO/doc ZEO/doc - New directory Message-ID: <200205210541.g4L5fbr26386@cvs.baymountain.com> Update of /cvs-repository/ZEO/doc In directory cvs.zope.org:/tmp/cvs-serv26380/doc Log Message: Directory /cvs-repository/ZEO/doc added to the repository --> Using per-directory sticky tag `ZEO2-branch' === Added directory ZEO/doc === From jeremy at zope.com Tue May 21 01:41:59 2002 From: jeremy at zope.com (Jeremy Hylton) Date: Sun Aug 10 16:31:21 2008 Subject: [ZEO-Checkins] CVS: ZEO - test.py:1.1.2.2 Message-ID: <200205210541.g4L5fxj26418@cvs.baymountain.com> Update of /cvs-repository/ZEO In directory cvs.zope.org:/tmp/cvs-serv26411 Modified Files: Tag: ZEO2-branch test.py Log Message: Add a bit of usage / doc string === ZEO/test.py 1.1.2.1 => 1.1.2.2 === +"""Test harness for ZEO + +usage: python test.py [options] [modulepath] [testcase] + +options: + -v -- verbose (can be repeated to increase verbosity) + -b -- run "setup.py -q build" before running the tests + -d -- run tests in debug mode + -L -- run tests in an infinite loop + -h -- print this message + +The optional modulepath and testcase arguments are regular expressions +that can be used to limit the number of tests that are run. The +modulepath regex must be found in the path of the module that contains +the tests. The testcase regex must be found in the name of the test case. + +When it finishes, the test harness prints a report of the tests run +and how long it took. If errors or failures occured, they will be +reported along with a traceback. If one -v is specified, a dot will +be printed as each test is run. If two -v's are specified, the name +of each test will be printed as it runs. +""" import os import re From jeremy at zope.com Tue May 21 01:43:04 2002 From: jeremy at zope.com (Jeremy Hylton) Date: Sun Aug 10 16:31:21 2008 Subject: [ZEO-Checkins] CVS: ZEO/doc - ZopeREADME.txt:1.1.2.1 start.txt:1.1.2.1 Message-ID: <200205210543.g4L5h4O26961@cvs.baymountain.com> Update of /cvs-repository/ZEO/doc In directory cvs.zope.org:/tmp/cvs-serv26953/doc Added Files: Tag: ZEO2-branch ZopeREADME.txt start.txt Log Message: Start moving stuff to the doc tree === Added File ZEO/doc/ZopeREADME.txt === Zope Enterprise Objects Installation ZEO 2.0 requires Zope 2.4 or higher and Python 2.1 or higher. If you use Python 2.1, we recommend the latest minor release (2.1.3 as of this writing) because it includes a few bug fixes that affect ZEO. Put the package (the ZEO directory, without any wrapping directory included in a distribution) in your Zope lib/python. The setup.py script in the top-level ZEO directory can also be used. Run "python setup.py install --home=ZOPE" where ZOPE is the top-level Zope directory. Starting (and configuring) the ZEO Server To start the storage server, go to your Zope install directory and:: python lib/python/ZEO/start.py -p port_number (Run start without arguments to see options.) Of course, the server and the client don't have to be on the same machine. If the server and client *are* on the same machine, then you can use a Unix domain socket:: python lib/python/ZEO/start.py -U filename The start script provides a number of options not documented here. See docs/start.txt for more information. Running Zope as a ZEO client To get Zope to use the server, create a custom_zodb module, custom_zodb.py, in your Zope install directory, so that Zope uses a ClientStorage:: from ZEO.ClientStorage import ClientStorage Storage = ClientStorage(('',port_number)) (See the misc/custom_zodb.py for an example.) You can specify a host name (rather than '') if you want. The port number is, of course, the port number used to start the storage server. You can also give the name of a Unix domain socket file:: import ZEO.ClientStorage Storage=ZEO.ClientStorage.ClientStorage(filename) There are a number of configuration options available for the ClientStorage. See doc/ClientStorage.txt for details. If you want a persistent client cache which retains cache contents across ClientStorage restarts, you need to define the environment variable, ZEO_CLIENT, to a unique name for the client. This is needed so that unique cache name files can be computed. Otherwise, the client cache is stored in temporary files which are removed when the ClientStorage shuts down. For example, to start two Zope processes with unique caches, use something like: python z2.py -P8700 ZEO_CLIENT=8700 python z2.py -P8800 ZEO_CLIENT=8800 Zope product installation Normally, Zope updates the Zope database during startup to reflect product changes or new products found. It makes no sense for multiple ZEO clients to do the same installation. Further, if different clients have different software installed, the correct state of the database is ambiguous. Starting in Zope 2.2, Zope will not modify the Zope database during product installation if the environment variable ZEO_CLIENT is set. Normally, Zope ZEO clients should be run with ZEO_CLIENT set so that product initialization is not performed. If you do install new Zope products, then you need to take a special step to cause the new products to be properly registered in the database. The easiest way to do this is to start Zope once with the environment variable FORCE_PRODUCT_LOAD set. The interaction between ZEO and Zope product installation is unfortunate. In the future, this interaction will be removed by === Added File ZEO/doc/start.txt === The ZEO Server start script, start.py ZEO provides a Python script for starting the ZEO server. The ZEO server is implemented as a Python class and could be used with other main programs, however, a simple ZEO server is provided for convenience. Basic usage To start the storage server, go to your Zope install directory and:: python lib/python/ZEO/start.py -p port_number (Run start without arguments to see options.) Of course, the server and the client don't have to be on the same machine. If the server and client *are* on the same machine, then you can use a Unix domain socket:: python lib/python/ZEO/start.py -U filename Serving custom storages or multiple storages with the storage server The Storage server can host multiple storages and can host any kind of storage. Each storage has a unique storage name. By default, the ZEO start.py script serves a standard FileStorage with the name '1'. You can control what storages are served by creating a Python file containing definitions for the storages and using the '-S' option to the start.py script to indicate the storage to be served. The form of the -S option is:: -Sstorage_name=module_path:attribute_name Where: storage_name -- is the storage name used in the ZEO protocol. This is the name that you give as the optional 'storage' keyword argument to the ClientStorage constructor. module_path -- This is the path to a Python module that defines the storage object(s) to be served. The module path should ommit the prefix (e.g. '.py'). attribute_name -- This is the name to which the storage object is assigned in the module. Consider the following example. I want to serve a FileStorage in read-only mode, which I define in the module file /stores/fs.py:: import ZODB.FileStorage Storage=FileStorage.FileStorage('/stores/fs1.fs', read_only=1) I then start start.py with the argument:: python lib/python/ZEO/start.py -U /xxx/var/zeo.sock \ -S 1=/stores/fs:Storage This option says to serve storage '1'. Storage '1' is found in attribute 'Storage' from the module '/stores/fs'. Now consider a more complicated example. I want to serve the storage from the previous example. I also want to serve two Oracle storages that are defined in the file '/stores/oracle.py':: import DCOracle, DCOracleStorage system=DCOracleStorage.DCOracleStorage( lambda : DCOracle.Connect('system/manager@spamservice') ) scott=DCOracleStorage.DCOracleStorage( lambda : DCOracle.Connect('scott/tiger@spamservice') ) I simply need to include three -S options:: python lib/python/ZEO/start.py -U /xxx/var/zeo.sock \ -Ssystem=/stores/oracle:system \ -Sscott=/stores/oracle:scott \ -S1=/stores/fs:Storage In this case, we made the storage and attribute name the same. To connect to the 'system' or 'scott' storage, we need to specify the storage in the ClientStorage constructor, as in:: import ZEO.ClientStorage Storage=ZEO.ClientStorage.ClientStorage( '/xxx/var/zeo.sock', storage='scott') Options The ZEO server start script is run with one or more command line options. An optional FileStorage file name may be provided after the options. The options are as follows: -D -- Run in debug mode In debug mode, the process is not run in the background and detailed debugging information is logged. Note that to actually log this information, you need to configure logging to include very low-severity (< -300) log entries. For example, to configure the stupid logger to log these messages, set the environment veriable 'STUPID_LOG_SEVERITY' to -999. -U -- Unix-domain socket file to listen on If you want to accept connections on a Unix domain socket, then use this option to specify the socket file name. -u username or uid number The username to run the ZEO server as. You may want to run the ZEO server as 'zope' or some other user with limited resouces. The only works under Unix, and if ZServer is started by root. If the server *is* started as root, the 'nobody' user if this option isn't used. -p port -- port to listen on Use this option together with the '-h' option to specify a host and port to listen on. -h adddress -- host address to listen on Use this option together with the '-p' option to specify a host and port to listen on. -s -- Don't use zdeamon This option has no effect on Unix. -S storage_name=module_path:attr_name -- A storage specification where: storage_name -- is the storage name used in the ZEO protocol. This is the name that you give as the optional 'storage' keyword argument to the ClientStorage constructor. module_path -- This is the path to a Python module that defines the storage object(s) to be served. The module path should ommit the prefix (e.g. '.py'). attr_name -- This is the name to which the storage object is assigned in the module. From jeremy at zope.com Tue May 21 01:43:20 2002 From: jeremy at zope.com (Jeremy Hylton) Date: Sun Aug 10 16:31:21 2008 Subject: [ZEO-Checkins] CVS: ZEO - .cvsignore:1.1.2.1 Message-ID: <200205210543.g4L5hK226987@cvs.baymountain.com> Update of /cvs-repository/ZEO In directory cvs.zope.org:/tmp/cvs-serv26980 Added Files: Tag: ZEO2-branch .cvsignore Log Message: ignore the distutils build directory === Added File ZEO/.cvsignore === build From jeremy at zope.com Tue May 21 01:43:45 2002 From: jeremy at zope.com (Jeremy Hylton) Date: Sun Aug 10 16:31:21 2008 Subject: [ZEO-Checkins] CVS: ZEO/docs - ZopeREADME.txt:NONE Message-ID: <200205210543.g4L5hjD27012@cvs.baymountain.com> Update of /cvs-repository/ZEO/docs In directory cvs.zope.org:/tmp/cvs-serv27006/docs Removed Files: Tag: ZEO2-branch ZopeREADME.txt Log Message: moved to doc === Removed File ZEO/docs/ZopeREADME.txt === From jeremy at zope.com Tue May 21 01:48:35 2002 From: jeremy at zope.com (Jeremy Hylton) Date: Sun Aug 10 16:31:21 2008 Subject: [ZEO-Checkins] CVS: ZEO/doc - ZopeREADME.txt:1.1.2.2 Message-ID: <200205210548.g4L5mZ128279@cvs.baymountain.com> Update of /cvs-repository/ZEO/doc In directory cvs.zope.org:/tmp/cvs-serv28272 Modified Files: Tag: ZEO2-branch ZopeREADME.txt Log Message: minor changes & an XXX comment === ZEO/doc/ZopeREADME.txt 1.1.2.1 => 1.1.2.2 === The start script provides a number of options not documented here. - See docs/start.txt for more information. + See doc/start.txt for more information. Running Zope as a ZEO client @@ -40,9 +40,7 @@ ClientStorage:: from ZEO.ClientStorage import ClientStorage - Storage = ClientStorage(('',port_number)) - - (See the misc/custom_zodb.py for an example.) + Storage = ClientStorage(('', port_number)) You can specify a host name (rather than '') if you want. The port number is, of course, the port number used to start the storage @@ -50,8 +48,8 @@ You can also give the name of a Unix domain socket file:: - import ZEO.ClientStorage - Storage=ZEO.ClientStorage.ClientStorage(filename) + from ZEO.ClientStorage import ClientStorage + Storage = ClientStorage(filename) There are a number of configuration options available for the ClientStorage. See doc/ClientStorage.txt for details. @@ -62,22 +60,23 @@ needed so that unique cache name files can be computed. Otherwise, the client cache is stored in temporary files which are removed when the ClientStorage shuts down. For example, to start two Zope - processes with unique caches, use something like: + processes with unique caches, use something like:: python z2.py -P8700 ZEO_CLIENT=8700 python z2.py -P8800 ZEO_CLIENT=8800 Zope product installation + XXX Does this still apply? + Normally, Zope updates the Zope database during startup to reflect product changes or new products found. It makes no sense for multiple ZEO clients to do the same installation. Further, if different clients have different software installed, the correct state of the database is ambiguous. - Starting in Zope 2.2, Zope will not modify the Zope database - during product installation if the environment variable ZEO_CLIENT - is set. + Zope will not modify the Zope database during product installation + if the environment variable ZEO_CLIENT is set. Normally, Zope ZEO clients should be run with ZEO_CLIENT set so that product initialization is not performed. From jeremy at zope.com Tue May 21 11:57:14 2002 From: jeremy at zope.com (Jeremy Hylton) Date: Sun Aug 10 16:31:21 2008 Subject: [ZEO-Checkins] CVS: ZEO - README.txt:1.1.4.3 Message-ID: <200205211557.g4LFvEC28170@cvs.baymountain.com> Update of /cvs-repository/ZEO In directory cvs.zope.org:/tmp/cvs-serv28163 Modified Files: Tag: ZEO2-branch README.txt Log Message: tweaks === ZEO/README.txt 1.1.4.2 => 1.1.4.3 === IMPORTANT: ZEO version 2 is not backwards compatible with ZEO 1.0. - A system that uses ZEO must upgrade all clients and the same time it - upgrades the server. + A system that uses ZEO must upgrade all clients and servers at the + same time. The ZEO package is contained in the directory named ZEO. - If you are using Zope, see docs/ZopeREADME.txt; otherwise, see - docs/NonZopeREADME.txt. + If you are using Zope, see doc/ZopeREADME.txt; otherwise, see + doc/NonZopeREADME.txt. From jeremy at zope.com Tue May 21 11:58:14 2002 From: jeremy at zope.com (Jeremy Hylton) Date: Sun Aug 10 16:31:21 2008 Subject: [ZEO-Checkins] CVS: ZEO/doc - ClientCache.txt:1.1.2.1 ClientStorage.txt:1.1.2.1 NonZopeREADME.txt:1.1.2.1 ZopeREADME.txt:1.1.2.3 Message-ID: <200205211558.g4LFwEc28643@cvs.baymountain.com> Update of /cvs-repository/ZEO/doc In directory cvs.zope.org:/tmp/cvs-serv28633/doc Modified Files: Tag: ZEO2-branch ZopeREADME.txt Added Files: Tag: ZEO2-branch ClientCache.txt ClientStorage.txt NonZopeREADME.txt Log Message: Add copies of files from ZEO/docs on the trunk. Update ZopeREADME and NonZopeREADME to be consistent in their explanations. Add brief explanation of test.py. === Added File ZEO/doc/ClientCache.txt === ZEO Client Cache The Client cache provides a disk based cache for each ZEO client. The client cache allows reads to be done from local disk rather than by remote access to the storage server. The cache may be persistent or transient. If the cache is persistent, then the cache files are retained for use after process restarts. A non-persistent cache uses temporary files that are removed when the client storage is closed. The client cache is managed as two files. The cache manager endeavors to maintain the two files at sizes less than or equal to one half the cache size. One of the cache files is designated the "current" cache file. The other cache file is designated the "old" cache file, if it exists. All writes are done to the current cache files. When transactions are committed on the client, transactions are not split between cache files. Large transactions may cause cache files to be larger than one half the target cache size. The life of the cache is as follows: - When the cache is created, the first of the two cache files is created and designated the "current" cache file. - Cache records are written to the cache file, either as transactions commit locally, or as data are loaded from the server. - When the cache file size exceeds one half the cache size, the second cache file is created and designated the "current" cache file. The first cache file becomes the "old" cache file. - Cache records are written to the new current cache file, either as transactions commit locally, or as data are loaded from the server. - When the current cache file size exceeds one half the cache size, the first cache file is recreated and designated the "current" cache file. The second cache file becomes the "old" cache file. and so on. Persistent cache files are created in the directory named in the 'var' argument to the ClientStorage (see ClientStorage.txt) or in the 'var' subdirectory of the directory given by the INSTANCE_HOME builtin (created by Zope), or in the current working directory. Persistent cache files have names of the form:: cstorage-client-n.zec where: storage -- the storage name client -- the client name, as given by the 'ZEO_CLIENT' environment variable or the 'client' argument provided when creating a client storage. n -- '0' for the first cache file and '1' for the second. For example, the second cache file for storage 'spam' and client 8881 would be named 'cspam-8881-1.zec'. === Added File ZEO/doc/ClientStorage.txt === ClientStorage The ClientStorage is a ZODB storage implementation that provides access to data served bt a ZEO server. To use a ClientStorage, create the SlientStorage and configure your application to use it. To configure Zope to use a ClientStorage, create the ClientStorage and and assign it to the 'Storage' variable in a 'custom_zodb' module (typically a Python file, with a '.py' suffix) in the instance home of your Zope installation. Creating a ClientStorage At a minimum, a client storage requires an argument (named connection) giving connection information. This argument should be a string, specifying a unix-domain socket file name, or a tuple consisting of a host and port. The host should be a string host name or IP number. The port should be a numeric port number. The ClientStorage constructor provides a number of additional options (arguments). The full list of arguments is: connection -- Connection information. This argument is either a string containing a socket file name or a tuple consisting of a string host name or ip number and an integer port. storage -- The name of the storage to connect to. A ZEO storage server can serve multiple storages. Each storage has a name, which is configured on the server. The server adminstrator should be able to provide this name. The default name for both the server and client is '1'. cache_size -- The number of bytes to allow for the client cache. The default is 20,000,000. For more information on client caches, see ClientCache.txt. name -- The name to use for the storage. This will be shown in Zope's control panel. The default name is a representation of the connection information. client -- The name to be used for the persistent client cache files. This parameter can be used instead of or to override the ZEO_CLIENT environment variable. It is generally better to use the environment variable because it's easier to change environment variables that it is to change Python code for creating the storage. Also note that, if you are using Zope, the ZEO_CLIENT environment variable effects whether products are initialized. For more information on client cache files, see ClientCache.txt. debug -- If this is provided, it should be a non-empty string. It indicates that client should log tracing and debugging information, using zLOG. var -- The directory in which persistent cache files should be written. If this option is provided, it is unnecessary to set INSTANCE_HOME in __builtins__. For more information on client cache files, see ClientCache.txt. min_disconnect_poll -- The minimum number of seconds to wait before retrying connections after connection failure. When trying to make a connection, if the connection fails, the ZEO client will wait a period of time before retrying the connection. The amount of time waited starts at the value given by 'min_disconnect_poll' and doubles on each attempt, but never exceeds the number of seconds given by 'max_disconnect_poll'. The default is 5 seconds. max_disconnect_poll -- The maximum number of seconds to wait before retrying connections after connection failure. See min_disconnect_poll. The default is 300 seconds. wait_for_server_on_starup -- Indicate whether the ClientStorage should block waiting for a storage server connection, or whether it should proceed, satisfying reads from the client cache. === Added File ZEO/doc/NonZopeREADME.txt === Zope Enterprize Objects Installation ZEO 2.0 requires Python 2.1 or higher when used without Zope. If you use Python 2.1, we recommend the latest minor release (2.1.3 as of this writing) because it includes a few bug fixes that affect ZEO. ZEO is packaged with distutils. To install it, run this command from the top-level ZEO directory:: python setup.py install The setup script will install the ZEO package in your Python site-packages directory. You can test ZEO before installing it with the test script:: python test.py -v Run the script with the -h option for a full list of options. The ZEO 2.0a1 release contains 87 unit tests on Unix. Starting (and configuring) the ZEO Server To start the storage server, go to your Zope install directory and run:: python lib/python/ZEO/start.py -p port_number This run the storage sever under zdaemon. zdaemon automatically restarts programs that exit unexpectedly. The server and the client don't have to be on the same machine. If they are on the same machine, then you can use a Unix domain socket:: python lib/python/ZEO/start.py -U filename The start script provides a number of options not documented here. See doc/start.txt for more information. Running a ZEO client In your application, create a ClientStorage, rather than, say, a FileStorage: import ZODB from ZEO.ClientStorage import ClientStorage Storage = ClientStorage(('', port_number)) db = ZODB.DB(Storage) You can specify a host name (rather than '') if you want. The port number is, of course, the port number used to start the storage server. You can also give the name of a Unix domain socket file:: import ZODB from ZEO.ClientStorage import ClientStorage Storage = ClientStorage(filename) db = ZODB.DB(Storage) There are a number of configuration options available for the ClientStorage. See ClientStorage.txt for details. If you want a persistent client cache which retains cache contents across ClientStorage restarts, you need to define the environment variable, ZEO_CLIENT, or set the client keyword argument to the constructor to a unique name for the client. This is needed so that unique cache name files can be computed. Otherwise, the client cache is stored in temporary files which are removed when the ClientStorage shuts down. Dependencies on other modules ZEO depends on other modules that are distributed with StandaloneZODB and with Zope. You can download StandaloneZODB from http://www.zope.org/Products/StandaloneZODB. === ZEO/doc/ZopeREADME.txt 1.1.2.2 => 1.1.2.3 === top-level Zope directory. + You can test ZEO before installing it with the test script:: + + python test.py -v + + Run the script with the -h option for a full list of options. The + ZEO 2.0a1 release contains 87 unit tests on Unix. + Starting (and configuring) the ZEO Server - To start the storage server, go to your Zope install directory and:: + To start the storage server, go to your Zope install directory and + run:: python lib/python/ZEO/start.py -p port_number - (Run start without arguments to see options.) - - Of course, the server and the client don't have to be on the same - machine. + This run the storage sever under zdaemon. zdaemon automatically + restarts programs that exit unexpectedly. - If the server and client *are* on the same machine, then you can use - a Unix domain socket:: + The server and the client don't have to be on the same machine. + If they are on the same machine, then you can use a Unix domain + socket:: python lib/python/ZEO/start.py -U filename @@ -56,9 +63,10 @@ If you want a persistent client cache which retains cache contents across ClientStorage restarts, you need to define the environment - variable, ZEO_CLIENT, to a unique name for the client. This is - needed so that unique cache name files can be computed. Otherwise, - the client cache is stored in temporary files which are removed when + variable, ZEO_CLIENT, or set the client keyword argument to the + constructor to a unique name for the client. This is needed so + that unique cache name files can be computed. Otherwise, the + client cache is stored in temporary files which are removed when the ClientStorage shuts down. For example, to start two Zope processes with unique caches, use something like:: From jeremy at zope.com Tue May 21 12:01:14 2002 From: jeremy at zope.com (Jeremy Hylton) Date: Sun Aug 10 16:31:21 2008 Subject: [ZEO-Checkins] CVS: StandaloneZODB/ZEO - README:NONE Message-ID: <200205211601.g4LG1E029414@cvs.baymountain.com> Update of /cvs-repository/StandaloneZODB/ZEO In directory cvs.zope.org:/tmp/cvs-serv29405 Removed Files: Tag: ZEO2-branch README Log Message: Remove out of date README file. The one in the directory above iscurrent. === Removed File StandaloneZODB/ZEO/README === From jeremy at zope.com Tue May 21 15:39:59 2002 From: jeremy at zope.com (Jeremy Hylton) Date: Sun Aug 10 16:31:21 2008 Subject: [ZEO-Checkins] CVS: ZEO/ZEO - ClientStorage.py:1.35.6.4.2.7 Message-ID: <200205211939.g4LJdxi31568@cvs.baymountain.com> Update of /cvs-repository/ZEO/ZEO In directory cvs.zope.org:/tmp/cvs-serv31557/ZEO Modified Files: Tag: ZEO2-branch ClientStorage.py Log Message: Remove unused debug kwarg to constructor === ZEO/ZEO/ClientStorage.py 1.35.6.4.2.6 => 1.35.6.4.2.7 === def __init__(self, addr, storage='1', cache_size=20000000, - name='', client='', debug=0, var=None, + name='', client='', var=None, min_disconnect_poll=5, max_disconnect_poll=300, wait=0, read_only=0): From jeremy at zope.com Tue May 21 15:41:32 2002 From: jeremy at zope.com (Jeremy Hylton) Date: Sun Aug 10 16:31:21 2008 Subject: [ZEO-Checkins] CVS: ZEO/doc - ClientStorage.txt:1.1.2.2 Message-ID: <200205211941.g4LJfWU32184@cvs.baymountain.com> Update of /cvs-repository/ZEO/doc In directory cvs.zope.org:/tmp/cvs-serv32177/doc Modified Files: Tag: ZEO2-branch ClientStorage.txt Log Message: Update description of keyword args to ClientStorage === ZEO/doc/ClientStorage.txt 1.1.2.1 => 1.1.2.2 === Creating a ClientStorage - At a minimum, a client storage requires an argument (named - connection) giving connection information. This argument should be - a string, specifying a unix-domain socket file name, or a tuple - consisting of a host and port. The host should be a string host - name or IP number. The port should be a numeric port number. + The ClientStorage requires at leats one argument, the address or + addresses of the server(s) to use. It accepts several other + optional keyword arguments. - The ClientStorage constructor provides a number of additional - options (arguments). The full list of arguments is: + The address argument can be one of: + + - a tuple containing hostname and port number - connection -- Connection information. + - a string specifying the path to a Unix domain socket - This argument is either a string containing a socket file name - or a tuple consisting of a string host name or ip number and an - integer port. + - a sequence of the previous two + + If a sequence of addresses is specified, the client will use the + first server from the list that it can connect to. + + The ClientStorage constructor provides a number of additional + options (arguments). The full list of arguments is: storage -- The name of the storage to connect to. @@ -33,7 +36,9 @@ default name for both the server and client is '1'. cache_size -- The number of bytes to allow for the client cache. - The default is 20,000,000. + The default is 20,000,000. A large cache can significantly + increase the performance of a ZEO system. For applications that + have a large database, the default size may be too small. For more information on client caches, see ClientCache.txt. @@ -54,10 +59,6 @@ For more information on client cache files, see ClientCache.txt. - debug -- If this is provided, it should be a non-empty string. It - indicates that client should log tracing and debugging - information, using zLOG. - var -- The directory in which persistent cache files should be written. If this option is provided, it is unnecessary to set INSTANCE_HOME in __builtins__. @@ -82,6 +83,13 @@ The default is 300 seconds. - wait_for_server_on_starup -- Indicate whether the ClientStorage - should block waiting for a storage server connection, or whether - it should proceed, satisfying reads from the client cache. + wait -- Indicate whether the ClientStorage should block waiting + for a storage server connection, or whether it should proceed, + satisfying reads from the client cache. + + read_only -- Open a read-only connection to the server. If the + client attempts to commit a transaction, it will get a + ReadOnlyError exception. + + Each storage served by a ZEO server can be configured as either + read-write or read-only. From jeremy at zope.com Tue May 21 15:48:53 2002 From: jeremy at zope.com (Jeremy Hylton) Date: Sun Aug 10 16:31:21 2008 Subject: [ZEO-Checkins] CVS: ZEO/ZEO/tests - forker.py:1.10.4.4.2.3 Message-ID: <200205211948.g4LJmr701200@cvs.baymountain.com> Update of /cvs-repository/ZEO/ZEO/tests In directory cvs.zope.org:/tmp/cvs-serv1189/tests Modified Files: Tag: ZEO2-branch forker.py Log Message: Track removal of debug kwarg === ZEO/ZEO/tests/forker.py 1.10.4.4.2.2 => 1.10.4.4.2.3 === pid, exit = start_zeo_server(storage_name, args, addr) s = ZEO.ClientStorage.ClientStorage(addr, storage_id, - debug=1, client=cache, + client=cache, cache_size=cache_size, min_disconnect_poll=0.5, wait=1) From jeremy at zope.com Tue May 21 16:00:00 2002 From: jeremy at zope.com (Jeremy Hylton) Date: Sun Aug 10 16:31:21 2008 Subject: [ZEO-Checkins] CVS: ZEO/doc - ZopeREADME.txt:1.1.2.4 Message-ID: <200205212000.g4LK00c04241@cvs.baymountain.com> Update of /cvs-repository/ZEO/doc In directory cvs.zope.org:/tmp/cvs-serv4234/doc Modified Files: Tag: ZEO2-branch ZopeREADME.txt Log Message: Remove XXX comment. It still applies. === ZEO/doc/ZopeREADME.txt 1.1.2.3 => 1.1.2.4 === Zope product installation - XXX Does this still apply? - Normally, Zope updates the Zope database during startup to reflect product changes or new products found. It makes no sense for multiple ZEO clients to do the same installation. Further, if From jeremy at zope.com Tue May 21 16:06:02 2002 From: jeremy at zope.com (Jeremy Hylton) Date: Sun Aug 10 16:31:21 2008 Subject: [ZEO-Checkins] CVS: ZEO/ZEO/tests - CommitLockTests.py:1.1.2.4 ThreadTests.py:1.1.2.3 Message-ID: <200205212006.g4LK62J06168@cvs.baymountain.com> Update of /cvs-repository/ZEO/ZEO/tests In directory cvs.zope.org:/tmp/cvs-serv6126/ZEO/tests Modified Files: Tag: ZEO2-branch CommitLockTests.py ThreadTests.py Log Message: Add licenses === ZEO/ZEO/tests/CommitLockTests.py 1.1.2.3 => 1.1.2.4 === +# +# Copyright (c) 2002 Zope Corporation and Contributors. +# All Rights Reserved. +# +# This software is subject to the provisions of the Zope Public License, +# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution. +# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED +# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS +# FOR A PARTICULAR PURPOSE +# +############################################################################## """Tests of the distributed commit lock.""" import threading === ZEO/ZEO/tests/ThreadTests.py 1.1.2.2 => 1.1.2.3 === +# +# Copyright (c) 2002 Zope Corporation and Contributors. +# All Rights Reserved. +# +# This software is subject to the provisions of the Zope Public License, +# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution. +# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED +# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS +# FOR A PARTICULAR PURPOSE +# +############################################################################## """Compromising positions involving threads.""" import threading From jeremy at zope.com Tue May 21 16:06:02 2002 From: jeremy at zope.com (Jeremy Hylton) Date: Sun Aug 10 16:31:21 2008 Subject: [ZEO-Checkins] CVS: ZEO - setup.py:1.1.2.2 test.py:1.1.2.3 Message-ID: <200205212006.g4LK62S06152@cvs.baymountain.com> Update of /cvs-repository/ZEO In directory cvs.zope.org:/tmp/cvs-serv6126 Modified Files: Tag: ZEO2-branch setup.py test.py Log Message: Add licenses === ZEO/setup.py 1.1.2.1 => 1.1.2.2 === +# +# Copyright (c) 2002 Zope Corporation and Contributors. +# All Rights Reserved. +# +# This software is subject to the provisions of the Zope Public License, +# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution. +# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED +# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS +# FOR A PARTICULAR PURPOSE +# +############################################################################## from distutils.core import setup packages = ['ZEO', 'ZEO.zrpc', 'ZEO.tests'] === ZEO/test.py 1.1.2.2 => 1.1.2.3 === +# +# Copyright (c) 2002 Zope Corporation and Contributors. +# All Rights Reserved. +# +# This software is subject to the provisions of the Zope Public License, +# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution. +# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED +# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS +# FOR A PARTICULAR PURPOSE +# +############################################################################## """Test harness for ZEO usage: python test.py [options] [modulepath] [testcase] From jeremy at zope.com Tue May 21 16:06:26 2002 From: jeremy at zope.com (Jeremy Hylton) Date: Sun Aug 10 16:31:21 2008 Subject: [ZEO-Checkins] CVS: ZEO - LICENSE.txt:1.1.4.1 Message-ID: <200205212006.g4LK6Qa06234@cvs.baymountain.com> Update of /cvs-repository/ZEO In directory cvs.zope.org:/tmp/cvs-serv6227 Added Files: Tag: ZEO2-branch LICENSE.txt Log Message: Add license === Added File ZEO/LICENSE.txt === Zope Public License (ZPL) Version 2.0 ----------------------------------------------- This software is Copyright (c) Zope Corporation (tm) and Contributors. All rights reserved. This license has been certified as open source. It has also been designated as GPL compatible by the Free Software Foundation (FSF). Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions in source code must retain the above copyright notice, this list of conditions, and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions, and the following disclaimer in the documentation and/or other materials provided with the distribution. 3. The name Zope Corporation (tm) must not be used to endorse or promote products derived from this software without prior written permission from Zope Corporation. 4. The right to distribute this software or to use it for any purpose does not give you the right to use Servicemarks (sm) or Trademarks (tm) of Zope Corporation. Use of them is covered in a separate agreement (see http://www.zope.com/Marks). 5. If any files are modified, you must cause the modified files to carry prominent notices stating that you changed the files and the date of any change. Disclaimer THIS SOFTWARE IS PROVIDED BY ZOPE CORPORATION ``AS IS'' AND ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL ZOPE CORPORATION OR ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. This software consists of contributions made by Zope Corporation and many individuals on behalf of Zope Corporation. Specific attributions are listed in the accompanying credits file. From jeremy at zope.com Tue May 21 16:27:06 2002 From: jeremy at zope.com (Jeremy Hylton) Date: Sun Aug 10 16:31:21 2008 Subject: [ZEO-Checkins] CVS: ZEO/ZEO/tests - testZEO.py:1.16.4.4.2.7 Message-ID: <200205212027.g4LKR6I11314@cvs.baymountain.com> Update of /cvs-repository/ZEO/ZEO/tests In directory cvs.zope.org:/tmp/cvs-serv11303 Modified Files: Tag: ZEO2-branch testZEO.py Log Message: Remove a debug kwarg that is only used on Windows. === ZEO/ZEO/tests/testZEO.py 1.16.4.4.2.6 => 1.16.4.4.2.7 === client=cache, cache_size=cache_size, - debug=1, wait=1) + wait=1) storage = PackWaitWrapper(base) storage.registerDB(DummyDB(), None) return storage From tim.one at comcast.net Tue May 21 18:28:57 2002 From: tim.one at comcast.net (Tim Peters) Date: Sun Aug 10 16:31:21 2008 Subject: [ZEO-Checkins] CVS: ZEO/ZEO - ClientCache.py:1.18.6.3.2.2 TransactionBuffer.py:1.3.2.2.2.2 Message-ID: <200205212228.g4LMSvE09119@cvs.baymountain.com> Update of /cvs-repository/ZEO/ZEO In directory cvs.zope.org:/tmp/cvs-serv9097/ZEO Modified Files: Tag: ZEO2-branch ClientCache.py TransactionBuffer.py Log Message: Some Windows-specific changes. === ZEO/ZEO/ClientCache.py 1.18.6.3.2.1 => 1.18.6.3.2.2 === for f in self._f: if f is not None: - f.close() + # In 2.1 on Windows, the TemporaryFileWrapper doesn't allow + # closing a file more than once. + try: + f.close() + except OSError: + pass def verify(self, verifyFunc): """Call the verifyFunc on every object in the cache. === ZEO/ZEO/TransactionBuffer.py 1.3.2.2.2.1 => 1.3.2.2.2.2 === self.pickler.fast = 1 - def close(self): - self.file.close() + def close(self): + try: + self.file.close() + except OSError: + pass + def store(self, oid, version, data): """Store oid, version, data for later retrieval""" From tim.one at comcast.net Tue May 21 18:28:57 2002 From: tim.one at comcast.net (Tim Peters) Date: Sun Aug 10 16:31:21 2008 Subject: [ZEO-Checkins] CVS: ZEO/ZEO/tests - CommitLockTests.py:1.1.2.5 forker.py:1.10.4.4.2.4 testZEO.py:1.16.4.4.2.8 Message-ID: <200205212228.g4LMSv909125@cvs.baymountain.com> Update of /cvs-repository/ZEO/ZEO/tests In directory cvs.zope.org:/tmp/cvs-serv9097/ZEO/tests Modified Files: Tag: ZEO2-branch CommitLockTests.py forker.py testZEO.py Log Message: Some Windows-specific changes. === ZEO/ZEO/tests/CommitLockTests.py 1.1.2.4 => 1.1.2.5 === # Copyright (c) 2002 Zope Corporation and Contributors. # All Rights Reserved. -# +# # This software is subject to the provisions of the Zope Public License, # Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution. # THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED # WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED # WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS # FOR A PARTICULAR PURPOSE -# +# ############################################################################## """Tests of the distributed commit lock.""" @@ -41,17 +41,17 @@ def run(self): try: self.storage.tpc_begin(self.trans) + oid = self.storage.new_oid() + self.storage.store(oid, ZERO, zodb_pickle(MinPO("c")), '', self.trans) + oid = self.storage.new_oid() + self.storage.store(oid, ZERO, zodb_pickle(MinPO("c")), '', self.trans) + self.storage.tpc_vote(self.trans) + if self.method == "tpc_finish": + self.storage.tpc_finish(self.trans) + else: + self.storage.tpc_abort(self.trans) except Disconnected: - return - oid = self.storage.new_oid() - self.storage.store(oid, ZERO, zodb_pickle(MinPO("c")), '', self.trans) - oid = self.storage.new_oid() - self.storage.store(oid, ZERO, zodb_pickle(MinPO("c")), '', self.trans) - self.storage.tpc_vote(self.trans) - if self.method == "tpc_finish": - self.storage.tpc_finish(self.trans) - else: - self.storage.tpc_abort(self.trans) + pass class CommitLockTests: @@ -63,7 +63,7 @@ # other transactions before they actually block. IOW, by the time # the other transactions get to the vote stage, the first # transaction has finished. - + def checkCommitLock1OnCommit(self): self._storages = [] try: === ZEO/ZEO/tests/forker.py 1.10.4.4.2.3 => 1.10.4.4.2.4 === if script.endswith('.pyc'): script = script[:-1] - args = (sys.executable, script, str(port), storage_name) + args + args = (sys.executable, script, str(port), storage_name) + args[0] d = os.environ.copy() d['PYTHONPATH'] = os.pathsep.join(sys.path) pid = os.spawnve(os.P_NOWAIT, sys.executable, args, d) === ZEO/ZEO/tests/testZEO.py 1.16.4.4.2.7 => 1.16.4.4.2.8 === os.waitpid(pid, 0) self.delStorage() + + def open(self, read_only=0): + # XXX Needed to support ReadOnlyStorage tests. Ought to be a + # cleaner way. + + # Is this the only way to get the address? + addr = self._storage._rpc_mgr.addr[0][1] + self._storage.close() + self._storage = ZEO.ClientStorage.ClientStorage(addr, read_only=1, + wait=1) def checkLargeUpdate(self): obj = MinPO("X" * (10 * 128 * 1024)) @@ -111,16 +121,6 @@ self.__fs_base = tempfile.mktemp() self.__super_setUp() - def open(self, read_only=0): - # XXX Needed to support ReadOnlyStorage tests. Ought to be a - # cleaner way. - - # Is this the only way to get the address? - addr = self._storage._rpc_mgr.addr[0][1] - self._storage.close() - self._storage = ZEO.ClientStorage.ClientStorage(addr, read_only=1, - wait=1) - def getStorage(self): self.__fs_base = tempfile.mktemp() return 'FileStorage', (self.__fs_base, '1') @@ -148,7 +148,7 @@ args = args[1:] zeo_addr, self.test_addr, self.test_pid = \ forker.start_zeo_server(name, args) - storage = ZEO.ClientStorage.ClientStorage(zeo_addr, debug=1, + storage = ZEO.ClientStorage.ClientStorage(zeo_addr, wait=1,#debug=1, min_disconnect_poll=0.1) self._storage = PackWaitWrapper(storage) storage.registerDB(DummyDB(), None) @@ -405,6 +405,7 @@ base = ZEO.ClientStorage.ClientStorage(self.addr, client=cache, cache_size=cache_size, + # debug=1, wait=1) storage = PackWaitWrapper(base) storage.registerDB(DummyDB(), None) From jeremy at zope.com Wed May 22 13:13:10 2002 From: jeremy at zope.com (Jeremy Hylton) Date: Sun Aug 10 16:31:21 2008 Subject: [ZEO-Checkins] CVS: ZEO/ZEO/tests - forker.py:1.10.4.4.2.5 testZEO.py:1.16.4.4.2.9 Message-ID: <200205221713.g4MHDAQ30248@cvs.baymountain.com> Update of /cvs-repository/ZEO/ZEO/tests In directory cvs.zope.org:/tmp/cvs-serv28899 Modified Files: Tag: ZEO2-branch forker.py testZEO.py Log Message: Try to make consistent use of the start_zeo_server() call on Windows. === ZEO/ZEO/tests/forker.py 1.10.4.4.2.4 => 1.10.4.4.2.5 === if script.endswith('.pyc'): script = script[:-1] - args = (sys.executable, script, str(port), storage_name) + args[0] + args = (sys.executable, script, str(port), storage_name) + args d = os.environ.copy() d['PYTHONPATH'] = os.pathsep.join(sys.path) pid = os.spawnve(os.P_NOWAIT, sys.executable, args, d) === ZEO/ZEO/tests/testZEO.py 1.16.4.4.2.8 => 1.16.4.4.2.9 === args = self.getStorageInfo() name = args[0] - args = args[1:] + args = args[1] zeo_addr, self.test_addr, self.test_pid = \ forker.start_zeo_server(name, args) - storage = ZEO.ClientStorage.ClientStorage(zeo_addr, wait=1,#debug=1, + storage = ZEO.ClientStorage.ClientStorage(zeo_addr, wait=1, min_disconnect_poll=0.1) self._storage = PackWaitWrapper(storage) storage.registerDB(DummyDB(), None) From matt at zope.com Thu May 23 09:26:50 2002 From: matt at zope.com (Matthew T. Kromer) Date: Sun Aug 10 16:31:21 2008 Subject: [ZEO-Checkins] CVS: ZEO/ZEO - StorageServer.py:1.32.6.3.2.6 Message-ID: <200205231326.g4NDQoN24016@cvs.baymountain.com> Update of /cvs-repository/ZEO/ZEO In directory cvs.zope.org:/tmp/cvs-serv23911 Modified Files: Tag: ZEO2-branch StorageServer.py Log Message: Add modifiedInVersion -- was missing -- causes a crash of ZEO2 when someone IMPORTs an object, since import calls modifiedInVersion. This just passes the call thru to the underlying storage. === ZEO/ZEO/StorageServer.py 1.32.6.3.2.5 => 1.32.6.3.2.6 === self.client.endVerify() + def modifiedInVersion(self, oid): + return self.__storage.modifiedInVersion(oid) + def pack(self, t, wait=0): t = threading.Thread(target=self._pack, args=(t, wait)) t.start() From jeremy at zope.com Thu May 30 17:46:15 2002 From: jeremy at zope.com (Jeremy Hylton) Date: Sun Aug 10 16:31:21 2008 Subject: [ZEO-Checkins] CVS: ZEO/ZEO/tests - testZEO.py:1.16.4.4.2.10 Message-ID: <200205302146.g4ULkFK07561@cvs.baymountain.com> Update of /cvs-repository/ZEO/ZEO/tests In directory cvs.zope.org:/tmp/cvs-serv7550 Modified Files: Tag: ZEO2-branch testZEO.py Log Message: Jim does it, too. === ZEO/ZEO/tests/testZEO.py 1.16.4.4.2.9 => 1.16.4.4.2.10 === from ZEO.smac import Disconnected -# Sorry Jim... from ZODB.tests import StorageTestBase, BasicStorage, VersionStorage, \ TransactionalUndoStorage, TransactionalUndoVersionStorage, \ PackableStorage, Synchronization, ConflictResolution, RevisionStorage, \ From jeremy at zope.com Thu May 30 17:52:55 2002 From: jeremy at zope.com (Jeremy Hylton) Date: Sun Aug 10 16:31:21 2008 Subject: [ZEO-Checkins] CVS: ZEO/ZEO/tests - testZEO.py:1.16.4.4.2.11 Message-ID: <200205302152.g4ULqtF08967@cvs.baymountain.com> Update of /cvs-repository/ZEO/ZEO/tests In directory cvs.zope.org:/tmp/cvs-serv8956/tests Modified Files: Tag: ZEO2-branch testZEO.py Log Message: Merge UnixConnectionTests back into ConnectionTests. There is no longer a difference between the Unix code and the Windows code. === ZEO/ZEO/tests/testZEO.py 1.16.4.4.2.10 => 1.16.4.4.2.11 === del i - def openClientStorage(self, cache='', cache_size=200000, wait=1): - # defined by subclasses - pass + def setUp(self): + """Start a ZEO server using a Unix domain socket + + The ZEO server uses the storage object returned by the + getStorage() method. + """ + self.running = 1 + self.file = tempfile.mktemp() + self.addr = [] + self._pids = [] + self._servers = [] + self._newAddr() + self._startServer() + + def _newAddr(self): + self.addr.append(self._getAddr()) - def shutdownServer(self): - # defined by subclasses - pass + def _getAddr(self): + return '', self.ports.pop() + + def _startServer(self, create=1, index=0): + path = "%s.%d" % (self.file, index) + addr = self.addr[index] + pid, server = forker.start_zeo_server('FileStorage', + (path, create), addr) + self._pids.append(pid) + self._servers.append(server) + + def shutdownServer(self, index=0): + if self.running: + self.running = 0 + self._servers[index].close() + try: + os.waitpid(self._pids[index], 0) + except os.error: + pass + + def openClientStorage(self, cache='', cache_size=200000, wait=1): + base = ZEO.ClientStorage.ClientStorage(self.addr, + client=cache, + cache_size=cache_size, + wait=wait, + min_disconnect_poll=0.1) + storage = PackWaitWrapper(base) + storage.registerDB(DummyDB(), None) + return storage def tearDown(self): """Try to cause the tests to halt""" @@ -332,96 +371,6 @@ # inherit from POSException. zLOG.LOG("checkReconnection", zLOG.INFO, "finished") -class UnixConnectionTests(ConnectionTests): - - def setUp(self): - """Start a ZEO server using a Unix domain socket - - The ZEO server uses the storage object returned by the - getStorage() method. - """ - self.running = 1 - self.file = tempfile.mktemp() - self.addr = [] - self._pids = [] - self._servers = [] - self._newAddr() - self._startServer() - - def _newAddr(self): - self.addr.append(self._getAddr()) - - def _getAddr(self): - return '', self.ports.pop() - - def _startServer(self, create=1, index=0): - path = "%s.%d" % (self.file, index) - addr = self.addr[index] - pid, server = forker.start_zeo_server('FileStorage', - (path, create), addr) - self._pids.append(pid) - self._servers.append(server) - - def openClientStorage(self, cache='', cache_size=200000, wait=1): - base = ZEO.ClientStorage.ClientStorage(self.addr, - client=cache, - cache_size=cache_size, - wait=wait, - min_disconnect_poll=0.1) - storage = PackWaitWrapper(base) - storage.registerDB(DummyDB(), None) - return storage - - def shutdownServer(self, index=0): - if self.running: - self.running = 0 - self._servers[index].close() - try: - os.waitpid(self._pids[index], 0) - except os.error: - pass - -class WindowsConnectionTests(ConnectionTests): - - # XXX these tests are now out-of-date - - def setUp(self): - self.file = tempfile.mktemp() - self._startServer() - - def _startServer(self, create=1): - if create == 0: - port = self.addr[1] - else: - port = None - self.addr, self.test_a, pid = forker.start_zeo_server('FileStorage', - (self.file, - str(create)), - port) - self.running = 1 - - def openClientStorage(self, cache='', cache_size=200000, wait=1): - base = ZEO.ClientStorage.ClientStorage(self.addr, - client=cache, - cache_size=cache_size, - # debug=1, - wait=1) - storage = PackWaitWrapper(base) - storage.registerDB(DummyDB(), None) - return storage - - def shutdownServer(self): - if self.running: - self.running = 0 - s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - s.connect(self.test_a) - s.close() - time.sleep(1.0) - - def tearDown(self): - self.shutdownServer() - self._storage.close() - def get_methods(klass): l = [klass] meth = {} @@ -435,10 +384,9 @@ return meth.keys() if os.name == "posix": - test_classes = ZEOFileStorageTests, UnixConnectionTests -## test_classes = UnixConnectionTests, + test_classes = ZEOFileStorageTests, ConnectionTests elif os.name == "nt": - test_classes = WindowsZEOFileStorageTests, WindowsConnectionTests + test_classes = WindowsZEOFileStorageTests, ConnectionTests else: raise RuntimeError, "unsupported os: %s" % os.name From jeremy at zope.com Thu May 30 18:02:22 2002 From: jeremy at zope.com (Jeremy Hylton) Date: Sun Aug 10 16:31:21 2008 Subject: [ZEO-Checkins] CVS: ZEO - test.py:1.1.2.4 Message-ID: <200205302202.g4UM2ML11523@cvs.baymountain.com> Update of /cvs-repository/ZEO In directory cvs.zope.org:/tmp/cvs-serv11516 Modified Files: Tag: ZEO2-branch test.py Log Message: Only catch an attribute error cause by a failure to find a test_suite() function. === ZEO/test.py 1.1.2.3 => 1.1.2.4 === mod = package_import(modname) try: - return mod.test_suite() - except AttributeError: + suite_factory = mod.test_suite + except AttributeError, err: return None + return suite_factory() def match(rx, s): if not rx: From jeremy at zope.com Thu May 30 18:07:51 2002 From: jeremy at zope.com (Jeremy Hylton) Date: Sun Aug 10 16:31:21 2008 Subject: [ZEO-Checkins] CVS: ZEO/ZEO/tests - testZEO.py:1.16.4.4.2.12 Message-ID: <200205302207.g4UM7pp12829@cvs.baymountain.com> Update of /cvs-repository/ZEO/ZEO/tests In directory cvs.zope.org:/tmp/cvs-serv12818 Modified Files: Tag: ZEO2-branch testZEO.py Log Message: get rid of some unit test mechanism kludges === ZEO/ZEO/tests/testZEO.py 1.16.4.4.2.11 => 1.16.4.4.2.12 === zLOG.LOG("checkReconnection", zLOG.INFO, "finished") -def get_methods(klass): - l = [klass] - meth = {} - while l: - klass = l.pop(0) - for base in klass.__bases__: - l.append(base) - for k, v in klass.__dict__.items(): - if callable(v): - meth[k] = 1 - return meth.keys() - if os.name == "posix": test_classes = ZEOFileStorageTests, ConnectionTests elif os.name == "nt": @@ -390,18 +378,12 @@ else: raise RuntimeError, "unsupported os: %s" % os.name -def makeTestSuite(testname=''): +def test_suite(): suite = unittest.TestSuite() - name = 'check' + testname - lname = len(name) for klass in test_classes: - for meth in get_methods(klass): - if meth[:lname] == name: - suite.addTest(klass(meth)) + sub = unittest.makeSuite(klass, 'check') + suite.addTest(sub) return suite - -def test_suite(): - return makeTestSuite() def main(): import sys, getopt From jeremy at zope.com Thu May 30 23:14:58 2002 From: jeremy at zope.com (Jeremy Hylton) Date: Sun Aug 10 16:31:21 2008 Subject: [ZEO-Checkins] CVS: StandaloneZODB/ZEO/tests - forker.py:1.10.4.4.2.6 Message-ID: <200205310314.g4V3Ew728199@cvs.baymountain.com> Update of /cvs-repository/StandaloneZODB/ZEO/tests In directory cvs.zope.org:/tmp/cvs-serv27497 Modified Files: Tag: ZEO2-branch forker.py Log Message: Make args to start_zeo_server() consistent for Windows and Unix. === StandaloneZODB/ZEO/tests/forker.py 1.10.4.4.2.5 => 1.10.4.4.2.6 === if os.name == "nt": - def start_zeo_server(storage_name, args, port=None): + def start_zeo_server(storage_name, args, addr=None): """Start a ZEO server in a separate process. Returns the ZEO port, the test server port, and the pid. """ import ZEO.tests.winserver - if port is None: + if addr is None: port = get_port() + else: + port = addr[1] script = ZEO.tests.winserver.__file__ if script.endswith('.pyc'): script = script[:-1] From jeremy at zope.com Thu May 30 23:17:55 2002 From: jeremy at zope.com (Jeremy Hylton) Date: Sun Aug 10 16:31:21 2008 Subject: [ZEO-Checkins] CVS: StandaloneZODB/ZEO/tests - testZEO.py:1.16.4.4.2.13 Message-ID: <200205310317.g4V3Htm01550@cvs.baymountain.com> Update of /cvs-repository/StandaloneZODB/ZEO/tests In directory cvs.zope.org:/tmp/cvs-serv32013 Modified Files: Tag: ZEO2-branch testZEO.py Log Message: Several small fixes to Windows test code. Define WindowsConnectionTests subclass to handle different mechanism for shutting down servers on Windows. Make sure socket addresses say 'localhost' because '' won't work on Windows. === StandaloneZODB/ZEO/tests/testZEO.py 1.16.4.4.2.12 => 1.16.4.4.2.13 === def _getAddr(self): - return '', self.ports.pop() - - def _startServer(self, create=1, index=0): - path = "%s.%d" % (self.file, index) - addr = self.addr[index] - pid, server = forker.start_zeo_server('FileStorage', - (path, create), addr) - self._pids.append(pid) - self._servers.append(server) - - def shutdownServer(self, index=0): - if self.running: - self.running = 0 - self._servers[index].close() - try: - os.waitpid(self._pids[index], 0) - except os.error: - pass + return 'localhost', self.ports.pop() def openClientStorage(self, cache='', cache_size=200000, wait=1): base = ZEO.ClientStorage.ClientStorage(self.addr, @@ -247,11 +230,17 @@ for ext in '', '.index', '.lock', '.tmp': path = "%s.%s%s" % (self.file, i, ext) if os.path.exists(path): - os.unlink(path) + try: + os.unlink(path) + except os.error: + pass for i in 0, 1: path = "c1-test-%d.zec" % i if os.path.exists(path): - os.unlink(path) + try: + os.unlink(path) + except os.error: + pass self.__super_tearDown() def checkMultipleAddresses(self): @@ -298,6 +287,7 @@ # To verify that the cache is being used, the test closes the # server and then starts a new client with the server down. + # When the server is down, a load() gets the data from its cache. self._storage = self.openClientStorage('test', 100000, wait=1) oid = self._storage.new_oid() @@ -371,10 +361,29 @@ # inherit from POSException. zLOG.LOG("checkReconnection", zLOG.INFO, "finished") +class WindowsConnectionTests(ConnectionTests): + + def _startServer(self, create=1, index=0): + path = "%s.%d" % (self.file, index) + addr = self.addr[index] + _addr, test_addr, test_pid = forker.start_zeo_server('FileStorage', + (path, str(create)), addr) + self._pids.append(test_pid) + self._servers.append(test_addr) + + def shutdownServer(self, index=0): + if self.running: + self.running = 0 + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + s.connect(self._servers[index]) + s.close() + # XXX waitpid() isn't available until Python 2.3 + time.sleep(0.5) + if os.name == "posix": test_classes = ZEOFileStorageTests, ConnectionTests elif os.name == "nt": - test_classes = WindowsZEOFileStorageTests, ConnectionTests + test_classes = WindowsZEOFileStorageTests, WindowsConnectionTests else: raise RuntimeError, "unsupported os: %s" % os.name From jeremy at zope.com Thu May 30 23:19:17 2002 From: jeremy at zope.com (Jeremy Hylton) Date: Sun Aug 10 16:31:21 2008 Subject: [ZEO-Checkins] CVS: StandaloneZODB/ZEO/zrpc - client.py:1.1.2.2.2.2 Message-ID: <200205310319.g4V3JHk03999@cvs.baymountain.com> Update of /cvs-repository/StandaloneZODB/ZEO/zrpc In directory cvs.zope.org:/tmp/cvs-serv1950 Modified Files: Tag: ZEO2-branch client.py Log Message: Fix connection logic to work better (correctly?) on Win2k. A comment in the code explains: It appears that winsock isn't behaving as expected on Win2k. It's possible for connect() to return 0, but the connection to have failed. In particular, in situations where I expect to get a Connection refused (10061), I'm seeing connect_ex() return 0. OTOH, it looks like select() is a more reliable indicator on Windows. === StandaloneZODB/ZEO/zrpc/client.py 1.1.2.2.2.1 => 1.1.2.2.2.2 === return 0 try: - r, w, x = select.select([], self.sockets.keys(), [], 1.0) + sockets = self.sockets.keys() + r, w, x = select.select([], sockets, sockets, 1.0) except select.error: continue + for s in x: + del self.sockets[s] + s.close() for s in w: # connect() raises Connected iff it succeeds self.connect(s) @@ -310,9 +314,29 @@ log("failed to connect to %s: %s" % (addr, msg), level=zLOG.ERROR) else: + log("connect_ex(%s) == %s" % (addr, e)) if e in _CONNECT_IN_PROGRESS: return elif e in _CONNECT_OK: + # special cases to deal with winsock oddities + if sys.platform.startswith("win") and e == 0: + + # It appears that winsock isn't behaving as + # expected on Win2k. It's possible for connect() + # to return 0, but the connection to have failed. + # In particular, in situations where I expect to + # get a Connection refused (10061), I'm seeing + # connect_ex() return 0. OTOH, it looks like + # select() is a more reliable indicator on + # Windows. + + r, w, x = select.select([s], [s], [s], 0.1) + if not (r or w or x): + return + if x: + # see comment at the end of the function + s.close() + del self.socket[s] c = self.test_connection(s, addr) if c: log("connected to %s" % repr(addr), level=zLOG.DEBUG) From jeremy at zope.com Thu May 30 23:55:48 2002 From: jeremy at zope.com (Jeremy Hylton) Date: Sun Aug 10 16:31:21 2008 Subject: [ZEO-Checkins] CVS: ZEO/ZEO/tests - testZEO.py:1.16.4.4.2.14 Message-ID: <200205310355.g4V3tmv16858@cvs.baymountain.com> Update of /cvs-repository/ZEO/ZEO/tests In directory cvs.zope.org:/tmp/cvs-serv15944 Modified Files: Tag: ZEO2-branch testZEO.py Log Message: One last fix. Looks like tests work on Unix and Windows now. Restore UnixConnectionTests to cover the differences in starting and stopping servers on Unix and Windows. The previous checkin inadvertantly deleted all the Unix-specific code. === ZEO/ZEO/tests/testZEO.py 1.16.4.4.2.13 => 1.16.4.4.2.14 === zLOG.LOG("checkReconnection", zLOG.INFO, "finished") +class UnixConnectionTests(ConnectionTests): + + def _startServer(self, create=1, index=0): + path = "%s.%d" % (self.file, index) + addr = self.addr[index] + pid, server = forker.start_zeo_server('FileStorage', + (path, create), addr) + self._pids.append(pid) + self._servers.append(server) + + def shutdownServer(self, index=0): + if self.running: + self.running = 0 + self._servers[index].close() + try: + os.waitpid(self._pids[index], 0) + except os.error: + pass + class WindowsConnectionTests(ConnectionTests): def _startServer(self, create=1, index=0): @@ -381,7 +400,7 @@ time.sleep(0.5) if os.name == "posix": - test_classes = ZEOFileStorageTests, ConnectionTests + test_classes = ZEOFileStorageTests, UnixConnectionTests elif os.name == "nt": test_classes = WindowsZEOFileStorageTests, WindowsConnectionTests else: From jeremy at zope.com Fri May 31 15:34:08 2002 From: jeremy at zope.com (Jeremy Hylton) Date: Sun Aug 10 16:31:21 2008 Subject: [ZEO-Checkins] CVS: StandaloneZODB/ZEO - ICache.py:1.1.2.1 ClientCache.py:1.18.6.3.2.3 Message-ID: <200205311934.g4VJY8c19483@cvs.baymountain.com> Update of /cvs-repository/StandaloneZODB/ZEO In directory cvs.zope.org:/tmp/cvs-serv19470 Modified Files: Tag: ZEO2-branch ClientCache.py Added Files: Tag: ZEO2-branch ICache.py Log Message: Add a provisional ICache that describes the ZEO client cache API. === Added File StandaloneZODB/ZEO/ICache.py === try: from Interface import Base except ImportError: class Base: # a dummy interface for use when Zope's is unavailable pass class ICache(Base): """ZEO client cache. __init__(storage, size, client, var) All arguments optional. storage -- name of storage size -- max size of cache in bytes client -- a string; if specified, cache is persistent. var -- var directory to store cache files in """ def open(): """Returns a sequence of object info tuples. An object info tuple is a pair containing an object id and a pair of serialnos, a non-version serialno and a version serialno: oid, (serial, ver_serial) This method builds an index of the cache and returns a sequence used for cache validation. """ def close(): """Closes the cache.""" def verify(func): """Call func on every object in cache. func is called with three arguments func(oid, serial, ver_serial) """ def invalidate(oid, version): """Remove object from cache.""" def load(oid, version): """Load object from cache. Return None if object not in cache. Return data, serialno if object is in cache. """ def store(oid, p, s, version, pv, sv): """Store a new object in the cache.""" def update(oid, serial, version, data): """Update an object already in the cache. XXX This method is called to update objects that were modified by a transaction. It's likely that it is already in the cache, and it may be possible for the implementation to operate more efficiently. """ def modifiedInVersion(oid): """Return the version an object is modified in. '' signifies the trunk. Returns None if the object is not in the cache. """ def checkSize(size): """Check if adding size bytes would exceed cache limit. This method is often called just before store or update. The size is a hint about the amount of data that is about to be stored. The cache may want to evict some data to make space. """ === StandaloneZODB/ZEO/ClientCache.py 1.18.6.3.2.2 => 1.18.6.3.2.3 === __version__ = "$Revision$"[11:-2] -import os, tempfile +import os +import sys +import tempfile from struct import pack, unpack from thread import allocate_lock -import sys import zLOG +from ZEO.ICache import ICache def log(msg, level=zLOG.INFO): zLOG.LOG("ZEC", level, msg) @@ -88,6 +90,8 @@ magic='ZEC0' class ClientCache: + + __implements__ = ICache def __init__(self, storage='', size=20000000, client=None, var=None):