From jeremy at digicool.com Tue May 1 14:43:54 2001 From: jeremy at digicool.com (jeremy@digicool.com) Date: Sun Aug 10 16:31:15 2008 Subject: [ZEO-Checkins] CVS: Packages/ZEO - forker.py:1.1.2.2 multi.py:1.1.2.3 Message-ID: <20010501184354.AE6AE510E3@korak.digicool.com> Update of /cvs-repository/Packages/ZEO/tests In directory korak:/tmp/cvs-serv2759/tests Modified Files: Tag: ZEO-ZRPC-Dev forker.py multi.py Log Message: Refactor forker so that it can be used to start StorageServer without starting ClientStorage. forker.py: Add start_zeo_server() function called by start_zeo() multi.py: Use start_zeo_server() and the ZEOClientExit object --- Updated File forker.py in package Packages/ZEO -- --- forker.py 2001/04/27 21:03:36 1.1.2.1 +++ forker.py 2001/05/01 18:43:53 1.1.2.2 @@ -32,23 +32,7 @@ def close(self): os.write(self.pipe, "done") -def start_zeo(storage, cache=None, cleanup=None, domain="AF_INET"): - """Setup ZEO client-server for storage. - - Returns a ClientStorage instance and a ZEOClientExit instance. - - XXX Don't know if os.pipe() will work on Windows. - """ - - if domain == "AF_INET": - import random - addr = '', random.randrange(2000, 3000) - elif domain == "AF_UNIX": - import tempfile - addr = tempfile.mktemp() - else: - raise ValueError, "bad domain: %s" % domain - +def start_zeo_server(storage, addr): rd, wr = os.pipe() pid = os.fork() if pid == 0: @@ -67,6 +51,26 @@ os._exit(0) else: os.close(rd) - s = ZEO.ClientStorage.ClientStorage(addr, debug=1, client=cache) - return s, ZEOClientExit(wr), pid + return pid, ZEOClientExit(wr) + +def start_zeo(storage, cache=None, cleanup=None, domain="AF_INET"): + """Setup ZEO client-server for storage. + + Returns a ClientStorage instance and a ZEOClientExit instance. + + XXX Don't know if os.pipe() will work on Windows. + """ + + if domain == "AF_INET": + import random + addr = '', random.randrange(2000, 3000) + elif domain == "AF_UNIX": + import tempfile + addr = tempfile.mktemp() + else: + raise ValueError, "bad domain: %s" % domain + + pid, exit = start_zeo_server(storage, addr) + s = ZEO.ClientStorage.ClientStorage(addr, debug=1, client=cache) + return s, exit, pid --- Updated File multi.py in package Packages/ZEO -- --- multi.py 2001/04/25 22:50:38 1.1.2.2 +++ multi.py 2001/05/01 18:43:53 1.1.2.3 @@ -3,6 +3,7 @@ import ZODB, ZODB.DB, ZODB.FileStorage, ZODB.POSException import Persistence import PersistentMapping +from ZEO.tests import forker import asyncore import os @@ -48,16 +49,9 @@ return fs def start_server(addr): - pid = os.fork() - if pid == 0: - import ZEO.StorageServer - storage = init_storage() - server = ZEO.StorageServer.StorageServer(addr, {'1': storage}) - print "Server ready:", os.getpid() - asyncore.loop() - os._exit(0) - else: - return pid + storage = init_storage() + pid, exit = forker.start_zeo_server(storage, addr) + return pid, exit def start_client(addr): pid = os.fork() @@ -123,7 +117,7 @@ print "Main process:", os.getpid() addr = tempfile.mktemp() t0 = time.time() - server_pid = start_server(addr) + server_pid, server = start_server(addr) t1 = time.time() pids = [start_client(addr) for i in range(CLIENTS)] for pid in pids: @@ -136,7 +130,7 @@ except os.error, err: print "waitpid(%s) failed: %s" % (repr(pid), err) t2 = time.time() - shutdown_server(addr) + server.close() os.waitpid(server_pid, 0) # XXX Should check that the results are consistent! --- Updated File forker.py in package Packages/ZEO -- --- forker.py 2001/04/27 21:03:36 1.1.2.1 +++ forker.py 2001/05/01 18:43:53 1.1.2.2 @@ -32,23 +32,7 @@ def close(self): os.write(self.pipe, "done") -def start_zeo(storage, cache=None, cleanup=None, domain="AF_INET"): - """Setup ZEO client-server for storage. - - Returns a ClientStorage instance and a ZEOClientExit instance. - - XXX Don't know if os.pipe() will work on Windows. - """ - - if domain == "AF_INET": - import random - addr = '', random.randrange(2000, 3000) - elif domain == "AF_UNIX": - import tempfile - addr = tempfile.mktemp() - else: - raise ValueError, "bad domain: %s" % domain - +def start_zeo_server(storage, addr): rd, wr = os.pipe() pid = os.fork() if pid == 0: @@ -67,6 +51,26 @@ os._exit(0) else: os.close(rd) - s = ZEO.ClientStorage.ClientStorage(addr, debug=1, client=cache) - return s, ZEOClientExit(wr), pid + return pid, ZEOClientExit(wr) + +def start_zeo(storage, cache=None, cleanup=None, domain="AF_INET"): + """Setup ZEO client-server for storage. + + Returns a ClientStorage instance and a ZEOClientExit instance. + + XXX Don't know if os.pipe() will work on Windows. + """ + + if domain == "AF_INET": + import random + addr = '', random.randrange(2000, 3000) + elif domain == "AF_UNIX": + import tempfile + addr = tempfile.mktemp() + else: + raise ValueError, "bad domain: %s" % domain + + pid, exit = start_zeo_server(storage, addr) + s = ZEO.ClientStorage.ClientStorage(addr, debug=1, client=cache) + return s, exit, pid --- Updated File multi.py in package Packages/ZEO -- --- multi.py 2001/04/25 22:50:38 1.1.2.2 +++ multi.py 2001/05/01 18:43:53 1.1.2.3 @@ -3,6 +3,7 @@ import ZODB, ZODB.DB, ZODB.FileStorage, ZODB.POSException import Persistence import PersistentMapping +from ZEO.tests import forker import asyncore import os @@ -48,16 +49,9 @@ return fs def start_server(addr): - pid = os.fork() - if pid == 0: - import ZEO.StorageServer - storage = init_storage() - server = ZEO.StorageServer.StorageServer(addr, {'1': storage}) - print "Server ready:", os.getpid() - asyncore.loop() - os._exit(0) - else: - return pid + storage = init_storage() + pid, exit = forker.start_zeo_server(storage, addr) + return pid, exit def start_client(addr): pid = os.fork() @@ -123,7 +117,7 @@ print "Main process:", os.getpid() addr = tempfile.mktemp() t0 = time.time() - server_pid = start_server(addr) + server_pid, server = start_server(addr) t1 = time.time() pids = [start_client(addr) for i in range(CLIENTS)] for pid in pids: @@ -136,7 +130,7 @@ except os.error, err: print "waitpid(%s) failed: %s" % (repr(pid), err) t2 = time.time() - shutdown_server(addr) + server.close() os.waitpid(server_pid, 0) # XXX Should check that the results are consistent! From jeremy at digicool.com Tue May 1 14:58:22 2001 From: jeremy at digicool.com (jeremy@digicool.com) Date: Sun Aug 10 16:31:15 2008 Subject: [ZEO-Checkins] CVS: Packages/ZEO - testZEO.py:1.1.2.8 Message-ID: <20010501185822.537A3510E3@korak.digicool.com> Update of /cvs-repository/Packages/ZEO/tests In directory korak:/tmp/cvs-serv3945/tests Modified Files: Tag: ZEO-ZRPC-Dev testZEO.py Log Message: add Synchronization tests --- Updated File testZEO.py in package Packages/ZEO -- --- testZEO.py 2001/04/27 21:03:36 1.1.2.7 +++ testZEO.py 2001/05/01 18:58:20 1.1.2.8 @@ -16,7 +16,7 @@ # Sorry Jim... from ZODB.tests import StorageTestBase, BasicStorage, VersionStorage, \ TransactionalUndoStorage, TransactionalUndoVersionStorage, \ - PackableStorage + PackableStorage, Synchronization ZERO = '\0'*8 import pickle @@ -62,6 +62,7 @@ s2 = self._get_serial(r2) self._storage.tpc_finish(self._transaction) # s1, s2 can be None or dict + assert not (s1 and s2) return s1 and s1[oid] or s2 and s2[oid] def _get_serial(self, r): @@ -79,8 +80,9 @@ class GenericTests(ZEOTestBase, BasicStorage.BasicStorage, VersionStorage.VersionStorage, - TransactionalUndoStorage.TransactionalUndoStorage, PackableStorage.PackableStorage, + Synchronization.SynchronizedStorage, + TransactionalUndoStorage.TransactionalUndoStorage, TransactionalUndoVersionStorage.TransactionalUndoVersionStorage, ): """An abstract base class for ZEO tests --- Updated File testZEO.py in package Packages/ZEO -- --- testZEO.py 2001/04/27 21:03:36 1.1.2.7 +++ testZEO.py 2001/05/01 18:58:20 1.1.2.8 @@ -16,7 +16,7 @@ # Sorry Jim... from ZODB.tests import StorageTestBase, BasicStorage, VersionStorage, \ TransactionalUndoStorage, TransactionalUndoVersionStorage, \ - PackableStorage + PackableStorage, Synchronization ZERO = '\0'*8 import pickle @@ -62,6 +62,7 @@ s2 = self._get_serial(r2) self._storage.tpc_finish(self._transaction) # s1, s2 can be None or dict + assert not (s1 and s2) return s1 and s1[oid] or s2 and s2[oid] def _get_serial(self, r): @@ -79,8 +80,9 @@ class GenericTests(ZEOTestBase, BasicStorage.BasicStorage, VersionStorage.VersionStorage, - TransactionalUndoStorage.TransactionalUndoStorage, PackableStorage.PackableStorage, + Synchronization.SynchronizedStorage, + TransactionalUndoStorage.TransactionalUndoStorage, TransactionalUndoVersionStorage.TransactionalUndoVersionStorage, ): """An abstract base class for ZEO tests From jeremy at digicool.com Tue May 1 15:50:44 2001 From: jeremy at digicool.com (jeremy@digicool.com) Date: Sun Aug 10 16:31:15 2008 Subject: [ZEO-Checkins] CVS: Packages/ZEO - speed.py:1.2 multi.py:1.2 Message-ID: <20010501195044.AF017510E3@korak.digicool.com> Update of /cvs-repository/Packages/ZEO/tests In directory korak:/tmp/cvs-serv7998 Added Files: speed.py multi.py Log Message: Some tests from ZEO2 --- Updated File speed.py in package Packages/ZEO -- --- Updated File multi.py in package Packages/ZEO -- --- Updated File speed.py in package Packages/ZEO -- --- Updated File multi.py in package Packages/ZEO -- From jeremy at digicool.com Tue May 1 15:54:34 2001 From: jeremy at digicool.com (jeremy@digicool.com) Date: Sun Aug 10 16:31:15 2008 Subject: [ZEO-Checkins] CVS: Packages/ZEO - forker.py:1.2 testZEO.py:1.3 Message-ID: <20010501195434.E8CAC510E3@korak.digicool.com> Update of /cvs-repository/Packages/ZEO/tests In directory korak:/tmp/cvs-serv8380/tests Modified Files: forker.py testZEO.py Log Message: Updates so that it passes all tests --- Updated File forker.py in package Packages/ZEO -- --- forker.py 2001/04/27 17:32:19 1.1 +++ forker.py 2001/05/01 19:54:33 1.2 @@ -32,23 +32,7 @@ def close(self): os.write(self.pipe, "done") -def start_zeo(storage, cache=None, cleanup=None, domain="AF_INET"): - """Setup ZEO client-server for storage. - - Returns a ClientStorage instance and a ZEOClientExit instance. - - XXX Don't know if os.pipe() will work on Windows. - """ - - if domain == "AF_INET": - import random - addr = '', random.randrange(2000, 3000) - elif domain == "AF_UNIX": - import tempfile - addr = tempfile.mktemp() - else: - raise ValueError, "bad domain: %s" % domain - +def start_zeo_server(storage, addr): rd, wr = os.pipe() pid = os.fork() if pid == 0: @@ -67,6 +51,26 @@ os._exit(0) else: os.close(rd) - s = ZEO.ClientStorage.ClientStorage(addr, debug=1, client=cache) - return s, ZEOClientExit(wr), pid + return pid, ZEOClientExit(wr) + +def start_zeo(storage, cache=None, cleanup=None, domain="AF_INET"): + """Setup ZEO client-server for storage. + + Returns a ClientStorage instance and a ZEOClientExit instance. + + XXX Don't know if os.pipe() will work on Windows. + """ + + if domain == "AF_INET": + import random + addr = '', random.randrange(2000, 3000) + elif domain == "AF_UNIX": + import tempfile + addr = tempfile.mktemp() + else: + raise ValueError, "bad domain: %s" % domain + + pid, exit = start_zeo_server(storage, addr) + s = ZEO.ClientStorage.ClientStorage(addr, debug=1, client=cache) + return s, exit, pid --- Updated File testZEO.py in package Packages/ZEO -- --- testZEO.py 2001/04/27 17:32:19 1.2 +++ testZEO.py 2001/05/01 19:54:33 1.3 @@ -1,8 +1,7 @@ """Test suite for ZEO based on ZODB.tests""" +import asyncore import os -import random -import signal import tempfile import time import types @@ -10,23 +9,31 @@ import ZEO.ClientStorage, ZEO.StorageServer import ThreadedAsync, ZEO.trigger -from ZEO.tests import forker -# XXX The ZODB.tests package contains a grab bad things, including, -# apparently, a collection of modules that define mixin classes -# containing tests cases. +from ZEO.tests import forker -from ZODB.tests import StorageTestBase, BasicStorage, VersionStorage +# Sorry Jim... +from ZODB.tests import StorageTestBase, BasicStorage, VersionStorage, \ + TransactionalUndoStorage, TransactionalUndoVersionStorage, \ + PackableStorage, Synchronization ZERO = '\0'*8 import pickle - -class FakeDB: - """A ClientStorage must be registered with a DB to function""" +class DummyDB: def invalidate(self, *args): pass +class PackWaitWrapper: + def __init__(self, storage): + self.storage = storage + + def __getattr__(self, attr): + return getattr(self.storage, attr) + + def pack(self, t, f): + self.storage.pack(t, f, wait=1) + class ZEOTestBase(StorageTestBase.StorageTestBase): """Version of the storage test class that supports ZEO. @@ -35,7 +42,8 @@ will get no later than the return value from vote. """ - def _dostore(self, oid=None, revid=None, data=None, version=None): + def _dostore(self, oid=None, revid=None, data=None, version=None, + already_pickled=0): """Do a complete storage transaction. The defaults are: @@ -51,8 +59,8 @@ if revid is None: revid = ZERO if data is None: - data = pickle.dumps(7) - else: + data = 7 + if not already_pickled: data = pickle.dumps(data) if version is None: version = '' @@ -67,6 +75,7 @@ s2 = self._get_serial(r2) self._storage.tpc_finish(self._transaction) # s1, s2 can be None or dict + assert not (s1 and s2) return s1 and s1[oid] or s2 and s2[oid] def _get_serial(self, r): @@ -78,15 +87,16 @@ raise RuntimeError, "unexpected ZEO response: no oid" else: for oid, serial in r: - if type(serial) != types.StringType: + if isinstance(serial, Exception): raise serial - else: - d[oid] = serial + d[oid] = serial return d class GenericTests(ZEOTestBase, BasicStorage.BasicStorage, VersionStorage.VersionStorage, + PackableStorage.PackableStorage, + Synchronization.SynchronizedStorage, ): """An abstract base class for ZEO tests @@ -106,27 +116,20 @@ getStorage() method. """ self.running = 1 - s = self.__storage = self.getStorage() - storage, exit, pid = forker.start_zeo(s) + client, exit, pid = forker.start_zeo(self.getStorage()) self._pid = pid - self._server_exit = exit - self._storage = storage - self._storage.registerDB(FakeDB(), None) + self._server = exit + self._storage = PackWaitWrapper(client) + client.registerDB(DummyDB(), None) self.__super_setUp() def tearDown(self): """Try to cause the tests to halt""" self.running = 0 - # XXX This only works on Unix - self._server_exit.close() + self._server.close() os.waitpid(self._pid, 0) - self.delStorage() self.__super_tearDown() - def checkFirst(self): - self._storage.tpc_begin(self._transaction) - self._storage.tpc_abort(self._transaction) - class ZEOFileStorageTests(GenericTests): __super_setUp = GenericTests.setUp @@ -143,8 +146,7 @@ # file storage appears to create three files for ext in '', '.index', '.lock', '.tmp': path = self.__fs_base + ext - if os.path.exists(path): - os.unlink(path) + os.unlink(path) def main(): import sys, getopt --- Updated File forker.py in package Packages/ZEO -- --- forker.py 2001/04/27 17:32:19 1.1 +++ forker.py 2001/05/01 19:54:33 1.2 @@ -32,23 +32,7 @@ def close(self): os.write(self.pipe, "done") -def start_zeo(storage, cache=None, cleanup=None, domain="AF_INET"): - """Setup ZEO client-server for storage. - - Returns a ClientStorage instance and a ZEOClientExit instance. - - XXX Don't know if os.pipe() will work on Windows. - """ - - if domain == "AF_INET": - import random - addr = '', random.randrange(2000, 3000) - elif domain == "AF_UNIX": - import tempfile - addr = tempfile.mktemp() - else: - raise ValueError, "bad domain: %s" % domain - +def start_zeo_server(storage, addr): rd, wr = os.pipe() pid = os.fork() if pid == 0: @@ -67,6 +51,26 @@ os._exit(0) else: os.close(rd) - s = ZEO.ClientStorage.ClientStorage(addr, debug=1, client=cache) - return s, ZEOClientExit(wr), pid + return pid, ZEOClientExit(wr) + +def start_zeo(storage, cache=None, cleanup=None, domain="AF_INET"): + """Setup ZEO client-server for storage. + + Returns a ClientStorage instance and a ZEOClientExit instance. + + XXX Don't know if os.pipe() will work on Windows. + """ + + if domain == "AF_INET": + import random + addr = '', random.randrange(2000, 3000) + elif domain == "AF_UNIX": + import tempfile + addr = tempfile.mktemp() + else: + raise ValueError, "bad domain: %s" % domain + + pid, exit = start_zeo_server(storage, addr) + s = ZEO.ClientStorage.ClientStorage(addr, debug=1, client=cache) + return s, exit, pid --- Updated File testZEO.py in package Packages/ZEO -- --- testZEO.py 2001/04/27 17:32:19 1.2 +++ testZEO.py 2001/05/01 19:54:33 1.3 @@ -1,8 +1,7 @@ """Test suite for ZEO based on ZODB.tests""" +import asyncore import os -import random -import signal import tempfile import time import types @@ -10,23 +9,31 @@ import ZEO.ClientStorage, ZEO.StorageServer import ThreadedAsync, ZEO.trigger -from ZEO.tests import forker -# XXX The ZODB.tests package contains a grab bad things, including, -# apparently, a collection of modules that define mixin classes -# containing tests cases. +from ZEO.tests import forker -from ZODB.tests import StorageTestBase, BasicStorage, VersionStorage +# Sorry Jim... +from ZODB.tests import StorageTestBase, BasicStorage, VersionStorage, \ + TransactionalUndoStorage, TransactionalUndoVersionStorage, \ + PackableStorage, Synchronization ZERO = '\0'*8 import pickle - -class FakeDB: - """A ClientStorage must be registered with a DB to function""" +class DummyDB: def invalidate(self, *args): pass +class PackWaitWrapper: + def __init__(self, storage): + self.storage = storage + + def __getattr__(self, attr): + return getattr(self.storage, attr) + + def pack(self, t, f): + self.storage.pack(t, f, wait=1) + class ZEOTestBase(StorageTestBase.StorageTestBase): """Version of the storage test class that supports ZEO. @@ -35,7 +42,8 @@ will get no later than the return value from vote. """ - def _dostore(self, oid=None, revid=None, data=None, version=None): + def _dostore(self, oid=None, revid=None, data=None, version=None, + already_pickled=0): """Do a complete storage transaction. The defaults are: @@ -51,8 +59,8 @@ if revid is None: revid = ZERO if data is None: - data = pickle.dumps(7) - else: + data = 7 + if not already_pickled: data = pickle.dumps(data) if version is None: version = '' @@ -67,6 +75,7 @@ s2 = self._get_serial(r2) self._storage.tpc_finish(self._transaction) # s1, s2 can be None or dict + assert not (s1 and s2) return s1 and s1[oid] or s2 and s2[oid] def _get_serial(self, r): @@ -78,15 +87,16 @@ raise RuntimeError, "unexpected ZEO response: no oid" else: for oid, serial in r: - if type(serial) != types.StringType: + if isinstance(serial, Exception): raise serial - else: - d[oid] = serial + d[oid] = serial return d class GenericTests(ZEOTestBase, BasicStorage.BasicStorage, VersionStorage.VersionStorage, + PackableStorage.PackableStorage, + Synchronization.SynchronizedStorage, ): """An abstract base class for ZEO tests @@ -106,27 +116,20 @@ getStorage() method. """ self.running = 1 - s = self.__storage = self.getStorage() - storage, exit, pid = forker.start_zeo(s) + client, exit, pid = forker.start_zeo(self.getStorage()) self._pid = pid - self._server_exit = exit - self._storage = storage - self._storage.registerDB(FakeDB(), None) + self._server = exit + self._storage = PackWaitWrapper(client) + client.registerDB(DummyDB(), None) self.__super_setUp() def tearDown(self): """Try to cause the tests to halt""" self.running = 0 - # XXX This only works on Unix - self._server_exit.close() + self._server.close() os.waitpid(self._pid, 0) - self.delStorage() self.__super_tearDown() - def checkFirst(self): - self._storage.tpc_begin(self._transaction) - self._storage.tpc_abort(self._transaction) - class ZEOFileStorageTests(GenericTests): __super_setUp = GenericTests.setUp @@ -143,8 +146,7 @@ # file storage appears to create three files for ext in '', '.index', '.lock', '.tmp': path = self.__fs_base + ext - if os.path.exists(path): - os.unlink(path) + os.unlink(path) def main(): import sys, getopt From jeremy at digicool.com Tue May 1 14:39:29 2001 From: jeremy at digicool.com (jeremy@digicool.com) Date: Sun Aug 10 16:31:15 2008 Subject: [ZEO-Checkins] CVS: Packages/ZEO - ClientStorage.py:1.26.4.16 Message-ID: <20010501183929.70A33510E3@korak.digicool.com> Update of /cvs-repository/Packages/ZEO In directory korak:/tmp/cvs-serv2484 Modified Files: Tag: ZEO-ZRPC-Dev ClientStorage.py Log Message: Tentative change: Get rid of the ThreadLock. There are not re-entrant calls. Furthermore, the condition variable around self._transaction plus a reasonable client should provide everything that is needed. --- Updated File ClientStorage.py in package Packages/ZEO -- --- ClientStorage.py 2001/04/27 20:57:18 1.26.4.15 +++ ClientStorage.py 2001/05/01 18:39:27 1.26.4.16 @@ -97,6 +97,7 @@ import sys import tempfile import thread +import threading import time from types import TupleType, StringType from struct import pack, unpack @@ -183,20 +184,30 @@ self.__name__ = name + # A ClientStorage only allows one client to commit at a time. + # A client enters the commit state by finding tpc_tid set to + # None and updating it to the new transaction's id. The + # tpc_tid variable is protected by tpc_cond. + self.tpc_cond = threading.Condition() + self._transaction = None + + # Cache synchronization + # We need to make sure the cache isn't accessed in an + # inconsistent state. If one client is doing a load while + # another is committing a transaction, the cache could contain + # partial results for the committing transaction. Thus, there + # needs to be locking so that only one thread is + # reading/writing the cache at a time. + self.cache_lock = threading.Lock() + commit_lock = thread.allocate_lock() self._commit_lock_acquire = commit_lock.acquire self._commit_lock_release = commit_lock.release - # What's the difference between thread and ThreadLock? - l = ThreadLock.allocate_lock() - self._lock_acquire = l.acquire - self._lock_release = l.release - t = time.time() t = self._ts = apply(TimeStamp,(time.gmtime(t)[:5]+(t%60,))) self._serial = `t` self._oid='\0\0\0\0\0\0\0\0' - self._transaction = None def registerDB(self, db, limit): """Register that the storage is controlled by the given DB.""" @@ -204,28 +215,19 @@ self._db = db def is_connected(self): - self._lock_acquire() - try: - if self._server: - return 1 - else: - return 0 - finally: - self._lock_release() + if self._server: + return 1 + else: + return 0 def notifyConnected(self, c): log2(INFO, "Connected to storage") - self._lock_acquire() - try: - self._server = ServerStub.StorageServer(c) + self._server = ServerStub.StorageServer(c) - self._oids = [] - - self._server.register(str(self._storage)) - self.verify_cache() + self._oids = [] - finally: - self._lock_release() + self._server.register(str(self._storage)) + self.verify_cache() def verify_cache(self): self._server.beginZeoVerify() @@ -247,113 +249,134 @@ def notifyDisconnected(self, ignored): log2(PROBLEM, "Disconnected from storage") self._transaction = None - try: - self._commit_lock_release() - except: - pass + if self._transaction: + self._transaction = None + self.tpc_cond.notifyAll() + self.tpc_cond.release() def __len__(self): return self._info['length'] - def abortVersion(self, src, transaction): - if transaction is not self._transaction: - raise POSException.StorageTransactionError(self, transaction) - self._lock_acquire() - try: - oids = self._server.abortVersion(src, self._serial) - invalidate = self._cache.invalidate - for oid in oids: - invalidate(oid, src) - return oids - finally: - self._lock_release() - - def close(self): - self._lock_acquire() - try: - # Close the manager first, so that it doesn't attempt to - # re-open the connection. - self._rpc_mgr.close() - self._server.rpc.close() - finally: - self._lock_release() - - def commitVersion(self, src, dest, transaction): - if transaction is not self._transaction: - raise POSException.StorageTransactionError(self, transaction) - self._lock_acquire() - try: - oids = self._server.commitVersion(src, dest, self._serial) - invalidate = self._cache.invalidate - if dest: - # just invalidate our version data - for oid in oids: - invalidate(oid, src) - else: - # dest is '', so invalidate version and non-version - for oid in oids: - invalidate(oid, dest) - return oids - finally: - self._lock_release() - def getName(self): return "%s (%s)" % (self.__name__, "XXX") def getSize(self): return self._info['size'] - def history(self, oid, version, length=1): - self._lock_acquire() + def supportsUndo(self): + return self._info['supportsUndo'] + + def supportsVersions(self): + return self._info['supportsVersions'] + + def supportsTransactionalUndo(self): + return self._info['supportsTransactionalUndo'] + + def _check_trans(self, trans, exc=None): + if self._transaction is not trans: + if exc is None: + return 0 + else: + raise exc(self._transaction, trans) + return 1 + + def _check_tid(self, tid, exc=None): + # XXX Is all this locking unnecessary? The only way to + # begin a transaction is to call tpc_begin(). If we assume + # clients are single-threaded and well-behaved, i.e. they call + # tpc_begin() first, then there appears to be no need for + # locking. If _check_tid() is called and self.tpc_tid != tid, + # then there is no way it can be come equal during the call. + # Thus, there should be no race. + + if self.tpc_tid != tid: + if exc is None: + return 0 + else: + raise exc(self.tpc_tid, tid) + return 1 + + # XXX But I'm not sure + + self.tpc_cond.acquire() try: - return self._server.history(oid, version, length) + if self.tpc_tid != tid: + if exc is None: + return 0 + else: + raise exc(self.tpc_tid, tid) + return 1 finally: - self._lock_release() + self.tpc_cond.release() + + def abortVersion(self, src, transaction): + self._check_trans(transaction, + POSException.StorageTransactionError) + oids = self._server.abortVersion(src, self._serial) + invalidate = self._cache.invalidate + for oid in oids: + invalidate(oid, src) + return oids + + def close(self): + # Close the manager first, so that it doesn't attempt to + # re-open the connection. + self._rpc_mgr.close() + self._server.rpc.close() + + def commitVersion(self, src, dest, transaction): + self._check_trans(transaction, + POSException.StorageTransactionError) + oids = self._server.commitVersion(src, dest, self._serial) + invalidate = self._cache.invalidate + if dest: + # just invalidate our version data + for oid in oids: + invalidate(oid, src) + else: + # dest is '', so invalidate version and non-version + for oid in oids: + invalidate(oid, dest) + return oids + + def history(self, oid, version, length=1): + # XXX sync + return self._server.history(oid, version, length) def loadSerial(self, oid, serial): - self._lock_acquire() - try: - return self._server.loadSerial(oid, serial) - finally: - self._lock_release() + # XXX sync + return self._server.loadSerial(oid, serial) def load(self, oid, version, _stuff=None): - self._lock_acquire() - try: - p = self._cache.load(oid, version) - if p: - return p - p, s, v, pv, sv = self._server.zeoLoad(oid) - self._cache.checkSize(0) - self._cache.store(oid, p, s, v, pv, sv) - if v and version and v == version: - return pv, sv - else: - if s: - return p, s - raise KeyError, oid # no non-version data for this - finally: - self._lock_release() + # XXX sync + p = self._cache.load(oid, version) + if p: + return p + p, s, v, pv, sv = self._server.zeoLoad(oid) + self._cache.checkSize(0) + self._cache.store(oid, p, s, v, pv, sv) + if v and version and v == version: + return pv, sv + else: + if s: + return p, s + raise KeyError, oid # no non-version data for this def modifiedInVersion(self, oid): - self._lock_acquire() - try: - v = self._cache.modifiedInVersion(oid) - if v is not None: - return v - return self._server.modifiedInVersion(oid) - finally: - self._lock_release() + # XXX sync + v = self._cache.modifiedInVersion(oid) + if v is not None: + return v + return self._server.modifiedInVersion(oid) def new_oid(self, last=None): - self._lock_acquire() - try: - if not self._oids: - self._oids = self._server.new_oids() - self._oids.reverse() - return self._oids.pop() - finally: - self._lock_release() +## if self._transaction is None: +## # XXX What exception? +## raise POSException.StorageTransactionError() + if not self._oids: + self._oids = self._server.new_oids() + self._oids.reverse() + return self._oids.pop() def pack(self, t=None, rf=None, wait=0, days=0): # Note that we ignore the rf argument. The server @@ -361,11 +384,7 @@ if t is None: t = time.time() t = t - (days * 86400) - self._lock_acquire() - try: - return self._server.pack(t, wait) - finally: - self._lock_release() + return self._server.pack(t, wait) def _check_serials(self): if self._serials: @@ -379,181 +398,124 @@ return r def store(self, oid, serial, data, version, transaction): - if transaction is not self._transaction: - raise POSException.StorageTransactionError(self, transaction) - self._lock_acquire() - try: - self._server.storea(oid, serial, data, version, self._serial) - self._tbuf.store(oid, version, data) - return self._check_serials() - finally: - self._lock_release() + self._check_trans(transaction, POSException.StorageTransactionError) + self._server.storea(oid, serial, data, version, self._serial) + self._tbuf.store(oid, version, data) + return self._check_serials() def tpc_vote(self, transaction): if transaction is not self._transaction: - raise POSException.StorageTransactionError(self, transaction) - self._lock_acquire() - try: - self._server.vote(self._serial) - return self._check_serials() - finally: - self._lock_release() + return + self._server.vote(self._serial) + return self._check_serials() - def supportsUndo(self): - return self._info['supportsUndo'] - - def supportsVersions(self): - return self._info['supportsVersions'] - - def supportsTransactionalUndo(self): - return self._info['supportsTransactionalUndo'] - def tpc_abort(self, transaction): - self._lock_acquire() - try: - if transaction is not self._transaction: - return - self._server.tpc_abort(self._serial) - self._transaction = None - self._tbuf.clear() - self._seriald.clear() - del self._serials[:] - self._commit_lock_release() - finally: - self._lock_release() + if transaction is not self._transaction: + return + self._server.tpc_abort(self._serial) + self._tbuf.clear() + self._seriald.clear() + del self._serials[:] + self._transaction = None + self.tpc_cond.notify() + self.tpc_cond.release() def tpc_begin(self, transaction): # XXX plan is to have begin be a local operation until the - # vote stage. - self._lock_acquire() - try: - if self._transaction is transaction: - return # can start the same transaction many times - self._lock_release() - self._commit_lock_acquire() - self._lock_acquire() - - self._ts = get_timestamp(self._ts) - id = `self._ts` + # vote stage. + self.tpc_cond.acquire() + while self._transaction is not None: + if self._transaction == transaction: + self.tpc_cond.release() + return + self.tpc_cond.wait() + + self._ts = get_timestamp(self._ts) + id = `self._ts` + self._transaction = transaction - try: - r = self._server.tpc_begin(id, - transaction.user, - transaction.description, - transaction._extension) - except: - self._commit_lock_release() - raise - - assert r is None - - # We have *BOTH* the local and distributed commit - # lock, now we can actually get ready to get started. - self._serial = id - self._seriald.clear() - del self._serials[:] + try: + r = self._server.tpc_begin(id, + transaction.user, + transaction.description, + transaction._extension) + except: + self.tpc_cond.release() + raise - self._transaction = transaction - finally: - self._lock_release() + self._serial = id + self._seriald.clear() + del self._serials[:] def tpc_finish(self, transaction, f=None): - self._lock_acquire() - try: - if transaction is not self._transaction: - return - if f is not None: # XXX what is f()? - f() + if transaction is not self._transaction: + return + if f is not None: # XXX what is f()? + f() - self._server.tpc_finish(self._serial) + self._server.tpc_finish(self._serial) - r = self._check_serials() - assert r is None or len(r) == 0, "unhandled serialnos: %s" % r + r = self._check_serials() + assert r is None or len(r) == 0, "unhandled serialnos: %s" % r - self._cache.checkSize(self._tbuf.get_size()) - - # Iterate over the objects in the transaction buffer and - # update or invalidate the cache. - self._tbuf.begin_iterate() - while 1: - try: - t = self._tbuf.next() - except ValueError, msg: - raise ClientStorageError, ( - "Unexpected error reading temporary file in " - "client storage: %s" % msg) - if t is None: - break - oid, v, p = t - s = self._seriald[oid] - if type(s) != StringType: - log2(INFO, "bad serialno: %s for %s" % \ - (repr(s), repr(oid))) - assert type(s) == StringType, "bad serialno: %s" % repr(s) - if s == ResolvedSerial: - self._cache.invalidate(oid, v) - else: - self._cache.update(oid, s, v, p) - self._tbuf.clear() + self._cache.checkSize(self._tbuf.get_size()) - self._transaction=None - self._commit_lock_release() - finally: self._lock_release() + # Iterate over the objects in the transaction buffer and + # update or invalidate the cache. + self._tbuf.begin_iterate() + while 1: + try: + t = self._tbuf.next() + except ValueError, msg: + raise ClientStorageError, ( + "Unexpected error reading temporary file in " + "client storage: %s" % msg) + if t is None: + break + oid, v, p = t + s = self._seriald[oid] + if type(s) != StringType: + log2(INFO, "bad serialno: %s for %s" % \ + (repr(s), repr(oid))) + assert type(s) == StringType, "bad serialno: %s" % repr(s) + if s == ResolvedSerial: + self._cache.invalidate(oid, v) + else: + self._cache.update(oid, s, v, p) + self._tbuf.clear() + self._transaction = None + self.tpc_cond.notify() + self.tpc_cond.release() + def transactionalUndo(self, trans_id, trans): - if trans is not self._transaction: - raise POSException.StorageTransactionError(self._transaction, - transaction) - self._lock_acquire() - try: - oids = self._server.transactionalUndo(trans_id, self._serial) - for oid in oids: - self._cache.invalidate(oid, '') - return oids - finally: - self._lock_release() + self._check_trans(trans, POSException.StorageTransactionError) + oids = self._server.transactionalUndo(trans_id, self._serial) + for oid in oids: + self._cache.invalidate(oid, '') + return oids def undo(self, transaction_id): - self._lock_acquire() - try: - oids = self._server.undo(transaction_id) - cinvalidate = self._cache.invalidate - for oid in oids: - cinvalidate(oid,'') - return oids - finally: self._lock_release() + oids = self._server.undo(transaction_id) + cinvalidate = self._cache.invalidate + for oid in oids: + cinvalidate(oid, '') + return oids - def undoInfo(self, first=0, last=-20, specification=None): - self._lock_acquire() - try: - return self._server.undoInfo(first, last, specification) - finally: - self._lock_release() + return self._server.undoInfo(first, last, specification) def undoLog(self, first, last, filter=None): if filter is not None: return () # XXX can't pass a filter to server - self._lock_acquire() - try: - return self._server.undoLog(first, last) # Eek! - finally: - self._lock_release() + return self._server.undoLog(first, last) # Eek! def versionEmpty(self, version): - self._lock_acquire() - try: - return self._server.versionEmpty(version) - finally: - self._lock_release() + return self._server.versionEmpty(version) def versions(self, max=None): - self._lock_acquire() - try: - return self._server.versions(max) - finally: - self._lock_release() + return self._server.versions(max) # below are methods invoked by the StorageServer --- Updated File ClientStorage.py in package Packages/ZEO -- --- ClientStorage.py 2001/04/27 20:57:18 1.26.4.15 +++ ClientStorage.py 2001/05/01 18:39:27 1.26.4.16 @@ -97,6 +97,7 @@ import sys import tempfile import thread +import threading import time from types import TupleType, StringType from struct import pack, unpack @@ -183,20 +184,30 @@ self.__name__ = name + # A ClientStorage only allows one client to commit at a time. + # A client enters the commit state by finding tpc_tid set to + # None and updating it to the new transaction's id. The + # tpc_tid variable is protected by tpc_cond. + self.tpc_cond = threading.Condition() + self._transaction = None + + # Cache synchronization + # We need to make sure the cache isn't accessed in an + # inconsistent state. If one client is doing a load while + # another is committing a transaction, the cache could contain + # partial results for the committing transaction. Thus, there + # needs to be locking so that only one thread is + # reading/writing the cache at a time. + self.cache_lock = threading.Lock() + commit_lock = thread.allocate_lock() self._commit_lock_acquire = commit_lock.acquire self._commit_lock_release = commit_lock.release - # What's the difference between thread and ThreadLock? - l = ThreadLock.allocate_lock() - self._lock_acquire = l.acquire - self._lock_release = l.release - t = time.time() t = self._ts = apply(TimeStamp,(time.gmtime(t)[:5]+(t%60,))) self._serial = `t` self._oid='\0\0\0\0\0\0\0\0' - self._transaction = None def registerDB(self, db, limit): """Register that the storage is controlled by the given DB.""" @@ -204,28 +215,19 @@ self._db = db def is_connected(self): - self._lock_acquire() - try: - if self._server: - return 1 - else: - return 0 - finally: - self._lock_release() + if self._server: + return 1 + else: + return 0 def notifyConnected(self, c): log2(INFO, "Connected to storage") - self._lock_acquire() - try: - self._server = ServerStub.StorageServer(c) + self._server = ServerStub.StorageServer(c) - self._oids = [] - - self._server.register(str(self._storage)) - self.verify_cache() + self._oids = [] - finally: - self._lock_release() + self._server.register(str(self._storage)) + self.verify_cache() def verify_cache(self): self._server.beginZeoVerify() @@ -247,113 +249,134 @@ def notifyDisconnected(self, ignored): log2(PROBLEM, "Disconnected from storage") self._transaction = None - try: - self._commit_lock_release() - except: - pass + if self._transaction: + self._transaction = None + self.tpc_cond.notifyAll() + self.tpc_cond.release() def __len__(self): return self._info['length'] - def abortVersion(self, src, transaction): - if transaction is not self._transaction: - raise POSException.StorageTransactionError(self, transaction) - self._lock_acquire() - try: - oids = self._server.abortVersion(src, self._serial) - invalidate = self._cache.invalidate - for oid in oids: - invalidate(oid, src) - return oids - finally: - self._lock_release() - - def close(self): - self._lock_acquire() - try: - # Close the manager first, so that it doesn't attempt to - # re-open the connection. - self._rpc_mgr.close() - self._server.rpc.close() - finally: - self._lock_release() - - def commitVersion(self, src, dest, transaction): - if transaction is not self._transaction: - raise POSException.StorageTransactionError(self, transaction) - self._lock_acquire() - try: - oids = self._server.commitVersion(src, dest, self._serial) - invalidate = self._cache.invalidate - if dest: - # just invalidate our version data - for oid in oids: - invalidate(oid, src) - else: - # dest is '', so invalidate version and non-version - for oid in oids: - invalidate(oid, dest) - return oids - finally: - self._lock_release() - def getName(self): return "%s (%s)" % (self.__name__, "XXX") def getSize(self): return self._info['size'] - def history(self, oid, version, length=1): - self._lock_acquire() + def supportsUndo(self): + return self._info['supportsUndo'] + + def supportsVersions(self): + return self._info['supportsVersions'] + + def supportsTransactionalUndo(self): + return self._info['supportsTransactionalUndo'] + + def _check_trans(self, trans, exc=None): + if self._transaction is not trans: + if exc is None: + return 0 + else: + raise exc(self._transaction, trans) + return 1 + + def _check_tid(self, tid, exc=None): + # XXX Is all this locking unnecessary? The only way to + # begin a transaction is to call tpc_begin(). If we assume + # clients are single-threaded and well-behaved, i.e. they call + # tpc_begin() first, then there appears to be no need for + # locking. If _check_tid() is called and self.tpc_tid != tid, + # then there is no way it can be come equal during the call. + # Thus, there should be no race. + + if self.tpc_tid != tid: + if exc is None: + return 0 + else: + raise exc(self.tpc_tid, tid) + return 1 + + # XXX But I'm not sure + + self.tpc_cond.acquire() try: - return self._server.history(oid, version, length) + if self.tpc_tid != tid: + if exc is None: + return 0 + else: + raise exc(self.tpc_tid, tid) + return 1 finally: - self._lock_release() + self.tpc_cond.release() + + def abortVersion(self, src, transaction): + self._check_trans(transaction, + POSException.StorageTransactionError) + oids = self._server.abortVersion(src, self._serial) + invalidate = self._cache.invalidate + for oid in oids: + invalidate(oid, src) + return oids + + def close(self): + # Close the manager first, so that it doesn't attempt to + # re-open the connection. + self._rpc_mgr.close() + self._server.rpc.close() + + def commitVersion(self, src, dest, transaction): + self._check_trans(transaction, + POSException.StorageTransactionError) + oids = self._server.commitVersion(src, dest, self._serial) + invalidate = self._cache.invalidate + if dest: + # just invalidate our version data + for oid in oids: + invalidate(oid, src) + else: + # dest is '', so invalidate version and non-version + for oid in oids: + invalidate(oid, dest) + return oids + + def history(self, oid, version, length=1): + # XXX sync + return self._server.history(oid, version, length) def loadSerial(self, oid, serial): - self._lock_acquire() - try: - return self._server.loadSerial(oid, serial) - finally: - self._lock_release() + # XXX sync + return self._server.loadSerial(oid, serial) def load(self, oid, version, _stuff=None): - self._lock_acquire() - try: - p = self._cache.load(oid, version) - if p: - return p - p, s, v, pv, sv = self._server.zeoLoad(oid) - self._cache.checkSize(0) - self._cache.store(oid, p, s, v, pv, sv) - if v and version and v == version: - return pv, sv - else: - if s: - return p, s - raise KeyError, oid # no non-version data for this - finally: - self._lock_release() + # XXX sync + p = self._cache.load(oid, version) + if p: + return p + p, s, v, pv, sv = self._server.zeoLoad(oid) + self._cache.checkSize(0) + self._cache.store(oid, p, s, v, pv, sv) + if v and version and v == version: + return pv, sv + else: + if s: + return p, s + raise KeyError, oid # no non-version data for this def modifiedInVersion(self, oid): - self._lock_acquire() - try: - v = self._cache.modifiedInVersion(oid) - if v is not None: - return v - return self._server.modifiedInVersion(oid) - finally: - self._lock_release() + # XXX sync + v = self._cache.modifiedInVersion(oid) + if v is not None: + return v + return self._server.modifiedInVersion(oid) def new_oid(self, last=None): - self._lock_acquire() - try: - if not self._oids: - self._oids = self._server.new_oids() - self._oids.reverse() - return self._oids.pop() - finally: - self._lock_release() +## if self._transaction is None: +## # XXX What exception? +## raise POSException.StorageTransactionError() + if not self._oids: + self._oids = self._server.new_oids() + self._oids.reverse() + return self._oids.pop() def pack(self, t=None, rf=None, wait=0, days=0): # Note that we ignore the rf argument. The server @@ -361,11 +384,7 @@ if t is None: t = time.time() t = t - (days * 86400) - self._lock_acquire() - try: - return self._server.pack(t, wait) - finally: - self._lock_release() + return self._server.pack(t, wait) def _check_serials(self): if self._serials: @@ -379,181 +398,124 @@ return r def store(self, oid, serial, data, version, transaction): - if transaction is not self._transaction: - raise POSException.StorageTransactionError(self, transaction) - self._lock_acquire() - try: - self._server.storea(oid, serial, data, version, self._serial) - self._tbuf.store(oid, version, data) - return self._check_serials() - finally: - self._lock_release() + self._check_trans(transaction, POSException.StorageTransactionError) + self._server.storea(oid, serial, data, version, self._serial) + self._tbuf.store(oid, version, data) + return self._check_serials() def tpc_vote(self, transaction): if transaction is not self._transaction: - raise POSException.StorageTransactionError(self, transaction) - self._lock_acquire() - try: - self._server.vote(self._serial) - return self._check_serials() - finally: - self._lock_release() + return + self._server.vote(self._serial) + return self._check_serials() - def supportsUndo(self): - return self._info['supportsUndo'] - - def supportsVersions(self): - return self._info['supportsVersions'] - - def supportsTransactionalUndo(self): - return self._info['supportsTransactionalUndo'] - def tpc_abort(self, transaction): - self._lock_acquire() - try: - if transaction is not self._transaction: - return - self._server.tpc_abort(self._serial) - self._transaction = None - self._tbuf.clear() - self._seriald.clear() - del self._serials[:] - self._commit_lock_release() - finally: - self._lock_release() + if transaction is not self._transaction: + return + self._server.tpc_abort(self._serial) + self._tbuf.clear() + self._seriald.clear() + del self._serials[:] + self._transaction = None + self.tpc_cond.notify() + self.tpc_cond.release() def tpc_begin(self, transaction): # XXX plan is to have begin be a local operation until the - # vote stage. - self._lock_acquire() - try: - if self._transaction is transaction: - return # can start the same transaction many times - self._lock_release() - self._commit_lock_acquire() - self._lock_acquire() - - self._ts = get_timestamp(self._ts) - id = `self._ts` + # vote stage. + self.tpc_cond.acquire() + while self._transaction is not None: + if self._transaction == transaction: + self.tpc_cond.release() + return + self.tpc_cond.wait() + + self._ts = get_timestamp(self._ts) + id = `self._ts` + self._transaction = transaction - try: - r = self._server.tpc_begin(id, - transaction.user, - transaction.description, - transaction._extension) - except: - self._commit_lock_release() - raise - - assert r is None - - # We have *BOTH* the local and distributed commit - # lock, now we can actually get ready to get started. - self._serial = id - self._seriald.clear() - del self._serials[:] + try: + r = self._server.tpc_begin(id, + transaction.user, + transaction.description, + transaction._extension) + except: + self.tpc_cond.release() + raise - self._transaction = transaction - finally: - self._lock_release() + self._serial = id + self._seriald.clear() + del self._serials[:] def tpc_finish(self, transaction, f=None): - self._lock_acquire() - try: - if transaction is not self._transaction: - return - if f is not None: # XXX what is f()? - f() + if transaction is not self._transaction: + return + if f is not None: # XXX what is f()? + f() - self._server.tpc_finish(self._serial) + self._server.tpc_finish(self._serial) - r = self._check_serials() - assert r is None or len(r) == 0, "unhandled serialnos: %s" % r + r = self._check_serials() + assert r is None or len(r) == 0, "unhandled serialnos: %s" % r - self._cache.checkSize(self._tbuf.get_size()) - - # Iterate over the objects in the transaction buffer and - # update or invalidate the cache. - self._tbuf.begin_iterate() - while 1: - try: - t = self._tbuf.next() - except ValueError, msg: - raise ClientStorageError, ( - "Unexpected error reading temporary file in " - "client storage: %s" % msg) - if t is None: - break - oid, v, p = t - s = self._seriald[oid] - if type(s) != StringType: - log2(INFO, "bad serialno: %s for %s" % \ - (repr(s), repr(oid))) - assert type(s) == StringType, "bad serialno: %s" % repr(s) - if s == ResolvedSerial: - self._cache.invalidate(oid, v) - else: - self._cache.update(oid, s, v, p) - self._tbuf.clear() + self._cache.checkSize(self._tbuf.get_size()) - self._transaction=None - self._commit_lock_release() - finally: self._lock_release() + # Iterate over the objects in the transaction buffer and + # update or invalidate the cache. + self._tbuf.begin_iterate() + while 1: + try: + t = self._tbuf.next() + except ValueError, msg: + raise ClientStorageError, ( + "Unexpected error reading temporary file in " + "client storage: %s" % msg) + if t is None: + break + oid, v, p = t + s = self._seriald[oid] + if type(s) != StringType: + log2(INFO, "bad serialno: %s for %s" % \ + (repr(s), repr(oid))) + assert type(s) == StringType, "bad serialno: %s" % repr(s) + if s == ResolvedSerial: + self._cache.invalidate(oid, v) + else: + self._cache.update(oid, s, v, p) + self._tbuf.clear() + self._transaction = None + self.tpc_cond.notify() + self.tpc_cond.release() + def transactionalUndo(self, trans_id, trans): - if trans is not self._transaction: - raise POSException.StorageTransactionError(self._transaction, - transaction) - self._lock_acquire() - try: - oids = self._server.transactionalUndo(trans_id, self._serial) - for oid in oids: - self._cache.invalidate(oid, '') - return oids - finally: - self._lock_release() + self._check_trans(trans, POSException.StorageTransactionError) + oids = self._server.transactionalUndo(trans_id, self._serial) + for oid in oids: + self._cache.invalidate(oid, '') + return oids def undo(self, transaction_id): - self._lock_acquire() - try: - oids = self._server.undo(transaction_id) - cinvalidate = self._cache.invalidate - for oid in oids: - cinvalidate(oid,'') - return oids - finally: self._lock_release() + oids = self._server.undo(transaction_id) + cinvalidate = self._cache.invalidate + for oid in oids: + cinvalidate(oid, '') + return oids - def undoInfo(self, first=0, last=-20, specification=None): - self._lock_acquire() - try: - return self._server.undoInfo(first, last, specification) - finally: - self._lock_release() + return self._server.undoInfo(first, last, specification) def undoLog(self, first, last, filter=None): if filter is not None: return () # XXX can't pass a filter to server - self._lock_acquire() - try: - return self._server.undoLog(first, last) # Eek! - finally: - self._lock_release() + return self._server.undoLog(first, last) # Eek! def versionEmpty(self, version): - self._lock_acquire() - try: - return self._server.versionEmpty(version) - finally: - self._lock_release() + return self._server.versionEmpty(version) def versions(self, max=None): - self._lock_acquire() - try: - return self._server.versions(max) - finally: - self._lock_release() + return self._server.versions(max) # below are methods invoked by the StorageServer From jeremy at digicool.com Tue May 1 18:35:05 2001 From: jeremy at digicool.com (jeremy@digicool.com) Date: Sun Aug 10 16:31:15 2008 Subject: [ZEO-Checkins] CVS: Packages/ZEO - zrpc2.py:1.1.2.12 Message-ID: <20010501223505.13684510E3@korak.digicool.com> Update of /cvs-repository/Packages/ZEO In directory korak:/tmp/cvs-serv20475 Modified Files: Tag: ZEO-ZRPC-Dev zrpc2.py Log Message: Add a call lock so that only a single call can be outstanding at one time. This isn't a long-term solution, but allows ClientStorage sans synchronized methods to function correctly. --- Updated File zrpc2.py in package Packages/ZEO -- --- zrpc2.py 2001/04/25 22:15:22 1.1.2.11 +++ zrpc2.py 2001/05/01 22:35:03 1.1.2.12 @@ -145,6 +145,7 @@ # waiting for a response self.__super_init(sock, addr) self._prepare_async() + self.__call_lock = thread.allocate_lock() self.__reply_lock = thread.allocate_lock() self.__reply_lock.acquire() if isinstance(obj, Handler): @@ -287,6 +288,13 @@ # synchronous calls def call(self, method, *args): + self.__call_lock.acquire() + try: + return self._call(method, args) + finally: + self.__call_lock.release() + + def _call(self, method, args): if self.closed: raise DisconnectedError("This action is temporarily unavailable") msgid = self.msgid @@ -310,6 +318,13 @@ return r_args def callAsync(self, method, *args): + self.__call_lock.acquire() + try: + self._callAsync(method, args) + finally: + self.__call_lock.release() + + def _callAsync(self, method, args): if self.closed: raise DisconnectedError("This action is temporarily unavailable") msgid = self.msgid --- Updated File zrpc2.py in package Packages/ZEO -- --- zrpc2.py 2001/04/25 22:15:22 1.1.2.11 +++ zrpc2.py 2001/05/01 22:35:03 1.1.2.12 @@ -145,6 +145,7 @@ # waiting for a response self.__super_init(sock, addr) self._prepare_async() + self.__call_lock = thread.allocate_lock() self.__reply_lock = thread.allocate_lock() self.__reply_lock.acquire() if isinstance(obj, Handler): @@ -287,6 +288,13 @@ # synchronous calls def call(self, method, *args): + self.__call_lock.acquire() + try: + return self._call(method, args) + finally: + self.__call_lock.release() + + def _call(self, method, args): if self.closed: raise DisconnectedError("This action is temporarily unavailable") msgid = self.msgid @@ -310,6 +318,13 @@ return r_args def callAsync(self, method, *args): + self.__call_lock.acquire() + try: + self._callAsync(method, args) + finally: + self.__call_lock.release() + + def _callAsync(self, method, args): if self.closed: raise DisconnectedError("This action is temporarily unavailable") msgid = self.msgid From jeremy at digicool.com Tue May 1 18:40:46 2001 From: jeremy at digicool.com (jeremy@digicool.com) Date: Sun Aug 10 16:31:15 2008 Subject: [ZEO-Checkins] CVS: Packages/ZEO - speed.py:1.1.2.2 Message-ID: <20010501224046.CADDF510E3@korak.digicool.com> Update of /cvs-repository/Packages/ZEO/tests In directory korak:/tmp/cvs-serv20902/tests Modified Files: Tag: ZEO-ZRPC-Dev speed.py Log Message: Modify to user forker Add -t n flag to specify number of concurrent threads Add -U flag to specify a Unix domain socket --- Updated File speed.py in package Packages/ZEO -- --- speed.py 2001/04/20 19:17:22 1.1.2.1 +++ speed.py 2001/05/01 22:40:45 1.1.2.2 @@ -106,6 +106,10 @@ -M Output means only -C Run with a persistent client cache + + -U Run ZEO using a Unix domain socket + + -t n Number of concurrent threads to run. """ import asyncore @@ -115,6 +119,8 @@ import ZODB, ZODB.FileStorage import Persistence import ZEO.ClientStorage, ZEO.StorageServer +from ZEO.tests import forker +from ZODB.POSException import ConflictError class P(Persistence.Persistent): pass @@ -141,8 +147,49 @@ os.unlink(fs_name + ".lock") os.unlink(fs_name + ".tmp") +def work(db, results, nrep, compress, data, detailed, minimize, threadno=None): + for j in range(nrep): + for r in 1, 10, 100, 1000: + t = time.time() + + jar = db.open() + while 1: + try: + get_transaction().begin() + rt = jar.root() + key = 's%s' % r + if rt.has_key(key): + p = rt[key] + else: + rt[key] = p =P() + for i in range(r): + v = getattr(p, str(i), P()) + if compress is not None: + v.d = compress(data) + else: + v.d = data + setattr(p, str(i), v) + get_transaction().commit() + except ConflictError: + pass + else: + break + jar.close() + + t = time.time() - t + if detailed: + if threadno is None: + print "%s\t%s\t%.4f" % (j, r, t) + else: + print "%s\t%s\t%.4f\t%d" % (j, r, t, threadno) + results[r] = results[r] + t + rt=d=p=v=None # release all references + if minimize: + time.sleep(3) + jar.cacheMinimize(3) + def main(args): - opts, args = getopt.getopt(args, 'zd:n:Ds:LM') + opts, args = getopt.getopt(args, 'zd:n:Ds:LMt:U') s = None compress = None data=sys.argv[0] @@ -150,6 +197,8 @@ minimize=0 detailed=1 cache = None + domain = 'AF_INET' + threads = 1 for o, v in opts: if o=='-n': nrep = int(v) elif o=='-d': data = v @@ -168,79 +217,52 @@ debug = 1 elif o == '-C': cache = 'speed' + elif o == '-U': + domain = 'AF_UNIX' + elif o == '-t': + threads = int(v) zeo_pipe = None if s: s = __import__(s, globals(), globals(), ('__doc__',)) s = s.Storage + server = None else: - rd, wr = os.pipe() - pid = os.fork() - if pid: - # in the child, run the storage server - os.close(wr) - import asyncore - ZEOExit(rd) - fs = ZODB.FileStorage.FileStorage(fs_name, create=1) - serv = ZEO.StorageServer.StorageServer(('', 1975), {'1':fs}) - asyncore.loop() - else: - os.close(rd) - zeo_pipe = wr - s = ZEO.ClientStorage.ClientStorage(('', 1975), debug=0, - client=cache) - if hasattr(s, 'is_connected'): - while not s.is_connected(): - time.sleep(0.1) - else: - time.sleep(1.0) + fs = ZODB.FileStorage.FileStorage(fs_name, create=1) + s, server, pid = forker.start_zeo(fs, domain=domain) + while not s.is_connected(): + time.sleep(0.1) data=open(data).read() db=ZODB.DB(s, # disable cache deactivation cache_size=4000, cache_deactivate_after=6000,) - db.open().root() + print "Beginning work..." results={1:0, 10:0, 100:0, 1000:0} - for j in range(nrep): - for r in 1, 10, 100, 1000: - t = time.time() - - jar = db.open() - get_transaction().begin() - rt = jar.root() - key = 's%s' % r - if rt.has_key(key): - p = rt[key] - else: - rt[key] = p =P() - for i in range(r): - v = getattr(p, str(i), P()) - if compress is not None: - v.d = compress(data) - else: - v.d = data - setattr(p, str(i), v) - get_transaction().commit() - jar.close() - - t = time.time() - t - if detailed: - print "%s\t%s\t%.4f" % (j, r, t) - results[r] = results[r] + t - rt=d=p=v=None # release all references - if minimize: - time.sleep(3) - jar.cacheMinimize(3) + if threads > 1: + import threading + l = [threading.Thread(target=work, + args=(db, results, nrep, compress, data, + detailed, minimize, i)) + for i in range(threads)] + for t in l: + t.start() + for t in l: + t.join() + + else: + work(db, results, nrep, compress, data, detailed, minimize) - if zeo_pipe: - os.write(zeo_pipe, "done") + if server is not None: + server.close() + os.waitpid(pid, 0) if detailed: print '-'*24 for r in 1, 10, 100, 1000: - t=results[r]/nrep + t=results[r]/(nrep * threads) print "mean:\t%s\t%.4f\t%.4f (s/o)" % (r, t, t/r) ##def compress(s): --- Updated File speed.py in package Packages/ZEO -- --- speed.py 2001/04/20 19:17:22 1.1.2.1 +++ speed.py 2001/05/01 22:40:45 1.1.2.2 @@ -106,6 +106,10 @@ -M Output means only -C Run with a persistent client cache + + -U Run ZEO using a Unix domain socket + + -t n Number of concurrent threads to run. """ import asyncore @@ -115,6 +119,8 @@ import ZODB, ZODB.FileStorage import Persistence import ZEO.ClientStorage, ZEO.StorageServer +from ZEO.tests import forker +from ZODB.POSException import ConflictError class P(Persistence.Persistent): pass @@ -141,8 +147,49 @@ os.unlink(fs_name + ".lock") os.unlink(fs_name + ".tmp") +def work(db, results, nrep, compress, data, detailed, minimize, threadno=None): + for j in range(nrep): + for r in 1, 10, 100, 1000: + t = time.time() + + jar = db.open() + while 1: + try: + get_transaction().begin() + rt = jar.root() + key = 's%s' % r + if rt.has_key(key): + p = rt[key] + else: + rt[key] = p =P() + for i in range(r): + v = getattr(p, str(i), P()) + if compress is not None: + v.d = compress(data) + else: + v.d = data + setattr(p, str(i), v) + get_transaction().commit() + except ConflictError: + pass + else: + break + jar.close() + + t = time.time() - t + if detailed: + if threadno is None: + print "%s\t%s\t%.4f" % (j, r, t) + else: + print "%s\t%s\t%.4f\t%d" % (j, r, t, threadno) + results[r] = results[r] + t + rt=d=p=v=None # release all references + if minimize: + time.sleep(3) + jar.cacheMinimize(3) + def main(args): - opts, args = getopt.getopt(args, 'zd:n:Ds:LM') + opts, args = getopt.getopt(args, 'zd:n:Ds:LMt:U') s = None compress = None data=sys.argv[0] @@ -150,6 +197,8 @@ minimize=0 detailed=1 cache = None + domain = 'AF_INET' + threads = 1 for o, v in opts: if o=='-n': nrep = int(v) elif o=='-d': data = v @@ -168,79 +217,52 @@ debug = 1 elif o == '-C': cache = 'speed' + elif o == '-U': + domain = 'AF_UNIX' + elif o == '-t': + threads = int(v) zeo_pipe = None if s: s = __import__(s, globals(), globals(), ('__doc__',)) s = s.Storage + server = None else: - rd, wr = os.pipe() - pid = os.fork() - if pid: - # in the child, run the storage server - os.close(wr) - import asyncore - ZEOExit(rd) - fs = ZODB.FileStorage.FileStorage(fs_name, create=1) - serv = ZEO.StorageServer.StorageServer(('', 1975), {'1':fs}) - asyncore.loop() - else: - os.close(rd) - zeo_pipe = wr - s = ZEO.ClientStorage.ClientStorage(('', 1975), debug=0, - client=cache) - if hasattr(s, 'is_connected'): - while not s.is_connected(): - time.sleep(0.1) - else: - time.sleep(1.0) + fs = ZODB.FileStorage.FileStorage(fs_name, create=1) + s, server, pid = forker.start_zeo(fs, domain=domain) + while not s.is_connected(): + time.sleep(0.1) data=open(data).read() db=ZODB.DB(s, # disable cache deactivation cache_size=4000, cache_deactivate_after=6000,) - db.open().root() + print "Beginning work..." results={1:0, 10:0, 100:0, 1000:0} - for j in range(nrep): - for r in 1, 10, 100, 1000: - t = time.time() - - jar = db.open() - get_transaction().begin() - rt = jar.root() - key = 's%s' % r - if rt.has_key(key): - p = rt[key] - else: - rt[key] = p =P() - for i in range(r): - v = getattr(p, str(i), P()) - if compress is not None: - v.d = compress(data) - else: - v.d = data - setattr(p, str(i), v) - get_transaction().commit() - jar.close() - - t = time.time() - t - if detailed: - print "%s\t%s\t%.4f" % (j, r, t) - results[r] = results[r] + t - rt=d=p=v=None # release all references - if minimize: - time.sleep(3) - jar.cacheMinimize(3) + if threads > 1: + import threading + l = [threading.Thread(target=work, + args=(db, results, nrep, compress, data, + detailed, minimize, i)) + for i in range(threads)] + for t in l: + t.start() + for t in l: + t.join() + + else: + work(db, results, nrep, compress, data, detailed, minimize) - if zeo_pipe: - os.write(zeo_pipe, "done") + if server is not None: + server.close() + os.waitpid(pid, 0) if detailed: print '-'*24 for r in 1, 10, 100, 1000: - t=results[r]/nrep + t=results[r]/(nrep * threads) print "mean:\t%s\t%.4f\t%.4f (s/o)" % (r, t, t/r) ##def compress(s): From jeremy at digicool.com Tue May 1 18:41:28 2001 From: jeremy at digicool.com (jeremy@digicool.com) Date: Sun Aug 10 16:31:15 2008 Subject: [ZEO-Checkins] CVS: Packages/ZEO - speed.py:1.3 Message-ID: <20010501224128.3CE54510E3@korak.digicool.com> Update of /cvs-repository/Packages/ZEO/tests In directory korak:/tmp/cvs-serv20967/tests Modified Files: speed.py Log Message: Modify to user forker Add -t n flag to specify number of concurrent threads Add -U flag to specify a Unix domain socket --- Updated File speed.py in package Packages/ZEO -- --- speed.py 2001/05/01 19:50:42 1.2 +++ speed.py 2001/05/01 22:41:26 1.3 @@ -106,6 +106,10 @@ -M Output means only -C Run with a persistent client cache + + -U Run ZEO using a Unix domain socket + + -t n Number of concurrent threads to run. """ import asyncore @@ -115,6 +119,8 @@ import ZODB, ZODB.FileStorage import Persistence import ZEO.ClientStorage, ZEO.StorageServer +from ZEO.tests import forker +from ZODB.POSException import ConflictError class P(Persistence.Persistent): pass @@ -141,8 +147,49 @@ os.unlink(fs_name + ".lock") os.unlink(fs_name + ".tmp") +def work(db, results, nrep, compress, data, detailed, minimize, threadno=None): + for j in range(nrep): + for r in 1, 10, 100, 1000: + t = time.time() + + jar = db.open() + while 1: + try: + get_transaction().begin() + rt = jar.root() + key = 's%s' % r + if rt.has_key(key): + p = rt[key] + else: + rt[key] = p =P() + for i in range(r): + v = getattr(p, str(i), P()) + if compress is not None: + v.d = compress(data) + else: + v.d = data + setattr(p, str(i), v) + get_transaction().commit() + except ConflictError: + pass + else: + break + jar.close() + + t = time.time() - t + if detailed: + if threadno is None: + print "%s\t%s\t%.4f" % (j, r, t) + else: + print "%s\t%s\t%.4f\t%d" % (j, r, t, threadno) + results[r] = results[r] + t + rt=d=p=v=None # release all references + if minimize: + time.sleep(3) + jar.cacheMinimize(3) + def main(args): - opts, args = getopt.getopt(args, 'zd:n:Ds:LM') + opts, args = getopt.getopt(args, 'zd:n:Ds:LMt:U') s = None compress = None data=sys.argv[0] @@ -150,6 +197,8 @@ minimize=0 detailed=1 cache = None + domain = 'AF_INET' + threads = 1 for o, v in opts: if o=='-n': nrep = int(v) elif o=='-d': data = v @@ -168,79 +217,50 @@ debug = 1 elif o == '-C': cache = 'speed' + elif o == '-U': + domain = 'AF_UNIX' + elif o == '-t': + threads = int(v) zeo_pipe = None if s: s = __import__(s, globals(), globals(), ('__doc__',)) s = s.Storage + server = None else: - rd, wr = os.pipe() - pid = os.fork() - if pid: - # in the child, run the storage server - os.close(wr) - import asyncore - ZEOExit(rd) - fs = ZODB.FileStorage.FileStorage(fs_name, create=1) - serv = ZEO.StorageServer.StorageServer(('', 1975), {'1':fs}) - asyncore.loop() - else: - os.close(rd) - zeo_pipe = wr - s = ZEO.ClientStorage.ClientStorage(('', 1975), debug=0, - client=cache) - if hasattr(s, 'is_connected'): - while not s.is_connected(): - time.sleep(0.1) - else: - time.sleep(1.0) + fs = ZODB.FileStorage.FileStorage(fs_name, create=1) + s, server, pid = forker.start_zeo(fs, domain=domain) data=open(data).read() db=ZODB.DB(s, # disable cache deactivation cache_size=4000, cache_deactivate_after=6000,) - db.open().root() + print "Beginning work..." results={1:0, 10:0, 100:0, 1000:0} - for j in range(nrep): - for r in 1, 10, 100, 1000: - t = time.time() - - jar = db.open() - get_transaction().begin() - rt = jar.root() - key = 's%s' % r - if rt.has_key(key): - p = rt[key] - else: - rt[key] = p =P() - for i in range(r): - v = getattr(p, str(i), P()) - if compress is not None: - v.d = compress(data) - else: - v.d = data - setattr(p, str(i), v) - get_transaction().commit() - jar.close() - - t = time.time() - t - if detailed: - print "%s\t%s\t%.4f" % (j, r, t) - results[r] = results[r] + t - rt=d=p=v=None # release all references - if minimize: - time.sleep(3) - jar.cacheMinimize(3) + if threads > 1: + import threading + l = [threading.Thread(target=work, + args=(db, results, nrep, compress, data, + detailed, minimize, i)) + for i in range(threads)] + for t in l: + t.start() + for t in l: + t.join() + + else: + work(db, results, nrep, compress, data, detailed, minimize) - if zeo_pipe: - os.write(zeo_pipe, "done") + if server is not None: + server.close() + os.waitpid(pid, 0) if detailed: print '-'*24 for r in 1, 10, 100, 1000: - t=results[r]/nrep + t=results[r]/(nrep * threads) print "mean:\t%s\t%.4f\t%.4f (s/o)" % (r, t, t/r) ##def compress(s): --- Updated File speed.py in package Packages/ZEO -- --- speed.py 2001/05/01 19:50:42 1.2 +++ speed.py 2001/05/01 22:41:26 1.3 @@ -106,6 +106,10 @@ -M Output means only -C Run with a persistent client cache + + -U Run ZEO using a Unix domain socket + + -t n Number of concurrent threads to run. """ import asyncore @@ -115,6 +119,8 @@ import ZODB, ZODB.FileStorage import Persistence import ZEO.ClientStorage, ZEO.StorageServer +from ZEO.tests import forker +from ZODB.POSException import ConflictError class P(Persistence.Persistent): pass @@ -141,8 +147,49 @@ os.unlink(fs_name + ".lock") os.unlink(fs_name + ".tmp") +def work(db, results, nrep, compress, data, detailed, minimize, threadno=None): + for j in range(nrep): + for r in 1, 10, 100, 1000: + t = time.time() + + jar = db.open() + while 1: + try: + get_transaction().begin() + rt = jar.root() + key = 's%s' % r + if rt.has_key(key): + p = rt[key] + else: + rt[key] = p =P() + for i in range(r): + v = getattr(p, str(i), P()) + if compress is not None: + v.d = compress(data) + else: + v.d = data + setattr(p, str(i), v) + get_transaction().commit() + except ConflictError: + pass + else: + break + jar.close() + + t = time.time() - t + if detailed: + if threadno is None: + print "%s\t%s\t%.4f" % (j, r, t) + else: + print "%s\t%s\t%.4f\t%d" % (j, r, t, threadno) + results[r] = results[r] + t + rt=d=p=v=None # release all references + if minimize: + time.sleep(3) + jar.cacheMinimize(3) + def main(args): - opts, args = getopt.getopt(args, 'zd:n:Ds:LM') + opts, args = getopt.getopt(args, 'zd:n:Ds:LMt:U') s = None compress = None data=sys.argv[0] @@ -150,6 +197,8 @@ minimize=0 detailed=1 cache = None + domain = 'AF_INET' + threads = 1 for o, v in opts: if o=='-n': nrep = int(v) elif o=='-d': data = v @@ -168,79 +217,50 @@ debug = 1 elif o == '-C': cache = 'speed' + elif o == '-U': + domain = 'AF_UNIX' + elif o == '-t': + threads = int(v) zeo_pipe = None if s: s = __import__(s, globals(), globals(), ('__doc__',)) s = s.Storage + server = None else: - rd, wr = os.pipe() - pid = os.fork() - if pid: - # in the child, run the storage server - os.close(wr) - import asyncore - ZEOExit(rd) - fs = ZODB.FileStorage.FileStorage(fs_name, create=1) - serv = ZEO.StorageServer.StorageServer(('', 1975), {'1':fs}) - asyncore.loop() - else: - os.close(rd) - zeo_pipe = wr - s = ZEO.ClientStorage.ClientStorage(('', 1975), debug=0, - client=cache) - if hasattr(s, 'is_connected'): - while not s.is_connected(): - time.sleep(0.1) - else: - time.sleep(1.0) + fs = ZODB.FileStorage.FileStorage(fs_name, create=1) + s, server, pid = forker.start_zeo(fs, domain=domain) data=open(data).read() db=ZODB.DB(s, # disable cache deactivation cache_size=4000, cache_deactivate_after=6000,) - db.open().root() + print "Beginning work..." results={1:0, 10:0, 100:0, 1000:0} - for j in range(nrep): - for r in 1, 10, 100, 1000: - t = time.time() - - jar = db.open() - get_transaction().begin() - rt = jar.root() - key = 's%s' % r - if rt.has_key(key): - p = rt[key] - else: - rt[key] = p =P() - for i in range(r): - v = getattr(p, str(i), P()) - if compress is not None: - v.d = compress(data) - else: - v.d = data - setattr(p, str(i), v) - get_transaction().commit() - jar.close() - - t = time.time() - t - if detailed: - print "%s\t%s\t%.4f" % (j, r, t) - results[r] = results[r] + t - rt=d=p=v=None # release all references - if minimize: - time.sleep(3) - jar.cacheMinimize(3) + if threads > 1: + import threading + l = [threading.Thread(target=work, + args=(db, results, nrep, compress, data, + detailed, minimize, i)) + for i in range(threads)] + for t in l: + t.start() + for t in l: + t.join() + + else: + work(db, results, nrep, compress, data, detailed, minimize) - if zeo_pipe: - os.write(zeo_pipe, "done") + if server is not None: + server.close() + os.waitpid(pid, 0) if detailed: print '-'*24 for r in 1, 10, 100, 1000: - t=results[r]/nrep + t=results[r]/(nrep * threads) print "mean:\t%s\t%.4f\t%.4f (s/o)" % (r, t, t/r) ##def compress(s): From jeremy at digicool.com Tue May 1 18:43:13 2001 From: jeremy at digicool.com (jeremy@digicool.com) Date: Sun Aug 10 16:31:15 2008 Subject: [ZEO-Checkins] CVS: Packages/ZEO - multi.py:1.1.2.4 Message-ID: <20010501224313.4513C510E3@korak.digicool.com> Update of /cvs-repository/Packages/ZEO/tests In directory korak:/tmp/cvs-serv21067 Modified Files: Tag: ZEO-ZRPC-Dev multi.py Log Message: remove unused shutdown_server() function --- Updated File multi.py in package Packages/ZEO -- --- multi.py 2001/05/01 18:43:53 1.1.2.3 +++ multi.py 2001/05/01 22:43:12 1.1.2.4 @@ -106,12 +106,6 @@ print "Client completed:", pid -def shutdown_server(addr): - import ZEO.ClientStorage - # XXX this doesn't work! - cli = ZEO.ClientStorage.ClientStorage(addr) - cli._server.rpc.callAsync('shutdown') - def main(): if VERBOSE: print "Main process:", os.getpid() --- Updated File multi.py in package Packages/ZEO -- --- multi.py 2001/05/01 18:43:53 1.1.2.3 +++ multi.py 2001/05/01 22:43:12 1.1.2.4 @@ -106,12 +106,6 @@ print "Client completed:", pid -def shutdown_server(addr): - import ZEO.ClientStorage - # XXX this doesn't work! - cli = ZEO.ClientStorage.ClientStorage(addr) - cli._server.rpc.callAsync('shutdown') - def main(): if VERBOSE: print "Main process:", os.getpid() From jeremy at digicool.com Wed May 2 16:44:23 2001 From: jeremy at digicool.com (jeremy@digicool.com) Date: Sun Aug 10 16:31:15 2008 Subject: [ZEO-Checkins] CVS: Packages/ZEO - ClientStorage.py:1.26.4.17 Message-ID: <20010502204423.12194510E3@korak.digicool.com> Update of /cvs-repository/Packages/ZEO In directory korak:/tmp/cvs-serv18647 Modified Files: Tag: ZEO-ZRPC-Dev ClientStorage.py Log Message: Clean up some aliasing in cache invalidation code. Note that last XXX comment in doc string says "it would be nice" which is another way of saying "the code wouldn't be buggy if" :-) --- Updated File ClientStorage.py in package Packages/ZEO -- --- ClientStorage.py 2001/05/01 18:39:27 1.26.4.16 +++ ClientStorage.py 2001/05/02 20:44:20 1.26.4.17 @@ -84,8 +84,11 @@ ############################################################################## """Network ZODB storage client -XXX support multiple outstanding requests up until the vote +XXX support multiple outstanding requests up until the vote? XXX is_connected() vis ClientDisconnected error +XXX it would be better to avoid invalidating for abortVersion, + commitVersion, transactionalUndo until the transaction actually + commits. """ __version__='$Revision$'[11:-2] @@ -313,9 +316,8 @@ self._check_trans(transaction, POSException.StorageTransactionError) oids = self._server.abortVersion(src, self._serial) - invalidate = self._cache.invalidate for oid in oids: - invalidate(oid, src) + self._cache.invalidate(oid, src) return oids def close(self): @@ -328,15 +330,14 @@ self._check_trans(transaction, POSException.StorageTransactionError) oids = self._server.commitVersion(src, dest, self._serial) - invalidate = self._cache.invalidate if dest: # just invalidate our version data for oid in oids: - invalidate(oid, src) + self._cache.invalidate(oid, src) else: # dest is '', so invalidate version and non-version for oid in oids: - invalidate(oid, dest) + self._cache.invalidate(oid, dest) return oids def history(self, oid, version, length=1): @@ -497,9 +498,8 @@ def undo(self, transaction_id): oids = self._server.undo(transaction_id) - cinvalidate = self._cache.invalidate for oid in oids: - cinvalidate(oid, '') + self._cache.invalidate(oid, '') return oids def undoInfo(self, first=0, last=-20, specification=None): --- Updated File ClientStorage.py in package Packages/ZEO -- --- ClientStorage.py 2001/05/01 18:39:27 1.26.4.16 +++ ClientStorage.py 2001/05/02 20:44:20 1.26.4.17 @@ -84,8 +84,11 @@ ############################################################################## """Network ZODB storage client -XXX support multiple outstanding requests up until the vote +XXX support multiple outstanding requests up until the vote? XXX is_connected() vis ClientDisconnected error +XXX it would be better to avoid invalidating for abortVersion, + commitVersion, transactionalUndo until the transaction actually + commits. """ __version__='$Revision$'[11:-2] @@ -313,9 +316,8 @@ self._check_trans(transaction, POSException.StorageTransactionError) oids = self._server.abortVersion(src, self._serial) - invalidate = self._cache.invalidate for oid in oids: - invalidate(oid, src) + self._cache.invalidate(oid, src) return oids def close(self): @@ -328,15 +330,14 @@ self._check_trans(transaction, POSException.StorageTransactionError) oids = self._server.commitVersion(src, dest, self._serial) - invalidate = self._cache.invalidate if dest: # just invalidate our version data for oid in oids: - invalidate(oid, src) + self._cache.invalidate(oid, src) else: # dest is '', so invalidate version and non-version for oid in oids: - invalidate(oid, dest) + self._cache.invalidate(oid, dest) return oids def history(self, oid, version, length=1): @@ -497,9 +498,8 @@ def undo(self, transaction_id): oids = self._server.undo(transaction_id) - cinvalidate = self._cache.invalidate for oid in oids: - cinvalidate(oid, '') + self._cache.invalidate(oid, '') return oids def undoInfo(self, first=0, last=-20, specification=None): From jeremy at digicool.com Wed May 2 16:55:22 2001 From: jeremy at digicool.com (jeremy@digicool.com) Date: Sun Aug 10 16:31:15 2008 Subject: [ZEO-Checkins] CVS: Packages/ZEO - testZEO.py:1.1.2.9 Message-ID: <20010502205522.571C0510E3@korak.digicool.com> Update of /cvs-repository/Packages/ZEO/tests In directory korak:/tmp/cvs-serv19570 Modified Files: Tag: ZEO-ZRPC-Dev testZEO.py Log Message: Update to use new zodb_pickle() func. Add ConflictResolution test cases for testZEO w/FileStorage. --- Updated File testZEO.py in package Packages/ZEO -- --- testZEO.py 2001/05/01 18:58:20 1.1.2.8 +++ testZEO.py 2001/05/02 20:55:20 1.1.2.9 @@ -16,10 +16,11 @@ # Sorry Jim... from ZODB.tests import StorageTestBase, BasicStorage, VersionStorage, \ TransactionalUndoStorage, TransactionalUndoVersionStorage, \ - PackableStorage, Synchronization + PackableStorage, Synchronization, ConflictResolution +from ZODB.tests.MinPO import MinPO + ZERO = '\0'*8 -import pickle class ZEOTestBase(StorageTestBase.StorageTestBase): """Version of the storage test class that supports ZEO. @@ -46,9 +47,9 @@ if revid is None: revid = ZERO if data is None: - data = 7 + data = MinPO(7) if not already_pickled: - data = pickle.dumps(data) + data = StorageTestBase.zodb_pickle(data) if version is None: version = '' # Begin the transaction @@ -82,6 +83,7 @@ VersionStorage.VersionStorage, PackableStorage.PackableStorage, Synchronization.SynchronizedStorage, + ConflictResolution.ConflictResolvingStorage, TransactionalUndoStorage.TransactionalUndoStorage, TransactionalUndoVersionStorage.TransactionalUndoVersionStorage, ): --- Updated File testZEO.py in package Packages/ZEO -- --- testZEO.py 2001/05/01 18:58:20 1.1.2.8 +++ testZEO.py 2001/05/02 20:55:20 1.1.2.9 @@ -16,10 +16,11 @@ # Sorry Jim... from ZODB.tests import StorageTestBase, BasicStorage, VersionStorage, \ TransactionalUndoStorage, TransactionalUndoVersionStorage, \ - PackableStorage, Synchronization + PackableStorage, Synchronization, ConflictResolution +from ZODB.tests.MinPO import MinPO + ZERO = '\0'*8 -import pickle class ZEOTestBase(StorageTestBase.StorageTestBase): """Version of the storage test class that supports ZEO. @@ -46,9 +47,9 @@ if revid is None: revid = ZERO if data is None: - data = 7 + data = MinPO(7) if not already_pickled: - data = pickle.dumps(data) + data = StorageTestBase.zodb_pickle(data) if version is None: version = '' # Begin the transaction @@ -82,6 +83,7 @@ VersionStorage.VersionStorage, PackableStorage.PackableStorage, Synchronization.SynchronizedStorage, + ConflictResolution.ConflictResolvingStorage, TransactionalUndoStorage.TransactionalUndoStorage, TransactionalUndoVersionStorage.TransactionalUndoVersionStorage, ): From jeremy at digicool.com Wed May 2 17:54:13 2001 From: jeremy at digicool.com (jeremy@digicool.com) Date: Sun Aug 10 16:31:15 2008 Subject: [ZEO-Checkins] CVS: Packages/ZEO - Cache.py:1.1.2.1 Message-ID: <20010502215413.8E392510E3@korak.digicool.com> Update of /cvs-repository/Packages/ZEO/tests In directory korak:/tmp/cvs-serv24104 Added Files: Tag: ZEO-ZRPC-Dev Cache.py Log Message: Single test that shows error with cache invalidation and transactional undo --- Added File Cache.py in package Packages/ZEO --- """Tests of the ZEO cache""" from ZODB.tests.MinPO import MinPO from ZODB.tests.StorageTestBase import zodb_unpickle class StorageWithCache: def checkUndoInvalidation(self): oid = self._storage.new_oid() revid = self._dostore(oid, data=MinPO(23)) revid = self._dostore(oid, revid=revid, data=MinPO(24)) revid = self._dostore(oid, revid=revid, data=MinPO(25)) info = self._storage.undoInfo() tid = info[0]['id'] # Now start an undo transaction self._transaction.note('undo1') self._storage.tpc_begin(self._transaction) oids = self._storage.transactionalUndo(tid, self._transaction) # Make sure this doesn't load invalid data into the cache self._storage.load(oid, '') self._storage.tpc_vote(self._transaction) self._storage.tpc_finish(self._transaction) assert len(oids) == 1 assert oids[0] == oid data, revid = self._storage.load(oid, '') obj = zodb_unpickle(data) assert obj == MinPO(24) def checkAbortVersionInvalidation(self): pass def checkCommitVersionInvalidation(self): pass --- Added File Cache.py in package Packages/ZEO --- """Tests of the ZEO cache""" from ZODB.tests.MinPO import MinPO from ZODB.tests.StorageTestBase import zodb_unpickle class StorageWithCache: def checkUndoInvalidation(self): oid = self._storage.new_oid() revid = self._dostore(oid, data=MinPO(23)) revid = self._dostore(oid, revid=revid, data=MinPO(24)) revid = self._dostore(oid, revid=revid, data=MinPO(25)) info = self._storage.undoInfo() tid = info[0]['id'] # Now start an undo transaction self._transaction.note('undo1') self._storage.tpc_begin(self._transaction) oids = self._storage.transactionalUndo(tid, self._transaction) # Make sure this doesn't load invalid data into the cache self._storage.load(oid, '') self._storage.tpc_vote(self._transaction) self._storage.tpc_finish(self._transaction) assert len(oids) == 1 assert oids[0] == oid data, revid = self._storage.load(oid, '') obj = zodb_unpickle(data) assert obj == MinPO(24) def checkAbortVersionInvalidation(self): pass def checkCommitVersionInvalidation(self): pass From jeremy at digicool.com Wed May 2 17:54:43 2001 From: jeremy at digicool.com (jeremy@digicool.com) Date: Sun Aug 10 16:31:15 2008 Subject: [ZEO-Checkins] CVS: Packages/ZEO - testZEO.py:1.1.2.10 Message-ID: <20010502215443.A162C510E3@korak.digicool.com> Update of /cvs-repository/Packages/ZEO/tests In directory korak:/tmp/cvs-serv24151 Modified Files: Tag: ZEO-ZRPC-Dev testZEO.py Log Message: Add Cache tests to ZEO test suite This causes one of the ZEO tests to fail. --- Updated File testZEO.py in package Packages/ZEO -- --- testZEO.py 2001/05/02 20:55:20 1.1.2.9 +++ testZEO.py 2001/05/02 21:54:41 1.1.2.10 @@ -11,7 +11,7 @@ import ThreadedAsync, ZEO.trigger from ZEO import zeolog -from ZEO.tests import forker +from ZEO.tests import forker, Cache # Sorry Jim... from ZODB.tests import StorageTestBase, BasicStorage, VersionStorage, \ @@ -79,6 +79,7 @@ return d class GenericTests(ZEOTestBase, + Cache.StorageWithCache, BasicStorage.BasicStorage, VersionStorage.VersionStorage, PackableStorage.PackableStorage, --- Updated File testZEO.py in package Packages/ZEO -- --- testZEO.py 2001/05/02 20:55:20 1.1.2.9 +++ testZEO.py 2001/05/02 21:54:41 1.1.2.10 @@ -11,7 +11,7 @@ import ThreadedAsync, ZEO.trigger from ZEO import zeolog -from ZEO.tests import forker +from ZEO.tests import forker, Cache # Sorry Jim... from ZODB.tests import StorageTestBase, BasicStorage, VersionStorage, \ @@ -79,6 +79,7 @@ return d class GenericTests(ZEOTestBase, + Cache.StorageWithCache, BasicStorage.BasicStorage, VersionStorage.VersionStorage, PackableStorage.PackableStorage, From jeremy at digicool.com Wed May 2 23:10:41 2001 From: jeremy at digicool.com (jeremy@digicool.com) Date: Sun Aug 10 16:31:15 2008 Subject: [ZEO-Checkins] CVS: Packages/ZEO - ClientStorage.py:1.26.4.18 TransactionBuffer.py:1.1.2.2 Message-ID: <20010503031041.75779510E3@korak.digicool.com> Update of /cvs-repository/Packages/ZEO In directory korak:/tmp/cvs-serv15940 Modified Files: Tag: ZEO-ZRPC-Dev ClientStorage.py TransactionBuffer.py Log Message: Quick fix for cache synchronization bug. Add the invalidation messages to the transaction buffer. They never have a pickle, so the invalidation messages are buffered with the pickle set to None. (pickles must be strings, too.) Do the invalidation after the server returns from tpc_finish(). There may still be some synchronization issues here, but I think they are less significant. If a load is performed after the server does tpc_finish() but before the invalidation, it will contain valid data. The invalidation will delete it unnecessarily, but will not introduce an inconsistency. XXX Still not sure if undo() should behave differently -- probably not since the undo() isn't part of a transaction. --- Updated File ClientStorage.py in package Packages/ZEO -- --- ClientStorage.py 2001/05/02 20:44:20 1.26.4.17 +++ ClientStorage.py 2001/05/03 03:10:37 1.26.4.18 @@ -317,7 +317,7 @@ POSException.StorageTransactionError) oids = self._server.abortVersion(src, self._serial) for oid in oids: - self._cache.invalidate(oid, src) + self._tbuf.invalidate(oid, src) return oids def close(self): @@ -333,11 +333,11 @@ if dest: # just invalidate our version data for oid in oids: - self._cache.invalidate(oid, src) + self._tbuf.invalidate(oid, src) else: # dest is '', so invalidate version and non-version for oid in oids: - self._cache.invalidate(oid, dest) + self._tbuf.invalidate(oid, dest) return oids def history(self, oid, version, length=1): @@ -474,12 +474,11 @@ if t is None: break oid, v, p = t - s = self._seriald[oid] - if type(s) != StringType: - log2(INFO, "bad serialno: %s for %s" % \ - (repr(s), repr(oid))) - assert type(s) == StringType, "bad serialno: %s" % repr(s) - if s == ResolvedSerial: + if p is None: # an invalidation + s = None + else: + s = self._seriald[oid] + if s == ResolvedSerial or s is None: self._cache.invalidate(oid, v) else: self._cache.update(oid, s, v, p) @@ -493,10 +492,11 @@ self._check_trans(trans, POSException.StorageTransactionError) oids = self._server.transactionalUndo(trans_id, self._serial) for oid in oids: - self._cache.invalidate(oid, '') + self._tbuf.invalidate(oid, '') return oids def undo(self, transaction_id): + # XXX what are the sync issues here? oids = self._server.undo(transaction_id) for oid in oids: self._cache.invalidate(oid, '') --- Updated File TransactionBuffer.py in package Packages/ZEO -- --- TransactionBuffer.py 2001/04/02 23:16:12 1.1.2.1 +++ TransactionBuffer.py 2001/05/03 03:10:37 1.1.2.2 @@ -28,6 +28,10 @@ if version: self.size = self.size + len(version) + 4 + def invalidate(self, oid, version): + marshal.dump((oid, version, None), self.file) + self.count = self.count + 1 + def clear(self): """Mark the buffer as empty""" self.file.seek(0) --- Updated File ClientStorage.py in package Packages/ZEO -- --- ClientStorage.py 2001/05/02 20:44:20 1.26.4.17 +++ ClientStorage.py 2001/05/03 03:10:37 1.26.4.18 @@ -317,7 +317,7 @@ POSException.StorageTransactionError) oids = self._server.abortVersion(src, self._serial) for oid in oids: - self._cache.invalidate(oid, src) + self._tbuf.invalidate(oid, src) return oids def close(self): @@ -333,11 +333,11 @@ if dest: # just invalidate our version data for oid in oids: - self._cache.invalidate(oid, src) + self._tbuf.invalidate(oid, src) else: # dest is '', so invalidate version and non-version for oid in oids: - self._cache.invalidate(oid, dest) + self._tbuf.invalidate(oid, dest) return oids def history(self, oid, version, length=1): @@ -474,12 +474,11 @@ if t is None: break oid, v, p = t - s = self._seriald[oid] - if type(s) != StringType: - log2(INFO, "bad serialno: %s for %s" % \ - (repr(s), repr(oid))) - assert type(s) == StringType, "bad serialno: %s" % repr(s) - if s == ResolvedSerial: + if p is None: # an invalidation + s = None + else: + s = self._seriald[oid] + if s == ResolvedSerial or s is None: self._cache.invalidate(oid, v) else: self._cache.update(oid, s, v, p) @@ -493,10 +492,11 @@ self._check_trans(trans, POSException.StorageTransactionError) oids = self._server.transactionalUndo(trans_id, self._serial) for oid in oids: - self._cache.invalidate(oid, '') + self._tbuf.invalidate(oid, '') return oids def undo(self, transaction_id): + # XXX what are the sync issues here? oids = self._server.undo(transaction_id) for oid in oids: self._cache.invalidate(oid, '') --- Updated File TransactionBuffer.py in package Packages/ZEO -- --- TransactionBuffer.py 2001/04/02 23:16:12 1.1.2.1 +++ TransactionBuffer.py 2001/05/03 03:10:37 1.1.2.2 @@ -28,6 +28,10 @@ if version: self.size = self.size + len(version) + 4 + def invalidate(self, oid, version): + marshal.dump((oid, version, None), self.file) + self.count = self.count + 1 + def clear(self): """Mark the buffer as empty""" self.file.seek(0) From jeremy at digicool.com Mon May 7 09:10:39 2001 From: jeremy at digicool.com (jeremy@digicool.com) Date: Sun Aug 10 16:31:15 2008 Subject: [ZEO-Checkins] CVS: Packages/ZEO - smac.py:1.9.6.5 Message-ID: <20010507131039.7ACBF510E3@korak.digicool.com> Update of /cvs-repository/Packages/ZEO In directory korak:/tmp/cvs-serv20721 Modified Files: Tag: ZEO-ZRPC-Dev smac.py Log Message: Various small changes. Swap order of if/elif, because one is far more common. Delete unused _handle_read() -- was a variant implementation of handle_read. Put debugging output in message_output() under if __debug__. Remove unused log_info and log methods. --- Updated File smac.py in package Packages/ZEO -- --- smac.py 2001/04/20 19:14:08 1.9.6.4 +++ smac.py 2001/05/07 13:10:37 1.9.6.5 @@ -142,10 +142,10 @@ return # keep waiting for more input # load all previous input and d into single string inp - if inp is None: - inp = d - elif type(inp) is StringType: + if isinstance(inp, StringType): inp = inp + d + elif inp is None: + inp = d else: inp.append(d) inp = string.join(inp, '') @@ -156,15 +156,7 @@ offset = offset + msg_size if state is None: # waiting for message - try: - msg_size = struct.unpack(">i", msg)[0] - except: - # XXX - print 'unpack(">i", %s)' % repr(msg) - print "state = %s" % state - print "msg_size = %(msg_size)d, "\ - "input_len= %(input_len)d" % locals() - raise + msg_size = struct.unpack(">i", msg)[0] state = 1 else: msg_size = 4 @@ -176,45 +168,6 @@ self.__inp = inp[offset:] self.__input_len = input_len - offset - def _handle_read(self): - d = self.recv(self.READ_SIZE) - if not d: - return - - inp = self.__inp - if inp is None: - inp = d - elif type(inp) is StringType: - inp = [inp, d] - else: - inp.append(d) - - input_len = self.__input_len + len(d) - msg_size = self.__msg_size - - while 1: - if msg_size <= input_len: - # Woo hoo, we have enough data - if type(inp) is not StringType: - inp = ''.join(inp) - d = inp[:msg_size] - inp = inp[msg_size:] - input_len = input_len - msg_size - if self.__state is None: - # waiting for message - msg_size = struct.unpack(">i", d)[0] - self.__state = 1 - else: - msg_size = 4 - self.__state = None - self.message_input(d) - else: - break # not enough data - - self.__msg_size = msg_size - self.__inp = inp - self.__input_len = input_len - def readable(self): return 1 @@ -239,12 +192,13 @@ self.close() def message_output(self, message): - if self._debug: - if len(message) > 40: - m = message[:40]+' ...' - else: - m = message - LOG(self._debug, TRACE, 'message_output %s' % `m`) + if __debug__: + if self._debug: + if len(message) > 40: + m = message[:40]+' ...' + else: + m = message + LOG(self._debug, TRACE, 'message_output %s' % `m`) if self.__closed is not None: raise Disconnected, ( @@ -254,15 +208,6 @@ # do two separate appends to avoid copying the message string self.__output.append(struct.pack(">i", len(message))) self.__output.append(message) - - def log_info(self, message, type='trace'): - if type == 'error': - type = ERROR - else: - type = TRACE - LOG('ZEO', type, message) - - log = log_info def close(self): if self.__closed is None: --- Updated File smac.py in package Packages/ZEO -- --- smac.py 2001/04/20 19:14:08 1.9.6.4 +++ smac.py 2001/05/07 13:10:37 1.9.6.5 @@ -142,10 +142,10 @@ return # keep waiting for more input # load all previous input and d into single string inp - if inp is None: - inp = d - elif type(inp) is StringType: + if isinstance(inp, StringType): inp = inp + d + elif inp is None: + inp = d else: inp.append(d) inp = string.join(inp, '') @@ -156,15 +156,7 @@ offset = offset + msg_size if state is None: # waiting for message - try: - msg_size = struct.unpack(">i", msg)[0] - except: - # XXX - print 'unpack(">i", %s)' % repr(msg) - print "state = %s" % state - print "msg_size = %(msg_size)d, "\ - "input_len= %(input_len)d" % locals() - raise + msg_size = struct.unpack(">i", msg)[0] state = 1 else: msg_size = 4 @@ -176,45 +168,6 @@ self.__inp = inp[offset:] self.__input_len = input_len - offset - def _handle_read(self): - d = self.recv(self.READ_SIZE) - if not d: - return - - inp = self.__inp - if inp is None: - inp = d - elif type(inp) is StringType: - inp = [inp, d] - else: - inp.append(d) - - input_len = self.__input_len + len(d) - msg_size = self.__msg_size - - while 1: - if msg_size <= input_len: - # Woo hoo, we have enough data - if type(inp) is not StringType: - inp = ''.join(inp) - d = inp[:msg_size] - inp = inp[msg_size:] - input_len = input_len - msg_size - if self.__state is None: - # waiting for message - msg_size = struct.unpack(">i", d)[0] - self.__state = 1 - else: - msg_size = 4 - self.__state = None - self.message_input(d) - else: - break # not enough data - - self.__msg_size = msg_size - self.__inp = inp - self.__input_len = input_len - def readable(self): return 1 @@ -239,12 +192,13 @@ self.close() def message_output(self, message): - if self._debug: - if len(message) > 40: - m = message[:40]+' ...' - else: - m = message - LOG(self._debug, TRACE, 'message_output %s' % `m`) + if __debug__: + if self._debug: + if len(message) > 40: + m = message[:40]+' ...' + else: + m = message + LOG(self._debug, TRACE, 'message_output %s' % `m`) if self.__closed is not None: raise Disconnected, ( @@ -254,15 +208,6 @@ # do two separate appends to avoid copying the message string self.__output.append(struct.pack(">i", len(message))) self.__output.append(message) - - def log_info(self, message, type='trace'): - if type == 'error': - type = ERROR - else: - type = TRACE - LOG('ZEO', type, message) - - log = log_info def close(self): if self.__closed is None: From jeremy at digicool.com Mon May 7 09:11:58 2001 From: jeremy at digicool.com (jeremy@digicool.com) Date: Sun Aug 10 16:31:15 2008 Subject: [ZEO-Checkins] CVS: Packages/ZEO - zrpc2.py:1.1.2.13 Message-ID: <20010507131158.C886251014@korak.digicool.com> Update of /cvs-repository/Packages/ZEO In directory korak:/tmp/cvs-serv20945 Modified Files: Tag: ZEO-ZRPC-Dev zrpc2.py Log Message: Connection is no longer its own socket map; manage explicit dict. Add useful defaults for tmin and tmax to Connection. --- Updated File zrpc2.py in package Packages/ZEO -- --- zrpc2.py 2001/05/01 22:35:03 1.1.2.12 +++ zrpc2.py 2001/05/07 13:11:56 1.1.2.13 @@ -144,6 +144,7 @@ # The reply lock is used to block when a synchronous call is # waiting for a response self.__super_init(sock, addr) + self._map = {self._fileno: self} self._prepare_async() self.__call_lock = thread.allocate_lock() self.__reply_lock = thread.allocate_lock() @@ -260,27 +261,6 @@ self.message_output(msg) self._do_io() - # The next five methods implement an asyncore socket map - - def keys(self): - return (self._fileno,) - - def values(self): - return (self,) - - def items(self): - return ((self._fileno, self),) - - def __len__(self): - return 1 - - def __getitem__(self, key): - if key == self._fileno: - return self - raise KeyError, key - - # The previous five methods implement an asyncore socket map - # The next two methods are used by clients to invoke methods on # remote objects @@ -371,10 +351,10 @@ if wait: # do loop only if lock is already acquired while not self.__reply_lock.acquire(0): - asyncore.poll(60.0, self) + asyncore.poll(60.0, self._map) self.__reply_lock.release() else: - asyncore.poll(0.0, self) + asyncore.poll(0.0, self._map) # XXX it seems that we need to release before returning if # called with wait==1. perhaps the caller need not acquire @@ -392,7 +372,7 @@ # XXX requires that obj implement notifyConnected and # notifyDisconnected. make this optional? - def __init__(self, addr, obj=None, debug=1, tmin=None, tmax=None): + def __init__(self, addr, obj=None, debug=1, tmin=1, tmax=180): self.addr = addr self.obj = obj self.tmin = tmin --- Updated File zrpc2.py in package Packages/ZEO -- --- zrpc2.py 2001/05/01 22:35:03 1.1.2.12 +++ zrpc2.py 2001/05/07 13:11:56 1.1.2.13 @@ -144,6 +144,7 @@ # The reply lock is used to block when a synchronous call is # waiting for a response self.__super_init(sock, addr) + self._map = {self._fileno: self} self._prepare_async() self.__call_lock = thread.allocate_lock() self.__reply_lock = thread.allocate_lock() @@ -260,27 +261,6 @@ self.message_output(msg) self._do_io() - # The next five methods implement an asyncore socket map - - def keys(self): - return (self._fileno,) - - def values(self): - return (self,) - - def items(self): - return ((self._fileno, self),) - - def __len__(self): - return 1 - - def __getitem__(self, key): - if key == self._fileno: - return self - raise KeyError, key - - # The previous five methods implement an asyncore socket map - # The next two methods are used by clients to invoke methods on # remote objects @@ -371,10 +351,10 @@ if wait: # do loop only if lock is already acquired while not self.__reply_lock.acquire(0): - asyncore.poll(60.0, self) + asyncore.poll(60.0, self._map) self.__reply_lock.release() else: - asyncore.poll(0.0, self) + asyncore.poll(0.0, self._map) # XXX it seems that we need to release before returning if # called with wait==1. perhaps the caller need not acquire @@ -392,7 +372,7 @@ # XXX requires that obj implement notifyConnected and # notifyDisconnected. make this optional? - def __init__(self, addr, obj=None, debug=1, tmin=None, tmax=None): + def __init__(self, addr, obj=None, debug=1, tmin=1, tmax=180): self.addr = addr self.obj = obj self.tmin = tmin From jeremy at digicool.com Mon May 7 09:12:56 2001 From: jeremy at digicool.com (jeremy@digicool.com) Date: Sun Aug 10 16:31:15 2008 Subject: [ZEO-Checkins] CVS: Packages/ZEO - forker.py:1.1.2.3 multi.py:1.1.2.5 testZEO.py:1.1.2.11 Message-ID: <20010507131256.37ECB510E3@korak.digicool.com> Update of /cvs-repository/Packages/ZEO/tests In directory korak:/tmp/cvs-serv21011 Modified Files: Tag: ZEO-ZRPC-Dev forker.py multi.py testZEO.py Log Message: Add profile support. Add ZEO unit test for large commits. --- Updated File forker.py in package Packages/ZEO -- --- forker.py 2001/05/01 18:43:53 1.1.2.2 +++ forker.py 2001/05/07 13:12:54 1.1.2.3 @@ -1,11 +1,17 @@ """Library for forking storage server and connecting client storage""" import asyncore +import atexit import os +import profile import sys +import time +import types import ThreadedAsync import ZEO.ClientStorage, ZEO.StorageServer +PROFILE = 0 + class ZEOServerExit(asyncore.file_dispatcher): """Used to exit ZEO.StorageServer when run is done""" @@ -36,23 +42,28 @@ rd, wr = os.pipe() pid = os.fork() if pid == 0: - # in the child, run the storage server - try: - os.close(wr) - ZEOServerExit(rd) - serv = ZEO.StorageServer.StorageServer(addr, {'1':storage}) - asyncore.loop() - storage.close() - if domain == "AF_UNIX": - os.unlink(addr) - if cleanup: - cleanup() - finally: - os._exit(0) + if PROFILE: + p = profile.Profile() + p.runctx("run_server(storage, addr, rd, wr)", globals(), + locals()) + p.dump_stats("stats.s.%d" % os.getpid()) + else: + run_server(storage, addr, rd, wr) + os._exit(0) else: os.close(rd) return pid, ZEOClientExit(wr) +def run_server(storage, addr, rd, wr): + # in the child, run the storage server + os.close(wr) + ZEOServerExit(rd) + serv = ZEO.StorageServer.StorageServer(addr, {'1':storage}) + asyncore.loop() + storage.close() + if isinstance(addr, types.StringType): + os.unlink(addr) + def start_zeo(storage, cache=None, cleanup=None, domain="AF_INET"): """Setup ZEO client-server for storage. @@ -72,5 +83,8 @@ pid, exit = start_zeo_server(storage, addr) s = ZEO.ClientStorage.ClientStorage(addr, debug=1, client=cache) + if hasattr(s, 'is_connected'): + while not s.is_connected(): + time.sleep(0.1) return s, exit, pid --- Updated File multi.py in package Packages/ZEO -- --- multi.py 2001/05/01 22:43:12 1.1.2.4 +++ multi.py 2001/05/07 13:12:54 1.1.2.5 @@ -11,6 +11,8 @@ import time import types +PROFILE = 0 + VERBOSE = 1 CLIENTS = 4 RECORDS_PER_CLIENT = 100 @@ -53,14 +55,17 @@ pid, exit = forker.start_zeo_server(storage, addr) return pid, exit -def start_client(addr): +def start_client(addr, client_func=None): pid = os.fork() if pid == 0: import ZEO.ClientStorage if VERBOSE: print "Client process started:", os.getpid() cli = ZEO.ClientStorage.ClientStorage(addr, client=CLIENT_CACHE) - run(cli) + if client_func is None: + run(cli) + else: + client_func(cli) cli.close() os._exit(0) else: @@ -106,14 +111,14 @@ print "Client completed:", pid -def main(): +def main(client_func=None): if VERBOSE: print "Main process:", os.getpid() addr = tempfile.mktemp() t0 = time.time() server_pid, server = start_server(addr) t1 = time.time() - pids = [start_client(addr) for i in range(CLIENTS)] + pids = [start_client(addr, client_func) for i in range(CLIENTS)] for pid in pids: assert type(pid) == types.IntType, "invalid pid type: %s (%s)" % \ (repr(pid), type(pid)) --- Updated File testZEO.py in package Packages/ZEO -- --- testZEO.py 2001/05/02 21:54:41 1.1.2.10 +++ testZEO.py 2001/05/07 13:12:54 1.1.2.11 @@ -77,6 +77,10 @@ for oid, serial in r: d[oid] = serial return d + + def checkLargeUpdate(self): + obj = MinPO("X" * (10 * 128 * 1024)) + self._dostore(data=obj) class GenericTests(ZEOTestBase, Cache.StorageWithCache, --- Updated File forker.py in package Packages/ZEO -- --- forker.py 2001/05/01 18:43:53 1.1.2.2 +++ forker.py 2001/05/07 13:12:54 1.1.2.3 @@ -1,11 +1,17 @@ """Library for forking storage server and connecting client storage""" import asyncore +import atexit import os +import profile import sys +import time +import types import ThreadedAsync import ZEO.ClientStorage, ZEO.StorageServer +PROFILE = 0 + class ZEOServerExit(asyncore.file_dispatcher): """Used to exit ZEO.StorageServer when run is done""" @@ -36,23 +42,28 @@ rd, wr = os.pipe() pid = os.fork() if pid == 0: - # in the child, run the storage server - try: - os.close(wr) - ZEOServerExit(rd) - serv = ZEO.StorageServer.StorageServer(addr, {'1':storage}) - asyncore.loop() - storage.close() - if domain == "AF_UNIX": - os.unlink(addr) - if cleanup: - cleanup() - finally: - os._exit(0) + if PROFILE: + p = profile.Profile() + p.runctx("run_server(storage, addr, rd, wr)", globals(), + locals()) + p.dump_stats("stats.s.%d" % os.getpid()) + else: + run_server(storage, addr, rd, wr) + os._exit(0) else: os.close(rd) return pid, ZEOClientExit(wr) +def run_server(storage, addr, rd, wr): + # in the child, run the storage server + os.close(wr) + ZEOServerExit(rd) + serv = ZEO.StorageServer.StorageServer(addr, {'1':storage}) + asyncore.loop() + storage.close() + if isinstance(addr, types.StringType): + os.unlink(addr) + def start_zeo(storage, cache=None, cleanup=None, domain="AF_INET"): """Setup ZEO client-server for storage. @@ -72,5 +83,8 @@ pid, exit = start_zeo_server(storage, addr) s = ZEO.ClientStorage.ClientStorage(addr, debug=1, client=cache) + if hasattr(s, 'is_connected'): + while not s.is_connected(): + time.sleep(0.1) return s, exit, pid --- Updated File multi.py in package Packages/ZEO -- --- multi.py 2001/05/01 22:43:12 1.1.2.4 +++ multi.py 2001/05/07 13:12:54 1.1.2.5 @@ -11,6 +11,8 @@ import time import types +PROFILE = 0 + VERBOSE = 1 CLIENTS = 4 RECORDS_PER_CLIENT = 100 @@ -53,14 +55,17 @@ pid, exit = forker.start_zeo_server(storage, addr) return pid, exit -def start_client(addr): +def start_client(addr, client_func=None): pid = os.fork() if pid == 0: import ZEO.ClientStorage if VERBOSE: print "Client process started:", os.getpid() cli = ZEO.ClientStorage.ClientStorage(addr, client=CLIENT_CACHE) - run(cli) + if client_func is None: + run(cli) + else: + client_func(cli) cli.close() os._exit(0) else: @@ -106,14 +111,14 @@ print "Client completed:", pid -def main(): +def main(client_func=None): if VERBOSE: print "Main process:", os.getpid() addr = tempfile.mktemp() t0 = time.time() server_pid, server = start_server(addr) t1 = time.time() - pids = [start_client(addr) for i in range(CLIENTS)] + pids = [start_client(addr, client_func) for i in range(CLIENTS)] for pid in pids: assert type(pid) == types.IntType, "invalid pid type: %s (%s)" % \ (repr(pid), type(pid)) --- Updated File testZEO.py in package Packages/ZEO -- --- testZEO.py 2001/05/02 21:54:41 1.1.2.10 +++ testZEO.py 2001/05/07 13:12:54 1.1.2.11 @@ -77,6 +77,10 @@ for oid, serial in r: d[oid] = serial return d + + def checkLargeUpdate(self): + obj = MinPO("X" * (10 * 128 * 1024)) + self._dostore(data=obj) class GenericTests(ZEOTestBase, Cache.StorageWithCache, From jeremy at digicool.com Mon May 7 09:15:08 2001 From: jeremy at digicool.com (jeremy@digicool.com) Date: Sun Aug 10 16:31:15 2008 Subject: [ZEO-Checkins] CVS: Packages/ZEO - ClientStorage.py:1.26.4.19 Message-ID: <20010507131508.01524510E3@korak.digicool.com> Update of /cvs-repository/Packages/ZEO In directory korak:/tmp/cvs-serv21141 Modified Files: Tag: ZEO-ZRPC-Dev ClientStorage.py Log Message: Remove cache_lock. (The cache locks itself.) Add oid_cond, so that only a single thread fetches new oids from the server. Remove unneeded XXX sync comments. Move cache invalidate/update logic on commit to separate method. --- Updated File ClientStorage.py in package Packages/ZEO -- --- ClientStorage.py 2001/05/03 03:10:37 1.26.4.18 +++ ClientStorage.py 2001/05/07 13:15:06 1.26.4.19 @@ -194,14 +194,10 @@ self.tpc_cond = threading.Condition() self._transaction = None - # Cache synchronization - # We need to make sure the cache isn't accessed in an - # inconsistent state. If one client is doing a load while - # another is committing a transaction, the cache could contain - # partial results for the committing transaction. Thus, there - # needs to be locking so that only one thread is - # reading/writing the cache at a time. - self.cache_lock = threading.Lock() + # Prevent multiple new_oid calls from going out. The _oids + # variable should only be modified while holding the + # oid_cond. + self.oid_cond = threading.Condition() commit_lock = thread.allocate_lock() self._commit_lock_acquire = commit_lock.acquire @@ -341,15 +337,12 @@ return oids def history(self, oid, version, length=1): - # XXX sync return self._server.history(oid, version, length) def loadSerial(self, oid, serial): - # XXX sync return self._server.loadSerial(oid, serial) def load(self, oid, version, _stuff=None): - # XXX sync p = self._cache.load(oid, version) if p: return p @@ -364,20 +357,22 @@ raise KeyError, oid # no non-version data for this def modifiedInVersion(self, oid): - # XXX sync v = self._cache.modifiedInVersion(oid) if v is not None: return v return self._server.modifiedInVersion(oid) def new_oid(self, last=None): -## if self._transaction is None: -## # XXX What exception? -## raise POSException.StorageTransactionError() + # We want to avoid a situation where multiple oid requests are + # made at the same time. + self.oid_cond.acquire() if not self._oids: self._oids = self._server.new_oids() self._oids.reverse() - return self._oids.pop() + self.oid_cond.notifyAll() + oid = self._oids.pop() + self.oid_cond.release() + return oid def pack(self, t=None, rf=None, wait=0, days=0): # Note that we ignore the rf argument. The server @@ -459,10 +454,16 @@ r = self._check_serials() assert r is None or len(r) == 0, "unhandled serialnos: %s" % r - self._cache.checkSize(self._tbuf.get_size()) + self._update_cache() + + self._transaction = None + self.tpc_cond.notify() + self.tpc_cond.release() + def _update_cache(self): # Iterate over the objects in the transaction buffer and # update or invalidate the cache. + self._cache.checkSize(self._tbuf.get_size()) self._tbuf.begin_iterate() while 1: try: @@ -483,10 +484,6 @@ else: self._cache.update(oid, s, v, p) self._tbuf.clear() - - self._transaction = None - self.tpc_cond.notify() - self.tpc_cond.release() def transactionalUndo(self, trans_id, trans): self._check_trans(trans, POSException.StorageTransactionError) --- Updated File ClientStorage.py in package Packages/ZEO -- --- ClientStorage.py 2001/05/03 03:10:37 1.26.4.18 +++ ClientStorage.py 2001/05/07 13:15:06 1.26.4.19 @@ -194,14 +194,10 @@ self.tpc_cond = threading.Condition() self._transaction = None - # Cache synchronization - # We need to make sure the cache isn't accessed in an - # inconsistent state. If one client is doing a load while - # another is committing a transaction, the cache could contain - # partial results for the committing transaction. Thus, there - # needs to be locking so that only one thread is - # reading/writing the cache at a time. - self.cache_lock = threading.Lock() + # Prevent multiple new_oid calls from going out. The _oids + # variable should only be modified while holding the + # oid_cond. + self.oid_cond = threading.Condition() commit_lock = thread.allocate_lock() self._commit_lock_acquire = commit_lock.acquire @@ -341,15 +337,12 @@ return oids def history(self, oid, version, length=1): - # XXX sync return self._server.history(oid, version, length) def loadSerial(self, oid, serial): - # XXX sync return self._server.loadSerial(oid, serial) def load(self, oid, version, _stuff=None): - # XXX sync p = self._cache.load(oid, version) if p: return p @@ -364,20 +357,22 @@ raise KeyError, oid # no non-version data for this def modifiedInVersion(self, oid): - # XXX sync v = self._cache.modifiedInVersion(oid) if v is not None: return v return self._server.modifiedInVersion(oid) def new_oid(self, last=None): -## if self._transaction is None: -## # XXX What exception? -## raise POSException.StorageTransactionError() + # We want to avoid a situation where multiple oid requests are + # made at the same time. + self.oid_cond.acquire() if not self._oids: self._oids = self._server.new_oids() self._oids.reverse() - return self._oids.pop() + self.oid_cond.notifyAll() + oid = self._oids.pop() + self.oid_cond.release() + return oid def pack(self, t=None, rf=None, wait=0, days=0): # Note that we ignore the rf argument. The server @@ -459,10 +454,16 @@ r = self._check_serials() assert r is None or len(r) == 0, "unhandled serialnos: %s" % r - self._cache.checkSize(self._tbuf.get_size()) + self._update_cache() + + self._transaction = None + self.tpc_cond.notify() + self.tpc_cond.release() + def _update_cache(self): # Iterate over the objects in the transaction buffer and # update or invalidate the cache. + self._cache.checkSize(self._tbuf.get_size()) self._tbuf.begin_iterate() while 1: try: @@ -483,10 +484,6 @@ else: self._cache.update(oid, s, v, p) self._tbuf.clear() - - self._transaction = None - self.tpc_cond.notify() - self.tpc_cond.release() def transactionalUndo(self, trans_id, trans): self._check_trans(trans, POSException.StorageTransactionError) From jeremy at digicool.com Tue May 8 16:36:12 2001 From: jeremy at digicool.com (jeremy@digicool.com) Date: Sun Aug 10 16:31:15 2008 Subject: [ZEO-Checkins] CVS: Packages/ZEO - ServerStub.py:1.1.2.7 StorageServer.py:1.21.4.12 Message-ID: <20010508203612.85717510E3@korak.digicool.com> Update of /cvs-repository/Packages/ZEO In directory korak:/tmp/cvs-serv12634 Modified Files: Tag: ZEO-ZRPC-Dev ServerStub.py StorageServer.py Log Message: Change register() to a synchronous call and raise ValueError when it is passed an invalid argument. This will cause the ClientStorage constructor to fail when it is passed an invalid argument. --- Updated File ServerStub.py in package Packages/ZEO -- --- ServerStub.py 2001/04/27 20:57:19 1.1.2.6 +++ ServerStub.py 2001/05/08 20:36:11 1.1.2.7 @@ -6,7 +6,7 @@ self.rpc = rpc def register(self, storage_name): - self.rpc.callAsync('register', storage_name) + self.rpc.call('register', storage_name) def get_info(self): return self.rpc.call('get_info') --- Updated File StorageServer.py in package Packages/ZEO -- --- StorageServer.py 2001/04/27 20:57:19 1.21.4.11 +++ StorageServer.py 2001/05/08 20:36:11 1.21.4.12 @@ -221,7 +221,7 @@ storage = self.server.storages.get(storage_id) if storage is None: self._log("unknown storage_id: %s" % storage_id) - self.get_caller.close() + raise ValueError, "unknown storage: %s" % storage_id self.__storage_id = storage_id self.__storage = storage --- Updated File ServerStub.py in package Packages/ZEO -- --- ServerStub.py 2001/04/27 20:57:19 1.1.2.6 +++ ServerStub.py 2001/05/08 20:36:11 1.1.2.7 @@ -6,7 +6,7 @@ self.rpc = rpc def register(self, storage_name): - self.rpc.callAsync('register', storage_name) + self.rpc.call('register', storage_name) def get_info(self): return self.rpc.call('get_info') --- Updated File StorageServer.py in package Packages/ZEO -- --- StorageServer.py 2001/04/27 20:57:19 1.21.4.11 +++ StorageServer.py 2001/05/08 20:36:11 1.21.4.12 @@ -221,7 +221,7 @@ storage = self.server.storages.get(storage_id) if storage is None: self._log("unknown storage_id: %s" % storage_id) - self.get_caller.close() + raise ValueError, "unknown storage: %s" % storage_id self.__storage_id = storage_id self.__storage = storage From jeremy at digicool.com Tue May 8 16:45:36 2001 From: jeremy at digicool.com (jeremy@digicool.com) Date: Sun Aug 10 16:31:15 2008 Subject: [ZEO-Checkins] CVS: Packages/ZEO - Cache.py:1.1.2.2 Message-ID: <20010508204536.66495510E3@korak.digicool.com> Update of /cvs-repository/Packages/ZEO/tests In directory korak:/tmp/cvs-serv13114/tests Modified Files: Tag: ZEO-ZRPC-Dev Cache.py Log Message: Add two new tests to check abortVersion() and commitVersion() cache invalidation. Separate transactionalUndo() test into separate class, since some storages do not support it, but do support abortVersion(). --- Updated File Cache.py in package Packages/ZEO -- --- Cache.py 2001/05/02 21:54:12 1.1.2.1 +++ Cache.py 2001/05/08 20:45:35 1.1.2.2 @@ -1,9 +1,10 @@ """Tests of the ZEO cache""" +from ZODB.Transaction import Transaction from ZODB.tests.MinPO import MinPO from ZODB.tests.StorageTestBase import zodb_unpickle -class StorageWithCache: +class TransUndoStorageWithCache: def checkUndoInvalidation(self): oid = self._storage.new_oid() @@ -31,8 +32,35 @@ obj = zodb_unpickle(data) assert obj == MinPO(24) +class StorageWithCache: + def checkAbortVersionInvalidation(self): - pass + oid = self._storage.new_oid() + revid = self._dostore(oid, data=MinPO(1)) + revid = self._dostore(oid, revid=revid, data=MinPO(2)) + revid = self._dostore(oid, revid=revid, data=MinPO(3), version="foo") + revid = self._dostore(oid, revid=revid, data=MinPO(4), version="foo") + t = Transaction() + self._storage.tpc_begin(t) + self._storage.abortVersion("foo", t) + self._storage.load(oid, "foo") + self._storage.tpc_vote(t) + self._storage.tpc_finish(t) + data, revid = self._storage.load(oid, "foo") + obj = zodb_unpickle(data) + assert obj == MinPO(2), obj def checkCommitVersionInvalidation(self): - pass + oid = self._storage.new_oid() + revid = self._dostore(oid, data=MinPO(1)) + revid = self._dostore(oid, revid=revid, data=MinPO(2)) + revid = self._dostore(oid, revid=revid, data=MinPO(3), version="foo") + t = Transaction() + self._storage.tpc_begin(t) + self._storage.commitVersion("foo", "", t) + self._storage.load(oid, "") + self._storage.tpc_vote(t) + self._storage.tpc_finish(t) + data, revid = self._storage.load(oid, "") + obj = zodb_unpickle(data) + assert obj == MinPO(3), obj --- Updated File Cache.py in package Packages/ZEO -- --- Cache.py 2001/05/02 21:54:12 1.1.2.1 +++ Cache.py 2001/05/08 20:45:35 1.1.2.2 @@ -1,9 +1,10 @@ """Tests of the ZEO cache""" +from ZODB.Transaction import Transaction from ZODB.tests.MinPO import MinPO from ZODB.tests.StorageTestBase import zodb_unpickle -class StorageWithCache: +class TransUndoStorageWithCache: def checkUndoInvalidation(self): oid = self._storage.new_oid() @@ -31,8 +32,35 @@ obj = zodb_unpickle(data) assert obj == MinPO(24) +class StorageWithCache: + def checkAbortVersionInvalidation(self): - pass + oid = self._storage.new_oid() + revid = self._dostore(oid, data=MinPO(1)) + revid = self._dostore(oid, revid=revid, data=MinPO(2)) + revid = self._dostore(oid, revid=revid, data=MinPO(3), version="foo") + revid = self._dostore(oid, revid=revid, data=MinPO(4), version="foo") + t = Transaction() + self._storage.tpc_begin(t) + self._storage.abortVersion("foo", t) + self._storage.load(oid, "foo") + self._storage.tpc_vote(t) + self._storage.tpc_finish(t) + data, revid = self._storage.load(oid, "foo") + obj = zodb_unpickle(data) + assert obj == MinPO(2), obj def checkCommitVersionInvalidation(self): - pass + oid = self._storage.new_oid() + revid = self._dostore(oid, data=MinPO(1)) + revid = self._dostore(oid, revid=revid, data=MinPO(2)) + revid = self._dostore(oid, revid=revid, data=MinPO(3), version="foo") + t = Transaction() + self._storage.tpc_begin(t) + self._storage.commitVersion("foo", "", t) + self._storage.load(oid, "") + self._storage.tpc_vote(t) + self._storage.tpc_finish(t) + data, revid = self._storage.load(oid, "") + obj = zodb_unpickle(data) + assert obj == MinPO(3), obj From jeremy at digicool.com Tue May 8 16:49:02 2001 From: jeremy at digicool.com (jeremy@digicool.com) Date: Sun Aug 10 16:31:15 2008 Subject: [ZEO-Checkins] CVS: Packages/ZEO - forker.py:1.1.2.4 Message-ID: <20010508204902.0E7CB510E3@korak.digicool.com> Update of /cvs-repository/Packages/ZEO/tests In directory korak:/tmp/cvs-serv13399/tests Modified Files: Tag: ZEO-ZRPC-Dev forker.py Log Message: Complicate start_zeo() to support tests where the ClientStorage constructor raises an error. The exception is caught and wrapped inside a RuntimeError that also passes the exit and pid to be used to shutdown the server. --- Updated File forker.py in package Packages/ZEO -- --- forker.py 2001/05/07 13:12:54 1.1.2.3 +++ forker.py 2001/05/08 20:49:01 1.1.2.4 @@ -64,7 +64,8 @@ if isinstance(addr, types.StringType): os.unlink(addr) -def start_zeo(storage, cache=None, cleanup=None, domain="AF_INET"): +def start_zeo(storage, cache=None, cleanup=None, domain="AF_INET", + storage_id="1"): """Setup ZEO client-server for storage. Returns a ClientStorage instance and a ZEOClientExit instance. @@ -82,9 +83,13 @@ raise ValueError, "bad domain: %s" % domain pid, exit = start_zeo_server(storage, addr) - s = ZEO.ClientStorage.ClientStorage(addr, debug=1, client=cache) - if hasattr(s, 'is_connected'): - while not s.is_connected(): - time.sleep(0.1) + try: + s = ZEO.ClientStorage.ClientStorage(addr, storage_id, + debug=1, client=cache) + if hasattr(s, 'is_connected'): + while not s.is_connected(): + time.sleep(0.1) + except Exception, err: + raise RuntimeError, (err, exit, pid) return s, exit, pid --- Updated File forker.py in package Packages/ZEO -- --- forker.py 2001/05/07 13:12:54 1.1.2.3 +++ forker.py 2001/05/08 20:49:01 1.1.2.4 @@ -64,7 +64,8 @@ if isinstance(addr, types.StringType): os.unlink(addr) -def start_zeo(storage, cache=None, cleanup=None, domain="AF_INET"): +def start_zeo(storage, cache=None, cleanup=None, domain="AF_INET", + storage_id="1"): """Setup ZEO client-server for storage. Returns a ClientStorage instance and a ZEOClientExit instance. @@ -82,9 +83,13 @@ raise ValueError, "bad domain: %s" % domain pid, exit = start_zeo_server(storage, addr) - s = ZEO.ClientStorage.ClientStorage(addr, debug=1, client=cache) - if hasattr(s, 'is_connected'): - while not s.is_connected(): - time.sleep(0.1) + try: + s = ZEO.ClientStorage.ClientStorage(addr, storage_id, + debug=1, client=cache) + if hasattr(s, 'is_connected'): + while not s.is_connected(): + time.sleep(0.1) + except Exception, err: + raise RuntimeError, (err, exit, pid) return s, exit, pid From jeremy at digicool.com Tue May 8 16:51:21 2001 From: jeremy at digicool.com (jeremy@digicool.com) Date: Sun Aug 10 16:31:15 2008 Subject: [ZEO-Checkins] CVS: Packages/ZEO - testZEO.py:1.1.2.12 Message-ID: <20010508205121.9CAF1510E3@korak.digicool.com> Update of /cvs-repository/Packages/ZEO/tests In directory korak:/tmp/cvs-serv13667/tests Modified Files: Tag: ZEO-ZRPC-Dev testZEO.py Log Message: Add new, specialized cache and conflict resolution test cases. Add new OtherZEOTests TestCases class that doesn't use the standard setUp() and tearDown(). --- Updated File testZEO.py in package Packages/ZEO -- --- testZEO.py 2001/05/07 13:12:54 1.1.2.11 +++ testZEO.py 2001/05/08 20:51:20 1.1.2.12 @@ -9,6 +9,7 @@ import ZEO.ClientStorage, ZEO.StorageServer import ThreadedAsync, ZEO.trigger +from ZODB.FileStorage import FileStorage from ZEO import zeolog from ZEO.tests import forker, Cache @@ -30,6 +31,31 @@ will get no later than the return value from vote. """ + __super_setUp = StorageTestBase.StorageTestBase.setUp + __super_tearDown = StorageTestBase.StorageTestBase.tearDown + + def setUp(self): + """Start a ZEO server using a Unix domain socket + + The ZEO server uses the storage object returned by the + getStorage() method. + """ + self.running = 1 + client, exit, pid = forker.start_zeo(self.getStorage()) + self._pid = pid + self._server = exit + self._storage = client + while not self._storage.is_connected(): + time.sleep(0.1) + self.__super_setUp() + + def tearDown(self): + """Try to cause the tests to halt""" + self.running = 0 + self._server.close() + os.waitpid(self._pid, 0) + self.__super_tearDown() + def _dostore(self, oid=None, revid=None, data=None, version=None, already_pickled=0): """Do a complete storage transaction. @@ -84,11 +110,13 @@ class GenericTests(ZEOTestBase, Cache.StorageWithCache, + Cache.TransUndoStorageWithCache, BasicStorage.BasicStorage, VersionStorage.VersionStorage, PackableStorage.PackableStorage, Synchronization.SynchronizedStorage, ConflictResolution.ConflictResolvingStorage, + ConflictResolution.ConflictResolvingTransUndoStorage, TransactionalUndoStorage.TransactionalUndoStorage, TransactionalUndoVersionStorage.TransactionalUndoVersionStorage, ): @@ -100,48 +128,43 @@ returns a specific storage, e.g. FileStorage. """ - __super_setUp = StorageTestBase.StorageTestBase.setUp - __super_tearDown = StorageTestBase.StorageTestBase.tearDown - - def setUp(self): - """Start a ZEO server using a Unix domain socket - - The ZEO server uses the storage object returned by the - getStorage() method. - """ - self.running = 1 - client, exit, pid = forker.start_zeo(self.getStorage()) - self._pid = pid - self._server = exit - self._storage = client - while not self._storage.is_connected(): - time.sleep(0.1) - self.__super_setUp() - - def tearDown(self): - """Try to cause the tests to halt""" - self.running = 0 - self._server.close() - os.waitpid(self._pid, 0) - self.__super_tearDown() - class ZEOFileStorageTests(GenericTests): + """Tests of storage behavior using FileStorage underneath ZEO""" __super_setUp = GenericTests.setUp - from ZODB.FileStorage import FileStorage - def setUp(self): self.__fs_base = tempfile.mktemp() self.__super_setUp() def getStorage(self): - return self.FileStorage(self.__fs_base, create=1) + return FileStorage(self.__fs_base, create=1) def delStorage(self): # file storage appears to create three files for ext in '', '.index', '.lock', '.tmp': path = self.__fs_base + ext os.unlink(path) + +class OtherZEOTests(unittest.TestCase): + """Tests of ZEO that are basically independent of the storage""" + + def checkBadStorageID(self): + tmp = tempfile.mktemp() + fs = FileStorage(tmp, create=1) + try: + client, exit, pid = forker.start_zeo(fs, storage_id="gotcha") + except RuntimeError, msg: + err, exit, pid = msg + assert isinstance(err, ValueError) + else: + assert 0, "Established ZEO connection with invalid storage id" + exit.close() + os.waitpid(pid, 0) + fs.close() + for ext in '', '.index', '.lock', '.tmp': + path = tmp + ext + if os.path.exists(path): + os.unlink(path) def main(): import sys, getopt @@ -158,6 +181,7 @@ return 0 tests = unittest.makeSuite(ZEOFileStorageTests, 'check' + name_of_test) + tests.addTest(OtherZEOTests("checkBadStorageID")) runner = unittest.TextTestRunner() runner.run(tests) --- Updated File testZEO.py in package Packages/ZEO -- --- testZEO.py 2001/05/07 13:12:54 1.1.2.11 +++ testZEO.py 2001/05/08 20:51:20 1.1.2.12 @@ -9,6 +9,7 @@ import ZEO.ClientStorage, ZEO.StorageServer import ThreadedAsync, ZEO.trigger +from ZODB.FileStorage import FileStorage from ZEO import zeolog from ZEO.tests import forker, Cache @@ -30,6 +31,31 @@ will get no later than the return value from vote. """ + __super_setUp = StorageTestBase.StorageTestBase.setUp + __super_tearDown = StorageTestBase.StorageTestBase.tearDown + + def setUp(self): + """Start a ZEO server using a Unix domain socket + + The ZEO server uses the storage object returned by the + getStorage() method. + """ + self.running = 1 + client, exit, pid = forker.start_zeo(self.getStorage()) + self._pid = pid + self._server = exit + self._storage = client + while not self._storage.is_connected(): + time.sleep(0.1) + self.__super_setUp() + + def tearDown(self): + """Try to cause the tests to halt""" + self.running = 0 + self._server.close() + os.waitpid(self._pid, 0) + self.__super_tearDown() + def _dostore(self, oid=None, revid=None, data=None, version=None, already_pickled=0): """Do a complete storage transaction. @@ -84,11 +110,13 @@ class GenericTests(ZEOTestBase, Cache.StorageWithCache, + Cache.TransUndoStorageWithCache, BasicStorage.BasicStorage, VersionStorage.VersionStorage, PackableStorage.PackableStorage, Synchronization.SynchronizedStorage, ConflictResolution.ConflictResolvingStorage, + ConflictResolution.ConflictResolvingTransUndoStorage, TransactionalUndoStorage.TransactionalUndoStorage, TransactionalUndoVersionStorage.TransactionalUndoVersionStorage, ): @@ -100,48 +128,43 @@ returns a specific storage, e.g. FileStorage. """ - __super_setUp = StorageTestBase.StorageTestBase.setUp - __super_tearDown = StorageTestBase.StorageTestBase.tearDown - - def setUp(self): - """Start a ZEO server using a Unix domain socket - - The ZEO server uses the storage object returned by the - getStorage() method. - """ - self.running = 1 - client, exit, pid = forker.start_zeo(self.getStorage()) - self._pid = pid - self._server = exit - self._storage = client - while not self._storage.is_connected(): - time.sleep(0.1) - self.__super_setUp() - - def tearDown(self): - """Try to cause the tests to halt""" - self.running = 0 - self._server.close() - os.waitpid(self._pid, 0) - self.__super_tearDown() - class ZEOFileStorageTests(GenericTests): + """Tests of storage behavior using FileStorage underneath ZEO""" __super_setUp = GenericTests.setUp - from ZODB.FileStorage import FileStorage - def setUp(self): self.__fs_base = tempfile.mktemp() self.__super_setUp() def getStorage(self): - return self.FileStorage(self.__fs_base, create=1) + return FileStorage(self.__fs_base, create=1) def delStorage(self): # file storage appears to create three files for ext in '', '.index', '.lock', '.tmp': path = self.__fs_base + ext os.unlink(path) + +class OtherZEOTests(unittest.TestCase): + """Tests of ZEO that are basically independent of the storage""" + + def checkBadStorageID(self): + tmp = tempfile.mktemp() + fs = FileStorage(tmp, create=1) + try: + client, exit, pid = forker.start_zeo(fs, storage_id="gotcha") + except RuntimeError, msg: + err, exit, pid = msg + assert isinstance(err, ValueError) + else: + assert 0, "Established ZEO connection with invalid storage id" + exit.close() + os.waitpid(pid, 0) + fs.close() + for ext in '', '.index', '.lock', '.tmp': + path = tmp + ext + if os.path.exists(path): + os.unlink(path) def main(): import sys, getopt @@ -158,6 +181,7 @@ return 0 tests = unittest.makeSuite(ZEOFileStorageTests, 'check' + name_of_test) + tests.addTest(OtherZEOTests("checkBadStorageID")) runner = unittest.TextTestRunner() runner.run(tests) From jeremy at digicool.com Tue May 8 17:14:03 2001 From: jeremy at digicool.com (jeremy@digicool.com) Date: Sun Aug 10 16:31:15 2008 Subject: [ZEO-Checkins] CVS: Packages/ZEO - StorageServer.py:1.25 Message-ID: <20010508211403.1143C510E3@korak.digicool.com> Update of /cvs-repository/Packages/ZEO In directory korak:/tmp/cvs-serv15015 Modified Files: StorageServer.py Log Message: Add explicit log output when client fails to connect because of bad storage id --- Updated File StorageServer.py in package Packages/ZEO -- --- StorageServer.py 2001/04/27 04:01:01 1.24 +++ StorageServer.py 2001/05/08 21:14:03 1.25 @@ -142,6 +142,7 @@ def register_connection(self, connection, storage_id): storage=self.__storages.get(storage_id, None) if storage is None: + LOG('ZEO Server', ERROR, "Unknown storage_id: %s" % storage_id) connection.close() return None, None --- Updated File StorageServer.py in package Packages/ZEO -- --- StorageServer.py 2001/04/27 04:01:01 1.24 +++ StorageServer.py 2001/05/08 21:14:03 1.25 @@ -142,6 +142,7 @@ def register_connection(self, connection, storage_id): storage=self.__storages.get(storage_id, None) if storage is None: + LOG('ZEO Server', ERROR, "Unknown storage_id: %s" % storage_id) connection.close() return None, None From jeremy at digicool.com Tue May 8 17:15:53 2001 From: jeremy at digicool.com (jeremy@digicool.com) Date: Sun Aug 10 16:31:15 2008 Subject: [ZEO-Checkins] CVS: Packages/ZEO - Cache.py:1.2 Message-ID: <20010508211553.D9E3E510E3@korak.digicool.com> Update of /cvs-repository/Packages/ZEO/tests In directory korak:/tmp/cvs-serv15106/tests Added Files: Cache.py Log Message: Cache tests from ZEO2 branch --- Updated File Cache.py in package Packages/ZEO -- --- Updated File Cache.py in package Packages/ZEO -- From jeremy at digicool.com Tue May 8 17:16:38 2001 From: jeremy at digicool.com (jeremy@digicool.com) Date: Sun Aug 10 16:31:15 2008 Subject: [ZEO-Checkins] CVS: Packages/ZEO - forker.py:1.3 Message-ID: <20010508211638.5B326510E3@korak.digicool.com> Update of /cvs-repository/Packages/ZEO/tests In directory korak:/tmp/cvs-serv15308/tests Modified Files: forker.py Log Message: Add run_server() method and profile support from ZEO2 --- Updated File forker.py in package Packages/ZEO -- --- forker.py 2001/05/01 19:54:33 1.2 +++ forker.py 2001/05/08 21:16:37 1.3 @@ -1,11 +1,17 @@ """Library for forking storage server and connecting client storage""" import asyncore +import atexit import os +import profile import sys +import time +import types import ThreadedAsync import ZEO.ClientStorage, ZEO.StorageServer +PROFILE = 0 + class ZEOServerExit(asyncore.file_dispatcher): """Used to exit ZEO.StorageServer when run is done""" @@ -36,24 +42,30 @@ rd, wr = os.pipe() pid = os.fork() if pid == 0: - # in the child, run the storage server - try: - os.close(wr) - ZEOServerExit(rd) - serv = ZEO.StorageServer.StorageServer(addr, {'1':storage}) - asyncore.loop() - storage.close() - if domain == "AF_UNIX": - os.unlink(addr) - if cleanup: - cleanup() - finally: - os._exit(0) + if PROFILE: + p = profile.Profile() + p.runctx("run_server(storage, addr, rd, wr)", globals(), + locals()) + p.dump_stats("stats.s.%d" % os.getpid()) + else: + run_server(storage, addr, rd, wr) + os._exit(0) else: os.close(rd) return pid, ZEOClientExit(wr) + +def run_server(storage, addr, rd, wr): + # in the child, run the storage server + os.close(wr) + ZEOServerExit(rd) + serv = ZEO.StorageServer.StorageServer(addr, {'1':storage}) + asyncore.loop() + storage.close() + if isinstance(addr, types.StringType): + os.unlink(addr) -def start_zeo(storage, cache=None, cleanup=None, domain="AF_INET"): +def start_zeo(storage, cache=None, cleanup=None, domain="AF_INET", + storage_id="1"): """Setup ZEO client-server for storage. Returns a ClientStorage instance and a ZEOClientExit instance. @@ -71,6 +83,10 @@ raise ValueError, "bad domain: %s" % domain pid, exit = start_zeo_server(storage, addr) - s = ZEO.ClientStorage.ClientStorage(addr, debug=1, client=cache) + s = ZEO.ClientStorage.ClientStorage(addr, storage_id, + debug=1, client=cache) + if hasattr(s, 'is_connected'): + while not s.is_connected(): + time.sleep(0.1) return s, exit, pid --- Updated File forker.py in package Packages/ZEO -- --- forker.py 2001/05/01 19:54:33 1.2 +++ forker.py 2001/05/08 21:16:37 1.3 @@ -1,11 +1,17 @@ """Library for forking storage server and connecting client storage""" import asyncore +import atexit import os +import profile import sys +import time +import types import ThreadedAsync import ZEO.ClientStorage, ZEO.StorageServer +PROFILE = 0 + class ZEOServerExit(asyncore.file_dispatcher): """Used to exit ZEO.StorageServer when run is done""" @@ -36,24 +42,30 @@ rd, wr = os.pipe() pid = os.fork() if pid == 0: - # in the child, run the storage server - try: - os.close(wr) - ZEOServerExit(rd) - serv = ZEO.StorageServer.StorageServer(addr, {'1':storage}) - asyncore.loop() - storage.close() - if domain == "AF_UNIX": - os.unlink(addr) - if cleanup: - cleanup() - finally: - os._exit(0) + if PROFILE: + p = profile.Profile() + p.runctx("run_server(storage, addr, rd, wr)", globals(), + locals()) + p.dump_stats("stats.s.%d" % os.getpid()) + else: + run_server(storage, addr, rd, wr) + os._exit(0) else: os.close(rd) return pid, ZEOClientExit(wr) + +def run_server(storage, addr, rd, wr): + # in the child, run the storage server + os.close(wr) + ZEOServerExit(rd) + serv = ZEO.StorageServer.StorageServer(addr, {'1':storage}) + asyncore.loop() + storage.close() + if isinstance(addr, types.StringType): + os.unlink(addr) -def start_zeo(storage, cache=None, cleanup=None, domain="AF_INET"): +def start_zeo(storage, cache=None, cleanup=None, domain="AF_INET", + storage_id="1"): """Setup ZEO client-server for storage. Returns a ClientStorage instance and a ZEOClientExit instance. @@ -71,6 +83,10 @@ raise ValueError, "bad domain: %s" % domain pid, exit = start_zeo_server(storage, addr) - s = ZEO.ClientStorage.ClientStorage(addr, debug=1, client=cache) + s = ZEO.ClientStorage.ClientStorage(addr, storage_id, + debug=1, client=cache) + if hasattr(s, 'is_connected'): + while not s.is_connected(): + time.sleep(0.1) return s, exit, pid From jeremy at digicool.com Tue May 8 17:25:25 2001 From: jeremy at digicool.com (jeremy@digicool.com) Date: Sun Aug 10 16:31:15 2008 Subject: [ZEO-Checkins] CVS: Packages/ZEO - testZEO.py:1.4 Message-ID: <20010508212525.6C3FD510E3@korak.digicool.com> Update of /cvs-repository/Packages/ZEO/tests In directory korak:/tmp/cvs-serv15788/tests Modified Files: testZEO.py Log Message: Add new tests from ZEO2. XXX The two cache invalidation tests fails currently. --- Updated File testZEO.py in package Packages/ZEO -- --- testZEO.py 2001/05/01 19:54:33 1.3 +++ testZEO.py 2001/05/08 21:25:24 1.4 @@ -9,16 +9,18 @@ import ZEO.ClientStorage, ZEO.StorageServer import ThreadedAsync, ZEO.trigger +from ZODB.FileStorage import FileStorage -from ZEO.tests import forker +from ZEO.tests import forker, Cache # Sorry Jim... from ZODB.tests import StorageTestBase, BasicStorage, VersionStorage, \ TransactionalUndoStorage, TransactionalUndoVersionStorage, \ - PackableStorage, Synchronization + PackableStorage, Synchronization, ConflictResolution +from ZODB.tests.MinPO import MinPO + ZERO = '\0'*8 -import pickle class DummyDB: def invalidate(self, *args): @@ -59,9 +61,9 @@ if revid is None: revid = ZERO if data is None: - data = 7 + data = MinPO(7) if not already_pickled: - data = pickle.dumps(data) + data = StorageTestBase.zodb_pickle(data) if version is None: version = '' # Begin the transaction @@ -91,12 +93,18 @@ raise serial d[oid] = serial return d + + def checkLargeUpdate(self): + obj = MinPO("X" * (10 * 128 * 1024)) + self._dostore(data=obj) class GenericTests(ZEOTestBase, + Cache.StorageWithCache, BasicStorage.BasicStorage, VersionStorage.VersionStorage, PackableStorage.PackableStorage, Synchronization.SynchronizedStorage, + ConflictResolution.ConflictResolvingStorage, ): """An abstract base class for ZEO tests @@ -133,14 +141,12 @@ class ZEOFileStorageTests(GenericTests): __super_setUp = GenericTests.setUp - from ZODB.FileStorage import FileStorage - def setUp(self): self.__fs_base = tempfile.mktemp() self.__super_setUp() def getStorage(self): - return self.FileStorage(self.__fs_base, create=1) + return FileStorage(self.__fs_base, create=1) def delStorage(self): # file storage appears to create three files --- Updated File testZEO.py in package Packages/ZEO -- --- testZEO.py 2001/05/01 19:54:33 1.3 +++ testZEO.py 2001/05/08 21:25:24 1.4 @@ -9,16 +9,18 @@ import ZEO.ClientStorage, ZEO.StorageServer import ThreadedAsync, ZEO.trigger +from ZODB.FileStorage import FileStorage -from ZEO.tests import forker +from ZEO.tests import forker, Cache # Sorry Jim... from ZODB.tests import StorageTestBase, BasicStorage, VersionStorage, \ TransactionalUndoStorage, TransactionalUndoVersionStorage, \ - PackableStorage, Synchronization + PackableStorage, Synchronization, ConflictResolution +from ZODB.tests.MinPO import MinPO + ZERO = '\0'*8 -import pickle class DummyDB: def invalidate(self, *args): @@ -59,9 +61,9 @@ if revid is None: revid = ZERO if data is None: - data = 7 + data = MinPO(7) if not already_pickled: - data = pickle.dumps(data) + data = StorageTestBase.zodb_pickle(data) if version is None: version = '' # Begin the transaction @@ -91,12 +93,18 @@ raise serial d[oid] = serial return d + + def checkLargeUpdate(self): + obj = MinPO("X" * (10 * 128 * 1024)) + self._dostore(data=obj) class GenericTests(ZEOTestBase, + Cache.StorageWithCache, BasicStorage.BasicStorage, VersionStorage.VersionStorage, PackableStorage.PackableStorage, Synchronization.SynchronizedStorage, + ConflictResolution.ConflictResolvingStorage, ): """An abstract base class for ZEO tests @@ -133,14 +141,12 @@ class ZEOFileStorageTests(GenericTests): __super_setUp = GenericTests.setUp - from ZODB.FileStorage import FileStorage - def setUp(self): self.__fs_base = tempfile.mktemp() self.__super_setUp() def getStorage(self): - return self.FileStorage(self.__fs_base, create=1) + return FileStorage(self.__fs_base, create=1) def delStorage(self): # file storage appears to create three files From jeremy at digicool.com Tue May 8 17:27:08 2001 From: jeremy at digicool.com (jeremy@digicool.com) Date: Sun Aug 10 16:31:15 2008 Subject: [ZEO-Checkins] CVS: Packages/ZEO - multi.py:1.3 Message-ID: <20010508212708.E1362510E3@korak.digicool.com> Update of /cvs-repository/Packages/ZEO/tests In directory korak:/tmp/cvs-serv16031/tests Modified Files: multi.py Log Message: Add new features from ZEO2. Support for using basic multi framework with other run() functions. --- Updated File multi.py in package Packages/ZEO -- --- multi.py 2001/05/01 19:50:42 1.2 +++ multi.py 2001/05/08 21:27:08 1.3 @@ -53,14 +53,17 @@ pid, exit = forker.start_zeo_server(storage, addr) return pid, exit -def start_client(addr): +def start_client(addr, client_func=None): pid = os.fork() if pid == 0: import ZEO.ClientStorage if VERBOSE: print "Client process started:", os.getpid() cli = ZEO.ClientStorage.ClientStorage(addr, client=CLIENT_CACHE) - run(cli) + if client_func is None: + run(cli) + else: + client_func(cli) cli.close() os._exit(0) else: @@ -105,21 +108,15 @@ get_transaction().commit() print "Client completed:", pid - -def shutdown_server(addr): - import ZEO.ClientStorage - # XXX this doesn't work! - cli = ZEO.ClientStorage.ClientStorage(addr) - cli._server.rpc.callAsync('shutdown') -def main(): +def main(client_func=None): if VERBOSE: print "Main process:", os.getpid() addr = tempfile.mktemp() t0 = time.time() server_pid, server = start_server(addr) t1 = time.time() - pids = [start_client(addr) for i in range(CLIENTS)] + pids = [start_client(addr, client_func) for i in range(CLIENTS)] for pid in pids: assert type(pid) == types.IntType, "invalid pid type: %s (%s)" % \ (repr(pid), type(pid)) --- Updated File multi.py in package Packages/ZEO -- --- multi.py 2001/05/01 19:50:42 1.2 +++ multi.py 2001/05/08 21:27:08 1.3 @@ -53,14 +53,17 @@ pid, exit = forker.start_zeo_server(storage, addr) return pid, exit -def start_client(addr): +def start_client(addr, client_func=None): pid = os.fork() if pid == 0: import ZEO.ClientStorage if VERBOSE: print "Client process started:", os.getpid() cli = ZEO.ClientStorage.ClientStorage(addr, client=CLIENT_CACHE) - run(cli) + if client_func is None: + run(cli) + else: + client_func(cli) cli.close() os._exit(0) else: @@ -105,21 +108,15 @@ get_transaction().commit() print "Client completed:", pid - -def shutdown_server(addr): - import ZEO.ClientStorage - # XXX this doesn't work! - cli = ZEO.ClientStorage.ClientStorage(addr) - cli._server.rpc.callAsync('shutdown') -def main(): +def main(client_func=None): if VERBOSE: print "Main process:", os.getpid() addr = tempfile.mktemp() t0 = time.time() server_pid, server = start_server(addr) t1 = time.time() - pids = [start_client(addr) for i in range(CLIENTS)] + pids = [start_client(addr, client_func) for i in range(CLIENTS)] for pid in pids: assert type(pid) == types.IntType, "invalid pid type: %s (%s)" % \ (repr(pid), type(pid)) From jeremy at digicool.com Tue May 8 18:33:54 2001 From: jeremy at digicool.com (jeremy@digicool.com) Date: Sun Aug 10 16:31:15 2008 Subject: [ZEO-Checkins] CVS: Packages/ZEO - ClientStorage.py:1.30 Message-ID: <20010508223354.42B62510E3@korak.digicool.com> Update of /cvs-repository/Packages/ZEO In directory korak:/tmp/cvs-serv20512 Modified Files: ClientStorage.py Log Message: Fix cache invalidation abortVersion() and commitVersion(). The invalidated oids (and versions) get written to the tempfile, and are invalidated during the tpc_finish(). --- Updated File ClientStorage.py in package Packages/ZEO -- --- ClientStorage.py 2001/04/29 17:08:23 1.29 +++ ClientStorage.py 2001/05/08 22:33:54 1.30 @@ -289,8 +289,9 @@ self._lock_acquire() try: oids=self._call('abortVersion', src, self._serial) - invalidate=self._cache.invalidate - for oid in oids: invalidate(oid, src) + vlen = pack(">H", len(src)) + for oid in oids: + self._tfile.write("i%s%s%s" % (oid, vlen, src)) return oids finally: self._lock_release() @@ -307,12 +308,15 @@ oids=self._call('commitVersion', src, dest, self._serial) invalidate=self._cache.invalidate if dest: + vlen = pack(">H", len(src)) # just invalidate our version data - for oid in oids: invalidate(oid, src) + for oid in oids: + self._tfile.write("i%s%s%s" % (oid, vlen, src)) else: + vlen = pack(">H", len(dest)) # dest is '', so invalidate version and non-version - for oid in oids: invalidate(oid, dest) - + for oid in oids: + self._tfile.write("i%s%s%s" % (oid, vlen, dest)) return oids finally: self._lock_release() @@ -385,8 +389,10 @@ data, version, self._serial) write=self._tfile.write - write(oid+pack(">HI", len(version), len(data))+version) - write(data) + buf = string.join(("s", oid, + pack(">HI", len(version), len(data)), + version, data), "") + write(buf) if self._serials: s=self._serials @@ -508,23 +514,32 @@ seek(0) i=0 while i < size: - oid=read(8) - s=seriald[oid] - h=read(6) - vlen, dlen = unpack(">HI", h) - if vlen: v=read(vlen) - else: v='' - p=read(dlen) - if len(p) != dlen: - raise ClientStorageError, ( - "Unexpected end of file in client storage " - "temporary file." - ) - if s==ResolvedSerial: - cache.invalidate(oid, v) - else: - update(oid, s, v, p) - i=i+14+vlen+dlen + opcode=read(1) + if opcode == "s": + oid=read(8) + s=seriald[oid] + h=read(6) + vlen, dlen = unpack(">HI", h) + if vlen: v=read(vlen) + else: v='' + p=read(dlen) + if len(p) != dlen: + raise ClientStorageError, ( + "Unexpected end of file in client storage " + "temporary file." + ) + if s==ResolvedSerial: + cache.invalidate(oid, v) + else: + update(oid, s, v, p) + i=i+15+vlen+dlen + elif opcode == "i": + oid=read(8) + h=read(2) + vlen=unpack(">H", h)[0] + v=read(vlen) + self._cache.invalidate(oid, v) + i=i+11+vlen seek(0) --- Updated File ClientStorage.py in package Packages/ZEO -- --- ClientStorage.py 2001/04/29 17:08:23 1.29 +++ ClientStorage.py 2001/05/08 22:33:54 1.30 @@ -289,8 +289,9 @@ self._lock_acquire() try: oids=self._call('abortVersion', src, self._serial) - invalidate=self._cache.invalidate - for oid in oids: invalidate(oid, src) + vlen = pack(">H", len(src)) + for oid in oids: + self._tfile.write("i%s%s%s" % (oid, vlen, src)) return oids finally: self._lock_release() @@ -307,12 +308,15 @@ oids=self._call('commitVersion', src, dest, self._serial) invalidate=self._cache.invalidate if dest: + vlen = pack(">H", len(src)) # just invalidate our version data - for oid in oids: invalidate(oid, src) + for oid in oids: + self._tfile.write("i%s%s%s" % (oid, vlen, src)) else: + vlen = pack(">H", len(dest)) # dest is '', so invalidate version and non-version - for oid in oids: invalidate(oid, dest) - + for oid in oids: + self._tfile.write("i%s%s%s" % (oid, vlen, dest)) return oids finally: self._lock_release() @@ -385,8 +389,10 @@ data, version, self._serial) write=self._tfile.write - write(oid+pack(">HI", len(version), len(data))+version) - write(data) + buf = string.join(("s", oid, + pack(">HI", len(version), len(data)), + version, data), "") + write(buf) if self._serials: s=self._serials @@ -508,23 +514,32 @@ seek(0) i=0 while i < size: - oid=read(8) - s=seriald[oid] - h=read(6) - vlen, dlen = unpack(">HI", h) - if vlen: v=read(vlen) - else: v='' - p=read(dlen) - if len(p) != dlen: - raise ClientStorageError, ( - "Unexpected end of file in client storage " - "temporary file." - ) - if s==ResolvedSerial: - cache.invalidate(oid, v) - else: - update(oid, s, v, p) - i=i+14+vlen+dlen + opcode=read(1) + if opcode == "s": + oid=read(8) + s=seriald[oid] + h=read(6) + vlen, dlen = unpack(">HI", h) + if vlen: v=read(vlen) + else: v='' + p=read(dlen) + if len(p) != dlen: + raise ClientStorageError, ( + "Unexpected end of file in client storage " + "temporary file." + ) + if s==ResolvedSerial: + cache.invalidate(oid, v) + else: + update(oid, s, v, p) + i=i+15+vlen+dlen + elif opcode == "i": + oid=read(8) + h=read(2) + vlen=unpack(">H", h)[0] + v=read(vlen) + self._cache.invalidate(oid, v) + i=i+11+vlen seek(0) From jeremy at digicool.com Tue May 8 18:34:57 2001 From: jeremy at digicool.com (jeremy@digicool.com) Date: Sun Aug 10 16:31:15 2008 Subject: [ZEO-Checkins] CVS: Packages/ZEO - Cache.py:1.3 Message-ID: <20010508223457.E6CFC510E3@korak.digicool.com> Update of /cvs-repository/Packages/ZEO/tests In directory korak:/tmp/cvs-serv20570/tests Modified Files: Cache.py Log Message: Add a second invalidation + commitVersion test, since the code path is different depending on whether the version is different. --- Updated File Cache.py in package Packages/ZEO -- --- Cache.py 2001/05/08 21:15:53 1.2 +++ Cache.py 2001/05/08 22:34:57 1.3 @@ -50,7 +50,7 @@ obj = zodb_unpickle(data) assert obj == MinPO(2), obj - def checkCommitVersionInvalidation(self): + def checkCommitEmptyVersionInvalidation(self): oid = self._storage.new_oid() revid = self._dostore(oid, data=MinPO(1)) revid = self._dostore(oid, revid=revid, data=MinPO(2)) @@ -62,5 +62,20 @@ self._storage.tpc_vote(t) self._storage.tpc_finish(t) data, revid = self._storage.load(oid, "") + obj = zodb_unpickle(data) + assert obj == MinPO(3), obj + + def checkCommitVersionInvalidation(self): + oid = self._storage.new_oid() + revid = self._dostore(oid, data=MinPO(1)) + revid = self._dostore(oid, revid=revid, data=MinPO(2)) + revid = self._dostore(oid, revid=revid, data=MinPO(3), version="foo") + t = Transaction() + self._storage.tpc_begin(t) + self._storage.commitVersion("foo", "bar", t) + self._storage.load(oid, "") + self._storage.tpc_vote(t) + self._storage.tpc_finish(t) + data, revid = self._storage.load(oid, "bar") obj = zodb_unpickle(data) assert obj == MinPO(3), obj --- Updated File Cache.py in package Packages/ZEO -- --- Cache.py 2001/05/08 21:15:53 1.2 +++ Cache.py 2001/05/08 22:34:57 1.3 @@ -50,7 +50,7 @@ obj = zodb_unpickle(data) assert obj == MinPO(2), obj - def checkCommitVersionInvalidation(self): + def checkCommitEmptyVersionInvalidation(self): oid = self._storage.new_oid() revid = self._dostore(oid, data=MinPO(1)) revid = self._dostore(oid, revid=revid, data=MinPO(2)) @@ -62,5 +62,20 @@ self._storage.tpc_vote(t) self._storage.tpc_finish(t) data, revid = self._storage.load(oid, "") + obj = zodb_unpickle(data) + assert obj == MinPO(3), obj + + def checkCommitVersionInvalidation(self): + oid = self._storage.new_oid() + revid = self._dostore(oid, data=MinPO(1)) + revid = self._dostore(oid, revid=revid, data=MinPO(2)) + revid = self._dostore(oid, revid=revid, data=MinPO(3), version="foo") + t = Transaction() + self._storage.tpc_begin(t) + self._storage.commitVersion("foo", "bar", t) + self._storage.load(oid, "") + self._storage.tpc_vote(t) + self._storage.tpc_finish(t) + data, revid = self._storage.load(oid, "bar") obj = zodb_unpickle(data) assert obj == MinPO(3), obj From jeremy at digicool.com Tue May 8 19:26:58 2001 From: jeremy at digicool.com (jeremy@digicool.com) Date: Sun Aug 10 16:31:15 2008 Subject: [ZEO-Checkins] CVS: Packages/ZEO - speed.py:1.4 Message-ID: <20010508232658.56532510E3@korak.digicool.com> Update of /cvs-repository/Packages/ZEO/tests In directory korak:/tmp/cvs-serv23717/tests Modified Files: speed.py Log Message: Print mean, min, and max for summary. Also print # conflicts for each test (appears to be zero). --- Updated File speed.py in package Packages/ZEO -- --- speed.py 2001/05/01 22:41:26 1.3 +++ speed.py 2001/05/08 23:26:57 1.4 @@ -151,6 +151,7 @@ for j in range(nrep): for r in 1, 10, 100, 1000: t = time.time() + conflicts = 0 jar = db.open() while 1: @@ -171,7 +172,7 @@ setattr(p, str(i), v) get_transaction().commit() except ConflictError: - pass + conflicts = conflicts + 1 else: break jar.close() @@ -179,10 +180,11 @@ t = time.time() - t if detailed: if threadno is None: - print "%s\t%s\t%.4f" % (j, r, t) + print "%s\t%s\t%.4f\t%d" % (j, r, t, conflicts) else: - print "%s\t%s\t%.4f\t%d" % (j, r, t, threadno) - results[r] = results[r] + t + print "%s\t%s\t%.4f\t%d\t%d" % (j, r, t, conflicts, + threadno) + results[r].append((t, conflicts)) rt=d=p=v=None # release all references if minimize: time.sleep(3) @@ -238,7 +240,7 @@ cache_deactivate_after=6000,) print "Beginning work..." - results={1:0, 10:0, 100:0, 1000:0} + results={1:[], 10:[], 100:[], 1000:[]} if threads > 1: import threading l = [threading.Thread(target=work, @@ -259,9 +261,17 @@ if detailed: print '-'*24 + print "num\tmean\tmin\tmax" for r in 1, 10, 100, 1000: - t=results[r]/(nrep * threads) - print "mean:\t%s\t%.4f\t%.4f (s/o)" % (r, t, t/r) + times = [time for time, conf in results[r]] + t = mean(times) + print "%d\t%.4f\t%.4f\t%.4f" % (r, t, min(times), max(times)) + +def mean(l): + tot = 0 + for v in l: + tot = tot + v + return tot / len(l) ##def compress(s): ## c = zlib.compressobj() --- Updated File speed.py in package Packages/ZEO -- --- speed.py 2001/05/01 22:41:26 1.3 +++ speed.py 2001/05/08 23:26:57 1.4 @@ -151,6 +151,7 @@ for j in range(nrep): for r in 1, 10, 100, 1000: t = time.time() + conflicts = 0 jar = db.open() while 1: @@ -171,7 +172,7 @@ setattr(p, str(i), v) get_transaction().commit() except ConflictError: - pass + conflicts = conflicts + 1 else: break jar.close() @@ -179,10 +180,11 @@ t = time.time() - t if detailed: if threadno is None: - print "%s\t%s\t%.4f" % (j, r, t) + print "%s\t%s\t%.4f\t%d" % (j, r, t, conflicts) else: - print "%s\t%s\t%.4f\t%d" % (j, r, t, threadno) - results[r] = results[r] + t + print "%s\t%s\t%.4f\t%d\t%d" % (j, r, t, conflicts, + threadno) + results[r].append((t, conflicts)) rt=d=p=v=None # release all references if minimize: time.sleep(3) @@ -238,7 +240,7 @@ cache_deactivate_after=6000,) print "Beginning work..." - results={1:0, 10:0, 100:0, 1000:0} + results={1:[], 10:[], 100:[], 1000:[]} if threads > 1: import threading l = [threading.Thread(target=work, @@ -259,9 +261,17 @@ if detailed: print '-'*24 + print "num\tmean\tmin\tmax" for r in 1, 10, 100, 1000: - t=results[r]/(nrep * threads) - print "mean:\t%s\t%.4f\t%.4f (s/o)" % (r, t, t/r) + times = [time for time, conf in results[r]] + t = mean(times) + print "%d\t%.4f\t%.4f\t%.4f" % (r, t, min(times), max(times)) + +def mean(l): + tot = 0 + for v in l: + tot = tot + v + return tot / len(l) ##def compress(s): ## c = zlib.compressobj() From jeremy at digicool.com Wed May 9 10:35:40 2001 From: jeremy at digicool.com (jeremy@digicool.com) Date: Sun Aug 10 16:31:15 2008 Subject: [ZEO-Checkins] CVS: Packages/ZEO - zrpc.py:1.15 Message-ID: <20010509143540.D6175510E3@korak.digicool.com> Update of /cvs-repository/Packages/ZEO In directory korak:/tmp/cvs-serv12875 Modified Files: zrpc.py Log Message: Provide more detailed logging when connection fails --- Updated File zrpc.py in package Packages/ZEO -- --- zrpc.py 2001/04/01 18:41:13 1.14 +++ zrpc.py 2001/05/09 14:35:40 1.15 @@ -140,9 +140,9 @@ else: s=socket.socket(socket.AF_INET, socket.SOCK_STREAM) s.connect(connection) - except: + except Exception, err: if debug: - LOG(debug, DEBUG, "Failed to connect to server") + LOG(debug, DEBUG, "Failed to connect to server: %s" % err) if tryonce: return 0 time.sleep(t) t=t*2 --- Updated File zrpc.py in package Packages/ZEO -- --- zrpc.py 2001/04/01 18:41:13 1.14 +++ zrpc.py 2001/05/09 14:35:40 1.15 @@ -140,9 +140,9 @@ else: s=socket.socket(socket.AF_INET, socket.SOCK_STREAM) s.connect(connection) - except: + except Exception, err: if debug: - LOG(debug, DEBUG, "Failed to connect to server") + LOG(debug, DEBUG, "Failed to connect to server: %s" % err) if tryonce: return 0 time.sleep(t) t=t*2 From jeremy at digicool.com Wed May 9 11:17:54 2001 From: jeremy at digicool.com (jeremy@digicool.com) Date: Sun Aug 10 16:31:15 2008 Subject: [ZEO-Checkins] CVS: Packages/ZEO - ClientStorage.py:1.26.4.20 Message-ID: <20010509151754.87BE0510E3@korak.digicool.com> Update of /cvs-repository/Packages/ZEO In directory korak:/tmp/cvs-serv15556 Modified Files: Tag: ZEO-ZRPC-Dev ClientStorage.py Log Message: Remove XXX comment about buggy cache invalidation --- Updated File ClientStorage.py in package Packages/ZEO -- --- ClientStorage.py 2001/05/07 13:15:06 1.26.4.19 +++ ClientStorage.py 2001/05/09 15:17:33 1.26.4.20 @@ -86,9 +86,6 @@ XXX support multiple outstanding requests up until the vote? XXX is_connected() vis ClientDisconnected error -XXX it would be better to avoid invalidating for abortVersion, - commitVersion, transactionalUndo until the transaction actually - commits. """ __version__='$Revision$'[11:-2] --- Updated File ClientStorage.py in package Packages/ZEO -- --- ClientStorage.py 2001/05/07 13:15:06 1.26.4.19 +++ ClientStorage.py 2001/05/09 15:17:33 1.26.4.20 @@ -86,9 +86,6 @@ XXX support multiple outstanding requests up until the vote? XXX is_connected() vis ClientDisconnected error -XXX it would be better to avoid invalidating for abortVersion, - commitVersion, transactionalUndo until the transaction actually - commits. """ __version__='$Revision$'[11:-2] From jeremy at digicool.com Wed May 9 12:03:20 2001 From: jeremy at digicool.com (jeremy@digicool.com) Date: Sun Aug 10 16:31:15 2008 Subject: [ZEO-Checkins] CVS: Packages/ZEO - ClientStorage.py:1.31 StorageServer.py:1.26 Message-ID: <20010509160320.329B1510E3@korak.digicool.com> Update of /cvs-repository/Packages/ZEO In directory korak:/tmp/cvs-serv18529 Modified Files: ClientStorage.py StorageServer.py Log Message: Add support for transactionalUndo --- Updated File ClientStorage.py in package Packages/ZEO -- --- ClientStorage.py 2001/05/08 22:33:54 1.30 +++ ClientStorage.py 2001/05/09 16:03:19 1.31 @@ -429,6 +429,8 @@ def supportsUndo(self): return self._info['supportsUndo'] def supportsVersions(self): return self._info['supportsVersions'] + def supportsTransactionalUndo(self): + return self._info['supportsTransactionalUndo'] def tpc_abort(self, transaction): self._lock_acquire() @@ -545,6 +547,18 @@ self._transaction=None self._commit_lock_release() + finally: self._lock_release() + + def transactionalUndo(self, trans_id, trans): + self._lock_acquire() + try: + if trans is not self._transaction: + raise POSException.StorageTransactionError(self, transaction) + oids = self._call('transactionalUndo', trans_id, self._serial) + for oid in oids: + # write invalidation records with no version + self._tfile.write("i%s\000\000" % oid) + return oids finally: self._lock_release() def undo(self, transaction_id): --- Updated File StorageServer.py in package Packages/ZEO -- --- StorageServer.py 2001/05/08 21:14:03 1.25 +++ StorageServer.py 2001/05/09 16:03:19 1.26 @@ -201,6 +201,7 @@ 'modifiedInVersion', 'new_oid', 'new_oids', 'pack', 'store', 'storea', 'tpc_abort', 'tpc_begin', 'tpc_begin_sync', 'tpc_finish', 'undo', 'undoLog', 'undoInfo', 'versionEmpty', 'versions', + 'transactionalUndo', 'vote', 'zeoLoad', 'zeoVerify', 'beginZeoVerify', 'endZeoVerify', ): storage_methods[n]=1 @@ -320,6 +321,7 @@ 'name': storage.getName(), 'supportsUndo': storage.supportsUndo(), 'supportsVersions': storage.supportsVersions(), + 'supportsTransactionalUndo': storage.supportsTransactionalUndo(), } def get_size_info(self): @@ -451,6 +453,12 @@ if t is None or id != t.id: raise POSException.StorageTransactionError(self, id) return self.__storage.tpc_vote(t) + + def transactionalUndo(self, trans_id, id): + t=self._transaction + if t is None or id != t.id: + raise POSException.StorageTransactionError(self, id) + return self.__storage.transactionalUndo(trans_id, self._transaction) def undo(self, transaction_id): oids=self.__storage.undo(transaction_id) --- Updated File ClientStorage.py in package Packages/ZEO -- --- ClientStorage.py 2001/05/08 22:33:54 1.30 +++ ClientStorage.py 2001/05/09 16:03:19 1.31 @@ -429,6 +429,8 @@ def supportsUndo(self): return self._info['supportsUndo'] def supportsVersions(self): return self._info['supportsVersions'] + def supportsTransactionalUndo(self): + return self._info['supportsTransactionalUndo'] def tpc_abort(self, transaction): self._lock_acquire() @@ -545,6 +547,18 @@ self._transaction=None self._commit_lock_release() + finally: self._lock_release() + + def transactionalUndo(self, trans_id, trans): + self._lock_acquire() + try: + if trans is not self._transaction: + raise POSException.StorageTransactionError(self, transaction) + oids = self._call('transactionalUndo', trans_id, self._serial) + for oid in oids: + # write invalidation records with no version + self._tfile.write("i%s\000\000" % oid) + return oids finally: self._lock_release() def undo(self, transaction_id): --- Updated File StorageServer.py in package Packages/ZEO -- --- StorageServer.py 2001/05/08 21:14:03 1.25 +++ StorageServer.py 2001/05/09 16:03:19 1.26 @@ -201,6 +201,7 @@ 'modifiedInVersion', 'new_oid', 'new_oids', 'pack', 'store', 'storea', 'tpc_abort', 'tpc_begin', 'tpc_begin_sync', 'tpc_finish', 'undo', 'undoLog', 'undoInfo', 'versionEmpty', 'versions', + 'transactionalUndo', 'vote', 'zeoLoad', 'zeoVerify', 'beginZeoVerify', 'endZeoVerify', ): storage_methods[n]=1 @@ -320,6 +321,7 @@ 'name': storage.getName(), 'supportsUndo': storage.supportsUndo(), 'supportsVersions': storage.supportsVersions(), + 'supportsTransactionalUndo': storage.supportsTransactionalUndo(), } def get_size_info(self): @@ -451,6 +453,12 @@ if t is None or id != t.id: raise POSException.StorageTransactionError(self, id) return self.__storage.tpc_vote(t) + + def transactionalUndo(self, trans_id, id): + t=self._transaction + if t is None or id != t.id: + raise POSException.StorageTransactionError(self, id) + return self.__storage.transactionalUndo(trans_id, self._transaction) def undo(self, transaction_id): oids=self.__storage.undo(transaction_id) From jeremy at digicool.com Wed May 9 12:22:11 2001 From: jeremy at digicool.com (jeremy@digicool.com) Date: Sun Aug 10 16:31:15 2008 Subject: [ZEO-Checkins] CVS: Packages/ZEO - testZEO.py:1.5 Message-ID: <20010509162211.61EEB510E3@korak.digicool.com> Update of /cvs-repository/Packages/ZEO/tests In directory korak:/tmp/cvs-serv19798 Modified Files: testZEO.py Log Message: Add new tests now that transactionalUndo is supported --- Updated File testZEO.py in package Packages/ZEO -- --- testZEO.py 2001/05/08 21:25:24 1.4 +++ testZEO.py 2001/05/09 16:22:10 1.5 @@ -100,11 +100,15 @@ class GenericTests(ZEOTestBase, Cache.StorageWithCache, + Cache.TransUndoStorageWithCache, BasicStorage.BasicStorage, VersionStorage.VersionStorage, PackableStorage.PackableStorage, Synchronization.SynchronizedStorage, ConflictResolution.ConflictResolvingStorage, + ConflictResolution.ConflictResolvingTransUndoStorage, + TransactionalUndoStorage.TransactionalUndoStorage, + TransactionalUndoVersionStorage.TransactionalUndoVersionStorage, ): """An abstract base class for ZEO tests --- Updated File testZEO.py in package Packages/ZEO -- --- testZEO.py 2001/05/08 21:25:24 1.4 +++ testZEO.py 2001/05/09 16:22:10 1.5 @@ -100,11 +100,15 @@ class GenericTests(ZEOTestBase, Cache.StorageWithCache, + Cache.TransUndoStorageWithCache, BasicStorage.BasicStorage, VersionStorage.VersionStorage, PackableStorage.PackableStorage, Synchronization.SynchronizedStorage, ConflictResolution.ConflictResolvingStorage, + ConflictResolution.ConflictResolvingTransUndoStorage, + TransactionalUndoStorage.TransactionalUndoStorage, + TransactionalUndoVersionStorage.TransactionalUndoVersionStorage, ): """An abstract base class for ZEO tests From jeremy at digicool.com Wed May 9 12:38:38 2001 From: jeremy at digicool.com (jeremy@digicool.com) Date: Sun Aug 10 16:31:15 2008 Subject: [ZEO-Checkins] CVS: Packages/ZEO - forker.py:1.4 Message-ID: <20010509163838.D053E510E3@korak.digicool.com> Update of /cvs-repository/Packages/ZEO/tests In directory korak:/tmp/cvs-serv20760 Modified Files: forker.py Log Message: Remove is_connected stuff (not supported w/ZEO1). Add optional cache_size argument. --- Updated File forker.py in package Packages/ZEO -- --- forker.py 2001/05/08 21:16:37 1.3 +++ forker.py 2001/05/09 16:38:38 1.4 @@ -65,7 +65,7 @@ os.unlink(addr) def start_zeo(storage, cache=None, cleanup=None, domain="AF_INET", - storage_id="1"): + storage_id="1", cache_size=20000000): """Setup ZEO client-server for storage. Returns a ClientStorage instance and a ZEOClientExit instance. @@ -84,9 +84,7 @@ pid, exit = start_zeo_server(storage, addr) s = ZEO.ClientStorage.ClientStorage(addr, storage_id, - debug=1, client=cache) - if hasattr(s, 'is_connected'): - while not s.is_connected(): - time.sleep(0.1) + debug=1, client=cache, + cache_size=cache_size) return s, exit, pid --- Updated File forker.py in package Packages/ZEO -- --- forker.py 2001/05/08 21:16:37 1.3 +++ forker.py 2001/05/09 16:38:38 1.4 @@ -65,7 +65,7 @@ os.unlink(addr) def start_zeo(storage, cache=None, cleanup=None, domain="AF_INET", - storage_id="1"): + storage_id="1", cache_size=20000000): """Setup ZEO client-server for storage. Returns a ClientStorage instance and a ZEOClientExit instance. @@ -84,9 +84,7 @@ pid, exit = start_zeo_server(storage, addr) s = ZEO.ClientStorage.ClientStorage(addr, storage_id, - debug=1, client=cache) - if hasattr(s, 'is_connected'): - while not s.is_connected(): - time.sleep(0.1) + debug=1, client=cache, + cache_size=cache_size) return s, exit, pid From jeremy at digicool.com Wed May 9 14:27:25 2001 From: jeremy at digicool.com (jeremy@digicool.com) Date: Sun Aug 10 16:31:16 2008 Subject: [ZEO-Checkins] CVS: Packages/ZEO - zrpc.py:1.16 Message-ID: <20010509182725.8F4B1510E3@korak.digicool.com> Update of /cvs-repository/Packages/ZEO In directory korak:/tmp/cvs-serv27472 Modified Files: zrpc.py Log Message: Make connect logging output more informative --- Updated File zrpc.py in package Packages/ZEO -- --- zrpc.py 2001/05/09 14:35:40 1.15 +++ zrpc.py 2001/05/09 18:27:24 1.16 @@ -133,7 +133,8 @@ connection = self._connection debug=self._debug while 1: - if log_type: LOG(log_type, INFO, 'Trying to connect to server') + if log_type: LOG(log_type, INFO, + 'Trying to connect to server: %s' % `connection`) try: if type(connection) is type(''): s=socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) --- Updated File zrpc.py in package Packages/ZEO -- --- zrpc.py 2001/05/09 14:35:40 1.15 +++ zrpc.py 2001/05/09 18:27:24 1.16 @@ -133,7 +133,8 @@ connection = self._connection debug=self._debug while 1: - if log_type: LOG(log_type, INFO, 'Trying to connect to server') + if log_type: LOG(log_type, INFO, + 'Trying to connect to server: %s' % `connection`) try: if type(connection) is type(''): s=socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) From jeremy at digicool.com Wed May 9 14:34:02 2001 From: jeremy at digicool.com (jeremy@digicool.com) Date: Sun Aug 10 16:31:16 2008 Subject: [ZEO-Checkins] CVS: Packages/ZEO - ClientCache.py:1.16 Message-ID: <20010509183402.3B9DB510E3@korak.digicool.com> Update of /cvs-repository/Packages/ZEO In directory korak:/tmp/cvs-serv27896 Modified Files: ClientCache.py Log Message: Add a close() method that closes the current file, guaranteeting that any recent updates are written to the file. Remove seek() and tell() at the beginning of read_index() because their results are not used. Add main() that prints the contents a cache file when executed as a script. Add a log call when the cache is being read. --- Updated File ClientCache.py in package Packages/ZEO -- --- ClientCache.py 2001/04/19 19:20:19 1.15 +++ ClientCache.py 2001/05/09 18:34:01 1.16 @@ -149,9 +149,13 @@ import os, tempfile from struct import pack, unpack from thread import allocate_lock +import zLOG magic='ZEC0' +def LOG(msg, level=zLOG.BLATHER): + zLOG.LOG("ZEC", level, msg) + class ClientCache: def __init__(self, storage='', size=20000000, client=None, var=None): @@ -210,6 +214,9 @@ self._limit=size/2 self._current=current + def close(self): + self._f[self._current].close() + def open(self): self._acquire() try: @@ -407,19 +414,19 @@ self._pos=pos+tlen def read_index(index, serial, f, current): + LOG("read_index(%s)" % f.name) seek=f.seek read=f.read pos=4 - seek(0,2) - size=f.tell() while 1: - f.seek(pos) + seek(pos) h=read(27) - + if len(h)==27 and h[8] in 'vni': tlen, vlen, dlen = unpack(">iHi", h[9:19]) - else: tlen=-1 + else: + break if tlen <= 0 or vlen < 0 or dlen < 0 or vlen+dlen > tlen: break @@ -454,3 +461,15 @@ except: pass return pos + +def main(files): + for file in files: + print file + index = {} + serial = {} + read_index(index, serial, open(file), 0) + print index.keys() + +if __name__ == "__main__": + import sys + main(sys.argv[1:]) --- Updated File ClientCache.py in package Packages/ZEO -- --- ClientCache.py 2001/04/19 19:20:19 1.15 +++ ClientCache.py 2001/05/09 18:34:01 1.16 @@ -149,9 +149,13 @@ import os, tempfile from struct import pack, unpack from thread import allocate_lock +import zLOG magic='ZEC0' +def LOG(msg, level=zLOG.BLATHER): + zLOG.LOG("ZEC", level, msg) + class ClientCache: def __init__(self, storage='', size=20000000, client=None, var=None): @@ -210,6 +214,9 @@ self._limit=size/2 self._current=current + def close(self): + self._f[self._current].close() + def open(self): self._acquire() try: @@ -407,19 +414,19 @@ self._pos=pos+tlen def read_index(index, serial, f, current): + LOG("read_index(%s)" % f.name) seek=f.seek read=f.read pos=4 - seek(0,2) - size=f.tell() while 1: - f.seek(pos) + seek(pos) h=read(27) - + if len(h)==27 and h[8] in 'vni': tlen, vlen, dlen = unpack(">iHi", h[9:19]) - else: tlen=-1 + else: + break if tlen <= 0 or vlen < 0 or dlen < 0 or vlen+dlen > tlen: break @@ -454,3 +461,15 @@ except: pass return pos + +def main(files): + for file in files: + print file + index = {} + serial = {} + read_index(index, serial, open(file), 0) + print index.keys() + +if __name__ == "__main__": + import sys + main(sys.argv[1:]) From jeremy at digicool.com Wed May 9 14:37:53 2001 From: jeremy at digicool.com (jeremy@digicool.com) Date: Sun Aug 10 16:31:16 2008 Subject: [ZEO-Checkins] CVS: Packages/ZEO - ClientStorage.py:1.32 Message-ID: <20010509183753.33287510E3@korak.digicool.com> Update of /cvs-repository/Packages/ZEO In directory korak:/tmp/cvs-serv28188 Modified Files: ClientStorage.py Log Message: close the cache when the storage is closed --- Updated File ClientStorage.py in package Packages/ZEO -- --- ClientStorage.py 2001/05/09 16:03:19 1.31 +++ ClientStorage.py 2001/05/09 18:37:52 1.32 @@ -297,7 +297,9 @@ def close(self): self._lock_acquire() - try: self._call.closeIntensionally() + try: + self._call.closeIntensionally() + self._cache.close() finally: self._lock_release() def commitVersion(self, src, dest, transaction): --- Updated File ClientStorage.py in package Packages/ZEO -- --- ClientStorage.py 2001/05/09 16:03:19 1.31 +++ ClientStorage.py 2001/05/09 18:37:52 1.32 @@ -297,7 +297,9 @@ def close(self): self._lock_acquire() - try: self._call.closeIntensionally() + try: + self._call.closeIntensionally() + self._cache.close() finally: self._lock_release() def commitVersion(self, src, dest, transaction): From jeremy at digicool.com Wed May 9 14:38:24 2001 From: jeremy at digicool.com (jeremy@digicool.com) Date: Sun Aug 10 16:31:16 2008 Subject: [ZEO-Checkins] CVS: Packages/ZEO - StorageServer.py:1.27 Message-ID: <20010509183824.A4E23510E3@korak.digicool.com> Update of /cvs-repository/Packages/ZEO In directory korak:/tmp/cvs-serv28232 Modified Files: StorageServer.py Log Message: Add log message when server is ready to accept connections --- Updated File StorageServer.py in package Packages/ZEO -- --- StorageServer.py 2001/05/09 16:03:19 1.26 +++ StorageServer.py 2001/05/09 18:38:23 1.27 @@ -135,8 +135,8 @@ self.create_socket(socket.AF_INET, socket.SOCK_STREAM) self.set_reuse_addr() + LOG('ZEO Server', INFO, 'Listening on %s' % repr(connection)) self.bind(connection) - self.listen(5) def register_connection(self, connection, storage_id): --- Updated File StorageServer.py in package Packages/ZEO -- --- StorageServer.py 2001/05/09 16:03:19 1.26 +++ StorageServer.py 2001/05/09 18:38:23 1.27 @@ -135,8 +135,8 @@ self.create_socket(socket.AF_INET, socket.SOCK_STREAM) self.set_reuse_addr() + LOG('ZEO Server', INFO, 'Listening on %s' % repr(connection)) self.bind(connection) - self.listen(5) def register_connection(self, connection, storage_id): From jeremy at digicool.com Wed May 9 14:58:00 2001 From: jeremy at digicool.com (jeremy@digicool.com) Date: Sun Aug 10 16:31:16 2008 Subject: [ZEO-Checkins] CVS: Packages/ZEO - testZEO.py:1.6 Message-ID: <20010509185800.2406D510E3@korak.digicool.com> Update of /cvs-repository/Packages/ZEO/tests In directory korak:/tmp/cvs-serv29564 Modified Files: testZEO.py Log Message: Add tests of persistent cache mechanism. Add makeTestSuite() helper that replaces clunky unittest mechanisms. --- Updated File testZEO.py in package Packages/ZEO -- --- testZEO.py 2001/05/09 16:22:10 1.5 +++ testZEO.py 2001/05/09 18:58:00 1.6 @@ -2,6 +2,7 @@ import asyncore import os +import random import tempfile import time import types @@ -12,12 +13,14 @@ from ZODB.FileStorage import FileStorage from ZEO.tests import forker, Cache +from ZEO.smac import Disconnected # Sorry Jim... from ZODB.tests import StorageTestBase, BasicStorage, VersionStorage, \ TransactionalUndoStorage, TransactionalUndoVersionStorage, \ PackableStorage, Synchronization, ConflictResolution from ZODB.tests.MinPO import MinPO +from ZODB.tests.StorageTestBase import zodb_unpickle ZERO = '\0'*8 @@ -93,10 +96,6 @@ raise serial d[oid] = serial return d - - def checkLargeUpdate(self): - obj = MinPO("X" * (10 * 128 * 1024)) - self._dostore(data=obj) class GenericTests(ZEOTestBase, Cache.StorageWithCache, @@ -142,6 +141,10 @@ os.waitpid(self._pid, 0) self.__super_tearDown() + def checkLargeUpdate(self): + obj = MinPO("X" * (10 * 128 * 1024)) + self._dostore(data=obj) + class ZEOFileStorageTests(GenericTests): __super_setUp = GenericTests.setUp @@ -153,11 +156,117 @@ return FileStorage(self.__fs_base, create=1) def delStorage(self): - # file storage appears to create three files + # file storage appears to create four files for ext in '', '.index', '.lock', '.tmp': path = self.__fs_base + ext os.unlink(path) - + +class PersistentCacheTests(ZEOTestBase): + __super_setUp = StorageTestBase.StorageTestBase.setUp + __super_tearDown = StorageTestBase.StorageTestBase.tearDown + + def setUp(self): + """Start a ZEO server using a Unix domain socket + + The ZEO server uses the storage object returned by the + getStorage() method. + """ + self.running = 1 + self.__fs_base = tempfile.mktemp() + fs = FileStorage(self.__fs_base, create=1) + self.addr = '', random.randrange(2000, 3000) + pid, exit = forker.start_zeo_server(fs, self.addr) + self._pid = pid + self._server = exit + self.__super_setUp() + + def openClientStorage(self, cache, cache_size, wait): + base = ZEO.ClientStorage.ClientStorage(self.addr, + client=cache, + cache_size=cache_size, + wait_for_server_on_startup=wait) + storage = PackWaitWrapper(base) + storage.registerDB(DummyDB(), None) + return storage + + def shutdownServer(self): + if self.running: + self.running = 0 + self._server.close() + os.waitpid(self._pid, 0) + + def tearDown(self): + """Try to cause the tests to halt""" + self.shutdownServer() + # file storage appears to create four files + for ext in '', '.index', '.lock', '.tmp': + path = self.__fs_base + ext + if os.path.exists(path): + os.unlink(path) + for i in 0, 1: + path = "c1-test-%d.zec" % i + if os.path.exists(path): + os.unlink(path) + self.__super_tearDown() + + def checkBasicPersistence(self): + """Verify cached data persists across client storage instances. + + To verify that the cache is being used, the test closes the + server and then starts a new client with the server down. + """ + self._storage = self.openClientStorage('test', 100000, 1) + oid = self._storage.new_oid() + obj = MinPO(12) + revid1 = self._dostore(oid, data=obj) + self._storage.close() + self.shutdownServer() + self._storage = self.openClientStorage('test', 100000, 0) + data, revid2 = self._storage.load(oid, '') + assert zodb_unpickle(data) == MinPO(12) + assert revid1 == revid2 + self._storage.close() + + def checkRollover(self): + """Check that the cache works when the files are swapped. + + In this case, only one object fits in a cache file. When the + cache files swap, the first object is effectively uncached. + """ + self._storage = self.openClientStorage('test', 1000, 1) + oid1 = self._storage.new_oid() + obj1 = MinPO("1" * 500) + revid1 = self._dostore(oid1, data=obj1) + oid2 = self._storage.new_oid() + obj2 = MinPO("2" * 500) + revid2 = self._dostore(oid2, data=obj2) + self._storage.close() + self.shutdownServer() + self._storage = self.openClientStorage('test', 1000, 0) + self._storage.load(oid2, '') + self.assertRaises(Disconnected, self._storage.load, oid1, '') + +def get_methods(klass): + l = [klass] + meth = {} + while l: + klass = l.pop(0) + for base in klass.__bases__: + l.append(base) + for k, v in klass.__dict__.items(): + if callable(v): + meth[k] = 1 + return meth.keys() + +def makeTestSuite(testname=''): + suite = unittest.TestSuite() + name = 'check' + testname + for klass in ZEOFileStorageTests, PersistentCacheTests: + for meth in get_methods(klass): + if meth.startswith(name): + suite.addTest(klass(meth)) + return suite + def main(): import sys, getopt @@ -172,7 +281,7 @@ print >> sys.stderr, "Did not expect arguments. Got %s" % args return 0 - tests = unittest.makeSuite(ZEOFileStorageTests, 'check' + name_of_test) + tests = makeTestSuite(name_of_test) runner = unittest.TextTestRunner() runner.run(tests) --- Updated File testZEO.py in package Packages/ZEO -- --- testZEO.py 2001/05/09 16:22:10 1.5 +++ testZEO.py 2001/05/09 18:58:00 1.6 @@ -2,6 +2,7 @@ import asyncore import os +import random import tempfile import time import types @@ -12,12 +13,14 @@ from ZODB.FileStorage import FileStorage from ZEO.tests import forker, Cache +from ZEO.smac import Disconnected # Sorry Jim... from ZODB.tests import StorageTestBase, BasicStorage, VersionStorage, \ TransactionalUndoStorage, TransactionalUndoVersionStorage, \ PackableStorage, Synchronization, ConflictResolution from ZODB.tests.MinPO import MinPO +from ZODB.tests.StorageTestBase import zodb_unpickle ZERO = '\0'*8 @@ -93,10 +96,6 @@ raise serial d[oid] = serial return d - - def checkLargeUpdate(self): - obj = MinPO("X" * (10 * 128 * 1024)) - self._dostore(data=obj) class GenericTests(ZEOTestBase, Cache.StorageWithCache, @@ -142,6 +141,10 @@ os.waitpid(self._pid, 0) self.__super_tearDown() + def checkLargeUpdate(self): + obj = MinPO("X" * (10 * 128 * 1024)) + self._dostore(data=obj) + class ZEOFileStorageTests(GenericTests): __super_setUp = GenericTests.setUp @@ -153,11 +156,117 @@ return FileStorage(self.__fs_base, create=1) def delStorage(self): - # file storage appears to create three files + # file storage appears to create four files for ext in '', '.index', '.lock', '.tmp': path = self.__fs_base + ext os.unlink(path) - + +class PersistentCacheTests(ZEOTestBase): + __super_setUp = StorageTestBase.StorageTestBase.setUp + __super_tearDown = StorageTestBase.StorageTestBase.tearDown + + def setUp(self): + """Start a ZEO server using a Unix domain socket + + The ZEO server uses the storage object returned by the + getStorage() method. + """ + self.running = 1 + self.__fs_base = tempfile.mktemp() + fs = FileStorage(self.__fs_base, create=1) + self.addr = '', random.randrange(2000, 3000) + pid, exit = forker.start_zeo_server(fs, self.addr) + self._pid = pid + self._server = exit + self.__super_setUp() + + def openClientStorage(self, cache, cache_size, wait): + base = ZEO.ClientStorage.ClientStorage(self.addr, + client=cache, + cache_size=cache_size, + wait_for_server_on_startup=wait) + storage = PackWaitWrapper(base) + storage.registerDB(DummyDB(), None) + return storage + + def shutdownServer(self): + if self.running: + self.running = 0 + self._server.close() + os.waitpid(self._pid, 0) + + def tearDown(self): + """Try to cause the tests to halt""" + self.shutdownServer() + # file storage appears to create four files + for ext in '', '.index', '.lock', '.tmp': + path = self.__fs_base + ext + if os.path.exists(path): + os.unlink(path) + for i in 0, 1: + path = "c1-test-%d.zec" % i + if os.path.exists(path): + os.unlink(path) + self.__super_tearDown() + + def checkBasicPersistence(self): + """Verify cached data persists across client storage instances. + + To verify that the cache is being used, the test closes the + server and then starts a new client with the server down. + """ + self._storage = self.openClientStorage('test', 100000, 1) + oid = self._storage.new_oid() + obj = MinPO(12) + revid1 = self._dostore(oid, data=obj) + self._storage.close() + self.shutdownServer() + self._storage = self.openClientStorage('test', 100000, 0) + data, revid2 = self._storage.load(oid, '') + assert zodb_unpickle(data) == MinPO(12) + assert revid1 == revid2 + self._storage.close() + + def checkRollover(self): + """Check that the cache works when the files are swapped. + + In this case, only one object fits in a cache file. When the + cache files swap, the first object is effectively uncached. + """ + self._storage = self.openClientStorage('test', 1000, 1) + oid1 = self._storage.new_oid() + obj1 = MinPO("1" * 500) + revid1 = self._dostore(oid1, data=obj1) + oid2 = self._storage.new_oid() + obj2 = MinPO("2" * 500) + revid2 = self._dostore(oid2, data=obj2) + self._storage.close() + self.shutdownServer() + self._storage = self.openClientStorage('test', 1000, 0) + self._storage.load(oid2, '') + self.assertRaises(Disconnected, self._storage.load, oid1, '') + +def get_methods(klass): + l = [klass] + meth = {} + while l: + klass = l.pop(0) + for base in klass.__bases__: + l.append(base) + for k, v in klass.__dict__.items(): + if callable(v): + meth[k] = 1 + return meth.keys() + +def makeTestSuite(testname=''): + suite = unittest.TestSuite() + name = 'check' + testname + for klass in ZEOFileStorageTests, PersistentCacheTests: + for meth in get_methods(klass): + if meth.startswith(name): + suite.addTest(klass(meth)) + return suite + def main(): import sys, getopt @@ -172,7 +281,7 @@ print >> sys.stderr, "Did not expect arguments. Got %s" % args return 0 - tests = unittest.makeSuite(ZEOFileStorageTests, 'check' + name_of_test) + tests = makeTestSuite(name_of_test) runner = unittest.TextTestRunner() runner.run(tests) From jeremy at digicool.com Wed May 9 15:00:21 2001 From: jeremy at digicool.com (jeremy@digicool.com) Date: Sun Aug 10 16:31:16 2008 Subject: [ZEO-Checkins] CVS: Packages/ZEO - ClientStorage.py:1.33 Message-ID: <20010509190021.534A2510E3@korak.digicool.com> Update of /cvs-repository/Packages/ZEO In directory korak:/tmp/cvs-serv29664 Modified Files: ClientStorage.py Log Message: XXX Add closed attribute to prevent re-opening the cache after it is closed. --- Updated File ClientStorage.py in package Packages/ZEO -- --- ClientStorage.py 2001/05/09 18:37:52 1.32 +++ ClientStorage.py 2001/05/09 19:00:20 1.33 @@ -140,6 +140,7 @@ name = name or str(connection) + self.closed = 0 self._tfile=tempfile.TemporaryFile() self._oids=[] self._serials=[] @@ -221,6 +222,9 @@ self._call.finishConnect(s) + if self.closed: + return + self._connected=1 self._oids=[] @@ -298,8 +302,10 @@ def close(self): self._lock_acquire() try: + LOG("ClientStorage", INFO, "close") self._call.closeIntensionally() self._cache.close() + self.closed = 1 finally: self._lock_release() def commitVersion(self, src, dest, transaction): --- Updated File ClientStorage.py in package Packages/ZEO -- --- ClientStorage.py 2001/05/09 18:37:52 1.32 +++ ClientStorage.py 2001/05/09 19:00:20 1.33 @@ -140,6 +140,7 @@ name = name or str(connection) + self.closed = 0 self._tfile=tempfile.TemporaryFile() self._oids=[] self._serials=[] @@ -221,6 +222,9 @@ self._call.finishConnect(s) + if self.closed: + return + self._connected=1 self._oids=[] @@ -298,8 +302,10 @@ def close(self): self._lock_acquire() try: + LOG("ClientStorage", INFO, "close") self._call.closeIntensionally() self._cache.close() + self.closed = 1 finally: self._lock_release() def commitVersion(self, src, dest, transaction): From jeremy at digicool.com Wed May 9 15:06:05 2001 From: jeremy at digicool.com (jeremy@digicool.com) Date: Sun Aug 10 16:31:16 2008 Subject: [ZEO-Checkins] CVS: Packages/ZEO - forker.py:1.5 Message-ID: <20010509190605.BC6A3510E3@korak.digicool.com> Update of /cvs-repository/Packages/ZEO/tests In directory korak:/tmp/cvs-serv30029 Modified Files: forker.py Log Message: Set the min_disconnect_poll variable to 0.5. (This is actually misnamed, because it's also the min_connect_poll and it makes the tests run much faster by waiting for a shorter period of time between connect attempts.) --- Updated File forker.py in package Packages/ZEO -- --- forker.py 2001/05/09 16:38:38 1.4 +++ forker.py 2001/05/09 19:06:05 1.5 @@ -85,6 +85,7 @@ pid, exit = start_zeo_server(storage, addr) s = ZEO.ClientStorage.ClientStorage(addr, storage_id, debug=1, client=cache, - cache_size=cache_size) + cache_size=cache_size, + min_disconnect_poll=0.5) return s, exit, pid --- Updated File forker.py in package Packages/ZEO -- --- forker.py 2001/05/09 16:38:38 1.4 +++ forker.py 2001/05/09 19:06:05 1.5 @@ -85,6 +85,7 @@ pid, exit = start_zeo_server(storage, addr) s = ZEO.ClientStorage.ClientStorage(addr, storage_id, debug=1, client=cache, - cache_size=cache_size) + cache_size=cache_size, + min_disconnect_poll=0.5) return s, exit, pid From jeremy at digicool.com Wed May 16 23:23:35 2001 From: jeremy at digicool.com (jeremy@digicool.com) Date: Sun Aug 10 16:31:16 2008 Subject: [ZEO-Checkins] CVS: Packages/ZEO - StorageServer.py:1.21.4.13 Message-ID: <20010517032335.2DF4151210@korak.digicool.com> Update of /cvs-repository/Packages/ZEO In directory korak.digicool.com:/tmp/cvs-serv7169 Modified Files: Tag: ZEO-ZRPC-Dev StorageServer.py Log Message: The file storage does not return a value from tpc_vote(). --- Updated File StorageServer.py in package Packages/ZEO -- --- StorageServer.py 2001/05/08 20:36:11 1.21.4.12 +++ StorageServer.py 2001/05/17 03:23:34 1.21.4.13 @@ -351,7 +351,7 @@ def vote(self, id): self._check_tid(id, exc=StorageTransactionError) - return self.__storage.tpc_vote(self._transaction) + self.__storage.tpc_vote(self._transaction) def transactionalUndo(self, trans_id, id): self._check_tid(id, exc=StorageTransactionError) --- Updated File StorageServer.py in package Packages/ZEO -- --- StorageServer.py 2001/05/08 20:36:11 1.21.4.12 +++ StorageServer.py 2001/05/17 03:23:34 1.21.4.13 @@ -351,7 +351,7 @@ def vote(self, id): self._check_tid(id, exc=StorageTransactionError) - return self.__storage.tpc_vote(self._transaction) + self.__storage.tpc_vote(self._transaction) def transactionalUndo(self, trans_id, id): self._check_tid(id, exc=StorageTransactionError) From jeremy at digicool.com Wed May 16 23:19:44 2001 From: jeremy at digicool.com (jeremy@digicool.com) Date: Sun Aug 10 16:31:16 2008 Subject: [ZEO-Checkins] CVS: Packages/ZEO - ClientStorage.py:1.26.4.21 Message-ID: <20010517031944.BE37C5120E@korak.digicool.com> Update of /cvs-repository/Packages/ZEO In directory korak.digicool.com:/tmp/cvs-serv7109 Modified Files: Tag: ZEO-ZRPC-Dev ClientStorage.py Log Message: Add wait_for_server_on_startup keyword arg support. --- Updated File ClientStorage.py in package Packages/ZEO -- --- ClientStorage.py 2001/05/09 15:17:33 1.26.4.20 +++ ClientStorage.py 2001/05/17 03:19:43 1.26.4.21 @@ -142,7 +142,7 @@ def __init__(self, addr, storage='1', cache_size=20000000, name='', client='', debug=0, var=None, min_disconnect_poll=5, max_disconnect_poll=300, - wait_for_server_on_starup=1): + wait_for_server_on_startup=1): # Decide whether to use non-temporary files client = client or os.environ.get('ZEO_CLIENT','') @@ -150,7 +150,7 @@ self._connection = addr self._storage = storage self._debug = debug - self._wait_for_server_on_starup = wait_for_server_on_starup + self._wait_for_server_on_startup = wait_for_server_on_startup self._info = {'length': 0, 'size': 0, 'name': 'ZEO Client', 'supportsUndo':0, 'supportsVersions': 0} @@ -175,9 +175,11 @@ tmin=min_disconnect_poll, tmax=max_disconnect_poll) self._server = None - # XXX make this method call the default CnMgr behavior? - if not self._rpc_mgr.attempt_connect(): - self._rpc_mgr.connect() + if wait_for_server_on_startup: + self._rpc_mgr.connect(1) + else: + if not self._rpc_mgr.attempt_connect(): + self._rpc_mgr.connect() def _basic_init(self, name): """Handle initialization activites of BaseStorage""" --- Updated File ClientStorage.py in package Packages/ZEO -- --- ClientStorage.py 2001/05/09 15:17:33 1.26.4.20 +++ ClientStorage.py 2001/05/17 03:19:43 1.26.4.21 @@ -142,7 +142,7 @@ def __init__(self, addr, storage='1', cache_size=20000000, name='', client='', debug=0, var=None, min_disconnect_poll=5, max_disconnect_poll=300, - wait_for_server_on_starup=1): + wait_for_server_on_startup=1): # Decide whether to use non-temporary files client = client or os.environ.get('ZEO_CLIENT','') @@ -150,7 +150,7 @@ self._connection = addr self._storage = storage self._debug = debug - self._wait_for_server_on_starup = wait_for_server_on_starup + self._wait_for_server_on_startup = wait_for_server_on_startup self._info = {'length': 0, 'size': 0, 'name': 'ZEO Client', 'supportsUndo':0, 'supportsVersions': 0} @@ -175,9 +175,11 @@ tmin=min_disconnect_poll, tmax=max_disconnect_poll) self._server = None - # XXX make this method call the default CnMgr behavior? - if not self._rpc_mgr.attempt_connect(): - self._rpc_mgr.connect() + if wait_for_server_on_startup: + self._rpc_mgr.connect(1) + else: + if not self._rpc_mgr.attempt_connect(): + self._rpc_mgr.connect() def _basic_init(self, name): """Handle initialization activites of BaseStorage""" From jeremy at digicool.com Mon May 14 15:08:20 2001 From: jeremy at digicool.com (jeremy@digicool.com) Date: Sun Aug 10 16:31:16 2008 Subject: [ZEO-Checkins] CVS: Packages/ZEO - StorageServer.py:1.28 Message-ID: <20010514190820.16BA751300@korak.digicool.com> Update of /cvs-repository/Packages/ZEO In directory korak.digicool.com:/tmp/cvs-serv26537 Modified Files: StorageServer.py Log Message: Fix get_info() so that it copes if the storage doesn't define a "supportsTransactionalUndo()" method. XXX untested XXX This mechanism for feature evolutional doesn't seem right, because it because a morass of backwards compatibility issues. I'd rather see the test by one of presence/absence of an attribute or a base class. --- Updated File StorageServer.py in package Packages/ZEO -- --- StorageServer.py 2001/05/09 18:38:23 1.27 +++ StorageServer.py 2001/05/14 19:08:19 1.28 @@ -315,14 +315,19 @@ def get_info(self): storage=self.__storage - return { + info = { 'length': len(storage), 'size': storage.getSize(), 'name': storage.getName(), - 'supportsUndo': storage.supportsUndo(), - 'supportsVersions': storage.supportsVersions(), - 'supportsTransactionalUndo': storage.supportsTransactionalUndo(), } + for feature in ('supportsUndo', + 'supportsVersions', + 'supportsTransactionalUndo',): + if hasattr(storage, feature): + info[feature] = getattr(storage, feature)() + else: + info[feature] = 0 + return info def get_size_info(self): storage=self.__storage --- Updated File StorageServer.py in package Packages/ZEO -- --- StorageServer.py 2001/05/09 18:38:23 1.27 +++ StorageServer.py 2001/05/14 19:08:19 1.28 @@ -315,14 +315,19 @@ def get_info(self): storage=self.__storage - return { + info = { 'length': len(storage), 'size': storage.getSize(), 'name': storage.getName(), - 'supportsUndo': storage.supportsUndo(), - 'supportsVersions': storage.supportsVersions(), - 'supportsTransactionalUndo': storage.supportsTransactionalUndo(), } + for feature in ('supportsUndo', + 'supportsVersions', + 'supportsTransactionalUndo',): + if hasattr(storage, feature): + info[feature] = getattr(storage, feature)() + else: + info[feature] = 0 + return info def get_size_info(self): storage=self.__storage From jeremy at digicool.com Tue May 22 10:15:05 2001 From: jeremy at digicool.com (jeremy@digicool.com) Date: Sun Aug 10 16:31:16 2008 Subject: [ZEO-Checkins] CVS: Packages/ZEO - zrpc2.py:1.1.2.14 Message-ID: <20010522141505.D71F5510E3@korak.digicool.com> Update of /cvs-repository/Packages/ZEO In directory korak.digicool.com:/tmp/cvs-serv6951 Modified Files: Tag: ZEO-ZRPC-Dev zrpc2.py Log Message: Enhanced several aspects of logging new_label(): New function that resets the label used with log messages, useful after a fork. __repr__(): Added to ClientStorage. --- Updated File zrpc2.py in package Packages/ZEO -- --- zrpc2.py 2001/05/07 13:11:56 1.1.2.13 +++ zrpc2.py 2001/05/22 14:15:04 1.1.2.14 @@ -41,10 +41,15 @@ REPLY = ".reply" # message name used for replies ASYNC = 1 -def log(message, level=zeolog.BLATHER, label="zrpc:%s" % os.getpid(), - error=None): - zeolog.LOG(label, level, message, error=error) +_label = "zrpc:%s" % os.getpid() +def new_label(): + global _label + _label = "zrpc:%s" % os.getpid() + +def log(message, level=zeolog.BLATHER, label=None, error=None): + zeolog.LOG(label or _label, level, message, error=error) + class ZRPCError(POSException.StorageError): pass @@ -154,6 +159,9 @@ else: self.set_caller = 0 + def __repr__(self): + return "<%s %s>" % (self.__class__.__name__, self.addr) + def close(self): if self.closed: return @@ -196,7 +204,8 @@ log("call %s%s on %s" % (name, repr(args)[:40], repr(self.obj)), zeolog.DEBUG) if not self.check_method(name): - raise ZRPCError("Invalid method name: %s" % name) + raise ZRPCError("Invalid method name: %s on %s" % (name, + `self.obj`)) meth = getattr(self.obj, name) try: @@ -248,7 +257,7 @@ self.handle_error("Exception raised during decoding") return if flags & ASYNC: - self.handle_error("Asynchronous call raised exception") + self.handle_error("Asynchronous call raised exception: %s" % self) return if type(err_value) is not types.InstanceType: err_value = err_type, err_value @@ -428,9 +437,10 @@ else: s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) s.connect(self.addr) - except socket.error: + except socket.error, msg: if self.debug: - log("Failed to connect to server", level=zeolog.DEBUG) + log("Failed to connect to server: %s" % msg, + level=zeolog.DEBUG) if repeat: t = self._wait(t) else: --- Updated File zrpc2.py in package Packages/ZEO -- --- zrpc2.py 2001/05/07 13:11:56 1.1.2.13 +++ zrpc2.py 2001/05/22 14:15:04 1.1.2.14 @@ -41,10 +41,15 @@ REPLY = ".reply" # message name used for replies ASYNC = 1 -def log(message, level=zeolog.BLATHER, label="zrpc:%s" % os.getpid(), - error=None): - zeolog.LOG(label, level, message, error=error) +_label = "zrpc:%s" % os.getpid() +def new_label(): + global _label + _label = "zrpc:%s" % os.getpid() + +def log(message, level=zeolog.BLATHER, label=None, error=None): + zeolog.LOG(label or _label, level, message, error=error) + class ZRPCError(POSException.StorageError): pass @@ -154,6 +159,9 @@ else: self.set_caller = 0 + def __repr__(self): + return "<%s %s>" % (self.__class__.__name__, self.addr) + def close(self): if self.closed: return @@ -196,7 +204,8 @@ log("call %s%s on %s" % (name, repr(args)[:40], repr(self.obj)), zeolog.DEBUG) if not self.check_method(name): - raise ZRPCError("Invalid method name: %s" % name) + raise ZRPCError("Invalid method name: %s on %s" % (name, + `self.obj`)) meth = getattr(self.obj, name) try: @@ -248,7 +257,7 @@ self.handle_error("Exception raised during decoding") return if flags & ASYNC: - self.handle_error("Asynchronous call raised exception") + self.handle_error("Asynchronous call raised exception: %s" % self) return if type(err_value) is not types.InstanceType: err_value = err_type, err_value @@ -428,9 +437,10 @@ else: s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) s.connect(self.addr) - except socket.error: + except socket.error, msg: if self.debug: - log("Failed to connect to server", level=zeolog.DEBUG) + log("Failed to connect to server: %s" % msg, + level=zeolog.DEBUG) if repeat: t = self._wait(t) else: From jeremy at digicool.com Tue May 22 14:05:44 2001 From: jeremy at digicool.com (jeremy@digicool.com) Date: Sun Aug 10 16:31:16 2008 Subject: [ZEO-Checkins] CVS: Packages/ZEO - zrpc.py:1.17 Message-ID: <20010522180544.59154510E3@korak.digicool.com> Update of /cvs-repository/Packages/ZEO In directory korak.digicool.com:/tmp/cvs-serv7551 Modified Files: zrpc.py Log Message: Add __closed attribute to asyncRPC object. __init__(): __closed = 0 connect(): In connecting thread, don't continue if __closed is true. close(): __closed = 1 --- Updated File zrpc.py in package Packages/ZEO -- --- zrpc.py 2001/05/09 18:27:24 1.16 +++ zrpc.py 2001/05/22 18:05:43 1.17 @@ -117,6 +117,7 @@ self._outOfBand=outOfBand self._tmin, self._tmax = tmin, tmax self._debug=debug + self.__closed = 0 l=allocate_lock() # Response lock used to wait for call results self.__la=l.acquire @@ -132,7 +133,7 @@ t=self._tmin connection = self._connection debug=self._debug - while 1: + while self.__closed == 0: if log_type: LOG(log_type, INFO, 'Trying to connect to server: %s' % `connection`) try: @@ -292,5 +293,6 @@ asyncRPC.inheritedAttribute('close')(self) self.aq_parent.notifyDisconnected(self) self.__r='E'+dump(sys.exc_info()[:2], 1) + self.__closed = 1 try: self.__lr() except: pass --- Updated File zrpc.py in package Packages/ZEO -- --- zrpc.py 2001/05/09 18:27:24 1.16 +++ zrpc.py 2001/05/22 18:05:43 1.17 @@ -117,6 +117,7 @@ self._outOfBand=outOfBand self._tmin, self._tmax = tmin, tmax self._debug=debug + self.__closed = 0 l=allocate_lock() # Response lock used to wait for call results self.__la=l.acquire @@ -132,7 +133,7 @@ t=self._tmin connection = self._connection debug=self._debug - while 1: + while self.__closed == 0: if log_type: LOG(log_type, INFO, 'Trying to connect to server: %s' % `connection`) try: @@ -292,5 +293,6 @@ asyncRPC.inheritedAttribute('close')(self) self.aq_parent.notifyDisconnected(self) self.__r='E'+dump(sys.exc_info()[:2], 1) + self.__closed = 1 try: self.__lr() except: pass From jeremy at digicool.com Tue May 22 14:15:56 2001 From: jeremy at digicool.com (jeremy@digicool.com) Date: Sun Aug 10 16:31:16 2008 Subject: [ZEO-Checkins] CVS: Packages/ZEO - testZEO.py:1.7 Message-ID: <20010522181556.0D48C510E3@korak.digicool.com> Update of /cvs-repository/Packages/ZEO/tests In directory korak.digicool.com:/tmp/cvs-serv7627 Modified Files: testZEO.py Log Message: tearDown(): Call delStorage() to clean up. --- Updated File testZEO.py in package Packages/ZEO -- --- testZEO.py 2001/05/09 18:58:00 1.6 +++ testZEO.py 2001/05/22 18:15:56 1.7 @@ -3,6 +3,7 @@ import asyncore import os import random +import sys import tempfile import time import types @@ -139,6 +140,7 @@ self.running = 0 self._server.close() os.waitpid(self._pid, 0) + self.delStorage() self.__super_tearDown() def checkLargeUpdate(self): @@ -165,6 +167,9 @@ __super_setUp = StorageTestBase.StorageTestBase.setUp __super_tearDown = StorageTestBase.StorageTestBase.tearDown + ports = range(29000, 30000, 10) # enough for 100 tests + random.shuffle(ports) + def setUp(self): """Start a ZEO server using a Unix domain socket @@ -174,7 +179,7 @@ self.running = 1 self.__fs_base = tempfile.mktemp() fs = FileStorage(self.__fs_base, create=1) - self.addr = '', random.randrange(2000, 3000) + self.addr = '', self.ports.pop() pid, exit = forker.start_zeo_server(fs, self.addr) self._pid = pid self._server = exit --- Updated File testZEO.py in package Packages/ZEO -- --- testZEO.py 2001/05/09 18:58:00 1.6 +++ testZEO.py 2001/05/22 18:15:56 1.7 @@ -3,6 +3,7 @@ import asyncore import os import random +import sys import tempfile import time import types @@ -139,6 +140,7 @@ self.running = 0 self._server.close() os.waitpid(self._pid, 0) + self.delStorage() self.__super_tearDown() def checkLargeUpdate(self): @@ -165,6 +167,9 @@ __super_setUp = StorageTestBase.StorageTestBase.setUp __super_tearDown = StorageTestBase.StorageTestBase.tearDown + ports = range(29000, 30000, 10) # enough for 100 tests + random.shuffle(ports) + def setUp(self): """Start a ZEO server using a Unix domain socket @@ -174,7 +179,7 @@ self.running = 1 self.__fs_base = tempfile.mktemp() fs = FileStorage(self.__fs_base, create=1) - self.addr = '', random.randrange(2000, 3000) + self.addr = '', self.ports.pop() pid, exit = forker.start_zeo_server(fs, self.addr) self._pid = pid self._server = exit From jeremy at digicool.com Tue May 29 22:04:17 2001 From: jeremy at digicool.com (jeremy@digicool.com) Date: Sun Aug 10 16:31:16 2008 Subject: [ZEO-Checkins] CVS: Packages/ZEO - smac.py:1.10 Message-ID: <20010530020417.88450510E3@korak.digicool.com> Update of /cvs-repository/Packages/ZEO In directory korak.digicool.com:/tmp/cvs-serv9644 Modified Files: smac.py Log Message: cosmetic cleanup of exception raising -- 4 lines down to 1 --- Updated File smac.py in package Packages/ZEO -- --- smac.py 2000/10/06 17:11:25 1.9 +++ smac.py 2001/05/30 02:04:16 1.10 @@ -181,10 +181,7 @@ append=self.__append if append is None: - raise Disconnected, ( - "This action is temporarily unavailable." - "

" - ) + raise Disconnected("This action is temporarily unavailable.

") append(pack(">i",len(message))+message) --- Updated File smac.py in package Packages/ZEO -- --- smac.py 2000/10/06 17:11:25 1.9 +++ smac.py 2001/05/30 02:04:16 1.10 @@ -181,10 +181,7 @@ append=self.__append if append is None: - raise Disconnected, ( - "This action is temporarily unavailable." - "

" - ) + raise Disconnected("This action is temporarily unavailable.

") append(pack(">i",len(message))+message) From jeremy at digicool.com Wed May 30 17:51:47 2001 From: jeremy at digicool.com (jeremy@digicool.com) Date: Sun Aug 10 16:31:16 2008 Subject: [ZEO-Checkins] CVS: Packages/ZEO - zrpc.py:1.18 Message-ID: <20010530215147.32339510E3@korak.digicool.com> Update of /cvs-repository/Packages/ZEO In directory korak.digicool.com:/tmp/cvs-serv16035 Modified Files: zrpc.py Log Message: Set self.__closed in closeIntensionally() rather than close(). closeIntensionally() is called when the storage's close() method is called. close() is called when the other end reset the connection. The __closed should only be set with the asyncRPC object should never re-open the connection. Reported by Dyon Balding. --- Updated File zrpc.py in package Packages/ZEO -- --- zrpc.py 2001/05/22 18:05:43 1.17 +++ zrpc.py 2001/05/30 21:51:46 1.18 @@ -287,12 +287,15 @@ # We aren't willing to close until told to by the main loop. # So we'll tell the main loop to tell us. :) self.__Wakeup(lambda self=self: self.close()) - else: self.close() + else: + self.close() + self.__closed = 1 def close(self): asyncRPC.inheritedAttribute('close')(self) self.aq_parent.notifyDisconnected(self) + # causes read call to raise last exception, which should be + # the socket error that caused asyncore to close the socket. self.__r='E'+dump(sys.exc_info()[:2], 1) - self.__closed = 1 try: self.__lr() except: pass --- Updated File zrpc.py in package Packages/ZEO -- --- zrpc.py 2001/05/22 18:05:43 1.17 +++ zrpc.py 2001/05/30 21:51:46 1.18 @@ -287,12 +287,15 @@ # We aren't willing to close until told to by the main loop. # So we'll tell the main loop to tell us. :) self.__Wakeup(lambda self=self: self.close()) - else: self.close() + else: + self.close() + self.__closed = 1 def close(self): asyncRPC.inheritedAttribute('close')(self) self.aq_parent.notifyDisconnected(self) + # causes read call to raise last exception, which should be + # the socket error that caused asyncore to close the socket. self.__r='E'+dump(sys.exc_info()[:2], 1) - self.__closed = 1 try: self.__lr() except: pass From jeremy at digicool.com Wed May 30 22:58:24 2001 From: jeremy at digicool.com (jeremy@digicool.com) Date: Sun Aug 10 16:31:16 2008 Subject: [ZEO-Checkins] CVS: Packages/ZEO - testZEO.py:1.8 Message-ID: <20010531025824.E297F510E3@korak.digicool.com> Update of /cvs-repository/Packages/ZEO/tests In directory korak.digicool.com:/tmp/cvs-serv17015 Modified Files: testZEO.py Log Message: Add new test to verify that client reconnects to restarted server. Add checkReconnection() to renamed ConnectionTests (was PersistentCacheTests). Close the client storage explicitly in the tear down to avoid spurious warnings from threads attempting to connect after the test is over. --- Updated File testZEO.py in package Packages/ZEO -- --- testZEO.py 2001/05/22 18:15:56 1.7 +++ testZEO.py 2001/05/31 02:58:23 1.8 @@ -138,6 +138,7 @@ def tearDown(self): """Try to cause the tests to halt""" self.running = 0 + self._storage.close() self._server.close() os.waitpid(self._pid, 0) self.delStorage() @@ -162,8 +163,14 @@ for ext in '', '.index', '.lock', '.tmp': path = self.__fs_base + ext os.unlink(path) + +class ConnectionTests(ZEOTestBase): + """Tests that explicitly manage the server process. -class PersistentCacheTests(ZEOTestBase): + To test the cache or re-connection, these test cases explicit + start and stop a ZEO storage server. + """ + __super_setUp = StorageTestBase.StorageTestBase.setUp __super_tearDown = StorageTestBase.StorageTestBase.tearDown @@ -178,14 +185,17 @@ """ self.running = 1 self.__fs_base = tempfile.mktemp() - fs = FileStorage(self.__fs_base, create=1) - self.addr = '', self.ports.pop() - pid, exit = forker.start_zeo_server(fs, self.addr) + self.addr = '', random.randrange(2000, 3000) + pid, exit = self._startServer() self._pid = pid self._server = exit self.__super_setUp() + + def _startServer(self, create=1): + fs = FileStorage(self.__fs_base, create=create) + return forker.start_zeo_server(fs, self.addr) - def openClientStorage(self, cache, cache_size, wait): + def openClientStorage(self, cache='', cache_size=200000, wait=1): base = ZEO.ClientStorage.ClientStorage(self.addr, client=cache, cache_size=cache_size, @@ -251,6 +261,27 @@ self._storage.load(oid2, '') self.assertRaises(Disconnected, self._storage.load, oid1, '') + def checkReconnection(self): + """Check that the client reconnects when a server restarts.""" + + from ZEO.ClientStorage import ClientDisconnected + self._storage = self.openClientStorage() + oid = self._storage.new_oid() + obj = MinPO(12) + revid1 = self._dostore(oid, data=obj) + self.shutdownServer() + self.running = 1 + self._pid, self._server = self._startServer(create=0) + oid = self._storage.new_oid() + obj = MinPO(12) + while 1: + try: + revid1 = self._dostore(oid, data=obj) + except ClientDisconnected: + time.sleep(0.1) + else: + break + def get_methods(klass): l = [klass] meth = {} @@ -266,7 +297,7 @@ def makeTestSuite(testname=''): suite = unittest.TestSuite() name = 'check' + testname - for klass in ZEOFileStorageTests, PersistentCacheTests: + for klass in ZEOFileStorageTests, ConnectionTests: for meth in get_methods(klass): if meth.startswith(name): suite.addTest(klass(meth)) @@ -283,7 +314,7 @@ name_of_test = val if args: - print >> sys.stderr, "Did not expect arguments. Got %s" % args + print "Did not expect arguments. Got %s" % args return 0 tests = makeTestSuite(name_of_test) --- Updated File testZEO.py in package Packages/ZEO -- --- testZEO.py 2001/05/22 18:15:56 1.7 +++ testZEO.py 2001/05/31 02:58:23 1.8 @@ -138,6 +138,7 @@ def tearDown(self): """Try to cause the tests to halt""" self.running = 0 + self._storage.close() self._server.close() os.waitpid(self._pid, 0) self.delStorage() @@ -162,8 +163,14 @@ for ext in '', '.index', '.lock', '.tmp': path = self.__fs_base + ext os.unlink(path) + +class ConnectionTests(ZEOTestBase): + """Tests that explicitly manage the server process. -class PersistentCacheTests(ZEOTestBase): + To test the cache or re-connection, these test cases explicit + start and stop a ZEO storage server. + """ + __super_setUp = StorageTestBase.StorageTestBase.setUp __super_tearDown = StorageTestBase.StorageTestBase.tearDown @@ -178,14 +185,17 @@ """ self.running = 1 self.__fs_base = tempfile.mktemp() - fs = FileStorage(self.__fs_base, create=1) - self.addr = '', self.ports.pop() - pid, exit = forker.start_zeo_server(fs, self.addr) + self.addr = '', random.randrange(2000, 3000) + pid, exit = self._startServer() self._pid = pid self._server = exit self.__super_setUp() + + def _startServer(self, create=1): + fs = FileStorage(self.__fs_base, create=create) + return forker.start_zeo_server(fs, self.addr) - def openClientStorage(self, cache, cache_size, wait): + def openClientStorage(self, cache='', cache_size=200000, wait=1): base = ZEO.ClientStorage.ClientStorage(self.addr, client=cache, cache_size=cache_size, @@ -251,6 +261,27 @@ self._storage.load(oid2, '') self.assertRaises(Disconnected, self._storage.load, oid1, '') + def checkReconnection(self): + """Check that the client reconnects when a server restarts.""" + + from ZEO.ClientStorage import ClientDisconnected + self._storage = self.openClientStorage() + oid = self._storage.new_oid() + obj = MinPO(12) + revid1 = self._dostore(oid, data=obj) + self.shutdownServer() + self.running = 1 + self._pid, self._server = self._startServer(create=0) + oid = self._storage.new_oid() + obj = MinPO(12) + while 1: + try: + revid1 = self._dostore(oid, data=obj) + except ClientDisconnected: + time.sleep(0.1) + else: + break + def get_methods(klass): l = [klass] meth = {} @@ -266,7 +297,7 @@ def makeTestSuite(testname=''): suite = unittest.TestSuite() name = 'check' + testname - for klass in ZEOFileStorageTests, PersistentCacheTests: + for klass in ZEOFileStorageTests, ConnectionTests: for meth in get_methods(klass): if meth.startswith(name): suite.addTest(klass(meth)) @@ -283,7 +314,7 @@ name_of_test = val if args: - print >> sys.stderr, "Did not expect arguments. Got %s" % args + print "Did not expect arguments. Got %s" % args return 0 tests = makeTestSuite(name_of_test)