From jeremy at zope.com Wed Jan 2 17:52:48 2002 From: jeremy at zope.com (Jeremy Hylton) Date: Sun Aug 10 16:31:17 2008 Subject: [ZEO-Checkins] CVS: StandaloneZODB/ZEO - smac.py:1.9.6.7 Message-ID: <200201022252.g02MqmG06165@cvs.baymountain.com> Update of /cvs-repository/StandaloneZODB/ZEO In directory cvs.zope.org:/tmp/cvs-serv6151 Modified Files: Tag: ZEO-ZRPC-Dev smac.py Log Message: Backport changes from ZEO 1.0 to make send/recv robust. === StandaloneZODB/ZEO/smac.py 1.9.6.6 => 1.9.6.7 === from types import StringType +import socket, errno + +# Use the dictionary to make sure we get the minimum number of errno +# entries. We expect that EWOULDBLOCK == EAGAIN on most systems -- +# or that only one is actually used. + +tmp_dict = {errno.EWOULDBLOCK: 0, + errno.EAGAIN: 0, + errno.EINTR: 0, + } +expected_socket_read_errors = tuple(tmp_dict.keys()) + +tmp_dict = {errno.EAGAIN: 0, + errno.EWOULDBLOCK: 0, + errno.ENOBUFS: 0, + errno.EINTR: 0, + } +expected_socket_write_errors = tuple(tmp_dict.keys()) +del tmp_dict + class SizedMessageAsyncConnection(asyncore.dispatcher): __super_init = asyncore.dispatcher.__init__ __super_close = asyncore.dispatcher.close @@ -123,7 +143,12 @@ def handle_read(self): # Use a single __inp buffer and integer indexes to make this # fast. - d = self.recv(self.READ_SIZE) + try: + d=self.recv(8096) + except socket.error, err: + if err[0] in expected_socket_read_errors: + return + raise if not d: return @@ -182,7 +207,12 @@ output = self.__output while output: v = output[0] - n = self.send(v) + try: + n=self.send(v) + except socket.error, err: + if err[0] in expected_socket_write_errors: + break # we couldn't write anything + raise if n < len(v): output[0] = v[n:] break # we can't write any more From jeremy at zope.com Thu Jan 3 17:29:20 2002 From: jeremy at zope.com (Jeremy Hylton) Date: Sun Aug 10 16:31:17 2008 Subject: [ZEO-Checkins] CVS: StandaloneZODB/ZEO - ClientStorage.py:1.26.4.24 Message-ID: <200201032229.g03MTKf29463@cvs.baymountain.com> Update of /cvs-repository/StandaloneZODB/ZEO In directory cvs.zope.org:/tmp/cvs-serv29452 Modified Files: Tag: ZEO-ZRPC-Dev ClientStorage.py Log Message: Don't assign to self._server until the end of notifyConnected(). No other thread should be able to use self._server until cache verification is complete. === StandaloneZODB/ZEO/ClientStorage.py 1.26.4.23 => 1.26.4.24 === def notifyConnected(self, c): log2(INFO, "Connected to storage") - self._server = ServerStub.StorageServer(c) + stub = ServerStub.StorageServer(c) self._oids = [] - self._server.register(str(self._storage)) - self.verify_cache() + stub.register(str(self._storage)) + self.verify_cache(stub) - def verify_cache(self): - self._server.beginZeoVerify() - self._cache.verify(self._server.zeoVerify) - self._server.endZeoVerify() + # Don't make the server available to clients until after + # validating the cache + self._server = stub + + def verify_cache(self, server): + server.beginZeoVerify() + self._cache.verify(server.zeoVerify) + server.endZeoVerify() ### Is there a race condition between notifyConnected and ### notifyDisconnected? In Particular, what if we get From jeremy at zope.com Thu Jan 3 17:35:26 2002 From: jeremy at zope.com (Jeremy Hylton) Date: Sun Aug 10 16:31:17 2008 Subject: [ZEO-Checkins] CVS: StandaloneZODB/ZEO - zrpc2.py:1.1.2.16 Message-ID: <200201032235.g03MZQu30825@cvs.baymountain.com> Update of /cvs-repository/StandaloneZODB/ZEO In directory cvs.zope.org:/tmp/cvs-serv30814 Modified Files: Tag: ZEO-ZRPC-Dev zrpc2.py Log Message: Fix problems with reconnecting to server and shutdown. _thread = None not __thread = None!!! The __connect() thread must assign None to _thread when it is done in order to allow future reconnection attempts to succeed. Exit __connect() thread if the manager is closed. In close() method, if there is a helper thread attempting to connect to server, wait() for it before returning. Add minimal doc string for __connect(). Change poorly named handle_error(). This method name is reserved by asyncore, but was also being used to log error messages in the rpc. Add a log_error() and call it from, the rpc code. Add a new handle_error() that calls log_error(). === StandaloneZODB/ZEO/zrpc2.py 1.1.2.15 => 1.1.2.16 === self.send_reply(msgid, ret) - def handle_error(self, msg="No error message supplied"): + def handle_error(self): + self.log_error() + self.close() + + def log_error(self, msg="No error message supplied"): error = sys.exc_info() log(msg, zeolog.ERROR, error=error) + del error def check_method(self, name): # XXX minimal security check should go here: Is name exported? @@ -255,10 +260,10 @@ def return_error(self, msgid, flags, err_type, err_value): if flags is None: - self.handle_error("Exception raised during decoding") + self.log_error("Exception raised during decoding") return if flags & ASYNC: - self.handle_error("Asynchronous call raised exception: %s" % self) + self.log_error("Asynchronous call raised exception: %s" % self) return if type(err_value) is not types.InstanceType: err_value = err_type, err_value @@ -396,9 +401,18 @@ self.closed = 0 ThreadedAsync.register_loop_callback(self.set_async) + def __repr__(self): + return "<%s for %s>" % (self.__class__.__name__, self.addr) + def close(self): """Prevent ConnectionManager from opening new connections""" self.closed = 1 + self._connect_lock.acquire() + try: + if self._thread is not None: + self._thread.join() + finally: + self._connect_lock.release() def register_object(self, obj): self.obj = obj @@ -419,7 +433,7 @@ self._thread.start() finally: self._connect_lock.release() - if sync: + if sync and self._thread is not None: self._thread.join() def attempt_connect(self): @@ -427,9 +441,16 @@ return self.connected def __connect(self, repeat=1): + """Attempt to connect to StorageServer. + + This method should always be called by attempt_connect() or by + connect(). + """ + tries = 0 t = self.tmin - while not self.connected and (repeat or (tries == 0)): + while not (self.connected or self.closed) \ + and (repeat or (tries == 0)): tries = tries + 1 log("Trying to connect to server") try: @@ -448,11 +469,11 @@ if self.debug: log("Connected to server", level=zeolog.DEBUG) self.connected = 1 - if self.connected: + if self.connected and not self.closed: c = ManagedConnection(s, self.addr, self.obj, self) log("Connection created: %s" % c) self.obj.notifyConnected(c) - self.__thread = None + self._thread = None def _wait(self, t): time.sleep(t) From jeremy at zope.com Thu Jan 3 19:14:18 2002 From: jeremy at zope.com (Jeremy Hylton) Date: Sun Aug 10 16:31:17 2008 Subject: [ZEO-Checkins] CVS: StandaloneZODB/ZEO/tests - forker.py:1.1.2.6 Message-ID: <200201040014.g040EIu22523@cvs.baymountain.com> Update of /cvs-repository/StandaloneZODB/ZEO/tests In directory cvs.zope.org:/tmp/cvs-serv22512/tests Modified Files: Tag: ZEO-ZRPC-Dev forker.py Log Message: Ignore os.error when attempting to shutdown a failed StorageServer. Print a nicer error message when a StorageServer fails unexpectedly. Don't let the exception get back to unittest, because the fork() confuses test runnner and it will continue trying to run tests. === StandaloneZODB/ZEO/tests/forker.py 1.1.2.5 => 1.1.2.6 === import socket import sys +import traceback import types import ZEO.ClientStorage, ZEO.StorageServer @@ -86,20 +87,27 @@ self.pipe = pipe def close(self): - os.write(self.pipe, "done") - os.close(self.pipe) + try: + os.write(self.pipe, "done") + os.close(self.pipe) + except os.error: + pass def start_zeo_server(storage, addr): rd, wr = os.pipe() pid = os.fork() if pid == 0: - if PROFILE: - p = profile.Profile() - p.runctx("run_server(storage, addr, rd, wr)", globals(), - locals()) - p.dump_stats("stats.s.%d" % os.getpid()) - else: - run_server(storage, addr, rd, wr) + try: + if PROFILE: + p = profile.Profile() + p.runctx("run_server(storage, addr, rd, wr)", globals(), + locals()) + p.dump_stats("stats.s.%d" % os.getpid()) + else: + run_server(storage, addr, rd, wr) + except: + print "Exception in ZEO server process" + traceback.print_exc() os._exit(0) else: os.close(rd) From jeremy at zope.com Thu Jan 3 19:15:19 2002 From: jeremy at zope.com (Jeremy Hylton) Date: Sun Aug 10 16:31:17 2008 Subject: [ZEO-Checkins] CVS: StandaloneZODB/ZEO - ClientStorage.py:1.26.4.25 Message-ID: <200201040015.g040FJE22646@cvs.baymountain.com> Update of /cvs-repository/StandaloneZODB/ZEO In directory cvs.zope.org:/tmp/cvs-serv22635 Modified Files: Tag: ZEO-ZRPC-Dev ClientStorage.py Log Message: Raise a ClientDisconnected in tpc_begin() if necessary. Remove an XXX comment that discusses a future design issue. === StandaloneZODB/ZEO/ClientStorage.py 1.26.4.24 => 1.26.4.25 === def tpc_begin(self, transaction): - # XXX plan is to have begin be a local operation until the - # vote stage. self.tpc_cond.acquire() while self._transaction is not None: if self._transaction == transaction: self.tpc_cond.release() return self.tpc_cond.wait() + + if self._server is None: + self.tpc_cond.release() + raise ClientDisconnected() self._ts = get_timestamp(self._ts) id = `self._ts` From jeremy at zope.com Thu Jan 3 19:18:54 2002 From: jeremy at zope.com (Jeremy Hylton) Date: Sun Aug 10 16:31:17 2008 Subject: [ZEO-Checkins] CVS: StandaloneZODB/ZEO - zrpc2.py:1.1.2.17 Message-ID: <200201040018.g040Is223698@cvs.baymountain.com> Update of /cvs-repository/StandaloneZODB/ZEO In directory cvs.zope.org:/tmp/cvs-serv23687 Modified Files: Tag: ZEO-ZRPC-Dev zrpc2.py Log Message: Three sets of changes. In a block _do_io() call, raise Disconnected() if the connection is closed. Refactor __connect() to put some of the socket logic in a separate method. _connect_socket() returns either a connected socket or None. Put __connect() in a try/finally that always clears _thread. Add readable() and writable() implementations to Connection() that prevent a closed connection from getting in a socket map. === StandaloneZODB/ZEO/zrpc2.py 1.1.2.16 => 1.1.2.17 === import zeolog import ThreadedAsync +from Exceptions import Disconnected REPLY = ".reply" # message name used for replies ASYNC = 1 @@ -140,6 +141,7 @@ """ __super_init = smac.SizedMessageAsyncConnection.__init__ __super_close = smac.SizedMessageAsyncConnection.close + __super_writable = smac.SizedMessageAsyncConnection.writable def __init__(self, sock, addr, obj=None, pickle=None): self.msgid = 0 @@ -163,6 +165,16 @@ def __repr__(self): return "<%s %s>" % (self.__class__.__name__, self.addr) + # XXX are the readable() and writable() methods necessary? + + def readable(self): + return not self.closed + + def writable(self): + if self.closed: + return 0 + return self.__super_writable() + def close(self): if self.closed: return @@ -366,7 +378,9 @@ if wait: # do loop only if lock is already acquired while not self.__reply_lock.acquire(0): - asyncore.poll(60.0, self._map) + asyncore.poll(10.0, self._map) + if self.closed: + raise Disconnected() self.__reply_lock.release() else: asyncore.poll(0.0, self._map) @@ -394,6 +408,8 @@ self.tmax = tmax self.debug = debug self.connected = 0 + # If _thread is not None, then there is a helper thread + # attempting to connect. _thread is protected by _connect_lock. self._thread = None self._connect_lock = threading.Lock() self.trigger = None @@ -446,34 +462,51 @@ This method should always be called by attempt_connect() or by connect(). """ - - tries = 0 - t = self.tmin - while not (self.connected or self.closed) \ - and (repeat or (tries == 0)): - tries = tries + 1 - log("Trying to connect to server") - try: - if type(self.addr) is types.StringType: - s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + + try: + tries = 0 + t = self.tmin + while not (self.connected or self.closed) \ + and (repeat or (tries == 0)): + tries = tries + 1 + print self, tries, self.closed + log("Trying to connect to server") + s = self._connect_socket() + if s is None: + if repeat: + t = self._wait(t) else: - s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - s.connect(self.addr) - except socket.error, msg: - if self.debug: - log("Failed to connect to server: %s" % msg, - level=zeolog.DEBUG) - if repeat: - t = self._wait(t) + if self.debug: + log("Connected to server", level=zeolog.DEBUG) + self.connected = 1 + if self.connected and not self.closed: + print "connected" + c = ManagedConnection(s, self.addr, self.obj, self) + log("Connection created: %s" % c) + try: + self.obj.notifyConnected(c) + except: + # XXX + c.close() + raise + finally: + # must always clear _thread on the way out + self._thread = None + + def _connect_socket(self): + try: + if type(self.addr) is types.StringType: + s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) else: - if self.debug: - log("Connected to server", level=zeolog.DEBUG) - self.connected = 1 - if self.connected and not self.closed: - c = ManagedConnection(s, self.addr, self.obj, self) - log("Connection created: %s" % c) - self.obj.notifyConnected(c) - self._thread = None + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + s.connect(self.addr) + except socket.error, msg: + if self.debug: + log("Failed to connect to server: %s" % msg, + level=zeolog.DEBUG) + s.close() + return None + return s def _wait(self, t): time.sleep(t) From jeremy at zope.com Thu Jan 3 19:19:48 2002 From: jeremy at zope.com (Jeremy Hylton) Date: Sun Aug 10 16:31:17 2008 Subject: [ZEO-Checkins] CVS: StandaloneZODB/ZEO - zrpc2.py:1.1.2.18 Message-ID: <200201040019.g040Jmp23821@cvs.baymountain.com> Update of /cvs-repository/StandaloneZODB/ZEO In directory cvs.zope.org:/tmp/cvs-serv23810 Modified Files: Tag: ZEO-ZRPC-Dev zrpc2.py Log Message: Remove debugging prints. === StandaloneZODB/ZEO/zrpc2.py 1.1.2.17 => 1.1.2.18 === and (repeat or (tries == 0)): tries = tries + 1 - print self, tries, self.closed log("Trying to connect to server") s = self._connect_socket() if s is None: @@ -480,7 +479,6 @@ log("Connected to server", level=zeolog.DEBUG) self.connected = 1 if self.connected and not self.closed: - print "connected" c = ManagedConnection(s, self.addr, self.obj, self) log("Connection created: %s" % c) try: From jeremy at zope.com Thu Jan 3 19:35:06 2002 From: jeremy at zope.com (Jeremy Hylton) Date: Sun Aug 10 16:31:17 2008 Subject: [ZEO-Checkins] CVS: StandaloneZODB/ZEO - zrpc2.py:1.1.2.19 Message-ID: <200201040035.g040Z6t27320@cvs.baymountain.com> Update of /cvs-repository/StandaloneZODB/ZEO In directory cvs.zope.org:/tmp/cvs-serv27309 Modified Files: Tag: ZEO-ZRPC-Dev zrpc2.py Log Message: readable() and writable() aren't necessary after all. === StandaloneZODB/ZEO/zrpc2.py 1.1.2.18 => 1.1.2.19 === return "<%s %s>" % (self.__class__.__name__, self.addr) - # XXX are the readable() and writable() methods necessary? - - def readable(self): - return not self.closed - - def writable(self): - if self.closed: - return 0 - return self.__super_writable() - def close(self): if self.closed: return From jeremy at zope.com Thu Jan 3 19:43:21 2002 From: jeremy at zope.com (Jeremy Hylton) Date: Sun Aug 10 16:31:17 2008 Subject: [ZEO-Checkins] CVS: StandaloneZODB/ZEO/tests - testZEO.py:1.1.2.15 Message-ID: <200201040043.g040hLq29073@cvs.baymountain.com> Update of /cvs-repository/StandaloneZODB/ZEO/tests In directory cvs.zope.org:/tmp/cvs-serv29062 Modified Files: Tag: ZEO-ZRPC-Dev testZEO.py Log Message: Several changes: Change doc strings to comments so that unittest prints the name of the test method rather than the first line of the doc string. Change checkReconnection() so that it catches the currently thrown errors. Ignore waitpid errors during cleanup. === StandaloneZODB/ZEO/tests/testZEO.py 1.1.2.14 => 1.1.2.15 === import os import random +import select import socket import sys import tempfile @@ -277,19 +278,19 @@ self.__super_tearDown() def checkDisconnectionError(self): - """Make sure we get a Disconnected when we try to read an - object when we're not connected to a storage server and the - object is not in the cache. """ + # Make sure we get a Disconnected when we try to read an + # object when we're not connected to a storage server and the + # object is not in the cache. self.shutdownServer() self._storage = self.openClientStorage('test', 1000, 0) self.assertRaises(Disconnected, self._storage.load, 'fredwash', '') def checkBasicPersistence(self): - """Verify cached data persists across client storage instances. + # Verify cached data persists across client storage instances. + + # To verify that the cache is being used, the test closes the + # server and then starts a new client with the server down. - To verify that the cache is being used, the test closes the - server and then starts a new client with the server down. - """ self._storage = self.openClientStorage('test', 100000, 1) oid = self._storage.new_oid() obj = MinPO(12) @@ -303,11 +304,11 @@ self._storage.close() def checkRollover(self): - """Check that the cache works when the files are swapped. + # Check that the cache works when the files are swapped. - In this case, only one object fits in a cache file. When the - cache files swap, the first object is effectively uncached. - """ + # In this case, only one object fits in a cache file. When the + # cache files swap, the first object is effectively uncached. + self._storage = self.openClientStorage('test', 1000, 1) oid1 = self._storage.new_oid() obj1 = MinPO("1" * 500) @@ -322,9 +323,8 @@ self._storage.load(oid2, '') def checkReconnection(self): - """Check that the client reconnects when a server restarts.""" + # Check that the client reconnects when a server restarts. - from ZEO.ClientStorage import ClientDisconnected self._storage = self.openClientStorage() oid = self._storage.new_oid() obj = MinPO(12) @@ -337,11 +337,10 @@ while 1: try: revid1 = self._dostore(oid, data=obj) - except (ClientDisconnected, thread.error, socket.error), err: - get_transaction().abort() - time.sleep(0.1) - else: break + except (Disconnected, select.error, thread.error, socket.error), err: + get_transaction().abort() + time.sleep(0.1) # XXX how long to sleep # XXX This is a bloody pain. We're placing a heavy burden # on users to catch a plethora of exceptions in order to # write robust code. Need to think about implementing @@ -380,7 +379,10 @@ if self.running: self.running = 0 self._server.close() - os.waitpid(self._pid, 0) + try: + os.waitpid(self._pid, 0) + except os.error: + pass class WindowsConnectionTests(ConnectionTests): __super_setUp = StorageTestBase.StorageTestBase.setUp From jeremy at zope.com Thu Jan 3 19:48:41 2002 From: jeremy at zope.com (Jeremy Hylton) Date: Sun Aug 10 16:31:17 2008 Subject: [ZEO-Checkins] CVS: StandaloneZODB/ZEO - ClientCache.py:1.14.4.6 ClientStorage.py:1.26.4.26 StorageServer.py:1.21.4.14 zrpc2.py:1.1.2.20 Message-ID: <200201040048.g040mfA30690@cvs.baymountain.com> Update of /cvs-repository/StandaloneZODB/ZEO In directory cvs.zope.org:/tmp/cvs-serv30673 Modified Files: Tag: ZEO-ZRPC-Dev ClientCache.py ClientStorage.py StorageServer.py zrpc2.py Log Message: Restore use of zLOG instead of zeolog. The performance issues that lead me to use zeolog in the first place have been fixed in zLOG. Also add helpful comment about where exception came from in zrpc2. === StandaloneZODB/ZEO/ClientCache.py 1.14.4.5 => 1.14.4.6 === import sys -import zeolog +import zLOG -def log(msg, level=zeolog.INFO): - zeolog.LOG("ZEC", level, msg) +def log(msg, level=zLOG.INFO): + zLOG.LOG("ZEC", level, msg) magic='ZEC0' === StandaloneZODB/ZEO/ClientStorage.py 1.26.4.25 => 1.26.4.26 === from ZODB import POSException from ZODB.TimeStamp import TimeStamp -from zeolog import LOG, PROBLEM, INFO, BLATHER +from zLOG import LOG, PROBLEM, INFO, BLATHER from Exceptions import Disconnected def log2(type, msg, subsys="ClientStorage %d" % os.getpid()): === StandaloneZODB/ZEO/StorageServer.py 1.21.4.13 => 1.21.4.14 === import ClientStub import zrpc2 -import zeolog +import zLOG from zrpc2 import Dispatcher, Handler, ManagedServerConnection, Delay from ZODB.POSException import StorageError, StorageTransactionError, \ @@ -114,9 +114,9 @@ pickler.fast = 1 # Don't use the memo dump = pickler.dump -def log(message, level=zeolog.INFO, label="ZEO Server:%s" % os.getpid(), +def log(message, level=zLOG.INFO, label="ZEO Server:%s" % os.getpid(), error=None): - zeolog.LOG(label, level, message, error=error) + zLOG.LOG(label, level, message, error=error) class StorageServerError(StorageError): pass @@ -180,8 +180,8 @@ return "" % (id(self), tid, stid) - def _log(self, msg, level=zeolog.INFO, error=None, pid=os.getpid()): - zeolog.LOG("ZEO Server %s %X" % (pid, id(self)), + def _log(self, msg, level=zLOG.INFO, error=None, pid=os.getpid()): + zLOG.LOG("ZEO Server %s %X" % (pid, id(self)), level, msg, error=error) def setup_delegation(self): @@ -198,7 +198,7 @@ caller = sys._getframe().f_back.f_code.co_name if self._transaction is None: self._log("no current transaction: %s()" % caller, - zeolog.PROBLEM) + zLOG.PROBLEM) if exc is not None: raise exc(None, tid) else: @@ -206,7 +206,7 @@ if self._transaction.id != tid: self._log("%s(%s) invalid; current transaction = %s" % \ (caller, repr(tid), repr(self._transaction.id)), - zeolog.PROBLEM) + zLOG.PROBLEM) if exc is not None: raise exc(self._transaction.id, tid) else: @@ -285,7 +285,7 @@ try: self.__storage.pack(t, referencesf) except: - self._log('ZEO Server', zeolog.ERROR, + self._log('ZEO Server', zLOG.ERROR, 'Pack failed for %s' % self.__storage_id, error=sys.exc_info()) if wait: @@ -331,7 +331,7 @@ # IOW, Anything that ends up here is evil enough to be logged. error = sys.exc_info() self._log('store error: %s: %s' % (error[0], error[1]), - zeolog.ERROR, error=error) + zLOG.ERROR, error=error) newserial = sys.exc_info()[1] else: if serial != '\0\0\0\0\0\0\0\0': @@ -341,7 +341,7 @@ nil = dump(newserial, 1) except: self._log("couldn't pickle newserial: %s" % repr(newserial), - zeolog.ERROR) + zLOG.ERROR) dump('', 1) # clear pickler r = StorageServerError("Couldn't pickle exception %s" % \ `newserial`) === StandaloneZODB/ZEO/zrpc2.py 1.1.2.19 => 1.1.2.20 === from ZEO import smac, trigger from Exceptions import Disconnected -import zeolog +import zLOG import ThreadedAsync from Exceptions import Disconnected @@ -49,8 +49,8 @@ global _label _label = "zrpc:%s" % os.getpid() -def log(message, level=zeolog.BLATHER, label=None, error=None): - zeolog.LOG(label or _label, level, message, error=error) +def log(message, level=zLOG.BLATHER, label=None, error=None): + zLOG.LOG(label or _label, level, message, error=error) class ZRPCError(POSException.StorageError): pass @@ -107,7 +107,7 @@ return unpickler.load() # msgid, flags, name, args msgid, flags, name, args = unpickler.load() except (cPickle.UnpicklingError, IndexError), err_msg: - log("can't decode %s" % repr(msg), level=zeolog.ERROR) + log("can't decode %s" % repr(msg), level=zLOG.ERROR) raise DecodingError(msg) class Delay: @@ -189,7 +189,7 @@ if __debug__: log("recv msg: %s, %s, %s, %s" % (msgid, flags, name, repr(args)[:40]), - level=zeolog.DEBUG) + level=zLOG.DEBUG) if name == REPLY: self.handle_reply(msgid, flags, args) else: @@ -198,14 +198,14 @@ def handle_reply(self, msgid, flags, args): if __debug__: log("recv reply: %s, %s, %s" % (msgid, flags, str(args)[:40]), - level=zeolog.DEBUG) + level=zLOG.DEBUG) self.__reply = msgid, flags, args self.__reply_lock.release() # will fail if lock is unlocked def handle_request(self, msgid, flags, name, args): if __debug__: log("call %s%s on %s" % (name, repr(args)[:40], repr(self.obj)), - zeolog.DEBUG) + zLOG.DEBUG) if not self.check_method(name): raise ZRPCError("Invalid method name: %s on %s" % (name, `self.obj`)) @@ -223,21 +223,21 @@ except (POSException.UndoError, POSException.VersionCommitError), msg: error = sys.exc_info() - log("%s() raised exception: %s" % (name, msg), zeolog.ERROR, error) + log("%s() raised exception: %s" % (name, msg), zLOG.ERROR, error) return self.return_error(msgid, flags, error[0], error[1]) except Exception, msg: error = sys.exc_info() - log("%s() raised exception: %s" % (name, msg), zeolog.ERROR, error) + log("%s() raised exception: %s" % (name, msg), zLOG.ERROR, error) return self.return_error(msgid, flags, error[0], error[1]) if flags & ASYNC: if ret is not None: log("async method %s returned value %s" % (name, repr(ret)), - zeolog.ERROR) + zLOG.ERROR) raise ZRPCError("async method returned value") else: if __debug__: - log("%s return %s" % (name, repr(ret)[:40]), zeolog.DEBUG) + log("%s return %s" % (name, repr(ret)[:40]), zLOG.DEBUG) if isinstance(ret, Delay): ret.set_sender(msgid, self.send_reply) else: @@ -249,7 +249,7 @@ def log_error(self, msg="No error message supplied"): error = sys.exc_info() - log(msg, zeolog.ERROR, error=error) + log(msg, zLOG.ERROR, error=error) del error def check_method(self, name): @@ -311,7 +311,7 @@ if type(r_args) == types.TupleType \ and type(r_args[0]) == types.ClassType \ and issubclass(r_args[0], Exception): - raise r_args[1] + raise r_args[1] # error raised by server return r_args def callAsync(self, method, *args): @@ -357,7 +357,7 @@ if __debug__: log("_do_io(wait=%d), async=%d" % (wait, self.is_async()), - level=zeolog.DEBUG) + level=zLOG.DEBUG) if self.is_async(): self.trigger.pull_trigger() if wait: @@ -466,7 +466,7 @@ t = self._wait(t) else: if self.debug: - log("Connected to server", level=zeolog.DEBUG) + log("Connected to server", level=zLOG.DEBUG) self.connected = 1 if self.connected and not self.closed: c = ManagedConnection(s, self.addr, self.obj, self) @@ -491,7 +491,7 @@ except socket.error, msg: if self.debug: log("Failed to connect to server: %s" % msg, - level=zeolog.DEBUG) + level=zLOG.DEBUG) s.close() return None return s From jeremy at zope.com Thu Jan 3 19:49:17 2002 From: jeremy at zope.com (Jeremy Hylton) Date: Sun Aug 10 16:31:17 2008 Subject: [ZEO-Checkins] CVS: StandaloneZODB/ZEO - zeolog.py:NONE Message-ID: <200201040049.g040nHi30779@cvs.baymountain.com> Update of /cvs-repository/StandaloneZODB/ZEO In directory cvs.zope.org:/tmp/cvs-serv30770 Removed Files: Tag: ZEO-ZRPC-Dev zeolog.py Log Message: No longer needed. === Removed File StandaloneZODB/ZEO/zeolog.py === From jeremy at zope.com Fri Jan 4 17:16:37 2002 From: jeremy at zope.com (Jeremy Hylton) Date: Sun Aug 10 16:31:17 2008 Subject: [ZEO-Checkins] CVS: StandaloneZODB/ZEO/tests - speed.py:1.1.2.4 Message-ID: <200201042216.g04MGbK12773@cvs.baymountain.com> Update of /cvs-repository/StandaloneZODB/ZEO/tests In directory cvs.zope.org:/tmp/cvs-serv12762/tests Modified Files: Tag: ZEO-ZRPC-Dev speed.py Log Message: Add a simple profile option === StandaloneZODB/ZEO/tests/speed.py 1.1.2.3 => 1.1.2.4 === -t n Number of concurrent threads to run. + + -P file Put profile data in this file. """ import asyncore @@ -116,12 +118,13 @@ jar.cacheMinimize(3) def main(args): - opts, args = getopt.getopt(args, 'zd:n:Ds:LMt:U') + opts, args = getopt.getopt(args, 'zd:n:Ds:LMt:UP:') s = None compress = None data=sys.argv[0] nrep=5 minimize=0 + profile=0 detailed=1 cache = None domain = 'AF_INET' @@ -148,6 +151,9 @@ domain = 'AF_UNIX' elif o == '-t': threads = int(v) + elif o == '-P': + profile = v + import hotshot zeo_pipe = None if s: @@ -180,7 +186,12 @@ t.join() else: - work(db, results, nrep, compress, data, detailed, minimize) + if profile: + p = hotshot.Profile(profile) + p.runctx("work(db, results, nrep, compress, data, detailed, minimize)", + globals(), locals()) + else: + work(db, results, nrep, compress, data, detailed, minimize) if server is not None: server.close() From jeremy at zope.com Fri Jan 4 17:18:43 2002 From: jeremy at zope.com (Jeremy Hylton) Date: Sun Aug 10 16:31:17 2008 Subject: [ZEO-Checkins] CVS: StandaloneZODB/ZEO/tests - testTransactionBuffer.py:1.1.2.1 Message-ID: <200201042218.g04MIhx13775@cvs.baymountain.com> Update of /cvs-repository/StandaloneZODB/ZEO/tests In directory cvs.zope.org:/tmp/cvs-serv13764/tests Added Files: Tag: ZEO-ZRPC-Dev testTransactionBuffer.py Log Message: Tests of the TransactionBuffer implementation === Added File StandaloneZODB/ZEO/tests/testTransactionBuffer.py === import random import unittest from ZEO.TransactionBuffer import TransactionBuffer def random_string(size): """Return a random string of size size.""" l = [chr(random.randrange(256)) for i in range(size)] return "".join(l) def new_store_data(): """Return arbitrary data to use as argument to store() method.""" return random_string(8), '', random_string(random.randrange(1000)) def new_invalidate_data(): """Return arbitrary data to use as argument to invalidate() method.""" return random_string(8), '' class TransBufTests(unittest.TestCase): def checkTypicalUsage(self): tbuf = TransactionBuffer() tbuf.store(*new_store_data()) tbuf.invalidate(*new_invalidate_data()) tbuf.begin_iterate() while 1: o = tbuf.next() if o is None: break tbuf.clear() def doUpdates(self, tbuf): data = [] for i in range(10): d = new_store_data() tbuf.store(*d) data.append(d) d = new_invalidate_data() tbuf.invalidate(*d) data.append(d) tbuf.begin_iterate() for i in range(len(data)): x = tbuf.next() if x[2] is None: # the tbuf add a dummy None to invalidates x = x[:2] self.assertEqual(x, data[i]) def checkOrderPreserved(self): tbuf = TransactionBuffer() self.doUpdates(tbuf) def checkReusable(self): tbuf = TransactionBuffer() self.doUpdates(tbuf) tbuf.clear() self.doUpdates(tbuf) tbuf.clear() self.doUpdates(tbuf) def test_suite(): return unittest.makeSuite(TransBufTests, 'check') From jeremy at zope.com Fri Jan 4 17:20:33 2002 From: jeremy at zope.com (Jeremy Hylton) Date: Sun Aug 10 16:31:17 2008 Subject: [ZEO-Checkins] CVS: StandaloneZODB/ZEO - ClientCache.py:1.14.4.7 Message-ID: <200201042220.g04MKXG13947@cvs.baymountain.com> Update of /cvs-repository/StandaloneZODB/ZEO In directory cvs.zope.org:/tmp/cvs-serv13936 Modified Files: Tag: ZEO-ZRPC-Dev ClientCache.py Log Message: Reformat and refactor store() implementation. Reformat based on guidelines from the Society for the Promotion of Whitespace. Accumulate individual strings to write in a list and do a single join and write at the end. Replaces several string + ops and several write calls. Undo use of temporary variables to avoid attribute lookups. === StandaloneZODB/ZEO/ClientCache.py 1.14.4.6 => 1.14.4.7 === def store(self, oid, p, s, version, pv, sv): self._acquire() - try: self._store(oid, p, s, version, pv, sv) - finally: self._release() + try: + self._store(oid, p, s, version, pv, sv) + finally: + self._release() def _store(self, oid, p, s, version, pv, sv): if not s: - p='' - s='\0\0\0\0\0\0\0\0' - tlen=31+len(p) + p = '' + s = '\0\0\0\0\0\0\0\0' + tlen = 31 + len(p) if version: - tlen=tlen+len(version)+12+len(pv) - vlen=len(version) + tlen = tlen + len(version) + 12 + len(pv) + vlen = len(version) else: - vlen=0 + vlen = 0 - pos=self._pos - current=self._current - f=self._f[current] - f.seek(pos) - stlen=pack(">I",tlen) - write=f.write - write(oid+'v'+stlen+pack(">HI", vlen, len(p))+s) - if p: write(p) + stlen = pack(">I", tlen) + # accumulate various data to write into a list + l = [oid, 'v', stlen, pack(">HI", vlen, len(p)), s] + if p: + l.append(p) if version: - write(version) - write(pack(">I", len(pv))) - write(pv) - write(sv) + l.extend([version, + pack(">I", len(pv)), + pv, sv]) + l.append(stlen) + f = self._f[self._current] + f.seek(self._pos) + f.write("".join(l)) - write(stlen) - - if current: self._index[oid]=-pos - else: self._index[oid]=pos + if self._current: + self._index[oid] = - self._pos + else: + self._index[oid] = self._pos - self._pos=pos+tlen + self._pos += tlen def read_index(index, serial, f, current): seek=f.seek From jeremy at zope.com Fri Jan 4 17:21:36 2002 From: jeremy at zope.com (Jeremy Hylton) Date: Sun Aug 10 16:31:17 2008 Subject: [ZEO-Checkins] CVS: StandaloneZODB/ZEO - TransactionBuffer.py:1.1.2.3 Message-ID: <200201042221.g04MLai14032@cvs.baymountain.com> Update of /cvs-repository/StandaloneZODB/ZEO In directory cvs.zope.org:/tmp/cvs-serv14021 Modified Files: Tag: ZEO-ZRPC-Dev TransactionBuffer.py Log Message: Use cPickle's fast Pickler() instead of marshal. Remove try/except in next() and trust self.count instead. === StandaloneZODB/ZEO/TransactionBuffer.py 1.1.2.2 => 1.1.2.3 === import tempfile -import marshal +import cPickle class TransactionBuffer: + def __init__(self): self.file = tempfile.TemporaryFile() self.count = 0 self.size = 0 + # It's safe to use a fast pickler because the only objects + # stored are builtin types -- strings or None. + self.pickler = cPickle.Pickler(self.file, 1) + self.pickler.fast = 1 def store(self, oid, version, data): """Store oid, version, data for later retrieval""" - marshal.dump((oid, version, data), self.file) - self.count = self.count + 1 + self.pickler.dump((oid, version, data)) + self.count += 1 # Estimate per-record cache size self.size = self.size + len(data) + (27 + 12) if version: self.size = self.size + len(version) + 4 def invalidate(self, oid, version): - marshal.dump((oid, version, None), self.file) - self.count = self.count + 1 + self.pickler.dump((oid, version, None)) + self.count += 1 def clear(self): """Mark the buffer as empty""" @@ -46,16 +51,15 @@ """Move the file pointer in advance of iteration""" self.file.flush() self.file.seek(0) + self.unpickler = cPickle.Unpickler(self.file) def next(self): """Return next tuple of data or None if EOF""" - if not self.count: + if self.count == 0: + del self.unpickler return None - try: - oid_ver_data = marshal.load(self.file) - except EOFError: - return None - self.count = self.count - 1 + oid_ver_data = self.unpickler.load() + self.count -= 1 return oid_ver_data def get_size(self): From jeremy at zope.com Fri Jan 4 18:05:18 2002 From: jeremy at zope.com (Jeremy Hylton) Date: Sun Aug 10 16:31:17 2008 Subject: [ZEO-Checkins] CVS: StandaloneZODB/ZEO - ClientStorage.py:1.26.4.27 Message-ID: <200201042305.g04N5I724703@cvs.baymountain.com> Update of /cvs-repository/StandaloneZODB/ZEO In directory cvs.zope.org:/tmp/cvs-serv24692 Modified Files: Tag: ZEO-ZRPC-Dev ClientStorage.py Log Message: No need for wait_for_server_on_startup to be an instance variable. It's irrelevant after the constructor returns. === StandaloneZODB/ZEO/ClientStorage.py 1.26.4.26 => 1.26.4.27 === self._storage = storage self._debug = debug - self._wait_for_server_on_startup = wait_for_server_on_startup self._info = {'length': 0, 'size': 0, 'name': 'ZEO Client', 'supportsUndo':0, 'supportsVersions': 0} @@ -177,7 +176,7 @@ tmax=max_disconnect_poll) self._server = None if wait_for_server_on_startup: - self._rpc_mgr.connect(1) + self._rpc_mgr.connect(sync=1) else: if not self._rpc_mgr.attempt_connect(): self._rpc_mgr.connect() From jeremy at zope.com Fri Jan 4 18:06:49 2002 From: jeremy at zope.com (Jeremy Hylton) Date: Sun Aug 10 16:31:17 2008 Subject: [ZEO-Checkins] CVS: StandaloneZODB/ZEO - zrpc2.py:1.1.2.21 Message-ID: <200201042306.g04N6nE24795@cvs.baymountain.com> Update of /cvs-repository/StandaloneZODB/ZEO In directory cvs.zope.org:/tmp/cvs-serv24784 Modified Files: Tag: ZEO-ZRPC-Dev zrpc2.py Log Message: Remove option of passing pickler object to Marshaller. (YAGNI) Add try/except to deal with harmless race for synchronous connect() call. === StandaloneZODB/ZEO/zrpc2.py 1.1.2.20 => 1.1.2.21 === return c -# We create a special fast pickler! This allows us -# to create slightly more efficient pickles and -# to create them a tad faster. -pickler = cPickle.Pickler() -pickler.fast = 1 # Don't use the memo -dump = pickler.dump - class Marshaller: """Marshal requests and replies to second across network""" - def __init__(self, pickle=None): - if pickle is not None: - self.pickle = pickle - self.errors = pickle.PickleError - else: - self.pickle = cPickle.Pickler().dump - self.errors = (cPickle.UnpickleableError, - cPickle.UnpicklingError, - cPickle.PickleError, - cPickle.PicklingError) + # It's okay to share a single Pickler as long as it's in fast + # mode, which means that it doesn't have a memo. + + pickler = cPickle.Pickler() + pickler.fast = 1 + pickle = pickler.dump + + errors = (cPickle.UnpickleableError, + cPickle.UnpicklingError, + cPickle.PickleError, + cPickle.PicklingError) def encode(self, msgid, flags, name, args): """Returns an encoded message""" @@ -105,7 +99,6 @@ try: return unpickler.load() # msgid, flags, name, args - msgid, flags, name, args = unpickler.load() except (cPickle.UnpicklingError, IndexError), err_msg: log("can't decode %s" % repr(msg), level=zLOG.ERROR) raise DecodingError(msg) @@ -143,10 +136,10 @@ __super_close = smac.SizedMessageAsyncConnection.close __super_writable = smac.SizedMessageAsyncConnection.writable - def __init__(self, sock, addr, obj=None, pickle=None): + def __init__(self, sock, addr, obj=None): self.msgid = 0 self.obj = obj - self.marshal = Marshaller(pickle) + self.marshal = Marshaller() self.closed = 0 self.async = 0 # The reply lock is used to block when a synchronous call is @@ -325,7 +318,7 @@ if self.closed: raise DisconnectedError("This action is temporarily unavailable") msgid = self.msgid - self.msgid = self.msgid + 1 + self.msgid += 1 if __debug__: log("send msg: %d, %d, %s, ..." % (msgid, ASYNC, method)) self.message_output(self.marshal.encode(msgid, ASYNC, method, args)) @@ -437,10 +430,14 @@ self._thread = threading.Thread(target=self.__connect, args=(1,)) self._thread.start() + if sync: + try: + self._thread.join() + except AttributeError: + # probably means the thread exited quickly + pass finally: self._connect_lock.release() - if sync and self._thread is not None: - self._thread.join() def attempt_connect(self): self.__connect(repeat=0) @@ -514,9 +511,9 @@ __super_init = Connection.__init__ __super_close = Connection.close - def __init__(self, sock, addr, obj, mgr, pickle=None): + def __init__(self, sock, addr, obj, mgr): self.__mgr = mgr - self.__super_init(sock, addr, obj, pickle) + self.__super_init(sock, addr, obj) def close(self): self.__super_close() @@ -531,14 +528,14 @@ __super_init = Connection.__init__ __super_close = Connection.close - def __init__(self, sock, addr, obj, mgr, pickle=None): + def __init__(self, sock, addr, obj, mgr): self.__mgr = mgr if self.__mgr.async: self.__async = 1 self.trigger = self.__mgr.trigger else: self.__async = None - self.__super_init(sock, addr, obj, pickle) + self.__super_init(sock, addr, obj) def _prepare_async(self): # Don't do the register_loop_callback that the superclass does From jeremy at zope.com Sun Jan 6 22:22:02 2002 From: jeremy at zope.com (Jeremy Hylton) Date: Sun Aug 10 16:31:17 2008 Subject: [ZEO-Checkins] CVS: StandaloneZODB/ZEO/tests - testZEO.py:1.1.2.16 Message-ID: <200201070322.g073M2d25928@cvs.baymountain.com> Update of /cvs-repository/StandaloneZODB/ZEO/tests In directory cvs.zope.org:/tmp/cvs-serv25917/ZEO/tests Modified Files: Tag: ZEO-ZRPC-Dev testZEO.py Log Message: Mixin MTStorage tests for all ZEOs. === StandaloneZODB/ZEO/tests/testZEO.py 1.1.2.15 => 1.1.2.16 === from ZODB.tests import StorageTestBase, BasicStorage, VersionStorage, \ TransactionalUndoStorage, TransactionalUndoVersionStorage, \ - PackableStorage, Synchronization, ConflictResolution, RevisionStorage + PackableStorage, Synchronization, ConflictResolution, RevisionStorage, \ + MTStorage from ZODB.tests.MinPO import MinPO from ZODB.tests.StorageTestBase import zodb_unpickle @@ -135,6 +136,7 @@ RevisionStorage.RevisionStorage, PackableStorage.PackableStorage, Synchronization.SynchronizedStorage, + MTStorage.MTStorage, ): """An abstract base class for ZEO tests From jeremy at zope.com Mon Jan 7 14:28:02 2002 From: jeremy at zope.com (Jeremy Hylton) Date: Sun Aug 10 16:31:17 2008 Subject: [ZEO-Checkins] CVS: StandaloneZODB/ZEO - ClientStorage.py:1.26.4.28 Message-ID: <200201071928.g07JS2H21895@cvs.baymountain.com> Update of /cvs-repository/StandaloneZODB/ZEO In directory cvs.zope.org:/tmp/cvs-serv21838 Modified Files: Tag: ZEO-ZRPC-Dev ClientStorage.py Log Message: Fix notifyDisconnected() to set _server to None. _server == None indicates that the client is disconnected. Without this assignment, the client will attempt to use a dead connection. Fix tpc_cond() release logic. The try/except in tpc_begin() should not release the lock if _server == None, because notifyDisconnected() will have already released it. === StandaloneZODB/ZEO/ClientStorage.py 1.26.4.27 => 1.26.4.28 === def notifyDisconnected(self, ignored): log2(PROBLEM, "Disconnected from storage") - self._transaction = None + self._server = None if self._transaction: self._transaction = None self.tpc_cond.notifyAll() @@ -449,7 +449,11 @@ transaction.description, transaction._extension) except: - self.tpc_cond.release() + # If _server is None, then the client disconnected during + # the tpc_begin() and notifyDisconnected() will have + # released the lock. + if self._server is not None: + self.tpc_cond.release() raise self._serial = id From jeremy at zope.com Mon Jan 7 15:56:45 2002 From: jeremy at zope.com (Jeremy Hylton) Date: Sun Aug 10 16:31:17 2008 Subject: [ZEO-Checkins] CVS: StandaloneZODB/ZEO/tests - forker.py:1.1.2.7 Message-ID: <200201072056.g07Kujs10030@cvs.baymountain.com> Update of /cvs-repository/StandaloneZODB/ZEO/tests In directory cvs.zope.org:/tmp/cvs-serv10015/tests Modified Files: Tag: ZEO-ZRPC-Dev forker.py Log Message: Make the server shutdown more robust. XXX Also break the profile option for versions of Python earlier than 2.2 by using hotshot rather than profile. === StandaloneZODB/ZEO/tests/forker.py 1.1.2.6 => 1.1.2.7 === import asyncore import os -import profile +import hotshot import random import socket import sys @@ -76,9 +76,11 @@ buf = self.recv(4) if buf: assert buf == "done" + server.close_server() asyncore.socket_map.clear() def handle_close(self): + server.close_server() asyncore.socket_map.clear() class ZEOClientExit: @@ -99,10 +101,10 @@ if pid == 0: try: if PROFILE: - p = profile.Profile() - p.runctx("run_server(storage, addr, rd, wr)", globals(), - locals()) - p.dump_stats("stats.s.%d" % os.getpid()) + p = hotshot.Profile("stats.s.%d" % os.getpid()) + p.runctx("run_server(storage, addr, rd, wr)", + globals(), locals()) + p.close() else: run_server(storage, addr, rd, wr) except: @@ -115,9 +117,10 @@ def run_server(storage, addr, rd, wr): # in the child, run the storage server + global server os.close(wr) ZEOServerExit(rd) - serv = ZEO.StorageServer.StorageServer(addr, {'1':storage}) + server = ZEO.StorageServer.StorageServer(addr, {'1':storage}) asyncore.loop() os.close(rd) storage.close() From jeremy at zope.com Mon Jan 7 16:00:15 2002 From: jeremy at zope.com (Jeremy Hylton) Date: Sun Aug 10 16:31:17 2008 Subject: [ZEO-Checkins] CVS: StandaloneZODB/ZEO - zrpc2.py:1.1.2.22 Message-ID: <200201072100.g07L0Fv11235@cvs.baymountain.com> Update of /cvs-repository/StandaloneZODB/ZEO In directory cvs.zope.org:/tmp/cvs-serv11199 Modified Files: Tag: ZEO-ZRPC-Dev zrpc2.py Log Message: Don't raise exception in __connect(). It's a thread. === StandaloneZODB/ZEO/zrpc2.py 1.1.2.21 => 1.1.2.22 === self.obj.notifyConnected(c) except: - # XXX c.close() - raise + # When the connection is closed, we'll trigger + # another attempt to reconnect. + finally: # must always clear _thread on the way out self._thread = None From jeremy at zope.com Mon Jan 7 16:02:24 2002 From: jeremy at zope.com (Jeremy Hylton) Date: Sun Aug 10 16:31:17 2008 Subject: [ZEO-Checkins] CVS: StandaloneZODB/ZEO - ClientStorage.py:1.26.4.29 Message-ID: <200201072102.g07L2OA11407@cvs.baymountain.com> Update of /cvs-repository/StandaloneZODB/ZEO In directory cvs.zope.org:/tmp/cvs-serv11395 Modified Files: Tag: ZEO-ZRPC-Dev ClientStorage.py Log Message: Use ClientDisconnectedStub for _server attribute of ClientStorage. The CDS always raises a ClientDisconnected() error. This is simpler than testing for _server == None and raising the error in every place where _server is used. === StandaloneZODB/ZEO/ClientStorage.py 1.26.4.28 => 1.26.4.29 === return t -class ClientStorage: +class DisconnectedServerStub: + """Raise ClientDisconnected on all attribute access.""" + + def __getattr__(self, attr): + raise ClientDisconnected() - _server = None +disconnected_stub = DisconnectedServerStub() + +class ClientStorage: def __init__(self, addr, storage='1', cache_size=20000000, name='', client='', debug=0, var=None, @@ -148,6 +154,7 @@ # Decide whether to use non-temporary files client = client or os.environ.get('ZEO_CLIENT','') + self._server = disconnected_stub self._connection = addr self._storage = storage self._debug = debug @@ -174,7 +181,6 @@ #debug=debug, tmin=min_disconnect_poll, tmax=max_disconnect_poll) - self._server = None if wait_for_server_on_startup: self._rpc_mgr.connect(sync=1) else: @@ -213,10 +219,10 @@ self._db = db def is_connected(self): - if self._server: - return 1 - else: + if self._server is disconnected_stub: return 0 + else: + return 1 def notifyConnected(self, c): log2(INFO, "Connected to storage") @@ -224,6 +230,8 @@ self._oids = [] + # XXX Why is this synchronous? If it were async, verification + # would start faster. stub.register(str(self._storage)) self.verify_cache(stub) @@ -250,7 +258,7 @@ def notifyDisconnected(self, ignored): log2(PROBLEM, "Disconnected from storage") - self._server = None + self._server = disconnected_stub if self._transaction: self._transaction = None self.tpc_cond.notifyAll() @@ -326,7 +334,8 @@ # Close the manager first, so that it doesn't attempt to # re-open the connection. self._rpc_mgr.close() - if self._server: + if self._server is not disconnected_stub: + # XXX why doesn't the manager close this for us? self._server.rpc.close() if self._cache is not None: self._cache.close() @@ -452,7 +461,7 @@ # If _server is None, then the client disconnected during # the tpc_begin() and notifyDisconnected() will have # released the lock. - if self._server is not None: + if self._server is not disconnected_stub: self.tpc_cond.release() raise From jeremy at zope.com Mon Jan 7 16:03:40 2002 From: jeremy at zope.com (Jeremy Hylton) Date: Sun Aug 10 16:31:17 2008 Subject: [ZEO-Checkins] CVS: StandaloneZODB/ZEO - StorageServer.py:1.21.4.15 Message-ID: <200201072103.g07L3ei12372@cvs.baymountain.com> Update of /cvs-repository/StandaloneZODB/ZEO In directory cvs.zope.org:/tmp/cvs-serv12361 Modified Files: Tag: ZEO-ZRPC-Dev StorageServer.py Log Message: Add a server_close() method that is called by forker to shutdown the server. === StandaloneZODB/ZEO/StorageServer.py 1.21.4.14 => 1.21.4.15 === for name, store in storages.items(): fixup_storage(store) - Dispatcher(addr, factory=self.newConnection, reuse_addr=1) + self.dispatcher = Dispatcher(addr, factory=self.newConnection, + reuse_addr=1) def newConnection(self, sock, addr, nil): c = ManagedServerConnection(sock, addr, None, self) @@ -153,7 +154,11 @@ else: p.client.info(info) + def close_server(self): + self.dispatcher.close() + def close(self, conn): + # XXX who calls this? # called when conn is closed # way too inefficient removed = 0 From jim at zope.com Mon Jan 7 16:20:32 2002 From: jim at zope.com (Jim Fulton) Date: Sun Aug 10 16:31:17 2008 Subject: [ZEO-Checkins] CVS: Packages/ZEO - Validators.py:1.1.2.1 list_cache.py:1.1.2.1 srp.py:1.1.2.1 srpdemo.py:1.1.2.1 ClientCache.py:1.7.2.1 ClientStorage.py:1.11.2.1 StorageServer.py:1.11.2.1 Message-ID: <200201072120.g07LKWe16161@cvs.baymountain.com> Update of /cvs-repository/Packages/ZEO In directory cvs.zope.org:/tmp/cvs-serv15436 Modified Files: Tag: ZEO-Auth-Dev ClientCache.py ClientStorage.py StorageServer.py Added Files: Tag: ZEO-Auth-Dev Validators.py list_cache.py srp.py srpdemo.py Log Message: Checking in some old attempts at adding authentication to ZEO for posterity. In particular, there's a reasonable clean SRP implementation here. === Added File Packages/ZEO/Validators.py === ############################################################################## # # Copyright (c) 2001 Zope Corporation and Contributors. All Rights Reserved. # # This software is subject to the provisions of the Zope Public License, # Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution. # THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED # WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED # WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS # FOR A PARTICULAR PURPOSE # ############################################################################## """ Revision information: $Id: Validators.py,v 1.1.2.1 2002/01/07 21:20:31 jim Exp $ """ """Objects that validate connections """ class HostValidator: def __init__(self, storage, allow=None, read_only=0): self._storage=storage self.__allow=allow self.__ro=read_only def validate(storage_id, server, connection): """Try to validate a connection""" # Need code to check connection address against allow list! server.register_storage( connection, self._storage, self.__ro and StorageServer.read_storage_method or storage_method ) class SRPValidator: def __init__(self, storage, allow=None, read_only=0, hash=None, n=None, g=None, rg=None): self._storage=storage self.__allow=allow self.__ro=read_only def validate(storage_id, server, connection): class SRPConnectionValidator: def __init__(self, validator, server, connection): self._validator=validator self.server=server === Added File Packages/ZEO/list_cache.py === try: import ZODB except: import sys sys.path.append('..') from ZODB.utils import U64 from ZODB.TimeStamp import TimeStamp from struct import unpack class list: def __init__(self, f): if type(f) is type(''): f=open(f,'rb') f.read(4) # header self.f=f self.i=-1 self.r='','','','','','' def read_record(self): read=self.f.read h=read(27) if len(h)==27 and h[8] in 'vni': tlen, vlen, dlen = unpack(">iHi", h[9:19]) else: return if tlen <= 0 or vlen < 0 or dlen <= 0 or vlen+dlen > tlen: return oid=h[:8] serial=TimeStamp(h[19:27]) data=read(dlen) if vlen: version = read(vlen) vdlen=read(4) if len(vdlen) != 4: return vdlen=unpack(">i", vdlen)[0] if vlen+dlen+42+vdlen > tlen: return vdata=read(vdlen) vs=TimeStamp(read(8)) else: vs=None vdata=version='' if read(4) != h[9:13]: return return oid, status, serial, data, version, vs, vdata def __getitem__(self, i): if not self.r or i < self.i: raise IndexError, i while i > self.i: self.r = self.read_record() self.i=self.i+1 if not self.r: raise IndexError, i return self.r def list_cache(f): for oid, status, serial, data, version, vs, vdata in list(f): print oid, status, serial, version, vs if __name__=='__main__': import sys list_cache(sys.argv[1]) === Added File Packages/ZEO/srp.py === ############################################################################## # # Copyright (c) 2001 Zope Corporation and Contributors. All Rights Reserved. # # This software is subject to the provisions of the Zope Public License, # Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution. # THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED # WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED # WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS # FOR A PARTICULAR PURPOSE # ############################################################################## """ Revision information: $Id: srp.py,v 1.1.2.1 2002/01/07 21:20:31 jim Exp $ """ """Simplest possible SRP implementation Usage: - Create a 'SRPConfig' object with same given parameters for clients and server. - On server, for each user, with user name 'U' and password, 'p', call 'user_info' on the configuration object passing the password to get a user salt, 's', and a verifier, 'v'. Save the salt and verifier for the user name. - When a client wants to connect, it creates a 'User' object, passing in the configuration, user name, and password. It calls 'session' on the 'User' object to get the user name, 'U', and a temporary public key, 'A'. It sends 'U' and 'A' to the server. - On the server, the user name, 'U', is used to look up the salt, 's', and verifier, 'v'. A Host object is created by passing 'U', 's', 'v', and 'A'. The 'session' method is called on the 'Host' object to get the salt, 's', a temporary public key, 'B', and a random number, 'u'. These are send to the client. - On the client, the 'key' method is called on the 'User' object, passing 's', 'B', and 'u' received from the server. A shared session key, 'K', and client proof, 'M', are returned. The client proof is sent to the server. - On the server, the client proof is validated by passing it to the 'validate' method on the 'Host' object. The 'validate' method returns the shared session key, 'K', and the host proof, which is sent to the client. - On the client, the host proof is validated by passing it to the 'validate' method on the 'User' object. The shared session key may be used to encrypt communication between the client and host using a symetric key encryption algorithm. """ LongType=type(0L) class SRPConfig: """Hold an SRP configuration. """ def __init__(self, h, g, N, random): """Create an SRP configuration: h -- A one-way has function that accepts a string and returns a string. g -- A (long) generator modulo N N -- A large safe prime (N = 2q+1, where q is prime) All arithmetic is done modulo N. random -- A t-bit random (long) number generator. """ self.h, self.g, self.N, self.random = ( h, g, N, random) # algorthm info self.ngh=stol(h(str(N))) ^ stol(h(str(g))) def H(self, *args): hash=self.h l=list(args) h='' while l: h=hash(str(l.pop())+h) return h def M(self, s, U, A, B, K): return self.H(self.ngh, self.H(U), s, A, B, K) def user_info(self, p, s=None): """Get the password verifier and user salt for server """ if s is None: s=self.random() x=stol(self.H(s, p)) v=pow(self.g, x, self.N) return s, v def stol(s): # Convert the bytes in a string to a long. r=0L for c in s: r=r*256+ord(c) return r class User: """Provide user side of SRP computations """ def __init__(self, config, U, p): self.config=config self.user=U, p def session(self): U, p = self.user a=self.a=self.config.random() A=self.A=pow(self.config.g, a, self.config.N) return U, A def key(self, s, B, u): if (B%self.config.N)==0: raise ValueError, B if u==0: raise ValueError, u U, p = self.user N=self.config.N x=stol(self.config.H(s, p)) v=pow(self.config.g, x, N) S=pow(B-v, self.a+u*x, N) self.K=K=self.config.H(S) self.M=M=self.config.M(s, U, self.A, B, K) return K, M def validate(self, proof): if proof != self.config.H(self.A, self.M, self.K): raise ValueError, proof class Host: """Provide host side of SRP computations. """ def __init__(self, config, U, s, v, A): self.config = config if A <= 0 or A >= self.config.N: raise ValueError, A self.user = U, s, v, A def session(self): U, s, v, A = self.user N=self.config.N b=self.b=self.config.random() B=self.B=v+pow(self.config.g, b, N) u=self.u=self.config.random() S=pow(A*pow(v, u, N), b, N) K=self.K=self.config.H(S) return s, self.B, self.u def validate(self, proof): U, s, v, A = self.user K=self.K M=self.config.M(s, U, A, self.B, K) if M != proof: raise ValueError, (proof, M) return K, self.config.H(A, M, K) === Added File Packages/ZEO/srpdemo.py === ############################################################################## # # Copyright (c) 2001 Zope Corporation and Contributors. All Rights Reserved. # # This software is subject to the provisions of the Zope Public License, # Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution. # THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED # WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED # WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS # FOR A PARTICULAR PURPOSE # ############################################################################## """ Revision information: $Id: srpdemo.py,v 1.1.2.1 2002/01/07 21:20:31 jim Exp $ """ import srp, sha, time, random def h(v): s=sha.new() s.update(v) return s.digest() def r(randint=random.randint): r=long(randint(0,1<<30)) for i in 1,2,3,4,5,6: r = (r << 24) + randint(0,1<<30) return r def r_(randint=random.randint): return long(randint(0,1<<30)) c=srp.SRPConfig( N=137656596376486790043182744734961384933899167257744121335064027192370741112305920493080254690601316526576747330553110881621319493425219214435734356437905637147670206858858966652975541347966997276817657605917471296442404150473520316654025988200256062845025470327802138620845134916799507318209468806715548156999L, g=8623462398472349872L, h=h, random=r, ) U='jim' p='zope rools and I know it' s, v = c.user_info(p) #print "U, p, s, x, v: ", U, p, s, v user=srp.User(c, U, p) U, A = user.session() #print 'U, A: ', U, A host=srp.Host(c, U, s, v, A) s, B, u = host.session() #print "s, B, u: ", s, B, u #print 'K', `host.K` K, M = user.key(s, B, u) #print 'K==K', host.K==K #print "K, M: ", `K`, `M` K, host_proof = host.validate(M) #print "host_proof:", `host_proof` user.validate(host_proof) === Packages/ZEO/ClientCache.py 1.7 => 1.7.2.1 === oid -- 8-byte object id - status -- 1-byte status v': valid, 'n': non-version valid, 'i': invalid + status -- 1-byte status 'v': valid, 'n': non-version valid, 'i': invalid tlen -- 4-byte (unsigned) record length @@ -108,7 +108,7 @@ vdata -- version data (if vlen > 0) - vserial -- 8-byte version serial (timestamp) + vserial -- 8-byte version serial (timestamp) (if vlen > 0) tlen -- 4-byte (unsigned) record length (for redundancy and backward traversal) @@ -240,20 +240,27 @@ if len(h)==27 and h[8] in 'nv' and h[:8]==oid: tlen, vlen, dlen = unpack(">iHi", h[9:19]) else: tlen=-1 - if tlen <= 0 or vlen < 0 or dlen <= 0 or vlen+dlen > tlen: + if tlen <= 0 or vlen < 0 or dlen < 0 or vlen+dlen > tlen: del self._index[oid] return None - if version and h[8]=='n': return None + if h[8]=='n': + if version: return None + if not dlen: + del self._index[oid] + return None if not vlen or not version: - return read(dlen), h[19:] + if dlen: return read(dlen), h[19:] + else: return None - seek(dlen, 1) + if dlen: seek(dlen, 1) v=read(vlen) - if version != v: - seek(-dlen-vlen, 1) - return read(dlen), h[19:] + if version != v: + if dlen: + seek(-dlen-vlen, 1) + return read(dlen), h[19:] + else: None dlen=unpack(">i", read(4))[0] return read(dlen), read(8) @@ -262,7 +269,8 @@ if version: # We need to find and include non-version data p=self._get(oid, None) - if p is None: return None + if p is None: + return self.store(oid, '', '', version, data, serial) f=self._f[p < 0] ap=abs(p) seek=f.seek @@ -271,13 +279,17 @@ h=read(27) if len(h)==27 and h[8] in 'nv' and h[:8]==oid: tlen, vlen, dlen = unpack(">iHi", h[9:19]) - else: tlen=-1 + else: + return self.store(oid, '', '', version, data, serial) + if tlen <= 0 or vlen < 0 or dlen <= 0 or vlen+dlen > tlen: - del self._index[oid] - return None + return self.store(oid, '', '', version, data, serial) - p=read(dlen) - s=h[19:] + if dlen: + p=read(dlen) + s=h[19:] + else: + return self.store(oid, '', '', version, data, serial) self.store(oid, p, s, version, data, serial) else: @@ -296,7 +308,7 @@ if len(h)==27 and h[8] in 'nv' and h[:8]==oid: tlen, vlen, dlen = unpack(">iHi", h[9:19]) else: tlen=-1 - if tlen <= 0 or vlen < 0 or dlen <= 0 or vlen+dlen > tlen: + if tlen <= 0 or vlen < 0 or dlen < 0 or vlen+dlen > tlen: del self._index[oid] return None @@ -318,6 +330,9 @@ def store(self, oid, p, s, version, pv, sv): + if not s: + p='' + s='\0\0\0\0\0\0\0\0' tlen=31+len(p) if version: tlen=tlen+len(version)+12+len(pv) @@ -332,11 +347,14 @@ stlen=pack(">I",tlen) write=f.write write(oid+'v'+stlen+pack(">HI", vlen, len(p))+s) - write(p) + if p: write(p) if version: + write(version) write(pack(">I", len(pv))) write(pv) - write(sv+stlen) + write(sv) + + write(stlen) if current: self._index[oid]=-pos else: self._index[oid]=pos @@ -357,7 +375,7 @@ if len(h)==27 and h[8] in 'vni': tlen, vlen, dlen = unpack(">iHi", h[9:19]) else: tlen=-1 - if tlen <= 0 or vlen < 0 or dlen <= 0 or vlen+dlen > tlen: + if tlen <= 0 or vlen < 0 or dlen < 0 or vlen+dlen > tlen: break oid=h[:8] @@ -379,7 +397,7 @@ serial[oid]=h[-8:], vs else: if serial.has_key(oid): - # We has a record for this oid, but it was invalidated! + # We have a record for this oid, but it was invalidated! del serial[oid] del index[oid] === Packages/ZEO/ClientStorage.py 1.11 => 1.11.2.1 === self._connected=0 thread.start_new_thread(self._call.connect,(0,)) + try: self._commit_lock_release() + except: pass + def becomeAsync(self, map): self._lock_acquire() @@ -229,7 +232,11 @@ if transaction is not self._transaction: raise POSException.StorageTransactionError(self, transaction) self._lock_acquire() - try: return self._call('abortVersion', src, self._serial) + try: + oids=self._call('abortVersion', src, self._serial) + invalidate=self._cache.invalidate + for oid in oids: invalidate(oid, src) + return oids finally: self._lock_release() def close(self): @@ -241,7 +248,17 @@ if transaction is not self._transaction: raise POSException.StorageTransactionError(self, transaction) self._lock_acquire() - try: return self._call('commitVersion', src, dest, self._serial) + try: + oids=self._call('commitVersion', src, dest, self._serial) + invalidate=self._cache.invalidate + if dest: + # just invalidate our version data + for oid in oids: invalidate(oid, src) + else: + # dest is '', so invalidate version and non-version + for oid in oids: invalidate(oid, dest) + + return oids finally: self._lock_release() def getName(self): @@ -265,11 +282,12 @@ self._lock_acquire() try: p = self._cache.load(oid, version) - if p is not None: return p + if p: return p p, s, v, pv, sv = self._call('zeoLoad', oid) self._cache.store(oid, p, s, v, pv, sv) if not v or not version or version != v: - return p, s + if s: return p, s + raise KeyError, oid # no non-version data for this return pv, sv finally: self._lock_release() @@ -465,5 +483,5 @@ def versions(self, max=None): self._lock_acquire() - try: return self._call('versionEmpty', max) + try: return self._call('versions', max) finally: self._lock_release() === Packages/ZEO/StorageServer.py 1.11 => 1.11.2.1 === return storage, storage_id + def register_storage(self, connection, storage, storage_method): + i=id(storage) + connections=self.__get_connections(i, None) + if connections is None: self.__connections[i]=[connection] + else: connections.append(connection) + connection.setStorage(i, storage, storage_method) + + def register_connection(self, connection, storage_id): + validator=self.__storages.get(storage_id, None) + if validator is None: + LOG('ZEO Server', PROBLEM, 'Invalid storage, %s' % storage_id) + connection.close() + return + + try: + validator.validate(storage_id, self, connection) + except: + if hasattr(validator, 'tpc_begin'): + # This is a storage, not a validator. Just register + # it. + self.register_storage(connection, validator, storage_method) + return + LOG('ZEO Server', ERROR, 'Invalid validator for %s' % storage_id) + connection.close() + return + def unregister_connection(self, connection, storage_id): connections=self.__get_connections(storage_id, None) @@ -185,16 +211,23 @@ log=log_info +read_storage_methods={} +for n in ( + 'get_info', 'history', 'load', 'loadSerial', 'modifiedInVersion', + 'tpc_abort', 'tpc_begin', 'tpc_begin_sync', 'tpc_finish', + 'undoLog', 'undoInfo', 'versionEmpty', 'versions', 'vote', + 'zeoLoad', 'zeoVerify', 'beginZeoVerify', 'endZeoVerify', + ): + read_storage_methods[n]=1 +read_storage_method=read_storage_methods.has_key + storage_methods={} for n in ( - 'get_info', 'abortVersion', 'commitVersion', - 'history', 'load', 'loadSerial', - 'modifiedInVersion', 'new_oid', 'new_oids', 'pack', 'store', - 'storea', 'tpc_abort', 'tpc_begin', 'tpc_begin_sync', - 'tpc_finish', 'undo', 'undoLog', 'undoInfo', 'versionEmpty', - 'vote', 'zeoLoad', 'zeoVerify', 'beginZeoVerify', 'endZeoVerify', + 'abortVersion', 'commitVersion', 'new_oid', 'new_oids', 'pack', + 'store', 'storea', 'undo', ): storage_methods[n]=1 +storage_methods.update(read_storage_methods) storage_method=storage_methods.has_key def find_global(module, name, @@ -238,6 +271,10 @@ self.__closed=1 SizedMessageAsyncConnection.close(self) + def setStorage(i, s, m): + self.__storage, self.__storage_id = s, i + self.storage_method=m + def message_input(self, message, dump=dump, Unpickler=Unpickler, StringIO=StringIO, None=None): @@ -247,8 +284,7 @@ blather('message_input', m) if self.__storage is None: - self.__storage, self.__storage_id = ( - self.__server.register_connection(self, message)) + self.__server.register_connection(self, message) return rt='R' @@ -262,10 +298,10 @@ name, args = args[0], args[1:] if __debug__: m=`tuple(args)` - if len(m) > 60: m=m[:60]+' ...' + if len(m) > 90: m=m[:90]+' ...' blather('call: %s%s' % (name, m)) - if not storage_method(name): + if not self.storage_method(name): raise 'Invalid Method Name', name if hasattr(self, name): r=apply(getattr(self, name), args) @@ -283,7 +319,14 @@ if len(m) > 60: m=m[:60]+' ...' blather('%s: %s' % (rt, m)) - r=dump(r,1) + try: r=dump(r,1) + except: + # Ugh, must be an unpicklable exception + r=StorageServerError("Couldn't pickle result %s" % `r`) + dump('',1) # clear pickler + r=dump(r,1) + rt='E' + self.message_output(rt+r) def get_info(self): @@ -301,8 +344,16 @@ v=storage.modifiedInVersion(oid) if v: pv, sv = storage.load(oid, v) else: pv=sv=None - p, s = storage.load(oid,'') + try: + p, s = storage.load(oid,'') + except KeyError: + if sv: + # Created in version, no non-version data + p=s=None + else: + raise return p, s, v, pv, sv + def beginZeoVerify(self): self.message_output('bN.') @@ -337,6 +388,26 @@ def _pack(self, t): self.__storage.pack(t, referencesf) self.message_output('S'+dump(self.get_info(), 1)) + + def abortVersion(self, src, id): + t=self._transaction + if t is None or id != t.id: + raise POSException.StorageTransactionError(self, id) + oids=self.__storage.abortVersion(src, t) + a=self.__invalidated.append + for oid in oids: a((oid,src)) + return oids + + def commitVersion(self, src, dest, id): + t=self._transaction + if t is None or id != t.id: + raise POSException.StorageTransactionError(self, id) + oids=self.__storage.commitVersion(src, dest, t) + a=self.__invalidated.append + for oid in oids: + a((oid,dest)) + if dest: a((oid,src)) + return oids def store(self, oid, serial, data, version, id): t=self._transaction From jeremy at zope.com Mon Jan 7 18:07:34 2002 From: jeremy at zope.com (Jeremy Hylton) Date: Sun Aug 10 16:31:17 2008 Subject: [ZEO-Checkins] CVS: StandaloneZODB/ZEO - ClientStorage.py:1.26.4.30 StorageServer.py:1.21.4.16 Message-ID: <200201072307.g07N7Yk08897@cvs.baymountain.com> Update of /cvs-repository/StandaloneZODB/ZEO In directory cvs.zope.org:/tmp/cvs-serv8884/ZEO Modified Files: Tag: ZEO-ZRPC-Dev ClientStorage.py StorageServer.py Log Message: Provisional read-only support for ZEO. The client does all the necessary checks, but the server should raise an exception if its storage is read-only and the client requests read-write. === StandaloneZODB/ZEO/ClientStorage.py 1.26.4.29 => 1.26.4.30 === class ClientStorage: - def __init__(self, addr, storage='1', cache_size=20000000, - name='', client='', debug=0, var=None, - min_disconnect_poll=5, max_disconnect_poll=300, + def __init__(self, addr, storage='1', read_only=0, + cache_size=20000000, name='', client='', debug=0, + var=None, min_disconnect_poll=5, + max_disconnect_poll=300, wait_for_server_on_startup=1): # Decide whether to use non-temporary files client = client or os.environ.get('ZEO_CLIENT','') self._server = disconnected_stub + self._is_read_only = read_only self._connection = addr self._storage = storage self._debug = debug @@ -232,7 +234,7 @@ # XXX Why is this synchronous? If it were async, verification # would start faster. - stub.register(str(self._storage)) + stub.register(str(self._storage), self._is_read_only) self.verify_cache(stub) # Don't make the server available to clients until after @@ -323,6 +325,8 @@ self.tpc_cond.release() def abortVersion(self, src, transaction): + if self._is_read_only: + raise POSException.ReadOnlyError() self._check_trans(transaction, POSException.StorageTransactionError) oids = self._server.abortVersion(src, self._serial) @@ -341,6 +345,8 @@ self._cache.close() def commitVersion(self, src, dest, transaction): + if self._is_read_only: + raise POSException.ReadOnlyError() self._check_trans(transaction, POSException.StorageTransactionError) oids = self._server.commitVersion(src, dest, self._serial) @@ -383,6 +389,8 @@ return self._server.modifiedInVersion(oid) def new_oid(self, last=None): + if self._is_read_only: + raise POSException.ReadOnlyError() # We want to avoid a situation where multiple oid requests are # made at the same time. self.oid_cond.acquire() @@ -395,6 +403,8 @@ return oid def pack(self, t=None, rf=None, wait=0, days=0): + if self._is_read_only: + raise POSException.ReadOnlyError() # Note that we ignore the rf argument. The server # will provide it's own implementation. if t is None: @@ -414,6 +424,8 @@ return r def store(self, oid, serial, data, version, transaction): + if self._is_read_only: + raise POSException.ReadOnlyError() self._check_trans(transaction, POSException.StorageTransactionError) self._server.storea(oid, serial, data, version, self._serial) self._tbuf.store(oid, version, data) @@ -512,6 +524,8 @@ self._tbuf.clear() def transactionalUndo(self, trans_id, trans): + if self._is_read_only: + raise POSException.ReadOnlyError() self._check_trans(trans, POSException.StorageTransactionError) oids = self._server.transactionalUndo(trans_id, self._serial) for oid in oids: @@ -519,6 +533,8 @@ return oids def undo(self, transaction_id): + if self._is_read_only: + raise POSException.ReadOnlyError() # XXX what are the sync issues here? oids = self._server.undo(transaction_id) for oid in oids: === StandaloneZODB/ZEO/StorageServer.py 1.21.4.15 => 1.21.4.16 === return 1 - def register(self, storage_id): + def register(self, storage_id, read_only): """Select the storage that this client will use This method must be the first one called by the client. From jeremy at zope.com Tue Jan 8 09:26:15 2002 From: jeremy at zope.com (Jeremy Hylton) Date: Sun Aug 10 16:31:17 2008 Subject: [ZEO-Checkins] CVS: StandaloneZODB/ZEO - smac.py:1.9.6.8 Message-ID: <200201081426.g08EQFh12152@cvs.baymountain.com> Update of /cvs-repository/StandaloneZODB/ZEO In directory cvs.zope.org:/tmp/cvs-serv12141 Modified Files: Tag: ZEO-ZRPC-Dev smac.py Log Message: Cleanup imports. Replace zeolog imports with zLOG. Replace string module with string method. Remove other unused imports. XXX This blasted code shouldn't have worked. If you remove a module from a distutils package, you also have to do a distutils clean. I didn't, which seems to have allowed broken code to creep into CVS. === StandaloneZODB/ZEO/smac.py 1.9.6.7 => 1.9.6.8 === __version__ = "$Revision$"[11:-2] -import asyncore, string, struct, sys, Acquisition +import asyncore, struct from Exceptions import Disconnected -from zeolog import LOG, TRACE, ERROR, INFO, BLATHER +from zLOG import LOG, TRACE, ERROR, INFO, BLATHER from types import StringType import socket, errno @@ -174,7 +174,7 @@ inp = d else: inp.append(d) - inp = string.join(inp, '') + inp = "".join(inp) offset = 0 while (offset + msg_size) <= input_len: From jeremy at zope.com Tue Jan 8 10:55:49 2002 From: jeremy at zope.com (Jeremy Hylton) Date: Sun Aug 10 16:31:17 2008 Subject: [ZEO-Checkins] CVS: StandaloneZODB/ZEO/tests - forker.py:1.1.2.8 Message-ID: <200201081555.g08FtnX00794@cvs.baymountain.com> Update of /cvs-repository/StandaloneZODB/ZEO/tests In directory cvs.zope.org:/tmp/cvs-serv783 Modified Files: Tag: ZEO-ZRPC-Dev forker.py Log Message: Don't import hotshot unless profiling is enabled. This allows the code to work with Py ver < 2.2. === StandaloneZODB/ZEO/tests/forker.py 1.1.2.7 => 1.1.2.8 === import asyncore import os -import hotshot import random import socket import sys @@ -19,7 +18,10 @@ import types import ZEO.ClientStorage, ZEO.StorageServer +# Change value of PROFILE to enable server-side profiling PROFILE = 0 +if PROFILE: + import hotshot def get_port(): """Return a port that is not in use. From jeremy at zope.com Tue Jan 8 10:58:10 2002 From: jeremy at zope.com (Jeremy Hylton) Date: Sun Aug 10 16:31:17 2008 Subject: [ZEO-Checkins] CVS: StandaloneZODB/ZEO - ServerStub.py:1.1.2.8 Message-ID: <200201081558.g08FwAH01480@cvs.baymountain.com> Update of /cvs-repository/StandaloneZODB/ZEO In directory cvs.zope.org:/tmp/cvs-serv1469 Modified Files: Tag: ZEO-ZRPC-Dev ServerStub.py Log Message: Track change to register() signature: It takes a second arg. === StandaloneZODB/ZEO/ServerStub.py 1.1.2.7 => 1.1.2.8 === self.rpc = rpc - def register(self, storage_name): - self.rpc.call('register', storage_name) + def register(self, storage_name, read_only): + self.rpc.call('register', storage_name, read_only) def get_info(self): return self.rpc.call('get_info') From jeremy at zope.com Tue Jan 8 10:58:47 2002 From: jeremy at zope.com (Jeremy Hylton) Date: Sun Aug 10 16:31:17 2008 Subject: [ZEO-Checkins] CVS: StandaloneZODB/ZEO - StorageServer.py:1.21.4.17 Message-ID: <200201081558.g08FwlI01956@cvs.baymountain.com> Update of /cvs-repository/StandaloneZODB/ZEO In directory cvs.zope.org:/tmp/cvs-serv1945 Modified Files: Tag: ZEO-ZRPC-Dev StorageServer.py Log Message: Make the newConnection factory return the connection. This doesn't have much effect currently, except to make the log message for new connections useful. === StandaloneZODB/ZEO/StorageServer.py 1.21.4.16 => 1.21.4.17 === c = ManagedServerConnection(sock, addr, None, self) c.register_object(StorageProxy(self, c)) + return c def register(self, storage_id, proxy): """Register a connection's use with a particular storage. From jeremy at zope.com Tue Jan 8 11:00:07 2002 From: jeremy at zope.com (Jeremy Hylton) Date: Sun Aug 10 16:31:17 2008 Subject: [ZEO-Checkins] CVS: StandaloneZODB/ZEO - zrpc2.py:1.1.2.23 Message-ID: <200201081600.g08G07s02113@cvs.baymountain.com> Update of /cvs-repository/StandaloneZODB/ZEO In directory cvs.zope.org:/tmp/cvs-serv2102 Modified Files: Tag: ZEO-ZRPC-Dev zrpc2.py Log Message: If an error occurs after a successful connect(), set _thread to None before closing the connection. This avoids a race where the start-connect-thread-on-close code sees that there is already a thread (the one that is about to exit with an error) and doesn't start a new thread. === StandaloneZODB/ZEO/zrpc2.py 1.1.2.22 => 1.1.2.23 === self.obj.notifyConnected(c) except: + self._thread = None c.close() # When the connection is closed, we'll trigger # another attempt to reconnect. - finally: # must always clear _thread on the way out self._thread = None From jeremy at zope.com Tue Jan 8 12:31:28 2002 From: jeremy at zope.com (Jeremy Hylton) Date: Sun Aug 10 16:31:17 2008 Subject: [ZEO-Checkins] CVS: StandaloneZODB/ZEO - ClientStorage.py:1.26.4.31 Message-ID: <200201081731.g08HVS324658@cvs.baymountain.com> Update of /cvs-repository/StandaloneZODB/ZEO In directory cvs.zope.org:/tmp/cvs-serv24647 Modified Files: Tag: ZEO-ZRPC-Dev ClientStorage.py Log Message: Cleanup constructor a bit. Remove some un-used instance variables. Put related uses of constructor args next to each other. === StandaloneZODB/ZEO/ClientStorage.py 1.26.4.30 => 1.26.4.31 === wait_for_server_on_startup=1): - # Decide whether to use non-temporary files - client = client or os.environ.get('ZEO_CLIENT','') - self._server = disconnected_stub self._is_read_only = read_only - self._connection = addr self._storage = storage - self._debug = debug self._info = {'length': 0, 'size': 0, 'name': 'ZEO Client', 'supportsUndo':0, 'supportsVersions': 0} - name = name or str(addr) - self._tbuf = TransactionBuffer() + self._db = None self._oids = [] # XXX It's confusing to have _serial, _serials, and _seriald. self._serials = [] self._seriald = {} - self._basic_init(name) + self._basic_init(name or str(addr)) - self._db = None + # Decide whether to use non-temporary files + client = client or os.environ.get('ZEO_CLIENT','') self._cache = ClientCache.ClientCache(storage, cache_size, client=client, var=var) self._cache.open() # XXX From jeremy at zope.com Tue Jan 8 13:12:13 2002 From: jeremy at zope.com (Jeremy Hylton) Date: Sun Aug 10 16:31:17 2008 Subject: [ZEO-Checkins] CVS: StandaloneZODB/ZEO - StorageServer.py:1.21.4.18 Message-ID: <200201081812.g08ICDN01755@cvs.baymountain.com> Update of /cvs-repository/StandaloneZODB/ZEO In directory cvs.zope.org:/tmp/cvs-serv1744 Modified Files: Tag: ZEO-ZRPC-Dev StorageServer.py Log Message: Raise ReadOnlyError in register() if storage is read-only and client isn't. === StandaloneZODB/ZEO/StorageServer.py 1.21.4.17 => 1.21.4.18 === raise ValueError, "unknown storage: %s" % storage_id + if not read_only and storage.isReadOnly(): + raise POSException.ReadOnlyError() + self.__storage_id = storage_id self.__storage = storage self.setup_delegation() From jeremy at zope.com Tue Jan 8 13:12:38 2002 From: jeremy at zope.com (Jeremy Hylton) Date: Sun Aug 10 16:31:17 2008 Subject: [ZEO-Checkins] CVS: StandaloneZODB/ZEO - ClientStorage.py:1.26.4.32 Message-ID: <200201081812.g08ICc602127@cvs.baymountain.com> Update of /cvs-repository/StandaloneZODB/ZEO In directory cvs.zope.org:/tmp/cvs-serv2116 Modified Files: Tag: ZEO-ZRPC-Dev ClientStorage.py Log Message: Move read_only to the end of the list of keyword args. === StandaloneZODB/ZEO/ClientStorage.py 1.26.4.31 => 1.26.4.32 === class ClientStorage: - def __init__(self, addr, storage='1', read_only=0, - cache_size=20000000, name='', client='', debug=0, - var=None, min_disconnect_poll=5, - max_disconnect_poll=300, - wait_for_server_on_startup=1): + def __init__(self, addr, storage='1', cache_size=20000000, + name='', client='', debug=0, var=None, + min_disconnect_poll=5, max_disconnect_poll=300, + wait_for_server_on_startup=1, read_only=0): self._server = disconnected_stub self._is_read_only = read_only @@ -281,6 +280,9 @@ return self._info['supportsTransactionalUndo'] except KeyError: return 0 + + def isReadOnly(self): + return self._is_read_only def _check_trans(self, trans, exc=None): if self._transaction is not trans: From jeremy at zope.com Tue Jan 8 13:12:57 2002 From: jeremy at zope.com (Jeremy Hylton) Date: Sun Aug 10 16:31:17 2008 Subject: [ZEO-Checkins] CVS: StandaloneZODB/ZEO/tests - testZEO.py:1.1.2.17 Message-ID: <200201081812.g08ICvW02224@cvs.baymountain.com> Update of /cvs-repository/StandaloneZODB/ZEO/tests In directory cvs.zope.org:/tmp/cvs-serv2213 Modified Files: Tag: ZEO-ZRPC-Dev testZEO.py Log Message: Enable ReadOnlyStorage tests for ZEO. === StandaloneZODB/ZEO/tests/testZEO.py 1.1.2.16 => 1.1.2.17 === TransactionalUndoStorage, TransactionalUndoVersionStorage, \ PackableStorage, Synchronization, ConflictResolution, RevisionStorage, \ - MTStorage + MTStorage, ReadOnlyStorage from ZODB.tests.MinPO import MinPO from ZODB.tests.StorageTestBase import zodb_unpickle @@ -137,6 +137,7 @@ PackableStorage.PackableStorage, Synchronization.SynchronizedStorage, MTStorage.MTStorage, + ReadOnlyStorage.ReadOnlyStorage, ): """An abstract base class for ZEO tests @@ -183,6 +184,15 @@ self.__fs_base = tempfile.mktemp() self.__super_setUp() + def open(self, read_only=0): + # XXX Needed to support ReadOnlyStorage tests. Ought to be a + # cleaner way. + + # Is this the only way to get the address? + addr = self._storage._rpc_mgr.addr + self._storage.close() + self._storage = ZEO.ClientStorage.ClientStorage(addr, read_only=1) + def getStorage(self): return FileStorage(self.__fs_base, create=1) @@ -372,7 +382,8 @@ base = ZEO.ClientStorage.ClientStorage(self.addr, client=cache, cache_size=cache_size, - wait_for_server_on_startup=wait) + wait_for_server_on_startup=wait, + min_disconnect_poll=0.1) storage = PackWaitWrapper(base) storage.registerDB(DummyDB(), None) return storage From jeremy at zope.com Tue Jan 8 15:06:32 2002 From: jeremy at zope.com (Jeremy Hylton) Date: Sun Aug 10 16:31:17 2008 Subject: [ZEO-Checkins] CVS: StandaloneZODB/ZEO - zrpc.py:NONE Message-ID: <200201082006.g08K6WV30287@cvs.baymountain.com> Update of /cvs-repository/StandaloneZODB/ZEO In directory cvs.zope.org:/tmp/cvs-serv30195 Removed Files: Tag: ZEO-ZRPC-Dev zrpc.py Log Message: Not needed for ZEO2. === Removed File StandaloneZODB/ZEO/zrpc.py === From jeremy at zope.com Tue Jan 8 23:40:16 2002 From: jeremy at zope.com (Jeremy Hylton) Date: Sun Aug 10 16:31:17 2008 Subject: [ZEO-Checkins] CVS: StandaloneZODB/ZEO - zrpc2.py:1.1.2.24 Message-ID: <200201090440.g094eGn03193@cvs.baymountain.com> Update of /cvs-repository/StandaloneZODB/ZEO In directory cvs.zope.org:/tmp/cvs-serv3178 Modified Files: Tag: ZEO-ZRPC-Dev zrpc2.py Log Message: Change ConnectionManager to support multiple server addresses. This is a significant reworking of the connect() logic in ConnectionManager. It works on an arbitrary list of server addresses. It opens a single socket for each address and does a non-blocking connect. It stops when it gets the first successful connection. The __m_connect() method is the entry point for the connection thread. The self.addr attribute has changed its use. It doesn't sort a single addresss, it stores a list of domain X address 2-tuples. The constructor accepts a single address or a list and parses each in advance to determine if it's a Unix domain socket or an Internet domain socket. Two other changes: - The ConnectionManager keeps a reference to its connection and closes the connection when the manager's close() is executed. - The notifyDisconnected() callback no longer passes an unused None. === StandaloneZODB/ZEO/zrpc2.py 1.1.2.23 => 1.1.2.24 === import asyncore +import errno import cPickle import os +import select import socket import sys import threading @@ -385,12 +387,13 @@ # notifyDisconnected. make this optional? def __init__(self, addr, obj=None, debug=1, tmin=1, tmax=180): - self.addr = addr + self.set_addr(addr) self.obj = obj self.tmin = tmin self.tmax = tmax self.debug = debug self.connected = 0 + self.connection = None # If _thread is not None, then there is a helper thread # attempting to connect. _thread is protected by _connect_lock. self._thread = None @@ -403,6 +406,38 @@ def __repr__(self): return "<%s for %s>" % (self.__class__.__name__, self.addr) + def set_addr(self, addr): + "Set one or more addresses to use for server." + + # For backwards compatibility (and simplicity?) the + # constructor accepts a single address in the addr argument -- + # a string for a Unix domain socket or a 2-tuple with a + # hostname and port. It can also accept a list of such addresses. + + addr_type = self._guess_type(addr) + if addr_type is not None: + self.addr = [(addr_type, addr)] + else: + self.addr = [] + for a in addr: + addr_type = self._guess_type(a) + if addr_type is None: + raise ValueError, "unknown address in list: %s" % repr(a) + self.addr.append((addr_type, a)) + + def _guess_type(self, addr): + if isinstance(addr, types.StringType): + return socket.AF_UNIX + + if (len(addr) == 2 + and isinstance(addr[0], types.StringType) + and isinstance(addr[1], types.IntType)): + return socket.AF_INET + + # not anything I know about + + return None + def close(self): """Prevent ConnectionManager from opening new connections""" self.closed = 1 @@ -412,6 +447,8 @@ self._thread.join() finally: self._connect_lock.release() + if self.connection: + self.connection.close() def register_object(self, obj): self.obj = obj @@ -427,8 +464,9 @@ self._connect_lock.acquire() try: if self._thread is None: - self._thread = threading.Thread(target=self.__connect, - args=(1,)) + zLOG.LOG(_label, zLOG.BLATHER, + "starting thread to connect to server") + self._thread = threading.Thread(target=self.__m_connect) self._thread.start() if sync: try: @@ -440,72 +478,115 @@ self._connect_lock.release() def attempt_connect(self): - self.__connect(repeat=0) + # XXX will _attempt_connects() take too long? think select(). + self._attempt_connects() return self.connected - def __connect(self, repeat=1): - """Attempt to connect to StorageServer. - - This method should always be called by attempt_connect() or by - connect(). - """ + def notify_closed(self, conn): + self.connected = 0 + self.connection = None + self.obj.notifyDisconnected() + if not self.closed: + self.connect() + class Connected(Exception): + def __init__(self, sock): + self.sock = sock + + def __m_connect(self): + # a new __connect that handles multiple addresses try: - tries = 0 - t = self.tmin - while not (self.connected or self.closed) \ - and (repeat or (tries == 0)): - tries = tries + 1 - log("Trying to connect to server") - s = self._connect_socket() - if s is None: - if repeat: - t = self._wait(t) - else: - if self.debug: - log("Connected to server", level=zLOG.DEBUG) - self.connected = 1 - if self.connected and not self.closed: - c = ManagedConnection(s, self.addr, self.obj, self) - log("Connection created: %s" % c) - try: - self.obj.notifyConnected(c) - except: - self._thread = None - c.close() - # When the connection is closed, we'll trigger - # another attempt to reconnect. + delay = self.tmin + while not (self.closed or self._attempt_connects()): + time.sleep(delay) + delay *= 2 + if delay > self.tmax: + delay = self.tmax finally: - # must always clear _thread on the way out self._thread = None - def _connect_socket(self): + def _attempt_connects(self): + "Return true if any connect attempt succeeds." + sockets = {} + + zLOG.LOG(_label, zLOG.BLATHER, + "attempting connection on %d sockets" % len(self.addr)) try: - if type(self.addr) is types.StringType: - s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) - else: - s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - s.connect(self.addr) - except socket.error, msg: - if self.debug: - log("Failed to connect to server: %s" % msg, - level=zLOG.DEBUG) - s.close() - return None - return s + for domain, addr in self.addr: + if __debug__: + zLOG.LOG(_label, zLOG.DEBUG, + "attempt connection to %s" % repr(addr)) + s = socket.socket(domain, socket.SOCK_STREAM) + s.setblocking(0) + # XXX can still block for a while if addr requires DNS + e = self._connect_ex(s, addr) + if e is not None: + sockets[s] = addr + + # next wait until the actually connect + while sockets: + if self.closed: + for s in sockets.keys(): + s.close() + return 0 + try: + r, w, x = select.select([], sockets.keys(), [], 1.0) + except select.error: + continue + for s in w: + e = self._connect_ex(s, sockets[s]) + if e is None: + del sockets[s] + except self.Connected, container: + s = container.sock + del sockets[s] + # close all the other sockets + for s in sockets.keys(): + s.close() + return 1 + return 0 + + def _connect_ex(self, s, addr): + """Call s.connect_ex(addr) and return true if loop should continue. + + We have to handle several possible return values from + connect_ex(). If the socket is connected and the initial ZEO + setup works, we're done. Report success by raising an + exception. Yes, the is odd, but we need to bail out of the + select() loop in the caller and an exception is a principled + way to do the abort. - def _wait(self, t): - time.sleep(t) - t = t * 2 - if t > self.tmax: - t = self.tmax - return t + If the socket sonnects and the initial ZEO setup fails or the + connect_ex() returns an error, we close the socket and ignore it. - def notify_closed(self, conn): - self.connected = 0 - self.obj.notifyDisconnected(None) - if not self.closed: - self.connect() + If connect_ex() returns EINPROGRESS, we need to try again later. + """ + + e = s.connect_ex(addr) + if e == errno.EINPROGRESS: + return 1 + elif e == 0: + c = self._test_connection(s, addr) + if c: + self.connected = 1 + raise self.Connected(s) + else: + if __debug__: + zLOG.LOG(_label, zLOG.DEBUG, + "error connecting to %s: %s" % (addr, + errno.errorcode[e])) + s.close() + + def _test_connection(self, s, addr): + c = ManagedConnection(s, addr, self.obj, self) + try: + self.obj.notifyConnected(c) + self.connection = c + return 1 + except: + # log something here? + c.close() + return 0 class ManagedServerConnection(ServerConnection): """A connection that notifies its ConnectionManager of closing""" From jeremy at zope.com Tue Jan 8 23:41:50 2002 From: jeremy at zope.com (Jeremy Hylton) Date: Sun Aug 10 16:31:17 2008 Subject: [ZEO-Checkins] CVS: StandaloneZODB/ZEO - ClientStorage.py:1.26.4.33 Message-ID: <200201090441.g094foC03310@cvs.baymountain.com> Update of /cvs-repository/StandaloneZODB/ZEO In directory cvs.zope.org:/tmp/cvs-serv3299 Modified Files: Tag: ZEO-ZRPC-Dev ClientStorage.py Log Message: A few small changes to track recent changes to zrpc2. notifyDisconnected() is no longer called with an argument. Don't make wait_for_server_on_startup the default. Don't explicitly close the zrpc2 connection in close(); the manager will take care of it. === StandaloneZODB/ZEO/ClientStorage.py 1.26.4.32 => 1.26.4.33 === name='', client='', debug=0, var=None, min_disconnect_poll=5, max_disconnect_poll=300, - wait_for_server_on_startup=1, read_only=0): + wait_for_server_on_startup=0, read_only=0): self._server = disconnected_stub self._is_read_only = read_only @@ -168,7 +168,7 @@ self._basic_init(name or str(addr)) # Decide whether to use non-temporary files - client = client or os.environ.get('ZEO_CLIENT','') + client = client or os.environ.get('ZEO_CLIENT', '') self._cache = ClientCache.ClientCache(storage, cache_size, client=client, var=var) self._cache.open() # XXX @@ -252,7 +252,7 @@ ### in the middle of notifyDisconnected, because *it's* ### responsible for starting the thread that makes the connection. - def notifyDisconnected(self, ignored): + def notifyDisconnected(self): log2(PROBLEM, "Disconnected from storage") self._server = disconnected_stub if self._transaction: @@ -335,9 +335,9 @@ # Close the manager first, so that it doesn't attempt to # re-open the connection. self._rpc_mgr.close() - if self._server is not disconnected_stub: - # XXX why doesn't the manager close this for us? - self._server.rpc.close() +## if self._server is not disconnected_stub: +## # XXX why doesn't the manager close this for us? +## self._server.rpc.close() if self._cache is not None: self._cache.close() From jeremy at zope.com Tue Jan 8 23:44:32 2002 From: jeremy at zope.com (Jeremy Hylton) Date: Sun Aug 10 16:31:17 2008 Subject: [ZEO-Checkins] CVS: StandaloneZODB/ZEO/tests - forker.py:1.1.2.9 Message-ID: <200201090444.g094iWV04392@cvs.baymountain.com> Update of /cvs-repository/StandaloneZODB/ZEO/tests In directory cvs.zope.org:/tmp/cvs-serv4381 Modified Files: Tag: ZEO-ZRPC-Dev forker.py Log Message: Add explicit wait_for_server_on_startup. The default changed. === StandaloneZODB/ZEO/tests/forker.py 1.1.2.8 => 1.1.2.9 === debug=1, client=cache, cache_size=cache_size, - min_disconnect_poll=0.5) + min_disconnect_poll=0.5, + wait_for_server_on_startup=1) return s, exit, pid From jeremy at zope.com Tue Jan 8 23:47:39 2002 From: jeremy at zope.com (Jeremy Hylton) Date: Sun Aug 10 16:31:17 2008 Subject: [ZEO-Checkins] CVS: StandaloneZODB/ZEO/tests - testZEO.py:1.1.2.18 Message-ID: <200201090447.g094ldc04996@cvs.baymountain.com> Update of /cvs-repository/StandaloneZODB/ZEO/tests In directory cvs.zope.org:/tmp/cvs-serv4985 Modified Files: Tag: ZEO-ZRPC-Dev testZEO.py Log Message: Add two tests of new zrpc2 multiple addresses feature. The tests are a bit kludgey and make the start/shutdownServer() machinery more complicated -- perhaps too complex. But I'm not yet sure what the right refactoring is/would be. The checkMultipleServers() test could be better, but it works without depending on Standby. It just does a dostore() at two different servers using a single ClientStorage. === StandaloneZODB/ZEO/tests/testZEO.py 1.1.2.17 => 1.1.2.18 === self.running = 1 client, exit, pid = forker.start_zeo(self.getStorage()) - self._pid = pid - self._server = exit + self._pids = [pid] + self._servers = [exit] self._storage = PackWaitWrapper(client) client.registerDB(DummyDB(), None) self.__super_setUp() @@ -168,8 +168,10 @@ """Try to cause the tests to halt""" self.running = 0 self._storage.close() - self._server.close() - os.waitpid(self._pid, 0) + for server in self._servers: + server.close() + for pid in self._pids: + os.waitpid(pid, 0) self.delStorage() self.__super_tearDown() @@ -189,9 +191,10 @@ # cleaner way. # Is this the only way to get the address? - addr = self._storage._rpc_mgr.addr + addr = self._storage._rpc_mgr.addr[0][1] self._storage.close() - self._storage = ZEO.ClientStorage.ClientStorage(addr, read_only=1) + self._storage = ZEO.ClientStorage.ClientStorage(addr, read_only=1, + wait_for_server_on_startup=1) def getStorage(self): return FileStorage(self.__fs_base, create=1) @@ -277,24 +280,58 @@ def tearDown(self): """Try to cause the tests to halt""" + if getattr(self, '_storage', None) is not None: + self._storage.close() self.shutdownServer() # file storage appears to create four files - for ext in '', '.index', '.lock', '.tmp': - path = self.file + ext - if os.path.exists(path): - os.unlink(path) + for i in range(len(self.addr)): + for ext in '', '.index', '.lock', '.tmp': + path = "%s.%s%s" % (self.file, i, ext) + if os.path.exists(path): + os.unlink(path) for i in 0, 1: path = "c1-test-%d.zec" % i if os.path.exists(path): os.unlink(path) self.__super_tearDown() + def checkMultipleAddresses(self): + for i in range(4): + self._newAddr() + self._storage = self.openClientStorage('test', 100000, wait=1) + oid = self._storage.new_oid() + obj = MinPO(12) + revid1 = self._dostore(oid, data=obj) + self._storage.close() + + def checkMultipleServers(self): + # XXX crude test at first -- just start two servers and do a + # commit at each one. + + self._newAddr() + self._storage = self.openClientStorage('test', 100000, wait=1) + self._dostore() + + self.shutdownServer(index=0) + self._startServer(index=1) + + # If we can still store after shutting down one of the + # servers, we must be reconnecting to the other server. + + for i in range(10): + try: + self._dostore() + break + except Disconnected: + time.sleep(0.5) + + def checkDisconnectionError(self): # Make sure we get a Disconnected when we try to read an # object when we're not connected to a storage server and the # object is not in the cache. self.shutdownServer() - self._storage = self.openClientStorage('test', 1000, 0) + self._storage = self.openClientStorage('test', 1000, wait=0) self.assertRaises(Disconnected, self._storage.load, 'fredwash', '') def checkBasicPersistence(self): @@ -303,13 +340,13 @@ # To verify that the cache is being used, the test closes the # server and then starts a new client with the server down. - self._storage = self.openClientStorage('test', 100000, 1) + self._storage = self.openClientStorage('test', 100000, wait=1) oid = self._storage.new_oid() obj = MinPO(12) revid1 = self._dostore(oid, data=obj) self._storage.close() self.shutdownServer() - self._storage = self.openClientStorage('test', 100000, 0) + self._storage = self.openClientStorage('test', 100000, wait=0) data, revid2 = self._storage.load(oid, '') assert zodb_unpickle(data) == MinPO(12) assert revid1 == revid2 @@ -321,7 +358,7 @@ # In this case, only one object fits in a cache file. When the # cache files swap, the first object is effectively uncached. - self._storage = self.openClientStorage('test', 1000, 1) + self._storage = self.openClientStorage('test', 1000, wait=1) oid1 = self._storage.new_oid() obj1 = MinPO("1" * 500) revid1 = self._dostore(oid1, data=obj1) @@ -330,13 +367,20 @@ revid2 = self._dostore(oid2, data=obj2) self._storage.close() self.shutdownServer() - self._storage = self.openClientStorage('test', 1000, 0) + self._storage = self.openClientStorage('test', 1000, wait=0) self._storage.load(oid1, '') self._storage.load(oid2, '') def checkReconnection(self): # Check that the client reconnects when a server restarts. + # XXX Seem to get occasional errors that look like this: + # File ZEO/zrpc2.py, line 217, in handle_request + # File ZEO/StorageServer.py, line 325, in storea + # File ZEO/StorageServer.py, line 209, in _check_tid + # StorageTransactionError: (None, ) + # could system reconnect and continue old transaction? + self._storage = self.openClientStorage() oid = self._storage.new_oid() obj = MinPO(12) @@ -350,7 +394,8 @@ try: revid1 = self._dostore(oid, data=obj) break - except (Disconnected, select.error, thread.error, socket.error), err: + except (Disconnected, select.error, thread.error, socket.error), \ + err: get_transaction().abort() time.sleep(0.1) # XXX how long to sleep # XXX This is a bloody pain. We're placing a heavy burden @@ -370,13 +415,25 @@ """ self.running = 1 self.file = tempfile.mktemp() - self.addr = '', self.ports.pop() + self.addr = [] + self._pids = [] + self._servers = [] + self._newAddr() self._startServer() self.__super_setUp() - def _startServer(self, create=1): - fs = FileStorage(self.file, create=create) - self._pid, self._server = forker.start_zeo_server(fs, self.addr) + def _newAddr(self): + self.addr.append(self._getAddr()) + + def _getAddr(self): + return '', self.ports.pop() + + def _startServer(self, create=1, index=0): + fs = FileStorage("%s.%d" % (self.file, index), create=create) + addr = self.addr[index] + pid, server = forker.start_zeo_server(fs, addr) + self._pids.append(pid) + self._servers.append(server) def openClientStorage(self, cache='', cache_size=200000, wait=1): base = ZEO.ClientStorage.ClientStorage(self.addr, @@ -388,18 +445,20 @@ storage.registerDB(DummyDB(), None) return storage - def shutdownServer(self): + def shutdownServer(self, index=0): if self.running: self.running = 0 - self._server.close() + self._servers[index].close() try: - os.waitpid(self._pid, 0) + os.waitpid(self._pids[index], 0) except os.error: pass class WindowsConnectionTests(ConnectionTests): __super_setUp = StorageTestBase.StorageTestBase.setUp + # XXX these tests are now out-of-date + def setUp(self): self.file = tempfile.mktemp() self._startServer() @@ -451,7 +510,8 @@ return meth.keys() if os.name == "posix": - test_classes = ZEOFileStorageTests, UnixConnectionTests +## test_classes = ZEOFileStorageTests, UnixConnectionTests + test_classes = UnixConnectionTests, elif os.name == "nt": test_classes = WindowsZEOFileStorageTests, WindowsConnectionTests else: From jeremy at zope.com Thu Jan 10 01:11:30 2002 From: jeremy at zope.com (Jeremy Hylton) Date: Sun Aug 10 16:31:17 2008 Subject: [ZEO-Checkins] CVS: StandaloneZODB/ZEO - StorageServer.py:1.21.4.19 Message-ID: <200201100611.g0A6BUS05595@cvs.baymountain.com> Update of /cvs-repository/StandaloneZODB/ZEO In directory cvs.zope.org:/tmp/cvs-serv5584 Modified Files: Tag: ZEO-ZRPC-Dev StorageServer.py Log Message: Fix raise of ReadOnlyError; don't include module name. === StandaloneZODB/ZEO/StorageServer.py 1.21.4.18 => 1.21.4.19 === from zrpc2 import Dispatcher, Handler, ManagedServerConnection, Delay from ZODB.POSException import StorageError, StorageTransactionError, \ - TransactionError + TransactionError, ReadOnlyError from ZODB.referencesf import referencesf from ZODB.Transaction import Transaction @@ -122,9 +122,11 @@ pass class StorageServer: - def __init__(self, addr, storages): + def __init__(self, addr, storages, read_only=0): + # XXX should read_only be a per-storage option? not yet... self.addr = addr self.storages = storages + self.read_only = read_only self.connections = {} for name, store in storages.items(): fixup_storage(store) @@ -229,8 +231,8 @@ self._log("unknown storage_id: %s" % storage_id) raise ValueError, "unknown storage: %s" % storage_id - if not read_only and storage.isReadOnly(): - raise POSException.ReadOnlyError() + if not read_only and (self.server.read_only or storage.isReadOnly()): + raise ReadOnlyError() self.__storage_id = storage_id self.__storage = storage From jeremy at zope.com Thu Jan 10 01:12:10 2002 From: jeremy at zope.com (Jeremy Hylton) Date: Sun Aug 10 16:31:17 2008 Subject: [ZEO-Checkins] CVS: StandaloneZODB/ZEO - ClientStorage.py:1.26.4.34 Message-ID: <200201100612.g0A6CAH05666@cvs.baymountain.com> Update of /cvs-repository/StandaloneZODB/ZEO In directory cvs.zope.org:/tmp/cvs-serv5655 Modified Files: Tag: ZEO-ZRPC-Dev ClientStorage.py Log Message: Add an XXX comment and delete some commented-out code. === StandaloneZODB/ZEO/ClientStorage.py 1.26.4.33 => 1.26.4.34 === tmin=min_disconnect_poll, tmax=max_disconnect_poll) + + # XXX What if we can only get a read-only connection and we + # want a read-write connection? Looks like the current code + # will block forever. + if wait_for_server_on_startup: self._rpc_mgr.connect(sync=1) else: @@ -332,12 +337,7 @@ return oids def close(self): - # Close the manager first, so that it doesn't attempt to - # re-open the connection. self._rpc_mgr.close() -## if self._server is not disconnected_stub: -## # XXX why doesn't the manager close this for us? -## self._server.rpc.close() if self._cache is not None: self._cache.close() From jeremy at zope.com Thu Jan 10 19:15:32 2002 From: jeremy at zope.com (Jeremy Hylton) Date: Sun Aug 10 16:31:17 2008 Subject: [ZEO-Checkins] CVS: StandaloneZODB/ZEO - zrpc2.py:1.1.2.25 Message-ID: <200201110015.g0B0FWj21712@cvs.baymountain.com> Update of /cvs-repository/StandaloneZODB/ZEO In directory cvs.zope.org:/tmp/cvs-serv21701 Modified Files: Tag: ZEO-ZRPC-Dev zrpc2.py Log Message: Add a missing zLOG call on successful connect(). Revise XXX comment about a missing one. === StandaloneZODB/ZEO/zrpc2.py 1.1.2.24 => 1.1.2.25 === elif e == 0: c = self._test_connection(s, addr) + zLOG.LOG(_label, zLOG.DEBUG, "connected to %s" % repr(addr)) if c: self.connected = 1 raise self.Connected(s) @@ -584,7 +585,7 @@ self.connection = c return 1 except: - # log something here? + # XXX zLOG the error c.close() return 0 From jeremy at zope.com Thu Jan 10 19:17:19 2002 From: jeremy at zope.com (Jeremy Hylton) Date: Sun Aug 10 16:31:17 2008 Subject: [ZEO-Checkins] CVS: StandaloneZODB/ZEO - StorageServer.py:1.21.4.20 Message-ID: <200201110017.g0B0HJP22365@cvs.baymountain.com> Update of /cvs-repository/StandaloneZODB/ZEO In directory cvs.zope.org:/tmp/cvs-serv22354 Modified Files: Tag: ZEO-ZRPC-Dev StorageServer.py Log Message: Improve close() so that it really shuts the servers down. Also some misc cleanup: No need to assert self._transaction is None, because it's assigned to just two lines above. Replace map + lambda with list comprehension. ZEO 2 will only work with Zope 2.5. === StandaloneZODB/ZEO/StorageServer.py 1.21.4.19 => 1.21.4.20 === """ +import asyncore import cPickle import os import sys @@ -158,7 +159,21 @@ p.client.info(info) def close_server(self): + # Close the dispatcher so that there are no new connections. self.dispatcher.close() + for conn_list in self.connections.values(): + for c in conn_list: + c.close() + for storage in self.storages.values(): + storage.close() + # Force the asyncore mainloop to exit by hackery, i.e. close + # every socket in the map. loop() will return when the map is + # empty. + for s in asyncore.socket_map.values(): + try: + s.close() + except: + pass def close(self, conn): # XXX who calls this? @@ -279,7 +294,6 @@ p, os, v, pv, osv = self.zeoLoad(oid) except: # except what? return None - p = pv = None # free the pickles if os != s: self.client.invalidate((oid, '')) elif osv != sv: @@ -427,7 +441,6 @@ if not self._handle_waiting(): self._transaction = None self.__invalidated = [] - assert self._transaction is None def tpc_abort(self, id): if not self._check_tid(id): @@ -438,7 +451,6 @@ if not self._handle_waiting(): self._transaction = None self.__invalidated = [] - assert self._transaction is None def _restart_delayed_transaction(self, delay, trans): self._transaction = trans @@ -458,7 +470,7 @@ """Return a sequence of n new oids, where n defaults to 100""" if n < 0: n = 1 - return map(lambda x, self=self: self.__storage.new_oid(), range(n)) + return [self.__storage.new_oid() for i in range(n)] def fixup_storage(storage): # backwards compatibility hack From jeremy at zope.com Thu Jan 10 19:21:18 2002 From: jeremy at zope.com (Jeremy Hylton) Date: Sun Aug 10 16:31:17 2008 Subject: [ZEO-Checkins] CVS: StandaloneZODB/ZEO/tests - forker.py:1.1.2.10 Message-ID: <200201110021.g0B0LI823040@cvs.baymountain.com> Update of /cvs-repository/StandaloneZODB/ZEO/tests In directory cvs.zope.org:/tmp/cvs-serv23029 Modified Files: Tag: ZEO-ZRPC-Dev forker.py Log Message: With new close_server() method, the ZEOServerExit fd will be closed when asyncore.loop() returns. === StandaloneZODB/ZEO/tests/forker.py 1.1.2.9 => 1.1.2.10 === server = ZEO.StorageServer.StorageServer(addr, {'1':storage}) asyncore.loop() - os.close(rd) storage.close() if isinstance(addr, types.StringType): os.unlink(addr) From jeremy at zope.com Thu Jan 10 19:22:12 2002 From: jeremy at zope.com (Jeremy Hylton) Date: Sun Aug 10 16:31:17 2008 Subject: [ZEO-Checkins] CVS: StandaloneZODB/ZEO - StorageServer.py:1.21.4.21 Message-ID: <200201110022.g0B0MCl23573@cvs.baymountain.com> Update of /cvs-repository/StandaloneZODB/ZEO In directory cvs.zope.org:/tmp/cvs-serv23562 Modified Files: Tag: ZEO-ZRPC-Dev StorageServer.py Log Message: Previous checkin was too thorough! StorageProxy objects don't have a close() method. === StandaloneZODB/ZEO/StorageServer.py 1.21.4.20 => 1.21.4.21 === # Close the dispatcher so that there are no new connections. self.dispatcher.close() - for conn_list in self.connections.values(): - for c in conn_list: - c.close() for storage in self.storages.values(): storage.close() # Force the asyncore mainloop to exit by hackery, i.e. close From jeremy at zope.com Fri Jan 11 14:29:52 2002 From: jeremy at zope.com (Jeremy Hylton) Date: Sun Aug 10 16:31:17 2008 Subject: [ZEO-Checkins] CVS: StandaloneZODB/ZEO/tests - testTransactionBuffer.py:1.2 forker.py:1.11 Message-ID: <200201111929.g0BJTqS29440@cvs.baymountain.com> Update of /cvs-repository/StandaloneZODB/ZEO/tests In directory cvs.zope.org:/tmp/cvs-serv29427 Modified Files: forker.py Added Files: testTransactionBuffer.py Log Message: Changes from the ZEO-ZRPC-Dev branch merge. === StandaloneZODB/ZEO/tests/testTransactionBuffer.py 1.1 => 1.2 === +import unittest + +from ZEO.TransactionBuffer import TransactionBuffer + +def random_string(size): + """Return a random string of size size.""" + l = [chr(random.randrange(256)) for i in range(size)] + return "".join(l) + +def new_store_data(): + """Return arbitrary data to use as argument to store() method.""" + return random_string(8), '', random_string(random.randrange(1000)) + +def new_invalidate_data(): + """Return arbitrary data to use as argument to invalidate() method.""" + return random_string(8), '' + +class TransBufTests(unittest.TestCase): + + def checkTypicalUsage(self): + tbuf = TransactionBuffer() + tbuf.store(*new_store_data()) + tbuf.invalidate(*new_invalidate_data()) + tbuf.begin_iterate() + while 1: + o = tbuf.next() + if o is None: + break + tbuf.clear() + + def doUpdates(self, tbuf): + data = [] + for i in range(10): + d = new_store_data() + tbuf.store(*d) + data.append(d) + d = new_invalidate_data() + tbuf.invalidate(*d) + data.append(d) + + tbuf.begin_iterate() + for i in range(len(data)): + x = tbuf.next() + if x[2] is None: + # the tbuf add a dummy None to invalidates + x = x[:2] + self.assertEqual(x, data[i]) + + def checkOrderPreserved(self): + tbuf = TransactionBuffer() + self.doUpdates(tbuf) + + def checkReusable(self): + tbuf = TransactionBuffer() + self.doUpdates(tbuf) + tbuf.clear() + self.doUpdates(tbuf) + tbuf.clear() + self.doUpdates(tbuf) + +def test_suite(): + return unittest.makeSuite(TransBufTests, 'check') + === StandaloneZODB/ZEO/tests/forker.py 1.10 => 1.11 === +# +# This software is subject to the provisions of the Zope Public License, +# Version 1.1 (ZPL). A copy of the ZPL should accompany this +# distribution. THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL +# EXPRESS OR IMPLIED WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST +# INFRINGEMENT, AND FITNESS FOR A PARTICULAR PURPOSE. + """Library for forking storage server and connecting client storage""" import asyncore import os -import profile import random import socket import sys +import traceback import types import ZEO.ClientStorage, ZEO.StorageServer +# Change value of PROFILE to enable server-side profiling PROFILE = 0 +if PROFILE: + import hotshot def get_port(): """Return a port that is not in use. @@ -66,9 +78,11 @@ buf = self.recv(4) if buf: assert buf == "done" + server.close_server() asyncore.socket_map.clear() def handle_close(self): + server.close_server() asyncore.socket_map.clear() class ZEOClientExit: @@ -77,20 +91,27 @@ self.pipe = pipe def close(self): - os.write(self.pipe, "done") - os.close(self.pipe) + try: + os.write(self.pipe, "done") + os.close(self.pipe) + except os.error: + pass def start_zeo_server(storage, addr): rd, wr = os.pipe() pid = os.fork() if pid == 0: - if PROFILE: - p = profile.Profile() - p.runctx("run_server(storage, addr, rd, wr)", globals(), - locals()) - p.dump_stats("stats.s.%d" % os.getpid()) - else: - run_server(storage, addr, rd, wr) + try: + if PROFILE: + p = hotshot.Profile("stats.s.%d" % os.getpid()) + p.runctx("run_server(storage, addr, rd, wr)", + globals(), locals()) + p.close() + else: + run_server(storage, addr, rd, wr) + except: + print "Exception in ZEO server process" + traceback.print_exc() os._exit(0) else: os.close(rd) @@ -98,11 +119,11 @@ def run_server(storage, addr, rd, wr): # in the child, run the storage server + global server os.close(wr) ZEOServerExit(rd) - serv = ZEO.StorageServer.StorageServer(addr, {'1':storage}) + server = ZEO.StorageServer.StorageServer(addr, {'1':storage}) asyncore.loop() - os.close(rd) storage.close() if isinstance(addr, types.StringType): os.unlink(addr) @@ -128,6 +149,7 @@ s = ZEO.ClientStorage.ClientStorage(addr, storage_id, debug=1, client=cache, cache_size=cache_size, - min_disconnect_poll=0.5) + min_disconnect_poll=0.5, + wait_for_server_on_startup=1) return s, exit, pid From jeremy at zope.com Fri Jan 11 14:33:20 2002 From: jeremy at zope.com (Jeremy Hylton) Date: Sun Aug 10 16:31:18 2008 Subject: [ZEO-Checkins] CVS: StandaloneZODB/ZEO - ClientStub.py:1.2 Exceptions.py:1.2 ServerStub.py:1.2 TransactionBuffer.py:1.2 zrpc2.py:1.2 ClientCache.py:1.19 ClientStorage.py:1.36 StorageServer.py:1.33 smac.py:1.12 start.py:1.27 Message-ID: <200201111933.g0BJXKu30700@cvs.baymountain.com> Update of /cvs-repository/StandaloneZODB/ZEO In directory cvs.zope.org:/tmp/cvs-serv30669 Modified Files: ClientCache.py ClientStorage.py StorageServer.py smac.py start.py Added Files: ClientStub.py Exceptions.py ServerStub.py TransactionBuffer.py zrpc2.py Log Message: Changes from the ZEO-ZRPC-Dev branch merge. === StandaloneZODB/ZEO/ClientStub.py 1.1 => 1.2 === + +class ClientStorage: + def __init__(self, rpc): + self.rpc = rpc + + def beginVerify(self): + self.rpc.callAsync('begin') + + # XXX what's the difference between these two? + + def invalidate(self, args): + self.rpc.callAsync('invalidate', args) + + def Invalidate(self, args): + self.rpc.callAsync('Invalidate', args) + + def endVerify(self): + self.rpc.callAsync('end') + + def serialno(self, arg): + self.rpc.callAsync('serialno', arg) + + def info(self, arg): + self.rpc.callAsync('info', arg) === StandaloneZODB/ZEO/Exceptions.py 1.1 => 1.2 === + +class Disconnected(Exception): + """Exception raised when a ZEO client is disconnected from the + ZEO server.""" === StandaloneZODB/ZEO/ServerStub.py 1.1 => 1.2 === + +class StorageServer: + + def __init__(self, rpc): + self.rpc = rpc + + def register(self, storage_name, read_only): + self.rpc.call('register', storage_name, read_only) + + def get_info(self): + return self.rpc.call('get_info') + + def get_size_info(self): + return self.rpc.call('get_size_info') + + def beginZeoVerify(self): + self.rpc.callAsync('beginZeoVerify') + + def zeoVerify(self, oid, s, sv): + self.rpc.callAsync('zeoVerify', oid, s, sv) + + def endZeoVerify(self): + self.rpc.callAsync('endZeoVerify') + + def new_oids(self, n=None): + if n is None: + return self.rpc.call('new_oids') + else: + return self.rpc.call('new_oids', n) + + def pack(self, t, wait=None): + if wait is None: + self.rpc.call('pack', t) + else: + self.rpc.call('pack', t, wait) + + def zeoLoad(self, oid): + return self.rpc.call('zeoLoad', oid) + + def storea(self, oid, serial, data, version, id): + self.rpc.callAsync('storea', oid, serial, data, version, id) + + def tpc_begin(self, id, user, descr, ext): + return self.rpc.call('tpc_begin', id, user, descr, ext) + + def vote(self, trans_id): + return self.rpc.call('vote', trans_id) + + def tpc_finish(self, id): + return self.rpc.call('tpc_finish', id) + + def tpc_abort(self, id): + self.rpc.callAsync('tpc_abort', id) + + def abortVersion(self, src, id): + return self.rpc.call('abortVersion', src, id) + + def commitVersion(self, src, dest, id): + return self.rpc.call('commitVersion', src, dest, id) + + def history(self, oid, version, length=None): + if length is not None: + return self.rpc.call('history', oid, version) + else: + return self.rpc.call('history', oid, version, length) + + def load(self, oid, version): + return self.rpc.call('load', oid, version) + + def loadSerial(self, oid, serial): + return self.rpc.call('loadSerial', oid, serial) + + def modifiedInVersion(self, oid): + return self.rpc.call('modifiedInVersion', oid) + + def new_oid(self, last=None): + if last is None: + return self.rpc.call('new_oid') + else: + return self.rpc.call('new_oid', last) + + def store(self, oid, serial, data, version, trans): + return self.rpc.call('store', oid, serial, data, version, trans) + + def transactionalUndo(self, trans_id, trans): + return self.rpc.call('transactionalUndo', trans_id, trans) + + def undo(self, trans_id): + return self.rpc.call('undo', trans_id) + + def undoLog(self, first, last): + # XXX filter not allowed across RPC + return self.rpc.call('undoLog', first, last) + + def undoInfo(self, first, last, spec): + return self.rpc.call('undoInfo', first, last, spec) + + def versionEmpty(self, vers): + return self.rpc.call('versionEmpty', vers) + + def versions(self, max=None): + if max is None: + return self.rpc.call('versions') + else: + return self.rpc.call('versions', max) + + === StandaloneZODB/ZEO/TransactionBuffer.py 1.1 => 1.2 === + +A transaction may generate enough data that it is not practical to +always hold pending updates in memory. Instead, a TransactionBuffer +is used to store the data until a commit or abort. +""" + +# XXX Figure out what a sensible storage format is + +# XXX A faster implementation might store trans data in memory until +# it reaches a certain size. + +import tempfile +import cPickle + +class TransactionBuffer: + + def __init__(self): + self.file = tempfile.TemporaryFile() + self.count = 0 + self.size = 0 + # It's safe to use a fast pickler because the only objects + # stored are builtin types -- strings or None. + self.pickler = cPickle.Pickler(self.file, 1) + self.pickler.fast = 1 + + def store(self, oid, version, data): + """Store oid, version, data for later retrieval""" + self.pickler.dump((oid, version, data)) + self.count += 1 + # Estimate per-record cache size + self.size = self.size + len(data) + (27 + 12) + if version: + self.size = self.size + len(version) + 4 + + def invalidate(self, oid, version): + self.pickler.dump((oid, version, None)) + self.count += 1 + + def clear(self): + """Mark the buffer as empty""" + self.file.seek(0) + self.count = 0 + self.size = 0 + + # XXX unchecked constraints: + # 1. can't call store() after begin_iterate() + # 2. must call clear() after iteration finishes + + def begin_iterate(self): + """Move the file pointer in advance of iteration""" + self.file.flush() + self.file.seek(0) + self.unpickler = cPickle.Unpickler(self.file) + + def next(self): + """Return next tuple of data or None if EOF""" + if self.count == 0: + del self.unpickler + return None + oid_ver_data = self.unpickler.load() + self.count -= 1 + return oid_ver_data + + def get_size(self): + """Return size of data stored in buffer (just a hint).""" + + return self.size === StandaloneZODB/ZEO/zrpc2.py 1.1 => 1.2 === (615/715 lines abridged) + +The basic protocol is as: +a pickled tuple containing: msgid, flags, method, args + +msgid is an integer. +flags is an integer. + The only currently defined flag is ASYNC (0x1), which means + the client does not expect a reply. +method is a string specifying the method to invoke. + For a reply, the method is ".reply". +args is a tuple of the argument to pass to method. + +XXX need to specify a version number that describes the protocol. +allow for future revision. + +XXX support multiple outstanding calls + +XXX factor out common pattern of deciding what protocol to use based +on whether address is tuple or string +""" + +import asyncore +import errno +import cPickle +import os +import select +import socket +import sys +import threading +import thread +import time +import traceback +import types + +from cStringIO import StringIO + +from ZODB import POSException +from ZEO import smac, trigger +from Exceptions import Disconnected +import zLOG +import ThreadedAsync +from Exceptions import Disconnected + +REPLY = ".reply" # message name used for replies +ASYNC = 1 + +_label = "zrpc:%s" % os.getpid() + +def new_label(): + global _label [-=- -=- -=- 615 lines omitted -=- -=- -=-] + + def readable(self): + return 1 + + def handle_accept(self): + try: + sock, addr = self.accept() + except socket.error, msg: + log("accepted failed: %s" % msg) + return + c = self.factory(sock, addr, self.obj) + log("connect from %s: %s" % (repr(addr), c)) + self.clients.append(c) + +class Handler: + """Base class used to handle RPC caller discovery""" + + def set_caller(self, addr): + self.__caller = addr + + def get_caller(self): + return self.__caller + + def clear_caller(self): + self.__caller = None + +_globals = globals() +_silly = ('__doc__',) + +def find_global(module, name): + """Helper for message unpickler""" + try: + m = __import__(module, _globals, _globals, _silly) + except ImportError, msg: + raise ZRPCError("import error %s: %s" % (module, msg)) + + try: + r = getattr(m, name) + except AttributeError: + raise ZRPCError("module %s has no global %s" % (module, name)) + + safe = getattr(r, '__no_side_effects__', 0) + if safe: + return r + + if type(r) == types.ClassType and issubclass(r, Exception): + return r + + raise ZRPCError("Unsafe global: %s.%s" % (module, name)) + === StandaloneZODB/ZEO/ClientCache.py 1.18 => 1.19 === from struct import pack, unpack from thread import allocate_lock -import zLOG -magic='ZEC0' +import sys +import zLOG -def LOG(msg, level=zLOG.BLATHER): +def log(msg, level=zLOG.INFO): zLOG.LOG("ZEC", level, msg) +magic='ZEC0' + class ClientCache: def __init__(self, storage='', size=20000000, client=None, var=None): @@ -211,16 +213,14 @@ f[0].write(magic) current=0 + log("cache opened. current = %s" % current) + self._limit=size/2 self._current=current - def close(self): - try: - self._f[self._current].close() - except (os.error, ValueError): - pass - def open(self): + # XXX open is overloaded to perform two tasks for + # optimization reasons self._acquire() try: self._index=index={} @@ -235,6 +235,19 @@ return serial.items() finally: self._release() + def close(self): + for f in self._f: + if f is not None: + f.close() + + def verify(self, verifyFunc): + """Call the verifyFunc on every object in the cache. + + verifyFunc(oid, serialno, version) + """ + for oid, (s, vs) in self.open(): + verifyFunc(oid, s, vs) + def invalidate(self, oid, version): self._acquire() try: @@ -373,8 +386,6 @@ self._f[current]=open(self._p[current],'w+b') else: # Temporary cache file: - if self._f[current] is not None: - self._f[current].close() self._f[current] = tempfile.TemporaryFile(suffix='.zec') self._f[current].write(magic) self._pos=pos=4 @@ -383,55 +394,57 @@ def store(self, oid, p, s, version, pv, sv): self._acquire() - try: self._store(oid, p, s, version, pv, sv) - finally: self._release() + try: + self._store(oid, p, s, version, pv, sv) + finally: + self._release() def _store(self, oid, p, s, version, pv, sv): if not s: - p='' - s='\0\0\0\0\0\0\0\0' - tlen=31+len(p) + p = '' + s = '\0\0\0\0\0\0\0\0' + tlen = 31 + len(p) if version: - tlen=tlen+len(version)+12+len(pv) - vlen=len(version) + tlen = tlen + len(version) + 12 + len(pv) + vlen = len(version) else: - vlen=0 + vlen = 0 - pos=self._pos - current=self._current - f=self._f[current] - f.seek(pos) - stlen=pack(">I",tlen) - write=f.write - write(oid+'v'+stlen+pack(">HI", vlen, len(p))+s) - if p: write(p) + stlen = pack(">I", tlen) + # accumulate various data to write into a list + l = [oid, 'v', stlen, pack(">HI", vlen, len(p)), s] + if p: + l.append(p) if version: - write(version) - write(pack(">I", len(pv))) - write(pv) - write(sv) - - write(stlen) + l.extend([version, + pack(">I", len(pv)), + pv, sv]) + l.append(stlen) + f = self._f[self._current] + f.seek(self._pos) + f.write("".join(l)) - if current: self._index[oid]=-pos - else: self._index[oid]=pos + if self._current: + self._index[oid] = - self._pos + else: + self._index[oid] = self._pos - self._pos=pos+tlen + self._pos += tlen def read_index(index, serial, f, current): - LOG("read_index(%s)" % f.name) seek=f.seek read=f.read pos=4 + seek(0,2) + size=f.tell() while 1: - seek(pos) + f.seek(pos) h=read(27) - + if len(h)==27 and h[8] in 'vni': tlen, vlen, dlen = unpack(">iHi", h[9:19]) - else: - break + else: tlen=-1 if tlen <= 0 or vlen < 0 or dlen < 0 or vlen+dlen > tlen: break @@ -466,15 +479,3 @@ except: pass return pos - -def main(files): - for file in files: - print file - index = {} - serial = {} - read_index(index, serial, open(file), 0) - print index.keys() - -if __name__ == "__main__": - import sys - main(sys.argv[1:]) === StandaloneZODB/ZEO/ClientStorage.py 1.35 => 1.36 === (868/968 lines abridged) ############################################################################## """Network ZODB storage client -""" +XXX support multiple outstanding requests up until the vote? +XXX is_connected() vis ClientDisconnected error +""" __version__='$Revision$'[11:-2] -import struct, time, os, socket, string, Sync, zrpc, ClientCache -import tempfile, Invalidator, ExtensionClass, thread -import ThreadedAsync - -now=time.time +import cPickle +import os +import socket +import string +import struct +import sys +import tempfile +import thread +import threading +import time +from types import TupleType, StringType from struct import pack, unpack -from ZODB import POSException, BaseStorage + +import ExtensionClass, Sync, ThreadLock +import ClientCache +import zrpc2 +import ServerStub +from TransactionBuffer import TransactionBuffer + +from ZODB import POSException from ZODB.TimeStamp import TimeStamp -from zLOG import LOG, PROBLEM, INFO +from zLOG import LOG, PROBLEM, INFO, BLATHER +from Exceptions import Disconnected -try: from ZODB.ConflictResolution import ResolvedSerial -except: ResolvedSerial='rs' +def log2(type, msg, subsys="ClientStorage %d" % os.getpid()): + LOG(subsys, type, msg) -TupleType=type(()) +try: + from ZODB.ConflictResolution import ResolvedSerial +except ImportError: + ResolvedSerial = 'rs' [-=- -=- -=- 868 lines omitted -=- -=- -=-] - _w.append(t) - return t + return self._server.versions(max) + + # below are methods invoked by the StorageServer + + def serialno(self, arg): + self._serials.append(arg) + + def info(self, dict): + self._info.update(dict) + + def begin(self): + self._tfile = tempfile.TemporaryFile() + self._pickler = cPickle.Pickler(self._tfile, 1) + self._pickler.fast = 1 # Don't use the memo + + def invalidate(self, args): + if self._pickler is None: + return + self._pickler.dump(args) + + def end(self): + if self._pickler is None: + return + self._pickler.dump((0,0)) +## self._pickler.dump = None + self._tfile.seek(0) + unpick = cPickle.Unpickler(self._tfile) + self._tfile = None + + while 1: + oid, version = unpick.load() + if not oid: + break + self._cache.invalidate(oid, version=version) + self._db.invalidate(oid, version=version) + + def Invalidate(self, args): + # XXX _db could be None + for oid, version in args: + self._cache.invalidate(oid, version=version) + try: + self._db.invalidate(oid, version=version) + except AttributeError, msg: + log2(PROBLEM, + "Invalidate(%s, %s) failed for _db: %s" % (repr(oid), + repr(version), + msg)) + === StandaloneZODB/ZEO/StorageServer.py 1.32 => 1.33 === (750/850 lines abridged) +############################################################################## # # Zope Public License (ZPL) Version 1.0 # ------------------------------------- @@ -59,7 +59,7 @@ # labeled as unofficial distributions. Modifications which do not # carry the name Zope may be packaged in any form, as long as they # conform to all of the clauses above. -# +# # # Disclaimer # @@ -82,527 +82,394 @@ # attributions are listed in the accompanying credits file. # ############################################################################## +"""Network ZODB storage server -__version__ = "$Revision$"[11:-2] +This server acts as a front-end for one or more real storages, like +file storage or Berkeley storage. -import asyncore, socket, string, sys, os -from smac import SizedMessageAsyncConnection -from ZODB import POSException +XXX Need some basic access control-- a declaration of the methods +exported for invocation by the server. +""" + +import asyncore import cPickle -from cPickle import Unpickler -from ZODB.POSException import TransactionError, UndoError, VersionCommitError -from ZODB.Transaction import Transaction -import traceback -from zLOG import LOG, INFO, ERROR, TRACE, BLATHER +import os +import sys +import threading +import types + +import ClientStub +import zrpc2 +import zLOG + +from zrpc2 import Dispatcher, Handler, ManagedServerConnection, Delay +from ZODB.POSException import StorageError, StorageTransactionError, \ + TransactionError, ReadOnlyError from ZODB.referencesf import referencesf [-=- -=- -=- 750 lines omitted -=- -=- -=-] - try: - port='', int(port) - except: - pass - - d = {'1': ZODB.FileStorage.FileStorage(name)} - StorageServer(port, d) - asyncwrap.loop() + self.server.invalidate(self, self.__storage_id, + self.__invalidated, + self.get_size_info()) + + if not self._handle_waiting(): + self._transaction = None + self.__invalidated = [] + + def tpc_abort(self, id): + if not self._check_tid(id): + return + r = self.__storage.tpc_abort(self._transaction) + assert self.__storage._transaction is None + + if not self._handle_waiting(): + self._transaction = None + self.__invalidated = [] + + def _restart_delayed_transaction(self, delay, trans): + self._transaction = trans + self.__storage.tpc_begin(trans) + self.__invalidated = [] + assert self._transaction.id == self.__storage._transaction.id + delay.reply(None) + + def _handle_waiting(self): + if self.__storage.__waiting: + delay, proxy, trans = self.__storage.__waiting.pop(0) + proxy._restart_delayed_transaction(delay, trans) + if self is proxy: + return 1 + + def new_oids(self, n=100): + """Return a sequence of n new oids, where n defaults to 100""" + if n < 0: + n = 1 + return [self.__storage.new_oid() for i in range(n)] + +def fixup_storage(storage): + # backwards compatibility hack + if not hasattr(storage,'tpc_vote'): + storage.tpc_vote = lambda *args: None === StandaloneZODB/ZEO/smac.py 1.11 => 1.12 === __version__ = "$Revision$"[11:-2] -import asyncore, string, struct, zLOG, sys, Acquisition +import asyncore, struct +from Exceptions import Disconnected +from zLOG import LOG, TRACE, ERROR, INFO, BLATHER +from types import StringType + import socket, errno -from zLOG import LOG, TRACE, ERROR, INFO # Use the dictionary to make sure we get the minimum number of errno # entries. We expect that EWOULDBLOCK == EAGAIN on most systems -- @@ -109,81 +112,101 @@ expected_socket_write_errors = tuple(tmp_dict.keys()) del tmp_dict -class SizedMessageAsyncConnection(Acquisition.Explicit, asyncore.dispatcher): +class SizedMessageAsyncConnection(asyncore.dispatcher): + __super_init = asyncore.dispatcher.__init__ + __super_close = asyncore.dispatcher.close + + __closed = 1 # Marker indicating that we're closed - __append=None # Marker indicating that we're closed + socket = None # to outwit Sam's getattr - socket=None # to outwit Sam's getattr + READ_SIZE = 8096 def __init__(self, sock, addr, map=None, debug=None): - SizedMessageAsyncConnection.inheritedAttribute( - '__init__')(self, sock, map) - self.addr=addr + self.__super_init(sock, map) + self.addr = addr if debug is not None: - self._debug=debug + self._debug = debug elif not hasattr(self, '_debug'): - self._debug=__debug__ and 'smac' - self.__state=None - self.__inp=None - self.__inpl=0 - self.__l=4 - self.__output=output=[] - self.__append=output.append - self.__pop=output.pop - - def handle_read(self, - join=string.join, StringType=type(''), _type=type, - _None=None): - + self._debug = __debug__ and 'smac' + self.__state = None + self.__inp = None # None, a single String, or a list + self.__input_len = 0 + self.__msg_size = 4 + self.__output = [] + self.__closed = None + + # XXX avoid expensive getattr calls? + def __nonzero__(self): + return 1 + + def handle_read(self): + # Use a single __inp buffer and integer indexes to make this + # fast. try: d=self.recv(8096) except socket.error, err: if err[0] in expected_socket_read_errors: return raise - if not d: return + if not d: + return - inp=self.__inp - if inp is _None: - inp=d - elif _type(inp) is StringType: - inp=[inp,d] + input_len = self.__input_len + len(d) + msg_size = self.__msg_size + state = self.__state + + inp = self.__inp + if msg_size > input_len: + if inp is None: + self.__inp = d + elif type(self.__inp) is StringType: + self.__inp = [self.__inp, d] + else: + self.__inp.append(d) + self.__input_len = input_len + return # keep waiting for more input + + # load all previous input and d into single string inp + if isinstance(inp, StringType): + inp = inp + d + elif inp is None: + inp = d else: inp.append(d) + inp = "".join(inp) - inpl=self.__inpl+len(d) - l=self.__l - - while 1: - - if l <= inpl: - # Woo hoo, we have enough data - if _type(inp) is not StringType: inp=join(inp,'') - d=inp[:l] - inp=inp[l:] - inpl=inpl-l - if self.__state is _None: - # waiting for message - l=struct.unpack(">i",d)[0] - self.__state=1 - else: - l=4 - self.__state=_None - self.message_input(d) + offset = 0 + while (offset + msg_size) <= input_len: + msg = inp[offset:offset + msg_size] + offset = offset + msg_size + if state is None: + # waiting for message + msg_size = struct.unpack(">i", msg)[0] + state = 1 else: - break # not enough data - - self.__l=l - self.__inp=inp - self.__inpl=inpl + msg_size = 4 + state = None + self.message_input(msg) + + self.__state = state + self.__msg_size = msg_size + self.__inp = inp[offset:] + self.__input_len = input_len - offset - def readable(self): return 1 - def writable(self): return not not self.__output + def readable(self): + return 1 + + def writable(self): + if len(self.__output) == 0: + return 0 + else: + return 1 def handle_write(self): - output=self.__output + output = self.__output while output: - v=output[0] + v = output[0] try: n=self.send(v) except socket.error, err: @@ -191,42 +214,33 @@ break # we couldn't write anything raise if n < len(v): - output[0]=v[n:] + output[0] = v[n:] break # we can't write any more else: del output[0] - #break # waaa - def handle_close(self): self.close() - def message_output(self, message, - pack=struct.pack, len=len): - if self._debug: - if len(message) > 40: m=message[:40]+' ...' - else: m=message - LOG(self._debug, TRACE, 'message_output %s' % `m`) - - append=self.__append - if append is None: - raise Disconnected("This action is temporarily unavailable.

") - - append(pack(">i",len(message))+message) - - def log_info(self, message, type='info'): - if type=='error': type=ERROR - else: type=INFO - LOG('ZEO', type, message) + def message_output(self, message): + if __debug__: + if self._debug: + if len(message) > 40: + m = message[:40]+' ...' + else: + m = message + LOG(self._debug, TRACE, 'message_output %s' % `m`) - log=log_info + if self.__closed is not None: + raise Disconnected, ( + "This action is temporarily unavailable." + "

" + ) + # do two separate appends to avoid copying the message string + self.__output.append(struct.pack(">i", len(message))) + self.__output.append(message) def close(self): - if self.__append is not None: - self.__append=None - SizedMessageAsyncConnection.inheritedAttribute('close')(self) - -class Disconnected(Exception): - """The client has become disconnected from the server - """ - + if self.__closed is None: + self.__closed = 1 + self.__super_close() === StandaloneZODB/ZEO/start.py 1.26 => 1.27 === import sys, os, getopt, string +import StorageServer +import asyncore + def directory(p, n=1): d=p while n: @@ -115,9 +118,11 @@ def main(argv): me=argv[0] - sys.path[:]==filter(None, sys.path) sys.path.insert(0, directory(me, 2)) + # XXX hack for profiling support + global unix, storages, zeo_pid, asyncore + args=[] last='' for a in argv[1:]: @@ -130,25 +135,13 @@ args.append(a) last=a - if os.environ.has_key('INSTANCE_HOME'): - INSTANCE_HOME=os.environ['INSTANCE_HOME'] - elif os.path.isdir(os.path.join(directory(me, 4),'var')): - INSTANCE_HOME=directory(me, 4) - else: - INSTANCE_HOME=os.getcwd() - - if os.path.isdir(os.path.join(INSTANCE_HOME, 'var')): - var=os.path.join(INSTANCE_HOME, 'var') - else: - var=INSTANCE_HOME + INSTANCE_HOME=os.environ.get('INSTANCE_HOME', directory(me, 4)) zeo_pid=os.environ.get('ZEO_SERVER_PID', - os.path.join(var, 'ZEO_SERVER.pid') + os.path.join(INSTANCE_HOME, 'var', 'ZEO_SERVER.pid') ) - opts, args = getopt.getopt(args, 'p:Ddh:U:sS:u:') - - fs=os.path.join(var, 'Data.fs') + fs=os.path.join(INSTANCE_HOME, 'var', 'Data.fs') usage="""%s [options] [filename] @@ -156,17 +149,14 @@ -D -- Run in debug mode - -d -- Generate detailed debug logging without running - in the foreground. - -U -- Unix-domain socket file to listen on -u username or uid number The username to run the ZEO server as. You may want to run the ZEO server as 'nobody' or some other user with limited - resouces. The only works under Unix, and if the storage - server is started by root. + resouces. The only works under Unix, and if ZServer is + started by root. -p port -- port to listen on @@ -189,23 +179,42 @@ attr_name -- This is the name to which the storage object is assigned in the module. + -P file -- Run under profile and dump output to file. Implies the + -s flag. + if no file name is specified, then %s is used. """ % (me, fs) + try: + opts, args = getopt.getopt(args, 'p:Dh:U:sS:u:P:') + except getopt.error, msg: + print usage + print msg + sys.exit(1) + port=None - debug=detailed=0 + debug=0 host='' unix=None Z=1 UID='nobody' + prof = None for o, v in opts: if o=='-p': port=string.atoi(v) elif o=='-h': host=v elif o=='-U': unix=v elif o=='-u': UID=v elif o=='-D': debug=1 - elif o=='-d': detailed=1 elif o=='-s': Z=0 + elif o=='-P': prof = v + + if prof: + Z = 0 + + try: + from ZServer.medusa import asyncore + sys.modules['asyncore']=asyncore + except: pass if port is None and unix is None: print usage @@ -219,10 +228,9 @@ sys.exit(1) fs=args[0] + __builtins__.__debug__=debug if debug: os.environ['Z_DEBUG_MODE']='1' - if detailed: os.environ['STUPID_LOG_SEVERITY']='-99999' - from zLOG import LOG, INFO, ERROR # Try to set uid to "-u" -provided uid. @@ -263,71 +271,54 @@ import zdaemon zdaemon.run(sys.argv, '') - try: - - import ZEO.StorageServer, asyncore - - storages={} - for o, v in opts: - if o=='-S': - n, m = string.split(v,'=') - if string.find(m,':'): - # we got an attribute name - m, a = string.split(m,':') - else: - # attribute name must be same as storage name - a=n - storages[n]=get_storage(m,a) - - if not storages: - import ZODB.FileStorage - storages['1']=ZODB.FileStorage.FileStorage(fs) - - # Try to set up a signal handler - try: - import signal - - signal.signal(signal.SIGTERM, - lambda sig, frame, s=storages: shutdown(s) - ) - signal.signal(signal.SIGINT, - lambda sig, frame, s=storages: shutdown(s, 0) - ) - try: signal.signal(signal.SIGHUP, rotate_logs_handler) - except: pass - - except: pass - - items=storages.items() - items.sort() - for kv in items: - LOG('ZEO Server', INFO, 'Serving %s:\t%s' % kv) - - if not unix: unix=host, port - - ZEO.StorageServer.StorageServer(unix, storages) - - try: ppid, pid = os.getppid(), os.getpid() - except: pass # getpid not supported - else: open(zeo_pid,'w').write("%s %s" % (ppid, pid)) - - except: - # Log startup exception and tell zdaemon not to restart us. - info=sys.exc_info() - try: - import zLOG - zLOG.LOG("z2", zLOG.PANIC, "Startup exception", - error=info) - except: - pass - - import traceback - apply(traceback.print_exception, info) - - sys.exit(0) + storages={} + for o, v in opts: + if o=='-S': + n, m = string.split(v,'=') + if string.find(m,':'): + # we got an attribute name + m, a = string.split(m,':') + else: + # attribute name must be same as storage name + a=n + storages[n]=get_storage(m,a) + + if not storages: + import ZODB.FileStorage + storages['1']=ZODB.FileStorage.FileStorage(fs) - asyncore.loop() + # Try to set up a signal handler + try: + import signal + signal.signal(signal.SIGTERM, + lambda sig, frame, s=storages: shutdown(s) + ) + signal.signal(signal.SIGINT, + lambda sig, frame, s=storages: shutdown(s, 0) + ) + signal.signal(signal.SIGHUP, rotate_logs_handler) + + finally: pass + + items=storages.items() + items.sort() + for kv in items: + LOG('ZEO Server', INFO, 'Serving %s:\t%s' % kv) + + if not unix: unix=host, port + + if prof: + cmds = \ + "StorageServer.StorageServer(unix, storages);" \ + 'open(zeo_pid,"w").write("%s %s" % (os.getppid(), os.getpid()));' \ + "asyncore.loop()" + import profile + profile.run(cmds, prof) + else: + StorageServer.StorageServer(unix, storages) + open(zeo_pid,'w').write("%s %s" % (os.getppid(), os.getpid())) + asyncore.loop() def rotate_logs(): import zLOG @@ -335,10 +326,7 @@ zLOG.log_write.reinitialize() else: # Hm, lets at least try to take care of the stupid logger: - if hasattr(zLOG, '_set_stupid_dest'): - zLOG._set_stupid_dest(None) - else: - zLOG._stupid_dest = None + zLOG._stupid_dest=None def rotate_logs_handler(signum, frame): rotate_logs() @@ -359,7 +347,7 @@ for storage in storages.values(): try: storage.close() - except: pass + finally: pass try: from zLOG import LOG, INFO From jeremy at zope.com Fri Jan 11 14:56:03 2002 From: jeremy at zope.com (Jeremy Hylton) Date: Sun Aug 10 16:31:18 2008 Subject: [ZEO-Checkins] CVS: StandaloneZODB/ZEO - ClientCache.py:1.20 ClientStorage.py:1.37 StorageServer.py:1.34 smac.py:1.13 start.py:1.28 ClientStub.py:NONE Exceptions.py:NONE ServerStub.py:NONE TransactionBuffer.py:NONE zrpc2.py:NONE Message-ID: <200201111956.g0BJu3603182@cvs.baymountain.com> Update of /cvs-repository/StandaloneZODB/ZEO In directory cvs.zope.org:/tmp/cvs-serv3118 Modified Files: ClientCache.py ClientStorage.py StorageServer.py smac.py start.py Removed Files: ClientStub.py Exceptions.py ServerStub.py TransactionBuffer.py zrpc2.py Log Message: Undo merge of ZEO-ZRPC-Dev branch into the trunk (for now). === StandaloneZODB/ZEO/ClientCache.py 1.19 => 1.20 === from struct import pack, unpack from thread import allocate_lock - -import sys import zLOG -def log(msg, level=zLOG.INFO): - zLOG.LOG("ZEC", level, msg) - magic='ZEC0' +def LOG(msg, level=zLOG.BLATHER): + zLOG.LOG("ZEC", level, msg) + class ClientCache: def __init__(self, storage='', size=20000000, client=None, var=None): @@ -213,14 +211,16 @@ f[0].write(magic) current=0 - log("cache opened. current = %s" % current) - self._limit=size/2 self._current=current + def close(self): + try: + self._f[self._current].close() + except (os.error, ValueError): + pass + def open(self): - # XXX open is overloaded to perform two tasks for - # optimization reasons self._acquire() try: self._index=index={} @@ -235,19 +235,6 @@ return serial.items() finally: self._release() - def close(self): - for f in self._f: - if f is not None: - f.close() - - def verify(self, verifyFunc): - """Call the verifyFunc on every object in the cache. - - verifyFunc(oid, serialno, version) - """ - for oid, (s, vs) in self.open(): - verifyFunc(oid, s, vs) - def invalidate(self, oid, version): self._acquire() try: @@ -386,6 +373,8 @@ self._f[current]=open(self._p[current],'w+b') else: # Temporary cache file: + if self._f[current] is not None: + self._f[current].close() self._f[current] = tempfile.TemporaryFile(suffix='.zec') self._f[current].write(magic) self._pos=pos=4 @@ -394,57 +383,55 @@ def store(self, oid, p, s, version, pv, sv): self._acquire() - try: - self._store(oid, p, s, version, pv, sv) - finally: - self._release() + try: self._store(oid, p, s, version, pv, sv) + finally: self._release() def _store(self, oid, p, s, version, pv, sv): if not s: - p = '' - s = '\0\0\0\0\0\0\0\0' - tlen = 31 + len(p) + p='' + s='\0\0\0\0\0\0\0\0' + tlen=31+len(p) if version: - tlen = tlen + len(version) + 12 + len(pv) - vlen = len(version) + tlen=tlen+len(version)+12+len(pv) + vlen=len(version) else: - vlen = 0 + vlen=0 - stlen = pack(">I", tlen) - # accumulate various data to write into a list - l = [oid, 'v', stlen, pack(">HI", vlen, len(p)), s] - if p: - l.append(p) + pos=self._pos + current=self._current + f=self._f[current] + f.seek(pos) + stlen=pack(">I",tlen) + write=f.write + write(oid+'v'+stlen+pack(">HI", vlen, len(p))+s) + if p: write(p) if version: - l.extend([version, - pack(">I", len(pv)), - pv, sv]) - l.append(stlen) - f = self._f[self._current] - f.seek(self._pos) - f.write("".join(l)) + write(version) + write(pack(">I", len(pv))) + write(pv) + write(sv) - if self._current: - self._index[oid] = - self._pos - else: - self._index[oid] = self._pos + write(stlen) + + if current: self._index[oid]=-pos + else: self._index[oid]=pos - self._pos += tlen + self._pos=pos+tlen def read_index(index, serial, f, current): + LOG("read_index(%s)" % f.name) seek=f.seek read=f.read pos=4 - seek(0,2) - size=f.tell() while 1: - f.seek(pos) + seek(pos) h=read(27) - + if len(h)==27 and h[8] in 'vni': tlen, vlen, dlen = unpack(">iHi", h[9:19]) - else: tlen=-1 + else: + break if tlen <= 0 or vlen < 0 or dlen < 0 or vlen+dlen > tlen: break @@ -479,3 +466,15 @@ except: pass return pos + +def main(files): + for file in files: + print file + index = {} + serial = {} + read_index(index, serial, open(file), 0) + print index.keys() + +if __name__ == "__main__": + import sys + main(sys.argv[1:]) === StandaloneZODB/ZEO/ClientStorage.py 1.36 => 1.37 === (868/968 lines abridged) ############################################################################## """Network ZODB storage client - -XXX support multiple outstanding requests up until the vote? -XXX is_connected() vis ClientDisconnected error """ -__version__='$Revision$'[11:-2] -import cPickle -import os -import socket -import string -import struct -import sys -import tempfile -import thread -import threading -import time -from types import TupleType, StringType -from struct import pack, unpack +__version__='$Revision$'[11:-2] -import ExtensionClass, Sync, ThreadLock -import ClientCache -import zrpc2 -import ServerStub -from TransactionBuffer import TransactionBuffer +import struct, time, os, socket, string, Sync, zrpc, ClientCache +import tempfile, Invalidator, ExtensionClass, thread +import ThreadedAsync -from ZODB import POSException +now=time.time +from struct import pack, unpack +from ZODB import POSException, BaseStorage from ZODB.TimeStamp import TimeStamp -from zLOG import LOG, PROBLEM, INFO, BLATHER -from Exceptions import Disconnected +from zLOG import LOG, PROBLEM, INFO -def log2(type, msg, subsys="ClientStorage %d" % os.getpid()): - LOG(subsys, type, msg) +try: from ZODB.ConflictResolution import ResolvedSerial +except: ResolvedSerial='rs' -try: - from ZODB.ConflictResolution import ResolvedSerial -except ImportError: - ResolvedSerial = 'rs' +TupleType=type(()) [-=- -=- -=- 868 lines omitted -=- -=- -=-] - def begin(self): - self._tfile = tempfile.TemporaryFile() - self._pickler = cPickle.Pickler(self._tfile, 1) - self._pickler.fast = 1 # Don't use the memo - - def invalidate(self, args): - if self._pickler is None: - return - self._pickler.dump(args) - - def end(self): - if self._pickler is None: - return - self._pickler.dump((0,0)) -## self._pickler.dump = None - self._tfile.seek(0) - unpick = cPickle.Unpickler(self._tfile) - self._tfile = None - - while 1: - oid, version = unpick.load() - if not oid: - break - self._cache.invalidate(oid, version=version) - self._db.invalidate(oid, version=version) - - def Invalidate(self, args): - # XXX _db could be None - for oid, version in args: - self._cache.invalidate(oid, version=version) - try: - self._db.invalidate(oid, version=version) - except AttributeError, msg: - log2(PROBLEM, - "Invalidate(%s, %s) failed for _db: %s" % (repr(oid), - repr(version), - msg)) - + self._lock_acquire() + try: return self._call('versions', max) + finally: self._lock_release() + + def sync(self): self._call.sync() + +def getWakeup(_w=[]): + if _w: return _w[0] + import trigger + t=trigger.trigger().pull_trigger + _w.append(t) + return t === StandaloneZODB/ZEO/StorageServer.py 1.33 => 1.34 === (750/850 lines abridged) +############################################################################# # # Zope Public License (ZPL) Version 1.0 # ------------------------------------- @@ -59,7 +59,7 @@ # labeled as unofficial distributions. Modifications which do not # carry the name Zope may be packaged in any form, as long as they # conform to all of the clauses above. -# +# # # Disclaimer # @@ -82,394 +82,527 @@ # attributions are listed in the accompanying credits file. # ############################################################################## -"""Network ZODB storage server -This server acts as a front-end for one or more real storages, like -file storage or Berkeley storage. +__version__ = "$Revision$"[11:-2] -XXX Need some basic access control-- a declaration of the methods -exported for invocation by the server. -""" - -import asyncore +import asyncore, socket, string, sys, os +from smac import SizedMessageAsyncConnection +from ZODB import POSException import cPickle -import os -import sys -import threading -import types - -import ClientStub -import zrpc2 -import zLOG - -from zrpc2 import Dispatcher, Handler, ManagedServerConnection, Delay -from ZODB.POSException import StorageError, StorageTransactionError, \ - TransactionError, ReadOnlyError -from ZODB.referencesf import referencesf +from cPickle import Unpickler +from ZODB.POSException import TransactionError, UndoError, VersionCommitError from ZODB.Transaction import Transaction +import traceback +from zLOG import LOG, INFO, ERROR, TRACE, BLATHER [-=- -=- -=- 750 lines omitted -=- -=- -=-] - assert self.__storage._transaction is None - - if not self._handle_waiting(): - self._transaction = None - self.__invalidated = [] - - def _restart_delayed_transaction(self, delay, trans): - self._transaction = trans - self.__storage.tpc_begin(trans) - self.__invalidated = [] - assert self._transaction.id == self.__storage._transaction.id - delay.reply(None) - - def _handle_waiting(self): - if self.__storage.__waiting: - delay, proxy, trans = self.__storage.__waiting.pop(0) - proxy._restart_delayed_transaction(delay, trans) - if self is proxy: - return 1 - - def new_oids(self, n=100): - """Return a sequence of n new oids, where n defaults to 100""" - if n < 0: - n = 1 - return [self.__storage.new_oid() for i in range(n)] - -def fixup_storage(storage): - # backwards compatibility hack - if not hasattr(storage,'tpc_vote'): - storage.tpc_vote = lambda *args: None + self.__server.invalidate(self, self.__storage_id, + self.__invalidated, + self.get_size_info()) + self.__invalidated=[] + +def init_storage(storage): + if not hasattr(storage,'tpc_vote'): storage.tpc_vote=lambda *args: None + +if __name__=='__main__': + import ZODB.FileStorage + name, port = sys.argv[1:3] + blather(name, port) + try: + port='', int(port) + except: + pass + + d = {'1': ZODB.FileStorage.FileStorage(name)} + StorageServer(port, d) + asyncwrap.loop() === StandaloneZODB/ZEO/smac.py 1.12 => 1.13 === __version__ = "$Revision$"[11:-2] -import asyncore, struct -from Exceptions import Disconnected -from zLOG import LOG, TRACE, ERROR, INFO, BLATHER -from types import StringType - +import asyncore, string, struct, zLOG, sys, Acquisition import socket, errno +from zLOG import LOG, TRACE, ERROR, INFO # Use the dictionary to make sure we get the minimum number of errno # entries. We expect that EWOULDBLOCK == EAGAIN on most systems -- @@ -112,101 +109,81 @@ expected_socket_write_errors = tuple(tmp_dict.keys()) del tmp_dict -class SizedMessageAsyncConnection(asyncore.dispatcher): - __super_init = asyncore.dispatcher.__init__ - __super_close = asyncore.dispatcher.close - - __closed = 1 # Marker indicating that we're closed +class SizedMessageAsyncConnection(Acquisition.Explicit, asyncore.dispatcher): - socket = None # to outwit Sam's getattr + __append=None # Marker indicating that we're closed - READ_SIZE = 8096 + socket=None # to outwit Sam's getattr def __init__(self, sock, addr, map=None, debug=None): - self.__super_init(sock, map) - self.addr = addr + SizedMessageAsyncConnection.inheritedAttribute( + '__init__')(self, sock, map) + self.addr=addr if debug is not None: - self._debug = debug + self._debug=debug elif not hasattr(self, '_debug'): - self._debug = __debug__ and 'smac' - self.__state = None - self.__inp = None # None, a single String, or a list - self.__input_len = 0 - self.__msg_size = 4 - self.__output = [] - self.__closed = None - - # XXX avoid expensive getattr calls? - def __nonzero__(self): - return 1 - - def handle_read(self): - # Use a single __inp buffer and integer indexes to make this - # fast. + self._debug=__debug__ and 'smac' + self.__state=None + self.__inp=None + self.__inpl=0 + self.__l=4 + self.__output=output=[] + self.__append=output.append + self.__pop=output.pop + + def handle_read(self, + join=string.join, StringType=type(''), _type=type, + _None=None): + try: d=self.recv(8096) except socket.error, err: if err[0] in expected_socket_read_errors: return raise - if not d: - return + if not d: return - input_len = self.__input_len + len(d) - msg_size = self.__msg_size - state = self.__state - - inp = self.__inp - if msg_size > input_len: - if inp is None: - self.__inp = d - elif type(self.__inp) is StringType: - self.__inp = [self.__inp, d] - else: - self.__inp.append(d) - self.__input_len = input_len - return # keep waiting for more input - - # load all previous input and d into single string inp - if isinstance(inp, StringType): - inp = inp + d - elif inp is None: - inp = d + inp=self.__inp + if inp is _None: + inp=d + elif _type(inp) is StringType: + inp=[inp,d] else: inp.append(d) - inp = "".join(inp) - offset = 0 - while (offset + msg_size) <= input_len: - msg = inp[offset:offset + msg_size] - offset = offset + msg_size - if state is None: - # waiting for message - msg_size = struct.unpack(">i", msg)[0] - state = 1 + inpl=self.__inpl+len(d) + l=self.__l + + while 1: + + if l <= inpl: + # Woo hoo, we have enough data + if _type(inp) is not StringType: inp=join(inp,'') + d=inp[:l] + inp=inp[l:] + inpl=inpl-l + if self.__state is _None: + # waiting for message + l=struct.unpack(">i",d)[0] + self.__state=1 + else: + l=4 + self.__state=_None + self.message_input(d) else: - msg_size = 4 - state = None - self.message_input(msg) - - self.__state = state - self.__msg_size = msg_size - self.__inp = inp[offset:] - self.__input_len = input_len - offset + break # not enough data + + self.__l=l + self.__inp=inp + self.__inpl=inpl - def readable(self): - return 1 - - def writable(self): - if len(self.__output) == 0: - return 0 - else: - return 1 + def readable(self): return 1 + def writable(self): return not not self.__output def handle_write(self): - output = self.__output + output=self.__output while output: - v = output[0] + v=output[0] try: n=self.send(v) except socket.error, err: @@ -214,33 +191,42 @@ break # we couldn't write anything raise if n < len(v): - output[0] = v[n:] + output[0]=v[n:] break # we can't write any more else: del output[0] + #break # waaa + def handle_close(self): self.close() - def message_output(self, message): - if __debug__: - if self._debug: - if len(message) > 40: - m = message[:40]+' ...' - else: - m = message - LOG(self._debug, TRACE, 'message_output %s' % `m`) + def message_output(self, message, + pack=struct.pack, len=len): + if self._debug: + if len(message) > 40: m=message[:40]+' ...' + else: m=message + LOG(self._debug, TRACE, 'message_output %s' % `m`) + + append=self.__append + if append is None: + raise Disconnected("This action is temporarily unavailable.

") + + append(pack(">i",len(message))+message) + + def log_info(self, message, type='info'): + if type=='error': type=ERROR + else: type=INFO + LOG('ZEO', type, message) - if self.__closed is not None: - raise Disconnected, ( - "This action is temporarily unavailable." - "

" - ) - # do two separate appends to avoid copying the message string - self.__output.append(struct.pack(">i", len(message))) - self.__output.append(message) + log=log_info def close(self): - if self.__closed is None: - self.__closed = 1 - self.__super_close() + if self.__append is not None: + self.__append=None + SizedMessageAsyncConnection.inheritedAttribute('close')(self) + +class Disconnected(Exception): + """The client has become disconnected from the server + """ + === StandaloneZODB/ZEO/start.py 1.27 => 1.28 === import sys, os, getopt, string -import StorageServer -import asyncore - def directory(p, n=1): d=p while n: @@ -118,11 +115,9 @@ def main(argv): me=argv[0] + sys.path[:]==filter(None, sys.path) sys.path.insert(0, directory(me, 2)) - # XXX hack for profiling support - global unix, storages, zeo_pid, asyncore - args=[] last='' for a in argv[1:]: @@ -135,13 +130,25 @@ args.append(a) last=a - INSTANCE_HOME=os.environ.get('INSTANCE_HOME', directory(me, 4)) + if os.environ.has_key('INSTANCE_HOME'): + INSTANCE_HOME=os.environ['INSTANCE_HOME'] + elif os.path.isdir(os.path.join(directory(me, 4),'var')): + INSTANCE_HOME=directory(me, 4) + else: + INSTANCE_HOME=os.getcwd() + + if os.path.isdir(os.path.join(INSTANCE_HOME, 'var')): + var=os.path.join(INSTANCE_HOME, 'var') + else: + var=INSTANCE_HOME zeo_pid=os.environ.get('ZEO_SERVER_PID', - os.path.join(INSTANCE_HOME, 'var', 'ZEO_SERVER.pid') + os.path.join(var, 'ZEO_SERVER.pid') ) - fs=os.path.join(INSTANCE_HOME, 'var', 'Data.fs') + opts, args = getopt.getopt(args, 'p:Ddh:U:sS:u:') + + fs=os.path.join(var, 'Data.fs') usage="""%s [options] [filename] @@ -149,14 +156,17 @@ -D -- Run in debug mode + -d -- Generate detailed debug logging without running + in the foreground. + -U -- Unix-domain socket file to listen on -u username or uid number The username to run the ZEO server as. You may want to run the ZEO server as 'nobody' or some other user with limited - resouces. The only works under Unix, and if ZServer is - started by root. + resouces. The only works under Unix, and if the storage + server is started by root. -p port -- port to listen on @@ -179,42 +189,23 @@ attr_name -- This is the name to which the storage object is assigned in the module. - -P file -- Run under profile and dump output to file. Implies the - -s flag. - if no file name is specified, then %s is used. """ % (me, fs) - try: - opts, args = getopt.getopt(args, 'p:Dh:U:sS:u:P:') - except getopt.error, msg: - print usage - print msg - sys.exit(1) - port=None - debug=0 + debug=detailed=0 host='' unix=None Z=1 UID='nobody' - prof = None for o, v in opts: if o=='-p': port=string.atoi(v) elif o=='-h': host=v elif o=='-U': unix=v elif o=='-u': UID=v elif o=='-D': debug=1 + elif o=='-d': detailed=1 elif o=='-s': Z=0 - elif o=='-P': prof = v - - if prof: - Z = 0 - - try: - from ZServer.medusa import asyncore - sys.modules['asyncore']=asyncore - except: pass if port is None and unix is None: print usage @@ -228,9 +219,10 @@ sys.exit(1) fs=args[0] - __builtins__.__debug__=debug if debug: os.environ['Z_DEBUG_MODE']='1' + if detailed: os.environ['STUPID_LOG_SEVERITY']='-99999' + from zLOG import LOG, INFO, ERROR # Try to set uid to "-u" -provided uid. @@ -271,54 +263,71 @@ import zdaemon zdaemon.run(sys.argv, '') - storages={} - for o, v in opts: - if o=='-S': - n, m = string.split(v,'=') - if string.find(m,':'): - # we got an attribute name - m, a = string.split(m,':') - else: - # attribute name must be same as storage name - a=n - storages[n]=get_storage(m,a) - - if not storages: - import ZODB.FileStorage - storages['1']=ZODB.FileStorage.FileStorage(fs) - - # Try to set up a signal handler try: - import signal - signal.signal(signal.SIGTERM, - lambda sig, frame, s=storages: shutdown(s) - ) - signal.signal(signal.SIGINT, - lambda sig, frame, s=storages: shutdown(s, 0) - ) - signal.signal(signal.SIGHUP, rotate_logs_handler) - - finally: pass - - items=storages.items() - items.sort() - for kv in items: - LOG('ZEO Server', INFO, 'Serving %s:\t%s' % kv) - - if not unix: unix=host, port - - if prof: - cmds = \ - "StorageServer.StorageServer(unix, storages);" \ - 'open(zeo_pid,"w").write("%s %s" % (os.getppid(), os.getpid()));' \ - "asyncore.loop()" - import profile - profile.run(cmds, prof) - else: - StorageServer.StorageServer(unix, storages) - open(zeo_pid,'w').write("%s %s" % (os.getppid(), os.getpid())) - asyncore.loop() + import ZEO.StorageServer, asyncore + + storages={} + for o, v in opts: + if o=='-S': + n, m = string.split(v,'=') + if string.find(m,':'): + # we got an attribute name + m, a = string.split(m,':') + else: + # attribute name must be same as storage name + a=n + storages[n]=get_storage(m,a) + + if not storages: + import ZODB.FileStorage + storages['1']=ZODB.FileStorage.FileStorage(fs) + + # Try to set up a signal handler + try: + import signal + + signal.signal(signal.SIGTERM, + lambda sig, frame, s=storages: shutdown(s) + ) + signal.signal(signal.SIGINT, + lambda sig, frame, s=storages: shutdown(s, 0) + ) + try: signal.signal(signal.SIGHUP, rotate_logs_handler) + except: pass + + except: pass + + items=storages.items() + items.sort() + for kv in items: + LOG('ZEO Server', INFO, 'Serving %s:\t%s' % kv) + + if not unix: unix=host, port + + ZEO.StorageServer.StorageServer(unix, storages) + + try: ppid, pid = os.getppid(), os.getpid() + except: pass # getpid not supported + else: open(zeo_pid,'w').write("%s %s" % (ppid, pid)) + + except: + # Log startup exception and tell zdaemon not to restart us. + info=sys.exc_info() + try: + import zLOG + zLOG.LOG("z2", zLOG.PANIC, "Startup exception", + error=info) + except: + pass + + import traceback + apply(traceback.print_exception, info) + + sys.exit(0) + + asyncore.loop() + def rotate_logs(): import zLOG @@ -326,7 +335,10 @@ zLOG.log_write.reinitialize() else: # Hm, lets at least try to take care of the stupid logger: - zLOG._stupid_dest=None + if hasattr(zLOG, '_set_stupid_dest'): + zLOG._set_stupid_dest(None) + else: + zLOG._stupid_dest = None def rotate_logs_handler(signum, frame): rotate_logs() @@ -347,7 +359,7 @@ for storage in storages.values(): try: storage.close() - finally: pass + except: pass try: from zLOG import LOG, INFO === Removed File StandaloneZODB/ZEO/ClientStub.py === === Removed File StandaloneZODB/ZEO/Exceptions.py === === Removed File StandaloneZODB/ZEO/ServerStub.py === === Removed File StandaloneZODB/ZEO/TransactionBuffer.py === === Removed File StandaloneZODB/ZEO/zrpc2.py === From jeremy at zope.com Fri Jan 11 14:56:30 2002 From: jeremy at zope.com (Jeremy Hylton) Date: Sun Aug 10 16:31:18 2008 Subject: [ZEO-Checkins] CVS: StandaloneZODB/ZEO/tests - forker.py:1.12 testTransactionBuffer.py:NONE Message-ID: <200201111956.g0BJuUF03276@cvs.baymountain.com> Update of /cvs-repository/StandaloneZODB/ZEO/tests In directory cvs.zope.org:/tmp/cvs-serv3265 Modified Files: forker.py Removed Files: testTransactionBuffer.py Log Message: Undo merge of ZEO-ZRPC-Dev branch into the trunk (for now). === StandaloneZODB/ZEO/tests/forker.py 1.11 => 1.12 === -# -# This software is subject to the provisions of the Zope Public License, -# Version 1.1 (ZPL). A copy of the ZPL should accompany this -# distribution. THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL -# EXPRESS OR IMPLIED WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST -# INFRINGEMENT, AND FITNESS FOR A PARTICULAR PURPOSE. - """Library for forking storage server and connecting client storage""" import asyncore import os +import profile import random import socket import sys -import traceback import types import ZEO.ClientStorage, ZEO.StorageServer -# Change value of PROFILE to enable server-side profiling PROFILE = 0 -if PROFILE: - import hotshot def get_port(): """Return a port that is not in use. @@ -78,11 +66,9 @@ buf = self.recv(4) if buf: assert buf == "done" - server.close_server() asyncore.socket_map.clear() def handle_close(self): - server.close_server() asyncore.socket_map.clear() class ZEOClientExit: @@ -91,27 +77,20 @@ self.pipe = pipe def close(self): - try: - os.write(self.pipe, "done") - os.close(self.pipe) - except os.error: - pass + os.write(self.pipe, "done") + os.close(self.pipe) def start_zeo_server(storage, addr): rd, wr = os.pipe() pid = os.fork() if pid == 0: - try: - if PROFILE: - p = hotshot.Profile("stats.s.%d" % os.getpid()) - p.runctx("run_server(storage, addr, rd, wr)", - globals(), locals()) - p.close() - else: - run_server(storage, addr, rd, wr) - except: - print "Exception in ZEO server process" - traceback.print_exc() + if PROFILE: + p = profile.Profile() + p.runctx("run_server(storage, addr, rd, wr)", globals(), + locals()) + p.dump_stats("stats.s.%d" % os.getpid()) + else: + run_server(storage, addr, rd, wr) os._exit(0) else: os.close(rd) @@ -119,11 +98,11 @@ def run_server(storage, addr, rd, wr): # in the child, run the storage server - global server os.close(wr) ZEOServerExit(rd) - server = ZEO.StorageServer.StorageServer(addr, {'1':storage}) + serv = ZEO.StorageServer.StorageServer(addr, {'1':storage}) asyncore.loop() + os.close(rd) storage.close() if isinstance(addr, types.StringType): os.unlink(addr) @@ -149,7 +128,6 @@ s = ZEO.ClientStorage.ClientStorage(addr, storage_id, debug=1, client=cache, cache_size=cache_size, - min_disconnect_poll=0.5, - wait_for_server_on_startup=1) + min_disconnect_poll=0.5) return s, exit, pid === Removed File StandaloneZODB/ZEO/tests/testTransactionBuffer.py === From jeremy at zope.com Fri Jan 11 15:24:21 2002 From: jeremy at zope.com (Jeremy Hylton) Date: Sun Aug 10 16:31:18 2008 Subject: [ZEO-Checkins] CVS: StandaloneZODB/ZEO - ClientStub.py:1.3.2.1 Exceptions.py:1.3.2.1 README:1.18.2.1 ServerStub.py:1.3.2.1 TransactionBuffer.py:1.3.2.1 zrpc2.py:1.3.2.1 ClientCache.py:1.18.6.2 ClientStorage.py:1.35.6.2 StorageServer.py:1.32.6.1 smac.py:1.11.4.2 start.py:1.26.4.2 Message-ID: <200201112024.g0BKOL110253@cvs.baymountain.com> Update of /cvs-repository/StandaloneZODB/ZEO In directory cvs.zope.org:/tmp/cvs-serv10221 Modified Files: Tag: Standby-branch ClientCache.py ClientStorage.py StorageServer.py smac.py start.py Added Files: Tag: Standby-branch ClientStub.py Exceptions.py README ServerStub.py TransactionBuffer.py zrpc2.py Log Message: Merge ZEO-ZRPC-Dev branch into Standby-branch. === Added File StandaloneZODB/ZEO/ClientStub.py === """Stub for interface exported by ClientStorage""" class ClientStorage: def __init__(self, rpc): self.rpc = rpc def beginVerify(self): self.rpc.callAsync('begin') # XXX what's the difference between these two? def invalidate(self, args): self.rpc.callAsync('invalidate', args) def Invalidate(self, args): self.rpc.callAsync('Invalidate', args) def endVerify(self): self.rpc.callAsync('end') def serialno(self, arg): self.rpc.callAsync('serialno', arg) def info(self, arg): self.rpc.callAsync('info', arg) === Added File StandaloneZODB/ZEO/Exceptions.py === """Exceptions for ZEO.""" class Disconnected(Exception): """Exception raised when a ZEO client is disconnected from the ZEO server.""" === Added File StandaloneZODB/ZEO/README === Zope Enterprize Objects, ZEO 0.2 Put this package (the ZEO directory, without any wrapping directory included in a distribution) in your Zope lib/python. Note -- This release of ZEO requires Zope 2.2.2 or a CVS checkout of Zope. See 'CHANGES.txt' for details. You also need to symbolically link (or copy) ZServer to lib/python:: cd lib/python ln -s ../../ZServer . Finally, you need to link the cPickle extension from your lib/python directory into your ZEO directory:: cd lib/python/ZEO ln -s ../cPickle.so . This is necessary because ZEO uses a (relatively) new cPickle feature that wasn't included in Python 1.5.2. Starting (and configuring) the ZEO Server To start the storage server, go to your Zope install directory and:: python lib/python/ZEO/start.py -p port_number (Run start without arguments to see options.) Of course, the server and the client don't have to be on the same machine. If the server and client *are* on the same machine, then you can use a Unix domain socket:: python lib/python/ZEO/start.py -U filename The start script provides a number of options not documented here. See doc/start.txt for more information. Running Zope as a ZEO client To get Zope to use the server, create a custom_zodb module, custom_zodb.py, in your Zope install directory, so that Zope uses a ClientStorage:: import ZEO.ClientStorage Storage=ZEO.ClientStorage.ClientStorage(('',port_number)) (See the misc/custom_zodb.py for an example.) You can specify a host name (rather than '') if you want. The port number is, of course, the port number used to start the storage server. You can also give the name of a Unix domain socket file:: import ZEO.ClientStorage Storage=ZEO.ClientStorage.ClientStorage(filename) There are a number of configuration options available for the ClientStorage. See doc/ClientStorage.txt for details. If you want a persistent client cache which retains cache contents across ClientStorage restarts, you need to define the environment variable, ZEO_CLIENT, to a unique name for the client. This is needed so that unique cache name files can be computed. Otherwise, the client cache is stored in temporary files which are removed when the ClientStorage shuts down. For example, to start two Zope processes with unique caches, use something like: python z2.py -P8700 ZEO_CLIENT=8700 python z2.py -P8800 ZEO_CLIENT=8800 Zope product installation Normally, Zope updates the Zope database during startup to reflect product changes or new products found. It makes no sense for multiple ZEO clients to do the same installation. Further, if different clients have different software installed, the correct state of the database is ambiguous. Starting in Zope 2.2, Zope will not modify the Zope database during product installation if the environment variable ZEO_CLIENT is set. Normally, Zope ZEO clients should be run with ZEO_CLIENT set so that product initialization is not performed. If you do install new Zope products, then you need to take a special step to cause the new products to be properly registered in the database. The easiest way to do this is to start Zope once without ZEO_CLIENT set. The interaction between ZEO and Zope product installation is unfortunate. In the future, this interaction will be removed by Notes for non Zope users First, we regret the amount of dependence on Zope. We intend for the dependence to decrease in the long run. Known dependencies: - The Zope lib/python/Shared directory must be in the Python path. This is due to a lame dependency on some Zope XML facilities used by ZODB for XML export and import. - ZServer should be in the Python path, or you should copy the version of asyncore.py from ZServer (from Zope 2.2 or CVS) to your Python path, or you should copy a version of a asyncore from the medusa CVS tree to your Python path. A recent change in asyncore is required. - The module, ThreadedAsync must be in the python path. - The version of cPickle from Zope, or from the python.org CVS tree must be used. It has a hook to provide control over which "global objects" (e.g. classes) may be pickled. === Added File StandaloneZODB/ZEO/ServerStub.py === """Stub for interface exposed by StorageServer""" class StorageServer: def __init__(self, rpc): self.rpc = rpc def register(self, storage_name, read_only): self.rpc.call('register', storage_name, read_only) def get_info(self): return self.rpc.call('get_info') def get_size_info(self): return self.rpc.call('get_size_info') def beginZeoVerify(self): self.rpc.callAsync('beginZeoVerify') def zeoVerify(self, oid, s, sv): self.rpc.callAsync('zeoVerify', oid, s, sv) def endZeoVerify(self): self.rpc.callAsync('endZeoVerify') def new_oids(self, n=None): if n is None: return self.rpc.call('new_oids') else: return self.rpc.call('new_oids', n) def pack(self, t, wait=None): if wait is None: self.rpc.call('pack', t) else: self.rpc.call('pack', t, wait) def zeoLoad(self, oid): return self.rpc.call('zeoLoad', oid) def storea(self, oid, serial, data, version, id): self.rpc.callAsync('storea', oid, serial, data, version, id) def tpc_begin(self, id, user, descr, ext): return self.rpc.call('tpc_begin', id, user, descr, ext) def vote(self, trans_id): return self.rpc.call('vote', trans_id) def tpc_finish(self, id): return self.rpc.call('tpc_finish', id) def tpc_abort(self, id): self.rpc.callAsync('tpc_abort', id) def abortVersion(self, src, id): return self.rpc.call('abortVersion', src, id) def commitVersion(self, src, dest, id): return self.rpc.call('commitVersion', src, dest, id) def history(self, oid, version, length=None): if length is not None: return self.rpc.call('history', oid, version) else: return self.rpc.call('history', oid, version, length) def load(self, oid, version): return self.rpc.call('load', oid, version) def loadSerial(self, oid, serial): return self.rpc.call('loadSerial', oid, serial) def modifiedInVersion(self, oid): return self.rpc.call('modifiedInVersion', oid) def new_oid(self, last=None): if last is None: return self.rpc.call('new_oid') else: return self.rpc.call('new_oid', last) def store(self, oid, serial, data, version, trans): return self.rpc.call('store', oid, serial, data, version, trans) def transactionalUndo(self, trans_id, trans): return self.rpc.call('transactionalUndo', trans_id, trans) def undo(self, trans_id): return self.rpc.call('undo', trans_id) def undoLog(self, first, last): # XXX filter not allowed across RPC return self.rpc.call('undoLog', first, last) def undoInfo(self, first, last, spec): return self.rpc.call('undoInfo', first, last, spec) def versionEmpty(self, vers): return self.rpc.call('versionEmpty', vers) def versions(self, max=None): if max is None: return self.rpc.call('versions') else: return self.rpc.call('versions', max) === Added File StandaloneZODB/ZEO/TransactionBuffer.py === """A TransactionBuffer store transaction updates until commit or abort. A transaction may generate enough data that it is not practical to always hold pending updates in memory. Instead, a TransactionBuffer is used to store the data until a commit or abort. """ # XXX Figure out what a sensible storage format is # XXX A faster implementation might store trans data in memory until # it reaches a certain size. import tempfile import cPickle class TransactionBuffer: def __init__(self): self.file = tempfile.TemporaryFile() self.count = 0 self.size = 0 # It's safe to use a fast pickler because the only objects # stored are builtin types -- strings or None. self.pickler = cPickle.Pickler(self.file, 1) self.pickler.fast = 1 def store(self, oid, version, data): """Store oid, version, data for later retrieval""" self.pickler.dump((oid, version, data)) self.count += 1 # Estimate per-record cache size self.size = self.size + len(data) + (27 + 12) if version: self.size = self.size + len(version) + 4 def invalidate(self, oid, version): self.pickler.dump((oid, version, None)) self.count += 1 def clear(self): """Mark the buffer as empty""" self.file.seek(0) self.count = 0 self.size = 0 # XXX unchecked constraints: # 1. can't call store() after begin_iterate() # 2. must call clear() after iteration finishes def begin_iterate(self): """Move the file pointer in advance of iteration""" self.file.flush() self.file.seek(0) self.unpickler = cPickle.Unpickler(self.file) def next(self): """Return next tuple of data or None if EOF""" if self.count == 0: del self.unpickler return None oid_ver_data = self.unpickler.load() self.count -= 1 return oid_ver_data def get_size(self): """Return size of data stored in buffer (just a hint).""" return self.size === Added File StandaloneZODB/ZEO/zrpc2.py === (616/716 lines abridged) """RPC protocol for ZEO based on asyncore The basic protocol is as: a pickled tuple containing: msgid, flags, method, args msgid is an integer. flags is an integer. The only currently defined flag is ASYNC (0x1), which means the client does not expect a reply. method is a string specifying the method to invoke. For a reply, the method is ".reply". args is a tuple of the argument to pass to method. XXX need to specify a version number that describes the protocol. allow for future revision. XXX support multiple outstanding calls XXX factor out common pattern of deciding what protocol to use based on whether address is tuple or string """ import asyncore import errno import cPickle import os import select import socket import sys import threading import thread import time import traceback import types from cStringIO import StringIO from ZODB import POSException from ZEO import smac, trigger from Exceptions import Disconnected import zLOG import ThreadedAsync from Exceptions import Disconnected REPLY = ".reply" # message name used for replies ASYNC = 1 _label = "zrpc:%s" % os.getpid() def new_label(): [-=- -=- -=- 616 lines omitted -=- -=- -=-] def readable(self): return 1 def handle_accept(self): try: sock, addr = self.accept() except socket.error, msg: log("accepted failed: %s" % msg) return c = self.factory(sock, addr, self.obj) log("connect from %s: %s" % (repr(addr), c)) self.clients.append(c) class Handler: """Base class used to handle RPC caller discovery""" def set_caller(self, addr): self.__caller = addr def get_caller(self): return self.__caller def clear_caller(self): self.__caller = None _globals = globals() _silly = ('__doc__',) def find_global(module, name): """Helper for message unpickler""" try: m = __import__(module, _globals, _globals, _silly) except ImportError, msg: raise ZRPCError("import error %s: %s" % (module, msg)) try: r = getattr(m, name) except AttributeError: raise ZRPCError("module %s has no global %s" % (module, name)) safe = getattr(r, '__no_side_effects__', 0) if safe: return r if type(r) == types.ClassType and issubclass(r, Exception): return r raise ZRPCError("Unsafe global: %s.%s" % (module, name)) === StandaloneZODB/ZEO/ClientCache.py 1.18.6.1 => 1.18.6.2 === -# -# This software is subject to the provisions of the Zope Public License, -# Version 1.1 (ZPL). A copy of the ZPL should accompany this -# distribution. THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL -# EXPRESS OR IMPLIED WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST -# INFRINGEMENT, AND FITNESS FOR A PARTICULAR PURPOSE. - +############################################################################## +# +# Zope Public License (ZPL) Version 1.0 +# ------------------------------------- +# +# Copyright (c) Digital Creations. All rights reserved. +# +# This license has been certified as Open Source(tm). +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# 1. Redistributions in source code must retain the above copyright +# notice, this list of conditions, and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions, and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# +# 3. Digital Creations requests that attribution be given to Zope +# in any manner possible. Zope includes a "Powered by Zope" +# button that is installed by default. While it is not a license +# violation to remove this button, it is requested that the +# attribution remain. A significant investment has been put +# into Zope, and this effort will continue if the Zope community +# continues to grow. This is one way to assure that growth. +# +# 4. All advertising materials and documentation mentioning +# features derived from or use of this software must display +# the following acknowledgement: +# +# "This product includes software developed by Digital Creations +# for use in the Z Object Publishing Environment +# (http://www.zope.org/)." +# +# In the event that the product being advertised includes an +# intact Zope distribution (with copyright and license included) +# then this clause is waived. +# +# 5. Names associated with Zope or Digital Creations must not be used to +# endorse or promote products derived from this software without +# prior written permission from Digital Creations. +# +# 6. Modified redistributions of any form whatsoever must retain +# the following acknowledgment: +# +# "This product includes software developed by Digital Creations +# for use in the Z Object Publishing Environment +# (http://www.zope.org/)." +# +# Intact (re-)distributions of any official Zope release do not +# require an external acknowledgement. +# +# 7. Modifications are encouraged but must be packaged separately as +# patches to official Zope releases. Distributions that do not +# clearly separate the patches from the original work must be clearly +# labeled as unofficial distributions. Modifications which do not +# carry the name Zope may be packaged in any form, as long as they +# conform to all of the clauses above. +# +# +# Disclaimer +# +# THIS SOFTWARE IS PROVIDED BY DIGITAL CREATIONS ``AS IS'' AND ANY +# EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL DIGITAL CREATIONS OR ITS +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF +# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT +# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +# SUCH DAMAGE. +# +# +# This software consists of contributions made by Digital Creations and +# many individuals on behalf of Digital Creations. Specific +# attributions are listed in the accompanying credits file. +# +############################################################################## """Implement a client cache The cache is managed as two files, var/c0.zec and var/c1.zec. @@ -74,13 +149,15 @@ import os, tempfile from struct import pack, unpack from thread import allocate_lock -import zLOG -magic='ZEC0' +import sys +import zLOG -def LOG(msg, level=zLOG.BLATHER): +def log(msg, level=zLOG.INFO): zLOG.LOG("ZEC", level, msg) +magic='ZEC0' + class ClientCache: def __init__(self, storage='', size=20000000, client=None, var=None): @@ -136,16 +213,14 @@ f[0].write(magic) current=0 + log("cache opened. current = %s" % current) + self._limit=size/2 self._current=current - def close(self): - try: - self._f[self._current].close() - except (os.error, ValueError): - pass - def open(self): + # XXX open is overloaded to perform two tasks for + # optimization reasons self._acquire() try: self._index=index={} @@ -160,6 +235,19 @@ return serial.items() finally: self._release() + def close(self): + for f in self._f: + if f is not None: + f.close() + + def verify(self, verifyFunc): + """Call the verifyFunc on every object in the cache. + + verifyFunc(oid, serialno, version) + """ + for oid, (s, vs) in self.open(): + verifyFunc(oid, s, vs) + def invalidate(self, oid, version): self._acquire() try: @@ -298,8 +386,6 @@ self._f[current]=open(self._p[current],'w+b') else: # Temporary cache file: - if self._f[current] is not None: - self._f[current].close() self._f[current] = tempfile.TemporaryFile(suffix='.zec') self._f[current].write(magic) self._pos=pos=4 @@ -308,55 +394,57 @@ def store(self, oid, p, s, version, pv, sv): self._acquire() - try: self._store(oid, p, s, version, pv, sv) - finally: self._release() + try: + self._store(oid, p, s, version, pv, sv) + finally: + self._release() def _store(self, oid, p, s, version, pv, sv): if not s: - p='' - s='\0\0\0\0\0\0\0\0' - tlen=31+len(p) + p = '' + s = '\0\0\0\0\0\0\0\0' + tlen = 31 + len(p) if version: - tlen=tlen+len(version)+12+len(pv) - vlen=len(version) + tlen = tlen + len(version) + 12 + len(pv) + vlen = len(version) else: - vlen=0 + vlen = 0 - pos=self._pos - current=self._current - f=self._f[current] - f.seek(pos) - stlen=pack(">I",tlen) - write=f.write - write(oid+'v'+stlen+pack(">HI", vlen, len(p))+s) - if p: write(p) + stlen = pack(">I", tlen) + # accumulate various data to write into a list + l = [oid, 'v', stlen, pack(">HI", vlen, len(p)), s] + if p: + l.append(p) if version: - write(version) - write(pack(">I", len(pv))) - write(pv) - write(sv) - - write(stlen) + l.extend([version, + pack(">I", len(pv)), + pv, sv]) + l.append(stlen) + f = self._f[self._current] + f.seek(self._pos) + f.write("".join(l)) - if current: self._index[oid]=-pos - else: self._index[oid]=pos + if self._current: + self._index[oid] = - self._pos + else: + self._index[oid] = self._pos - self._pos=pos+tlen + self._pos += tlen def read_index(index, serial, f, current): - LOG("read_index(%s)" % f.name) seek=f.seek read=f.read pos=4 + seek(0,2) + size=f.tell() while 1: - seek(pos) + f.seek(pos) h=read(27) - + if len(h)==27 and h[8] in 'vni': tlen, vlen, dlen = unpack(">iHi", h[9:19]) - else: - break + else: tlen=-1 if tlen <= 0 or vlen < 0 or dlen < 0 or vlen+dlen > tlen: break @@ -391,15 +479,3 @@ except: pass return pos - -def main(files): - for file in files: - print file - index = {} - serial = {} - read_index(index, serial, open(file), 0) - print index.keys() - -if __name__ == "__main__": - import sys - main(sys.argv[1:]) === StandaloneZODB/ZEO/ClientStorage.py 1.35.6.1 => 1.35.6.2 === (959/1059 lines abridged) -# -# This software is subject to the provisions of the Zope Public License, -# Version 1.1 (ZPL). A copy of the ZPL should accompany this -# distribution. THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL -# EXPRESS OR IMPLIED WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST -# INFRINGEMENT, AND FITNESS FOR A PARTICULAR PURPOSE. - +############################################################################## +# +# Zope Public License (ZPL) Version 1.0 +# ------------------------------------- +# +# Copyright (c) Digital Creations. All rights reserved. +# +# This license has been certified as Open Source(tm). +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# 1. Redistributions in source code must retain the above copyright +# notice, this list of conditions, and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions, and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# +# 3. Digital Creations requests that attribution be given to Zope +# in any manner possible. Zope includes a "Powered by Zope" +# button that is installed by default. While it is not a license +# violation to remove this button, it is requested that the +# attribution remain. A significant investment has been put +# into Zope, and this effort will continue if the Zope community +# continues to grow. This is one way to assure that growth. +# +# 4. All advertising materials and documentation mentioning +# features derived from or use of this software must display +# the following acknowledgement: +# +# "This product includes software developed by Digital Creations +# for use in the Z Object Publishing Environment +# (http://www.zope.org/)." +# +# In the event that the product being advertised includes an +# intact Zope distribution (with copyright and license included) +# then this clause is waived. +# +# 5. Names associated with Zope or Digital Creations must not be used to [-=- -=- -=- 959 lines omitted -=- -=- -=-] - _w.append(t) - return t + return self._server.versions(max) + + # below are methods invoked by the StorageServer + + def serialno(self, arg): + self._serials.append(arg) + + def info(self, dict): + self._info.update(dict) + + def begin(self): + self._tfile = tempfile.TemporaryFile() + self._pickler = cPickle.Pickler(self._tfile, 1) + self._pickler.fast = 1 # Don't use the memo + + def invalidate(self, args): + if self._pickler is None: + return + self._pickler.dump(args) + + def end(self): + if self._pickler is None: + return + self._pickler.dump((0,0)) +## self._pickler.dump = None + self._tfile.seek(0) + unpick = cPickle.Unpickler(self._tfile) + self._tfile = None + + while 1: + oid, version = unpick.load() + if not oid: + break + self._cache.invalidate(oid, version=version) + self._db.invalidate(oid, version=version) + + def Invalidate(self, args): + # XXX _db could be None + for oid, version in args: + self._cache.invalidate(oid, version=version) + try: + self._db.invalidate(oid, version=version) + except AttributeError, msg: + log2(PROBLEM, + "Invalidate(%s, %s) failed for _db: %s" % (repr(oid), + repr(version), + msg)) + === StandaloneZODB/ZEO/StorageServer.py 1.32 => 1.32.6.1 === (750/850 lines abridged) +############################################################################## # # Zope Public License (ZPL) Version 1.0 # ------------------------------------- @@ -59,7 +59,7 @@ # labeled as unofficial distributions. Modifications which do not # carry the name Zope may be packaged in any form, as long as they # conform to all of the clauses above. -# +# # # Disclaimer # @@ -82,527 +82,394 @@ # attributions are listed in the accompanying credits file. # ############################################################################## +"""Network ZODB storage server -__version__ = "$Revision$"[11:-2] +This server acts as a front-end for one or more real storages, like +file storage or Berkeley storage. -import asyncore, socket, string, sys, os -from smac import SizedMessageAsyncConnection -from ZODB import POSException +XXX Need some basic access control-- a declaration of the methods +exported for invocation by the server. +""" + +import asyncore import cPickle -from cPickle import Unpickler -from ZODB.POSException import TransactionError, UndoError, VersionCommitError -from ZODB.Transaction import Transaction -import traceback -from zLOG import LOG, INFO, ERROR, TRACE, BLATHER +import os +import sys +import threading +import types + +import ClientStub +import zrpc2 +import zLOG + +from zrpc2 import Dispatcher, Handler, ManagedServerConnection, Delay +from ZODB.POSException import StorageError, StorageTransactionError, \ + TransactionError, ReadOnlyError from ZODB.referencesf import referencesf [-=- -=- -=- 750 lines omitted -=- -=- -=-] - try: - port='', int(port) - except: - pass - - d = {'1': ZODB.FileStorage.FileStorage(name)} - StorageServer(port, d) - asyncwrap.loop() + self.server.invalidate(self, self.__storage_id, + self.__invalidated, + self.get_size_info()) + + if not self._handle_waiting(): + self._transaction = None + self.__invalidated = [] + + def tpc_abort(self, id): + if not self._check_tid(id): + return + r = self.__storage.tpc_abort(self._transaction) + assert self.__storage._transaction is None + + if not self._handle_waiting(): + self._transaction = None + self.__invalidated = [] + + def _restart_delayed_transaction(self, delay, trans): + self._transaction = trans + self.__storage.tpc_begin(trans) + self.__invalidated = [] + assert self._transaction.id == self.__storage._transaction.id + delay.reply(None) + + def _handle_waiting(self): + if self.__storage.__waiting: + delay, proxy, trans = self.__storage.__waiting.pop(0) + proxy._restart_delayed_transaction(delay, trans) + if self is proxy: + return 1 + + def new_oids(self, n=100): + """Return a sequence of n new oids, where n defaults to 100""" + if n < 0: + n = 1 + return [self.__storage.new_oid() for i in range(n)] + +def fixup_storage(storage): + # backwards compatibility hack + if not hasattr(storage,'tpc_vote'): + storage.tpc_vote = lambda *args: None === StandaloneZODB/ZEO/smac.py 1.11.4.1 => 1.11.4.2 === -# -# This software is subject to the provisions of the Zope Public License, -# Version 1.1 (ZPL). A copy of the ZPL should accompany this -# distribution. THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL -# EXPRESS OR IMPLIED WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST -# INFRINGEMENT, AND FITNESS FOR A PARTICULAR PURPOSE. - +############################################################################## +# +# Zope Public License (ZPL) Version 1.0 +# ------------------------------------- +# +# Copyright (c) Digital Creations. All rights reserved. +# +# This license has been certified as Open Source(tm). +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# 1. Redistributions in source code must retain the above copyright +# notice, this list of conditions, and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions, and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# +# 3. Digital Creations requests that attribution be given to Zope +# in any manner possible. Zope includes a "Powered by Zope" +# button that is installed by default. While it is not a license +# violation to remove this button, it is requested that the +# attribution remain. A significant investment has been put +# into Zope, and this effort will continue if the Zope community +# continues to grow. This is one way to assure that growth. +# +# 4. All advertising materials and documentation mentioning +# features derived from or use of this software must display +# the following acknowledgement: +# +# "This product includes software developed by Digital Creations +# for use in the Z Object Publishing Environment +# (http://www.zope.org/)." +# +# In the event that the product being advertised includes an +# intact Zope distribution (with copyright and license included) +# then this clause is waived. +# +# 5. Names associated with Zope or Digital Creations must not be used to +# endorse or promote products derived from this software without +# prior written permission from Digital Creations. +# +# 6. Modified redistributions of any form whatsoever must retain +# the following acknowledgment: +# +# "This product includes software developed by Digital Creations +# for use in the Z Object Publishing Environment +# (http://www.zope.org/)." +# +# Intact (re-)distributions of any official Zope release do not +# require an external acknowledgement. +# +# 7. Modifications are encouraged but must be packaged separately as +# patches to official Zope releases. Distributions that do not +# clearly separate the patches from the original work must be clearly +# labeled as unofficial distributions. Modifications which do not +# carry the name Zope may be packaged in any form, as long as they +# conform to all of the clauses above. +# +# +# Disclaimer +# +# THIS SOFTWARE IS PROVIDED BY DIGITAL CREATIONS ``AS IS'' AND ANY +# EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL DIGITAL CREATIONS OR ITS +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF +# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT +# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +# SUCH DAMAGE. +# +# +# This software consists of contributions made by Digital Creations and +# many individuals on behalf of Digital Creations. Specific +# attributions are listed in the accompanying credits file. +# +############################################################################## """Sized message async connections """ __version__ = "$Revision$"[11:-2] -import asyncore, string, struct, zLOG, sys, Acquisition +import asyncore, struct +from Exceptions import Disconnected +from zLOG import LOG, TRACE, ERROR, INFO, BLATHER +from types import StringType + import socket, errno -from zLOG import LOG, TRACE, ERROR, INFO # Use the dictionary to make sure we get the minimum number of errno # entries. We expect that EWOULDBLOCK == EAGAIN on most systems -- @@ -34,81 +112,101 @@ expected_socket_write_errors = tuple(tmp_dict.keys()) del tmp_dict -class SizedMessageAsyncConnection(Acquisition.Explicit, asyncore.dispatcher): +class SizedMessageAsyncConnection(asyncore.dispatcher): + __super_init = asyncore.dispatcher.__init__ + __super_close = asyncore.dispatcher.close + + __closed = 1 # Marker indicating that we're closed - __append=None # Marker indicating that we're closed + socket = None # to outwit Sam's getattr - socket=None # to outwit Sam's getattr + READ_SIZE = 8096 def __init__(self, sock, addr, map=None, debug=None): - SizedMessageAsyncConnection.inheritedAttribute( - '__init__')(self, sock, map) - self.addr=addr + self.__super_init(sock, map) + self.addr = addr if debug is not None: - self._debug=debug + self._debug = debug elif not hasattr(self, '_debug'): - self._debug=__debug__ and 'smac' - self.__state=None - self.__inp=None - self.__inpl=0 - self.__l=4 - self.__output=output=[] - self.__append=output.append - self.__pop=output.pop - - def handle_read(self, - join=string.join, StringType=type(''), _type=type, - _None=None): - + self._debug = __debug__ and 'smac' + self.__state = None + self.__inp = None # None, a single String, or a list + self.__input_len = 0 + self.__msg_size = 4 + self.__output = [] + self.__closed = None + + # XXX avoid expensive getattr calls? + def __nonzero__(self): + return 1 + + def handle_read(self): + # Use a single __inp buffer and integer indexes to make this + # fast. try: d=self.recv(8096) except socket.error, err: if err[0] in expected_socket_read_errors: return raise - if not d: return + if not d: + return - inp=self.__inp - if inp is _None: - inp=d - elif _type(inp) is StringType: - inp=[inp,d] + input_len = self.__input_len + len(d) + msg_size = self.__msg_size + state = self.__state + + inp = self.__inp + if msg_size > input_len: + if inp is None: + self.__inp = d + elif type(self.__inp) is StringType: + self.__inp = [self.__inp, d] + else: + self.__inp.append(d) + self.__input_len = input_len + return # keep waiting for more input + + # load all previous input and d into single string inp + if isinstance(inp, StringType): + inp = inp + d + elif inp is None: + inp = d else: inp.append(d) + inp = "".join(inp) - inpl=self.__inpl+len(d) - l=self.__l - - while 1: - - if l <= inpl: - # Woo hoo, we have enough data - if _type(inp) is not StringType: inp=join(inp,'') - d=inp[:l] - inp=inp[l:] - inpl=inpl-l - if self.__state is _None: - # waiting for message - l=struct.unpack(">i",d)[0] - self.__state=1 - else: - l=4 - self.__state=_None - self.message_input(d) + offset = 0 + while (offset + msg_size) <= input_len: + msg = inp[offset:offset + msg_size] + offset = offset + msg_size + if state is None: + # waiting for message + msg_size = struct.unpack(">i", msg)[0] + state = 1 else: - break # not enough data - - self.__l=l - self.__inp=inp - self.__inpl=inpl + msg_size = 4 + state = None + self.message_input(msg) + + self.__state = state + self.__msg_size = msg_size + self.__inp = inp[offset:] + self.__input_len = input_len - offset - def readable(self): return 1 - def writable(self): return not not self.__output + def readable(self): + return 1 + + def writable(self): + if len(self.__output) == 0: + return 0 + else: + return 1 def handle_write(self): - output=self.__output + output = self.__output while output: - v=output[0] + v = output[0] try: n=self.send(v) except socket.error, err: @@ -116,42 +214,33 @@ break # we couldn't write anything raise if n < len(v): - output[0]=v[n:] + output[0] = v[n:] break # we can't write any more else: del output[0] - #break # waaa - def handle_close(self): self.close() - def message_output(self, message, - pack=struct.pack, len=len): - if self._debug: - if len(message) > 40: m=message[:40]+' ...' - else: m=message - LOG(self._debug, TRACE, 'message_output %s' % `m`) - - append=self.__append - if append is None: - raise Disconnected("This action is temporarily unavailable.

") - - append(pack(">i",len(message))+message) - - def log_info(self, message, type='info'): - if type=='error': type=ERROR - else: type=INFO - LOG('ZEO', type, message) + def message_output(self, message): + if __debug__: + if self._debug: + if len(message) > 40: + m = message[:40]+' ...' + else: + m = message + LOG(self._debug, TRACE, 'message_output %s' % `m`) - log=log_info + if self.__closed is not None: + raise Disconnected, ( + "This action is temporarily unavailable." + "

" + ) + # do two separate appends to avoid copying the message string + self.__output.append(struct.pack(">i", len(message))) + self.__output.append(message) def close(self): - if self.__append is not None: - self.__append=None - SizedMessageAsyncConnection.inheritedAttribute('close')(self) - -class Disconnected(Exception): - """The client has become disconnected from the server - """ - + if self.__closed is None: + self.__closed = 1 + self.__super_close() === StandaloneZODB/ZEO/start.py 1.26.4.1 => 1.26.4.2 === -# -# This software is subject to the provisions of the Zope Public License, -# Version 1.1 (ZPL). A copy of the ZPL should accompany this -# distribution. THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL -# EXPRESS OR IMPLIED WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST -# INFRINGEMENT, AND FITNESS FOR A PARTICULAR PURPOSE. - +############################################################################## +# +# Zope Public License (ZPL) Version 1.0 +# ------------------------------------- +# +# Copyright (c) Digital Creations. All rights reserved. +# +# This license has been certified as Open Source(tm). +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# 1. Redistributions in source code must retain the above copyright +# notice, this list of conditions, and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions, and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# +# 3. Digital Creations requests that attribution be given to Zope +# in any manner possible. Zope includes a "Powered by Zope" +# button that is installed by default. While it is not a license +# violation to remove this button, it is requested that the +# attribution remain. A significant investment has been put +# into Zope, and this effort will continue if the Zope community +# continues to grow. This is one way to assure that growth. +# +# 4. All advertising materials and documentation mentioning +# features derived from or use of this software must display +# the following acknowledgement: +# +# "This product includes software developed by Digital Creations +# for use in the Z Object Publishing Environment +# (http://www.zope.org/)." +# +# In the event that the product being advertised includes an +# intact Zope distribution (with copyright and license included) +# then this clause is waived. +# +# 5. Names associated with Zope or Digital Creations must not be used to +# endorse or promote products derived from this software without +# prior written permission from Digital Creations. +# +# 6. Modified redistributions of any form whatsoever must retain +# the following acknowledgment: +# +# "This product includes software developed by Digital Creations +# for use in the Z Object Publishing Environment +# (http://www.zope.org/)." +# +# Intact (re-)distributions of any official Zope release do not +# require an external acknowledgement. +# +# 7. Modifications are encouraged but must be packaged separately as +# patches to official Zope releases. Distributions that do not +# clearly separate the patches from the original work must be clearly +# labeled as unofficial distributions. Modifications which do not +# carry the name Zope may be packaged in any form, as long as they +# conform to all of the clauses above. +# +# +# Disclaimer +# +# THIS SOFTWARE IS PROVIDED BY DIGITAL CREATIONS ``AS IS'' AND ANY +# EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL DIGITAL CREATIONS OR ITS +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF +# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT +# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +# SUCH DAMAGE. +# +# +# This software consists of contributions made by Digital Creations and +# many individuals on behalf of Digital Creations. Specific +# attributions are listed in the accompanying credits file. +# +############################################################################## """Start the server storage. """ @@ -15,6 +90,9 @@ import sys, os, getopt, string +import StorageServer +import asyncore + def directory(p, n=1): d=p while n: @@ -40,9 +118,11 @@ def main(argv): me=argv[0] - sys.path[:]==filter(None, sys.path) sys.path.insert(0, directory(me, 2)) + # XXX hack for profiling support + global unix, storages, zeo_pid, asyncore + args=[] last='' for a in argv[1:]: @@ -55,25 +135,13 @@ args.append(a) last=a - if os.environ.has_key('INSTANCE_HOME'): - INSTANCE_HOME=os.environ['INSTANCE_HOME'] - elif os.path.isdir(os.path.join(directory(me, 4),'var')): - INSTANCE_HOME=directory(me, 4) - else: - INSTANCE_HOME=os.getcwd() - - if os.path.isdir(os.path.join(INSTANCE_HOME, 'var')): - var=os.path.join(INSTANCE_HOME, 'var') - else: - var=INSTANCE_HOME + INSTANCE_HOME=os.environ.get('INSTANCE_HOME', directory(me, 4)) zeo_pid=os.environ.get('ZEO_SERVER_PID', - os.path.join(var, 'ZEO_SERVER.pid') + os.path.join(INSTANCE_HOME, 'var', 'ZEO_SERVER.pid') ) - opts, args = getopt.getopt(args, 'p:Ddh:U:sS:u:') - - fs=os.path.join(var, 'Data.fs') + fs=os.path.join(INSTANCE_HOME, 'var', 'Data.fs') usage="""%s [options] [filename] @@ -81,17 +149,14 @@ -D -- Run in debug mode - -d -- Generate detailed debug logging without running - in the foreground. - -U -- Unix-domain socket file to listen on -u username or uid number The username to run the ZEO server as. You may want to run the ZEO server as 'nobody' or some other user with limited - resouces. The only works under Unix, and if the storage - server is started by root. + resouces. The only works under Unix, and if ZServer is + started by root. -p port -- port to listen on @@ -114,23 +179,42 @@ attr_name -- This is the name to which the storage object is assigned in the module. + -P file -- Run under profile and dump output to file. Implies the + -s flag. + if no file name is specified, then %s is used. """ % (me, fs) + try: + opts, args = getopt.getopt(args, 'p:Dh:U:sS:u:P:') + except getopt.error, msg: + print usage + print msg + sys.exit(1) + port=None - debug=detailed=0 + debug=0 host='' unix=None Z=1 UID='nobody' + prof = None for o, v in opts: if o=='-p': port=string.atoi(v) elif o=='-h': host=v elif o=='-U': unix=v elif o=='-u': UID=v elif o=='-D': debug=1 - elif o=='-d': detailed=1 elif o=='-s': Z=0 + elif o=='-P': prof = v + + if prof: + Z = 0 + + try: + from ZServer.medusa import asyncore + sys.modules['asyncore']=asyncore + except: pass if port is None and unix is None: print usage @@ -144,10 +228,9 @@ sys.exit(1) fs=args[0] + __builtins__.__debug__=debug if debug: os.environ['Z_DEBUG_MODE']='1' - if detailed: os.environ['STUPID_LOG_SEVERITY']='-99999' - from zLOG import LOG, INFO, ERROR # Try to set uid to "-u" -provided uid. @@ -188,71 +271,54 @@ import zdaemon zdaemon.run(sys.argv, '') - try: - - import ZEO.StorageServer, asyncore - - storages={} - for o, v in opts: - if o=='-S': - n, m = string.split(v,'=') - if string.find(m,':'): - # we got an attribute name - m, a = string.split(m,':') - else: - # attribute name must be same as storage name - a=n - storages[n]=get_storage(m,a) - - if not storages: - import ZODB.FileStorage - storages['1']=ZODB.FileStorage.FileStorage(fs) - - # Try to set up a signal handler - try: - import signal - - signal.signal(signal.SIGTERM, - lambda sig, frame, s=storages: shutdown(s) - ) - signal.signal(signal.SIGINT, - lambda sig, frame, s=storages: shutdown(s, 0) - ) - try: signal.signal(signal.SIGHUP, rotate_logs_handler) - except: pass - - except: pass - - items=storages.items() - items.sort() - for kv in items: - LOG('ZEO Server', INFO, 'Serving %s:\t%s' % kv) - - if not unix: unix=host, port - - ZEO.StorageServer.StorageServer(unix, storages) - - try: ppid, pid = os.getppid(), os.getpid() - except: pass # getpid not supported - else: open(zeo_pid,'w').write("%s %s" % (ppid, pid)) - - except: - # Log startup exception and tell zdaemon not to restart us. - info=sys.exc_info() - try: - import zLOG - zLOG.LOG("z2", zLOG.PANIC, "Startup exception", - error=info) - except: - pass - - import traceback - apply(traceback.print_exception, info) - - sys.exit(0) + storages={} + for o, v in opts: + if o=='-S': + n, m = string.split(v,'=') + if string.find(m,':'): + # we got an attribute name + m, a = string.split(m,':') + else: + # attribute name must be same as storage name + a=n + storages[n]=get_storage(m,a) + + if not storages: + import ZODB.FileStorage + storages['1']=ZODB.FileStorage.FileStorage(fs) - asyncore.loop() + # Try to set up a signal handler + try: + import signal + signal.signal(signal.SIGTERM, + lambda sig, frame, s=storages: shutdown(s) + ) + signal.signal(signal.SIGINT, + lambda sig, frame, s=storages: shutdown(s, 0) + ) + signal.signal(signal.SIGHUP, rotate_logs_handler) + + finally: pass + + items=storages.items() + items.sort() + for kv in items: + LOG('ZEO Server', INFO, 'Serving %s:\t%s' % kv) + + if not unix: unix=host, port + + if prof: + cmds = \ + "StorageServer.StorageServer(unix, storages);" \ + 'open(zeo_pid,"w").write("%s %s" % (os.getppid(), os.getpid()));' \ + "asyncore.loop()" + import profile + profile.run(cmds, prof) + else: + StorageServer.StorageServer(unix, storages) + open(zeo_pid,'w').write("%s %s" % (os.getppid(), os.getpid())) + asyncore.loop() def rotate_logs(): import zLOG @@ -260,10 +326,7 @@ zLOG.log_write.reinitialize() else: # Hm, lets at least try to take care of the stupid logger: - if hasattr(zLOG, '_set_stupid_dest'): - zLOG._set_stupid_dest(None) - else: - zLOG._stupid_dest = None + zLOG._stupid_dest=None def rotate_logs_handler(signum, frame): rotate_logs() @@ -284,7 +347,7 @@ for storage in storages.values(): try: storage.close() - except: pass + finally: pass try: from zLOG import LOG, INFO From jeremy at zope.com Fri Jan 11 15:24:33 2002 From: jeremy at zope.com (Jeremy Hylton) Date: Sun Aug 10 16:31:18 2008 Subject: [ZEO-Checkins] CVS: StandaloneZODB/ZEO/doc - ClientCache.txt:1.2.2.1 ClientStorage.txt:1.2.2.1 start.txt:1.2.2.1 Message-ID: <200201112024.g0BKOXW10275@cvs.baymountain.com> Update of /cvs-repository/StandaloneZODB/ZEO/doc In directory cvs.zope.org:/tmp/cvs-serv10260 Added Files: Tag: Standby-branch ClientCache.txt ClientStorage.txt start.txt Log Message: Merge ZEO-ZRPC-Dev branch into Standby-branch. === Added File StandaloneZODB/ZEO/doc/ClientCache.txt === ZEO Client Cache The Client cache provides a disk based cache for each ZEO client. The client cache allows reads to be done from local disk rather than by remote access to the storage server. The cache may be persistent or transient. If the cache is persistent, then the cache files are retained for use after process restarts. A non-persistent cache uses temporary files that are removed when the client storage is closed. The client cache is managed as two files. The cache manager endeavors to maintain the two files at sizes less than or equal to one half the cache size. One of the cache files is designated the "current" cache file. The other cache file is designated the "old" cache file, if it exists. All writes are done to the current cache files. When transactions are committed on the client, transactions are not split between cache files. Large transactions may cause cache files to be larger than one half the target cache size. The life of the cache is as follows: - When the cache is created, the first of the two cache files is created and designated the "current" cache file. - Cache records are written to the cache file, either as transactions commit locally, or as data are loaded from the server. - When the cache file size exceeds one half the cache size, the second cache file is created and designated the "current" cache file. The first cache file becomes the "old" cache file. - Cache records are written to the new current cache file, either as transactions commit locally, or as data are loaded from the server. - When the current cache file size exceeds one half the cache size, the first cache file is recreated and designated the "current" cache file. The second cache file becomes the "old" cache file. and so on. Persistent cache files are created in the directory named in the 'var' argument to the ClientStorage (see ClientStorage.txt) or in the 'var' subdirectory of the directory given by the INSTANCE_HOME builtin (created by Zope), or in the current working directory. Persistent cache files have names of the form:: cstorage-client-n.zec where: storage -- the storage name client -- the client name, as given by the 'ZEO_CLIENT' environment variable or the 'client' argument provided when creating a client storage. n -- '0' for the first cache file and '1' for the second. For example, the second cache file for storage 'spam' and client 8881 would be named 'cspam-8881-1.zec'. === Added File StandaloneZODB/ZEO/doc/ClientStorage.txt === ClientStorage The ClientStorage is a ZODB storage implementation that provides access to data served bt a ZEO server. To use a ClientStorage, create the SlientStorage and configure your application to use it. To configure Zope to use a ClientStorage, create the ClientStorage and and assign it to the 'Storage' variable in a 'custom_zodb' module (typically a Python file, with a '.py' suffix) in the instance home of your Zope installation. Creating a ClientStorage At a minimum, a client storage requires an argument (named connection) giving connection information. This argument should be a string, specifying a unix-domain socket file name, or a tuple consisting of a host and port. The host should be a string host name or IP number. The port should be a numeric port number. The ClientStorage constructor provides a number of additional options (arguments). The full list of arguments is: connection -- Connection information. This argument is either a string containing a socket file name or a tuple consisting of a string host name or ip number and an integer port. storage -- The name of the storage to connect to. A ZEO storage server can serve multiple storages. Each storage has a name, which is configured on the server. The server adminstrator should be able to provide this name. The default name for both the server and client is '1'. cache_size -- The number of bytes to allow for the client cache. The default is 20,000,000. For more information on client caches, see ClientCache.txt. name -- The name to use for the storage. This will be shown in Zope's control panel. The default name is a representation of the connection information. client -- The name to be used for the persistent client cache files. This parameter can be used instead of or to override the ZEO_CLIENT environment variable. It is generally better to use the environment variable because it's easier to change environment variables that it is to change Python code for creating the storage. Also note that, if you are using Zope, the ZEO_CLIENT environment variable effects whether products are initialized. For more information on client cache files, see ClientCache.txt. debug -- If this is provided, it should be a non-empty string. It indicates that client should log tracing and debugging information, using zLOG. var -- The directory in which persistent cache files should be written. If this option is provided, it is unnecessary to set INSTANCE_HOME in __builtins__. For more information on client cache files, see ClientCache.txt. min_disconnect_poll -- The minimum number of seconds to wait before retrying connections after connection failure. When trying to make a connection, if the connection fails, the ZEO client will wait a period of time before retrying the connection. The amount of time waited starts at the value given by 'min_disconnect_poll' and doubles on each attempt, but never exceeds the number of seconds given by 'max_disconnect_poll'. The default is 5 seconds. max_disconnect_poll -- The maximum number of seconds to wait before retrying connections after connection failure. See min_disconnect_poll. The default is 300 seconds. wait_for_server_on_starup -- Indicate whether the ClientStorage should block waiting for a storage server connection, or whether it should proceed, satisfying reads from the client cache. === Added File StandaloneZODB/ZEO/doc/start.txt === The ZEO Server start script, start.py ZEO provides a Python script for starting the ZEO server. The ZEO server is implemented as a Python class and could be used with other main programs, however, a simple ZEO server is provided for convenience. Basic usage To start the storage server, go to your Zope install directory and:: python lib/python/ZEO/start.py -p port_number (Run start without arguments to see options.) Of course, the server and the client don't have to be on the same machine. If the server and client *are* on the same machine, then you can use a Unix domain socket:: python lib/python/ZEO/start.py -U filename Serving custom storages or multiple storages with the storage server The Storage server can host multiple storages and can host any kind of storage. Each storage has a unique storage name. By default, the ZEO start.py script serves a standard FileStorage with the name '1'. You can control what storages are served by creating a Python file containing definitions for the storages and using the '-S' option to the start.py script to indicate the storage to be served. The form of the -S option is:: -Sstorage_name=module_path:attribute_name Where: storage_name -- is the storage name used in the ZEO protocol. This is the name that you give as the optional 'storage' keyword argument to the ClientStorage constructor. module_path -- This is the path to a Python module that defines the storage object(s) to be served. The module path should ommit the prefix (e.g. '.py'). attribute_name -- This is the name to which the storage object is assigned in the module. Consider the following example. I want to serve a FileStorage in read-only mode, which I define in the module file /stores/fs.py:: import ZODB.FileStorage Storage=FileStorage.FileStorage('/stores/fs1.fs', read_only=1) I then start start.py with the argument:: python lib/python/ZEO/start.py -U /xxx/var/zeo.sock \ -S 1=/stores/fs:Storage This option says to serve storage '1'. Storage '1' is found in attribute 'Storage' from the module '/stores/fs'. Now consider a more complicated example. I want to serve the storage from the previous example. I also want to serve two Oracle storages that are defined in the file '/stores/oracle.py':: import DCOracle, DCOracleStorage system=DCOracleStorage.DCOracleStorage( lambda : DCOracle.Connect('system/manager@spamservice') ) scott=DCOracleStorage.DCOracleStorage( lambda : DCOracle.Connect('scott/tiger@spamservice') ) I simply need to include three -S options:: python lib/python/ZEO/start.py -U /xxx/var/zeo.sock \ -Ssystem=/stores/oracle:system \ -Sscott=/stores/oracle:scott \ -S1=/stores/fs:Storage In this case, we made the storage and attribute name the same. To connect to the 'system' or 'scott' storage, we need to specify the storage in the ClientStorage constructor, as in:: import ZEO.ClientStorage Storage=ZEO.ClientStorage.ClientStorage( '/xxx/var/zeo.sock', storage='scott') Options The ZEO server start script is run with one or more command line options. An optional FileStorage file name may be provided after the options. The options are as follows: -D -- Run in debug mode In debug mode, the process is not run in the background and detailed debugging information is logged. Note that to actually log this information, you need to configure logging to include very low-severity (< -300) log entries. For example, to configure the stupid logger to log these messages, set the environment veriable 'STUPID_LOG_SEVERITY' to -999. -U -- Unix-domain socket file to listen on If you want to accept connections on a Unix domain socket, then use this option to specify the socket file name. -u username or uid number The username to run the ZEO server as. You may want to run the ZEO server as 'zope' or some other user with limited resouces. The only works under Unix, and if ZServer is started by root. If the server *is* started as root, the 'nobody' user if this option isn't used. -p port -- port to listen on Use this option together with the '-h' option to specify a host and port to listen on. -h adddress -- host address to listen on Use this option together with the '-p' option to specify a host and port to listen on. -s -- Don't use zdeamon This option has no effect on Unix. -S storage_name=module_path:attr_name -- A storage specification where: storage_name -- is the storage name used in the ZEO protocol. This is the name that you give as the optional 'storage' keyword argument to the ClientStorage constructor. module_path -- This is the path to a Python module that defines the storage object(s) to be served. The module path should ommit the prefix (e.g. '.py'). attr_name -- This is the name to which the storage object is assigned in the module. From jeremy at zope.com Fri Jan 11 15:24:49 2002 From: jeremy at zope.com (Jeremy Hylton) Date: Sun Aug 10 16:31:18 2008 Subject: [ZEO-Checkins] CVS: StandaloneZODB/ZEO/misc - README:1.2.2.1 custom_zodb.py:1.2.2.1 start_zeo:1.2.2.1 stop_zeo:1.2.2.1 Message-ID: <200201112024.g0BKOn510297@cvs.baymountain.com> Update of /cvs-repository/StandaloneZODB/ZEO/misc In directory cvs.zope.org:/tmp/cvs-serv10280 Added Files: Tag: Standby-branch README custom_zodb.py start_zeo stop_zeo Log Message: Merge ZEO-ZRPC-Dev branch into Standby-branch. === Added File StandaloneZODB/ZEO/misc/README === This directory contains some sampe scripts for starting/stopping ZEO, etc. Please feel free to submit improvements. :) === Added File StandaloneZODB/ZEO/misc/custom_zodb.py === # Sample custom_zodb.py __version__ = "$Revision: 1.2.2.1 $"[11:-2] # In situations where we switch between different storages, we've # found it useful to use if-elif-else pattern. import os if 0: # Change the 0 to 1 to enable! # ZEO Unix Domain Socket # This import isn't strictly necessary but is helpful when # debugging and while Zope's and Python's asyncore are out of sync # to make sure we get the right version of asyncore. import ZServer import ZEO.ClientStorage Storage=ZEO.ClientStorage.ClientStorage( os.path.join(INSTANCE_HOME, 'var', 'zeo.soc'), # If no name is given, then connection info will be shown: name="ZEO Storage", # You can specify the storage name, which defaults to "1": storage="1", ) else: # Default FileStorage import ZODB.FileStorage Storage=ZODB.FileStorage.FileStorage( os.path.join(INSTANCE_HOME, 'var', 'Data.fs'), ) === Added File StandaloneZODB/ZEO/misc/start_zeo === #!/bin/sh reldir=`dirname $0` exec python $reldir/lib/python/ZEO/start.py \ -U $reldir/var/zeo.soc \ ZEO_SERVER_PID=$reldir/var/ZEO_SERVER.pid \ STUPID_LOG_FILE=$reldir/var/ZEO_EVENTS.log \ "$@" === Added File StandaloneZODB/ZEO/misc/stop_zeo === #!/bin/sh reldir=`dirname $0` kill `cat $reldir/var/ZEO_SERVER.pid` From jeremy at zope.com Fri Jan 11 15:25:15 2002 From: jeremy at zope.com (Jeremy Hylton) Date: Sun Aug 10 16:31:18 2008 Subject: [ZEO-Checkins] CVS: StandaloneZODB/ZEO/tests - testTransactionBuffer.py:1.3.2.1 forker.py:1.10.4.3 Message-ID: <200201112025.g0BKPFg10389@cvs.baymountain.com> Update of /cvs-repository/StandaloneZODB/ZEO/tests In directory cvs.zope.org:/tmp/cvs-serv10361 Modified Files: Tag: Standby-branch forker.py Added Files: Tag: Standby-branch testTransactionBuffer.py Log Message: Merge ZEO-ZRPC-Dev branch into Standby-branch. === Added File StandaloneZODB/ZEO/tests/testTransactionBuffer.py === import random import unittest from ZEO.TransactionBuffer import TransactionBuffer def random_string(size): """Return a random string of size size.""" l = [chr(random.randrange(256)) for i in range(size)] return "".join(l) def new_store_data(): """Return arbitrary data to use as argument to store() method.""" return random_string(8), '', random_string(random.randrange(1000)) def new_invalidate_data(): """Return arbitrary data to use as argument to invalidate() method.""" return random_string(8), '' class TransBufTests(unittest.TestCase): def checkTypicalUsage(self): tbuf = TransactionBuffer() tbuf.store(*new_store_data()) tbuf.invalidate(*new_invalidate_data()) tbuf.begin_iterate() while 1: o = tbuf.next() if o is None: break tbuf.clear() def doUpdates(self, tbuf): data = [] for i in range(10): d = new_store_data() tbuf.store(*d) data.append(d) d = new_invalidate_data() tbuf.invalidate(*d) data.append(d) tbuf.begin_iterate() for i in range(len(data)): x = tbuf.next() if x[2] is None: # the tbuf add a dummy None to invalidates x = x[:2] self.assertEqual(x, data[i]) def checkOrderPreserved(self): tbuf = TransactionBuffer() self.doUpdates(tbuf) def checkReusable(self): tbuf = TransactionBuffer() self.doUpdates(tbuf) tbuf.clear() self.doUpdates(tbuf) tbuf.clear() self.doUpdates(tbuf) def test_suite(): return unittest.makeSuite(TransBufTests, 'check') === StandaloneZODB/ZEO/tests/forker.py 1.10.4.2 => 1.10.4.3 === import asyncore import os -import profile import random import socket import sys +import traceback import types import ZEO.ClientStorage, ZEO.StorageServer +# Change value of PROFILE to enable server-side profiling PROFILE = 0 +if PROFILE: + import hotshot def get_port(): """Return a port that is not in use. @@ -57,7 +60,7 @@ args = (sys.executable, script, str(port), storage_name) + args d = os.environ.copy() d['PYTHONPATH'] = os.pathsep.join(sys.path) - pid = os.spawnve(os.P_NOWAIT, sys.executable, args, d) + pid = os.spawnve(os.P_NOWAIT, sys.executable, args, os.environ) return ('localhost', port), ('localhost', port + 1), pid else: @@ -75,9 +78,11 @@ buf = self.recv(4) if buf: assert buf == "done" + server.close_server() asyncore.socket_map.clear() def handle_close(self): + server.close_server() asyncore.socket_map.clear() class ZEOClientExit: @@ -86,20 +91,27 @@ self.pipe = pipe def close(self): - os.write(self.pipe, "done") - os.close(self.pipe) + try: + os.write(self.pipe, "done") + os.close(self.pipe) + except os.error: + pass def start_zeo_server(storage, addr): rd, wr = os.pipe() pid = os.fork() if pid == 0: - if PROFILE: - p = profile.Profile() - p.runctx("run_server(storage, addr, rd, wr)", globals(), - locals()) - p.dump_stats("stats.s.%d" % os.getpid()) - else: - run_server(storage, addr, rd, wr) + try: + if PROFILE: + p = hotshot.Profile("stats.s.%d" % os.getpid()) + p.runctx("run_server(storage, addr, rd, wr)", + globals(), locals()) + p.close() + else: + run_server(storage, addr, rd, wr) + except: + print "Exception in ZEO server process" + traceback.print_exc() os._exit(0) else: os.close(rd) @@ -107,11 +119,11 @@ def run_server(storage, addr, rd, wr): # in the child, run the storage server + global server os.close(wr) ZEOServerExit(rd) - serv = ZEO.StorageServer.StorageServer(addr, {'1':storage}) + server = ZEO.StorageServer.StorageServer(addr, {'1':storage}) asyncore.loop() - os.close(rd) storage.close() if isinstance(addr, types.StringType): os.unlink(addr) @@ -137,6 +149,7 @@ s = ZEO.ClientStorage.ClientStorage(addr, storage_id, debug=1, client=cache, cache_size=cache_size, - min_disconnect_poll=0.5) + min_disconnect_poll=0.5, + wait_for_server_on_startup=1) return s, exit, pid From jeremy at zope.com Tue Jan 15 10:22:22 2002 From: jeremy at zope.com (Jeremy Hylton) Date: Sun Aug 10 16:31:18 2008 Subject: [ZEO-Checkins] CVS: StandaloneZODB/ZEO/tests - testZEO.py:1.1.2.19 Message-ID: <200201151522.g0FFMM604384@cvs.baymountain.com> Update of /cvs-repository/StandaloneZODB/ZEO/tests In directory cvs.zope.org:/tmp/cvs-serv4373 Modified Files: Tag: ZEO-ZRPC-Dev testZEO.py Log Message: Fix missing storage in ZEO tests. The StandbyTestBase was changed to set self._storage = None to initialize it. But the GenericTests in testZEO initialized self._storage and then called StandbyTestBase setUp(). Fixed by calling superclass setUp() first. Also, remove some doc strings and re-enable all the ZEOFileStorageTests. === StandaloneZODB/ZEO/tests/testZEO.py 1.1.2.18 => 1.1.2.19 === def setUp(self): - """Start a ZEO server using a Unix domain socket - - The ZEO server uses the storage object returned by the - getStorage() method. - """ + self.__super_setUp() self.running = 1 client, exit, pid = forker.start_zeo(self.getStorage()) self._pids = [pid] self._servers = [exit] self._storage = PackWaitWrapper(client) client.registerDB(DummyDB(), None) - self.__super_setUp() def tearDown(self): - """Try to cause the tests to halt""" self.running = 0 self._storage.close() for server in self._servers: @@ -510,8 +504,8 @@ return meth.keys() if os.name == "posix": -## test_classes = ZEOFileStorageTests, UnixConnectionTests - test_classes = UnixConnectionTests, + test_classes = ZEOFileStorageTests, UnixConnectionTests +## test_classes = UnixConnectionTests, elif os.name == "nt": test_classes = WindowsZEOFileStorageTests, WindowsConnectionTests else: From jeremy at zope.com Tue Jan 15 10:40:20 2002 From: jeremy at zope.com (Jeremy Hylton) Date: Sun Aug 10 16:31:18 2008 Subject: [ZEO-Checkins] CVS: StandaloneZODB/ZEO/tests - testZEO.py:1.16.4.2 Message-ID: <200201151540.g0FFeKt08675@cvs.baymountain.com> Update of /cvs-repository/StandaloneZODB/ZEO/tests In directory cvs.zope.org:/tmp/cvs-serv8664 Modified Files: Tag: Standby-branch testZEO.py Log Message: Track changes for ZEO-ZRPC-Dev branch. (XXX I might stop work on that branch and just work on this branch. It's too confusing to have two branches for the same thing.) === StandaloneZODB/ZEO/tests/testZEO.py 1.16.4.1 => 1.16.4.2 === def setUp(self): - """Start a ZEO server using a Unix domain socket - - The ZEO server uses the storage object returned by the - getStorage() method. - """ + self.__super_setUp() self.running = 1 client, exit, pid = forker.start_zeo(self.getStorage()) self._pid = pid self._server = exit self._storage = PackWaitWrapper(client) client.registerDB(DummyDB(), None) - self.__super_setUp() def tearDown(self): - """Try to cause the tests to halt""" self.running = 0 self._storage.close() self._server.close() From jeremy at zope.com Tue Jan 15 10:52:49 2002 From: jeremy at zope.com (Jeremy Hylton) Date: Sun Aug 10 16:31:18 2008 Subject: [ZEO-Checkins] CVS: StandaloneZODB/ZEO/tests - testZEO.py:1.16.4.3 Message-ID: <200201151552.g0FFqn111609@cvs.baymountain.com> Update of /cvs-repository/StandaloneZODB/ZEO/tests In directory cvs.zope.org:/tmp/cvs-serv11598 Modified Files: Tag: Standby-branch testZEO.py Log Message: More explicit merge of testZEO from ZEO-ZRPC-Dev branch. Don't know what went wrong before but there were a raft of differences between the branches. I checked out the other branch and copied this file from that branch into my checkout of the Standby-branch. === StandaloneZODB/ZEO/tests/testZEO.py 1.16.4.2 => 1.16.4.3 === import os import random +import select import socket import sys import tempfile @@ -30,7 +31,8 @@ # Sorry Jim... from ZODB.tests import StorageTestBase, BasicStorage, VersionStorage, \ TransactionalUndoStorage, TransactionalUndoVersionStorage, \ - PackableStorage, Synchronization, ConflictResolution, RevisionStorage + PackableStorage, Synchronization, ConflictResolution, RevisionStorage, \ + MTStorage, ReadOnlyStorage from ZODB.tests.MinPO import MinPO from ZODB.tests.StorageTestBase import zodb_unpickle @@ -134,6 +136,8 @@ RevisionStorage.RevisionStorage, PackableStorage.PackableStorage, Synchronization.SynchronizedStorage, + MTStorage.MTStorage, + ReadOnlyStorage.ReadOnlyStorage, ): """An abstract base class for ZEO tests @@ -150,16 +154,18 @@ self.__super_setUp() self.running = 1 client, exit, pid = forker.start_zeo(self.getStorage()) - self._pid = pid - self._server = exit + self._pids = [pid] + self._servers = [exit] self._storage = PackWaitWrapper(client) client.registerDB(DummyDB(), None) def tearDown(self): self.running = 0 self._storage.close() - self._server.close() - os.waitpid(self._pid, 0) + for server in self._servers: + server.close() + for pid in self._pids: + os.waitpid(pid, 0) self.delStorage() self.__super_tearDown() @@ -174,6 +180,16 @@ self.__fs_base = tempfile.mktemp() self.__super_setUp() + def open(self, read_only=0): + # XXX Needed to support ReadOnlyStorage tests. Ought to be a + # cleaner way. + + # Is this the only way to get the address? + addr = self._storage._rpc_mgr.addr[0][1] + self._storage.close() + self._storage = ZEO.ClientStorage.ClientStorage(addr, read_only=1, + wait_for_server_on_startup=1) + def getStorage(self): return FileStorage(self.__fs_base, create=1) @@ -258,43 +274,85 @@ def tearDown(self): """Try to cause the tests to halt""" + if getattr(self, '_storage', None) is not None: + self._storage.close() self.shutdownServer() # file storage appears to create four files - for ext in '', '.index', '.lock', '.tmp': - path = self.file + ext - if os.path.exists(path): - os.unlink(path) + for i in range(len(self.addr)): + for ext in '', '.index', '.lock', '.tmp': + path = "%s.%s%s" % (self.file, i, ext) + if os.path.exists(path): + os.unlink(path) for i in 0, 1: path = "c1-test-%d.zec" % i if os.path.exists(path): os.unlink(path) self.__super_tearDown() + def checkMultipleAddresses(self): + for i in range(4): + self._newAddr() + self._storage = self.openClientStorage('test', 100000, wait=1) + oid = self._storage.new_oid() + obj = MinPO(12) + revid1 = self._dostore(oid, data=obj) + self._storage.close() + + def checkMultipleServers(self): + # XXX crude test at first -- just start two servers and do a + # commit at each one. + + self._newAddr() + self._storage = self.openClientStorage('test', 100000, wait=1) + self._dostore() + + self.shutdownServer(index=0) + self._startServer(index=1) + + # If we can still store after shutting down one of the + # servers, we must be reconnecting to the other server. + + for i in range(10): + try: + self._dostore() + break + except Disconnected: + time.sleep(0.5) + + + def checkDisconnectionError(self): + # Make sure we get a Disconnected when we try to read an + # object when we're not connected to a storage server and the + # object is not in the cache. + self.shutdownServer() + self._storage = self.openClientStorage('test', 1000, wait=0) + self.assertRaises(Disconnected, self._storage.load, 'fredwash', '') + def checkBasicPersistence(self): - """Verify cached data persists across client storage instances. + # Verify cached data persists across client storage instances. - To verify that the cache is being used, the test closes the - server and then starts a new client with the server down. - """ - self._storage = self.openClientStorage('test', 100000, 1) + # To verify that the cache is being used, the test closes the + # server and then starts a new client with the server down. + + self._storage = self.openClientStorage('test', 100000, wait=1) oid = self._storage.new_oid() obj = MinPO(12) revid1 = self._dostore(oid, data=obj) self._storage.close() self.shutdownServer() - self._storage = self.openClientStorage('test', 100000, 0) + self._storage = self.openClientStorage('test', 100000, wait=0) data, revid2 = self._storage.load(oid, '') assert zodb_unpickle(data) == MinPO(12) assert revid1 == revid2 self._storage.close() def checkRollover(self): - """Check that the cache works when the files are swapped. + # Check that the cache works when the files are swapped. - In this case, only one object fits in a cache file. When the - cache files swap, the first object is effectively uncached. - """ - self._storage = self.openClientStorage('test', 1000, 1) + # In this case, only one object fits in a cache file. When the + # cache files swap, the first object is effectively uncached. + + self._storage = self.openClientStorage('test', 1000, wait=1) oid1 = self._storage.new_oid() obj1 = MinPO("1" * 500) revid1 = self._dostore(oid1, data=obj1) @@ -303,14 +361,20 @@ revid2 = self._dostore(oid2, data=obj2) self._storage.close() self.shutdownServer() - self._storage = self.openClientStorage('test', 1000, 0) + self._storage = self.openClientStorage('test', 1000, wait=0) + self._storage.load(oid1, '') self._storage.load(oid2, '') - self.assertRaises(Disconnected, self._storage.load, oid1, '') def checkReconnection(self): - """Check that the client reconnects when a server restarts.""" + # Check that the client reconnects when a server restarts. + + # XXX Seem to get occasional errors that look like this: + # File ZEO/zrpc2.py, line 217, in handle_request + # File ZEO/StorageServer.py, line 325, in storea + # File ZEO/StorageServer.py, line 209, in _check_tid + # StorageTransactionError: (None, ) + # could system reconnect and continue old transaction? - from ZEO.ClientStorage import ClientDisconnected self._storage = self.openClientStorage() oid = self._storage.new_oid() obj = MinPO(12) @@ -323,11 +387,11 @@ while 1: try: revid1 = self._dostore(oid, data=obj) - except (ClientDisconnected, thread.error, socket.error), err: - get_transaction().abort() - time.sleep(0.1) - else: break + except (Disconnected, select.error, thread.error, socket.error), \ + err: + get_transaction().abort() + time.sleep(0.1) # XXX how long to sleep # XXX This is a bloody pain. We're placing a heavy burden # on users to catch a plethora of exceptions in order to # write robust code. Need to think about implementing @@ -345,32 +409,50 @@ """ self.running = 1 self.file = tempfile.mktemp() - self.addr = '', self.ports.pop() + self.addr = [] + self._pids = [] + self._servers = [] + self._newAddr() self._startServer() self.__super_setUp() - def _startServer(self, create=1): - fs = FileStorage(self.file, create=create) - self._pid, self._server = forker.start_zeo_server(fs, self.addr) + def _newAddr(self): + self.addr.append(self._getAddr()) + + def _getAddr(self): + return '', self.ports.pop() + + def _startServer(self, create=1, index=0): + fs = FileStorage("%s.%d" % (self.file, index), create=create) + addr = self.addr[index] + pid, server = forker.start_zeo_server(fs, addr) + self._pids.append(pid) + self._servers.append(server) def openClientStorage(self, cache='', cache_size=200000, wait=1): base = ZEO.ClientStorage.ClientStorage(self.addr, client=cache, cache_size=cache_size, - wait_for_server_on_startup=wait) + wait_for_server_on_startup=wait, + min_disconnect_poll=0.1) storage = PackWaitWrapper(base) storage.registerDB(DummyDB(), None) return storage - def shutdownServer(self): + def shutdownServer(self, index=0): if self.running: self.running = 0 - self._server.close() - os.waitpid(self._pid, 0) + self._servers[index].close() + try: + os.waitpid(self._pids[index], 0) + except os.error: + pass class WindowsConnectionTests(ConnectionTests): __super_setUp = StorageTestBase.StorageTestBase.setUp + # XXX these tests are now out-of-date + def setUp(self): self.file = tempfile.mktemp() self._startServer() @@ -423,6 +505,7 @@ if os.name == "posix": test_classes = ZEOFileStorageTests, UnixConnectionTests +## test_classes = UnixConnectionTests, elif os.name == "nt": test_classes = WindowsZEOFileStorageTests, WindowsConnectionTests else: From jeremy at zope.com Tue Jan 15 12:22:11 2002 From: jeremy at zope.com (Jeremy Hylton) Date: Sun Aug 10 16:31:18 2008 Subject: [ZEO-Checkins] CVS: StandaloneZODB/ZEO - trigger.py:1.3.4.2 Message-ID: <200201151722.g0FHMBZ00484@cvs.baymountain.com> Update of /cvs-repository/StandaloneZODB/ZEO In directory cvs.zope.org:/tmp/cvs-serv473 Modified Files: Tag: Standby-branch trigger.py Log Message: Fix trigger close/__del__. The close() mechanism for an asyncore file_dispatcher is not safe to call multiple times. It's calling os.close() on a file descriptor (int). Guido observed that if you call close() twice, you could be in trouble: 1) First close() call closes FD 6. 2) Another bit of code opens a new file, getting FD 6. 3) Second close() call closes FD 6. Waah! FD 6 is some other file. The workaround attempt here is to define a close() method on a trigger that only closes the file descriptors the first time. Also, make sure that both file descriptors are closed. The previous version only closed the read-end of the pipe. === StandaloneZODB/ZEO/trigger.py 1.3.4.1 => 1.3.4.2 === # INFRINGEMENT, AND FITNESS FOR A PARTICULAR PURPOSE. - # This module is a simplified version of the select_trigger module # from Sam Rushing's Medusa server. - import asyncore -#import asynchat import os import socket @@ -60,10 +57,20 @@ asyncore.file_dispatcher.__init__ (self, r) self.lock = thread.allocate_lock() self.thunks = [] + self._closed = None - def __del__(self): - os.close(self._fds[0]) - os.close(self._fds[1]) + # Override the asyncore close() method, because it seems that + # it would only close the r file descriptor and not w. The + # constructor calls file_dispactcher.__init__ and passes r, + # which would get stored in a file_wrapper and get closed by + # the default close. But that would leave w open... + + def close(self): + if self._closed is None: + self._closed = 1 + self.del_channel() + for fd in self._fds: + os.close(fd) def __repr__ (self): return '' % id(self) @@ -78,7 +85,6 @@ pass def pull_trigger (self, thunk=None): - # print 'PULL_TRIGGER: ', len(self.thunks) if thunk: try: self.lock.acquire() From jeremy at zope.com Tue Jan 15 12:23:26 2002 From: jeremy at zope.com (Jeremy Hylton) Date: Sun Aug 10 16:31:18 2008 Subject: [ZEO-Checkins] CVS: StandaloneZODB/ZEO - zrpc2.py:1.3.2.2 Message-ID: <200201151723.g0FHNQp00599@cvs.baymountain.com> Update of /cvs-repository/StandaloneZODB/ZEO In directory cvs.zope.org:/tmp/cvs-serv588 Modified Files: Tag: Standby-branch zrpc2.py Log Message: Make sure triggers are closed. === StandaloneZODB/ZEO/zrpc2.py 1.3.2.1 => 1.3.2.2 === self.closed = 0 self.async = 0 + self.trigger = None # The reply lock is used to block when a synchronous call is # waiting for a response self.__super_init(sock, addr) @@ -164,8 +165,13 @@ if self.closed: return self.closed = 1 + self.close_trigger() self.__super_close() + def close_trigger(self): + if self.trigger is not None: + self.trigger.close() + def register_object(self, obj): """Register obj as the true object to invoke methods on""" self.obj = obj @@ -394,6 +400,7 @@ self.debug = debug self.connected = 0 self.connection = None + self.trigger = None # If _thread is not None, then there is a helper thread # attempting to connect. _thread is protected by _connect_lock. self._thread = None @@ -441,6 +448,8 @@ def close(self): """Prevent ConnectionManager from opening new connections""" self.closed = 1 + if self.trigger is not None: + self.trigger.close() self._connect_lock.acquire() try: if self._thread is not None: @@ -619,6 +628,10 @@ else: self.__async = None self.__super_init(sock, addr, obj) + + def close_trigger(self): + # the manager should actually close the trigger + del self.trigger def _prepare_async(self): # Don't do the register_loop_callback that the superclass does From jeremy at zope.com Tue Jan 15 12:27:02 2002 From: jeremy at zope.com (Jeremy Hylton) Date: Sun Aug 10 16:31:18 2008 Subject: [ZEO-Checkins] CVS: StandaloneZODB/ZEO/tests - stress.py:1.2.4.2 Message-ID: <200201151727.g0FHR2m01704@cvs.baymountain.com> Update of /cvs-repository/StandaloneZODB/ZEO/tests In directory cvs.zope.org:/tmp/cvs-serv1665 Modified Files: Tag: Standby-branch stress.py Log Message: Add wait_for_server_on_startup to ClientStorage constructor call. === StandaloneZODB/ZEO/tests/stress.py 1.2.4.1 => 1.2.4.2 === -# -# This software is subject to the provisions of the Zope Public License, -# Version 1.1 (ZPL). A copy of the ZPL should accompany this -# distribution. THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL -# EXPRESS OR IMPLIED WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST -# INFRINGEMENT, AND FITNESS FOR A PARTICULAR PURPOSE. - """A ZEO client-server stress test to look for leaks. The stress test should run in an infinite loop and should involve @@ -100,7 +91,8 @@ if pid != 0: return pid - storage = ClientStorage(zaddr, debug=1, min_disconnect_poll=0.5) + storage = ClientStorage(zaddr, debug=1, min_disconnect_poll=0.5, + wait_for_server_on_startup=1) db = ZODB.DB(storage, pool_size=NUM_CONNECTIONS) setup(db.open()) conns = [] From jeremy at zope.com Tue Jan 15 17:38:41 2002 From: jeremy at zope.com (Jeremy Hylton) Date: Sun Aug 10 16:31:18 2008 Subject: [ZEO-Checkins] CVS: StandaloneZODB/ZEO - zrpc2.py:1.3.2.3 Message-ID: <200201152238.g0FMcfX12081@cvs.baymountain.com> Update of /cvs-repository/StandaloneZODB/ZEO In directory cvs.zope.org:/tmp/cvs-serv12065 Modified Files: Tag: Standby-branch zrpc2.py Log Message: Rework the is_async() logic to actually work in cases where it is true. Implement the various classes that care using thr_async variable -- to avoid confusion with other async things like SizedMessageAsyncConnection and asynchronous messages. As a comment in the Connection constructor explains: # A Connection either uses asyncore directly or relies on an # asyncore mainloop running in a separate thread. If # thr_async is true, then the mainloop is running in a # separate thread. If thr_async is true, then the asyncore # trigger (self.trigger) is used to notify that thread of # activity on the current thread. Make sure that ManagedConnection sets thr_async and trigger properly, based on the ConnectionManager. === StandaloneZODB/ZEO/zrpc2.py 1.3.2.2 => 1.3.2.3 === def __init__(self, sock, addr, obj=None): - self.msgid = 0 self.obj = obj self.marshal = Marshaller() self.closed = 0 - self.async = 0 - self.trigger = None - # The reply lock is used to block when a synchronous call is - # waiting for a response + self.msgid = 0 self.__super_init(sock, addr) - self._map = {self._fileno: self} + # A Connection either uses asyncore directly or relies on an + # asyncore mainloop running in a separate thread. If + # thr_async is true, then the mainloop is running in a + # separate thread. If thr_async is true, then the asyncore + # trigger (self.trigger) is used to notify that thread of + # activity on the current thread. + self.thr_async = 0 + self.trigger = None self._prepare_async() + self._map = {self._fileno: self} self.__call_lock = thread.allocate_lock() + # The reply lock is used to block when a synchronous call is + # waiting for a response self.__reply_lock = thread.allocate_lock() self.__reply_lock.acquire() + # If the object implements the Handler interface (XXX checked + # by isinstance), it wants to know who the caller is. if isinstance(obj, Handler): self.set_caller = 1 else: self.set_caller = 0 def __repr__(self): - return "<%s %s>" % (self.__class__.__name__, self.addr) + return "<%s %s %s>" % (self.__class__.__name__, self.addr, + hex(id(self))) def close(self): if self.closed: @@ -338,7 +347,7 @@ pass # XXX what is this supposed to do? def _prepare_async(self): - self._async = 0 + self.thr_async = 0 ThreadedAsync.register_loop_callback(self.set_async) # XXX If we are not in async mode, this will cause dead # Connections to be leaked. @@ -346,18 +355,21 @@ def set_async(self, map): # XXX do we need a lock around this? I'm not sure there is # any harm to a race with _do_io(). - self._async = 1 self.trigger = trigger.trigger() + self.thr_async = 1 def is_async(self): - return self._async + if self.thr_async: + return 1 + else: + return 0 def _do_io(self, wait=0): # XXX need better name # XXX invariant? lock must be held when calling with wait==1 # otherwise, in non-async mode, there will be no poll if __debug__: - log("_do_io(wait=%d), async=%d" % (wait, self.is_async()), + log("_do_io(wait=%d), async=%s" % (wait, self.is_async()), level=zLOG.DEBUG) if self.is_async(): self.trigger.pull_trigger() @@ -406,7 +418,7 @@ self._thread = None self._connect_lock = threading.Lock() self.trigger = None - self.async = 0 + self.thr_async = 0 self.closed = 0 ThreadedAsync.register_loop_callback(self.set_async) @@ -464,8 +476,8 @@ def set_async(self, map): # XXX need each connection started with async==0 to have a callback - self.async = 1 # XXX needs to be set on the Connection self.trigger = trigger.trigger() + self.thr_async = 1 # XXX needs to be set on the Connection def connect(self, sync=0): if self.connected == 1: @@ -594,7 +606,9 @@ self.connection = c return 1 except: - # XXX zLOG the error + zLOG.LOG(_label, zLOG.ERROR, + "error connecting to server: %s" % str(addr), + error=sys.exc_info()) c.close() return 0 @@ -622,29 +636,30 @@ def __init__(self, sock, addr, obj, mgr): self.__mgr = mgr - if self.__mgr.async: - self.__async = 1 - self.trigger = self.__mgr.trigger - else: - self.__async = None self.__super_init(sock, addr, obj) + self.check_mgr_async() def close_trigger(self): # the manager should actually close the trigger del self.trigger def _prepare_async(self): - # Don't do the register_loop_callback that the superclass does + # Don't do the register_loop_callback; the manager handles it pass + def check_mgr_async(self): + if not self.thr_async and self.__mgr.thr_async: + assert self.__mgr.trigger is not None, \ + "manager (%s) has no trigger" % self.__mgr + self.thr_async = 1 + self.trigger = self.__mgr.trigger + return 1 + return 0 + def is_async(self): - if self.__async: + if self.thr_async: return 1 - async = self.__mgr.async - if async: - self.__async = 1 - self.trigger = self.__mgr.trigger - return async + return self.check_mgr_async() def close(self): self.__super_close() @@ -726,4 +741,3 @@ return r raise ZRPCError("Unsafe global: %s.%s" % (module, name)) - From jeremy at zope.com Wed Jan 16 09:51:33 2002 From: jeremy at zope.com (Jeremy Hylton) Date: Sun Aug 10 16:31:18 2008 Subject: [ZEO-Checkins] CVS: StandaloneZODB/ZEO/zrpc StandaloneZODB/ZEO/zrpc - New directory Message-ID: <200201161451.g0GEpXj22326@cvs.baymountain.com> Update of /cvs-repository/StandaloneZODB/ZEO/zrpc In directory cvs.zope.org:/tmp/cvs-serv22317/zrpc Log Message: Directory /cvs-repository/StandaloneZODB/ZEO/zrpc added to the repository --> Using per-directory sticky tag `Standby-branch' === Added directory StandaloneZODB/ZEO/zrpc === From jeremy at zope.com Wed Jan 16 09:52:39 2002 From: jeremy at zope.com (Jeremy Hylton) Date: Sun Aug 10 16:31:18 2008 Subject: [ZEO-Checkins] CVS: StandaloneZODB/ZEO/zrpc - NOTES:1.1.2.1 __init__.py:1.1.2.1 client.py:1.1.2.1 connection.py:1.1.2.1 error.py:1.1.2.1 log.py:1.1.2.1 marshal.py:1.1.2.1 server.py:1.1.2.1 trigger.py:1.1.2.1 Message-ID: <200201161452.g0GEqdj22556@cvs.baymountain.com> Update of /cvs-repository/StandaloneZODB/ZEO/zrpc In directory cvs.zope.org:/tmp/cvs-serv22529/zrpc Added Files: Tag: Standby-branch NOTES __init__.py client.py connection.py error.py log.py marshal.py server.py trigger.py Log Message: Convert zrpc from module zrpc2 to zrpc package. === Added File StandaloneZODB/ZEO/zrpc/NOTES === The Connection object should be extended to support more flexible handling for outstanding calls. In particular, it should be possible to have multiple calls with return values outstanding. The mechanism described here is based on the promises mechanism in Argus, which was influenced by futures in Multilisp. Promises: Linguistic Support for Efficient Asynchronous Procedure Calls in Distributed Systems. Barbara Liskov and Liuba Shrira. Proc. of Conf. on Programming Language Design and Implementation (PLDI), June 1988. We want to support two different kinds of calls: - send : invoke a method that returns no value - call : invoke a method that returns a value On the client, a call immediately returns a promise. A promise is an object that can be used to claim the return value when it becomes available. - ready(): returns true if the return value is ready or an exception occurred - claim(): returns the call's return value or raises an exception, blocking if necessary The server side of a zrpc connection can be implemented using asyncore. In that case, a method call blocks other RPC activity until it returns. If a call needs to return a value, but can't return immediately, it returns a delay object (ZEO.zrpc.server.Delay). When the zrpc connection receives a Delay object, it does not immediately return to the caller. Instead, it returns when the reply() method is called. A Delay has two methods: - set_sender() - reply(obj): returns obj to the sender ----------------------------------------- Open issues: Delayed exception There is currently no mechanism to raise an exception from a delayed pcall. Synchronization The following item is part of Argus, but the motivation isn't entirely clear. For any two calls, C1 and C2, C1 always starts on the server first. For the promises, C2 is ready() iff C1 is also ready(). The promises can be claimed in any order. A related notion: The connection should also support a synch() method that returns only when all outstanding calls have completed. If any of these calls raised an exception, the synch() call raises an exception. XXX synch() sounds potentially useful, but it's not clear if it would be useful for ZEO. In ZEO a single connection object handles multiple threads, each thread is going to make independent calls. When a particular tpc_begin() returns and a thread commits its transaction, it makes more calls. These calls will before any of the other tpc_begin() calls. I think the Argus approach would be to use separate handlers for each thread (not sure Argus had threads), so that a single thread could rely on ordering guarantees. Multithreaded server There are lots of issues to work out here. Delays may not be necessary if the connecftion handler runs in a different thread than the object the handles the calls. === Added File StandaloneZODB/ZEO/zrpc/__init__.py === # zrpc is a package with the following modules # error -- exceptions raised by zrpc # marshal -- internal, handles basic protocol issues # connection -- object dispatcher # client -- manages connection creation to remote server # server -- manages incoming connections from remote clients # trigger -- medusa's trigger === Added File StandaloneZODB/ZEO/zrpc/client.py === import errno import select import socket import sys import threading import time import types import ThreadedAsync import zLOG from ZEO.zrpc.log import log from ZEO.zrpc.trigger import trigger from ZEO.zrpc.connection import ManagedConnection class ConnectionManager: """Keeps a connection up over time""" def __init__(self, addr, client, tmin=1, tmax=180): self.set_addr(addr) self.client = client self.tmin = tmin self.tmax = tmax self.connected = 0 self.connection = None # If _thread is not None, then there is a helper thread # attempting to connect. _thread is protected by _connect_lock. self._thread = None self._connect_lock = threading.Lock() self.trigger = None self.thr_async = 0 self.closed = 0 ThreadedAsync.register_loop_callback(self.set_async) def __repr__(self): return "<%s for %s>" % (self.__class__.__name__, self.addr) def set_addr(self, addr): "Set one or more addresses to use for server." # For backwards compatibility (and simplicity?) the # constructor accepts a single address in the addr argument -- # a string for a Unix domain socket or a 2-tuple with a # hostname and port. It can also accept a list of such addresses. addr_type = self._guess_type(addr) if addr_type is not None: self.addr = [(addr_type, addr)] else: self.addr = [] for a in addr: addr_type = self._guess_type(a) if addr_type is None: raise ValueError, "unknown address in list: %s" % repr(a) self.addr.append((addr_type, a)) def _guess_type(self, addr): if isinstance(addr, types.StringType): return socket.AF_UNIX if (len(addr) == 2 and isinstance(addr[0], types.StringType) and isinstance(addr[1], types.IntType)): return socket.AF_INET # not anything I know about return None def close(self): """Prevent ConnectionManager from opening new connections""" self.closed = 1 self._connect_lock.acquire() try: if self._thread is not None: # XXX race on _thread self._thread.stop() self._thread.join() finally: self._connect_lock.release() if self.connection: self.connection.close() # XXX get rid of this? def register_client(self, client): self.client = client def set_async(self, map): # XXX need each connection started with async==0 to have a callback self.trigger = trigger() self.thr_async = 1 # XXX needs to be set on the Connection def attempt_connect(self): # XXX will a single attempt take too long? self.connect() try: event = self._thread.one_attempt except AttributeError: pass else: event.wait() return self.connected def connect(self, sync=0): if self.connected == 1: return self._connect_lock.acquire() try: if self._thread is None: log("starting thread to connect to server") self._thread = ConnectThread(self, self.client, self.addr, self.tmin, self.tmax) self._thread.start() if sync: try: self._thread.join() except AttributeError: # probably means the thread exited quickly pass finally: self._connect_lock.release() def connect_done(self, c): log("connect_done()") self.connected = 1 self.connection = c self._thread = None def notify_closed(self, conn): self.connected = 0 self.connection = None self.client.notifyDisconnected() if not self.closed: self.connect() class Connected(Exception): # helper for non-local exit def __init__(self, sock): self.sock = sock class ConnectThread(threading.Thread): __super_init = threading.Thread.__init__ def __init__(self, mgr, client, addr, tmin, tmax): self.__super_init(name="Connect(%s)" % addr) self.mgr = mgr self.client = client self.addr = addr self.tmin = tmin self.tmax = tmax self.stopped = 0 self.one_attempt = threading.Event() def stop(self): self.stopped = 1 def run(self): delay = self.tmin while not (self.stopped or self.attempt_connects()): if not self.one_attempt.isSet(): self.one_attempt.set() time.sleep(delay) delay *= 2 if delay > self.tmax: delay = self.tmax log("thread exiting: %s" % self.getName()) def attempt_connects(self): "Return true if any connect attempt succeeds." sockets = {} log("attempting connection on %d sockets" % len(self.addr)) try: for domain, addr in self.addr: if __debug__: log("attempt connection to %s" % repr(addr), level=zLOG.DEBUG) s = socket.socket(domain, socket.SOCK_STREAM) s.setblocking(0) # XXX can still block for a while if addr requires DNS e = self.connect(s, addr) if e is not None: sockets[s] = addr # next wait until they actually connect while sockets: if self.stopped: for s in sockets.keys(): s.close() return 0 try: r, w, x = select.select([], sockets.keys(), [], 1.0) except select.error: continue for s in w: e = self.connect(s, sockets[s]) if e is None: del sockets[s] except Connected, container: s = container.sock del sockets[s] # close all the other sockets for s in sockets.keys(): s.close() return 1 return 0 def connect(self, s, addr): """Call s.connect_ex(addr) and return true if loop should continue. We have to handle several possible return values from connect_ex(). If the socket is connected and the initial ZEO setup works, we're done. Report success by raising an exception. Yes, the is odd, but we need to bail out of the select() loop in the caller and an exception is a principled way to do the abort. If the socket sonnects and the initial ZEO setup fails or the connect_ex() returns an error, we close the socket and ignore it. If connect_ex() returns EINPROGRESS, we need to try again later. """ e = s.connect_ex(addr) if e == errno.EINPROGRESS: return 1 elif e == 0: c = self.test_connection(s, addr) log("connected to %s" % repr(addr), level=zLOG.DEBUG) if c: raise Connected(s) else: if __debug__: log("error connecting to %s: %s" % (addr, errno.errorcode[e]), level=zLOG.DEBUG) s.close() def test_connection(self, s, addr): c = ManagedConnection(s, addr, self.client, self.mgr) try: self.client.notifyConnected(c) except: log("error connecting to server: %s" % str(addr), level=zLOG.ERROR, error=sys.exc_info()) c.close() return 0 self.mgr.connect_done(c) return 1 === Added File StandaloneZODB/ZEO/zrpc/connection.py === import asyncore import sys import threading import types import ThreadedAsync from ZEO import smac # XXX put smac in zrpc? from ZEO.zrpc.error import ZRPCError, DisconnectedError, DecodingError from ZEO.zrpc.log import log from ZEO.zrpc.marshal import Marshaller from ZEO.zrpc.trigger import trigger import zLOG from ZODB import POSException REPLY = ".reply" # message name used for replies ASYNC = 1 # XXX get rid of this class and use hasattr() class Handler: """Base class used to handle RPC caller discovery""" def set_caller(self, addr): self.__caller = addr def get_caller(self): return self.__caller def clear_caller(self): self.__caller = None class Delay: """Used to delay response to client for synchronous calls When a synchronous call is made and the original handler returns without handling the call, it returns a Delay object that prevents the mainloop from sending a response. """ def set_sender(self, msgid, send_reply): self.msgid = msgid self.send_reply = send_reply def reply(self, obj): self.send_reply(self.msgid, obj) class Connection(smac.SizedMessageAsyncConnection): """Dispatcher for RPC on object The connection supports synchronous calls, which expect a return, and asynchronous calls that do not. It uses the Marshaller class to handle encoding and decoding of method calls are arguments. A Connection is designed for use in a multithreaded application, where a synchronous call must block until a response is ready. The current design only allows a single synchronous call to be outstanding. """ __super_init = smac.SizedMessageAsyncConnection.__init__ __super_close = smac.SizedMessageAsyncConnection.close __super_writable = smac.SizedMessageAsyncConnection.writable def __init__(self, sock, addr, obj=None): self.obj = obj self.marshal = Marshaller() self.closed = 0 self.msgid = 0 self.__super_init(sock, addr) # A Connection either uses asyncore directly or relies on an # asyncore mainloop running in a separate thread. If # thr_async is true, then the mainloop is running in a # separate thread. If thr_async is true, then the asyncore # trigger (self.trigger) is used to notify that thread of # activity on the current thread. self.thr_async = 0 self.trigger = None self._prepare_async() self._map = {self._fileno: self} self.__call_lock = threading.Lock() # The reply lock is used to block when a synchronous call is # waiting for a response self.__reply_lock = threading.Lock() self.__reply_lock.acquire() # If the object implements the Handler interface (XXX checked # by isinstance), it wants to know who the caller is. if isinstance(obj, Handler): self.set_caller = 1 else: self.set_caller = 0 def __repr__(self): return "<%s %s>" % (self.__class__.__name__, self.addr) def close(self): caller = sys._getframe(1).f_code.co_name log("close() caller=%s" % caller) if self.closed: return self.closed = 1 self.close_trigger() self.__super_close() def close_trigger(self): if self.trigger is not None: self.trigger.close() def register_object(self, obj): """Register obj as the true object to invoke methods on""" self.obj = obj def message_input(self, message): """Decoding an incoming message and dispatch it""" # XXX Not sure what to do with errors that reach this level. # Need to catch ZRPCErrors in handle_reply() and # handle_request() so that they get back to the client. try: msgid, flags, name, args = self.marshal.decode(message) except DecodingError, msg: return self.return_error(None, None, DecodingError, msg) if __debug__: log("recv msg: %s, %s, %s, %s" % (msgid, flags, name, repr(args)[:40]), level=zLOG.DEBUG) if name == REPLY: self.handle_reply(msgid, flags, args) else: self.handle_request(msgid, flags, name, args) def handle_reply(self, msgid, flags, args): if __debug__: log("recv reply: %s, %s, %s" % (msgid, flags, str(args)[:40]), level=zLOG.DEBUG) self.__reply = msgid, flags, args self.__reply_lock.release() # will fail if lock is unlocked def handle_request(self, msgid, flags, name, args): if __debug__: log("call %s%s on %s" % (name, repr(args)[:40], repr(self.obj)), zLOG.DEBUG) if not self.check_method(name): raise ZRPCError("Invalid method name: %s on %s" % (name, `self.obj`)) meth = getattr(self.obj, name) try: if self.set_caller: self.obj.set_caller(self) try: ret = meth(*args) finally: self.obj.clear_caller() else: ret = meth(*args) except (POSException.UndoError, POSException.VersionCommitError), msg: error = sys.exc_info()[:2] log("%s() raised exception: %s" % (name, msg), zLOG.ERROR, error) return self.return_error(msgid, flags, error[0], error[1]) except Exception, msg: error = sys.exc_info()[:2] log("%s() raised exception: %s" % (name, msg), zLOG.ERROR, error) return self.return_error(msgid, flags, error[0], error[1]) if flags & ASYNC: if ret is not None: log("async method %s returned value %s" % (name, repr(ret)), zLOG.ERROR) raise ZRPCError("async method returned value") else: if __debug__: log("%s return %s" % (name, repr(ret)[:40]), zLOG.DEBUG) if isinstance(ret, Delay): ret.set_sender(msgid, self.send_reply) else: self.send_reply(msgid, ret) def handle_error(self): self.log_error() self.close() def log_error(self, msg="No error message supplied"): error = sys.exc_info() log(msg, zLOG.ERROR, error=error) del error def check_method(self, name): # XXX minimal security check should go here: Is name exported? return hasattr(self.obj, name) def send_reply(self, msgid, ret): msg = self.marshal.encode(msgid, 0, REPLY, ret) self.message_output(msg) def return_error(self, msgid, flags, err_type, err_value): if flags is None: self.log_error("Exception raised during decoding") return if flags & ASYNC: self.log_error("Asynchronous call raised exception: %s" % self) return if type(err_value) is not types.InstanceType: err_value = err_type, err_value try: msg = self.marshal.encode(msgid, 0, REPLY, (err_type, err_value)) except self.marshal.errors: err = ZRPCError("Couldn't pickle error %s" % `err_value`) msg = self.marshal.encode(msgid, 0, REPLY, (ZRPCError, err)) self.message_output(msg) self._do_io() # The next two methods are used by clients to invoke methods on # remote objects # XXX Should revise design to allow multiple outstanding # synchronous calls def call(self, method, *args): self.__call_lock.acquire() try: return self._call(method, args) finally: self.__call_lock.release() def _call(self, method, args): if self.closed: raise DisconnectedError("This action is temporarily unavailable") msgid = self.msgid self.msgid = self.msgid + 1 if __debug__: log("send msg: %d, 0, %s, ..." % (msgid, method)) self.message_output(self.marshal.encode(msgid, 0, method, args)) self.__reply = None # lock is currently held self._do_io(wait=1) # lock is held again... r_msgid, r_flags, r_args = self.__reply self.__reply_lock.acquire() assert r_msgid == msgid, "%s != %s: %s" % (r_msgid, msgid, r_args) if type(r_args) == types.TupleType \ and type(r_args[0]) == types.ClassType \ and issubclass(r_args[0], Exception): raise r_args[1] # error raised by server return r_args def callAsync(self, method, *args): self.__call_lock.acquire() try: self._callAsync(method, args) finally: self.__call_lock.release() def _callAsync(self, method, args): if self.closed: raise DisconnectedError("This action is temporarily unavailable") msgid = self.msgid self.msgid += 1 if __debug__: log("send msg: %d, %d, %s, ..." % (msgid, ASYNC, method)) self.message_output(self.marshal.encode(msgid, ASYNC, method, args)) self._do_io() # handle IO, possibly in async mode def _prepare_async(self): self.thr_async = 0 ThreadedAsync.register_loop_callback(self.set_async) # XXX If we are not in async mode, this will cause dead # Connections to be leaked. def set_async(self, map): # XXX do we need a lock around this? I'm not sure there is # any harm to a race with _do_io(). self.trigger = trigger() self.thr_async = 1 def is_async(self): if self.thr_async: return 1 else: return 0 def _do_io(self, wait=0): # XXX need better name # XXX invariant? lock must be held when calling with wait==1 # otherwise, in non-async mode, there will be no poll if __debug__: log("_do_io(wait=%d), async=%d" % (wait, self.is_async()), level=zLOG.DEBUG) if self.is_async(): self.trigger.pull_trigger() if wait: self.__reply_lock.acquire() # wait until reply... self.__reply_lock.release() else: if wait: # do loop only if lock is already acquired while not self.__reply_lock.acquire(0): asyncore.poll(10.0, self._map) if self.closed: raise DisconnectedError() self.__reply_lock.release() else: asyncore.poll(0.0, self._map) # XXX it seems that we need to release before returning if # called with wait==1. perhaps the caller need not acquire # upon return... class ServerConnection(Connection): # XXX this is a hack def _do_io(self, wait=0): """If this is a server, there is no explicit IO to do""" pass class ManagedServerConnection(ServerConnection): """A connection that notifies its ConnectionManager of closing""" __super_init = Connection.__init__ __super_close = Connection.close def __init__(self, sock, addr, obj, mgr): self.__mgr = mgr self.__super_init(sock, addr, obj) def close(self): self.__super_close() self.__mgr.close(self) class ManagedConnection(Connection): """A connection that notifies its ConnectionManager of closing. A managed connection also defers the ThreadedAsync work to its manager. """ __super_init = Connection.__init__ __super_close = Connection.close def __init__(self, sock, addr, obj, mgr): self.__mgr = mgr self.__super_init(sock, addr, obj) self.check_mgr_async() def close_trigger(self): # the manager should actually close the trigger del self.trigger def _prepare_async(self): # Don't do the register_loop_callback that the superclass does pass def check_mgr_async(self): if not self.thr_async and self.__mgr.thr_async: assert self.__mgr.trigger is not None, \ "manager (%s) has no trigger" % self.__mgr self.thr_async = 1 self.trigger = self.__mgr.trigger return 1 return 0 def is_async(self): if self.thr_async: return 1 return self.check_mgr_async() def close(self): self.__super_close() self.__mgr.notify_closed(self) === Added File StandaloneZODB/ZEO/zrpc/error.py === from ZODB import POSException from ZEO.Exceptions import Disconnected class ZRPCError(POSException.StorageError): pass class DecodingError(ZRPCError): """A ZRPC message could not be decoded.""" class DisconnectedError(ZRPCError, Disconnected): """The database storage is disconnected from the storage server.""" === Added File StandaloneZODB/ZEO/zrpc/log.py === import os import zLOG _label = "zrpc:%s" % os.getpid() def new_label(): global _label _label = "zrpc:%s" % os.getpid() def log(message, level=zLOG.BLATHER, label=None, error=None): zLOG.LOG(label or _label, level, message, error=error) === Added File StandaloneZODB/ZEO/zrpc/marshal.py === import cPickle from cStringIO import StringIO import types class Marshaller: """Marshal requests and replies to second across network""" # It's okay to share a single Pickler as long as it's in fast # mode, which means that it doesn't have a memo. pickler = cPickle.Pickler() pickler.fast = 1 pickle = pickler.dump errors = (cPickle.UnpickleableError, cPickle.UnpicklingError, cPickle.PickleError, cPickle.PicklingError) def encode(self, msgid, flags, name, args): """Returns an encoded message""" return self.pickle((msgid, flags, name, args), 1) def decode(self, msg): """Decodes msg and returns its parts""" unpickler = cPickle.Unpickler(StringIO(msg)) unpickler.find_global = find_global try: return unpickler.load() # msgid, flags, name, args except (cPickle.UnpicklingError, IndexError), err_msg: log("can't decode %s" % repr(msg), level=zLOG.ERROR) raise DecodingError(msg) _globals = globals() _silly = ('__doc__',) def find_global(module, name): """Helper for message unpickler""" try: m = __import__(module, _globals, _globals, _silly) except ImportError, msg: raise ZRPCError("import error %s: %s" % (module, msg)) try: r = getattr(m, name) except AttributeError: raise ZRPCError("module %s has no global %s" % (module, name)) safe = getattr(r, '__no_side_effects__', 0) if safe: return r # XXX what's a better way to do this? esp w/ 2.1 & 2.2 if type(r) == types.ClassType and issubclass(r, Exception): return r raise ZRPCError("Unsafe global: %s.%s" % (module, name)) === Added File StandaloneZODB/ZEO/zrpc/server.py === import asyncore import socket import types from ZEO.zrpc.connection import Connection, Delay from ZEO.zrpc.log import log # Export the main asyncore loop loop = asyncore.loop class Dispatcher(asyncore.dispatcher): """A server that accepts incoming RPC connections""" __super_init = asyncore.dispatcher.__init__ reuse_addr = 1 def __init__(self, addr, factory=Connection, reuse_addr=None): self.__super_init() self.addr = addr self.factory = factory self.clients = [] if reuse_addr is not None: self.reuse_addr = reuse_addr self._open_socket() def _open_socket(self): if type(self.addr) == types.TupleType: self.create_socket(socket.AF_INET, socket.SOCK_STREAM) else: self.create_socket(socket.AF_UNIX, socket.SOCK_STREAM) self.set_reuse_addr() log("listening on %s" % str(self.addr)) self.bind(self.addr) self.listen(5) def writable(self): return 0 def readable(self): return 1 def handle_accept(self): try: sock, addr = self.accept() except socket.error, msg: log("accepted failed: %s" % msg) return c = self.factory(sock, addr) log("connect from %s: %s" % (repr(addr), c)) self.clients.append(c) === Added File StandaloneZODB/ZEO/zrpc/trigger.py === ############################################################################## # # Zope Public License (ZPL) Version 1.0 # ------------------------------------- # # Copyright (c) Digital Creations. All rights reserved. # # This license has been certified as Open Source(tm). # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions in source code must retain the above copyright # notice, this list of conditions, and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions, and the following disclaimer in # the documentation and/or other materials provided with the # distribution. # # 3. Digital Creations requests that attribution be given to Zope # in any manner possible. Zope includes a "Powered by Zope" # button that is installed by default. While it is not a license # violation to remove this button, it is requested that the # attribution remain. A significant investment has been put # into Zope, and this effort will continue if the Zope community # continues to grow. This is one way to assure that growth. # # 4. All advertising materials and documentation mentioning # features derived from or use of this software must display # the following acknowledgement: # # "This product includes software developed by Digital Creations # for use in the Z Object Publishing Environment # (http://www.zope.org/)." # # In the event that the product being advertised includes an # intact Zope distribution (with copyright and license included) # then this clause is waived. # # 5. Names associated with Zope or Digital Creations must not be used to # endorse or promote products derived from this software without # prior written permission from Digital Creations. # # 6. Modified redistributions of any form whatsoever must retain # the following acknowledgment: # # "This product includes software developed by Digital Creations # for use in the Z Object Publishing Environment # (http://www.zope.org/)." # # Intact (re-)distributions of any official Zope release do not # require an external acknowledgement. # # 7. Modifications are encouraged but must be packaged separately as # patches to official Zope releases. Distributions that do not # clearly separate the patches from the original work must be clearly # labeled as unofficial distributions. Modifications which do not # carry the name Zope may be packaged in any form, as long as they # conform to all of the clauses above. # # # Disclaimer # # THIS SOFTWARE IS PROVIDED BY DIGITAL CREATIONS ``AS IS'' AND ANY # EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL DIGITAL CREATIONS OR ITS # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF # USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND # ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT # OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF # SUCH DAMAGE. # # # This software consists of contributions made by Digital Creations and # many individuals on behalf of Digital Creations. Specific # attributions are listed in the accompanying credits file. # ############################################################################## # This module is a simplified version of the select_trigger module # from Sam Rushing's Medusa server. import asyncore #import asynchat import os import socket import string import thread if os.name == 'posix': class trigger (asyncore.file_dispatcher): "Wake up a call to select() running in the main thread" # This is useful in a context where you are using Medusa's I/O # subsystem to deliver data, but the data is generated by another # thread. Normally, if Medusa is in the middle of a call to # select(), new output data generated by another thread will have # to sit until the call to select() either times out or returns. # If the trigger is 'pulled' by another thread, it should immediately # generate a READ event on the trigger object, which will force the # select() invocation to return. # A common use for this facility: letting Medusa manage I/O for a # large number of connections; but routing each request through a # thread chosen from a fixed-size thread pool. When a thread is # acquired, a transaction is performed, but output data is # accumulated into buffers that will be emptied more efficiently # by Medusa. [picture a server that can process database queries # rapidly, but doesn't want to tie up threads waiting to send data # to low-bandwidth connections] # The other major feature provided by this class is the ability to # move work back into the main thread: if you call pull_trigger() # with a thunk argument, when select() wakes up and receives the # event it will call your thunk from within that thread. The main # purpose of this is to remove the need to wrap thread locks around # Medusa's data structures, which normally do not need them. [To see # why this is true, imagine this scenario: A thread tries to push some # new data onto a channel's outgoing data queue at the same time that # the main thread is trying to remove some] def __init__ (self): r, w = os.pipe() self.trigger = w asyncore.file_dispatcher.__init__ (self, r) self.lock = thread.allocate_lock() self.thunks = [] def __repr__ (self): return '' % id(self) def readable (self): return 1 def writable (self): return 0 def handle_connect (self): pass def pull_trigger (self, thunk=None): # print 'PULL_TRIGGER: ', len(self.thunks) if thunk: try: self.lock.acquire() self.thunks.append (thunk) finally: self.lock.release() os.write (self.trigger, 'x') def handle_read (self): self.recv (8192) try: self.lock.acquire() for thunk in self.thunks: try: thunk() except: (file, fun, line), t, v, tbinfo = asyncore.compact_traceback() print 'exception in trigger thunk: (%s:%s %s)' % (t, v, tbinfo) self.thunks = [] finally: self.lock.release() else: # win32-safe version class trigger (asyncore.dispatcher): address = ('127.9.9.9', 19999) def __init__ (self): a = socket.socket (socket.AF_INET, socket.SOCK_STREAM) w = socket.socket (socket.AF_INET, socket.SOCK_STREAM) # set TCP_NODELAY to true to avoid buffering w.setsockopt(socket.IPPROTO_TCP, 1, 1) # tricky: get a pair of connected sockets host='127.0.0.1' port=19999 while 1: try: self.address=(host, port) a.bind(self.address) break except: if port <= 19950: raise 'Bind Error', 'Cannot bind trigger!' port=port - 1 a.listen (1) w.setblocking (0) try: w.connect (self.address) except: pass r, addr = a.accept() a.close() w.setblocking (1) self.trigger = w asyncore.dispatcher.__init__ (self, r) self.lock = thread.allocate_lock() self.thunks = [] self._trigger_connected = 0 def __repr__ (self): return '' % id(self) def readable (self): return 1 def writable (self): return 0 def handle_connect (self): pass def pull_trigger (self, thunk=None): if thunk: try: self.lock.acquire() self.thunks.append (thunk) finally: self.lock.release() self.trigger.send ('x') def handle_read (self): self.recv (8192) try: self.lock.acquire() for thunk in self.thunks: try: thunk() except: (file, fun, line), t, v, tbinfo = asyncore.compact_traceback() print 'exception in trigger thunk: (%s:%s %s)' % (t, v, tbinfo) self.thunks = [] finally: self.lock.release() From jeremy at zope.com Wed Jan 16 09:58:39 2002 From: jeremy at zope.com (Jeremy Hylton) Date: Sun Aug 10 16:31:18 2008 Subject: [ZEO-Checkins] CVS: StandaloneZODB/ZEO - ClientStorage.py:1.35.6.3 StorageServer.py:1.32.6.2 zrpc2.py:NONE Message-ID: <200201161458.g0GEwdX23853@cvs.baymountain.com> Update of /cvs-repository/StandaloneZODB/ZEO In directory cvs.zope.org:/tmp/cvs-serv23840 Modified Files: Tag: Standby-branch ClientStorage.py StorageServer.py Removed Files: Tag: Standby-branch zrpc2.py Log Message: Various cleanups and simplifications. Convert from zrpc2 module to zrpc package. This includes some changes to client code: - Dispatcher factory (StorageServer.newConnection) doesn't have an unused third argument anymore. Remove unused imports. Remove fixup_storage(). I think this backwards compatibility feature is not needed. Happy to add it back if someone misses it. Fix possible leaks of traceback (sys.exc_info() stored in local). Cleanup comments in ClientStorage and removed unreachable code. Replace uses of thread module with threading module. === StandaloneZODB/ZEO/ClientStorage.py 1.35.6.2 => 1.35.6.3 === ############################################################################## """Network ZODB storage client - -XXX support multiple outstanding requests up until the vote? -XXX is_connected() vis ClientDisconnected error """ __version__='$Revision$'[11:-2] import cPickle import os -import socket -import string -import struct -import sys import tempfile -import thread import threading import time -from types import TupleType, StringType -from struct import pack, unpack -import ExtensionClass, Sync, ThreadLock -import ClientCache -import zrpc2 -import ServerStub -from TransactionBuffer import TransactionBuffer +from ZEO import ClientCache, ServerStub +from ZEO.TransactionBuffer import TransactionBuffer +from ZEO.Exceptions import Disconnected +from ZEO.zrpc.client import ConnectionManager from ZODB import POSException from ZODB.TimeStamp import TimeStamp from zLOG import LOG, PROBLEM, INFO, BLATHER -from Exceptions import Disconnected def log2(type, msg, subsys="ClientStorage %d" % os.getpid()): LOG(subsys, type, msg) @@ -130,10 +118,11 @@ class ClientDisconnected(ClientStorageError, Disconnected): """The database storage is disconnected from the storage.""" -def get_timestamp(prev_ts): +def get_timestamp(prev_ts=None): t = time.time() t = apply(TimeStamp, (time.gmtime(t)[:5] + (t % 60,))) - t = t.laterThan(prev_ts) + if prev_ts is not None: + t = t.laterThan(prev_ts) return t class DisconnectedServerStub: @@ -161,7 +150,9 @@ self._tbuf = TransactionBuffer() self._db = None self._oids = [] - # XXX It's confusing to have _serial, _serials, and _seriald. + # _serials: stores (oid, serialno) as returned by server + # _seriald: _check_serials() moves from _serials to _seriald, + # which maps oid to serialno self._serials = [] self._seriald = {} @@ -171,17 +162,15 @@ client = client or os.environ.get('ZEO_CLIENT', '') self._cache = ClientCache.ClientCache(storage, cache_size, client=client, var=var) - self._cache.open() # XXX + self._cache.open() # XXX open now? - self._rpc_mgr = zrpc2.ConnectionManager(addr, self, - #debug=debug, - tmin=min_disconnect_poll, - tmax=max_disconnect_poll) + self._rpc_mgr = ConnectionManager(addr, self, + tmin=min_disconnect_poll, + tmax=max_disconnect_poll) # XXX What if we can only get a read-only connection and we # want a read-write connection? Looks like the current code # will block forever. - if wait_for_server_on_startup: self._rpc_mgr.connect(sync=1) else: @@ -191,6 +180,7 @@ def _basic_init(self, name): """Handle initialization activites of BaseStorage""" + # XXX does anything depend on attr being __name__ self.__name__ = name # A ClientStorage only allows one client to commit at a time. @@ -205,15 +195,19 @@ # oid_cond. self.oid_cond = threading.Condition() - commit_lock = thread.allocate_lock() + commit_lock = threading.Lock() self._commit_lock_acquire = commit_lock.acquire self._commit_lock_release = commit_lock.release - t = time.time() - t = self._ts = apply(TimeStamp,(time.gmtime(t)[:5]+(t%60,))) + t = self._ts = get_timestamp() self._serial = `t` self._oid='\0\0\0\0\0\0\0\0' + def close(self): + self._rpc_mgr.close() + if self._cache is not None: + self._cache.close() + def registerDB(self, db, limit): """Register that the storage is controlled by the given DB.""" log2(INFO, "registerDB(%s, %s)" % (repr(db), repr(limit))) @@ -253,10 +247,6 @@ ### notifyDisconnected had to get the instance lock. There's ### nothing to gain by getting the instance lock. - ### Note that we *don't* have to worry about getting connected - ### in the middle of notifyDisconnected, because *it's* - ### responsible for starting the thread that makes the connection. - def notifyDisconnected(self): log2(PROBLEM, "Disconnected from storage") self._server = disconnected_stub @@ -298,14 +288,6 @@ return 1 def _check_tid(self, tid, exc=None): - # XXX Is all this locking unnecessary? The only way to - # begin a transaction is to call tpc_begin(). If we assume - # clients are single-threaded and well-behaved, i.e. they call - # tpc_begin() first, then there appears to be no need for - # locking. If _check_tid() is called and self.tpc_tid != tid, - # then there is no way it can be come equal during the call. - # Thus, there should be no race. - if self.tpc_tid != tid: if exc is None: return 0 @@ -313,19 +295,6 @@ raise exc(self.tpc_tid, tid) return 1 - # XXX But I'm not sure - - self.tpc_cond.acquire() - try: - if self.tpc_tid != tid: - if exc is None: - return 0 - else: - raise exc(self.tpc_tid, tid) - return 1 - finally: - self.tpc_cond.release() - def abortVersion(self, src, transaction): if self._is_read_only: raise POSException.ReadOnlyError() @@ -388,8 +357,7 @@ def new_oid(self, last=None): if self._is_read_only: raise POSException.ReadOnlyError() - # We want to avoid a situation where multiple oid requests are - # made at the same time. + # avoid multiple oid requests to server at the same time self.oid_cond.acquire() if not self._oids: self._oids = self._server.new_oids() @@ -402,8 +370,7 @@ def pack(self, t=None, rf=None, wait=0, days=0): if self._is_read_only: raise POSException.ReadOnlyError() - # Note that we ignore the rf argument. The server - # will provide it's own implementation. + # rf argument ignored; server will provide it's own implementation if t is None: t = time.time() t = t - (days * 86400) @@ -467,9 +434,8 @@ transaction.description, transaction._extension) except: - # If _server is None, then the client disconnected during - # the tpc_begin() and notifyDisconnected() will have - # released the lock. + # Client may have disconnected during the tpc_begin(). + # Then notifyDisconnected() will have released the lock. if self._server is not disconnected_stub: self.tpc_cond.release() raise @@ -481,7 +447,7 @@ def tpc_finish(self, transaction, f=None): if transaction is not self._transaction: return - if f is not None: # XXX what is f()? + if f is not None: f() self._server.tpc_finish(self._serial) @@ -575,7 +541,6 @@ if self._pickler is None: return self._pickler.dump((0,0)) -## self._pickler.dump = None self._tfile.seek(0) unpick = cPickle.Unpickler(self._tfile) self._tfile = None @@ -588,7 +553,6 @@ self._db.invalidate(oid, version=version) def Invalidate(self, args): - # XXX _db could be None for oid, version in args: self._cache.invalidate(oid, version=version) try: @@ -598,4 +562,3 @@ "Invalidate(%s, %s) failed for _db: %s" % (repr(oid), repr(version), msg)) - === StandaloneZODB/ZEO/StorageServer.py 1.32.6.1 => 1.32.6.2 === import sys import threading -import types -import ClientStub -import zrpc2 -import zLOG +from ZEO import ClientStub +from ZEO.zrpc.server import Dispatcher +from ZEO.zrpc.connection import ManagedServerConnection, Handler, Delay -from zrpc2 import Dispatcher, Handler, ManagedServerConnection, Delay +import zLOG from ZODB.POSException import StorageError, StorageTransactionError, \ TransactionError, ReadOnlyError from ZODB.referencesf import referencesf @@ -129,14 +128,14 @@ self.storages = storages self.read_only = read_only self.connections = {} - for name, store in storages.items(): - fixup_storage(store) self.dispatcher = Dispatcher(addr, factory=self.newConnection, reuse_addr=1) - def newConnection(self, sock, addr, nil): + def newConnection(self, sock, addr): + # XXX figure out how to do the connection / proxy dance better c = ManagedServerConnection(sock, addr, None, self) c.register_object(StorageProxy(self, c)) + log("new connection %s: %s" % (addr, `c`)) return c def register(self, storage_id, proxy): @@ -187,6 +186,7 @@ self.server = server self.client = ClientStub.ClientStorage(conn) self.__storage = None + self.__storage_id = "uninitialized" self.__invalidated = [] self._transaction = None @@ -201,7 +201,7 @@ stid) def _log(self, msg, level=zLOG.INFO, error=None, pid=os.getpid()): - zLOG.LOG("ZEO Server %s %X" % (pid, id(self)), + zLOG.LOG("ZEO Server:%s:%s" % (pid, self.__storage_id), level, msg, error=error) def setup_delegation(self): @@ -313,6 +313,7 @@ if wait: raise else: + # XXX Why doesn't we broadcast on wait? if not wait: # Broadcast new size statistics self.server.invalidate(0, self.__storage_id, (), @@ -338,7 +339,6 @@ self._check_tid(id, exc=StorageTransactionError) try: # XXX does this stmt need to be in the try/except? - newserial = self.__storage.store(oid, serial, data, version, self._transaction) except TransactionError, v: @@ -354,11 +354,13 @@ error = sys.exc_info() self._log('store error: %s: %s' % (error[0], error[1]), zLOG.ERROR, error=error) - newserial = sys.exc_info()[1] + newserial = error[1] + del error else: if serial != '\0\0\0\0\0\0\0\0': self.__invalidated.append((oid, version)) + # Is all this error checking necessary? try: nil = dump(newserial, 1) except: @@ -415,7 +417,7 @@ t._extension = ext if self.__storage._transaction is not None: - d = zrpc2.Delay() + d = Delay() self.__storage.__waiting.append((d, self, t)) return d @@ -468,8 +470,3 @@ if n < 0: n = 1 return [self.__storage.new_oid() for i in range(n)] - -def fixup_storage(storage): - # backwards compatibility hack - if not hasattr(storage,'tpc_vote'): - storage.tpc_vote = lambda *args: None === Removed File StandaloneZODB/ZEO/zrpc2.py === From jeremy at zope.com Wed Jan 16 10:00:34 2002 From: jeremy at zope.com (Jeremy Hylton) Date: Sun Aug 10 16:31:18 2008 Subject: [ZEO-Checkins] CVS: StandaloneZODB/ZEO - smac.py:1.11.4.3 Message-ID: <200201161500.g0GF0YT24456@cvs.baymountain.com> Update of /cvs-repository/StandaloneZODB/ZEO In directory cvs.zope.org:/tmp/cvs-serv24445 Modified Files: Tag: Standby-branch smac.py Log Message: Possible hack! Call asyncore.dispatcher.__init__ at end of constructor. I saw a traceback in a thread about a smac object that didn't have an __output variable. My guess is that the superclass constructor gets the object registered with a mainloop running in another thread. Then the mainloop invokes methods on the object before the constructor completes. === StandaloneZODB/ZEO/smac.py 1.11.4.2 => 1.11.4.3 === def __init__(self, sock, addr, map=None, debug=None): - self.__super_init(sock, map) self.addr = addr if debug is not None: self._debug = debug @@ -135,6 +134,7 @@ self.__msg_size = 4 self.__output = [] self.__closed = None + self.__super_init(sock, map) # XXX avoid expensive getattr calls? def __nonzero__(self): From jeremy at zope.com Wed Jan 16 10:04:14 2002 From: jeremy at zope.com (Jeremy Hylton) Date: Sun Aug 10 16:31:18 2008 Subject: [ZEO-Checkins] CVS: StandaloneZODB/ZEO/tests - testZEO.py:1.16.4.4 Message-ID: <200201161504.g0GF4El25157@cvs.baymountain.com> Update of /cvs-repository/StandaloneZODB/ZEO/tests In directory cvs.zope.org:/tmp/cvs-serv25146/tests Modified Files: Tag: Standby-branch testZEO.py Log Message: Add a zLOG call during the setUp() for each test. === StandaloneZODB/ZEO/tests/testZEO.py 1.16.4.3 => 1.16.4.4 === from ZODB.FileStorage import FileStorage import thread +import zLOG from ZEO.tests import forker, Cache from ZEO.smac import Disconnected @@ -152,6 +153,7 @@ def setUp(self): self.__super_setUp() + zLOG.LOG("testZEO", zLOG.BLATHER, "setUp() new test") self.running = 1 client, exit, pid = forker.start_zeo(self.getStorage()) self._pids = [pid] @@ -230,7 +232,6 @@ s.connect(self.test_addr) s.close() # the connection should cause the storage server to die -## os.waitpid(self.test_pid, 0) time.sleep(0.5) self.delStorage() self.__super_tearDown() From jeremy at zope.com Wed Jan 16 10:05:27 2002 From: jeremy at zope.com (Jeremy Hylton) Date: Sun Aug 10 16:31:18 2008 Subject: [ZEO-Checkins] CVS: StandaloneZODB/ZEO/tests - forker.py:1.10.4.4 Message-ID: <200201161505.g0GF5R925676@cvs.baymountain.com> Update of /cvs-repository/StandaloneZODB/ZEO/tests In directory cvs.zope.org:/tmp/cvs-serv25665/tests Modified Files: Tag: Standby-branch forker.py Log Message: Attempt to import ZEO stuff after the fork. There's crufty logging code that includes os.getpid() in the subsystem name. It's very helpful when it's correct, but it's even more unhelpful when it's wrong. XXX Need to fix the logging code so that the pid is looked up in a more useful. Perhaps just on every call. === StandaloneZODB/ZEO/tests/forker.py 1.10.4.3 => 1.10.4.4 === import traceback import types -import ZEO.ClientStorage, ZEO.StorageServer +import ZEO.ClientStorage # Change value of PROFILE to enable server-side profiling PROFILE = 0 @@ -101,6 +101,8 @@ rd, wr = os.pipe() pid = os.fork() if pid == 0: + import ZEO.zrpc.log + reload(ZEO.zrpc.log) try: if PROFILE: p = hotshot.Profile("stats.s.%d" % os.getpid()) @@ -122,8 +124,9 @@ global server os.close(wr) ZEOServerExit(rd) + import ZEO.StorageServer, ZEO.zrpc.server server = ZEO.StorageServer.StorageServer(addr, {'1':storage}) - asyncore.loop() + ZEO.zrpc.server.loop() storage.close() if isinstance(addr, types.StringType): os.unlink(addr) From jeremy at zope.com Wed Jan 16 20:34:48 2002 From: jeremy at zope.com (Jeremy Hylton) Date: Sun Aug 10 16:31:18 2008 Subject: [ZEO-Checkins] CVS: StandaloneZODB/ZEO/zrpc - log.py:1.1.2.2 Message-ID: <200201170134.g0H1YmY12023@cvs.baymountain.com> Update of /cvs-repository/StandaloneZODB/ZEO/zrpc In directory cvs.zope.org:/tmp/cvs-serv12012/zrpc Modified Files: Tag: Standby-branch log.py Log Message: Add a complicated short_repr() function. This makes the same sort of change that was made late in the beta release for ZEO 1.0. We were seeing memory problems exacerbated by the cost of building reprs of large strings and tuples that were going to be truncated anyway. === StandaloneZODB/ZEO/zrpc/log.py 1.1.2.1 => 1.1.2.2 === +import types import zLOG _label = "zrpc:%s" % os.getpid() @@ -9,3 +10,25 @@ def log(message, level=zLOG.BLATHER, label=None, error=None): zLOG.LOG(label or _label, level, message, error=error) + +REPR_LIMIT = 40 + +def short_repr(obj): + "Return an object repr limited to REPR_LIMIT bytes." + # Some of the objects being repr'd are large strings. It's wastes + # a lot of memory to repr them and then truncate, so special case + # them in this function. + # Also handle short repr of a tuple containing a long string. + if isinstance(obj, types.StringType): + obj = obj[:REPR_LIMIT] + elif isinstance(obj, types.TupleType): + elts = [] + size = 0 + for elt in obj: + r = repr(elt) + elts.append(r) + size += len(r) + if size > REPR_LIMIT: + break + obj = tuple(elts) + return repr(obj)[:REPR_LIMIT] From jeremy at zope.com Wed Jan 16 20:35:32 2002 From: jeremy at zope.com (Jeremy Hylton) Date: Sun Aug 10 16:31:18 2008 Subject: [ZEO-Checkins] CVS: StandaloneZODB/ZEO/zrpc - client.py:1.1.2.2 Message-ID: <200201170135.g0H1ZWN12555@cvs.baymountain.com> Update of /cvs-repository/StandaloneZODB/ZEO/zrpc In directory cvs.zope.org:/tmp/cvs-serv12544/zrpc Modified Files: Tag: Standby-branch client.py Log Message: Get rid of unused method. === StandaloneZODB/ZEO/zrpc/client.py 1.1.2.1 => 1.1.2.2 === self.connection.close() - # XXX get rid of this? - def register_client(self, client): - self.client = client - def set_async(self, map): # XXX need each connection started with async==0 to have a callback self.trigger = trigger() From jeremy at zope.com Wed Jan 16 20:40:41 2002 From: jeremy at zope.com (Jeremy Hylton) Date: Sun Aug 10 16:31:18 2008 Subject: [ZEO-Checkins] CVS: StandaloneZODB/ZEO/zrpc - connection.py:1.1.2.2 Message-ID: <200201170140.g0H1ef013785@cvs.baymountain.com> Update of /cvs-repository/StandaloneZODB/ZEO/zrpc In directory cvs.zope.org:/tmp/cvs-serv13774/zrpc Modified Files: Tag: Standby-branch connection.py Log Message: Many small cleanups and improvements. Get rid of Handler and set_caller() mechanism. It was unused. Replace _do_io() with two specialized methods, one of which is only called by _call(). This should have been more obvious because the behavior of _do_io() depended entirely on whether the wait kwarg with 0 or 1. Use short_repr() in conjunction with low-level log() calls. Extend doc string on Connection. Simplify error/logging for return val from async call. Add _ check to check_method() as suggested by Jim. Extend ManagedServerConnection protocol to use notifyConnected(). This simplifies the newConnection() dance in StorageServer. === StandaloneZODB/ZEO/zrpc/connection.py 1.1.2.1 => 1.1.2.2 === from ZEO import smac # XXX put smac in zrpc? from ZEO.zrpc.error import ZRPCError, DisconnectedError, DecodingError -from ZEO.zrpc.log import log +from ZEO.zrpc.log import log, short_repr from ZEO.zrpc.marshal import Marshaller from ZEO.zrpc.trigger import trigger import zLOG @@ -15,19 +15,6 @@ REPLY = ".reply" # message name used for replies ASYNC = 1 -# XXX get rid of this class and use hasattr() -class Handler: - """Base class used to handle RPC caller discovery""" - - def set_caller(self, addr): - self.__caller = addr - - def get_caller(self): - return self.__caller - - def clear_caller(self): - self.__caller = None - class Delay: """Used to delay response to client for synchronous calls @@ -44,7 +31,7 @@ self.send_reply(self.msgid, obj) class Connection(smac.SizedMessageAsyncConnection): - """Dispatcher for RPC on object + """Dispatcher for RPC on object on both sides of socket. The connection supports synchronous calls, which expect a return, and asynchronous calls that do not. @@ -55,7 +42,11 @@ A Connection is designed for use in a multithreaded application, where a synchronous call must block until a response is ready. The current design only allows a single synchronous call to be - outstanding. + outstanding. + + A socket connection between a client and a server allows either + side to invoke methods on the other side. The processes on each + end of the socket use a Connection object to manage communication. """ __super_init = smac.SizedMessageAsyncConnection.__init__ @@ -63,7 +54,7 @@ __super_writable = smac.SizedMessageAsyncConnection.writable def __init__(self, sock, addr, obj=None): - self.obj = obj + self.obj = None self.marshal = Marshaller() self.closed = 0 self.msgid = 0 @@ -83,19 +74,12 @@ # waiting for a response self.__reply_lock = threading.Lock() self.__reply_lock.acquire() - # If the object implements the Handler interface (XXX checked - # by isinstance), it wants to know who the caller is. - if isinstance(obj, Handler): - self.set_caller = 1 - else: - self.set_caller = 0 + self.register_object(obj) def __repr__(self): return "<%s %s>" % (self.__class__.__name__, self.addr) def close(self): - caller = sys._getframe(1).f_code.co_name - log("close() caller=%s" % caller) if self.closed: return self.closed = 1 @@ -103,6 +87,7 @@ self.__super_close() def close_trigger(self): + # overridden by ManagedConnection if self.trigger is not None: self.trigger.close() @@ -122,7 +107,7 @@ if __debug__: log("recv msg: %s, %s, %s, %s" % (msgid, flags, name, - repr(args)[:40]), + short_repr(args)), level=zLOG.DEBUG) if name == REPLY: self.handle_reply(msgid, flags, args) @@ -137,28 +122,13 @@ self.__reply_lock.release() # will fail if lock is unlocked def handle_request(self, msgid, flags, name, args): - if __debug__: - log("call %s%s on %s" % (name, repr(args)[:40], repr(self.obj)), - zLOG.DEBUG) if not self.check_method(name): raise ZRPCError("Invalid method name: %s on %s" % (name, `self.obj`)) meth = getattr(self.obj, name) try: - if self.set_caller: - self.obj.set_caller(self) - try: - ret = meth(*args) - finally: - self.obj.clear_caller() - else: - ret = meth(*args) - except (POSException.UndoError, - POSException.VersionCommitError), msg: - error = sys.exc_info()[:2] - log("%s() raised exception: %s" % (name, msg), zLOG.ERROR, error) - return self.return_error(msgid, flags, error[0], error[1]) + ret = meth(*args) except Exception, msg: error = sys.exc_info()[:2] log("%s() raised exception: %s" % (name, msg), zLOG.ERROR, error) @@ -166,12 +136,11 @@ if flags & ASYNC: if ret is not None: - log("async method %s returned value %s" % (name, repr(ret)), - zLOG.ERROR) - raise ZRPCError("async method returned value") + raise ZRPCError("async method %s returned value %s" % + (name, repr(ret)) else: if __debug__: - log("%s return %s" % (name, repr(ret)[:40]), zLOG.DEBUG) + log("%s return %s" % (name, short_repr(ret)), zLOG.DEBUG) if isinstance(ret, Delay): ret.set_sender(msgid, self.send_reply) else: @@ -182,12 +151,12 @@ self.close() def log_error(self, msg="No error message supplied"): - error = sys.exc_info() - log(msg, zLOG.ERROR, error=error) - del error + log(msg, zLOG.ERROR, error=sys.exc_info()) def check_method(self, name): - # XXX minimal security check should go here: Is name exported? + # XXX Is this sufficient "security" for now? + if name.startswith('_'): + return None return hasattr(self.obj, name) def send_reply(self, msgid, ret): @@ -210,13 +179,10 @@ err = ZRPCError("Couldn't pickle error %s" % `err_value`) msg = self.marshal.encode(msgid, 0, REPLY, (ZRPCError, err)) self.message_output(msg) - self._do_io() + self._do_async_poll() - # The next two methods are used by clients to invoke methods on - # remote objects - - # XXX Should revise design to allow multiple outstanding - # synchronous calls + # The next two public methods (call and callAsync) are used by + # clients to invoke methods on remote objects def call(self, method, *args): self.__call_lock.acquire() @@ -234,10 +200,12 @@ log("send msg: %d, 0, %s, ..." % (msgid, method)) self.message_output(self.marshal.encode(msgid, 0, method, args)) + # XXX implementation of promises starts here + self.__reply = None - # lock is currently held - self._do_io(wait=1) - # lock is held again... + # reply lock is currently held + self._do_async_loop() + # reply lock is held again... r_msgid, r_flags, r_args = self.__reply self.__reply_lock.acquire() assert r_msgid == msgid, "%s != %s: %s" % (r_msgid, msgid, r_args) @@ -263,7 +231,10 @@ if __debug__: log("send msg: %d, %d, %s, ..." % (msgid, ASYNC, method)) self.message_output(self.marshal.encode(msgid, ASYNC, method, args)) - self._do_io() + # XXX The message won't go out right away in this case. It + # will wait for the asyncore loop to get control again. Seems + # okay to comment our for now, but need to understand better. +## self._do_async_poll() # handle IO, possibly in async mode @@ -274,8 +245,6 @@ # Connections to be leaked. def set_async(self, map): - # XXX do we need a lock around this? I'm not sure there is - # any harm to a race with _do_io(). self.trigger = trigger() self.thr_async = 1 @@ -284,41 +253,45 @@ return 1 else: return 0 - - def _do_io(self, wait=0): # XXX need better name - # XXX invariant? lock must be held when calling with wait==1 - # otherwise, in non-async mode, there will be no poll + def _do_async_loop(self): + "Invoke asyncore mainloop and wait for reply." if __debug__: - log("_do_io(wait=%d), async=%d" % (wait, self.is_async()), + log("_do_async_loop() async=%d" % self.is_async(), level=zLOG.DEBUG) if self.is_async(): self.trigger.pull_trigger() - if wait: - self.__reply_lock.acquire() - # wait until reply... - self.__reply_lock.release() + self.__reply_lock.acquire() + # wait until reply... else: - if wait: - # do loop only if lock is already acquired - while not self.__reply_lock.acquire(0): - asyncore.poll(10.0, self._map) - if self.closed: - raise DisconnectedError() - self.__reply_lock.release() - else: - asyncore.poll(0.0, self._map) + # Do loop only if lock is already acquired. XXX But can't + # we already guarantee that the lock is already acquired? + while not self.__reply_lock.acquire(0): + asyncore.poll(10.0, self._map) + if self.closed: + raise DisconnectedError() + self.__reply_lock.release() + + def _do_async_poll(self, wait_for_reply=0): + "Invoke asyncore mainloop to get pending message out." - # XXX it seems that we need to release before returning if - # called with wait==1. perhaps the caller need not acquire - # upon return... + if __debug__: + log("_do_async_poll(), async=%d" % self.is_async(), + level=zLOG.DEBUG) + if self.is_async(): + self.trigger.pull_trigger() + else: + asyncore.poll(0.0, self._map) class ServerConnection(Connection): - # XXX this is a hack - def _do_io(self, wait=0): + + def _do_async_poll(self, wait=0): """If this is a server, there is no explicit IO to do""" pass + # XXX _do_async_loop is never called. Should it be defined as + # above anyway? + class ManagedServerConnection(ServerConnection): """A connection that notifies its ConnectionManager of closing""" __super_init = Connection.__init__ @@ -327,6 +300,7 @@ def __init__(self, sock, addr, obj, mgr): self.__mgr = mgr self.__super_init(sock, addr, obj) + obj.notifyConnected(self) def close(self): self.__super_close() From jeremy at zope.com Wed Jan 16 20:41:00 2002 From: jeremy at zope.com (Jeremy Hylton) Date: Sun Aug 10 16:31:18 2008 Subject: [ZEO-Checkins] CVS: StandaloneZODB/ZEO - smac.py:1.11.4.4 Message-ID: <200201170141.g0H1f0814156@cvs.baymountain.com> Update of /cvs-repository/StandaloneZODB/ZEO In directory cvs.zope.org:/tmp/cvs-serv14145 Modified Files: Tag: Standby-branch smac.py Log Message: Improve comment? === StandaloneZODB/ZEO/smac.py 1.11.4.3 => 1.11.4.4 === self.__super_init(sock, map) - # XXX avoid expensive getattr calls? + # XXX avoid expensive getattr calls? Can't remember exactly what + # this comment was supposed to mean, but it has something to do + # with the way asyncore uses getattr and uses if sock: def __nonzero__(self): return 1 From jeremy at zope.com Wed Jan 16 20:41:23 2002 From: jeremy at zope.com (Jeremy Hylton) Date: Sun Aug 10 16:31:18 2008 Subject: [ZEO-Checkins] CVS: StandaloneZODB/ZEO - TransactionBuffer.py:1.3.2.2 Message-ID: <200201170141.g0H1fNZ14289@cvs.baymountain.com> Update of /cvs-repository/StandaloneZODB/ZEO In directory cvs.zope.org:/tmp/cvs-serv14278 Modified Files: Tag: Standby-branch TransactionBuffer.py Log Message: Change some XXX comments to clearer non-XXX comments. === StandaloneZODB/ZEO/TransactionBuffer.py 1.3.2.1 => 1.3.2.2 === """ -# XXX Figure out what a sensible storage format is - -# XXX A faster implementation might store trans data in memory until -# it reaches a certain size. +# A faster implementation might store trans data in memory until it +# reaches a certain size. import tempfile import cPickle @@ -43,7 +41,7 @@ self.count = 0 self.size = 0 - # XXX unchecked constraints: + # unchecked constraints: # 1. can't call store() after begin_iterate() # 2. must call clear() after iteration finishes From jeremy at zope.com Wed Jan 16 20:42:58 2002 From: jeremy at zope.com (Jeremy Hylton) Date: Sun Aug 10 16:31:18 2008 Subject: [ZEO-Checkins] CVS: StandaloneZODB/ZEO - StorageServer.py:1.32.6.3 Message-ID: <200201170142.g0H1gwd14448@cvs.baymountain.com> Update of /cvs-repository/StandaloneZODB/ZEO In directory cvs.zope.org:/tmp/cvs-serv14437 Modified Files: Tag: Standby-branch StorageServer.py Log Message: Various cleanups. Rename StorageProxy to ZEOStorage. The latter seems more consistent with usage like FileStorage. It's ZEOStorage because it implements the storage API exported by the storage server. Track removal of Handler class in zrpc.connection. Simplify newConnection() and extend StorageServer to do new notifyConnected() protocol. Fiddle some comments. === StandaloneZODB/ZEO/StorageServer.py 1.32.6.2 => 1.32.6.3 === from ZEO import ClientStub from ZEO.zrpc.server import Dispatcher -from ZEO.zrpc.connection import ManagedServerConnection, Handler, Delay +from ZEO.zrpc.connection import ManagedServerConnection, Delay import zLOG from ZODB.POSException import StorageError, StorageTransactionError, \ @@ -132,9 +132,7 @@ reuse_addr=1) def newConnection(self, sock, addr): - # XXX figure out how to do the connection / proxy dance better - c = ManagedServerConnection(sock, addr, None, self) - c.register_object(StorageProxy(self, c)) + c = ManagedServerConnection(sock, addr, ZEOStorage(self), self) log("new connection %s: %s" % (addr, `c`)) return c @@ -147,7 +145,8 @@ if l is None: l = self.connections[storage_id] = [] # intialize waiting list - self.storages[storage_id]._StorageProxy__waiting = [] + # XXX why are we using a mangled name ?!? + self.storages[storage_id]._ZEOStorage__waiting = [] l.append(proxy) def invalidate(self, conn, storage_id, invalidated=(), info=0): @@ -172,24 +171,24 @@ pass def close(self, conn): - # XXX who calls this? - # called when conn is closed - # way too inefficient removed = 0 for sid, cl in self.connections.items(): if conn.obj in cl: cl.remove(conn.obj) removed = 1 -class StorageProxy(Handler): - def __init__(self, server, conn): +class ZEOStorage: + def __init__(self, server): self.server = server - self.client = ClientStub.ClientStorage(conn) + self.client = None self.__storage = None self.__storage_id = "uninitialized" self.__invalidated = [] self._transaction = None + def notifyConnected(self, conn): + self.client = ClientStub.ClientStorage(conn) + def __repr__(self): tid = self._transaction and repr(self._transaction.id) if self.__storage: @@ -337,8 +336,9 @@ def storea(self, oid, serial, data, version, id): self._check_tid(id, exc=StorageTransactionError) + # XXX The try/except seems to be doing a lot of work. How + # worried are we about errors that can't be pickled. try: - # XXX does this stmt need to be in the try/except? newserial = self.__storage.store(oid, serial, data, version, self._transaction) except TransactionError, v: From jeremy at zope.com Wed Jan 16 20:43:17 2002 From: jeremy at zope.com (Jeremy Hylton) Date: Sun Aug 10 16:31:18 2008 Subject: [ZEO-Checkins] CVS: StandaloneZODB/ZEO - ClientStub.py:1.3.2.2 Message-ID: <200201170143.g0H1hHs14523@cvs.baymountain.com> Update of /cvs-repository/StandaloneZODB/ZEO In directory cvs.zope.org:/tmp/cvs-serv14512 Modified Files: Tag: Standby-branch ClientStub.py Log Message: Add XXX about need to rename invalidate() and/or Invalidate(). === StandaloneZODB/ZEO/ClientStub.py 1.3.2.1 => 1.3.2.2 === self.rpc.callAsync('begin') - # XXX what's the difference between these two? + # XXX must rename the two invalidate messages. I can never + # remember which is which def invalidate(self, args): self.rpc.callAsync('invalidate', args) From jeremy at zope.com Wed Jan 16 20:43:32 2002 From: jeremy at zope.com (Jeremy Hylton) Date: Sun Aug 10 16:31:18 2008 Subject: [ZEO-Checkins] CVS: StandaloneZODB/ZEO - ClientStorage.py:1.35.6.4 Message-ID: <200201170143.g0H1hWP14557@cvs.baymountain.com> Update of /cvs-repository/StandaloneZODB/ZEO In directory cvs.zope.org:/tmp/cvs-serv14546 Modified Files: Tag: Standby-branch ClientStorage.py Log Message: Fiddle XXX comments. === StandaloneZODB/ZEO/ClientStorage.py 1.35.6.3 => 1.35.6.4 === self._cache = ClientCache.ClientCache(storage, cache_size, client=client, var=var) - self._cache.open() # XXX open now? + self._cache.open() # XXX open now? or later? self._rpc_mgr = ConnectionManager(addr, self, tmin=min_disconnect_poll, @@ -170,7 +170,7 @@ # XXX What if we can only get a read-only connection and we # want a read-write connection? Looks like the current code - # will block forever. + # will block forever. (Future feature) if wait_for_server_on_startup: self._rpc_mgr.connect(sync=1) else: @@ -509,7 +509,7 @@ def undoLog(self, first, last, filter=None): if filter is not None: - return () # XXX can't pass a filter to server + return () # can't pass a filter to server return self._server.undoLog(first, last) # Eek! @@ -533,6 +533,7 @@ self._pickler.fast = 1 # Don't use the memo def invalidate(self, args): + # Queue an invalidate for the end the transaction if self._pickler is None: return self._pickler.dump(args) From jeremy at zope.com Wed Jan 16 20:43:49 2002 From: jeremy at zope.com (Jeremy Hylton) Date: Sun Aug 10 16:31:18 2008 Subject: [ZEO-Checkins] CVS: StandaloneZODB/ZEO - ClientCache.py:1.18.6.3 Message-ID: <200201170143.g0H1hn414582@cvs.baymountain.com> Update of /cvs-repository/StandaloneZODB/ZEO In directory cvs.zope.org:/tmp/cvs-serv14571 Modified Files: Tag: Standby-branch ClientCache.py Log Message: Reindent comment. === StandaloneZODB/ZEO/ClientCache.py 1.18.6.2 => 1.18.6.3 === def open(self): - # XXX open is overloaded to perform two tasks for - # optimization reasons + # XXX open is overloaded to perform two tasks for + # optimization reasons self._acquire() try: self._index=index={} From jeremy at zope.com Thu Jan 17 19:32:58 2002 From: jeremy at zope.com (Jeremy Hylton) Date: Sun Aug 10 16:31:18 2008 Subject: [ZEO-Checkins] CVS: StandaloneZODB/ZEO/zrpc - connection.py:1.1.2.3 Message-ID: <200201180032.g0I0WwA18217@cvs.baymountain.com> Update of /cvs-repository/StandaloneZODB/ZEO/zrpc In directory cvs.zope.org:/tmp/cvs-serv18206/ZEO/zrpc Modified Files: Tag: Standby-branch connection.py Log Message: Add missing closing paren. === StandaloneZODB/ZEO/zrpc/connection.py 1.1.2.2 => 1.1.2.3 === if ret is not None: raise ZRPCError("async method %s returned value %s" % - (name, repr(ret)) + (name, repr(ret))) else: if __debug__: log("%s return %s" % (name, short_repr(ret)), zLOG.DEBUG) From jeremy at zope.com Thu Jan 17 23:51:38 2002 From: jeremy at zope.com (Jeremy Hylton) Date: Sun Aug 10 16:31:18 2008 Subject: [ZEO-Checkins] CVS: StandaloneZODB/ZEO - StorageServer.py:1.32.6.4 Message-ID: <200201180451.g0I4pc732668@cvs.baymountain.com> Update of /cvs-repository/StandaloneZODB/ZEO In directory cvs.zope.org:/tmp/cvs-serv32657/ZEO Modified Files: Tag: Standby-branch StorageServer.py Log Message: Fix repr() to print class name and not constant "StorageProxy" === StandaloneZODB/ZEO/StorageServer.py 1.32.6.3 => 1.32.6.4 === else: stid = None - return "" % (id(self), tid, - stid) + name = self.__class__.__name__ + return "<%s %X trans=%s s_trans=%s>" % (name, id(self), tid, stid) def _log(self, msg, level=zLOG.INFO, error=None, pid=os.getpid()): zLOG.LOG("ZEO Server:%s:%s" % (pid, self.__storage_id), From jeremy at zope.com Thu Jan 17 23:54:30 2002 From: jeremy at zope.com (Jeremy Hylton) Date: Sun Aug 10 16:31:18 2008 Subject: [ZEO-Checkins] CVS: StandaloneZODB/ZEO/zrpc - connection.py:1.1.2.4 Message-ID: <200201180454.g0I4sUX00535@cvs.baymountain.com> Update of /cvs-repository/StandaloneZODB/ZEO/zrpc In directory cvs.zope.org:/tmp/cvs-serv523/ZEO/zrpc Modified Files: Tag: Standby-branch connection.py Log Message: Re-enable the _do_async_poll() for _callAsync(). Found a case where the lack of an explicit poll() call caused the application to hang. === StandaloneZODB/ZEO/zrpc/connection.py 1.1.2.3 => 1.1.2.4 === # will wait for the asyncore loop to get control again. Seems # okay to comment our for now, but need to understand better. -## self._do_async_poll() + self._do_async_poll() # handle IO, possibly in async mode From jeremy at zope.com Fri Jan 18 12:24:59 2002 From: jeremy at zope.com (Jeremy Hylton) Date: Sun Aug 10 16:31:18 2008 Subject: [ZEO-Checkins] CVS: StandaloneZODB/ZEO - ClientStorage.py:1.35.6.5 ServerStub.py:1.3.2.2 StorageServer.py:1.32.6.5 Message-ID: <200201181724.g0IHOx405040@cvs.baymountain.com> Update of /cvs-repository/StandaloneZODB/ZEO In directory cvs.zope.org:/tmp/cvs-serv5025/ZEO Modified Files: Tag: Standby-branch ClientStorage.py ServerStub.py StorageServer.py Log Message: Add support for optional 2nd & 3rd args to tpc_begin(). === StandaloneZODB/ZEO/ClientStorage.py 1.35.6.4 => 1.35.6.5 === self.tpc_cond.release() - def tpc_begin(self, transaction): + def tpc_begin(self, transaction, tid=None, status=' '): self.tpc_cond.acquire() while self._transaction is not None: if self._transaction == transaction: @@ -432,7 +432,8 @@ r = self._server.tpc_begin(id, transaction.user, transaction.description, - transaction._extension) + transaction._extension, + tid, status) except: # Client may have disconnected during the tpc_begin(). # Then notifyDisconnected() will have released the lock. === StandaloneZODB/ZEO/ServerStub.py 1.3.2.1 => 1.3.2.2 === self.rpc.callAsync('storea', oid, serial, data, version, id) - def tpc_begin(self, id, user, descr, ext): - return self.rpc.call('tpc_begin', id, user, descr, ext) + def tpc_begin(self, id, user, descr, ext, tid, status): + return self.rpc.call('tpc_begin', id, user, descr, ext, tid, status) def vote(self, trans_id): return self.rpc.call('vote', trans_id) === StandaloneZODB/ZEO/StorageServer.py 1.32.6.4 => 1.32.6.5 === # and continues from there. - def tpc_begin(self, id, user, description, ext): + def tpc_begin(self, id, user, description, ext, tid, status): if self._transaction is not None: if self._transaction.id == id: self._log("duplicate tpc_begin(%s)" % repr(id)) @@ -418,11 +418,11 @@ if self.__storage._transaction is not None: d = Delay() - self.__storage.__waiting.append((d, self, t)) + self.__storage.__waiting.append((d, self, t, tid, status)) return d self._transaction = t - self.__storage.tpc_begin(t) + self.__storage.tpc_begin(t, tid, status) self.__invalidated = [] def tpc_finish(self, id): @@ -451,17 +451,17 @@ self._transaction = None self.__invalidated = [] - def _restart_delayed_transaction(self, delay, trans): + def _restart_delayed_transaction(self, delay, trans, tid, status): self._transaction = trans - self.__storage.tpc_begin(trans) + self.__storage.tpc_begin(trans, tid, status) self.__invalidated = [] assert self._transaction.id == self.__storage._transaction.id delay.reply(None) def _handle_waiting(self): if self.__storage.__waiting: - delay, proxy, trans = self.__storage.__waiting.pop(0) - proxy._restart_delayed_transaction(delay, trans) + delay, proxy, trans, tid, status = self.__storage.__waiting.pop(0) + proxy._restart_delayed_transaction(delay, trans, tid, status) if self is proxy: return 1 From jeremy at zope.com Mon Jan 21 15:21:18 2002 From: jeremy at zope.com (Jeremy Hylton) Date: Sun Aug 10 16:31:18 2008 Subject: [ZEO-Checkins] CVS: ZEO/ZEO/tests - forker.py:1.13 Message-ID: <200201212021.g0LKLIW30368@cvs.baymountain.com> Update of /cvs-repository/ZEO/ZEO/tests In directory cvs.zope.org:/tmp/cvs-serv30357 Modified Files: forker.py Log Message: Make sure the child process doesn't raise an exception and continue running the rest of the tests in the child. === ZEO/ZEO/tests/forker.py 1.12 => 1.13 === pid = os.fork() if pid == 0: - if PROFILE: - p = profile.Profile() - p.runctx("run_server(storage, addr, rd, wr)", globals(), - locals()) - p.dump_stats("stats.s.%d" % os.getpid()) - else: - run_server(storage, addr, rd, wr) - os._exit(0) + try: + if PROFILE: + p = profile.Profile() + p.runctx("run_server(storage, addr, rd, wr)", globals(), + locals()) + p.dump_stats("stats.s.%d" % os.getpid()) + else: + run_server(storage, addr, rd, wr) + finally: + os._exit(0) else: os.close(rd) return pid, ZEOClientExit(wr) From jeremy at zope.com Mon Jan 21 15:33:51 2002 From: jeremy at zope.com (Jeremy Hylton) Date: Sun Aug 10 16:31:18 2008 Subject: [ZEO-Checkins] CVS: StandaloneZODB/ZEO/tests - testZEO.py:1.17 Message-ID: <200201212033.g0LKXpG00946@cvs.baymountain.com> Update of /cvs-repository/StandaloneZODB/ZEO/tests In directory cvs.zope.org:/tmp/cvs-serv935/tests Modified Files: testZEO.py Log Message: Call super setUp() right away, because it sets _storage to None. Let super tearDown() close the storage. Tracking changes to ZODB/tests/StorageTestBase. === StandaloneZODB/ZEO/tests/testZEO.py 1.16 => 1.17 === getStorage() method. """ + self.__super_setUp() self.running = 1 client, exit, pid = forker.start_zeo(self.getStorage()) self._pid = pid self._server = exit self._storage = PackWaitWrapper(client) client.registerDB(DummyDB(), None) - self.__super_setUp() def tearDown(self): """Try to cause the tests to halt""" self.running = 0 - self._storage.close() self._server.close() os.waitpid(self._pid, 0) self.delStorage() From guido at python.org Mon Jan 21 15:36:39 2002 From: guido at python.org (Guido van Rossum) Date: Sun Aug 10 16:31:18 2008 Subject: [ZEO-Checkins] CVS: StandaloneZODB/ZEO/tests - multi.py:1.5 Message-ID: <200201212036.g0LKadu01626@cvs.baymountain.com> Update of /cvs-repository/StandaloneZODB/ZEO/tests In directory cvs.zope.org:/tmp/cvs-serv1615 Modified Files: multi.py Log Message: Make sure the child process doesn't raise an exception and continue running the rest of the tests in the child. === StandaloneZODB/ZEO/tests/multi.py 1.4 => 1.5 === pid = os.fork() if pid == 0: - import ZEO.ClientStorage - if VERBOSE: - print "Client process started:", os.getpid() - cli = ZEO.ClientStorage.ClientStorage(addr, client=CLIENT_CACHE) - if client_func is None: - run(cli) - else: - client_func(cli) - cli.close() - os._exit(0) + try: + import ZEO.ClientStorage + if VERBOSE: + print "Client process started:", os.getpid() + cli = ZEO.ClientStorage.ClientStorage(addr, client=CLIENT_CACHE) + if client_func is None: + run(cli) + else: + client_func(cli) + cli.close() + finally: + os._exit(0) else: return pid From guido at python.org Mon Jan 21 15:36:30 2002 From: guido at python.org (Guido van Rossum) Date: Sun Aug 10 16:31:18 2008 Subject: [ZEO-Checkins] CVS: StandaloneZODB/ZEO/tests - stress.py:1.3 Message-ID: <200201212036.g0LKaU901598@cvs.baymountain.com> Update of /cvs-repository/StandaloneZODB/ZEO/tests In directory cvs.zope.org:/tmp/cvs-serv1587 Modified Files: stress.py Log Message: Make sure the child process doesn't raise an exception and continue running the rest of the tests in the child. === StandaloneZODB/ZEO/tests/stress.py 1.2 => 1.3 === return pid - storage = ClientStorage(zaddr, debug=1, min_disconnect_poll=0.5) - db = ZODB.DB(storage, pool_size=NUM_CONNECTIONS) - setup(db.open()) - conns = [] - conn_count = 0 + try: + storage = ClientStorage(zaddr, debug=1, min_disconnect_poll=0.5) + db = ZODB.DB(storage, pool_size=NUM_CONNECTIONS) + setup(db.open()) + conns = [] + conn_count = 0 - for i in range(NUM_CONNECTIONS): - c = db.open() - c.__count = 0 - conns.append(c) - conn_count += 1 - - while conn_count < 25: - c = random.choice(conns) - if c.__count > NUM_TRANSACTIONS_PER_CONN: - conns.remove(c) - c.close() - conn_count += 1 + for i in range(NUM_CONNECTIONS): c = db.open() c.__count = 0 conns.append(c) - else: - c.__count += 1 - work(c) + conn_count += 1 + + while conn_count < 25: + c = random.choice(conns) + if c.__count > NUM_TRANSACTIONS_PER_CONN: + conns.remove(c) + c.close() + conn_count += 1 + c = db.open() + c.__count = 0 + conns.append(c) + else: + c.__count += 1 + work(c) - os._exit(0) + finally: + os._exit(0) if __name__ == "__main__": main() From jeremy at zope.com Mon Jan 21 16:20:28 2002 From: jeremy at zope.com (Jeremy Hylton) Date: Sun Aug 10 16:31:18 2008 Subject: [ZEO-Checkins] CVS: ZEO/ZEO/tests - multi.py:1.4.4.2 stress.py:1.2.4.3 Message-ID: <200201212120.g0LLKSk12157@cvs.baymountain.com> Update of /cvs-repository/ZEO/ZEO/tests In directory cvs.zope.org:/tmp/cvs-serv12086 Modified Files: Tag: Standby-branch multi.py stress.py Log Message: Make sure os.fork() doesn't run away with the machine. Tracking similar changes on the trunk. === ZEO/ZEO/tests/multi.py 1.4.4.1 => 1.4.4.2 === pid = os.fork() if pid == 0: - import ZEO.ClientStorage - if VERBOSE: - print "Client process started:", os.getpid() - cli = ZEO.ClientStorage.ClientStorage(addr, client=CLIENT_CACHE) - if client_func is None: - run(cli) - else: - client_func(cli) - cli.close() - os._exit(0) + try: + import ZEO.ClientStorage + if VERBOSE: + print "Client process started:", os.getpid() + cli = ZEO.ClientStorage.ClientStorage(addr, client=CLIENT_CACHE) + if client_func is None: + run(cli) + else: + client_func(cli) + cli.close() + finally: + os._exit(0) else: return pid === ZEO/ZEO/tests/stress.py 1.2.4.2 => 1.2.4.3 === if pid != 0: return pid - + try: + _start_child(zaddr) + finally: + os._exit(0) + +def _start_child(zaddr): storage = ClientStorage(zaddr, debug=1, min_disconnect_poll=0.5, wait_for_server_on_startup=1) db = ZODB.DB(storage, pool_size=NUM_CONNECTIONS) @@ -116,8 +121,6 @@ else: c.__count += 1 work(c) - - os._exit(0) if __name__ == "__main__": main() From jeremy at zope.com Mon Jan 21 16:27:13 2002 From: jeremy at zope.com (Jeremy Hylton) Date: Sun Aug 10 16:31:18 2008 Subject: [ZEO-Checkins] CVS: StandaloneZODB/ZEO/tests - testZEO.py:1.18 Message-ID: <200201212127.g0LLRDv13947@cvs.baymountain.com> Update of /cvs-repository/StandaloneZODB/ZEO/tests In directory cvs.zope.org:/tmp/cvs-serv13936 Modified Files: testZEO.py Log Message: Prevent checkTwoArgBegin() from being run for ZEO 1. === StandaloneZODB/ZEO/tests/testZEO.py 1.17 => 1.18 === self._dostore(data=obj) + def checkTwoArgBegin(self): + pass # ZEO 1 doesn't support two-arg begin + class ZEOFileStorageTests(GenericTests): __super_setUp = GenericTests.setUp From jeremy at zope.com Thu Jan 24 15:56:43 2002 From: jeremy at zope.com (Jeremy Hylton) Date: Sun Aug 10 16:31:18 2008 Subject: [ZEO-Checkins] CVS: StandaloneZODB/ZEO/tests - testZEO.py:1.19 Message-ID: <200201242056.g0OKuh502049@cvs.baymountain.com> Update of /cvs-repository/StandaloneZODB/ZEO/tests In directory cvs.zope.org:/tmp/cvs-serv2038/ZEO/tests Modified Files: testZEO.py Log Message: Get rid of custom _dostore() for ZEO. The main test suite handles ZEO now. === StandaloneZODB/ZEO/tests/testZEO.py 1.18 => 1.19 === class ZEOTestBase(StorageTestBase.StorageTestBase): - """Version of the storage test class that supports ZEO. + """Version of the storage test class that supports ZEO.""" + pass - For ZEO, we don't always get the serialno/exception for a - particular store as the return value from the store. But we - will get no later than the return value from vote. - """ - - def _dostore(self, oid=None, revid=None, data=None, version=None, - already_pickled=0): - """Do a complete storage transaction. - - The defaults are: - - oid=None, ask the storage for a new oid - - revid=None, use a revid of ZERO - - data=None, pickle up some arbitrary data (the integer 7) - - version=None, use the empty string version - - Returns the object's new revision id. - """ - if oid is None: - oid = self._storage.new_oid() - if revid is None: - revid = ZERO - if data is None: - data = MinPO(7) - if not already_pickled: - data = StorageTestBase.zodb_pickle(data) - if version is None: - version = '' - # Begin the transaction - self._storage.tpc_begin(self._transaction) - # Store an object - r1 = self._storage.store(oid, revid, data, version, - self._transaction) - s1 = self._get_serial(r1) - # Finish the transaction - r2 = self._storage.tpc_vote(self._transaction) - s2 = self._get_serial(r2) - self._storage.tpc_finish(self._transaction) - # s1, s2 can be None or dict - assert not (s1 and s2) - return s1 and s1[oid] or s2 and s2[oid] - - def _get_serial(self, r): - """Return oid -> serialno dict from sequence of ZEO replies.""" - d = {} - if r is None: - return None - if type(r) == types.StringType: - raise RuntimeError, "unexpected ZEO response: no oid" - else: - for oid, serial in r: - if isinstance(serial, Exception): - raise serial - d[oid] = serial - return d - # Some of the ZEO tests depend on the version of FileStorage available # for the tests. If we run these tests using Zope 2.3, FileStorage # doesn't support TransactionalUndo. From jeremy at zope.com Thu Jan 24 15:56:09 2002 From: jeremy at zope.com (Jeremy Hylton) Date: Sun Aug 10 16:31:18 2008 Subject: [ZEO-Checkins] CVS: StandaloneZODB/ZEO/tests - Cache.py:1.5 Message-ID: <200201242056.g0OKu9402012@cvs.baymountain.com> Update of /cvs-repository/StandaloneZODB/ZEO/tests In directory cvs.zope.org:/tmp/cvs-serv2001/ZEO/tests Modified Files: Cache.py Log Message: Get rid of more self._transaction. === StandaloneZODB/ZEO/tests/Cache.py 1.4 => 1.5 === # Now start an undo transaction - self._transaction.note('undo1') - self._storage.tpc_begin(self._transaction) + t = Transaction() + t.note('undo1') + self._storage.tpc_begin(t) - oids = self._storage.transactionalUndo(tid, self._transaction) + oids = self._storage.transactionalUndo(tid, t) # Make sure this doesn't load invalid data into the cache self._storage.load(oid, '') - self._storage.tpc_vote(self._transaction) - self._storage.tpc_finish(self._transaction) + self._storage.tpc_vote(t) + self._storage.tpc_finish(t) assert len(oids) == 1 assert oids[0] == oid From jeremy at zope.com Thu Jan 24 17:08:24 2002 From: jeremy at zope.com (Jeremy Hylton) Date: Sun Aug 10 16:31:18 2008 Subject: [ZEO-Checkins] CVS: ZEO/ZEO/tests - Cache.py:1.4.4.2 Message-ID: <200201242208.g0OM8Oj20683@cvs.baymountain.com> Update of /cvs-repository/ZEO/ZEO/tests In directory cvs.zope.org:/tmp/cvs-serv20672 Modified Files: Tag: Standby-branch Cache.py Log Message: Track removal of _transaction on trunk. === ZEO/ZEO/tests/Cache.py 1.4.4.1 => 1.4.4.2 === -# -# This software is subject to the provisions of the Zope Public License, -# Version 1.1 (ZPL). A copy of the ZPL should accompany this -# distribution. THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL -# EXPRESS OR IMPLIED WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST -# INFRINGEMENT, AND FITNESS FOR A PARTICULAR PURPOSE. - """Tests of the ZEO cache""" from ZODB.Transaction import Transaction @@ -34,16 +25,17 @@ return # Now start an undo transaction - self._transaction.note('undo1') - self._storage.tpc_begin(self._transaction) + t = Transaction() + t.note('undo1') + self._storage.tpc_begin(t) - oids = self._storage.transactionalUndo(tid, self._transaction) + oids = self._storage.transactionalUndo(tid, t) # Make sure this doesn't load invalid data into the cache self._storage.load(oid, '') - self._storage.tpc_vote(self._transaction) - self._storage.tpc_finish(self._transaction) + self._storage.tpc_vote(t) + self._storage.tpc_finish(t) assert len(oids) == 1 assert oids[0] == oid From jeremy at zope.com Thu Jan 24 17:20:51 2002 From: jeremy at zope.com (Jeremy Hylton) Date: Sun Aug 10 16:31:18 2008 Subject: [ZEO-Checkins] CVS: ZEO/ZEO/tests - testZEO.py:1.16.4.5 Message-ID: <200201242220.g0OMKp424309@cvs.baymountain.com> Update of /cvs-repository/ZEO/ZEO/tests In directory cvs.zope.org:/tmp/cvs-serv24298 Modified Files: Tag: Standby-branch testZEO.py Log Message: Get rid of ZEOTestBase and VersionDependentTests. The ZEOTestBase provided features now in the StorageTestBase. The version-dependent tests are a non-issue because this code don't need to work with Zope 2.3. === ZEO/ZEO/tests/testZEO.py 1.16.4.4 => 1.16.4.5 === self.storage.pack(t, f, wait=1) -class ZEOTestBase(StorageTestBase.StorageTestBase): - """Version of the storage test class that supports ZEO. - - For ZEO, we don't always get the serialno/exception for a - particular store as the return value from the store. But we - will get no later than the return value from vote. - """ - - def _dostore(self, oid=None, revid=None, data=None, version=None, - already_pickled=0): - """Do a complete storage transaction. - - The defaults are: - - oid=None, ask the storage for a new oid - - revid=None, use a revid of ZERO - - data=None, pickle up some arbitrary data (the integer 7) - - version=None, use the empty string version - - Returns the object's new revision id. - """ - if oid is None: - oid = self._storage.new_oid() - if revid is None: - revid = ZERO - if data is None: - data = MinPO(7) - if not already_pickled: - data = StorageTestBase.zodb_pickle(data) - if version is None: - version = '' - # Begin the transaction - self._storage.tpc_begin(self._transaction) - # Store an object - r1 = self._storage.store(oid, revid, data, version, - self._transaction) - s1 = self._get_serial(r1) - # Finish the transaction - r2 = self._storage.tpc_vote(self._transaction) - s2 = self._get_serial(r2) - self._storage.tpc_finish(self._transaction) - # s1, s2 can be None or dict - assert not (s1 and s2) - return s1 and s1[oid] or s2 and s2[oid] - - def _get_serial(self, r): - """Return oid -> serialno dict from sequence of ZEO replies.""" - d = {} - if r is None: - return None - if type(r) == types.StringType: - raise RuntimeError, "unexpected ZEO response: no oid" - else: - for oid, serial in r: - if isinstance(serial, Exception): - raise serial - d[oid] = serial - return d - -# Some of the ZEO tests depend on the version of FileStorage available -# for the tests. If we run these tests using Zope 2.3, FileStorage -# doesn't support TransactionalUndo. - -if hasattr(FileStorage, 'supportsTransactionalUndo'): - # XXX Assume that a FileStorage that supports transactional undo - # also supports conflict resolution. - class VersionDependentTests( - TransactionalUndoStorage.TransactionalUndoStorage, - TransactionalUndoVersionStorage.TransactionalUndoVersionStorage, - ConflictResolution.ConflictResolvingStorage, - ConflictResolution.ConflictResolvingTransUndoStorage): - pass -else: - class VersionDependentTests: - pass - -class GenericTests(ZEOTestBase, - VersionDependentTests, +class GenericTests(StorageTestBase.StorageTestBase, + TransactionalUndoStorage.TransactionalUndoStorage, + TransactionalUndoVersionStorage.TransactionalUndoVersionStorage, + ConflictResolution.ConflictResolvingStorage, + ConflictResolution.ConflictResolvingTransUndoStorage, Cache.StorageWithCache, Cache.TransUndoStorageWithCache, BasicStorage.BasicStorage, @@ -213,7 +141,7 @@ """ __super_setUp = StorageTestBase.StorageTestBase.setUp __super_tearDown = StorageTestBase.StorageTestBase.tearDown - + def setUp(self): self.__super_setUp() args = self.getStorageInfo() @@ -251,7 +179,7 @@ except os.error: pass -class ConnectionTests(ZEOTestBase): +class ConnectionTests(StorageTestBase.StorageTestBase): """Tests that explicitly manage the server process. To test the cache or re-connection, these test cases explicit From jeremy at zope.com Fri Jan 25 11:32:04 2002 From: jeremy at zope.com (Jeremy Hylton) Date: Sun Aug 10 16:31:18 2008 Subject: [ZEO-Checkins] CVS: StandaloneZODB/ZEO/tests - testZEO.py:1.20 Message-ID: <200201251632.g0PGW4D24759@cvs.baymountain.com> Update of /cvs-repository/StandaloneZODB/ZEO/tests In directory cvs.zope.org:/tmp/cvs-serv24748 Modified Files: testZEO.py Log Message: Remove dependence on StorageTestBase setUp() and tearDown(). === StandaloneZODB/ZEO/tests/testZEO.py 1.19 => 1.20 === class VersionDependentTests: pass - + class GenericTests(ZEOTestBase, VersionDependentTests, Cache.StorageWithCache, @@ -80,16 +80,12 @@ returns a specific storage, e.g. FileStorage. """ - __super_setUp = StorageTestBase.StorageTestBase.setUp - __super_tearDown = StorageTestBase.StorageTestBase.tearDown - def setUp(self): """Start a ZEO server using a Unix domain socket The ZEO server uses the storage object returned by the getStorage() method. """ - self.__super_setUp() self.running = 1 client, exit, pid = forker.start_zeo(self.getStorage()) self._pid = pid @@ -100,10 +96,10 @@ def tearDown(self): """Try to cause the tests to halt""" self.running = 0 + self._storage.close() self._server.close() os.waitpid(self._pid, 0) self.delStorage() - self.__super_tearDown() def checkLargeUpdate(self): obj = MinPO("X" * (10 * 128 * 1024)) @@ -138,11 +134,8 @@ can't be created in the parent process and passed to the child. All the work has to be done in the server's process. """ - __super_setUp = StorageTestBase.StorageTestBase.setUp - __super_tearDown = StorageTestBase.StorageTestBase.tearDown def setUp(self): - self.__super_setUp() args = self.getStorageInfo() name = args[0] args = args[1:] @@ -162,7 +155,6 @@ ## os.waitpid(self.test_pid, 0) time.sleep(0.5) self.delStorage() - self.__super_tearDown() class WindowsZEOFileStorageTests(WindowsGenericTests): @@ -186,8 +178,6 @@ start and stop a ZEO storage server. """ - __super_tearDown = StorageTestBase.StorageTestBase.tearDown - ports = [] for i in range(200): ports.append(random.randrange(25000, 30000)) @@ -203,6 +193,7 @@ def tearDown(self): """Try to cause the tests to halt""" + self._storage.close() self.shutdownServer() # file storage appears to create four files for ext in '', '.index', '.lock', '.tmp': @@ -213,7 +204,6 @@ path = "c1-test-%d.zec" % i if os.path.exists(path): os.unlink(path) - self.__super_tearDown() def checkBasicPersistence(self): """Verify cached data persists across client storage instances. From jeremy at zope.com Fri Jan 25 19:02:43 2002 From: jeremy at zope.com (Jeremy Hylton) Date: Sun Aug 10 16:31:18 2008 Subject: [ZEO-Checkins] CVS: ZEO/ZEO/zrpc - client.py:1.1.2.3 Message-ID: <200201260002.g0Q02he24490@cvs.baymountain.com> Update of /cvs-repository/ZEO/ZEO/zrpc In directory cvs.zope.org:/tmp/cvs-serv24474 Modified Files: Tag: Standby-branch client.py Log Message: Fix KeyError bug (I think). The problem was in the logic of attempt_connects(). It has two parts: one where it creates the sockets and initiates the connection, another where it waits for the first connection to succeed. But it's possible that the connect returns right away. In this case, the socket wasn't put in the sockets dict yet and the cleanup code failed with a KeyError. As part of the fix, I reworked the entire handling of the sockets dict. I made it an instance variable so that attempt_connects() has to do less explicit management of it. The various helper methods can modify it directly. === ZEO/ZEO/zrpc/client.py 1.1.2.2 => 1.1.2.3 === self.stopped = 1 + def close_sockets(self): + for s in self.sockets.keys(): + s.close() + def run(self): delay = self.tmin while not (self.stopped or self.attempt_connects()): @@ -163,7 +167,7 @@ def attempt_connects(self): "Return true if any connect attempt succeeds." - sockets = {} + self.sockets = {} log("attempting connection on %d sockets" % len(self.addr)) try: @@ -173,35 +177,30 @@ level=zLOG.DEBUG) s = socket.socket(domain, socket.SOCK_STREAM) s.setblocking(0) + self.sockets[s] = addr # XXX can still block for a while if addr requires DNS - e = self.connect(s, addr) - if e is not None: - sockets[s] = addr + e = self.connect(s) # next wait until they actually connect - while sockets: + while self.sockets: if self.stopped: - for s in sockets.keys(): - s.close() + self.close_sockets() return 0 try: - r, w, x = select.select([], sockets.keys(), [], 1.0) + r, w, x = select.select([], self.sockets.keys(), [], 1.0) except select.error: continue for s in w: - e = self.connect(s, sockets[s]) - if e is None: - del sockets[s] + # connect() will raise Connected if it succeeds + e = self.connect(s) except Connected, container: s = container.sock - del sockets[s] - # close all the other sockets - for s in sockets.keys(): - s.close() + del self.sockets[s] # don't close the newly connected socket + self.close_sockets() return 1 return 0 - def connect(self, s, addr): + def connect(self, s): """Call s.connect_ex(addr) and return true if loop should continue. We have to handle several possible return values from @@ -213,10 +212,12 @@ If the socket sonnects and the initial ZEO setup fails or the connect_ex() returns an error, we close the socket and ignore it. + When the socket is closed, it is removed from self.sockets. If connect_ex() returns EINPROGRESS, we need to try again later. """ - + + addr = self.sockets[s] e = s.connect_ex(addr) if e == errno.EINPROGRESS: return 1 @@ -230,6 +231,7 @@ log("error connecting to %s: %s" % (addr, errno.errorcode[e]), level=zLOG.DEBUG) s.close() + del self.sockets[s] def test_connection(self, s, addr): c = ManagedConnection(s, addr, self.client, self.mgr) From jeremy at zope.com Fri Jan 25 23:07:55 2002 From: jeremy at zope.com (Jeremy Hylton) Date: Sun Aug 10 16:31:18 2008 Subject: [ZEO-Checkins] CVS: ZEO/ZEO/tests - testZEO.py:1.16.4.6 Message-ID: <200201260407.g0Q47tU20063@cvs.baymountain.com> Update of /cvs-repository/ZEO/ZEO/tests In directory cvs.zope.org:/tmp/cvs-serv20052 Modified Files: Tag: Standby-branch testZEO.py Log Message: Don't call StorageTestBase setUp() or tearDown(). === ZEO/ZEO/tests/testZEO.py 1.16.4.5 => 1.16.4.6 === """ - __super_setUp = StorageTestBase.StorageTestBase.setUp - __super_tearDown = StorageTestBase.StorageTestBase.tearDown - def setUp(self): - self.__super_setUp() zLOG.LOG("testZEO", zLOG.BLATHER, "setUp() new test") self.running = 1 client, exit, pid = forker.start_zeo(self.getStorage()) @@ -97,7 +93,6 @@ for pid in self._pids: os.waitpid(pid, 0) self.delStorage() - self.__super_tearDown() def checkLargeUpdate(self): obj = MinPO("X" * (10 * 128 * 1024)) @@ -139,11 +134,8 @@ can't be created in the parent process and passed to the child. All the work has to be done in the server's process. """ - __super_setUp = StorageTestBase.StorageTestBase.setUp - __super_tearDown = StorageTestBase.StorageTestBase.tearDown def setUp(self): - self.__super_setUp() args = self.getStorageInfo() name = args[0] args = args[1:] @@ -162,7 +154,6 @@ # the connection should cause the storage server to die time.sleep(0.5) self.delStorage() - self.__super_tearDown() class WindowsZEOFileStorageTests(WindowsGenericTests): @@ -328,7 +319,6 @@ # inherit from POSException. class UnixConnectionTests(ConnectionTests): - __super_setUp = StorageTestBase.StorageTestBase.setUp def setUp(self): """Start a ZEO server using a Unix domain socket @@ -343,7 +333,6 @@ self._servers = [] self._newAddr() self._startServer() - self.__super_setUp() def _newAddr(self): self.addr.append(self._getAddr()) @@ -378,14 +367,12 @@ pass class WindowsConnectionTests(ConnectionTests): - __super_setUp = StorageTestBase.StorageTestBase.setUp # XXX these tests are now out-of-date def setUp(self): self.file = tempfile.mktemp() self._startServer() - self.__super_setUp() def _startServer(self, create=1): if create == 0: @@ -418,7 +405,7 @@ def tearDown(self): self.shutdownServer() - + self._storage.close() def get_methods(klass): l = [klass] From jeremy at zope.com Fri Jan 25 23:31:36 2002 From: jeremy at zope.com (Jeremy Hylton) Date: Sun Aug 10 16:31:18 2008 Subject: [ZEO-Checkins] CVS: ZEO/ZEO/zrpc - client.py:1.1.2.4 Message-ID: <200201260431.g0Q4VaG26243@cvs.baymountain.com> Update of /cvs-repository/ZEO/ZEO/zrpc In directory cvs.zope.org:/tmp/cvs-serv26232 Modified Files: Tag: Standby-branch client.py Log Message: Add a few comments and doc strings that may help Tim understand the classes and methods. === ZEO/ZEO/zrpc/client.py 1.1.2.3 => 1.1.2.4 === self.connected = 0 self.connection = None + self.closed = 0 # If _thread is not None, then there is a helper thread # attempting to connect. _thread is protected by _connect_lock. self._thread = None self._connect_lock = threading.Lock() self.trigger = None self.thr_async = 0 - self.closed = 0 ThreadedAsync.register_loop_callback(self.set_async) def __repr__(self): @@ -86,11 +86,24 @@ self.thr_async = 1 # XXX needs to be set on the Connection def attempt_connect(self): - # XXX will a single attempt take too long? + """Attempt a connection to the server without blocking too long. + + There isn't a crisp definition for too long. When a + ClientStorage is created, it attempts to connect to the + server. If the server isn't immediately available, it can + operate from the cache. This method will start the background + connection thread and wait a little while to see if it + finishes quickly. + """ + + # XXX will a single attempt take too long? self.connect() try: event = self._thread.one_attempt except AttributeError: + # An AttributeError means that (1) _thread is None and (2) + # as a consquence of (1) that the connect thread has + # already exited. pass else: event.wait() @@ -134,9 +147,23 @@ self.sock = sock class ConnectThread(threading.Thread): + """Thread that tries to connect to server given one or more addresses. + + The thread is passed a ConnectionManager and the manager's client + as arguments. It calls notifyConnected() on the client when a + socket connects. If notifyConnected() returns without raising an + exception, the thread is done; it calls connect_done() on the + manager and exits. + + The thread will continue to run, attempting connections, until a + successful notifyConnected() or stop() is called. + """ __super_init = threading.Thread.__init__ + # We don't expect clients to call any methods of this Thread other + # than close() and those defined by the Thread API. + def __init__(self, mgr, client, addr, tmin, tmax): self.__super_init(name="Connect(%s)" % addr) self.mgr = mgr @@ -146,13 +173,16 @@ self.tmax = tmax self.stopped = 0 self.one_attempt = threading.Event() + # A ConnectThread keeps track of whether it has finished a + # call to attempt_connects(). This allows the + # ConnectionManager to make an attempt to connect right away, + # but not block for too long if the server isn't immediately + # available. def stop(self): self.stopped = 1 - def close_sockets(self): - for s in self.sockets.keys(): - s.close() + # Every method from run() to the end is used internally by the Thread. def run(self): delay = self.tmin @@ -165,6 +195,10 @@ delay = self.tmax log("thread exiting: %s" % self.getName()) + def close_sockets(self): + for s in self.sockets.keys(): + s.close() + def attempt_connects(self): "Return true if any connect attempt succeeds." self.sockets = {} @@ -216,7 +250,6 @@ If connect_ex() returns EINPROGRESS, we need to try again later. """ - addr = self.sockets[s] e = s.connect_ex(addr) if e == errno.EINPROGRESS: From tim.one at home.com Sat Jan 26 13:01:32 2002 From: tim.one at home.com (Tim Peters) Date: Sun Aug 10 16:31:18 2008 Subject: [ZEO-Checkins] CVS: ZEO/ZEO/zrpc - client.py:1.1.2.5 Message-ID: <200201261801.g0QI1WH27626@cvs.baymountain.com> Update of /cvs-repository/ZEO/ZEO/zrpc In directory cvs.zope.org:/tmp/cvs-serv27609/zrpc Modified Files: Tag: Standby-branch client.py Log Message: Added a YYY question to an unclear XXX comment. Whitespace normalization. === ZEO/ZEO/zrpc/client.py 1.1.2.4 => 1.1.2.5 === finishes quickly. """ - - # XXX will a single attempt take too long? + + # XXX will a single attempt take too long? self.connect() try: event = self._thread.one_attempt @@ -145,7 +145,7 @@ # helper for non-local exit def __init__(self, sock): self.sock = sock - + class ConnectThread(threading.Thread): """Thread that tries to connect to server given one or more addresses. @@ -177,7 +177,7 @@ # call to attempt_connects(). This allows the # ConnectionManager to make an attempt to connect right away, # but not block for too long if the server isn't immediately - # available. + # available. def stop(self): self.stopped = 1 @@ -194,7 +194,7 @@ if delay > self.tmax: delay = self.tmax log("thread exiting: %s" % self.getName()) - + def close_sockets(self): for s in self.sockets.keys(): s.close() @@ -213,6 +213,8 @@ s.setblocking(0) self.sockets[s] = addr # XXX can still block for a while if addr requires DNS + # YYY What is that XXX comment trying to say? self.connect + # YYY explicitly tolerates EINPROGRESS. e = self.connect(s) # next wait until they actually connect From tim.one at home.com Sat Jan 26 13:03:52 2002 From: tim.one at home.com (Tim Peters) Date: Sun Aug 10 16:31:18 2008 Subject: [ZEO-Checkins] CVS: ZEO/ZEO/zrpc - connection.py:1.1.2.5 error.py:1.1.2.2 marshal.py:1.1.2.2 server.py:1.1.2.2 trigger.py:1.1.2.2 Message-ID: <200201261803.g0QI3qg28358@cvs.baymountain.com> Update of /cvs-repository/ZEO/ZEO/zrpc In directory cvs.zope.org:/tmp/cvs-serv28311/zrpc Modified Files: Tag: Standby-branch connection.py error.py marshal.py server.py trigger.py Log Message: Whitespace normalization. === ZEO/ZEO/zrpc/connection.py 1.1.2.4 => 1.1.2.5 === msg = self.marshal.encode(msgid, 0, REPLY, ret) self.message_output(msg) - + def return_error(self, msgid, flags, err_type, err_value): if flags is None: self.log_error("Exception raised during decoding") @@ -271,7 +271,7 @@ if self.closed: raise DisconnectedError() self.__reply_lock.release() - + def _do_async_poll(self, wait_for_reply=0): "Invoke asyncore mainloop to get pending message out." @@ -310,7 +310,7 @@ """A connection that notifies its ConnectionManager of closing. A managed connection also defers the ThreadedAsync work to its - manager. + manager. """ __super_init = Connection.__init__ __super_close = Connection.close @@ -345,4 +345,3 @@ def close(self): self.__super_close() self.__mgr.notify_closed(self) - === ZEO/ZEO/zrpc/error.py 1.1.2.1 => 1.1.2.2 === class DisconnectedError(ZRPCError, Disconnected): """The database storage is disconnected from the storage server.""" - === ZEO/ZEO/zrpc/marshal.py 1.1.2.1 => 1.1.2.2 === # It's okay to share a single Pickler as long as it's in fast # mode, which means that it doesn't have a memo. - + pickler = cPickle.Pickler() pickler.fast = 1 pickle = pickler.dump @@ -46,7 +46,7 @@ r = getattr(m, name) except AttributeError: raise ZRPCError("module %s has no global %s" % (module, name)) - + safe = getattr(r, '__no_side_effects__', 0) if safe: return r === ZEO/ZEO/zrpc/server.py 1.1.2.1 => 1.1.2.2 === reuse_addr = 1 - def __init__(self, addr, factory=Connection, reuse_addr=None): + def __init__(self, addr, factory=Connection, reuse_addr=None): self.__super_init() self.addr = addr self.factory = factory === ZEO/ZEO/zrpc/trigger.py 1.1.2.1 => 1.1.2.2 === -# +# # Zope Public License (ZPL) Version 1.0 # ------------------------------------- -# +# # Copyright (c) Digital Creations. All rights reserved. -# +# # This license has been certified as Open Source(tm). -# +# # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: -# +# # 1. Redistributions in source code must retain the above copyright # notice, this list of conditions, and the following disclaimer. -# +# # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions, and the following disclaimer in # the documentation and/or other materials provided with the # distribution. -# +# # 3. Digital Creations requests that attribution be given to Zope # in any manner possible. Zope includes a "Powered by Zope" # button that is installed by default. While it is not a license @@ -26,43 +26,43 @@ # attribution remain. A significant investment has been put # into Zope, and this effort will continue if the Zope community # continues to grow. This is one way to assure that growth. -# +# # 4. All advertising materials and documentation mentioning # features derived from or use of this software must display # the following acknowledgement: -# +# # "This product includes software developed by Digital Creations # for use in the Z Object Publishing Environment # (http://www.zope.org/)." -# +# # In the event that the product being advertised includes an # intact Zope distribution (with copyright and license included) # then this clause is waived. -# +# # 5. Names associated with Zope or Digital Creations must not be used to # endorse or promote products derived from this software without # prior written permission from Digital Creations. -# +# # 6. Modified redistributions of any form whatsoever must retain # the following acknowledgment: -# +# # "This product includes software developed by Digital Creations # for use in the Z Object Publishing Environment # (http://www.zope.org/)." -# +# # Intact (re-)distributions of any official Zope release do not # require an external acknowledgement. -# +# # 7. Modifications are encouraged but must be packaged separately as # patches to official Zope releases. Distributions that do not # clearly separate the patches from the original work must be clearly # labeled as unofficial distributions. Modifications which do not # carry the name Zope may be packaged in any form, as long as they # conform to all of the clauses above. -# -# +# +# # Disclaimer -# +# # THIS SOFTWARE IS PROVIDED BY DIGITAL CREATIONS ``AS IS'' AND ANY # EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR @@ -75,12 +75,12 @@ # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT # OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF # SUCH DAMAGE. -# -# +# +# # This software consists of contributions made by Digital Creations and # many individuals on behalf of Digital Creations. Specific # attributions are listed in the accompanying credits file. -# +# ############################################################################## # This module is a simplified version of the select_trigger module @@ -94,7 +94,7 @@ import socket import string import thread - + if os.name == 'posix': class trigger (asyncore.file_dispatcher): @@ -199,7 +199,7 @@ if port <= 19950: raise 'Bind Error', 'Cannot bind trigger!' port=port - 1 - + a.listen (1) w.setblocking (0) try: @@ -250,7 +250,3 @@ self.thunks = [] finally: self.lock.release() - - - - From tim.one at home.com Sat Jan 26 13:03:52 2002 From: tim.one at home.com (Tim Peters) Date: Sun Aug 10 16:31:18 2008 Subject: [ZEO-Checkins] CVS: ZEO/ZEO/tests - Cache.py:1.4.4.3 forker.py:1.10.4.5 speed.py:1.5.4.2 testTransactionBuffer.py:1.3.2.2 testZEO.py:1.16.4.7 Message-ID: <200201261803.g0QI3q928350@cvs.baymountain.com> Update of /cvs-repository/ZEO/ZEO/tests In directory cvs.zope.org:/tmp/cvs-serv28311/tests Modified Files: Tag: Standby-branch Cache.py forker.py speed.py testTransactionBuffer.py testZEO.py Log Message: Whitespace normalization. === ZEO/ZEO/tests/Cache.py 1.4.4.2 => 1.4.4.3 === # Make sure this doesn't load invalid data into the cache self._storage.load(oid, '') - + self._storage.tpc_vote(t) self._storage.tpc_finish(t) === ZEO/ZEO/tests/forker.py 1.10.4.4 => 1.10.4.5 === wait_for_server_on_startup=1) return s, exit, pid - === ZEO/ZEO/tests/speed.py 1.5.4.1 => 1.5.4.2 === """ -import asyncore +import asyncore import sys, os, getopt, string, time ##sys.path.insert(0, os.getcwd()) @@ -77,7 +77,7 @@ for r in 1, 10, 100, 1000: t = time.time() conflicts = 0 - + jar = db.open() while 1: try: @@ -101,7 +101,7 @@ else: break jar.close() - + t = time.time() - t if detailed: if threadno is None: @@ -201,11 +201,11 @@ for v in l: tot = tot + v return tot / len(l) - + ##def compress(s): ## c = zlib.compressobj() ## o = c.compress(s) -## return o + c.flush() +## return o + c.flush() if __name__=='__main__': main(sys.argv[1:]) === ZEO/ZEO/tests/testTransactionBuffer.py 1.3.2.1 => 1.3.2.2 === x = x[:2] self.assertEqual(x, data[i]) - + def checkOrderPreserved(self): tbuf = TransactionBuffer() self.doUpdates(tbuf) @@ -61,4 +61,3 @@ def test_suite(): return unittest.makeSuite(TransBufTests, 'check') - === ZEO/ZEO/tests/testZEO.py 1.16.4.6 => 1.16.4.7 === class ZEOFileStorageTests(GenericTests): __super_setUp = GenericTests.setUp - + def setUp(self): self.__fs_base = tempfile.mktemp() self.__super_setUp() @@ -108,7 +108,7 @@ def open(self, read_only=0): # XXX Needed to support ReadOnlyStorage tests. Ought to be a # cleaner way. - + # Is this the only way to get the address? addr = self._storage._rpc_mgr.addr[0][1] self._storage.close() @@ -134,7 +134,7 @@ can't be created in the parent process and passed to the child. All the work has to be done in the server's process. """ - + def setUp(self): args = self.getStorageInfo() name = args[0] @@ -176,7 +176,7 @@ To test the cache or re-connection, these test cases explicit start and stop a ZEO storage server. """ - + __super_tearDown = StorageTestBase.StorageTestBase.tearDown ports = [] @@ -221,14 +221,14 @@ def checkMultipleServers(self): # XXX crude test at first -- just start two servers and do a # commit at each one. - + self._newAddr() self._storage = self.openClientStorage('test', 100000, wait=1) self._dostore() self.shutdownServer(index=0) self._startServer(index=1) - + # If we can still store after shutting down one of the # servers, we must be reconnecting to the other server. @@ -238,7 +238,7 @@ break except Disconnected: time.sleep(0.5) - + def checkDisconnectionError(self): # Make sure we get a Disconnected when we try to read an @@ -271,7 +271,7 @@ # In this case, only one object fits in a cache file. When the # cache files swap, the first object is effectively uncached. - + self._storage = self.openClientStorage('test', 1000, wait=1) oid1 = self._storage.new_oid() obj1 = MinPO("1" * 500) @@ -316,7 +316,7 @@ # on users to catch a plethora of exceptions in order to # write robust code. Need to think about implementing # John Heintz's suggestion to make sure all exceptions - # inherit from POSException. + # inherit from POSException. class UnixConnectionTests(ConnectionTests): @@ -453,7 +453,7 @@ if args: print "Did not expect arguments. Got %s" % args return 0 - + tests = makeTestSuite(name_of_test) runner = unittest.TextTestRunner() runner.run(tests) From tim.one at home.com Sat Jan 26 13:04:21 2002 From: tim.one at home.com (Tim Peters) Date: Sun Aug 10 16:31:18 2008 Subject: [ZEO-Checkins] CVS: ZEO/ZEO/misc - custom_zodb.py:1.2.2.2 Message-ID: <200201261804.g0QI4LB28499@cvs.baymountain.com> Update of /cvs-repository/ZEO/ZEO/misc In directory cvs.zope.org:/tmp/cvs-serv28311/misc Modified Files: Tag: Standby-branch custom_zodb.py Log Message: Whitespace normalization. === ZEO/ZEO/misc/custom_zodb.py 1.2.2.1 => 1.2.2.2 === if 0: # Change the 0 to 1 to enable! - + # ZEO Unix Domain Socket # This import isn't strictly necessary but is helpful when @@ -30,4 +30,3 @@ Storage=ZODB.FileStorage.FileStorage( os.path.join(INSTANCE_HOME, 'var', 'Data.fs'), ) - From tim.one at home.com Sat Jan 26 13:04:22 2002 From: tim.one at home.com (Tim Peters) Date: Sun Aug 10 16:31:18 2008 Subject: [ZEO-Checkins] CVS: ZEO/ZEO - ClientCache.py:1.18.6.4 ClientStorage.py:1.35.6.6 ClientStub.py:1.3.2.3 Invalidator.py:1.6.6.2 ServerStub.py:1.3.2.3 StorageServer.py:1.32.6.6 TransactionBuffer.py:1.3.2.3 asyncwrap.py:1.2.4.2 fap.py:1.5.6.2 smac.py:1.11.4.5 start.py:1.26.4.3 trigger.py:1.3.4.3 zrpc.py:1.20.4.2 Message-ID: <200201261804.g0QI4M228517@cvs.baymountain.com> Update of /cvs-repository/ZEO/ZEO In directory cvs.zope.org:/tmp/cvs-serv28311 Modified Files: Tag: Standby-branch ClientCache.py ClientStorage.py ClientStub.py Invalidator.py ServerStub.py StorageServer.py TransactionBuffer.py asyncwrap.py fap.py smac.py start.py trigger.py zrpc.py Log Message: Whitespace normalization. === ZEO/ZEO/ClientCache.py 1.18.6.3 => 1.18.6.4 === -# +# # Zope Public License (ZPL) Version 1.0 # ------------------------------------- -# +# # Copyright (c) Digital Creations. All rights reserved. -# +# # This license has been certified as Open Source(tm). -# +# # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: -# +# # 1. Redistributions in source code must retain the above copyright # notice, this list of conditions, and the following disclaimer. -# +# # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions, and the following disclaimer in # the documentation and/or other materials provided with the # distribution. -# +# # 3. Digital Creations requests that attribution be given to Zope # in any manner possible. Zope includes a "Powered by Zope" # button that is installed by default. While it is not a license @@ -26,43 +26,43 @@ # attribution remain. A significant investment has been put # into Zope, and this effort will continue if the Zope community # continues to grow. This is one way to assure that growth. -# +# # 4. All advertising materials and documentation mentioning # features derived from or use of this software must display # the following acknowledgement: -# +# # "This product includes software developed by Digital Creations # for use in the Z Object Publishing Environment # (http://www.zope.org/)." -# +# # In the event that the product being advertised includes an # intact Zope distribution (with copyright and license included) # then this clause is waived. -# +# # 5. Names associated with Zope or Digital Creations must not be used to # endorse or promote products derived from this software without # prior written permission from Digital Creations. -# +# # 6. Modified redistributions of any form whatsoever must retain # the following acknowledgment: -# +# # "This product includes software developed by Digital Creations # for use in the Z Object Publishing Environment # (http://www.zope.org/)." -# +# # Intact (re-)distributions of any official Zope release do not # require an external acknowledgement. -# +# # 7. Modifications are encouraged but must be packaged separately as # patches to official Zope releases. Distributions that do not # clearly separate the patches from the original work must be clearly # labeled as unofficial distributions. Modifications which do not # carry the name Zope may be packaged in any form, as long as they # conform to all of the clauses above. -# -# +# +# # Disclaimer -# +# # THIS SOFTWARE IS PROVIDED BY DIGITAL CREATIONS ``AS IS'' AND ANY # EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR @@ -75,15 +75,15 @@ # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT # OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF # SUCH DAMAGE. -# -# +# +# # This software consists of contributions made by Digital Creations and # many individuals on behalf of Digital Creations. Specific # attributions are listed in the accompanying credits file. -# +# ############################################################################## """Implement a client cache - + The cache is managed as two files, var/c0.zec and var/c1.zec. Each cache file is a sequence of records of the form: @@ -195,7 +195,7 @@ # If we found a non-zero serial, then use the file if s[i] != '\0\0\0\0\0\0\0\0': f[i]=fi fi=None - + # Whoever has the larger serial is the current if s[1] > s[0]: current=1 elif s[0] > s[1]: current=0 @@ -220,7 +220,7 @@ def open(self): # XXX open is overloaded to perform two tasks for - # optimization reasons + # optimization reasons self._acquire() try: self._index=index={} @@ -284,11 +284,11 @@ del self._index[oid] return None - if h[8]=='n': + if h[8]=='n': if version: return None if not dlen: del self._index[oid] - return None + return None if not vlen or not version: if dlen: return read(dlen), h[19:] @@ -332,7 +332,7 @@ if dlen: p=read(dlen) s=h[19:] - else: + else: return self._store(oid, '', '', version, data, serial) self._store(oid, p, s, version, data, serial) @@ -390,7 +390,7 @@ self._f[current].write(magic) self._pos=pos=4 finally: self._release() - + def store(self, oid, p, s, version, pv, sv): self._acquire() @@ -416,7 +416,7 @@ if p: l.append(p) if version: - l.extend([version, + l.extend([version, pack(">I", len(pv)), pv, sv]) l.append(stlen) @@ -441,7 +441,7 @@ while 1: f.seek(pos) h=read(27) - + if len(h)==27 and h[8] in 'vni': tlen, vlen, dlen = unpack(">iHi", h[9:19]) else: tlen=-1 @@ -470,12 +470,12 @@ # We have a record for this oid, but it was invalidated! del serial[oid] del index[oid] - - + + pos=pos+tlen f.seek(pos) try: f.truncate() except: pass - + return pos === ZEO/ZEO/ClientStorage.py 1.35.6.5 => 1.35.6.6 === -# +# # Zope Public License (ZPL) Version 1.0 # ------------------------------------- -# +# # Copyright (c) Digital Creations. All rights reserved. -# +# # This license has been certified as Open Source(tm). -# +# # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: -# +# # 1. Redistributions in source code must retain the above copyright # notice, this list of conditions, and the following disclaimer. -# +# # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions, and the following disclaimer in # the documentation and/or other materials provided with the # distribution. -# +# # 3. Digital Creations requests that attribution be given to Zope # in any manner possible. Zope includes a "Powered by Zope" # button that is installed by default. While it is not a license @@ -26,43 +26,43 @@ # attribution remain. A significant investment has been put # into Zope, and this effort will continue if the Zope community # continues to grow. This is one way to assure that growth. -# +# # 4. All advertising materials and documentation mentioning # features derived from or use of this software must display # the following acknowledgement: -# +# # "This product includes software developed by Digital Creations # for use in the Z Object Publishing Environment # (http://www.zope.org/)." -# +# # In the event that the product being advertised includes an # intact Zope distribution (with copyright and license included) # then this clause is waived. -# +# # 5. Names associated with Zope or Digital Creations must not be used to # endorse or promote products derived from this software without # prior written permission from Digital Creations. -# +# # 6. Modified redistributions of any form whatsoever must retain # the following acknowledgment: -# +# # "This product includes software developed by Digital Creations # for use in the Z Object Publishing Environment # (http://www.zope.org/)." -# +# # Intact (re-)distributions of any official Zope release do not # require an external acknowledgement. -# +# # 7. Modifications are encouraged but must be packaged separately as # patches to official Zope releases. Distributions that do not # clearly separate the patches from the original work must be clearly # labeled as unofficial distributions. Modifications which do not # carry the name Zope may be packaged in any form, as long as they # conform to all of the clauses above. -# -# +# +# # Disclaimer -# +# # THIS SOFTWARE IS PROVIDED BY DIGITAL CREATIONS ``AS IS'' AND ANY # EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR @@ -75,12 +75,12 @@ # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT # OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF # SUCH DAMAGE. -# -# +# +# # This software consists of contributions made by Digital Creations and # many individuals on behalf of Digital Creations. Specific # attributions are listed in the accompanying credits file. -# +# ############################################################################## """Network ZODB storage client """ @@ -192,7 +192,7 @@ # Prevent multiple new_oid calls from going out. The _oids # variable should only be modified while holding the - # oid_cond. + # oid_cond. self.oid_cond = threading.Condition() commit_lock = threading.Lock() @@ -207,7 +207,7 @@ self._rpc_mgr.close() if self._cache is not None: self._cache.close() - + def registerDB(self, db, limit): """Register that the storage is controlled by the given DB.""" log2(INFO, "registerDB(%s, %s)" % (repr(db), repr(limit))) @@ -263,10 +263,10 @@ def getSize(self): return self._info['size'] - + def supportsUndo(self): return self._info['supportsUndo'] - + def supportsVersions(self): return self._info['supportsVersions'] @@ -286,7 +286,7 @@ else: raise exc(self._transaction, trans) return 1 - + def _check_tid(self, tid, exc=None): if self.tpc_tid != tid: if exc is None: @@ -309,7 +309,7 @@ self._rpc_mgr.close() if self._cache is not None: self._cache.close() - + def commitVersion(self, src, dest, transaction): if self._is_read_only: raise POSException.ReadOnlyError() @@ -327,10 +327,10 @@ return oids def history(self, oid, version, length=1): - return self._server.history(oid, version, length) - + return self._server.history(oid, version, length) + def loadSerial(self, oid, serial): - return self._server.loadSerial(oid, serial) + return self._server.loadSerial(oid, serial) def load(self, oid, version, _stuff=None): p = self._cache.load(oid, version) @@ -347,7 +347,7 @@ if s: return p, s raise KeyError, oid # no non-version data for this - + def modifiedInVersion(self, oid): v = self._cache.modifiedInVersion(oid) if v is not None: @@ -366,7 +366,7 @@ oid = self._oids.pop() self.oid_cond.release() return oid - + def pack(self, t=None, rf=None, wait=0, days=0): if self._is_read_only: raise POSException.ReadOnlyError() @@ -391,7 +391,7 @@ if self._is_read_only: raise POSException.ReadOnlyError() self._check_trans(transaction, POSException.StorageTransactionError) - self._server.storea(oid, serial, data, version, self._serial) + self._server.storea(oid, serial, data, version, self._serial) self._tbuf.store(oid, version, data) return self._check_serials() @@ -400,7 +400,7 @@ return self._server.vote(self._serial) return self._check_serials() - + def tpc_abort(self, transaction): if transaction is not self._transaction: return @@ -423,7 +423,7 @@ if self._server is None: self.tpc_cond.release() raise ClientDisconnected() - + self._ts = get_timestamp(self._ts) id = `self._ts` self._transaction = transaction @@ -464,7 +464,7 @@ def _update_cache(self): # Iterate over the objects in the transaction buffer and - # update or invalidate the cache. + # update or invalidate the cache. self._cache.checkSize(self._tbuf.get_size()) self._tbuf.begin_iterate() while 1: @@ -477,7 +477,7 @@ if t is None: break oid, v, p = t - if p is None: # an invalidation + if p is None: # an invalidation s = None else: s = self._seriald[oid] @@ -502,7 +502,7 @@ # XXX what are the sync issues here? oids = self._server.undo(transaction_id) for oid in oids: - self._cache.invalidate(oid, '') + self._cache.invalidate(oid, '') return oids def undoInfo(self, first=0, last=-20, specification=None): @@ -511,7 +511,7 @@ def undoLog(self, first, last, filter=None): if filter is not None: return () # can't pass a filter to server - + return self._server.undoLog(first, last) # Eek! def versionEmpty(self, version): === ZEO/ZEO/ClientStub.py 1.3.2.2 => 1.3.2.3 === def __init__(self, rpc): self.rpc = rpc - + def beginVerify(self): self.rpc.callAsync('begin') === ZEO/ZEO/Invalidator.py 1.6.6.1 => 1.6.6.2 === _d=None - + def __init__(self, dinvalidate, cinvalidate): self.dinvalidate=dinvalidate self.cinvalidate=cinvalidate === ZEO/ZEO/ServerStub.py 1.3.2.2 => 1.3.2.3 === else: return self.rpc.call('new_oids', n) - + def pack(self, t, wait=None): if wait is None: self.rpc.call('pack', t) @@ -58,7 +58,7 @@ def commitVersion(self, src, dest, id): return self.rpc.call('commitVersion', src, dest, id) - + def history(self, oid, version, length=None): if length is not None: return self.rpc.call('history', oid, version) @@ -104,5 +104,3 @@ return self.rpc.call('versions') else: return self.rpc.call('versions', max) - - === ZEO/ZEO/StorageServer.py 1.32.6.5 => 1.32.6.6 === -# +# # Zope Public License (ZPL) Version 1.0 # ------------------------------------- -# +# # Copyright (c) Digital Creations. All rights reserved. -# +# # This license has been certified as Open Source(tm). -# +# # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: -# +# # 1. Redistributions in source code must retain the above copyright # notice, this list of conditions, and the following disclaimer. -# +# # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions, and the following disclaimer in # the documentation and/or other materials provided with the # distribution. -# +# # 3. Digital Creations requests that attribution be given to Zope # in any manner possible. Zope includes a "Powered by Zope" # button that is installed by default. While it is not a license @@ -26,33 +26,33 @@ # attribution remain. A significant investment has been put # into Zope, and this effort will continue if the Zope community # continues to grow. This is one way to assure that growth. -# +# # 4. All advertising materials and documentation mentioning # features derived from or use of this software must display # the following acknowledgement: -# +# # "This product includes software developed by Digital Creations # for use in the Z Object Publishing Environment # (http://www.zope.org/)." -# +# # In the event that the product being advertised includes an # intact Zope distribution (with copyright and license included) # then this clause is waived. -# +# # 5. Names associated with Zope or Digital Creations must not be used to # endorse or promote products derived from this software without # prior written permission from Digital Creations. -# +# # 6. Modified redistributions of any form whatsoever must retain # the following acknowledgment: -# +# # "This product includes software developed by Digital Creations # for use in the Z Object Publishing Environment # (http://www.zope.org/)." -# +# # Intact (re-)distributions of any official Zope release do not # require an external acknowledgement. -# +# # 7. Modifications are encouraged but must be packaged separately as # patches to official Zope releases. Distributions that do not # clearly separate the patches from the original work must be clearly @@ -60,9 +60,9 @@ # carry the name Zope may be packaged in any form, as long as they # conform to all of the clauses above. # -# +# # Disclaimer -# +# # THIS SOFTWARE IS PROVIDED BY DIGITAL CREATIONS ``AS IS'' AND ANY # EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR @@ -75,12 +75,12 @@ # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT # OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF # SUCH DAMAGE. -# -# +# +# # This software consists of contributions made by Digital Creations and # many individuals on behalf of Digital Creations. Specific # attributions are listed in the accompanying credits file. -# +# ############################################################################## """Network ZODB storage server @@ -135,7 +135,7 @@ c = ManagedServerConnection(sock, addr, ZEOStorage(self), self) log("new connection %s: %s" % (addr, `c`)) return c - + def register(self, storage_id, proxy): """Register a connection's use with a particular storage. @@ -188,7 +188,7 @@ def notifyConnected(self, conn): self.client = ClientStub.ClientStorage(conn) - + def __repr__(self): tid = self._transaction and repr(self._transaction.id) if self.__storage: @@ -303,7 +303,7 @@ t.start() def _pack(self, t, wait=0): - try: + try: self.__storage.pack(t, referencesf) except: self._log('ZEO Server', zLOG.ERROR, @@ -380,7 +380,7 @@ def transactionalUndo(self, trans_id, id): self._check_tid(id, exc=StorageTransactionError) return self.__storage.transactionalUndo(trans_id, self._transaction) - + def undo(self, transaction_id): oids = self.__storage.undo(transaction_id) if oids: @@ -464,7 +464,7 @@ proxy._restart_delayed_transaction(delay, trans, tid, status) if self is proxy: return 1 - + def new_oids(self, n=100): """Return a sequence of n new oids, where n defaults to 100""" if n < 0: === ZEO/ZEO/TransactionBuffer.py 1.3.2.2 => 1.3.2.3 === class TransactionBuffer: - + def __init__(self): self.file = tempfile.TemporaryFile() self.count = 0 @@ -59,8 +59,8 @@ oid_ver_data = self.unpickler.load() self.count -= 1 return oid_ver_data - + def get_size(self): """Return size of data stored in buffer (just a hint).""" - + return self.size === ZEO/ZEO/asyncwrap.py 1.2.4.1 => 1.2.4.2 === else: break - + def poll(*args, **kwargs): try: apply(asyncore.poll, args, kwargs) === ZEO/ZEO/fap.py 1.5.6.1 => 1.5.6.2 === except: return 0 else: return 1 - + def fap(): # if we are using an old version of Python, our asyncore is likely to @@ -33,12 +33,12 @@ except: # Try a little harder to import ZServer import os, imp - + location = package_home() location = os.path.split(location)[0] location = os.path.split(location)[0] location = os.path.split(location)[0] - + if whiff(location): sys.path.append(location) try: === ZEO/ZEO/smac.py 1.11.4.4 => 1.11.4.5 === -# +# # Zope Public License (ZPL) Version 1.0 # ------------------------------------- -# +# # Copyright (c) Digital Creations. All rights reserved. -# +# # This license has been certified as Open Source(tm). -# +# # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: -# +# # 1. Redistributions in source code must retain the above copyright # notice, this list of conditions, and the following disclaimer. -# +# # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions, and the following disclaimer in # the documentation and/or other materials provided with the # distribution. -# +# # 3. Digital Creations requests that attribution be given to Zope # in any manner possible. Zope includes a "Powered by Zope" # button that is installed by default. While it is not a license @@ -26,43 +26,43 @@ # attribution remain. A significant investment has been put # into Zope, and this effort will continue if the Zope community # continues to grow. This is one way to assure that growth. -# +# # 4. All advertising materials and documentation mentioning # features derived from or use of this software must display # the following acknowledgement: -# +# # "This product includes software developed by Digital Creations # for use in the Z Object Publishing Environment # (http://www.zope.org/)." -# +# # In the event that the product being advertised includes an # intact Zope distribution (with copyright and license included) # then this clause is waived. -# +# # 5. Names associated with Zope or Digital Creations must not be used to # endorse or promote products derived from this software without # prior written permission from Digital Creations. -# +# # 6. Modified redistributions of any form whatsoever must retain # the following acknowledgment: -# +# # "This product includes software developed by Digital Creations # for use in the Z Object Publishing Environment # (http://www.zope.org/)." -# +# # Intact (re-)distributions of any official Zope release do not # require an external acknowledgement. -# +# # 7. Modifications are encouraged but must be packaged separately as # patches to official Zope releases. Distributions that do not # clearly separate the patches from the original work must be clearly # labeled as unofficial distributions. Modifications which do not # carry the name Zope may be packaged in any form, as long as they # conform to all of the clauses above. -# -# +# +# # Disclaimer -# +# # THIS SOFTWARE IS PROVIDED BY DIGITAL CREATIONS ``AS IS'' AND ANY # EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR @@ -75,12 +75,12 @@ # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT # OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF # SUCH DAMAGE. -# -# +# +# # This software consists of contributions made by Digital Creations and # many individuals on behalf of Digital Creations. Specific # attributions are listed in the accompanying credits file. -# +# ############################################################################## """Sized message async connections """ @@ -198,7 +198,7 @@ def readable(self): return 1 - + def writable(self): if len(self.__output) == 0: return 0 === ZEO/ZEO/start.py 1.26.4.2 => 1.26.4.3 === -# +# # Zope Public License (ZPL) Version 1.0 # ------------------------------------- -# +# # Copyright (c) Digital Creations. All rights reserved. -# +# # This license has been certified as Open Source(tm). -# +# # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: -# +# # 1. Redistributions in source code must retain the above copyright # notice, this list of conditions, and the following disclaimer. -# +# # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions, and the following disclaimer in # the documentation and/or other materials provided with the # distribution. -# +# # 3. Digital Creations requests that attribution be given to Zope # in any manner possible. Zope includes a "Powered by Zope" # button that is installed by default. While it is not a license @@ -26,43 +26,43 @@ # attribution remain. A significant investment has been put # into Zope, and this effort will continue if the Zope community # continues to grow. This is one way to assure that growth. -# +# # 4. All advertising materials and documentation mentioning # features derived from or use of this software must display # the following acknowledgement: -# +# # "This product includes software developed by Digital Creations # for use in the Z Object Publishing Environment # (http://www.zope.org/)." -# +# # In the event that the product being advertised includes an # intact Zope distribution (with copyright and license included) # then this clause is waived. -# +# # 5. Names associated with Zope or Digital Creations must not be used to # endorse or promote products derived from this software without # prior written permission from Digital Creations. -# +# # 6. Modified redistributions of any form whatsoever must retain # the following acknowledgment: -# +# # "This product includes software developed by Digital Creations # for use in the Z Object Publishing Environment # (http://www.zope.org/)." -# +# # Intact (re-)distributions of any official Zope release do not # require an external acknowledgement. -# +# # 7. Modifications are encouraged but must be packaged separately as # patches to official Zope releases. Distributions that do not # clearly separate the patches from the original work must be clearly # labeled as unofficial distributions. Modifications which do not # carry the name Zope may be packaged in any form, as long as they # conform to all of the clauses above. -# -# +# +# # Disclaimer -# +# # THIS SOFTWARE IS PROVIDED BY DIGITAL CREATIONS ``AS IS'' AND ANY # EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR @@ -75,12 +75,12 @@ # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT # OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF # SUCH DAMAGE. -# -# +# +# # This software consists of contributions made by Digital Creations and # many individuals on behalf of Digital Creations. Specific # attributions are listed in the accompanying credits file. -# +# ############################################################################## """Start the server storage. @@ -99,7 +99,7 @@ d=os.path.split(d)[0] if not d or d=='.': d=os.getcwd() n=n-1 - + return d def get_storage(m, n, cache={}): @@ -150,7 +150,7 @@ -D -- Run in debug mode -U -- Unix-domain socket file to listen on - + -u username or uid number The username to run the ZEO server as. You may want to run @@ -191,7 +191,7 @@ print usage print msg sys.exit(1) - + port=None debug=0 host='' @@ -234,7 +234,7 @@ from zLOG import LOG, INFO, ERROR # Try to set uid to "-u" -provided uid. - # Try to set gid to "-u" user's primary group. + # Try to set gid to "-u" user's primary group. # This will only work if this script is run by root. try: import pwd @@ -249,7 +249,7 @@ uid = pwd.getpwuid(UID)[2] gid = pwd.getpwuid(UID)[3] else: - raise KeyError + raise KeyError try: if gid is not None: try: @@ -355,7 +355,7 @@ "Shutting down (%s)" % (die and "shutdown" or "restart") ) except: pass - + if die: sys.exit(0) else: sys.exit(1) === ZEO/ZEO/trigger.py 1.3.4.2 => 1.3.4.3 === import string import thread - + if os.name == 'posix': class trigger (asyncore.file_dispatcher): @@ -138,7 +138,7 @@ if port <= 19950: raise 'Bind Error', 'Cannot bind trigger!' port=port - 1 - + a.listen (1) w.setblocking (0) try: === ZEO/ZEO/zrpc.py 1.20.4.1 => 1.20.4.2 === # Flag indicating whether a main loop is running. If one isn't running, # then we'll have to provide our own main loop at times. - __haveMainLoop=0 + __haveMainLoop=0 def __Wakeup(*args): pass def __init__(self, connection, outOfBand=None, tmin=5, tmax=300, debug=0): @@ -69,7 +69,7 @@ s=socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) else: s=socket.socket(socket.AF_INET, socket.SOCK_STREAM) - s.connect(connection) + s.connect(connection) except Exception, err: if debug: LOG(debug, DEBUG, "Failed to connect to server: %s" % err) @@ -80,13 +80,13 @@ else: if debug: LOG(debug, DEBUG, "Connected to server") - + # Make sure the result lock is set, se we don't # get an old result (e.g. the exception that # we generated on close). self.__r=None self.__la(0) - + self.aq_parent.notifyConnected(s) return 1 @@ -131,7 +131,7 @@ self.__Wakeup=Wakeup - + def __call__(self, *args): self.__call_la() try: @@ -148,7 +148,7 @@ if c=='R': if r=='RN.': return None # Common case! return loads(r[1:]) - + # If c == 'E', an error occured on the server. In # this case, the return value is a pickled exception. # Unpickle it and raise it on the client side. The @@ -162,7 +162,7 @@ raise UnUnPickleableError(r[1:]) if type(r) is TupleType: raise r[0], r[1] # see server log for real traceback - raise r + raise r oob=self._outOfBand if oob is not None: r=r[1:] @@ -227,12 +227,12 @@ if self.__haveMainLoop: # We aren't willing to close until told to by the main loop. # So we'll tell the main loop to tell us. :) - self.__Wakeup(lambda self=self: self.close()) + self.__Wakeup(lambda self=self: self.close()) else: self.close() self._outOfBand = None self.__closed = 1 - + def close(self): asyncRPC.inheritedAttribute('close')(self) self.aq_parent.notifyDisconnected(self) From jeremy at zope.com Sat Jan 26 13:25:52 2002 From: jeremy at zope.com (Jeremy Hylton) Date: Sun Aug 10 16:31:18 2008 Subject: [ZEO-Checkins] CVS: ZEO/ZEO/zrpc - client.py:1.1.2.6 Message-ID: <200201261825.g0QIPqJ01702@cvs.baymountain.com> Update of /cvs-repository/ZEO/ZEO/zrpc In directory cvs.zope.org:/tmp/cvs-serv1691 Modified Files: Tag: Standby-branch client.py Log Message: In connect() method, treat EISCONN as success. Also rename ConnectThread's attr "addr" to "addrs", because it is a sequence. === ZEO/ZEO/zrpc/client.py 1.1.2.5 => 1.1.2.6 === # than close() and those defined by the Thread API. - def __init__(self, mgr, client, addr, tmin, tmax): - self.__super_init(name="Connect(%s)" % addr) + def __init__(self, mgr, client, addrs, tmin, tmax): + self.__super_init(name="Connect(%s)" % addrs) self.mgr = mgr self.client = client - self.addr = addr + self.addrs = addrs self.tmin = tmin self.tmax = tmax self.stopped = 0 @@ -203,9 +203,9 @@ "Return true if any connect attempt succeeds." self.sockets = {} - log("attempting connection on %d sockets" % len(self.addr)) + log("attempting connection on %d sockets" % len(self.addrs)) try: - for domain, addr in self.addr: + for domain, addr in self.addrs: if __debug__: log("attempt connection to %s" % repr(addr), level=zLOG.DEBUG) @@ -256,7 +256,7 @@ e = s.connect_ex(addr) if e == errno.EINPROGRESS: return 1 - elif e == 0: + elif e == 0 or e == errno.EISCONN: c = self.test_connection(s, addr) log("connected to %s" % repr(addr), level=zLOG.DEBUG) if c: From tim.one at home.com Sat Jan 26 13:37:31 2002 From: tim.one at home.com (Tim Peters) Date: Sun Aug 10 16:31:18 2008 Subject: [ZEO-Checkins] CVS: ZEO/ZEO/zrpc - client.py:1.1.2.7 Message-ID: <200201261837.g0QIbVQ04612@cvs.baymountain.com> Update of /cvs-repository/ZEO/ZEO/zrpc In directory cvs.zope.org:/tmp/cvs-serv4531/zrpc Modified Files: Tag: Standby-branch client.py Log Message: ConnectThread.connect(): add enough errno smarts so that checkWriteClient passes on Windows too. === ZEO/ZEO/zrpc/client.py 1.1.2.6 => 1.1.2.7 === self.sock = sock +# When trying to do a connect on a non-blocking socket, some outcomes +# are expected. Set _CONNECT_IN_PROGRESS to the errno value(s) expected +# when an initial connect can't complete immediately. Set _CONNECT_OK +# to the errno value(s) expected if the connect succeeds *or* if it's +# already connected (our code can attempt redundant connects). +if hasattr(errno, "WSAEWOULDBLOCK"): # Windows + _CONNECT_IN_PROGRESS = (errno.WSAEWOULDBLOCK,) + _CONNECT_OK = (0, errno.WSAEISCONN) +else: # Unix + _CONNECT_IN_PROGRESS = (errno.EINPROGRESS,) + _CONNECT_OK = (0, errno.EISCONN) + class ConnectThread(threading.Thread): """Thread that tries to connect to server given one or more addresses. @@ -213,8 +225,6 @@ s.setblocking(0) self.sockets[s] = addr # XXX can still block for a while if addr requires DNS - # YYY What is that XXX comment trying to say? self.connect - # YYY explicitly tolerates EINPROGRESS. e = self.connect(s) # next wait until they actually connect @@ -254,9 +264,9 @@ """ addr = self.sockets[s] e = s.connect_ex(addr) - if e == errno.EINPROGRESS: + if e in _CONNECT_IN_PROGRESS: return 1 - elif e == 0 or e == errno.EISCONN: + elif e in _CONNECT_OK: c = self.test_connection(s, addr) log("connected to %s" % repr(addr), level=zLOG.DEBUG) if c: From guido at python.org Sat Jan 26 16:02:18 2002 From: guido at python.org (Guido van Rossum) Date: Sun Aug 10 16:31:18 2008 Subject: [ZEO-Checkins] CVS: ZEO/ZEO/tests - testZEO.py:1.16.4.8 Message-ID: <200201262102.g0QL2If10512@cvs.baymountain.com> Update of /cvs-repository/ZEO/ZEO/tests In directory cvs.zope.org:/tmp/cvs-serv10501 Modified Files: Tag: Standby-branch testZEO.py Log Message: Be a little more informative in the zLOG message in setUp: mention the test name, and raise priority to INFO (every test spits out several messages in INFO mode anyway). === ZEO/ZEO/tests/testZEO.py 1.16.4.7 => 1.16.4.8 === def setUp(self): - zLOG.LOG("testZEO", zLOG.BLATHER, "setUp() new test") + zLOG.LOG("testZEO", zLOG.INFO, "setUp() %s" % self.id()) self.running = 1 client, exit, pid = forker.start_zeo(self.getStorage()) self._pids = [pid] From tim.one at home.com Sun Jan 27 14:09:13 2002 From: tim.one at home.com (Tim Peters) Date: Sun Aug 10 16:31:18 2008 Subject: [ZEO-Checkins] CVS: ZEO/ZEO/zrpc - client.py:1.1.2.8 Message-ID: <200201271909.g0RJ9D403604@cvs.baymountain.com> Update of /cvs-repository/ZEO/ZEO/zrpc In directory cvs.zope.org:/tmp/cvs-serv3077/zrpc Modified Files: Tag: Standby-branch client.py Log Message: XXX comments for Jeremy. Minor docstring repair and code cleanup. === ZEO/ZEO/zrpc/client.py 1.1.2.7 => 1.1.2.8 === def attempt_connects(self): - "Return true if any connect attempt succeeds." - self.sockets = {} + """Try connecting to all self.addrs addresses. + + If at least one succeeds, pick a success arbitrarily, close all other + successes (if any), and return true. If none succeed, return false. + """ + + self.sockets = {} # {open socket: connection address} log("attempting connection on %d sockets" % len(self.addrs)) try: @@ -224,8 +229,9 @@ s = socket.socket(domain, socket.SOCK_STREAM) s.setblocking(0) self.sockets[s] = addr + # connect() raises Connected iff it succeeds # XXX can still block for a while if addr requires DNS - e = self.connect(s) + self.connect(s) # next wait until they actually connect while self.sockets: @@ -237,8 +243,8 @@ except select.error: continue for s in w: - # connect() will raise Connected if it succeeds - e = self.connect(s) + # connect() raises Connected iff it succeeds + self.connect(s) except Connected, container: s = container.sock del self.sockets[s] # don't close the newly connected socket @@ -247,7 +253,7 @@ return 0 def connect(self, s): - """Call s.connect_ex(addr) and return true if loop should continue. + """Call s.connect_ex(addr); raise Connected iff connection succeeds. We have to handle several possible return values from connect_ex(). If the socket is connected and the initial ZEO @@ -258,6 +264,9 @@ If the socket sonnects and the initial ZEO setup fails or the connect_ex() returns an error, we close the socket and ignore it. + XXX If test_connection() fails (I assume that's what's meant by + XXX "the initial ZEO setup fails", the code below doesn't do anything + XXX with the socket. So the comment or the code is wrong. When the socket is closed, it is removed from self.sockets. If connect_ex() returns EINPROGRESS, we need to try again later. @@ -265,7 +274,7 @@ addr = self.sockets[s] e = s.connect_ex(addr) if e in _CONNECT_IN_PROGRESS: - return 1 + pass elif e in _CONNECT_OK: c = self.test_connection(s, addr) log("connected to %s" % repr(addr), level=zLOG.DEBUG) @@ -286,6 +295,9 @@ log("error connecting to server: %s" % str(addr), level=zLOG.ERROR, error=sys.exc_info()) c.close() + # XXX what's the state of s at this point? If s got closed, + # XXX self.sockets contains a closed socket, and the + # XXX next attempt_connects() select() will blow up. return 0 self.mgr.connect_done(c) return 1 From tim.one at home.com Sun Jan 27 14:19:48 2002 From: tim.one at home.com (Tim Peters) Date: Sun Aug 10 16:31:18 2008 Subject: [ZEO-Checkins] CVS: ZEO/ZEO/zrpc - client.py:1.1.2.9 Message-ID: <200201271919.g0RJJmk08368@cvs.baymountain.com> Update of /cvs-repository/ZEO/ZEO/zrpc In directory cvs.zope.org:/tmp/cvs-serv8115/zrpc Modified Files: Tag: Standby-branch client.py Log Message: Another XXX: the log says we're connected even if test_connection fails. Doesn't seem right. === ZEO/ZEO/zrpc/client.py 1.1.2.8 => 1.1.2.9 === elif e in _CONNECT_OK: c = self.test_connection(s, addr) + # XXX we should't log that we got connected if c==0, should we? log("connected to %s" % repr(addr), level=zLOG.DEBUG) if c: raise Connected(s) From tim.one at home.com Sun Jan 27 15:31:53 2002 From: tim.one at home.com (Tim Peters) Date: Sun Aug 10 16:31:19 2008 Subject: [ZEO-Checkins] CVS: ZEO/ZEO - ClientCache.py:1.18.6.5 Message-ID: <200201272031.g0RKVrQ09329@cvs.baymountain.com> Update of /cvs-repository/ZEO/ZEO In directory cvs.zope.org:/tmp/cvs-serv9214 Modified Files: Tag: Standby-branch ClientCache.py Log Message: Added whitespace throughout so that (a) I can read this, and (b) can insert debugging prints. === ZEO/ZEO/ClientCache.py 1.18.6.4 => 1.18.6.5 === # Allocate locks: - l=allocate_lock() - self._acquire=l.acquire - self._release=l.release + L = allocate_lock() + self._acquire = L.acquire + self._release = L.release if client: # Create a persistent cache if var is None: - try: var=CLIENT_HOME + try: + var = CLIENT_HOME except: - try: var=os.path.join(INSTANCE_HOME,'var') - except: var=os.getcwd() + try: + var = os.path.join(INSTANCE_HOME, 'var') + except: + var = os.getcwd() # Get the list of cache file names - self._p=p=map(lambda i, p=storage, var=var, c=client: - os.path.join(var,'c%s-%s-%s.zec' % (p, c, i)), - (0,1)) + self._p = p = map(lambda i, p=storage, var=var, c=client: + os.path.join(var, 'c%s-%s-%s.zec' % (p, c, i)), + (0, 1)) # get the list of cache files - self._f=f=[None, None] + self._f = f = [None, None] # initialize cache serial numbers s=['\0\0\0\0\0\0\0\0', '\0\0\0\0\0\0\0\0'] - for i in 0,1: + for i in 0, 1: if os.path.exists(p[i]): - fi=open(p[i],'r+b') - if fi.read(4)==magic: # Minimal sanity - fi.seek(0,2) + fi = open(p[i],'r+b') + if fi.read(4) == magic: # Minimal sanity + fi.seek(0, 2) if fi.tell() > 30: fi.seek(22) - s[i]=fi.read(8) + s[i] = fi.read(8) # If we found a non-zero serial, then use the file - if s[i] != '\0\0\0\0\0\0\0\0': f[i]=fi - fi=None + if s[i] != '\0\0\0\0\0\0\0\0': + f[i] = fi + fi = None # Whoever has the larger serial is the current - if s[1] > s[0]: current=1 - elif s[0] > s[1]: current=0 + if s[1] > s[0]: + current = 1 + elif s[0] > s[1]: + current = 0 else: if f[0] is None: # We started, open the first cache file - f[0]=open(p[0], 'w+b') + f[0] = open(p[0], 'w+b') f[0].write(magic) - current=0 - f[1]=None + current = 0 + f[1] = None else: self._f = f = [tempfile.TemporaryFile(suffix='.zec'), None] - # self._p file names 'None' signifies unnamed temp files. + # self._p file name 'None' signifies an unnamed temp file. self._p = p = [None, None] f[0].write(magic) - current=0 + current = 0 log("cache opened. current = %s" % current) - self._limit=size/2 - self._current=current + self._limit = size / 2 + self._current = current def open(self): # XXX open is overloaded to perform two tasks for @@ -224,16 +230,17 @@ self._acquire() try: self._index=index={} - self._get=index.get - serial={} - f=self._f - current=self._current + self._get = index.get + serial = {} + f = self._f + current = self._current if f[not current] is not None: read_index(index, serial, f[not current], not current) - self._pos=read_index(index, serial, f[current], current) + self._pos = read_index(index, serial, f[current], current) return serial.items() - finally: self._release() + finally: + self._release() def close(self): for f in self._f: @@ -251,51 +258,60 @@ def invalidate(self, oid, version): self._acquire() try: - p=self._get(oid, None) - if p is None: return None - f=self._f[p < 0] - ap=abs(p) + p = self._get(oid, None) + if p is None: + return None + f = self._f[p < 0] + ap = abs(p) f.seek(ap) - h=f.read(8) - if h != oid: return - f.seek(8,1) # Dang, we shouldn't have to do this. Bad Solaris & NT + h = f.read(8) + if h != oid: + return + f.seek(8, 1) # Dang, we shouldn't have to do this. Bad Solaris & NT if version: f.write('n') else: del self._index[oid] f.write('i') - finally: self._release() + finally: + self._release() def load(self, oid, version): self._acquire() try: - p=self._get(oid, None) - if p is None: return None - f=self._f[p < 0] - ap=abs(p) - seek=f.seek - read=f.read + p = self._get(oid, None) + if p is None: + return None + f = self._f[p < 0] + ap = abs(p) + seek = f.seek + read = f.read seek(ap) - h=read(27) + h = read(27) if len(h)==27 and h[8] in 'nv' and h[:8]==oid: tlen, vlen, dlen = unpack(">iHi", h[9:19]) - else: tlen=-1 + else: + tlen = -1 if tlen <= 0 or vlen < 0 or dlen < 0 or vlen+dlen > tlen: del self._index[oid] return None if h[8]=='n': - if version: return None + if version: + return None if not dlen: del self._index[oid] return None if not vlen or not version: - if dlen: return read(dlen), h[19:] - else: return None + if dlen: + return read(dlen), h[19:] + else: + return None - if dlen: seek(dlen, 1) - v=read(vlen) + if dlen: + seek(dlen, 1) + v = read(vlen) if version != v: if dlen: seek(-dlen-vlen, 1) @@ -303,24 +319,25 @@ else: return None - dlen=unpack(">i", read(4))[0] + dlen = unpack(">i", read(4))[0] return read(dlen), read(8) - finally: self._release() + finally: + self._release() def update(self, oid, serial, version, data): self._acquire() try: if version: # We need to find and include non-version data - p=self._get(oid, None) + p = self._get(oid, None) if p is None: return self._store(oid, '', '', version, data, serial) - f=self._f[p < 0] - ap=abs(p) - seek=f.seek - read=f.read + f = self._f[p < 0] + ap = abs(p) + seek = f.seek + read = f.read seek(ap) - h=read(27) + h = read(27) if len(h)==27 and h[8] in 'nv' and h[:8]==oid: tlen, vlen, dlen = unpack(">iHi", h[9:19]) else: @@ -330,8 +347,8 @@ return self._store(oid, '', '', version, data, serial) if dlen: - p=read(dlen) - s=h[19:] + p = read(dlen) + s = h[19:] else: return self._store(oid, '', '', version, data, serial) @@ -339,41 +356,47 @@ else: # Simple case, just store new data: self._store(oid, data, serial, '', None, None) - finally: self._release() + finally: + self._release() def modifiedInVersion(self, oid): self._acquire() try: - p=self._get(oid, None) - if p is None: return None - f=self._f[p < 0] - ap=abs(p) - seek=f.seek - read=f.read + p = self._get(oid, None) + if p is None: + return None + f = self._f[p < 0] + ap = abs(p) + seek = f.seek + read = f.read seek(ap) - h=read(27) + h = read(27) if len(h)==27 and h[8] in 'nv' and h[:8]==oid: tlen, vlen, dlen = unpack(">iHi", h[9:19]) - else: tlen=-1 + else: + tlen = -1 if tlen <= 0 or vlen < 0 or dlen < 0 or vlen+dlen > tlen: del self._index[oid] return None - if h[8]=='n': return None + if h[8] == 'n': + return None - if not vlen: return '' + if not vlen: + return '' seek(dlen, 1) return read(vlen) - finally: self._release() + finally: + self._release() def checkSize(self, size): self._acquire() try: # Make sure we aren't going to exceed the target size. # If we are, then flip the cache. - if self._pos+size > self._limit: - current=not self._current - self._current=current + if self._pos + size > self._limit: + current = not self._current + self._current = current if self._p[current] is not None: # Persistent cache file: # Note that due to permission madness, waaa, @@ -381,15 +404,18 @@ # we open the new one. Waaaaaaaaaa. if self._f[current] is not None: self._f[current].close() - try: os.remove(self._p[current]) - except: pass - self._f[current]=open(self._p[current],'w+b') + try: + os.remove(self._p[current]) + except: + pass + self._f[current] = open(self._p[current],'w+b') else: # Temporary cache file: self._f[current] = tempfile.TemporaryFile(suffix='.zec') self._f[current].write(magic) - self._pos=pos=4 - finally: self._release() + self._pos = pos = 4 + finally: + self._release() def store(self, oid, p, s, version, pv, sv): @@ -432,39 +458,46 @@ self._pos += tlen def read_index(index, serial, f, current): - seek=f.seek - read=f.read - pos=4 - seek(0,2) - size=f.tell() + seek = f.seek + read = f.read + pos = 4 + seek(0, 2) + size = f.tell() while 1: f.seek(pos) - h=read(27) + h = read(27) if len(h)==27 and h[8] in 'vni': tlen, vlen, dlen = unpack(">iHi", h[9:19]) - else: tlen=-1 - if tlen <= 0 or vlen < 0 or dlen < 0 or vlen+dlen > tlen: + else: + tlen = -1 + if tlen <= 0 or vlen < 0 or dlen < 0 or vlen + dlen > tlen: break - oid=h[:8] + oid = h[:8] if h[8]=='v' and vlen: seek(dlen+vlen, 1) - vdlen=read(4) - if len(vdlen) != 4: break - vdlen=unpack(">i", vdlen)[0] - if vlen+dlen+42+vdlen > tlen: break + vdlen = read(4) + if len(vdlen) != 4: + break + vdlen = unpack(">i", vdlen)[0] + if vlen+dlen+42+vdlen > tlen: + break seek(vdlen, 1) - vs=read(8) - if read(4) != h[9:13]: break - else: vs=None + vs = read(8) + if read(4) != h[9:13]: + break + else: + vs = None if h[8] in 'vn': - if current: index[oid]=-pos - else: index[oid]=pos - serial[oid]=h[-8:], vs + if current: + index[oid] = -pos + else: + index[oid] = pos + serial[oid] = h[-8:], vs else: if serial.has_key(oid): # We have a record for this oid, but it was invalidated! @@ -472,10 +505,12 @@ del index[oid] - pos=pos+tlen + pos = pos + tlen f.seek(pos) - try: f.truncate() - except: pass + try: + f.truncate() + except: + pass return pos From jeremy at zope.com Mon Jan 28 13:47:24 2002 From: jeremy at zope.com (Jeremy Hylton) Date: Sun Aug 10 16:31:19 2008 Subject: [ZEO-Checkins] CVS: ZEO/ZEO/zrpc - client.py:1.1.2.10 Message-ID: <200201281847.g0SIlOm10318@cvs.baymountain.com> Update of /cvs-repository/ZEO/ZEO/zrpc In directory cvs.zope.org:/tmp/cvs-serv10307 Modified Files: Tag: Standby-branch client.py Log Message: Fix several corner cases in the ConnectThread logic. Catch and log errors from more socket calls. It's possible for even connect_ex() to raise an exception, e.g. "host not found." Make sure that every path through connect() method does one of the following: - Returns without doing anything if connect_ex() returns EINPROGRESS or similar. - Raises a Connected() exception if the zrpc connection is established. - Closes the socket, removes it from self.sockets, and logs an error in all other cases. Address Tim's XXX comments. === ZEO/ZEO/zrpc/client.py 1.1.2.9 => 1.1.2.10 === log("attempt connection to %s" % repr(addr), level=zLOG.DEBUG) - s = socket.socket(domain, socket.SOCK_STREAM) + try: + s = socket.socket(domain, socket.SOCK_STREAM) + except socket.error, err: + log("Failed to create socket with domain=%s: %s" % ( + domain, err), level=zLOG.ERROR) + continue s.setblocking(0) self.sockets[s] = addr # connect() raises Connected iff it succeeds @@ -262,33 +267,42 @@ select() loop in the caller and an exception is a principled way to do the abort. - If the socket sonnects and the initial ZEO setup fails or the - connect_ex() returns an error, we close the socket and ignore it. - XXX If test_connection() fails (I assume that's what's meant by - XXX "the initial ZEO setup fails", the code below doesn't do anything - XXX with the socket. So the comment or the code is wrong. - When the socket is closed, it is removed from self.sockets. + If the socket sonnects and the initial ZEO setup + (notifyConnected()) fails or the connect_ex() returns an + error, we close the socket, remove it from self.sockets, and + proceed with the other sockets. If connect_ex() returns EINPROGRESS, we need to try again later. """ addr = self.sockets[s] - e = s.connect_ex(addr) - if e in _CONNECT_IN_PROGRESS: - pass - elif e in _CONNECT_OK: - c = self.test_connection(s, addr) - # XXX we should't log that we got connected if c==0, should we? - log("connected to %s" % repr(addr), level=zLOG.DEBUG) - if c: - raise Connected(s) + try: + e = s.connect_ex(addr) + except socket.error, msg: + log("failed to connect to %s: %s" % (addr, msg), + level=zLOG.ERROR) else: - if __debug__: + if e in _CONNECT_IN_PROGRESS: + return + elif e in _CONNECT_OK: + c = self.test_connection(s, addr) + if c: + log("connected to %s" % repr(addr), level=zLOG.DEBUG) + raise Connected(s) + else: log("error connecting to %s: %s" % (addr, errno.errorcode[e]), level=zLOG.DEBUG) - s.close() - del self.sockets[s] + # Any execution that doesn't raise Connected() or return + # because of CONNECT_IN_PROGRESS is an error. Make sure the + # socket is closed and remove it from the dict of pending + # sockets. + s.close() + del self.sockets[s] def test_connection(self, s, addr): + # Establish a connection at the zrpc level and call the + # client's notifyConnected(), giving the zrpc application a + # chance to do app-level check of whether the connection is + # okay. c = ManagedConnection(s, addr, self.client, self.mgr) try: self.client.notifyConnected(c) @@ -296,9 +310,8 @@ log("error connecting to server: %s" % str(addr), level=zLOG.ERROR, error=sys.exc_info()) c.close() - # XXX what's the state of s at this point? If s got closed, - # XXX self.sockets contains a closed socket, and the - # XXX next attempt_connects() select() will blow up. + # Closing the ZRPC connection will eventually close the + # socket, somewhere in asyncore. return 0 self.mgr.connect_done(c) return 1 From jeremy at zope.com Mon Jan 28 14:25:44 2002 From: jeremy at zope.com (Jeremy Hylton) Date: Sun Aug 10 16:31:19 2008 Subject: [ZEO-Checkins] CVS: ZEO/ZEO/tests - testZEO.py:1.16.4.9 Message-ID: <200201281925.g0SJPiY29285@cvs.baymountain.com> Update of /cvs-repository/ZEO/ZEO/tests In directory cvs.zope.org:/tmp/cvs-serv29274 Modified Files: Tag: Standby-branch testZEO.py Log Message: Add '.old' to the list of filestorage extensions to delete. === ZEO/ZEO/tests/testZEO.py 1.16.4.8 => 1.16.4.9 === def delStorage(self): # file storage appears to create four files - for ext in '', '.index', '.lock', '.tmp': + for ext in '', '.index', '.lock', '.tmp', '.old': path = self.__fs_base + ext try: os.remove(path) From jeremy at zope.com Mon Jan 28 16:56:51 2002 From: jeremy at zope.com (Jeremy Hylton) Date: Sun Aug 10 16:31:19 2008 Subject: [ZEO-Checkins] CVS: ZEO/ZEO - ClientStorage.py:1.35.6.7 Message-ID: <200201282156.g0SLupP01223@cvs.baymountain.com> Update of /cvs-repository/ZEO/ZEO In directory cvs.zope.org:/tmp/cvs-serv1204 Modified Files: Tag: Standby-branch ClientStorage.py Log Message: Make sure _tbuf is closed. Also add suffix to tbuf temp file, so you can see who to blame if it isn't closed. === ZEO/ZEO/ClientStorage.py 1.35.6.6 => 1.35.6.7 === def close(self): - self._rpc_mgr.close() + if self._tbuf is not None: + self._tbuf.close() if self._cache is not None: self._cache.close() + self._rpc_mgr.close() def registerDB(self, db, limit): """Register that the storage is controlled by the given DB.""" @@ -529,7 +531,7 @@ self._info.update(dict) def begin(self): - self._tfile = tempfile.TemporaryFile() + self._tfile = tempfile.TemporaryFile(suffix=".inv") self._pickler = cPickle.Pickler(self._tfile, 1) self._pickler.fast = 1 # Don't use the memo @@ -545,6 +547,7 @@ self._pickler.dump((0,0)) self._tfile.seek(0) unpick = cPickle.Unpickler(self._tfile) + f = self._tfile self._tfile = None while 1: @@ -553,6 +556,7 @@ break self._cache.invalidate(oid, version=version) self._db.invalidate(oid, version=version) + f.close() def Invalidate(self, args): for oid, version in args: From jeremy at zope.com Mon Jan 28 16:57:13 2002 From: jeremy at zope.com (Jeremy Hylton) Date: Sun Aug 10 16:31:19 2008 Subject: [ZEO-Checkins] CVS: ZEO/ZEO - Invalidator.py:NONE Message-ID: <200201282157.g0SLvDh01451@cvs.baymountain.com> Update of /cvs-repository/ZEO/ZEO In directory cvs.zope.org:/tmp/cvs-serv1432 Removed Files: Tag: Standby-branch Invalidator.py Log Message: Not used by ZEO2 === Removed File ZEO/ZEO/Invalidator.py === From jeremy at zope.com Mon Jan 28 16:58:24 2002 From: jeremy at zope.com (Jeremy Hylton) Date: Sun Aug 10 16:31:19 2008 Subject: [ZEO-Checkins] CVS: ZEO/ZEO - TransactionBuffer.py:1.3.2.4 Message-ID: <200201282158.g0SLwOB01998@cvs.baymountain.com> Update of /cvs-repository/ZEO/ZEO In directory cvs.zope.org:/tmp/cvs-serv1986 Modified Files: Tag: Standby-branch TransactionBuffer.py Log Message: Add close() method to TransactionBuffer. Give the TemporaryFile a suffix so we know who to blame. === ZEO/ZEO/TransactionBuffer.py 1.3.2.3 => 1.3.2.4 === def __init__(self): - self.file = tempfile.TemporaryFile() + self.file = tempfile.TemporaryFile(suffix=".tbuf") self.count = 0 self.size = 0 # It's safe to use a fast pickler because the only objects # stored are builtin types -- strings or None. self.pickler = cPickle.Pickler(self.file, 1) self.pickler.fast = 1 + + def close(self): + self.file.close() def store(self, oid, version, data): """Store oid, version, data for later retrieval""" From jeremy at zope.com Mon Jan 28 17:00:34 2002 From: jeremy at zope.com (Jeremy Hylton) Date: Sun Aug 10 16:31:19 2008 Subject: [ZEO-Checkins] CVS: ZEO/ZEO/tests - forker.py:1.10.4.6 testZEO.py:1.16.4.10 Message-ID: <200201282200.g0SM0Yw03232@cvs.baymountain.com> Update of /cvs-repository/ZEO/ZEO/tests In directory cvs.zope.org:/tmp/cvs-serv3219 Modified Files: Tag: Standby-branch forker.py testZEO.py Log Message: Change the Unix version of forker to be more like Windows. XXX I should really unify Unix and Windows code here, but I can't be bothered. I'm just fixing this to get at a different bug. Don't create a storage and pass it across the fork. There seem to be too many hard-to-explain problems linked to the fds that are open in parent when child is forked. === ZEO/ZEO/tests/forker.py 1.10.4.5 => 1.10.4.6 === pass - def start_zeo_server(storage, addr): + def start_zeo_server(addr, storage_name, args): + assert isinstance(args, types.TupleType) rd, wr = os.pipe() pid = os.fork() if pid == 0: @@ -110,7 +111,7 @@ globals(), locals()) p.close() else: - run_server(storage, addr, rd, wr) + run_server(addr, rd, wr, storage_name, args) except: print "Exception in ZEO server process" traceback.print_exc() @@ -119,20 +120,27 @@ os.close(rd) return pid, ZEOClientExit(wr) - def run_server(storage, addr, rd, wr): + def load_storage(name, args): + package = __import__("ZODB." + name) + mod = getattr(package, name) + klass = getattr(mod, name) + return klass(*args) + + def run_server(addr, rd, wr, storage_name, args): # in the child, run the storage server global server os.close(wr) ZEOServerExit(rd) import ZEO.StorageServer, ZEO.zrpc.server + storage = load_storage(storage_name, args) server = ZEO.StorageServer.StorageServer(addr, {'1':storage}) ZEO.zrpc.server.loop() storage.close() if isinstance(addr, types.StringType): os.unlink(addr) - def start_zeo(storage, cache=None, cleanup=None, domain="AF_INET", - storage_id="1", cache_size=20000000): + def start_zeo(storage_name, args, cache=None, cleanup=None, + domain="AF_INET", storage_id="1", cache_size=20000000): """Setup ZEO client-server for storage. Returns a ClientStorage instance and a ZEOClientExit instance. @@ -148,7 +156,7 @@ else: raise ValueError, "bad domain: %s" % domain - pid, exit = start_zeo_server(storage, addr) + pid, exit = start_zeo_server(addr, storage_name, args) s = ZEO.ClientStorage.ClientStorage(addr, storage_id, debug=1, client=cache, cache_size=cache_size, === ZEO/ZEO/tests/testZEO.py 1.16.4.9 => 1.16.4.10 === class GenericTests(StorageTestBase.StorageTestBase, TransactionalUndoStorage.TransactionalUndoStorage, - TransactionalUndoVersionStorage.TransactionalUndoVersionStorage, + TransactionalUndoVersionStorage.TransactionalUndoVersionStorage, ConflictResolution.ConflictResolvingStorage, ConflictResolution.ConflictResolvingTransUndoStorage, Cache.StorageWithCache, @@ -79,7 +79,7 @@ def setUp(self): zLOG.LOG("testZEO", zLOG.INFO, "setUp() %s" % self.id()) self.running = 1 - client, exit, pid = forker.start_zeo(self.getStorage()) + client, exit, pid = forker.start_zeo(*self.getStorage()) self._pids = [pid] self._servers = [exit] self._storage = PackWaitWrapper(client) @@ -116,7 +116,8 @@ wait_for_server_on_startup=1) def getStorage(self): - return FileStorage(self.__fs_base, create=1) + self.__fs_base = tempfile.mktemp() + return 'FileStorage', (self.__fs_base, '1') def delStorage(self): # file storage appears to create four files @@ -159,7 +160,7 @@ def getStorageInfo(self): self.__fs_base = tempfile.mktemp() - return 'FileStorage', self.__fs_base, '1' + return 'FileStorage', (self.__fs_base, '1') # create=1 def delStorage(self): # file storage appears to create four files @@ -342,8 +343,10 @@ def _startServer(self, create=1, index=0): fs = FileStorage("%s.%d" % (self.file, index), create=create) + path = "%s.%d" % (self.file, index) addr = self.addr[index] - pid, server = forker.start_zeo_server(fs, addr) + pid, server = forker.start_zeo_server(addr, 'FileStorage', + (path, create)) self._pids.append(pid) self._servers.append(server) @@ -379,10 +382,10 @@ port = self.addr[1] else: port = None - self.addr, self.test_a, pid = forker.start_zeo_server('FileStorage', + self.addr, self.test_a, pid = forker.start_zeo_server(port, + 'FileStorage', (self.file, - str(create)), - port) + str(create))) self.running = 1 def openClientStorage(self, cache='', cache_size=200000, wait=1): From jeremy at zope.com Mon Jan 28 17:39:05 2002 From: jeremy at zope.com (Jeremy Hylton) Date: Sun Aug 10 16:31:19 2008 Subject: [ZEO-Checkins] CVS: ZEO/ZEO - ClientStorage.py:1.35.6.8 Message-ID: <200201282239.g0SMd5r21045@cvs.baymountain.com> Update of /cvs-repository/ZEO/ZEO In directory cvs.zope.org:/tmp/cvs-serv21034 Modified Files: Tag: Standby-branch ClientStorage.py Log Message: Delete second def close() statement. Why does this always happen to me? === ZEO/ZEO/ClientStorage.py 1.35.6.7 => 1.35.6.8 === return oids - def close(self): - self._rpc_mgr.close() - if self._cache is not None: - self._cache.close() - def commitVersion(self, src, dest, transaction): if self._is_read_only: raise POSException.ReadOnlyError() From jeremy at zope.com Mon Jan 28 17:54:24 2002 From: jeremy at zope.com (Jeremy Hylton) Date: Sun Aug 10 16:31:19 2008 Subject: [ZEO-Checkins] CVS: ZEO/ZEO/tests - testZEO.py:1.16.4.11 Message-ID: <200201282254.g0SMsOT26929@cvs.baymountain.com> Update of /cvs-repository/ZEO/ZEO/tests In directory cvs.zope.org:/tmp/cvs-serv26918 Modified Files: Tag: Standby-branch testZEO.py Log Message: Argh! Don't create a FileStorage in _startServer(), that's what the previous checkin was supposed to prevent. === ZEO/ZEO/tests/testZEO.py 1.16.4.10 => 1.16.4.11 === def _startServer(self, create=1, index=0): - fs = FileStorage("%s.%d" % (self.file, index), create=create) path = "%s.%d" % (self.file, index) addr = self.addr[index] pid, server = forker.start_zeo_server(addr, 'FileStorage', From jeremy at zope.com Mon Jan 28 18:23:19 2002 From: jeremy at zope.com (Jeremy Hylton) Date: Sun Aug 10 16:31:19 2008 Subject: [ZEO-Checkins] CVS: ZEO/ZEO/zrpc - client.py:1.1.2.11 Message-ID: <200201282323.g0SNNJk07391@cvs.baymountain.com> Update of /cvs-repository/ZEO/ZEO/zrpc In directory cvs.zope.org:/tmp/cvs-serv7380 Modified Files: Tag: Standby-branch client.py Log Message: close trigger if it exists. === ZEO/ZEO/zrpc/client.py 1.1.2.10 => 1.1.2.11 === if self.connection: self.connection.close() + if self.trigger is not None: + self.trigger.close() def set_async(self, map): # XXX need each connection started with async==0 to have a callback From jeremy at zope.com Mon Jan 28 18:42:56 2002 From: jeremy at zope.com (Jeremy Hylton) Date: Sun Aug 10 16:31:19 2008 Subject: [ZEO-Checkins] CVS: ZEO/ZEO/zrpc - trigger.py:1.1.2.3 Message-ID: <200201282342.g0SNguJ15327@cvs.baymountain.com> Update of /cvs-repository/ZEO/ZEO/zrpc In directory cvs.zope.org:/tmp/cvs-serv15310 Modified Files: Tag: Standby-branch trigger.py Log Message: Implement a close() that closes both sides of the pipe. === ZEO/ZEO/zrpc/trigger.py 1.1.2.2 => 1.1.2.3 === import asyncore -#import asynchat import os import socket @@ -135,6 +134,11 @@ asyncore.file_dispatcher.__init__ (self, r) self.lock = thread.allocate_lock() self.thunks = [] + + def close(self): + self.del_channel() + self.socket.close() # the read side of the pipe + os.close(self.trigger) # the write side of the pipe def __repr__ (self): return '' % id(self) From jeremy at zope.com Mon Jan 28 19:59:52 2002 From: jeremy at zope.com (Jeremy Hylton) Date: Sun Aug 10 16:31:19 2008 Subject: [ZEO-Checkins] CVS: ZEO/ZEO/zrpc - client.py:1.1.2.12 Message-ID: <200201290059.g0T0xqU18104@cvs.baymountain.com> Update of /cvs-repository/ZEO/ZEO/zrpc In directory cvs.zope.org:/tmp/cvs-serv18054 Modified Files: Tag: Standby-branch client.py Log Message: Fix leaking trigger pips caused by test suite. See comment for details. === ZEO/ZEO/zrpc/client.py 1.1.2.11 => 1.1.2.12 === self.connection.close() if self.trigger is not None: + print repr(self), "close" self.trigger.close() def set_async(self, map): - # XXX need each connection started with async==0 to have a callback - self.trigger = trigger() - self.thr_async = 1 # XXX needs to be set on the Connection + # This is the callback registered with ThreadedAsync. The + # callback might be called multiple times, so it shouldn't + # create a trigger every time and should never do anything + # after it's closed. + + # It may be that the only case where it is called multiple + # times is in the test suite, where ThreadedAsync's loop can + # be started in a child process after a fork. Regardless, + # it's good to be defensive. + + # XXX need each connection started with async==0 to have a + # callback + if not self.closed and self.trigger is None: + self.trigger = trigger() + self.thr_async = 1 # XXX needs to be set on the Connection def attempt_connect(self): """Attempt a connection to the server without blocking too long. @@ -162,7 +175,6 @@ class ConnectThread(threading.Thread): """Thread that tries to connect to server given one or more addresses. - The thread is passed a ConnectionManager and the manager's client as arguments. It calls notifyConnected() on the client when a socket connects. If notifyConnected() returns without raising an From jeremy at zope.com Mon Jan 28 20:03:05 2002 From: jeremy at zope.com (Jeremy Hylton) Date: Sun Aug 10 16:31:19 2008 Subject: [ZEO-Checkins] CVS: ZEO/ZEO/zrpc - client.py:1.1.2.13 Message-ID: <200201290103.g0T135M22976@cvs.baymountain.com> Update of /cvs-repository/ZEO/ZEO/zrpc In directory cvs.zope.org:/tmp/cvs-serv22919 Modified Files: Tag: Standby-branch client.py Log Message: Remove debugging print. Why do I always notice those in the diff in the checkin mail, but not in the diff I generate before the checkin? === ZEO/ZEO/zrpc/client.py 1.1.2.12 => 1.1.2.13 === self.connection.close() if self.trigger is not None: - print repr(self), "close" self.trigger.close() def set_async(self, map): From jeremy at zope.com Tue Jan 29 09:34:43 2002 From: jeremy at zope.com (Jeremy Hylton) Date: Sun Aug 10 16:31:19 2008 Subject: [ZEO-Checkins] CVS: ZEO/ZEO/zrpc - trigger.py:1.1.2.4 Message-ID: <200201291434.g0TEYh804965@cvs.baymountain.com> Update of /cvs-repository/ZEO/ZEO/zrpc In directory cvs.zope.org:/tmp/cvs-serv4954 Modified Files: Tag: Standby-branch trigger.py Log Message: Remove unused import. === ZEO/ZEO/zrpc/trigger.py 1.1.2.3 => 1.1.2.4 === import os import socket -import string import thread if os.name == 'posix': From jeremy at zope.com Tue Jan 29 09:35:34 2002 From: jeremy at zope.com (Jeremy Hylton) Date: Sun Aug 10 16:31:19 2008 Subject: [ZEO-Checkins] CVS: ZEO/ZEO/tests - forker.py:1.10.4.7 Message-ID: <200201291435.g0TEZYS06112@cvs.baymountain.com> Update of /cvs-repository/ZEO/ZEO/tests In directory cvs.zope.org:/tmp/cvs-serv6076 Modified Files: Tag: Standby-branch forker.py Log Message: Make start_zeo_server() consist across platforms. On Windows, make sure modified environment is passed to spawn. === ZEO/ZEO/tests/forker.py 1.10.4.6 => 1.10.4.7 === d = os.environ.copy() d['PYTHONPATH'] = os.pathsep.join(sys.path) - pid = os.spawnve(os.P_NOWAIT, sys.executable, args, os.environ) + pid = os.spawnve(os.P_NOWAIT, sys.executable, args, d) return ('localhost', port), ('localhost', port + 1), pid else: @@ -97,7 +97,7 @@ except os.error: pass - def start_zeo_server(addr, storage_name, args): + def start_zeo_server(storage_name, args, addr): assert isinstance(args, types.TupleType) rd, wr = os.pipe() pid = os.fork() @@ -156,7 +156,7 @@ else: raise ValueError, "bad domain: %s" % domain - pid, exit = start_zeo_server(addr, storage_name, args) + pid, exit = start_zeo_server(storage_name, args, addr) s = ZEO.ClientStorage.ClientStorage(addr, storage_id, debug=1, client=cache, cache_size=cache_size, From jeremy at zope.com Tue Jan 29 09:42:49 2002 From: jeremy at zope.com (Jeremy Hylton) Date: Sun Aug 10 16:31:19 2008 Subject: [ZEO-Checkins] CVS: ZEO/ZEO/tests - testZEO.py:1.16.4.12 Message-ID: <200201291442.g0TEgnT08429@cvs.baymountain.com> Update of /cvs-repository/ZEO/ZEO/tests In directory cvs.zope.org:/tmp/cvs-serv8415 Modified Files: Tag: Standby-branch testZEO.py Log Message: Call start_zeo_server() with args in correct order. === ZEO/ZEO/tests/testZEO.py 1.16.4.11 => 1.16.4.12 === path = "%s.%d" % (self.file, index) addr = self.addr[index] - pid, server = forker.start_zeo_server(addr, 'FileStorage', - (path, create)) + pid, server = forker.start_zeo_server('FileStorage', + (path, create), addr) self._pids.append(pid) self._servers.append(server) @@ -381,10 +381,10 @@ port = self.addr[1] else: port = None - self.addr, self.test_a, pid = forker.start_zeo_server(port, - 'FileStorage', + self.addr, self.test_a, pid = forker.start_zeo_server('FileStorage', (self.file, - str(create))) + str(create)), + port) self.running = 1 def openClientStorage(self, cache='', cache_size=200000, wait=1): From jeremy at zope.com Tue Jan 29 10:14:35 2002 From: jeremy at zope.com (Jeremy Hylton) Date: Sun Aug 10 16:31:19 2008 Subject: [ZEO-Checkins] CVS: ZEO/ZEO/tests - testZEO.py:1.16.4.13 Message-ID: <200201291514.g0TFEZd19581@cvs.baymountain.com> Update of /cvs-repository/ZEO/ZEO/tests In directory cvs.zope.org:/tmp/cvs-serv19570 Modified Files: Tag: Standby-branch testZEO.py Log Message: Add some extra zLOG calls to track the progress of checkReconnection(). === ZEO/ZEO/tests/testZEO.py 1.16.4.12 => 1.16.4.13 === obj = MinPO(12) revid1 = self._dostore(oid, data=obj) + zLOG.LOG("checkReconnection", zLOG.INFO, + "About to shutdown server") self.shutdownServer() self.running = 1 + zLOG.LOG("checkReconnection", zLOG.INFO, + "About to restart server") self._startServer(create=0) oid = self._storage.new_oid() obj = MinPO(12) @@ -311,6 +315,9 @@ break except (Disconnected, select.error, thread.error, socket.error), \ err: + zLOG.LOG("checkReconnection", zLOG.INFO, + "Error after server restart; retrying.", + error=sys.exc_info()) get_transaction().abort() time.sleep(0.1) # XXX how long to sleep # XXX This is a bloody pain. We're placing a heavy burden @@ -318,6 +325,7 @@ # write robust code. Need to think about implementing # John Heintz's suggestion to make sure all exceptions # inherit from POSException. + zLOG.LOG("checkReconnection", zLOG.INFO, "finished") class UnixConnectionTests(ConnectionTests):