From jeremy at zope.com Wed Sep 5 01:13:39 2001 From: jeremy at zope.com (Jeremy Hylton) Date: Sun Aug 10 16:31:16 2008 Subject: [ZEO-Checkins] CVS: ZEO - README:1.20.2.1 Message-ID: <200109050513.f855Dd306620@cvs.baymountain.com> Update of /cvs-repository/ZEO In directory cvs.zope.org:/tmp/cvs-serv6608 Modified Files: Tag: zeo-1_0-branch README Log Message: Add note about required versions of Python and Zope === ZEO/README 1.20 => 1.20.2.1 === If you are using Zope, see docs/ZopeREADME.txt; otherwise, see docs/NonZopeREADME.txt. + +Requirements + + ZEO requires Python 1.5.2 and Zope 2.2 or higher. It can also be + used with StandaloneZODB. + From jeremy at zope.com Fri Sep 7 15:32:27 2001 From: jeremy at zope.com (Jeremy Hylton) Date: Sun Aug 10 16:31:16 2008 Subject: [ZEO-Checkins] CVS: StandaloneZODB/ZEO/tests - forker.py:1.5.2.4 Message-ID: <200109071932.f87JWRR14426@cvs.baymountain.com> Update of /cvs-repository/StandaloneZODB/ZEO/tests In directory cvs.zope.org:/tmp/cvs-serv14410 Modified Files: Tag: zeo-1_0-branch forker.py Log Message: Choose a port for the new server carefully (attempt to detect conflicts) === StandaloneZODB/ZEO/tests/forker.py 1.5.2.3 => 1.5.2.4 === import profile import random +import socket import sys import types import ZEO.ClientStorage, ZEO.StorageServer PROFILE = 0 +def get_port(): + """Return a port that is not in use. + + Checks if a port is in use by trying to connect to it. Assumes it + is not in use if connect raises an exception. + + Raises RuntimeError after 10 tries. + """ + for i in range(10): + port = random.randrange(20000, 30000) + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + try: + s.connect(('localhost', port)) + except socket.error: + # XXX check value of error? + return port + raise RuntimeError, "Can't find port" + if os.name == "nt": def start_zeo_server(storage_name, args, port=None): @@ -18,8 +37,7 @@ Returns the ZEO port, the test server port, and the pid. """ import ZEO.tests.winserver - if port is None: - port = random.randrange(20000, 30000) + port = get_port() script = ZEO.tests.winserver.__file__ if script.endswith('.pyc'): script = script[:-1] @@ -93,8 +111,7 @@ """ if domain == "AF_INET": - import random - addr = '', random.randrange(2000, 3000) + addr = '', get_port() elif domain == "AF_UNIX": import tempfile addr = tempfile.mktemp() From jeremy at zope.com Fri Sep 7 15:36:41 2001 From: jeremy at zope.com (Jeremy Hylton) Date: Sun Aug 10 16:31:16 2008 Subject: [ZEO-Checkins] CVS: StandaloneZODB/ZEO - asyncwrap.py:1.2 Message-ID: <200109071936.f87Jafj15091@cvs.baymountain.com> Update of /cvs-repository/StandaloneZODB/ZEO In directory cvs.zope.org:/tmp/cvs-serv15075 Added Files: asyncwrap.py Log Message: New file from zeo-1_0-branch. Wrap asyncore calls and be careful about exceptions. === StandaloneZODB/ZEO/asyncwrap.py 1.1 => 1.2 === + +The poll() and loop() calls exported by asyncore can raise exceptions. +asyncore uses either the select() or poll() system call. It is +possible for those system calls to fail, returning, for example, +EINTR. Python raises a select.error when an error occurs. If the +program using asyncore doesn't catch the exception, it will die with +an uncaught exception. + +This module exports safer versions of loop() and poll() that wrap the +asyncore calls in try/except handlers that catch the errors and do the +right thing. In most cases, it is safe to catch the error and simply +retry the call. + +XXX Operations on asyncore sockets can also fail with exceptions that +can safely be caught and ignored by user programs. It's not clear if +it would be useful to extend this module with wrappers for those +errors. +""" + +# XXX The current implementation requires Python 2.0. Not sure if +# that's acceptable, depends on how many users want to combine ZEO 1.0 +# and Zope 2.3. + +import asyncore +import errno +import select + +def loop(*args, **kwargs): + while 1: + try: + apply(asyncore.loop, args, kwargs) + except select.error, err: + if err[0] != errno.EINTR: + raise + else: + break + +def poll(*args, **kwargs): + try: + apply(asyncore.poll, args, kwargs) + except select.error, err: + if err[0] != errno.EINTR: + raise From jeremy at zope.com Fri Sep 7 15:37:56 2001 From: jeremy at zope.com (Jeremy Hylton) Date: Sun Aug 10 16:31:16 2008 Subject: [ZEO-Checkins] CVS: StandaloneZODB/ZEO - ClientStorage.py:1.34 Message-ID: <200109071937.f87JbuP15696@cvs.baymountain.com> Update of /cvs-repository/StandaloneZODB/ZEO In directory cvs.zope.org:/tmp/cvs-serv15673 Modified Files: ClientStorage.py Log Message: Merge in zeo-1_0-branch Don't release the commit lock unless there is a transaction in progress. This appears to cause the "release unlocked lock" error to be replaced with a socket.error: "bad file descriptor". That's progress? === StandaloneZODB/ZEO/ClientStorage.py 1.33 => 1.34 === self._transaction=None thread.start_new_thread(self._call.connect,(0,)) - try: self._commit_lock_release() - except: pass + if self._transaction is not None: + try: + self._commit_lock_release() + except: + pass def becomeAsync(self, map): self._lock_acquire() @@ -479,6 +482,7 @@ "This action is temporarily unavailable.

") r=self._call(self.__begin, id, user, desc, ext) except: + # XXX can't seem to guarantee that the lock is held here. self._commit_lock_release() raise From jeremy at zope.com Fri Sep 7 15:39:15 2001 From: jeremy at zope.com (Jeremy Hylton) Date: Sun Aug 10 16:31:16 2008 Subject: [ZEO-Checkins] CVS: StandaloneZODB/ZEO - StorageServer.py:1.29 Message-ID: <200109071939.f87JdFx15850@cvs.baymountain.com> Update of /cvs-repository/StandaloneZODB/ZEO In directory cvs.zope.org:/tmp/cvs-serv15834 Modified Files: StorageServer.py Log Message: Merge in zeo-1_0-branch Fixed a bug in building undo invalidation info that caused client storages to raise an exception that caused connections to be reset and that, for some of reason, caused undo's to not propigate to the clients. Avoid long delays at the end of a pack. When a client calls pack, a separate thread is started to call the pack call on the storage. When the thread finishes, it calls message_output() to send a response to the client. If asyncore is currently in a poll call when this happens, the output won't be detected until the next poll call. If there is little I/O on the connection, this won't happen until the select call times out after 30 seconds. The trigger is the standard gimmick for one thread to notify a mainloop in another thread that the first thread has some I/O to do. It exits the current select. The next poll call detects that the triggering thread is ready to do I/O and handles it. Use asyncwrap to call asyncore.loop() and asyncore.poll() === StandaloneZODB/ZEO/StorageServer.py 1.28 => 1.29 === from ZODB.Transaction import Transaction import traceback -from zLOG import LOG, INFO, ERROR, TRACE +from zLOG import LOG, INFO, ERROR, TRACE, BLATHER from ZODB.referencesf import referencesf from thread import start_new_thread from cStringIO import StringIO +from ZEO import trigger +from ZEO import asyncwrap class StorageServerError(POSException.StorageError): pass @@ -124,7 +126,6 @@ self.__connections={} self.__get_connections=self.__connections.get - asyncore.dispatcher.__init__(self) if type(connection) is type(''): @@ -184,8 +185,8 @@ sock, addr = self.accept() except socket.error: sys.stderr.write('warning: accept failed\n') - - ZEOConnection(self, sock, addr) + else: + ZEOConnection(self, sock, addr) def log_info(self, message, type='info'): if type=='error': type=ERROR @@ -234,6 +235,7 @@ self.__server=server self.__invalidated=[] self.__closed=None + self._pack_trigger = trigger.trigger() if __debug__: debug='ZEO Server' else: debug=0 SizedMessageAsyncConnection.__init__(self, sock, addr, debug=debug) @@ -384,16 +386,21 @@ if wait: return _noreturn def _pack(self, t, wait=0): - try: + try: + LOG('ZEO Server', BLATHER, 'pack begin') self.__storage.pack(t, referencesf) + LOG('ZEO Server', BLATHER, 'pack end') except: LOG('ZEO Server', ERROR, 'Pack failed for %s' % self.__storage_id, error=sys.exc_info()) - if wait: self.return_error(sys.exc_info()[0], sys.exc_info()[1]) + if wait: + self.return_error(sys.exc_info()[0], sys.exc_info()[1]) + self._pack_trigger.pull_trigger() else: if wait: self.message_output('RN.') + self._pack_trigger.pull_trigger() else: # Broadcast new size statistics self.__server.invalidate(0, self.__storage_id, (), @@ -469,7 +476,7 @@ oids=self.__storage.undo(transaction_id) if oids: self.__server.invalidate( - self, self.__storage_id, map(lambda oid: (oid,None,''), oids) + self, self.__storage_id, map(lambda oid: (oid,None), oids) ) return oids return () @@ -571,7 +578,10 @@ import ZODB.FileStorage name, port = sys.argv[1:3] blather(name, port) - try: port='',string.atoi(port) - except: pass + try: + port='', int(port) + except: + pass + StorageServer(port, ZODB.FileStorage.FileStorage(name)) - asyncore.loop() + asyncwrap.loop() From jeremy at zope.com Fri Sep 7 15:40:47 2001 From: jeremy at zope.com (Jeremy Hylton) Date: Sun Aug 10 16:31:16 2008 Subject: [ZEO-Checkins] CVS: StandaloneZODB/ZEO - trigger.py:1.2 Message-ID: <200109071940.f87Jelo16057@cvs.baymountain.com> Update of /cvs-repository/StandaloneZODB/ZEO In directory cvs.zope.org:/tmp/cvs-serv16041 Modified Files: trigger.py Log Message: Merge in zeo-1_0-branch Shorten two long lines, one by wrapping, another by removing an unnecessary tuple unpacking. === StandaloneZODB/ZEO/trigger.py 1.1 => 1.2 === thunk() except: - (file, fun, line), t, v, tbinfo = asyncore.compact_traceback() - print 'exception in trigger thunk: (%s:%s %s)' % (t, v, tbinfo) + nil, t, v, tbinfo = asyncore.compact_traceback() + print ('exception in trigger thunk:' + ' (%s:%s %s)' % (t, v, tbinfo)) self.thunks = [] finally: self.lock.release() else: + # XXX Should define a base class that has the common methods and + # then put the platform-specific in a subclass named trigger. + # win32-safe version class trigger (asyncore.dispatcher): @@ -245,12 +249,9 @@ try: thunk() except: - (file, fun, line), t, v, tbinfo = asyncore.compact_traceback() - print 'exception in trigger thunk: (%s:%s %s)' % (t, v, tbinfo) + nil, t, v, tbinfo = asyncore.compact_traceback() + print ('exception in trigger thunk:' + ' (%s:%s %s)' % (t, v, tbinfo)) self.thunks = [] finally: self.lock.release() - - - - From jeremy at zope.com Fri Sep 7 15:40:02 2001 From: jeremy at zope.com (Jeremy Hylton) Date: Sun Aug 10 16:31:16 2008 Subject: [ZEO-Checkins] CVS: StandaloneZODB/ZEO - smac.py:1.11 Message-ID: <200109071940.f87Je2h16002@cvs.baymountain.com> Update of /cvs-repository/StandaloneZODB/ZEO In directory cvs.zope.org:/tmp/cvs-serv15978 Modified Files: smac.py Log Message: Merge in zeo-1_0-branch Wrap send() and recv() calls in try/except The try/except catches errors like EAGAIN that indicate transient failures. In those cases, treat as send() and recv() of no data and return from method gracefully. === StandaloneZODB/ZEO/smac.py 1.10 => 1.11 === import asyncore, string, struct, zLOG, sys, Acquisition +import socket, errno from zLOG import LOG, TRACE, ERROR, INFO +# Use the dictionary to make sure we get the minimum number of errno +# entries. We expect that EWOULDBLOCK == EAGAIN on most systems -- +# or that only one is actually used. + +tmp_dict = {errno.EWOULDBLOCK: 0, + errno.EAGAIN: 0, + errno.EINTR: 0, + } +expected_socket_read_errors = tuple(tmp_dict.keys()) + +tmp_dict = {errno.EAGAIN: 0, + errno.EWOULDBLOCK: 0, + errno.ENOBUFS: 0, + errno.EINTR: 0, + } +expected_socket_write_errors = tuple(tmp_dict.keys()) +del tmp_dict + class SizedMessageAsyncConnection(Acquisition.Explicit, asyncore.dispatcher): __append=None # Marker indicating that we're closed @@ -116,7 +135,12 @@ join=string.join, StringType=type(''), _type=type, _None=None): - d=self.recv(8096) + try: + d=self.recv(8096) + except socket.error, err: + if err[0] in expected_socket_read_errors: + return + raise if not d: return inp=self.__inp @@ -160,7 +184,12 @@ output=self.__output while output: v=output[0] - n=self.send(v) + try: + n=self.send(v) + except socket.error, err: + if err[0] in expected_socket_write_errors: + break # we couldn't write anything + raise if n < len(v): output[0]=v[n:] break # we can't write any more From jeremy at zope.com Fri Sep 7 15:41:24 2001 From: jeremy at zope.com (Jeremy Hylton) Date: Sun Aug 10 16:31:16 2008 Subject: [ZEO-Checkins] CVS: StandaloneZODB/ZEO - zrpc.py:1.19 Message-ID: <200109071941.f87JfO716382@cvs.baymountain.com> Update of /cvs-repository/StandaloneZODB/ZEO In directory cvs.zope.org:/tmp/cvs-serv16366 Modified Files: zrpc.py Log Message: Merge in zeo-1_0-branch Use asyncwrap to call asyncore.loop() and asyncore.poll(). Add definition for EINTR. === StandaloneZODB/ZEO/zrpc.py 1.18 => 1.19 === from smac import SizedMessageAsyncConnection import socket, string, struct, asyncore, sys, time, select -TupleType=type(()) from zLOG import LOG, TRACE, DEBUG, INFO +from ZEO import asyncwrap + +from errno import EINTR +TupleType=type(()) # We create a special fast pickler! This allows us # to create slightly more efficient pickles and @@ -184,13 +187,13 @@ try: r, w, e = select.select([self._fileno],[],[],0.0) except select.error, v: if v[0] != EINTR: raise - if r: asyncore.poll(0.0, self) + if r: asyncwrap.poll(0.0, self) else: break def readLoop(self): la=self.__la while not la(0): - asyncore.poll(60.0, self) + asyncwrap.poll(60.0, self) self.__lr() def setLoop(self, map=None, Wakeup=lambda : None): @@ -240,7 +243,7 @@ self.message_output(dump(args,1)) if self.__haveMainLoop: self.__Wakeup() # Wake up the main loop - else: asyncore.poll(0.0, self) + else: asyncwrap.poll(0.0, self) def setOutOfBand(self, f): """Define a call-back function for handling out-of-band communication From jeremy at zope.com Fri Sep 7 15:41:41 2001 From: jeremy at zope.com (Jeremy Hylton) Date: Sun Aug 10 16:31:16 2008 Subject: [ZEO-Checkins] CVS: StandaloneZODB/ZEO/tests - multi.py:1.4 Message-ID: <200109071941.f87Jffv16434@cvs.baymountain.com> Update of /cvs-repository/StandaloneZODB/ZEO/tests In directory cvs.zope.org:/tmp/cvs-serv16417 Modified Files: multi.py Log Message: Merge in zeo-1_0-branch Remove Python 2.x-isms so that tests run under Python 1.5.2 === StandaloneZODB/ZEO/tests/multi.py 1.3 => 1.4 === server_pid, server = start_server(addr) t1 = time.time() - pids = [start_client(addr, client_func) for i in range(CLIENTS)] + pids = [] + for i in range(CLIENTS): + pids.append(start_client(addr, client_func)) for pid in pids: assert type(pid) == types.IntType, "invalid pid type: %s (%s)" % \ (repr(pid), type(pid)) From jeremy at zope.com Fri Sep 7 15:41:52 2001 From: jeremy at zope.com (Jeremy Hylton) Date: Sun Aug 10 16:31:16 2008 Subject: [ZEO-Checkins] CVS: StandaloneZODB/ZEO/tests - speed.py:1.5 Message-ID: <200109071941.f87Jfqp16466@cvs.baymountain.com> Update of /cvs-repository/StandaloneZODB/ZEO/tests In directory cvs.zope.org:/tmp/cvs-serv16450 Modified Files: speed.py Log Message: Merge in zeo-1_0-branch Remove Python 2.x-isms so that tests run under Python 1.5.2 === StandaloneZODB/ZEO/tests/speed.py 1.4 => 1.5 === if threads > 1: import threading - l = [threading.Thread(target=work, - args=(db, results, nrep, compress, data, - detailed, minimize, i)) - for i in range(threads)] + l = [] + for i in range(threads): + t = threading.Thread(target=work, + args=(db, results, nrep, compress, data, + detailed, minimize, i)) + l.append(t) for t in l: t.start() for t in l: @@ -263,7 +265,9 @@ print '-'*24 print "num\tmean\tmin\tmax" for r in 1, 10, 100, 1000: - times = [time for time, conf in results[r]] + times = [] + for time, conf in results[r]: + times.append(time) t = mean(times) print "%d\t%.4f\t%.4f\t%.4f" % (r, t, min(times), max(times)) From jeremy at zope.com Fri Sep 7 16:46:38 2001 From: jeremy at zope.com (Jeremy Hylton) Date: Sun Aug 10 16:31:16 2008 Subject: [ZEO-Checkins] CVS: StandaloneZODB/ZEO/tests - forker.py:1.8 testZEO.py:1.14 Message-ID: <200109072046.f87Kkcd00815@cvs.baymountain.com> Update of /cvs-repository/StandaloneZODB/ZEO/tests In directory cvs.zope.org:/tmp/cvs-serv796 Modified Files: forker.py testZEO.py Log Message: Merge in zeo-1_0-branch === StandaloneZODB/ZEO/tests/forker.py 1.7 => 1.8 === import profile import random +import socket import sys import types import ZEO.ClientStorage, ZEO.StorageServer PROFILE = 0 +def get_port(): + """Return a port that is not in use. + + Checks if a port is in use by trying to connect to it. Assumes it + is not in use if connect raises an exception. + + Raises RuntimeError after 10 tries. + """ + for i in range(10): + port = random.randrange(20000, 30000) + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + try: + s.connect(('localhost', port)) + except socket.error: + # XXX check value of error? + return port + raise RuntimeError, "Can't find port" + if os.name == "nt": def start_zeo_server(storage_name, args, port=None): @@ -18,8 +37,7 @@ Returns the ZEO port, the test server port, and the pid. """ import ZEO.tests.winserver - if port is None: - port = random.randrange(20000, 30000) + port = get_port() script = ZEO.tests.winserver.__file__ if script.endswith('.pyc'): script = script[:-1] @@ -93,8 +111,7 @@ """ if domain == "AF_INET": - import random - addr = '', random.randrange(2000, 3000) + addr = '', get_port() elif domain == "AF_UNIX": import tempfile addr = tempfile.mktemp() === StandaloneZODB/ZEO/tests/testZEO.py 1.13 => 1.14 === """ - __super_setUp = StorageTestBase.StorageTestBase.setUp + __super_tearDown = StorageTestBase.StorageTestBase.tearDown ports = [] for i in range(200): @@ -242,7 +242,7 @@ self.shutdownServer() # file storage appears to create four files for ext in '', '.index', '.lock', '.tmp': - path = self.__fs_base + ext + path = self.file + ext if os.path.exists(path): os.unlink(path) for i in 0, 1: @@ -325,13 +325,13 @@ getStorage() method. """ self.running = 1 - self.__fs_base = tempfile.mktemp() + self.file = tempfile.mktemp() self.addr = '', self.ports.pop() self._startServer() self.__super_setUp() def _startServer(self, create=1): - fs = FileStorage(self.__fs_base, create=create) + fs = FileStorage(self.file, create=create) self._pid, self._server = forker.start_zeo_server(fs, self.addr) def openClientStorage(self, cache='', cache_size=200000, wait=1): From jeremy at zope.com Fri Sep 28 18:58:04 2001 From: jeremy at zope.com (Jeremy Hylton) Date: Sun Aug 10 16:31:16 2008 Subject: [ZEO-Checkins] CVS: StandaloneZODB/ZEO/tests - forker.py:1.9 Message-ID: <200109282258.f8SMw4J04036@cvs.baymountain.com> Update of /cvs-repository/StandaloneZODB/ZEO/tests In directory cvs.zope.org:/tmp/cvs-serv4024 Modified Files: forker.py Log Message: Pass the correct environment to the spawned process === StandaloneZODB/ZEO/tests/forker.py 1.8 => 1.9 === d = os.environ.copy() d['PYTHONPATH'] = os.pathsep.join(sys.path) - pid = os.spawnve(os.P_NOWAIT, sys.executable, args, os.environ) + pid = os.spawnve(os.P_NOWAIT, sys.executable, args, d) return ('localhost', port), ('localhost', port + 1), pid else: