From jeremy at digicool.com Fri Mar 16 19:40:47 2001 From: jeremy at digicool.com (jeremy@digicool.com) Date: Sun Aug 10 16:31:13 2008 Subject: [ZEO-Checkins] CVS: Packages/ZEO - StorageServer.py:1.19.2.2 Message-ID: <20010317004047.3AAD051014@korak.digicool.com> Update of /cvs-repository/Packages/ZEO In directory korak:/tmp/cvs-serv27129 Modified Files: Tag: ZEO-Quorum-Dev StorageServer.py Log Message: New StorageServer based on zrpc2 Replace function blather with function log. Rename first argument to StorageServer to addr (instead of connection). Defer most of the real work from StorageServer to StorageProxy class. Each StorageProxy is attached to a single connection. The StorageProxy defers all the message dispatch to zrpc2. Replace message_output() calls with explicit RPC calls on self.client Use threading module instead of thread module. --- Updated File StorageServer.py in package Packages/ZEO -- --- StorageServer.py 2000/12/19 22:14:09 1.19.2.1 +++ StorageServer.py 2001/03/17 00:40:46 1.19.2.2 @@ -98,6 +98,7 @@ from cPickle import Unpickler from cStringIO import StringIO import types +import ClientStub class StorageServerError(POSException.StorageError): pass @@ -484,7 +485,7 @@ newserial = self.__storage.store(oid, serial, data, version, t) except TransactionError, v: - # This is a normal transaction errorm such as a conflict error + # This is a normal transaction error such as a conflict error # or a version lock or conflict error. It doen't need to be # logged. newserial = v @@ -550,6 +551,7 @@ if self.__closed: return self.message_output(zeoproto.UNLOCK, None) + # Why doesn't this return None? def tpc_begin(self, id, user, description, ext): t = self._transaction @@ -640,13 +642,3 @@ if not hasattr(storage,'tpc_vote'): storage.tpc_vote = lambda *args: None -if __name__ == '__main__': - import ZODB.FileStorage - name, port = sys.argv[1:3] - blather(name, port) - try: - port = '', int(port) - except: # XXX What exception is expected here? - pass - StorageServer(port, ZODB.FileStorage.FileStorage(name)) - asyncore.loop() From jim at digicool.com Tue Mar 27 18:35:36 2001 From: jim at digicool.com (Jim Fulton) Date: Sun Aug 10 16:31:13 2008 Subject: [ZEO-Checkins] CVS: Packages/ZEO - CHANGES.txt:1.19 Message-ID: <20010327233536.4BF15510E2@korak.digicool.com> Update of /cvs-repository/Packages/ZEO In directory korak:/tmp/cvs-serv12423 Modified Files: CHANGES.txt Log Message: *** empty log message *** --- Updated File CHANGES.txt in package Packages/ZEO -- --- CHANGES.txt 2001/02/23 18:50:50 1.18 +++ CHANGES.txt 2001/03/27 23:35:34 1.19 @@ -1,14 +1,7 @@ Zope Enterprise Objects (ZEO) Revision History - ZEO 1.0.0 + ZEO 0.5.0 - Bugs Fixed - - - A forgotten argument made it unreliable to start a ClientStorage - after the main loop has started. - - ZEO 1.0.0 beta 1 - New Features - The server can be made to reopen it's log file @@ -27,6 +20,21 @@ wait for the pack results. This is handy when packing as part of cron jobs. + - It is no longer necessary to symbolically link cPickle or + ZServer. ZServer is no longer necessary at all. + + - A Zope-style INSTANCE_HOME and var directory are no longer + needed. + + - If ZServer *is* available, the medusa monitor server can be + used in the storage server. + + - An option, -d, was added to facilitate generation of a + detailed debug log while running in the background. + + - The documentation has been simplified and spread over multiple + files in the doc subdirectory. + Bugs Fixed - A possible (but unobserved) race condition that could @@ -50,6 +58,19 @@ while the client storage was disconnected from the server, no further write transactions would be allowed, even after reconnection, and the site would eventually hang. + + - A forgotten argument made it unreliable to start a ClientStorage + after the main loop has started. + + - In combination with recent changes in zdeamon, startup errors + could cause infinite loops. + + - The handling of the Python global, __debug__, was not + compatible with Python 2.1. + + - If an exception raised on the server which could not be + unpickled on the client could cause the client connection to + fail. Planned for (future) ZEO releases --- Updated File CHANGES.txt in package Packages/ZEO -- --- CHANGES.txt 2001/02/23 18:50:50 1.18 +++ CHANGES.txt 2001/03/27 23:35:34 1.19 @@ -1,14 +1,7 @@ Zope Enterprise Objects (ZEO) Revision History - ZEO 1.0.0 + ZEO 0.5.0 - Bugs Fixed - - - A forgotten argument made it unreliable to start a ClientStorage - after the main loop has started. - - ZEO 1.0.0 beta 1 - New Features - The server can be made to reopen it's log file @@ -27,6 +20,21 @@ wait for the pack results. This is handy when packing as part of cron jobs. + - It is no longer necessary to symbolically link cPickle or + ZServer. ZServer is no longer necessary at all. + + - A Zope-style INSTANCE_HOME and var directory are no longer + needed. + + - If ZServer *is* available, the medusa monitor server can be + used in the storage server. + + - An option, -d, was added to facilitate generation of a + detailed debug log while running in the background. + + - The documentation has been simplified and spread over multiple + files in the doc subdirectory. + Bugs Fixed - A possible (but unobserved) race condition that could @@ -50,6 +58,19 @@ while the client storage was disconnected from the server, no further write transactions would be allowed, even after reconnection, and the site would eventually hang. + + - A forgotten argument made it unreliable to start a ClientStorage + after the main loop has started. + + - In combination with recent changes in zdeamon, startup errors + could cause infinite loops. + + - The handling of the Python global, __debug__, was not + compatible with Python 2.1. + + - If an exception raised on the server which could not be + unpickled on the client could cause the client connection to + fail. Planned for (future) ZEO releases From jim at digicool.com Tue Mar 27 18:36:24 2001 From: jim at digicool.com (Jim Fulton) Date: Sun Aug 10 16:31:13 2008 Subject: [ZEO-Checkins] CVS: Packages/ZEO - fap.py:1.1 Message-ID: <20010327233624.D4AC8510E2@korak.digicool.com> Update of /cvs-repository/Packages/ZEO In directory korak:/tmp/cvs-serv12555 Added Files: fap.py Log Message: *** empty log message *** --- Added File fap.py in package Packages/ZEO --- ############################################################################## # # Zope Public License (ZPL) Version 1.0 # ------------------------------------- # # Copyright (c) Digital Creations. All rights reserved. # # This license has been certified as Open Source(tm). # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions in source code must retain the above copyright # notice, this list of conditions, and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions, and the following disclaimer in # the documentation and/or other materials provided with the # distribution. # # 3. Digital Creations requests that attribution be given to Zope # in any manner possible. Zope includes a "Powered by Zope" # button that is installed by default. While it is not a license # violation to remove this button, it is requested that the # attribution remain. A significant investment has been put # into Zope, and this effort will continue if the Zope community # continues to grow. This is one way to assure that growth. # # 4. All advertising materials and documentation mentioning # features derived from or use of this software must display # the following acknowledgement: # # "This product includes software developed by Digital Creations # for use in the Z Object Publishing Environment # (http://www.zope.org/)." # # In the event that the product being advertised includes an # intact Zope distribution (with copyright and license included) # then this clause is waived. # # 5. Names associated with Zope or Digital Creations must not be used to # endorse or promote products derived from this software without # prior written permission from Digital Creations. # # 6. Modified redistributions of any form whatsoever must retain # the following acknowledgment: # # "This product includes software developed by Digital Creations # for use in the Z Object Publishing Environment # (http://www.zope.org/)." # # Intact (re-)distributions of any official Zope release do not # require an external acknowledgement. # # 7. Modifications are encouraged but must be packaged separately as # patches to official Zope releases. Distributions that do not # clearly separate the patches from the original work must be clearly # labeled as unofficial distributions. Modifications which do not # carry the name Zope may be packaged in any form, as long as they # conform to all of the clauses above. # # # Disclaimer # # THIS SOFTWARE IS PROVIDED BY DIGITAL CREATIONS ``AS IS'' AND ANY # EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL DIGITAL CREATIONS OR ITS # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF # USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND # ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT # OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF # SUCH DAMAGE. # # # This software consists of contributions made by Digital Creations and # many individuals on behalf of Digital Creations. Specific # attributions are listed in the accompanying credits file. # ############################################################################## """ZEO depends on recent versions of asyncore and cPickle Try to fix up the imports of these to make these dependencies work, localizing the hacks^H^H^H^H^Hchanges here. """ import sys # if we are using an old version of Python, our asyncore is likely to # be out of date. If ZServer is sitting around, we can get a current # version of ayncore from it. In any case, if we are going to be used # with Zope, it's important to use the version from Zope. try: from ZServer.medusa import asyncore except: # Try a little harder to import ZServer import os, imp try: m=imp.find_module('ZServer', ['.']) except: try: m=imp.find_module('ZServer', [os.path.join('..','..')]) except: import asyncore else: sys.path.append(os.path.join('..','..')) from ZServer.medusa import asyncore else: sys.path.append('.') from ZServer.medusa import asyncore if sys.version[:1] < '2' and asyncore.loop.func_code.co_argcount < 3: raise ImportError, 'Cannot import an up-to-date asyncore' sys.modules['ZEO.asyncore']=asyncore # We need a recent version of cPickle too. if sys.version[:3] < '1.6': try: from ZODB import cPickle sys.modules['ZEO.cPickle']=cPickle except: # Try a little harder import cPickle else: import cPickle import cStringIO p=cPickle.Pickler(cStringIO.StringIO(),1) try: p.fast=1 except: raise ImportError, 'Cannot import an up-to-date cPickle' --- Added File fap.py in package Packages/ZEO --- ############################################################################## # # Zope Public License (ZPL) Version 1.0 # ------------------------------------- # # Copyright (c) Digital Creations. All rights reserved. # # This license has been certified as Open Source(tm). # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions in source code must retain the above copyright # notice, this list of conditions, and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions, and the following disclaimer in # the documentation and/or other materials provided with the # distribution. # # 3. Digital Creations requests that attribution be given to Zope # in any manner possible. Zope includes a "Powered by Zope" # button that is installed by default. While it is not a license # violation to remove this button, it is requested that the # attribution remain. A significant investment has been put # into Zope, and this effort will continue if the Zope community # continues to grow. This is one way to assure that growth. # # 4. All advertising materials and documentation mentioning # features derived from or use of this software must display # the following acknowledgement: # # "This product includes software developed by Digital Creations # for use in the Z Object Publishing Environment # (http://www.zope.org/)." # # In the event that the product being advertised includes an # intact Zope distribution (with copyright and license included) # then this clause is waived. # # 5. Names associated with Zope or Digital Creations must not be used to # endorse or promote products derived from this software without # prior written permission from Digital Creations. # # 6. Modified redistributions of any form whatsoever must retain # the following acknowledgment: # # "This product includes software developed by Digital Creations # for use in the Z Object Publishing Environment # (http://www.zope.org/)." # # Intact (re-)distributions of any official Zope release do not # require an external acknowledgement. # # 7. Modifications are encouraged but must be packaged separately as # patches to official Zope releases. Distributions that do not # clearly separate the patches from the original work must be clearly # labeled as unofficial distributions. Modifications which do not # carry the name Zope may be packaged in any form, as long as they # conform to all of the clauses above. # # # Disclaimer # # THIS SOFTWARE IS PROVIDED BY DIGITAL CREATIONS ``AS IS'' AND ANY # EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL DIGITAL CREATIONS OR ITS # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF # USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND # ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT # OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF # SUCH DAMAGE. # # # This software consists of contributions made by Digital Creations and # many individuals on behalf of Digital Creations. Specific # attributions are listed in the accompanying credits file. # ############################################################################## """ZEO depends on recent versions of asyncore and cPickle Try to fix up the imports of these to make these dependencies work, localizing the hacks^H^H^H^H^Hchanges here. """ import sys # if we are using an old version of Python, our asyncore is likely to # be out of date. If ZServer is sitting around, we can get a current # version of ayncore from it. In any case, if we are going to be used # with Zope, it's important to use the version from Zope. try: from ZServer.medusa import asyncore except: # Try a little harder to import ZServer import os, imp try: m=imp.find_module('ZServer', ['.']) except: try: m=imp.find_module('ZServer', [os.path.join('..','..')]) except: import asyncore else: sys.path.append(os.path.join('..','..')) from ZServer.medusa import asyncore else: sys.path.append('.') from ZServer.medusa import asyncore if sys.version[:1] < '2' and asyncore.loop.func_code.co_argcount < 3: raise ImportError, 'Cannot import an up-to-date asyncore' sys.modules['ZEO.asyncore']=asyncore # We need a recent version of cPickle too. if sys.version[:3] < '1.6': try: from ZODB import cPickle sys.modules['ZEO.cPickle']=cPickle except: # Try a little harder import cPickle else: import cPickle import cStringIO p=cPickle.Pickler(cStringIO.StringIO(),1) try: p.fast=1 except: raise ImportError, 'Cannot import an up-to-date cPickle' From jim at digicool.com Tue Mar 27 18:37:34 2001 From: jim at digicool.com (Jim Fulton) Date: Sun Aug 10 16:31:13 2008 Subject: [ZEO-Checkins] CVS: Packages/ZEO - start.py:1.17 Message-ID: <20010327233734.A2903510E2@korak.digicool.com> Update of /cvs-repository/Packages/ZEO In directory korak:/tmp/cvs-serv12718 Modified Files: start.py Log Message: - It is no longer necessary to symbolically link cPickle or ZServer. ZServer is no longer necessary at all. - A Zope-style INSTANCE_HOME and var directory are no longer needed. - If ZServer *is* available, the medusa monitor server can be used in the storage server. - An option, -d, was added to facilitate generation of a detailed debug log while running in the background. --- Updated File start.py in package Packages/ZEO -- --- start.py 2001/01/11 21:16:47 1.16 +++ start.py 2001/03/27 23:37:33 1.17 @@ -130,17 +130,25 @@ args.append(a) last=a - INSTANCE_HOME=os.environ.get('INSTANCE_HOME', directory(me, 4)) + if os.environ.has_key('INSTANCE_HOME'): + INSTANCE_HOME=os.environ['INSTANCE_HOME'] + elif os.path.isdir(os.path.join(directory(me, 4),'var')): + INSTANCE_HOME=directory(me, 4) + else: + INSTANCE_HOME=os.getcwd() + + if os.path.isdir(os.path.join(INSTANCE_HOME, 'var')): + var=os.path.join(INSTANCE_HOME, 'var') + else: + var=INSTANCE_HOME zeo_pid=os.environ.get('ZEO_SERVER_PID', - os.path.join(INSTANCE_HOME, 'var', 'ZEO_SERVER.pid') + os.path.join(var, 'ZEO_SERVER.pid') ) - opts, args = getopt.getopt(args, 'p:Dh:U:sS:u:') + opts, args = getopt.getopt(args, 'p:Ddh:U:sS:u:') - - - fs=os.path.join(INSTANCE_HOME, 'var', 'Data.fs') + fs=os.path.join(var, 'Data.fs') usage="""%s [options] [filename] @@ -148,6 +156,9 @@ -D -- Run in debug mode + -d -- Generate detailed debug logging without running + in the foreground. + -U -- Unix-domain socket file to listen on -u username or uid number @@ -182,7 +193,7 @@ """ % (me, fs) port=None - debug=0 + debug=detailed=0 host='' unix=None Z=1 @@ -193,12 +204,10 @@ elif o=='-U': unix=v elif o=='-u': UID=v elif o=='-D': debug=1 + elif o=='-d': detailed=1 elif o=='-s': Z=0 - try: - from ZServer.medusa import asyncore - sys.modules['asyncore']=asyncore - except: pass + import fap # fixup asyncore/cPickle dependencies if port is None and unix is None: print usage @@ -212,9 +221,10 @@ sys.exit(1) fs=args[0] - __builtins__.__debug__=debug if debug: os.environ['Z_DEBUG_MODE']='1' + if detailed: os.environ['STUPID_LOG_SEVERITY']='-99999' + from zLOG import LOG, INFO, ERROR # Try to set uid to "-u" -provided uid. @@ -255,52 +265,67 @@ import zdaemon zdaemon.run(sys.argv, '') - import ZEO.StorageServer, asyncore + try: - storages={} - for o, v in opts: - if o=='-S': - n, m = string.split(v,'=') - if string.find(m,':'): - # we got an attribute name - m, a = string.split(m,':') - else: - # attribute name must be same as storage name - a=n - storages[n]=get_storage(m,a) - - if not storages: - import ZODB.FileStorage - storages['1']=ZODB.FileStorage.FileStorage(fs) + import ZEO.StorageServer, asyncore - # Try to set up a signal handler - try: - import signal + storages={} + for o, v in opts: + if o=='-S': + n, m = string.split(v,'=') + if string.find(m,':'): + # we got an attribute name + m, a = string.split(m,':') + else: + # attribute name must be same as storage name + a=n + storages[n]=get_storage(m,a) + + if not storages: + import ZODB.FileStorage + storages['1']=ZODB.FileStorage.FileStorage(fs) - signal.signal(signal.SIGTERM, - lambda sig, frame, s=storages: shutdown(s) - ) - signal.signal(signal.SIGINT, - lambda sig, frame, s=storages: shutdown(s, 0) - ) - signal.signal(signal.SIGHUP, rotate_logs_handler) + # Try to set up a signal handler + try: + import signal - finally: pass + signal.signal(signal.SIGTERM, + lambda sig, frame, s=storages: shutdown(s) + ) + signal.signal(signal.SIGINT, + lambda sig, frame, s=storages: shutdown(s, 0) + ) + signal.signal(signal.SIGHUP, rotate_logs_handler) - items=storages.items() - items.sort() - for kv in items: - LOG('ZEO Server', INFO, 'Serving %s:\t%s' % kv) + finally: pass - if not unix: unix=host, port + items=storages.items() + items.sort() + for kv in items: + LOG('ZEO Server', INFO, 'Serving %s:\t%s' % kv) - ZEO.StorageServer.StorageServer(unix, storages) + if not unix: unix=host, port + ZEO.StorageServer.StorageServer(unix, storages) - open(zeo_pid,'w').write("%s %s" % (os.getppid(), os.getpid())) - - asyncore.loop() + open(zeo_pid,'w').write("%s %s" % (os.getppid(), os.getpid())) + + asyncore.loop() + except: + # Log startup exception and tell zdaemon not to restart us. + info=sys.exc_info() + try: + import zLOG + zLOG.LOG("z2", zLOG.PANIC, "Startup exception", + error=info) + except: + info2=sys.exc_info() + import traceback + traceback.print_exception(*info) + traceback.print_exception(*info2) + + sys.exit(0) def rotate_logs(): import zLOG --- Updated File start.py in package Packages/ZEO -- --- start.py 2001/01/11 21:16:47 1.16 +++ start.py 2001/03/27 23:37:33 1.17 @@ -130,17 +130,25 @@ args.append(a) last=a - INSTANCE_HOME=os.environ.get('INSTANCE_HOME', directory(me, 4)) + if os.environ.has_key('INSTANCE_HOME'): + INSTANCE_HOME=os.environ['INSTANCE_HOME'] + elif os.path.isdir(os.path.join(directory(me, 4),'var')): + INSTANCE_HOME=directory(me, 4) + else: + INSTANCE_HOME=os.getcwd() + + if os.path.isdir(os.path.join(INSTANCE_HOME, 'var')): + var=os.path.join(INSTANCE_HOME, 'var') + else: + var=INSTANCE_HOME zeo_pid=os.environ.get('ZEO_SERVER_PID', - os.path.join(INSTANCE_HOME, 'var', 'ZEO_SERVER.pid') + os.path.join(var, 'ZEO_SERVER.pid') ) - opts, args = getopt.getopt(args, 'p:Dh:U:sS:u:') + opts, args = getopt.getopt(args, 'p:Ddh:U:sS:u:') - - - fs=os.path.join(INSTANCE_HOME, 'var', 'Data.fs') + fs=os.path.join(var, 'Data.fs') usage="""%s [options] [filename] @@ -148,6 +156,9 @@ -D -- Run in debug mode + -d -- Generate detailed debug logging without running + in the foreground. + -U -- Unix-domain socket file to listen on -u username or uid number @@ -182,7 +193,7 @@ """ % (me, fs) port=None - debug=0 + debug=detailed=0 host='' unix=None Z=1 @@ -193,12 +204,10 @@ elif o=='-U': unix=v elif o=='-u': UID=v elif o=='-D': debug=1 + elif o=='-d': detailed=1 elif o=='-s': Z=0 - try: - from ZServer.medusa import asyncore - sys.modules['asyncore']=asyncore - except: pass + import fap # fixup asyncore/cPickle dependencies if port is None and unix is None: print usage @@ -212,9 +221,10 @@ sys.exit(1) fs=args[0] - __builtins__.__debug__=debug if debug: os.environ['Z_DEBUG_MODE']='1' + if detailed: os.environ['STUPID_LOG_SEVERITY']='-99999' + from zLOG import LOG, INFO, ERROR # Try to set uid to "-u" -provided uid. @@ -255,52 +265,67 @@ import zdaemon zdaemon.run(sys.argv, '') - import ZEO.StorageServer, asyncore + try: - storages={} - for o, v in opts: - if o=='-S': - n, m = string.split(v,'=') - if string.find(m,':'): - # we got an attribute name - m, a = string.split(m,':') - else: - # attribute name must be same as storage name - a=n - storages[n]=get_storage(m,a) - - if not storages: - import ZODB.FileStorage - storages['1']=ZODB.FileStorage.FileStorage(fs) + import ZEO.StorageServer, asyncore - # Try to set up a signal handler - try: - import signal + storages={} + for o, v in opts: + if o=='-S': + n, m = string.split(v,'=') + if string.find(m,':'): + # we got an attribute name + m, a = string.split(m,':') + else: + # attribute name must be same as storage name + a=n + storages[n]=get_storage(m,a) + + if not storages: + import ZODB.FileStorage + storages['1']=ZODB.FileStorage.FileStorage(fs) - signal.signal(signal.SIGTERM, - lambda sig, frame, s=storages: shutdown(s) - ) - signal.signal(signal.SIGINT, - lambda sig, frame, s=storages: shutdown(s, 0) - ) - signal.signal(signal.SIGHUP, rotate_logs_handler) + # Try to set up a signal handler + try: + import signal - finally: pass + signal.signal(signal.SIGTERM, + lambda sig, frame, s=storages: shutdown(s) + ) + signal.signal(signal.SIGINT, + lambda sig, frame, s=storages: shutdown(s, 0) + ) + signal.signal(signal.SIGHUP, rotate_logs_handler) - items=storages.items() - items.sort() - for kv in items: - LOG('ZEO Server', INFO, 'Serving %s:\t%s' % kv) + finally: pass - if not unix: unix=host, port + items=storages.items() + items.sort() + for kv in items: + LOG('ZEO Server', INFO, 'Serving %s:\t%s' % kv) - ZEO.StorageServer.StorageServer(unix, storages) + if not unix: unix=host, port + ZEO.StorageServer.StorageServer(unix, storages) - open(zeo_pid,'w').write("%s %s" % (os.getppid(), os.getpid())) - - asyncore.loop() + open(zeo_pid,'w').write("%s %s" % (os.getppid(), os.getpid())) + + asyncore.loop() + except: + # Log startup exception and tell zdaemon not to restart us. + info=sys.exc_info() + try: + import zLOG + zLOG.LOG("z2", zLOG.PANIC, "Startup exception", + error=info) + except: + info2=sys.exc_info() + import traceback + traceback.print_exception(*info) + traceback.print_exception(*info2) + + sys.exit(0) def rotate_logs(): import zLOG From jim at digicool.com Tue Mar 27 18:39:03 2001 From: jim at digicool.com (Jim Fulton) Date: Sun Aug 10 16:31:13 2008 Subject: [ZEO-Checkins] CVS: Packages/ZEO - __init__.py:1.5 Message-ID: <20010327233903.DE8EA510E2@korak.digicool.com> Update of /cvs-repository/Packages/ZEO In directory korak:/tmp/cvs-serv13100 Modified Files: __init__.py Log Message: - It is no longer necessary to symbolically link cPickle or ZServer. ZServer is no longer necessary at all. --- Updated File __init__.py in package Packages/ZEO -- --- __init__.py 2001/01/11 21:15:48 1.4 +++ __init__.py 2001/03/27 23:39:02 1.5 @@ -83,10 +83,4 @@ # ############################################################################## -# if we are using an old version of Python, our asyncore is likely to -# be out of date. If ZServer is sitting around, we can get a current -# version of ayncore from it. In any case, if we are going to be used -# with Zope, it's important to use the version from Zope. -try: import ZServer -except: pass - +import fap # fixup asyncore/cPickle dependencies --- Updated File __init__.py in package Packages/ZEO -- --- __init__.py 2001/01/11 21:15:48 1.4 +++ __init__.py 2001/03/27 23:39:02 1.5 @@ -83,10 +83,4 @@ # ############################################################################## -# if we are using an old version of Python, our asyncore is likely to -# be out of date. If ZServer is sitting around, we can get a current -# version of ayncore from it. In any case, if we are going to be used -# with Zope, it's important to use the version from Zope. -try: import ZServer -except: pass - +import fap # fixup asyncore/cPickle dependencies From jim at digicool.com Tue Mar 27 18:40:30 2001 From: jim at digicool.com (Jim Fulton) Date: Sun Aug 10 16:31:13 2008 Subject: [ZEO-Checkins] CVS: Packages/ZEO - Invalidator.py:1.5 Message-ID: <20010327234030.536F8510E2@korak.digicool.com> Update of /cvs-repository/Packages/ZEO In directory korak:/tmp/cvs-serv13316 Modified Files: Invalidator.py Log Message: It is no longer necessary to symbolically link cPickle or ZServer. ZServer is no longer necessary at all. --- Updated File Invalidator.py in package Packages/ZEO -- --- Invalidator.py 2001/01/11 21:10:36 1.4 +++ Invalidator.py 2001/03/27 23:40:28 1.5 @@ -87,7 +87,7 @@ Note that this is not *really* atomic, but it is close enough. """ -from ZODB import cPickle +import cPickle import tempfile class Invalidator: --- Updated File Invalidator.py in package Packages/ZEO -- --- Invalidator.py 2001/01/11 21:10:36 1.4 +++ Invalidator.py 2001/03/27 23:40:28 1.5 @@ -87,7 +87,7 @@ Note that this is not *really* atomic, but it is close enough. """ -from ZODB import cPickle +import cPickle import tempfile class Invalidator: From jim at digicool.com Tue Mar 27 18:41:53 2001 From: jim at digicool.com (Jim Fulton) Date: Sun Aug 10 16:31:13 2008 Subject: [ZEO-Checkins] CVS: Packages/ZEO - StorageServer.py:1.22 Message-ID: <20010327234153.31172510E2@korak.digicool.com> Update of /cvs-repository/Packages/ZEO In directory korak:/tmp/cvs-serv13528 Modified Files: StorageServer.py Log Message: It is no longer necessary to symbolically link cPickle or ZServer. ZServer is no longer necessary at all. --- Updated File StorageServer.py in package Packages/ZEO -- --- StorageServer.py 2001/01/11 21:55:30 1.21 +++ StorageServer.py 2001/03/27 23:41:51 1.22 @@ -87,8 +87,9 @@ import asyncore, socket, string, sys, os from smac import SizedMessageAsyncConnection -from ZODB import POSException, cPickle -from ZODB.cPickle import Unpickler +from ZODB import POSException +import cPickle +from cPickle import Unpickler from ZODB.POSException import TransactionError, UndoError, VersionCommitError from ZODB.Transaction import Transaction import traceback --- Updated File StorageServer.py in package Packages/ZEO -- --- StorageServer.py 2001/01/11 21:55:30 1.21 +++ StorageServer.py 2001/03/27 23:41:51 1.22 @@ -87,8 +87,9 @@ import asyncore, socket, string, sys, os from smac import SizedMessageAsyncConnection -from ZODB import POSException, cPickle -from ZODB.cPickle import Unpickler +from ZODB import POSException +import cPickle +from cPickle import Unpickler from ZODB.POSException import TransactionError, UndoError, VersionCommitError from ZODB.Transaction import Transaction import traceback From jim at digicool.com Tue Mar 27 18:43:33 2001 From: jim at digicool.com (Jim Fulton) Date: Sun Aug 10 16:31:13 2008 Subject: [ZEO-Checkins] CVS: Packages/ZEO - zrpc.py:1.13 Message-ID: <20010327234333.E03FC510E2@korak.digicool.com> Update of /cvs-repository/Packages/ZEO In directory korak:/tmp/cvs-serv13952 Modified Files: zrpc.py Log Message: It is no longer necessary to symbolically link cPickle or ZServer. ZServer is no longer necessary at all. Fixed a race condition in __call__. If an exception raised on the server which could not be unpickled on the client could cause the client connection to fail. --- Updated File zrpc.py in package Packages/ZEO -- --- zrpc.py 2001/02/23 17:57:50 1.12 +++ zrpc.py 2001/03/27 23:43:32 1.13 @@ -87,8 +87,8 @@ __version__ = "$Revision$"[11:-2] -from ZODB.cPickle import loads -from ZODB import cPickle +from cPickle import loads +import cPickle from thread import allocate_lock from smac import SizedMessageAsyncConnection import socket, string, struct, asyncore, sys, time, select @@ -102,6 +102,9 @@ pickler.fast=1 # Don't use the memo dump=pickler.dump +class UnUnPickleableError(Exception): + "Couldn't unpickle a remote exception" + class asyncRPC(SizedMessageAsyncConnection): __map=0 @@ -119,6 +122,10 @@ self.__r=None l.acquire() + l=allocate_lock() # Response lock used to wait for call results + self.__call_la=l.acquire + self.__call_lr=l.release + def connect(self, tryonce=1, log_type='client'): t=self._tmin connection = self._connection @@ -152,7 +159,7 @@ return 1 def finishConnect(self, s): - SizedMessageAsyncConnection.__init__(self, s, s.getpeername(), {}) + SizedMessageAsyncConnection.__init__(self, s, '', {}) # we are our own socket map! def keys(self): return (self._fileno,) @@ -190,30 +197,37 @@ def __call__(self, *args): - args=dump(args,1) - self.message_output(args) - - if self.__map: self.__Wakeup() # You dumb bastard - else: self.readLoop() - - while 1: - r=self._read() - c=r[:1] - if c=='R': - if r=='RN.': return None # Common case! - return loads(r[1:]) - if c=='E': - r=loads(r[1:]) - if type(r) is TupleType: raise r[0], r[1] - raise r - oob=self._outOfBand - if oob is not None: - r=r[1:] - if r=='N.': r=None # Common case! - else: r=loads(r) - oob(c, r) - else: - raise UnrecognizedResult, r + self.__call_la() + try: + self._last_args=args=dump(args,1) + self.message_output(args) + + if self.__map: self.__Wakeup() # You dumb bastard + else: self.readLoop() + + while 1: + r=self._read() + c=r[:1] + if c=='R': + if r=='RN.': return None # Common case! + return loads(r[1:]) + if c=='E': + try: r=loads(r[1:]) + except: + raise UnUnPickleableError(r[1:]) + if type(r) is TupleType: raise r[0], r[1] + raise r + oob=self._outOfBand + if oob is not None: + r=r[1:] + if r=='N.': r=None # Common case! + else: r=loads(r) + oob(c, r) + else: + raise UnrecognizedResult, r + finally: + self._last_args='' + self.__call_lr() def sendMessage(self, *args): self.message_output(dump(args,1)) --- Updated File zrpc.py in package Packages/ZEO -- --- zrpc.py 2001/02/23 17:57:50 1.12 +++ zrpc.py 2001/03/27 23:43:32 1.13 @@ -87,8 +87,8 @@ __version__ = "$Revision$"[11:-2] -from ZODB.cPickle import loads -from ZODB import cPickle +from cPickle import loads +import cPickle from thread import allocate_lock from smac import SizedMessageAsyncConnection import socket, string, struct, asyncore, sys, time, select @@ -102,6 +102,9 @@ pickler.fast=1 # Don't use the memo dump=pickler.dump +class UnUnPickleableError(Exception): + "Couldn't unpickle a remote exception" + class asyncRPC(SizedMessageAsyncConnection): __map=0 @@ -119,6 +122,10 @@ self.__r=None l.acquire() + l=allocate_lock() # Response lock used to wait for call results + self.__call_la=l.acquire + self.__call_lr=l.release + def connect(self, tryonce=1, log_type='client'): t=self._tmin connection = self._connection @@ -152,7 +159,7 @@ return 1 def finishConnect(self, s): - SizedMessageAsyncConnection.__init__(self, s, s.getpeername(), {}) + SizedMessageAsyncConnection.__init__(self, s, '', {}) # we are our own socket map! def keys(self): return (self._fileno,) @@ -190,30 +197,37 @@ def __call__(self, *args): - args=dump(args,1) - self.message_output(args) - - if self.__map: self.__Wakeup() # You dumb bastard - else: self.readLoop() - - while 1: - r=self._read() - c=r[:1] - if c=='R': - if r=='RN.': return None # Common case! - return loads(r[1:]) - if c=='E': - r=loads(r[1:]) - if type(r) is TupleType: raise r[0], r[1] - raise r - oob=self._outOfBand - if oob is not None: - r=r[1:] - if r=='N.': r=None # Common case! - else: r=loads(r) - oob(c, r) - else: - raise UnrecognizedResult, r + self.__call_la() + try: + self._last_args=args=dump(args,1) + self.message_output(args) + + if self.__map: self.__Wakeup() # You dumb bastard + else: self.readLoop() + + while 1: + r=self._read() + c=r[:1] + if c=='R': + if r=='RN.': return None # Common case! + return loads(r[1:]) + if c=='E': + try: r=loads(r[1:]) + except: + raise UnUnPickleableError(r[1:]) + if type(r) is TupleType: raise r[0], r[1] + raise r + oob=self._outOfBand + if oob is not None: + r=r[1:] + if r=='N.': r=None # Common case! + else: r=loads(r) + oob(c, r) + else: + raise UnrecognizedResult, r + finally: + self._last_args='' + self.__call_lr() def sendMessage(self, *args): self.message_output(dump(args,1)) From jim at digicool.com Tue Mar 27 18:46:42 2001 From: jim at digicool.com (Jim Fulton) Date: Sun Aug 10 16:31:13 2008 Subject: [ZEO-Checkins] CVS: Packages/ZEO - ZopeREADME.txt:1.1 Message-ID: <20010327234642.578DF510E2@korak.digicool.com> Update of /cvs-repository/Packages/ZEO/doc In directory korak:/tmp/cvs-serv14385 Added Files: ZopeREADME.txt Log Message: *** empty log message *** --- Added File ZopeREADME.txt in package Packages/ZEO --- Zope Enterprize Objects Put this package (the ZEO directory, without any wrapping directory included in a distribution) in your Zope lib/python. Starting (and configuring) the ZEO Server To start the storage server, go to your Zope install directory and:: python lib/python/ZEO/start.py -p port_number (Run start without arguments to see options.) Of course, the server and the client don't have to be on the same machine. If the server and client *are* on the same machine, then you can use a Unix domain socket:: python lib/python/ZEO/start.py -U filename The start script provides a number of options not documented here. See doc/start.txt for more information. Running Zope as a ZEO client To get Zope to use the server, create a custom_zodb module, custom_zodb.py, in your Zope install directory, so that Zope uses a ClientStorage:: import ZEO.ClientStorage Storage=ZEO.ClientStorage.ClientStorage(('',port_number)) (See the misc/custom_zodb.py for an example.) You can specify a host name (rather than '') if you want. The port number is, of course, the port number used to start the storage server. You can also give the name of a Unix domain socket file:: import ZEO.ClientStorage Storage=ZEO.ClientStorage.ClientStorage(filename) There are a number of configuration options available for the ClientStorage. See doc/ClientStorage.txt for details. If you want a persistent client cache which retains cache contents across ClientStorage restarts, you need to define the environment variable, ZEO_CLIENT, to a unique name for the client. This is needed so that unique cache name files can be computed. Otherwise, the client cache is stored in temporary files which are removed when the ClientStorage shuts down. For example, to start two Zope processes with unique caches, use something like: python z2.py -P8700 ZEO_CLIENT=8700 python z2.py -P8800 ZEO_CLIENT=8800 Zope product installation Normally, Zope updates the Zope database during startup to reflect product changes or new products found. It makes no sense for multiple ZEO clients to do the same installation. Further, if different clients have different software installed, the correct state of the database is ambiguous. Starting in Zope 2.2, Zope will not modify the Zope database during product installation if the environment variable ZEO_CLIENT is set. Normally, Zope ZEO clients should be run with ZEO_CLIENT set so that product initialization is not performed. If you do install new Zope products, then you need to take a special step to cause the new products to be properly registered in the database. The easiest way to do this is to start Zope once without ZEO_CLIENT set. The interaction between ZEO and Zope product installation is unfortunate. In the future, this interaction will be removed by --- Added File ZopeREADME.txt in package Packages/ZEO --- Zope Enterprize Objects Put this package (the ZEO directory, without any wrapping directory included in a distribution) in your Zope lib/python. Starting (and configuring) the ZEO Server To start the storage server, go to your Zope install directory and:: python lib/python/ZEO/start.py -p port_number (Run start without arguments to see options.) Of course, the server and the client don't have to be on the same machine. If the server and client *are* on the same machine, then you can use a Unix domain socket:: python lib/python/ZEO/start.py -U filename The start script provides a number of options not documented here. See doc/start.txt for more information. Running Zope as a ZEO client To get Zope to use the server, create a custom_zodb module, custom_zodb.py, in your Zope install directory, so that Zope uses a ClientStorage:: import ZEO.ClientStorage Storage=ZEO.ClientStorage.ClientStorage(('',port_number)) (See the misc/custom_zodb.py for an example.) You can specify a host name (rather than '') if you want. The port number is, of course, the port number used to start the storage server. You can also give the name of a Unix domain socket file:: import ZEO.ClientStorage Storage=ZEO.ClientStorage.ClientStorage(filename) There are a number of configuration options available for the ClientStorage. See doc/ClientStorage.txt for details. If you want a persistent client cache which retains cache contents across ClientStorage restarts, you need to define the environment variable, ZEO_CLIENT, to a unique name for the client. This is needed so that unique cache name files can be computed. Otherwise, the client cache is stored in temporary files which are removed when the ClientStorage shuts down. For example, to start two Zope processes with unique caches, use something like: python z2.py -P8700 ZEO_CLIENT=8700 python z2.py -P8800 ZEO_CLIENT=8800 Zope product installation Normally, Zope updates the Zope database during startup to reflect product changes or new products found. It makes no sense for multiple ZEO clients to do the same installation. Further, if different clients have different software installed, the correct state of the database is ambiguous. Starting in Zope 2.2, Zope will not modify the Zope database during product installation if the environment variable ZEO_CLIENT is set. Normally, Zope ZEO clients should be run with ZEO_CLIENT set so that product initialization is not performed. If you do install new Zope products, then you need to take a special step to cause the new products to be properly registered in the database. The easiest way to do this is to start Zope once without ZEO_CLIENT set. The interaction between ZEO and Zope product installation is unfortunate. In the future, this interaction will be removed by From klm at serenade.digicool.com Tue Mar 27 18:54:22 2001 From: klm at serenade.digicool.com (klm@serenade.digicool.com) Date: Sun Aug 10 16:31:13 2008 Subject: [ZEO-Checkins] CVS: Packages/ZEO - README:1.16 Message-ID: <200103272354.SAA31618@serenade.digicool.com> Update of /cvs-repository/Packages/ZEO In directory serenade.digicool.com:/home/klm/checkouts/ZEO Modified Files: README Log Message: Small (but prominent) spelling correction. --- Updated File README in package Packages/ZEO -- --- README 2000/09/27 11:21:20 1.15 +++ README 2001/03/27 23:54:21 1.16 @@ -1,4 +1,4 @@ -Zope Enterprize Objects, ZEO 0.2 +Zope Enterprise Objects, ZEO 0.2 Put this package (the ZEO directory, without any wrapping directory included in a distribution) in your Zope lib/python. --- Updated File README in package Packages/ZEO -- --- README 2000/09/27 11:21:20 1.15 +++ README 2001/03/27 23:54:21 1.16 @@ -1,4 +1,4 @@ -Zope Enterprize Objects, ZEO 0.2 +Zope Enterprise Objects, ZEO 0.2 Put this package (the ZEO directory, without any wrapping directory included in a distribution) in your Zope lib/python. From klm at serenade.digicool.com Tue Mar 27 18:54:43 2001 From: klm at serenade.digicool.com (klm@serenade.digicool.com) Date: Sun Aug 10 16:31:13 2008 Subject: [ZEO-Checkins] CVS: Packages/ZEO - ZopeREADME.txt:1.2 Message-ID: <200103272354.SAA31642@serenade.digicool.com> Update of /cvs-repository/Packages/ZEO/doc In directory serenade.digicool.com:/home/klm/checkouts/ZEO/doc Modified Files: ZopeREADME.txt Log Message: Small (but prominent) spelling correction. --- Updated File ZopeREADME.txt in package Packages/ZEO -- --- ZopeREADME.txt 2001/03/27 23:46:40 1.1 +++ ZopeREADME.txt 2001/03/27 23:54:42 1.2 @@ -1,4 +1,4 @@ -Zope Enterprize Objects +Zope Enterprise Objects Put this package (the ZEO directory, without any wrapping directory included in a distribution) in your Zope lib/python. --- Updated File ZopeREADME.txt in package Packages/ZEO -- --- ZopeREADME.txt 2001/03/27 23:46:40 1.1 +++ ZopeREADME.txt 2001/03/27 23:54:42 1.2 @@ -1,4 +1,4 @@ -Zope Enterprize Objects +Zope Enterprise Objects Put this package (the ZEO directory, without any wrapping directory included in a distribution) in your Zope lib/python. From jim at digicool.com Tue Mar 27 19:12:59 2001 From: jim at digicool.com (Jim Fulton) Date: Sun Aug 10 16:31:13 2008 Subject: [ZEO-Checkins] CVS: Packages/ZEO - NonZopeREADME.txt:1.1 Message-ID: <20010328001259.AB604510E2@korak.digicool.com> Update of /cvs-repository/Packages/ZEO/doc In directory korak:/tmp/cvs-serv18445 Added Files: NonZopeREADME.txt Log Message: *** empty log message *** --- Added File NonZopeREADME.txt in package Packages/ZEO --- Zope Enterprize Objects Put this package (the ZEO directory, without any wrapping directory included in a distribution) in your python path. Starting (and configuring) the ZEO Server To start the storage server, go to your Zope install directory and:: python ZEO/start.py -p port_number (Run start without arguments to see options.) Of course, the server and the client don't have to be on the same machine. If the server and client *are* on the same machine, then you can use a Unix domain socket:: python ZEO/start.py -U filename The start script provides a number of options not documented here. See doc/start.txt for more information. Running a ZEO client In your application, create a ClientStorage, rather than, say, a FileStorage: import ZODB, ZEO.ClientStorage Storage=ZEO.ClientStorage.ClientStorage(('',port_number)) db=ZODB.DB(Storage) You can specify a host name (rather than '') if you want. The port number is, of course, the port number used to start the storage server. You can also give the name of a Unix domain socket file:: import ZODB, ZEO.ClientStorage Storage=ZEO.ClientStorage.ClientStorage(filename) db=ZODB.DB(Storage) There are a number of configuration options available for the ClientStorage. See doc/ClientStorage.txt for details. If you want a persistent client cache which retains cache contents across ClientStorage restarts, you need to define the environment variable, ZEO_CLIENT, to a unique name for the client. This is needed so that unique cache name files can be computed. Otherwise, the client cache is stored in temporary files which are removed when the ClientStorage shuts down. Dependencies on other modules - The module, ThreadedAsync must be in the python path. - The zdaemon module is necessary if you want to run your storage server as a daemon that automatically restarts itself if there is a fatal error. - The zLOG module provides a handy logging capability. If you are using a version of Python before Python 2: - ZServer should be in the Python path, or you should copy the version of asyncore.py from ZServer (from Zope 2.2 or CVS) to your Python path, or you should copy a version of a asyncore from the medusa CVS tree to your Python path. A recent change in asyncore is required. - The version of cPickle from Zope, or from the python.org CVS tree must be used. It has a hook to provide control over which "global objects" (e.g. classes) may be pickled. --- Added File NonZopeREADME.txt in package Packages/ZEO --- Zope Enterprize Objects Put this package (the ZEO directory, without any wrapping directory included in a distribution) in your python path. Starting (and configuring) the ZEO Server To start the storage server, go to your Zope install directory and:: python ZEO/start.py -p port_number (Run start without arguments to see options.) Of course, the server and the client don't have to be on the same machine. If the server and client *are* on the same machine, then you can use a Unix domain socket:: python ZEO/start.py -U filename The start script provides a number of options not documented here. See doc/start.txt for more information. Running a ZEO client In your application, create a ClientStorage, rather than, say, a FileStorage: import ZODB, ZEO.ClientStorage Storage=ZEO.ClientStorage.ClientStorage(('',port_number)) db=ZODB.DB(Storage) You can specify a host name (rather than '') if you want. The port number is, of course, the port number used to start the storage server. You can also give the name of a Unix domain socket file:: import ZODB, ZEO.ClientStorage Storage=ZEO.ClientStorage.ClientStorage(filename) db=ZODB.DB(Storage) There are a number of configuration options available for the ClientStorage. See doc/ClientStorage.txt for details. If you want a persistent client cache which retains cache contents across ClientStorage restarts, you need to define the environment variable, ZEO_CLIENT, to a unique name for the client. This is needed so that unique cache name files can be computed. Otherwise, the client cache is stored in temporary files which are removed when the ClientStorage shuts down. Dependencies on other modules - The module, ThreadedAsync must be in the python path. - The zdaemon module is necessary if you want to run your storage server as a daemon that automatically restarts itself if there is a fatal error. - The zLOG module provides a handy logging capability. If you are using a version of Python before Python 2: - ZServer should be in the Python path, or you should copy the version of asyncore.py from ZServer (from Zope 2.2 or CVS) to your Python path, or you should copy a version of a asyncore from the medusa CVS tree to your Python path. A recent change in asyncore is required. - The version of cPickle from Zope, or from the python.org CVS tree must be used. It has a hook to provide control over which "global objects" (e.g. classes) may be pickled. From jim at digicool.com Tue Mar 27 19:15:15 2001 From: jim at digicool.com (Jim Fulton) Date: Sun Aug 10 16:31:13 2008 Subject: [ZEO-Checkins] CVS: Packages/ZEO - README:1.17 Message-ID: <20010328001515.C02D5510E2@korak.digicool.com> Update of /cvs-repository/Packages/ZEO In directory korak:/tmp/cvs-serv18922 Modified Files: README Log Message: Split Zope and non-Zope readme files. --- Updated File README in package Packages/ZEO -- --- README 2001/03/27 23:54:21 1.16 +++ README 2001/03/28 00:15:14 1.17 @@ -1,120 +1,4 @@ -Zope Enterprise Objects, ZEO 0.2 +Zope Enterprise Objects - Put this package (the ZEO directory, without any wrapping directory - included in a distribution) in your Zope lib/python. - - Note -- This release of ZEO requires Zope 2.2.2 or a CVS checkout - of Zope. See 'CHANGES.txt' for details. - - You also need to symbolically link (or copy) ZServer to lib/python:: - - cd lib/python - ln -s ../../ZServer . - - Finally, you need to link the cPickle extension from your lib/python - directory into your ZEO directory:: - - cd lib/python/ZEO - ln -s ../cPickle.so . - - This is necessary because ZEO uses a (relatively) new - cPickle feature that wasn't included in Python 1.5.2. - - Starting (and configuring) the ZEO Server - - To start the storage server, go to your Zope install directory and:: - - python lib/python/ZEO/start.py -p port_number - - (Run start without arguments to see options.) - - Of course, the server and the client don't have to be on the same - machine. - - If the server and client *are* on the same machine, then you can use - a Unix domain socket:: - - python lib/python/ZEO/start.py -U filename - - The start script provides a number of options not documented here. - See doc/start.txt for more information. - - Running Zope as a ZEO client - - To get Zope to use the server, create a custom_zodb module, - custom_zodb.py, in your Zope install directory, so that Zope uses a - ClientStorage:: - - import ZEO.ClientStorage - Storage=ZEO.ClientStorage.ClientStorage(('',port_number)) - - (See the misc/custom_zodb.py for an example.) - - You can specify a host name (rather than '') if you want. The port - number is, of course, the port number used to start the storage - server. - - You can also give the name of a Unix domain socket file:: - - import ZEO.ClientStorage - Storage=ZEO.ClientStorage.ClientStorage(filename) - - There are a number of configuration options available for the - ClientStorage. See doc/ClientStorage.txt for details. - - If you want a persistent client cache which retains cache contents - across ClientStorage restarts, you need to define the environment - variable, ZEO_CLIENT, to a unique name for the client. This is - needed so that unique cache name files can be computed. Otherwise, - the client cache is stored in temporary files which are removed when - the ClientStorage shuts down. For example, to start two Zope - processes with unique caches, use something like: - - python z2.py -P8700 ZEO_CLIENT=8700 - python z2.py -P8800 ZEO_CLIENT=8800 - - Zope product installation - - Normally, Zope updates the Zope database during startup to reflect - product changes or new products found. It makes no sense for - multiple ZEO clients to do the same installation. Further, if - different clients have different software installed, the correct - state of the database is ambiguous. - - Starting in Zope 2.2, Zope will not modify the Zope database - during product installation if the environment variable ZEO_CLIENT - is set. - - Normally, Zope ZEO clients should be run with ZEO_CLIENT set so - that product initialization is not performed. - - If you do install new Zope products, then you need to take a - special step to cause the new products to be properly registered - in the database. The easiest way to do this is to start Zope - once without ZEO_CLIENT set. - - The interaction between ZEO and Zope product installation is - unfortunate. In the future, this interaction will be removed by - - Notes for non Zope users - - First, we regret the amount of dependence on Zope. We intend for - the dependence to decrease in the long run. - - Known dependencies: - - - The Zope lib/python/Shared directory must be in the Python - path. This is due to a lame dependency on some Zope XML - facilities used by ZODB for XML export and import. - - - ZServer should be in the Python path, or you should copy the - version of asyncore.py from ZServer (from Zope 2.2 or CVS) to - your Python path, or you should copy a version of a asyncore - from the medusa CVS tree to your Python path. A recent change - in asyncore is required. - - - The module, ThreadedAsync must be in the python path. - - - The version of cPickle from Zope, or from the python.org CVS - tree must be used. It has a hook to provide control over which - "global objects" (e.g. classes) may be pickled. + If you are using Zope, see docs/ZopeREADME.txt, otherwise, see + docs/NonZopeREADME.txt. --- Updated File README in package Packages/ZEO -- --- README 2001/03/27 23:54:21 1.16 +++ README 2001/03/28 00:15:14 1.17 @@ -1,120 +1,4 @@ -Zope Enterprise Objects, ZEO 0.2 +Zope Enterprise Objects - Put this package (the ZEO directory, without any wrapping directory - included in a distribution) in your Zope lib/python. - - Note -- This release of ZEO requires Zope 2.2.2 or a CVS checkout - of Zope. See 'CHANGES.txt' for details. - - You also need to symbolically link (or copy) ZServer to lib/python:: - - cd lib/python - ln -s ../../ZServer . - - Finally, you need to link the cPickle extension from your lib/python - directory into your ZEO directory:: - - cd lib/python/ZEO - ln -s ../cPickle.so . - - This is necessary because ZEO uses a (relatively) new - cPickle feature that wasn't included in Python 1.5.2. - - Starting (and configuring) the ZEO Server - - To start the storage server, go to your Zope install directory and:: - - python lib/python/ZEO/start.py -p port_number - - (Run start without arguments to see options.) - - Of course, the server and the client don't have to be on the same - machine. - - If the server and client *are* on the same machine, then you can use - a Unix domain socket:: - - python lib/python/ZEO/start.py -U filename - - The start script provides a number of options not documented here. - See doc/start.txt for more information. - - Running Zope as a ZEO client - - To get Zope to use the server, create a custom_zodb module, - custom_zodb.py, in your Zope install directory, so that Zope uses a - ClientStorage:: - - import ZEO.ClientStorage - Storage=ZEO.ClientStorage.ClientStorage(('',port_number)) - - (See the misc/custom_zodb.py for an example.) - - You can specify a host name (rather than '') if you want. The port - number is, of course, the port number used to start the storage - server. - - You can also give the name of a Unix domain socket file:: - - import ZEO.ClientStorage - Storage=ZEO.ClientStorage.ClientStorage(filename) - - There are a number of configuration options available for the - ClientStorage. See doc/ClientStorage.txt for details. - - If you want a persistent client cache which retains cache contents - across ClientStorage restarts, you need to define the environment - variable, ZEO_CLIENT, to a unique name for the client. This is - needed so that unique cache name files can be computed. Otherwise, - the client cache is stored in temporary files which are removed when - the ClientStorage shuts down. For example, to start two Zope - processes with unique caches, use something like: - - python z2.py -P8700 ZEO_CLIENT=8700 - python z2.py -P8800 ZEO_CLIENT=8800 - - Zope product installation - - Normally, Zope updates the Zope database during startup to reflect - product changes or new products found. It makes no sense for - multiple ZEO clients to do the same installation. Further, if - different clients have different software installed, the correct - state of the database is ambiguous. - - Starting in Zope 2.2, Zope will not modify the Zope database - during product installation if the environment variable ZEO_CLIENT - is set. - - Normally, Zope ZEO clients should be run with ZEO_CLIENT set so - that product initialization is not performed. - - If you do install new Zope products, then you need to take a - special step to cause the new products to be properly registered - in the database. The easiest way to do this is to start Zope - once without ZEO_CLIENT set. - - The interaction between ZEO and Zope product installation is - unfortunate. In the future, this interaction will be removed by - - Notes for non Zope users - - First, we regret the amount of dependence on Zope. We intend for - the dependence to decrease in the long run. - - Known dependencies: - - - The Zope lib/python/Shared directory must be in the Python - path. This is due to a lame dependency on some Zope XML - facilities used by ZODB for XML export and import. - - - ZServer should be in the Python path, or you should copy the - version of asyncore.py from ZServer (from Zope 2.2 or CVS) to - your Python path, or you should copy a version of a asyncore - from the medusa CVS tree to your Python path. A recent change - in asyncore is required. - - - The module, ThreadedAsync must be in the python path. - - - The version of cPickle from Zope, or from the python.org CVS - tree must be used. It has a hook to provide control over which - "global objects" (e.g. classes) may be pickled. + If you are using Zope, see docs/ZopeREADME.txt, otherwise, see + docs/NonZopeREADME.txt. From klm at serenade.digicool.com Wed Mar 28 16:55:08 2001 From: klm at serenade.digicool.com (klm@serenade.digicool.com) Date: Sun Aug 10 16:31:13 2008 Subject: [ZEO-Checkins] CVS: Packages/ZEO - start.py:1.18 Message-ID: <200103282155.QAA04652@serenade.digicool.com> Update of /cvs-repository/Packages/ZEO In directory serenade.digicool.com:/usr/local/dc/ZopeSoftwareHome/lib/python/ZEO Modified Files: start.py Log Message: Retain backwards python1.5.2 compatibility - convert python2 function call: 'func(*args)' to old, python1.5.2: 'apply(func, args)'. (When working on pre-zope2.4 code, you should avoid developing in an environemnt where python2 is the default python...) --- Updated File start.py in package Packages/ZEO -- --- start.py 2001/03/27 23:37:33 1.17 +++ start.py 2001/03/28 21:55:06 1.18 @@ -322,8 +322,8 @@ except: info2=sys.exc_info() import traceback - traceback.print_exception(*info) - traceback.print_exception(*info2) + apply(traceback.print_exception, info) + apply(traceback.print_exception, info2) sys.exit(0) --- Updated File start.py in package Packages/ZEO -- --- start.py 2001/03/27 23:37:33 1.17 +++ start.py 2001/03/28 21:55:06 1.18 @@ -322,8 +322,8 @@ except: info2=sys.exc_info() import traceback - traceback.print_exception(*info) - traceback.print_exception(*info2) + apply(traceback.print_exception, info) + apply(traceback.print_exception, info2) sys.exit(0) From jeremy at digicool.com Thu Mar 29 08:29:04 2001 From: jeremy at digicool.com (jeremy@digicool.com) Date: Sun Aug 10 16:31:13 2008 Subject: [ZEO-Checkins] CVS: Packages/ZEO - ClientStorage.py:1.26.4.2 Message-ID: <20010329132904.458CB510E2@korak.digicool.com> Update of /cvs-repository/Packages/ZEO In directory korak:/tmp/cvs-serv9299 Modified Files: Tag: ZEO-ZRPC-Dev ClientStorage.py Log Message: cleanup: remove debugging print; add whitespace --- Updated File ClientStorage.py in package Packages/ZEO -- --- ClientStorage.py 2001/03/17 00:35:11 1.26.4.1 +++ ClientStorage.py 2001/03/29 13:29:01 1.26.4.2 @@ -248,10 +248,9 @@ pass def becomeAsync(self, map): - print "becomeAsync()" self._lock_acquire() try: - self._async=1 + self._async = 1 if self._connected: self._call.setLoop(map, getWakeup()) self.__begin='tpc_begin' --- Updated File ClientStorage.py in package Packages/ZEO -- --- ClientStorage.py 2001/03/17 00:35:11 1.26.4.1 +++ ClientStorage.py 2001/03/29 13:29:01 1.26.4.2 @@ -248,10 +248,9 @@ pass def becomeAsync(self, map): - print "becomeAsync()" self._lock_acquire() try: - self._async=1 + self._async = 1 if self._connected: self._call.setLoop(map, getWakeup()) self.__begin='tpc_begin' From jeremy at digicool.com Thu Mar 29 08:30:34 2001 From: jeremy at digicool.com (jeremy@digicool.com) Date: Sun Aug 10 16:31:13 2008 Subject: [ZEO-Checkins] CVS: Packages/ZEO - smac.py:1.9.6.2 Message-ID: <20010329133034.EDA4351014@korak.digicool.com> Update of /cvs-repository/Packages/ZEO In directory korak:/tmp/cvs-serv9470 Modified Files: Tag: ZEO-ZRPC-Dev smac.py Log Message: changes to aid debugging: commented out "read_inprogress" checks handler for unpack exceptions --- Updated File smac.py in package Packages/ZEO -- --- smac.py 2001/03/17 00:25:36 1.9.6.1 +++ smac.py 2001/03/29 13:30:33 1.9.6.2 @@ -119,6 +119,8 @@ def __nonzero__(self): return 1 +## __read_inprogress = None + def handle_read(self): # Use a single __inp buffer and integer indexes to make this # fast. @@ -126,8 +128,10 @@ if not d: return +## self.__read_inprogress = 1 input_len = self.__input_len + len(d) msg_size = self.__msg_size + state = self.__state inp = self.__inp if msg_size > input_len: @@ -151,17 +155,28 @@ offset = 0 while (offset + msg_size) <= input_len: - msg = inp[offset:offset+msg_size] +## if state is None and msg_size != 4: +## print "OUT OF LUCK" + msg = inp[offset:offset + msg_size] offset = offset + msg_size - if self.__state is None: + if state is None: # waiting for message - msg_size = struct.unpack(">i", msg)[0] - self.__state = 1 + try: + msg_size = struct.unpack(">i", msg)[0] + except: + # XXX + print 'unpack(">i", %s)' % repr(msg) + print "state = %s" % state + print "msg_size = %(msg_size)d, "\ + "input_len= %(input_len)d" % locals() + raise + state = 1 else: msg_size = 4 - self.__state = None + state = None self.message_input(msg) - + + self.__state = state self.__msg_size = msg_size self.__inp = inp[offset:] self.__input_len = input_len - offset @@ -209,8 +224,10 @@ return 1 def writable(self): - # inscrutable - return not not self.__output + if len(self.__output) == 0: + return 0 + else: + return 1 def handle_write(self): output = self.__output @@ -222,6 +239,7 @@ break # we can't write any more else: del output[0] +# LOG(self._debug, INFO, "output written; remain=%d" % len(self.__output)) def handle_close(self): self.close() @@ -242,6 +260,7 @@ # do two separate appends to avoid copying the message string self.__output.append(struct.pack(">i", len(message))) self.__output.append(message) +# LOG(self._debug, INFO, "output remain=%d" % len(self.__output)) def log_info(self, message, type='info'): if type == 'error': --- Updated File smac.py in package Packages/ZEO -- --- smac.py 2001/03/17 00:25:36 1.9.6.1 +++ smac.py 2001/03/29 13:30:33 1.9.6.2 @@ -119,6 +119,8 @@ def __nonzero__(self): return 1 +## __read_inprogress = None + def handle_read(self): # Use a single __inp buffer and integer indexes to make this # fast. @@ -126,8 +128,10 @@ if not d: return +## self.__read_inprogress = 1 input_len = self.__input_len + len(d) msg_size = self.__msg_size + state = self.__state inp = self.__inp if msg_size > input_len: @@ -151,17 +155,28 @@ offset = 0 while (offset + msg_size) <= input_len: - msg = inp[offset:offset+msg_size] +## if state is None and msg_size != 4: +## print "OUT OF LUCK" + msg = inp[offset:offset + msg_size] offset = offset + msg_size - if self.__state is None: + if state is None: # waiting for message - msg_size = struct.unpack(">i", msg)[0] - self.__state = 1 + try: + msg_size = struct.unpack(">i", msg)[0] + except: + # XXX + print 'unpack(">i", %s)' % repr(msg) + print "state = %s" % state + print "msg_size = %(msg_size)d, "\ + "input_len= %(input_len)d" % locals() + raise + state = 1 else: msg_size = 4 - self.__state = None + state = None self.message_input(msg) - + + self.__state = state self.__msg_size = msg_size self.__inp = inp[offset:] self.__input_len = input_len - offset @@ -209,8 +224,10 @@ return 1 def writable(self): - # inscrutable - return not not self.__output + if len(self.__output) == 0: + return 0 + else: + return 1 def handle_write(self): output = self.__output @@ -222,6 +239,7 @@ break # we can't write any more else: del output[0] +# LOG(self._debug, INFO, "output written; remain=%d" % len(self.__output)) def handle_close(self): self.close() @@ -242,6 +260,7 @@ # do two separate appends to avoid copying the message string self.__output.append(struct.pack(">i", len(message))) self.__output.append(message) +# LOG(self._debug, INFO, "output remain=%d" % len(self.__output)) def log_info(self, message, type='info'): if type == 'error': From jeremy at digicool.com Thu Mar 29 08:34:33 2001 From: jeremy at digicool.com (jeremy@digicool.com) Date: Sun Aug 10 16:31:14 2008 Subject: [ZEO-Checkins] CVS: Packages/ZEO - zrpc2.py:1.1.2.2 Message-ID: <20010329133433.88C1F510E2@korak.digicool.com> Update of /cvs-repository/Packages/ZEO In directory korak:/tmp/cvs-serv10202 Modified Files: Tag: ZEO-ZRPC-Dev zrpc2.py Log Message: Add proper mainloop maintenance to zrpc2 Add _do_io() method to Connection that handles logic for when to trigger a ThreadedAsync loop and when to call poll directly. Hack alert: Add ServerConnection() that has empty _do_io() method. On the server side, the connection is driven by a single asyncore.poll() and doesn't need to explicitly restart loops. Fix name errors in exception handlers; err namespace no longer exists. Add DebugLock that adds logging around acquire and release calls. XXX Depends on sys._getframe() Use thread lock instead of threading lock; re-entrant is not what is needed. Fiddle with logging calls. --- Updated File zrpc2.py in package Packages/ZEO -- --- zrpc2.py 2001/03/17 00:14:51 1.1.2.1 +++ zrpc2.py 2001/03/29 13:34:32 1.1.2.2 @@ -1,4 +1,4 @@ -"""RPC protocol for ZEO +"""RPC protocol for ZEO based on asyncore The basic protocol is as: a pickled tuple containing: msgid, flags, method, args @@ -18,6 +18,7 @@ import socket import sys import threading +import thread import time import traceback import types @@ -26,6 +27,7 @@ from ZODB import POSException import smac +import trigger import zLOG REPLY = ".reply" # message name used for replies @@ -82,7 +84,7 @@ try: msgid, flags, name, args = unpickler.load() except (cPickle.UnpicklingError, IndexError), msg: - raise err.DecodingError(msg) + raise DecodingError(msg) return msgid, flags, name, args class Delay: @@ -99,6 +101,31 @@ def reply(self, obj): self.send_reply(self.msgid, obj) + +class DebugLock: + def __init__(self): + self.lock = thread.allocate_lock() + + def _debug(self): + method = sys._getframe().f_back + caller = method.f_back + filename = os.path.split(caller.f_code.co_filename)[1] + log("LOCK %s: %s called by %s, %s, line %s" % (id(self.lock), + method.f_code.co_name, + caller.f_code.co_name, + filename, + caller.f_lineno)) + + def acquire(self, wait=None): + self._debug() + if wait is not None: + return self.lock.acquire(wait) + else: + return self.lock.acquire() + + def release(self): + self._debug() + return self.lock.release() class Connection(smac.SizedMessageAsyncConnection): """Dispatcher for RPC on object @@ -122,9 +149,10 @@ self.obj = obj self.marshal = Marshaller(pickle) self.closed = 0 + self.async = 0 # The reply lock is used to block when a synchronous call is # waiting for a response - self.__reply_lock = threading.Lock() + self.__reply_lock = thread.allocate_lock() self.__reply_lock.acquire() self.__super_init(sock, addr) if isinstance(obj, Handler): @@ -146,24 +174,26 @@ """Decoding an incoming message and dispatch it""" try: msgid, flags, name, args = self.marshal.decode(message) - except err.DecodingError, msg: + except DecodingError, msg: return self.return_error(None, None, sys.exc_info()[0], sys.exc_info()[1]) + log("message: %s, %s, %s, %s" % (msgid, flags, name, repr(args)[:40]), + level=zLOG.TRACE) if name == REPLY: self.handle_reply(msgid, flags, args) else: self.handle_request(msgid, flags, name, args) def handle_reply(self, msgid, flags, args): + log("reply: %s, %s, %s" % (msgid, flags, str(args)[:40])) self.__reply = msgid, flags, args -# self.__lock.release() + self.__reply_lock.release() # will fail if lock is unlocked def handle_request(self, msgid, flags, name, args): - if __debug__: - log("%s%s" % (name, args), zLOG.TRACE) + log("%s%s" % (name, repr(args)[:40]), zLOG.BLATHER) if not self.check_method(name): - raise err.ZRPCError("Invalid method name: %s" % name) + raise ZRPCError("Invalid method name: %s" % name) meth = getattr(self.obj, name) try: @@ -185,8 +215,9 @@ if flags & ASYNC: if ret is not None: - raise err.ZRPCError("async method returned value") + raise ZRPCError("async method returned value") else: + log("%s reply %s" % (name, repr(ret)[:40]), zLOG.BLATHER) if isinstance(ret, Delay): ret.set_sender(msgid, self.send_reply) else: @@ -194,7 +225,6 @@ def handle_error(self): t, v, tb = sys.exc_info() - print "%s: %s" % (str(t), v) traceback.print_tb(tb) def check_method(self, name): @@ -205,6 +235,10 @@ self.message_output(msg) def return_error(self, msgid, flags, err_type, err_value): + if flags is None: + print "Exception raised during decoding" + self.handle_error() + return if flags & ASYNC: print "Asynchronous call raised exception:" self.handle_error() @@ -220,6 +254,7 @@ err = ZRPCError("Couldn't pickle error %s" % `err_value`) msg = self.marshal.encode(msgid, 0, REPLY, (ZRPCError, err)) self.message_output(msg) + self._do_io() print "Sent error message for:" self.handle_error() @@ -244,10 +279,6 @@ # The previous five methods implement an asyncore socket map - def _mainloop(self): - """Invoke the asyncore mainloop""" - - # The next two methods are used by clients to invoke methods on # remote objects @@ -258,17 +289,17 @@ self.message_output(self.marshal.encode(msgid, 0, method, args)) self.__reply = None - while self.__reply is None: - # this is where you want to call the main loop - asyncore.poll(60.0) + self._do_io(wait=1) r_msgid, r_flags, r_args = self.__reply + self.__reply_lock.acquire() assert r_msgid == msgid, "%s != %s: %s" % (r_msgid, msgid, r_args) + if type(r_args) == types.TupleType \ and type(r_args[0]) == types.ClassType \ and issubclass(r_args[0], Exception): - log("error") - print repr(r_args[1]) + log("call %s %s raised error" % (msgid, method)) raise r_args[1] + log("call %s %s returned" % (msgid, method)) return r_args def callAsync(self, method, *args): @@ -276,6 +307,33 @@ self.msgid += 1 log("async %s %s" % (msgid, method)) self.message_output(self.marshal.encode(msgid, ASYNC, method, args)) + self._do_io() + + # handle IO, possibly in async mode + + def _do_io(self, wait=0): # XXX need better name + # XXX invariant? lock must be held when calling with wait==1 + # otherwise, in non-async mode, there will be no poll + + log("_do_io(wait=%d), async=%d" % (wait, self.async), + level=zLOG.BLATHER) + if self.async: + self.trigger.pull_trigger() + if wait: + self.__reply_lock.acquire() + else: + if wait: + # do loop only if lock is already acquired + while not self.__reply_lock.acquire(0): + asyncore.poll(60.0, self) + self.__reply_lock.release() + else: + asyncore.poll(0.0, self) + +class ServerConnection(Connection): + def _do_io(self, wait=0): + """If this is a server, there is no explicit IO to do""" + pass class ConnectionManager: """Keeps a connection up over time""" @@ -293,6 +351,10 @@ def register_object(self, obj): self.obj = obj + def set_async(self): + self.async = 1 + self.trigger = trigger.trigger() + def connect(self, sync=0, callback=None): if self.connected == 1: return @@ -330,6 +392,7 @@ log("Connected to server", level=zLOG.DEBUG) self.connected = 1 if self.connected: + # XXX how do we get here with s being defined? c = ManagedConnection(s, self.addr, self.obj, self) log("Connection created: %s" % c) log("callback = %s" % self._callback) @@ -345,8 +408,22 @@ return t def closed(self, conn): + self.connected = 0 self.connect() +class ManagedServerConnection(ServerConnection): + """A connection that notifies its ConnectionManager of closing""" + __super_init = Connection.__init__ + __super_close = Connection.close + + def __init__(self, sock, addr, obj, mgr, pickle=None): + self.__mgr = mgr + self.__super_init(sock, addr, obj, pickle) + + def close(self): + self.__super_close() + self.__mgr.closed(self) + class ManagedConnection(Connection): """A connection that notifies its ConnectionManager of closing""" __super_init = Connection.__init__ @@ -418,12 +495,12 @@ try: m = __import__(module, _globals, _globals, _silly) except ImportError, msg: - raise err.ZRPCError("import error %s: %s" % (module, msg)) + raise ZRPCError("import error %s: %s" % (module, msg)) try: r = getattr(m, name) except AttributeError: - raise err.ZRPCError("module %s has no global %s" % (module, name)) + raise ZRPCError("module %s has no global %s" % (module, name)) safe = getattr(r, '__no_side_effects__', 0) if safe: @@ -432,5 +509,5 @@ if type(r) == types.ClassType and issubclass(r, Exception): return r - raise err.ZRPCError("Unsafe global: %s.%s" % (module, name)) + raise ZRPCError("Unsafe global: %s.%s" % (module, name)) --- Updated File zrpc2.py in package Packages/ZEO -- --- zrpc2.py 2001/03/17 00:14:51 1.1.2.1 +++ zrpc2.py 2001/03/29 13:34:32 1.1.2.2 @@ -1,4 +1,4 @@ -"""RPC protocol for ZEO +"""RPC protocol for ZEO based on asyncore The basic protocol is as: a pickled tuple containing: msgid, flags, method, args @@ -18,6 +18,7 @@ import socket import sys import threading +import thread import time import traceback import types @@ -26,6 +27,7 @@ from ZODB import POSException import smac +import trigger import zLOG REPLY = ".reply" # message name used for replies @@ -82,7 +84,7 @@ try: msgid, flags, name, args = unpickler.load() except (cPickle.UnpicklingError, IndexError), msg: - raise err.DecodingError(msg) + raise DecodingError(msg) return msgid, flags, name, args class Delay: @@ -99,6 +101,31 @@ def reply(self, obj): self.send_reply(self.msgid, obj) + +class DebugLock: + def __init__(self): + self.lock = thread.allocate_lock() + + def _debug(self): + method = sys._getframe().f_back + caller = method.f_back + filename = os.path.split(caller.f_code.co_filename)[1] + log("LOCK %s: %s called by %s, %s, line %s" % (id(self.lock), + method.f_code.co_name, + caller.f_code.co_name, + filename, + caller.f_lineno)) + + def acquire(self, wait=None): + self._debug() + if wait is not None: + return self.lock.acquire(wait) + else: + return self.lock.acquire() + + def release(self): + self._debug() + return self.lock.release() class Connection(smac.SizedMessageAsyncConnection): """Dispatcher for RPC on object @@ -122,9 +149,10 @@ self.obj = obj self.marshal = Marshaller(pickle) self.closed = 0 + self.async = 0 # The reply lock is used to block when a synchronous call is # waiting for a response - self.__reply_lock = threading.Lock() + self.__reply_lock = thread.allocate_lock() self.__reply_lock.acquire() self.__super_init(sock, addr) if isinstance(obj, Handler): @@ -146,24 +174,26 @@ """Decoding an incoming message and dispatch it""" try: msgid, flags, name, args = self.marshal.decode(message) - except err.DecodingError, msg: + except DecodingError, msg: return self.return_error(None, None, sys.exc_info()[0], sys.exc_info()[1]) + log("message: %s, %s, %s, %s" % (msgid, flags, name, repr(args)[:40]), + level=zLOG.TRACE) if name == REPLY: self.handle_reply(msgid, flags, args) else: self.handle_request(msgid, flags, name, args) def handle_reply(self, msgid, flags, args): + log("reply: %s, %s, %s" % (msgid, flags, str(args)[:40])) self.__reply = msgid, flags, args -# self.__lock.release() + self.__reply_lock.release() # will fail if lock is unlocked def handle_request(self, msgid, flags, name, args): - if __debug__: - log("%s%s" % (name, args), zLOG.TRACE) + log("%s%s" % (name, repr(args)[:40]), zLOG.BLATHER) if not self.check_method(name): - raise err.ZRPCError("Invalid method name: %s" % name) + raise ZRPCError("Invalid method name: %s" % name) meth = getattr(self.obj, name) try: @@ -185,8 +215,9 @@ if flags & ASYNC: if ret is not None: - raise err.ZRPCError("async method returned value") + raise ZRPCError("async method returned value") else: + log("%s reply %s" % (name, repr(ret)[:40]), zLOG.BLATHER) if isinstance(ret, Delay): ret.set_sender(msgid, self.send_reply) else: @@ -194,7 +225,6 @@ def handle_error(self): t, v, tb = sys.exc_info() - print "%s: %s" % (str(t), v) traceback.print_tb(tb) def check_method(self, name): @@ -205,6 +235,10 @@ self.message_output(msg) def return_error(self, msgid, flags, err_type, err_value): + if flags is None: + print "Exception raised during decoding" + self.handle_error() + return if flags & ASYNC: print "Asynchronous call raised exception:" self.handle_error() @@ -220,6 +254,7 @@ err = ZRPCError("Couldn't pickle error %s" % `err_value`) msg = self.marshal.encode(msgid, 0, REPLY, (ZRPCError, err)) self.message_output(msg) + self._do_io() print "Sent error message for:" self.handle_error() @@ -244,10 +279,6 @@ # The previous five methods implement an asyncore socket map - def _mainloop(self): - """Invoke the asyncore mainloop""" - - # The next two methods are used by clients to invoke methods on # remote objects @@ -258,17 +289,17 @@ self.message_output(self.marshal.encode(msgid, 0, method, args)) self.__reply = None - while self.__reply is None: - # this is where you want to call the main loop - asyncore.poll(60.0) + self._do_io(wait=1) r_msgid, r_flags, r_args = self.__reply + self.__reply_lock.acquire() assert r_msgid == msgid, "%s != %s: %s" % (r_msgid, msgid, r_args) + if type(r_args) == types.TupleType \ and type(r_args[0]) == types.ClassType \ and issubclass(r_args[0], Exception): - log("error") - print repr(r_args[1]) + log("call %s %s raised error" % (msgid, method)) raise r_args[1] + log("call %s %s returned" % (msgid, method)) return r_args def callAsync(self, method, *args): @@ -276,6 +307,33 @@ self.msgid += 1 log("async %s %s" % (msgid, method)) self.message_output(self.marshal.encode(msgid, ASYNC, method, args)) + self._do_io() + + # handle IO, possibly in async mode + + def _do_io(self, wait=0): # XXX need better name + # XXX invariant? lock must be held when calling with wait==1 + # otherwise, in non-async mode, there will be no poll + + log("_do_io(wait=%d), async=%d" % (wait, self.async), + level=zLOG.BLATHER) + if self.async: + self.trigger.pull_trigger() + if wait: + self.__reply_lock.acquire() + else: + if wait: + # do loop only if lock is already acquired + while not self.__reply_lock.acquire(0): + asyncore.poll(60.0, self) + self.__reply_lock.release() + else: + asyncore.poll(0.0, self) + +class ServerConnection(Connection): + def _do_io(self, wait=0): + """If this is a server, there is no explicit IO to do""" + pass class ConnectionManager: """Keeps a connection up over time""" @@ -293,6 +351,10 @@ def register_object(self, obj): self.obj = obj + def set_async(self): + self.async = 1 + self.trigger = trigger.trigger() + def connect(self, sync=0, callback=None): if self.connected == 1: return @@ -330,6 +392,7 @@ log("Connected to server", level=zLOG.DEBUG) self.connected = 1 if self.connected: + # XXX how do we get here with s being defined? c = ManagedConnection(s, self.addr, self.obj, self) log("Connection created: %s" % c) log("callback = %s" % self._callback) @@ -345,8 +408,22 @@ return t def closed(self, conn): + self.connected = 0 self.connect() +class ManagedServerConnection(ServerConnection): + """A connection that notifies its ConnectionManager of closing""" + __super_init = Connection.__init__ + __super_close = Connection.close + + def __init__(self, sock, addr, obj, mgr, pickle=None): + self.__mgr = mgr + self.__super_init(sock, addr, obj, pickle) + + def close(self): + self.__super_close() + self.__mgr.closed(self) + class ManagedConnection(Connection): """A connection that notifies its ConnectionManager of closing""" __super_init = Connection.__init__ @@ -418,12 +495,12 @@ try: m = __import__(module, _globals, _globals, _silly) except ImportError, msg: - raise err.ZRPCError("import error %s: %s" % (module, msg)) + raise ZRPCError("import error %s: %s" % (module, msg)) try: r = getattr(m, name) except AttributeError: - raise err.ZRPCError("module %s has no global %s" % (module, name)) + raise ZRPCError("module %s has no global %s" % (module, name)) safe = getattr(r, '__no_side_effects__', 0) if safe: @@ -432,5 +509,5 @@ if type(r) == types.ClassType and issubclass(r, Exception): return r - raise err.ZRPCError("Unsafe global: %s.%s" % (module, name)) + raise ZRPCError("Unsafe global: %s.%s" % (module, name)) From jeremy at digicool.com Thu Mar 29 08:35:19 2001 From: jeremy at digicool.com (jeremy@digicool.com) Date: Sun Aug 10 16:31:14 2008 Subject: [ZEO-Checkins] CVS: Packages/ZEO - StorageServer.py:1.21.4.2 Message-ID: <20010329133519.1B436510E2@korak.digicool.com> Update of /cvs-repository/Packages/ZEO In directory korak:/tmp/cvs-serv10282 Modified Files: Tag: ZEO-ZRPC-Dev StorageServer.py Log Message: Hack alert: Use zrpc2.ManagedServerConnection instead of zrpc.ManagedConnection --- Updated File StorageServer.py in package Packages/ZEO -- --- StorageServer.py 2001/03/17 00:41:48 1.21.4.1 +++ StorageServer.py 2001/03/29 13:35:17 1.21.4.2 @@ -7,7 +7,7 @@ import zrpc2 import zLOG -from zrpc2 import Dispatcher, Handler, ManagedConnection, Delay +from zrpc2 import Dispatcher, Handler, ManagedServerConnection, Delay from ZODB.POSException import StorageError, StorageTransactionError, \ TransactionError from ZODB.referencesf import referencesf @@ -36,7 +36,7 @@ Dispatcher(addr, factory=self.newConnection, reuse_addr=1) def newConnection(self, sock, addr, nil): - c = ManagedConnection(sock, addr, None, self) + c = ManagedServerConnection(sock, addr, None, self) c.register_object(StorageProxy(self, c)) def register(self, storage_id, proxy): --- Updated File StorageServer.py in package Packages/ZEO -- --- StorageServer.py 2001/03/17 00:41:48 1.21.4.1 +++ StorageServer.py 2001/03/29 13:35:17 1.21.4.2 @@ -7,7 +7,7 @@ import zrpc2 import zLOG -from zrpc2 import Dispatcher, Handler, ManagedConnection, Delay +from zrpc2 import Dispatcher, Handler, ManagedServerConnection, Delay from ZODB.POSException import StorageError, StorageTransactionError, \ TransactionError from ZODB.referencesf import referencesf @@ -36,7 +36,7 @@ Dispatcher(addr, factory=self.newConnection, reuse_addr=1) def newConnection(self, sock, addr, nil): - c = ManagedConnection(sock, addr, None, self) + c = ManagedServerConnection(sock, addr, None, self) c.register_object(StorageProxy(self, c)) def register(self, storage_id, proxy): From jeremy at digicool.com Thu Mar 29 16:31:58 2001 From: jeremy at digicool.com (jeremy@digicool.com) Date: Sun Aug 10 16:31:14 2008 Subject: [ZEO-Checkins] CVS: Packages/ZEO - ClientCache.py:1.14.4.1 Message-ID: <20010329213158.3E21D510E2@korak.digicool.com> Update of /cvs-repository/Packages/ZEO In directory korak:/tmp/cvs-serv19284 Modified Files: Tag: ZEO-ZRPC-Dev ClientCache.py Log Message: Add some notes about planned redesign of ClientCache --- Updated File ClientCache.py in package Packages/ZEO -- --- ClientCache.py 2000/12/11 18:02:13 1.14 +++ ClientCache.py 2001/03/29 21:31:57 1.14.4.1 @@ -211,6 +211,8 @@ self._current=current def open(self): + # XXX open is overloaded to perform two tasks for + # optimization reasons self._acquire() try: self._index=index={} @@ -224,6 +226,12 @@ return serial.items() finally: self._release() + + def verify(self, verifyFunc): + """Call the verifyFunc on every object in the cache. + + verifyFunc(oid, serialno, version) + """ def invalidate(self, oid, version): self._acquire() --- Updated File ClientCache.py in package Packages/ZEO -- --- ClientCache.py 2000/12/11 18:02:13 1.14 +++ ClientCache.py 2001/03/29 21:31:57 1.14.4.1 @@ -211,6 +211,8 @@ self._current=current def open(self): + # XXX open is overloaded to perform two tasks for + # optimization reasons self._acquire() try: self._index=index={} @@ -224,6 +226,12 @@ return serial.items() finally: self._release() + + def verify(self, verifyFunc): + """Call the verifyFunc on every object in the cache. + + verifyFunc(oid, serialno, version) + """ def invalidate(self, oid, version): self._acquire() From jeremy at digicool.com Thu Mar 29 16:36:25 2001 From: jeremy at digicool.com (jeremy@digicool.com) Date: Sun Aug 10 16:31:14 2008 Subject: [ZEO-Checkins] CVS: Packages/ZEO - ClientStorage.py:1.26.4.3 Message-ID: <20010329213625.2EA4B510E2@korak.digicool.com> Update of /cvs-repository/Packages/ZEO In directory korak:/tmp/cvs-serv19392 Modified Files: Tag: ZEO-ZRPC-Dev ClientStorage.py Log Message: Mondo refactoring based on workday with Jim Push more startup work and all ThreadedAsync work into ConnectionManager. Open cache before registerDB() call. Remove __begin hackery. Merge Invalidator into ClientStorage (need to revise this code and remove Invalidator, but that comes later). Leads to removal of ClientProxy object. --- Updated File ClientStorage.py in package Packages/ZEO -- --- ClientStorage.py 2001/03/29 13:29:01 1.26.4.2 +++ ClientStorage.py 2001/03/29 21:36:23 1.26.4.3 @@ -111,8 +111,8 @@ class ClientStorage(ExtensionClass.Base, BaseStorage.BaseStorage): - _connected = _async = 0 - __begin = 'tpc_begin_sync' + _connected = 0 # XXX is this needed? can probably use self._server + _server = None def __init__(self, addr, storage='1', cache_size=20000000, name='', client='', debug=0, var=None, @@ -130,10 +130,6 @@ self._info = {'length': 0, 'size': 0, 'name': 'ZEO Client', 'supportsUndo':0, 'supportsVersions': 0} - self._rpc_mgr = zrpc2.ConnectionManager(addr, #debug=debug, - tmin=min_disconnect_poll, - tmax=max_disconnect_poll) - name = name or str(addr) self._tfile = tempfile.TemporaryFile() @@ -145,79 +141,40 @@ ClientStorage.inheritedAttribute('__init__')(self, name) self.__lock_acquire = self._lock_acquire # XXX - - self._cache=ClientCache.ClientCache(storage, cache_size, - client=client, var=var) - # XXX This stuff doesn't work right yet. - ThreadedAsync.register_loop_callback(self.becomeAsync) + self._cache = ClientCache.ClientCache(storage, cache_size, + client=client, var=var) + self._cache.open() # XXX - # IMPORTANT: Note that we aren't fully "there" yet. - # In particular, we don't actually connect to the server - # until we have a controlling database set with registerDB - # below. + self._rpc_mgr = zrpc2.ConnectionManager(addr, self, + #debug=debug, + tmin=min_disconnect_poll, + tmax=max_disconnect_poll) + # XXX make this method call the default CnMgr behavior + self._rpc_mgr.connect(callback=self.notifyConnected) def registerDB(self, db, limit): """Register that the storage is controlled by the given DB. """ + self._db = db - # Among other things, we know that our data methods won't get - # called until after this call. - - # XXX Don't need to create separate invalidator object and - # proxy. JF: The separate object exists only to avoid a - # circular reference. - invalidator = Invalidator.Invalidator(db.invalidate, - self._cache.invalidate) - self._rpc_mgr.register_object(ClientProxy(invalidator, self)) - # Now that we have our callback system in place, we can - # try to connect - self._startup() - - def _startup(self): - if not self._rpc_mgr.attempt_connect(self.notifyConnected): - # If we can't connect right away, go ahead and open the cache - # and start a separate thread to try and reconnect. - - LOG("ClientStorage", PROBLEM, "Failed to connect to storage") - self._cache.open() - self._rpc_mgr.connect(callback=self.notifyConnected) - # If the connect succeeds then this work will be done by - # notifyConnected - else: - LOG("ClientStorage", INFO, "connected to storage") - def notifyConnected(self, c): LOG("ClientStorage", INFO, "Connected to storage") self._lock_acquire() try: - # We let the connection keep coming up now that - # we have the storage lock. This way, we know no calls - # will be made while in the process of coming up. self._server = ServerStub.StorageServer(c) self._connected = 1 self._oids = [] - # we do synchronous commits until we are sure that - # we have and are ready for a main loop. - - # Hm. This is a little silly. If self._async, then - # we will really never do a synchronous commit. - # See below. - self.__begin='tpc_begin_sync' self._server.register(str(self._storage)) - self.open_cache() + self.verify_cache() finally: self._lock_release() - if self._async: - import asyncore - self.becomeAsync(asyncore.socket_map) - - def open_cache(self): - cached = self._cache.open() + def verify_cache(self): + cached = self._cache.open() # XXX ### This is a little expensive for large caches if cached: self._server.beginZeoVerify() @@ -239,25 +196,17 @@ def notifyDisconnected(self, ignored): LOG("ClientStorage", PROBLEM, "Disconnected from storage") + # XXX does this need to be called by ConnectionManager? + # oh yes! need to release the lock self._connected = 0 self._transaction = None - thread.start_new_thread(self._call.connect, (0,)) try: self._commit_lock_release() except: pass - - def becomeAsync(self, map): - self._lock_acquire() - try: - self._async = 1 - if self._connected: - self._call.setLoop(map, getWakeup()) - self.__begin='tpc_begin' - finally: - self._lock_release() - def __len__(self): return self._info['length'] + def __len__(self): + return self._info['length'] def abortVersion(self, src, transaction): if transaction is not self._transaction: @@ -275,7 +224,8 @@ def close(self): self._lock_acquire() try: - self._call.closeIntensionally() # XXX + self_.call.closeIntensionally() # XXX this can't work + self._rpc_mgr.close_nicely() # XXX or self._server finally: self._lock_release() @@ -454,10 +404,7 @@ if not self._connected: raise ClientDisconnected("This action is temporarily " "unavailable.") - if self.__begin == "tpc_begin_sync": - r = self._server.tpc_begin_sync(id, user, desc, ext) - else: - r = self._server.tpc_begin(id, user, desc, ext) + r = self._server.tpc_begin(id, user, desc, ext) except: self._commit_lock_release() raise @@ -481,6 +428,8 @@ if transaction is not self._transaction: return if f is not None: f() + # XXX what happens if the network dies RIGHT NOW? + self._server.tpc_finish(self._serial, transaction.user, transaction.description, @@ -566,41 +515,36 @@ finally: self._lock_release() -class ClientProxy: - """Define methods called by the StorageServer""" - - def __init__(self, invalidator, storage): - self._inv = invalidator - self._sto = storage - def begin(self): - self._inv.begin() + self._tfile=tempfile.TemporaryFile() + pickler=cPickle.Pickler(self._tfile, 1) + pickler.fast=1 # Don't use the memo + self._d=pickler.dump - def end(self): - self._inv.end() - def invalidate(self, args): - self._inv.invalidate(args) - - def Invalidate(self, args): - self._inv.Invalidate(args) - - def unlock(self): - self._sto.commit_lock_release() - - def serialno(self, arg): - self._sto._serials.append(arg) - - def info(self, dict): - self._sto._info.update(dict) + if self._d is None: return + self._d(args) -class Wakeup: - def __init__(self): - self.trigger = None - - def __call__(self): - if self.trigger is None: - self.trigger = trigger.trigger() - return self.trigger.pull_trigger + def end(self): + if self._d is None: return + self._d((0,0)) + self._d=None + self._tfile.seek(0) + load=cPickle.Unpickler(self._tfile).load + self._tfile=None + + cinvalidate=self._cache.invalidate + dinvalidate=self._db.invalidate + + while 1: + oid, version = load() + if not oid: break + cinvalidate(oid, version=version) + dinvalidate(oid, version=version) -getWakeup = Wakeup() + def Invalidate(self, args): + cinvalidate=self._cache.invalidate + dinvalidate=self._db.invalidate + for oid, version in args: + cinvalidate(oid, version=version) + dinvalidate(oid, version=version) --- Updated File ClientStorage.py in package Packages/ZEO -- --- ClientStorage.py 2001/03/29 13:29:01 1.26.4.2 +++ ClientStorage.py 2001/03/29 21:36:23 1.26.4.3 @@ -111,8 +111,8 @@ class ClientStorage(ExtensionClass.Base, BaseStorage.BaseStorage): - _connected = _async = 0 - __begin = 'tpc_begin_sync' + _connected = 0 # XXX is this needed? can probably use self._server + _server = None def __init__(self, addr, storage='1', cache_size=20000000, name='', client='', debug=0, var=None, @@ -130,10 +130,6 @@ self._info = {'length': 0, 'size': 0, 'name': 'ZEO Client', 'supportsUndo':0, 'supportsVersions': 0} - self._rpc_mgr = zrpc2.ConnectionManager(addr, #debug=debug, - tmin=min_disconnect_poll, - tmax=max_disconnect_poll) - name = name or str(addr) self._tfile = tempfile.TemporaryFile() @@ -145,79 +141,40 @@ ClientStorage.inheritedAttribute('__init__')(self, name) self.__lock_acquire = self._lock_acquire # XXX - - self._cache=ClientCache.ClientCache(storage, cache_size, - client=client, var=var) - # XXX This stuff doesn't work right yet. - ThreadedAsync.register_loop_callback(self.becomeAsync) + self._cache = ClientCache.ClientCache(storage, cache_size, + client=client, var=var) + self._cache.open() # XXX - # IMPORTANT: Note that we aren't fully "there" yet. - # In particular, we don't actually connect to the server - # until we have a controlling database set with registerDB - # below. + self._rpc_mgr = zrpc2.ConnectionManager(addr, self, + #debug=debug, + tmin=min_disconnect_poll, + tmax=max_disconnect_poll) + # XXX make this method call the default CnMgr behavior + self._rpc_mgr.connect(callback=self.notifyConnected) def registerDB(self, db, limit): """Register that the storage is controlled by the given DB. """ + self._db = db - # Among other things, we know that our data methods won't get - # called until after this call. - - # XXX Don't need to create separate invalidator object and - # proxy. JF: The separate object exists only to avoid a - # circular reference. - invalidator = Invalidator.Invalidator(db.invalidate, - self._cache.invalidate) - self._rpc_mgr.register_object(ClientProxy(invalidator, self)) - # Now that we have our callback system in place, we can - # try to connect - self._startup() - - def _startup(self): - if not self._rpc_mgr.attempt_connect(self.notifyConnected): - # If we can't connect right away, go ahead and open the cache - # and start a separate thread to try and reconnect. - - LOG("ClientStorage", PROBLEM, "Failed to connect to storage") - self._cache.open() - self._rpc_mgr.connect(callback=self.notifyConnected) - # If the connect succeeds then this work will be done by - # notifyConnected - else: - LOG("ClientStorage", INFO, "connected to storage") - def notifyConnected(self, c): LOG("ClientStorage", INFO, "Connected to storage") self._lock_acquire() try: - # We let the connection keep coming up now that - # we have the storage lock. This way, we know no calls - # will be made while in the process of coming up. self._server = ServerStub.StorageServer(c) self._connected = 1 self._oids = [] - # we do synchronous commits until we are sure that - # we have and are ready for a main loop. - - # Hm. This is a little silly. If self._async, then - # we will really never do a synchronous commit. - # See below. - self.__begin='tpc_begin_sync' self._server.register(str(self._storage)) - self.open_cache() + self.verify_cache() finally: self._lock_release() - if self._async: - import asyncore - self.becomeAsync(asyncore.socket_map) - - def open_cache(self): - cached = self._cache.open() + def verify_cache(self): + cached = self._cache.open() # XXX ### This is a little expensive for large caches if cached: self._server.beginZeoVerify() @@ -239,25 +196,17 @@ def notifyDisconnected(self, ignored): LOG("ClientStorage", PROBLEM, "Disconnected from storage") + # XXX does this need to be called by ConnectionManager? + # oh yes! need to release the lock self._connected = 0 self._transaction = None - thread.start_new_thread(self._call.connect, (0,)) try: self._commit_lock_release() except: pass - - def becomeAsync(self, map): - self._lock_acquire() - try: - self._async = 1 - if self._connected: - self._call.setLoop(map, getWakeup()) - self.__begin='tpc_begin' - finally: - self._lock_release() - def __len__(self): return self._info['length'] + def __len__(self): + return self._info['length'] def abortVersion(self, src, transaction): if transaction is not self._transaction: @@ -275,7 +224,8 @@ def close(self): self._lock_acquire() try: - self._call.closeIntensionally() # XXX + self_.call.closeIntensionally() # XXX this can't work + self._rpc_mgr.close_nicely() # XXX or self._server finally: self._lock_release() @@ -454,10 +404,7 @@ if not self._connected: raise ClientDisconnected("This action is temporarily " "unavailable.") - if self.__begin == "tpc_begin_sync": - r = self._server.tpc_begin_sync(id, user, desc, ext) - else: - r = self._server.tpc_begin(id, user, desc, ext) + r = self._server.tpc_begin(id, user, desc, ext) except: self._commit_lock_release() raise @@ -481,6 +428,8 @@ if transaction is not self._transaction: return if f is not None: f() + # XXX what happens if the network dies RIGHT NOW? + self._server.tpc_finish(self._serial, transaction.user, transaction.description, @@ -566,41 +515,36 @@ finally: self._lock_release() -class ClientProxy: - """Define methods called by the StorageServer""" - - def __init__(self, invalidator, storage): - self._inv = invalidator - self._sto = storage - def begin(self): - self._inv.begin() + self._tfile=tempfile.TemporaryFile() + pickler=cPickle.Pickler(self._tfile, 1) + pickler.fast=1 # Don't use the memo + self._d=pickler.dump - def end(self): - self._inv.end() - def invalidate(self, args): - self._inv.invalidate(args) - - def Invalidate(self, args): - self._inv.Invalidate(args) - - def unlock(self): - self._sto.commit_lock_release() - - def serialno(self, arg): - self._sto._serials.append(arg) - - def info(self, dict): - self._sto._info.update(dict) + if self._d is None: return + self._d(args) -class Wakeup: - def __init__(self): - self.trigger = None - - def __call__(self): - if self.trigger is None: - self.trigger = trigger.trigger() - return self.trigger.pull_trigger + def end(self): + if self._d is None: return + self._d((0,0)) + self._d=None + self._tfile.seek(0) + load=cPickle.Unpickler(self._tfile).load + self._tfile=None + + cinvalidate=self._cache.invalidate + dinvalidate=self._db.invalidate + + while 1: + oid, version = load() + if not oid: break + cinvalidate(oid, version=version) + dinvalidate(oid, version=version) -getWakeup = Wakeup() + def Invalidate(self, args): + cinvalidate=self._cache.invalidate + dinvalidate=self._db.invalidate + for oid, version in args: + cinvalidate(oid, version=version) + dinvalidate(oid, version=version) From jeremy at digicool.com Thu Mar 29 16:37:34 2001 From: jeremy at digicool.com (jeremy@digicool.com) Date: Sun Aug 10 16:31:14 2008 Subject: [ZEO-Checkins] CVS: Packages/ZEO - zrpc2.py:1.1.2.3 Message-ID: <20010329213734.5ABD3510E2@korak.digicool.com> Update of /cvs-repository/Packages/ZEO In directory korak:/tmp/cvs-serv20142 Modified Files: Tag: ZEO-ZRPC-Dev zrpc2.py Log Message: Register ConnectionManager with ThreadedAsync. XXX The async flag needs to be threaded through to the Connection. Add some comments about threads. --- Updated File zrpc2.py in package Packages/ZEO -- --- zrpc2.py 2001/03/29 13:34:32 1.1.2.2 +++ zrpc2.py 2001/03/29 21:37:32 1.1.2.3 @@ -154,6 +154,9 @@ # waiting for a response self.__reply_lock = thread.allocate_lock() self.__reply_lock.acquire() + # The async mode lock is used to prevent a race between the + # write of self.async by set_async() and its use in _do_io() + self.__async_mode_lock = thread.allocate_lock() self.__super_init(sock, addr) if isinstance(obj, Handler): self.set_caller = 1 @@ -282,6 +285,9 @@ # The next two methods are used by clients to invoke methods on # remote objects + # XXX these two methods should raise an nice exception if there + # are called after the connection is closed + def call(self, method, *args): msgid = self.msgid self.msgid += 1 @@ -347,18 +353,20 @@ self.connected = 0 self._thread = None self._callback = None + ThreadedAsync.register_loop_callback(self.set_async) def register_object(self, obj): self.obj = obj def set_async(self): - self.async = 1 + self.async = 1 # XXX needs to be set on the Connection self.trigger = trigger.trigger() def connect(self, sync=0, callback=None): if self.connected == 1: return self._callback = callback + # XXX probably need lock around self._thread if self._thread is None: self._thread = threading.Thread(target=self.__connect, args=(1,)) self._thread.start() @@ -409,6 +417,7 @@ def closed(self, conn): self.connected = 0 + # perhaps call ClientStorage(XXX) and tell it we closed self.connect() class ManagedServerConnection(ServerConnection): --- Updated File zrpc2.py in package Packages/ZEO -- --- zrpc2.py 2001/03/29 13:34:32 1.1.2.2 +++ zrpc2.py 2001/03/29 21:37:32 1.1.2.3 @@ -154,6 +154,9 @@ # waiting for a response self.__reply_lock = thread.allocate_lock() self.__reply_lock.acquire() + # The async mode lock is used to prevent a race between the + # write of self.async by set_async() and its use in _do_io() + self.__async_mode_lock = thread.allocate_lock() self.__super_init(sock, addr) if isinstance(obj, Handler): self.set_caller = 1 @@ -282,6 +285,9 @@ # The next two methods are used by clients to invoke methods on # remote objects + # XXX these two methods should raise an nice exception if there + # are called after the connection is closed + def call(self, method, *args): msgid = self.msgid self.msgid += 1 @@ -347,18 +353,20 @@ self.connected = 0 self._thread = None self._callback = None + ThreadedAsync.register_loop_callback(self.set_async) def register_object(self, obj): self.obj = obj def set_async(self): - self.async = 1 + self.async = 1 # XXX needs to be set on the Connection self.trigger = trigger.trigger() def connect(self, sync=0, callback=None): if self.connected == 1: return self._callback = callback + # XXX probably need lock around self._thread if self._thread is None: self._thread = threading.Thread(target=self.__connect, args=(1,)) self._thread.start() @@ -409,6 +417,7 @@ def closed(self, conn): self.connected = 0 + # perhaps call ClientStorage(XXX) and tell it we closed self.connect() class ManagedServerConnection(ServerConnection): From jeremy at digicool.com Thu Mar 29 23:47:00 2001 From: jeremy at digicool.com (jeremy@digicool.com) Date: Sun Aug 10 16:31:14 2008 Subject: [ZEO-Checkins] CVS: Packages/ZEO - ClientStorage.py:1.26.4.4 Message-ID: <20010330044700.349A5510E2@korak.digicool.com> Update of /cvs-repository/Packages/ZEO In directory korak:/tmp/cvs-serv22381 Modified Files: Tag: ZEO-ZRPC-Dev ClientStorage.py Log Message: In ClientStorage constructor, make an explicit attempt_connect() call on the ConnectionManager. Otherwise, a script starting a fresh ClientStorage will usually get a Disconnected exception before the connection is established. Add methods defined on ClientProxy() that were removed in haste by the previous checkin. Remove unneeded imports (ThreadedAsync, Invalidator). --- Updated File ClientStorage.py in package Packages/ZEO -- --- ClientStorage.py 2001/03/29 21:36:23 1.26.4.3 +++ ClientStorage.py 2001/03/30 04:46:59 1.26.4.4 @@ -87,8 +87,7 @@ __version__='$Revision$'[11:-2] import struct, time, os, socket, string, Sync, ClientCache -import tempfile, Invalidator, ExtensionClass, thread -import ThreadedAsync +import tempfile, ExtensionClass, thread import zrpc2 import ServerStub @@ -151,7 +150,8 @@ tmin=min_disconnect_poll, tmax=max_disconnect_poll) # XXX make this method call the default CnMgr behavior - self._rpc_mgr.connect(callback=self.notifyConnected) + if not self._rpc_mgr.attempt_connect(callback=self.notifyConnected): + self._rpc_mgr.connect(callback=self.notifyConnected) def registerDB(self, db, limit): """Register that the storage is controlled by the given DB. @@ -515,26 +515,38 @@ finally: self._lock_release() + # below are methods invoked by the StorageServer + + def unlock(self): + self.commit_lock_release() + + def serialno(self, arg): + self._serials.append(arg) + + def info(self, dict): + self._info.update(dict) + def begin(self): - self._tfile=tempfile.TemporaryFile() - pickler=cPickle.Pickler(self._tfile, 1) - pickler.fast=1 # Don't use the memo - self._d=pickler.dump + self._tfile = tempfile.TemporaryFile() + self._pickler = cPickle.Pickler(self._tfile, 1) + self._pickler.fast = 1 # Don't use the memo def invalidate(self, args): - if self._d is None: return - self._d(args) + if self._pickler is None: + return + self._pickler.dump(args) def end(self): - if self._d is None: return - self._d((0,0)) - self._d=None + if self._pickler is None: + return + self._pickler.dump((0,0)) + self._pickler.dump=None self._tfile.seek(0) load=cPickle.Unpickler(self._tfile).load self._tfile=None cinvalidate=self._cache.invalidate - dinvalidate=self._db.invalidate + dinvalidate=self._pickler.dumpb.invalidate while 1: oid, version = load() @@ -544,7 +556,7 @@ def Invalidate(self, args): cinvalidate=self._cache.invalidate - dinvalidate=self._db.invalidate + dinvalidate=self._pickler.dumpb.invalidate for oid, version in args: cinvalidate(oid, version=version) dinvalidate(oid, version=version) --- Updated File ClientStorage.py in package Packages/ZEO -- --- ClientStorage.py 2001/03/29 21:36:23 1.26.4.3 +++ ClientStorage.py 2001/03/30 04:46:59 1.26.4.4 @@ -87,8 +87,7 @@ __version__='$Revision$'[11:-2] import struct, time, os, socket, string, Sync, ClientCache -import tempfile, Invalidator, ExtensionClass, thread -import ThreadedAsync +import tempfile, ExtensionClass, thread import zrpc2 import ServerStub @@ -151,7 +150,8 @@ tmin=min_disconnect_poll, tmax=max_disconnect_poll) # XXX make this method call the default CnMgr behavior - self._rpc_mgr.connect(callback=self.notifyConnected) + if not self._rpc_mgr.attempt_connect(callback=self.notifyConnected): + self._rpc_mgr.connect(callback=self.notifyConnected) def registerDB(self, db, limit): """Register that the storage is controlled by the given DB. @@ -515,26 +515,38 @@ finally: self._lock_release() + # below are methods invoked by the StorageServer + + def unlock(self): + self.commit_lock_release() + + def serialno(self, arg): + self._serials.append(arg) + + def info(self, dict): + self._info.update(dict) + def begin(self): - self._tfile=tempfile.TemporaryFile() - pickler=cPickle.Pickler(self._tfile, 1) - pickler.fast=1 # Don't use the memo - self._d=pickler.dump + self._tfile = tempfile.TemporaryFile() + self._pickler = cPickle.Pickler(self._tfile, 1) + self._pickler.fast = 1 # Don't use the memo def invalidate(self, args): - if self._d is None: return - self._d(args) + if self._pickler is None: + return + self._pickler.dump(args) def end(self): - if self._d is None: return - self._d((0,0)) - self._d=None + if self._pickler is None: + return + self._pickler.dump((0,0)) + self._pickler.dump=None self._tfile.seek(0) load=cPickle.Unpickler(self._tfile).load self._tfile=None cinvalidate=self._cache.invalidate - dinvalidate=self._db.invalidate + dinvalidate=self._pickler.dumpb.invalidate while 1: oid, version = load() @@ -544,7 +556,7 @@ def Invalidate(self, args): cinvalidate=self._cache.invalidate - dinvalidate=self._db.invalidate + dinvalidate=self._pickler.dumpb.invalidate for oid, version in args: cinvalidate(oid, version=version) dinvalidate(oid, version=version) From jeremy at digicool.com Thu Mar 29 23:47:32 2001 From: jeremy at digicool.com (jeremy@digicool.com) Date: Sun Aug 10 16:31:14 2008 Subject: [ZEO-Checkins] CVS: Packages/ZEO - zrpc2.py:1.1.2.4 Message-ID: <20010330044732.5D2B2510E2@korak.digicool.com> Update of /cvs-repository/Packages/ZEO In directory korak:/tmp/cvs-serv22822 Modified Files: Tag: ZEO-ZRPC-Dev zrpc2.py Log Message: Add import ThreadedAsync (doh!) --- Updated File zrpc2.py in package Packages/ZEO -- --- zrpc2.py 2001/03/29 21:37:32 1.1.2.3 +++ zrpc2.py 2001/03/30 04:47:30 1.1.2.4 @@ -29,6 +29,7 @@ import smac import trigger import zLOG +import ThreadedAsync REPLY = ".reply" # message name used for replies ASYNC = 1 @@ -231,6 +232,7 @@ traceback.print_tb(tb) def check_method(self, name): + # XXX minimal security check should go here: Is name exported? return hasattr(self.obj, name) def send_reply(self, msgid, ret): --- Updated File zrpc2.py in package Packages/ZEO -- --- zrpc2.py 2001/03/29 21:37:32 1.1.2.3 +++ zrpc2.py 2001/03/30 04:47:30 1.1.2.4 @@ -29,6 +29,7 @@ import smac import trigger import zLOG +import ThreadedAsync REPLY = ".reply" # message name used for replies ASYNC = 1 @@ -231,6 +232,7 @@ traceback.print_tb(tb) def check_method(self, name): + # XXX minimal security check should go here: Is name exported? return hasattr(self.obj, name) def send_reply(self, msgid, ret): From jeremy at digicool.com Fri Mar 30 16:16:05 2001 From: jeremy at digicool.com (jeremy@digicool.com) Date: Sun Aug 10 16:31:14 2008 Subject: [ZEO-Checkins] CVS: Packages/ZEO - ClientStorage.py:1.26.4.5 Message-ID: <20010330211605.1DBCC510E2@korak.digicool.com> Update of /cvs-repository/Packages/ZEO In directory korak:/tmp/cvs-serv14082 Modified Files: Tag: ZEO-ZRPC-Dev ClientStorage.py Log Message: XXX Update ClientStorage to work with Zope. It still doesn't work correctly, but is getting closer to full functionality. Remove _connected attr from ClientStorage. Use __super_init to get at BaseStore.__init__ rather than using "inheritedAttribute". Initialize _db. "Fix" getName(). It's doesn't fail, but it doesn't say if the client is connected either. Defer DisconnectedError to zrpc2, rather than raising in tpc_begin(). Reformat and rename invalidator code -- and import cPickle so that it works. --- Updated File ClientStorage.py in package Packages/ZEO -- --- ClientStorage.py 2001/03/30 04:46:59 1.26.4.4 +++ ClientStorage.py 2001/03/30 21:16:02 1.26.4.5 @@ -91,6 +91,8 @@ import zrpc2 import ServerStub +import cPickle + from struct import pack, unpack from ZODB import POSException, BaseStorage from ZODB.TimeStamp import TimeStamp @@ -109,8 +111,8 @@ """The database storage is disconnected from the storage.""" class ClientStorage(ExtensionClass.Base, BaseStorage.BaseStorage): + __super_init = BaseStorage.BaseStorage.__init__ - _connected = 0 # XXX is this needed? can probably use self._server _server = None def __init__(self, addr, storage='1', cache_size=20000000, @@ -136,11 +138,10 @@ # XXX It's confusing to have _serial, _serials, and _seriald. self._serials = [] self._seriald = {} - - ClientStorage.inheritedAttribute('__init__')(self, name) - self.__lock_acquire = self._lock_acquire # XXX + self.__super_init(name) + self._db = None self._cache = ClientCache.ClientCache(storage, cache_size, client=client, var=var) self._cache.open() # XXX @@ -149,13 +150,13 @@ #debug=debug, tmin=min_disconnect_poll, tmax=max_disconnect_poll) - # XXX make this method call the default CnMgr behavior - if not self._rpc_mgr.attempt_connect(callback=self.notifyConnected): - self._rpc_mgr.connect(callback=self.notifyConnected) + self._server = None + # XXX make this method call the default CnMgr behavior? + if not self._rpc_mgr.attempt_connect(): + self._rpc_mgr.connect() def registerDB(self, db, limit): - """Register that the storage is controlled by the given DB. - """ + """Register that the storage is controlled by the given DB.""" self._db = db def notifyConnected(self, c): @@ -164,7 +165,6 @@ try: self._server = ServerStub.StorageServer(c) - self._connected = 1 self._oids = [] self._server.register(str(self._storage)) @@ -196,9 +196,6 @@ def notifyDisconnected(self, ignored): LOG("ClientStorage", PROBLEM, "Disconnected from storage") - # XXX does this need to be called by ConnectionManager? - # oh yes! need to release the lock - self._connected = 0 self._transaction = None try: self._commit_lock_release() @@ -249,8 +246,9 @@ self._lock_release() def getName(self): - return "%s (%s)" % (self.__name__, - self._connected and 'connected' or 'disconnected') + return "%s (%s)" % (self.__name__, "XXX") +## # XXX self._connected is gone +## self._connected and 'connected' or 'disconnected') def getSize(self): return self._info['size'] @@ -380,6 +378,8 @@ self._lock_release() def tpc_begin(self, transaction): + # XXX plan is to have begin be a local operation until the + # vote stage. self._lock_acquire() try: if self._transaction is transaction: return @@ -401,26 +401,25 @@ id=`t` try: - if not self._connected: - raise ClientDisconnected("This action is temporarily " - "unavailable.") r = self._server.tpc_begin(id, user, desc, ext) except: self._commit_lock_release() raise - if r is None: break + if r is None: + break # We have *BOTH* the local and distributed commit # lock, now we can actually get ready to get started. - self._serial=id + self._serial = id self._tfile.seek(0) self._seriald.clear() del self._serials[:] - self._transaction=transaction + self._transaction = transaction - finally: self._lock_release() + finally: + self._lock_release() def tpc_finish(self, transaction, f=None): self._lock_acquire() @@ -540,23 +539,19 @@ if self._pickler is None: return self._pickler.dump((0,0)) - self._pickler.dump=None + self._pickler.dump = None self._tfile.seek(0) - load=cPickle.Unpickler(self._tfile).load - self._tfile=None - - cinvalidate=self._cache.invalidate - dinvalidate=self._pickler.dumpb.invalidate + load = cPickle.Unpickler(self._tfile).load + self._tfile = None while 1: oid, version = load() - if not oid: break - cinvalidate(oid, version=version) - dinvalidate(oid, version=version) + if not oid: + break + self._cache.invalidate(oid, version=version) + self._db.invalidate(oid, version=version) def Invalidate(self, args): - cinvalidate=self._cache.invalidate - dinvalidate=self._pickler.dumpb.invalidate for oid, version in args: - cinvalidate(oid, version=version) - dinvalidate(oid, version=version) + self._cache.invalidate(oid, version=version) + self._db.invalidate(oid, version=version) --- Updated File ClientStorage.py in package Packages/ZEO -- --- ClientStorage.py 2001/03/30 04:46:59 1.26.4.4 +++ ClientStorage.py 2001/03/30 21:16:02 1.26.4.5 @@ -91,6 +91,8 @@ import zrpc2 import ServerStub +import cPickle + from struct import pack, unpack from ZODB import POSException, BaseStorage from ZODB.TimeStamp import TimeStamp @@ -109,8 +111,8 @@ """The database storage is disconnected from the storage.""" class ClientStorage(ExtensionClass.Base, BaseStorage.BaseStorage): + __super_init = BaseStorage.BaseStorage.__init__ - _connected = 0 # XXX is this needed? can probably use self._server _server = None def __init__(self, addr, storage='1', cache_size=20000000, @@ -136,11 +138,10 @@ # XXX It's confusing to have _serial, _serials, and _seriald. self._serials = [] self._seriald = {} - - ClientStorage.inheritedAttribute('__init__')(self, name) - self.__lock_acquire = self._lock_acquire # XXX + self.__super_init(name) + self._db = None self._cache = ClientCache.ClientCache(storage, cache_size, client=client, var=var) self._cache.open() # XXX @@ -149,13 +150,13 @@ #debug=debug, tmin=min_disconnect_poll, tmax=max_disconnect_poll) - # XXX make this method call the default CnMgr behavior - if not self._rpc_mgr.attempt_connect(callback=self.notifyConnected): - self._rpc_mgr.connect(callback=self.notifyConnected) + self._server = None + # XXX make this method call the default CnMgr behavior? + if not self._rpc_mgr.attempt_connect(): + self._rpc_mgr.connect() def registerDB(self, db, limit): - """Register that the storage is controlled by the given DB. - """ + """Register that the storage is controlled by the given DB.""" self._db = db def notifyConnected(self, c): @@ -164,7 +165,6 @@ try: self._server = ServerStub.StorageServer(c) - self._connected = 1 self._oids = [] self._server.register(str(self._storage)) @@ -196,9 +196,6 @@ def notifyDisconnected(self, ignored): LOG("ClientStorage", PROBLEM, "Disconnected from storage") - # XXX does this need to be called by ConnectionManager? - # oh yes! need to release the lock - self._connected = 0 self._transaction = None try: self._commit_lock_release() @@ -249,8 +246,9 @@ self._lock_release() def getName(self): - return "%s (%s)" % (self.__name__, - self._connected and 'connected' or 'disconnected') + return "%s (%s)" % (self.__name__, "XXX") +## # XXX self._connected is gone +## self._connected and 'connected' or 'disconnected') def getSize(self): return self._info['size'] @@ -380,6 +378,8 @@ self._lock_release() def tpc_begin(self, transaction): + # XXX plan is to have begin be a local operation until the + # vote stage. self._lock_acquire() try: if self._transaction is transaction: return @@ -401,26 +401,25 @@ id=`t` try: - if not self._connected: - raise ClientDisconnected("This action is temporarily " - "unavailable.") r = self._server.tpc_begin(id, user, desc, ext) except: self._commit_lock_release() raise - if r is None: break + if r is None: + break # We have *BOTH* the local and distributed commit # lock, now we can actually get ready to get started. - self._serial=id + self._serial = id self._tfile.seek(0) self._seriald.clear() del self._serials[:] - self._transaction=transaction + self._transaction = transaction - finally: self._lock_release() + finally: + self._lock_release() def tpc_finish(self, transaction, f=None): self._lock_acquire() @@ -540,23 +539,19 @@ if self._pickler is None: return self._pickler.dump((0,0)) - self._pickler.dump=None + self._pickler.dump = None self._tfile.seek(0) - load=cPickle.Unpickler(self._tfile).load - self._tfile=None - - cinvalidate=self._cache.invalidate - dinvalidate=self._pickler.dumpb.invalidate + load = cPickle.Unpickler(self._tfile).load + self._tfile = None while 1: oid, version = load() - if not oid: break - cinvalidate(oid, version=version) - dinvalidate(oid, version=version) + if not oid: + break + self._cache.invalidate(oid, version=version) + self._db.invalidate(oid, version=version) def Invalidate(self, args): - cinvalidate=self._cache.invalidate - dinvalidate=self._pickler.dumpb.invalidate for oid, version in args: - cinvalidate(oid, version=version) - dinvalidate(oid, version=version) + self._cache.invalidate(oid, version=version) + self._db.invalidate(oid, version=version) From jeremy at digicool.com Fri Mar 30 16:19:38 2001 From: jeremy at digicool.com (jeremy@digicool.com) Date: Sun Aug 10 16:31:14 2008 Subject: [ZEO-Checkins] CVS: Packages/ZEO - zrpc2.py:1.1.2.5 Message-ID: <20010330211938.E1180510E2@korak.digicool.com> Update of /cvs-repository/Packages/ZEO In directory korak:/tmp/cvs-serv14682 Modified Files: Tag: ZEO-ZRPC-Dev zrpc2.py Log Message: XXX Update zrpc2 to work with Zope. Still doesn't work quite right. Change default log() output to zLOG.BLATHER. Define DisconnectedError. Raise it if call() or callAsync() is called when connection is closed. DebugLock: replace builtin object id with simple counter print the thread id too Move async handling, inc. ThreadedAsync callback into methods that can be overriden by ManagedConnection. A ManagedConnection uses the trigger of its manager. handle_error() -- print the actual exception too Add __reply_lock.release() in _do_io() async==1, wait==1. I don't think this is right, but it works for now. --- Updated File zrpc2.py in package Packages/ZEO -- --- zrpc2.py 2001/03/30 04:47:30 1.1.2.4 +++ zrpc2.py 2001/03/30 21:19:36 1.1.2.5 @@ -34,15 +34,18 @@ REPLY = ".reply" # message name used for replies ASYNC = 1 -def log(message, level=zLOG.INFO, label="zrpc:%s" % os.getpid()): +def log(message, level=zLOG.BLATHER, label="zrpc:%s" % os.getpid()): zLOG.LOG(label, level, message) -class ZRPCError(Exception): +class ZRPCError(POSException.StorageError): pass class DecodingError(ZRPCError): """A ZRPC message could not be decoded.""" +class DisconnectedError(ZRPCError): + """The database storage is disconnected from the storage server.""" + # Export the mainloop function from asycnore to zrpc clients loop = asyncore.loop @@ -84,7 +87,8 @@ try: msgid, flags, name, args = unpickler.load() - except (cPickle.UnpicklingError, IndexError), msg: + except (cPickle.UnpicklingError, IndexError), err_msg: + log("can't decode %s" % repr(msg), level=zLOG.ERROR) raise DecodingError(msg) return msgid, flags, name, args @@ -104,18 +108,26 @@ self.send_reply(self.msgid, obj) class DebugLock: + + __locks = 0 + def __init__(self): self.lock = thread.allocate_lock() + # XXX this actually needs to be locked too + self.__lock_id = self.__locks + self.__locks += 1 def _debug(self): method = sys._getframe().f_back caller = method.f_back filename = os.path.split(caller.f_code.co_filename)[1] - log("LOCK %s: %s called by %s, %s, line %s" % (id(self.lock), - method.f_code.co_name, - caller.f_code.co_name, - filename, - caller.f_lineno)) + log("LOCK %s (tid %s): " \ + "%s called by %s, %s, line %s" % (self.__lock_id, + thread.get_ident(), + method.f_code.co_name, + caller.f_code.co_name, + filename, + caller.f_lineno)) def acquire(self, wait=None): self._debug() @@ -127,6 +139,10 @@ def release(self): self._debug() return self.lock.release() + + def locked(self): + self._debug() + return self.lock.locked() class Connection(smac.SizedMessageAsyncConnection): """Dispatcher for RPC on object @@ -153,12 +169,11 @@ self.async = 0 # The reply lock is used to block when a synchronous call is # waiting for a response - self.__reply_lock = thread.allocate_lock() - self.__reply_lock.acquire() - # The async mode lock is used to prevent a race between the - # write of self.async by set_async() and its use in _do_io() - self.__async_mode_lock = thread.allocate_lock() self.__super_init(sock, addr) + self._prepare_async() + self.__reply_lock = DebugLock() +## self.__reply_lock = thread.allocate_lock() + self.__reply_lock.acquire() if isinstance(obj, Handler): self.set_caller = 1 else: @@ -195,7 +210,7 @@ self.__reply_lock.release() # will fail if lock is unlocked def handle_request(self, msgid, flags, name, args): - log("%s%s" % (name, repr(args)[:40]), zLOG.BLATHER) + log("%s%s" % (name, repr(args)[:40]), zLOG.TRACE) if not self.check_method(name): raise ZRPCError("Invalid method name: %s" % name) @@ -221,7 +236,7 @@ if ret is not None: raise ZRPCError("async method returned value") else: - log("%s reply %s" % (name, repr(ret)[:40]), zLOG.BLATHER) + log("%s reply %s" % (name, repr(ret)[:40]), zLOG.TRACE) if isinstance(ret, Delay): ret.set_sender(msgid, self.send_reply) else: @@ -229,6 +244,7 @@ def handle_error(self): t, v, tb = sys.exc_info() + print t, v traceback.print_tb(tb) def check_method(self, name): @@ -287,19 +303,24 @@ # The next two methods are used by clients to invoke methods on # remote objects - # XXX these two methods should raise an nice exception if there - # are called after the connection is closed + # XXX Should revise design to allow multiple outstanding + # synchronous calls def call(self, method, *args): + if self.closed: + raise DisconnectedError("This action is temporarily unavailable") msgid = self.msgid self.msgid += 1 log("call %s %s" % (msgid, method)) self.message_output(self.marshal.encode(msgid, 0, method, args)) self.__reply = None + # lock is currently held self._do_io(wait=1) + # lock is held again... r_msgid, r_flags, r_args = self.__reply self.__reply_lock.acquire() + log("call acquired lock") assert r_msgid == msgid, "%s != %s: %s" % (r_msgid, msgid, r_args) if type(r_args) == types.TupleType \ @@ -311,6 +332,8 @@ return r_args def callAsync(self, method, *args): + if self.closed: + raise DisconnectedError("This action is temporarily unavailable") msgid = self.msgid self.msgid += 1 log("async %s %s" % (msgid, method)) @@ -319,16 +342,33 @@ # handle IO, possibly in async mode + def _prepare_async(self): + self._async = 0 + ThreadedAsync.register_loop_callback(self.set_async) + # XXX If we are not in async mode, this will cause dead + # Connections to be leaked. + + def set_async(self, map): + # XXX do we need a lock around this? I'm not sure there is + # any harm to a race with _do_io(). + self._async = 1 + self.trigger = trigger.trigger() + + def is_async(self): + return self._async + def _do_io(self, wait=0): # XXX need better name # XXX invariant? lock must be held when calling with wait==1 # otherwise, in non-async mode, there will be no poll - log("_do_io(wait=%d), async=%d" % (wait, self.async), - level=zLOG.BLATHER) - if self.async: + log("_do_io(wait=%d), async=%d" % (wait, self.is_async()), + level=zLOG.TRACE) + if self.is_async(): self.trigger.pull_trigger() if wait: self.__reply_lock.acquire() + # wait until reply... + self.__reply_lock.release() else: if wait: # do loop only if lock is already acquired @@ -338,7 +378,12 @@ else: asyncore.poll(0.0, self) + # XXX it seems that we need to release before returning if + # called with wait==1. perhaps the caller need not acquire + # upon return... + class ServerConnection(Connection): + # XXX this is a hack def _do_io(self, wait=0): """If this is a server, there is no explicit IO to do""" pass @@ -346,6 +391,9 @@ class ConnectionManager: """Keeps a connection up over time""" + # XXX requires that obj implement notifyConnected and + # notifyDisconnected. make this optional? + def __init__(self, addr, obj=None, debug=1, tmin=None, tmax=None): self.addr = addr self.obj = obj @@ -354,29 +402,34 @@ self.debug = debug self.connected = 0 self._thread = None - self._callback = None + self._connect_lock = threading.Lock() + self.trigger = None + self.async = 0 ThreadedAsync.register_loop_callback(self.set_async) def register_object(self, obj): self.obj = obj - def set_async(self): + def set_async(self, map): + # XXX need each connection started with async==0 to have a callback self.async = 1 # XXX needs to be set on the Connection self.trigger = trigger.trigger() - def connect(self, sync=0, callback=None): + def connect(self, sync=0): if self.connected == 1: return - self._callback = callback - # XXX probably need lock around self._thread - if self._thread is None: - self._thread = threading.Thread(target=self.__connect, args=(1,)) - self._thread.start() + self._connect_lock.acquire() + try: + if self._thread is None: + self._thread = threading.Thread(target=self.__connect, + args=(1,)) + self._thread.start() + finally: + self._connect_lock.release() if sync: self._thread.join() - def attempt_connect(self, callback=None): - self._callback = callback + def attempt_connect(self): self.__connect(repeat=0) return self.connected @@ -402,12 +455,9 @@ log("Connected to server", level=zLOG.DEBUG) self.connected = 1 if self.connected: - # XXX how do we get here with s being defined? c = ManagedConnection(s, self.addr, self.obj, self) log("Connection created: %s" % c) - log("callback = %s" % self._callback) - if self._callback: - self._callback(c) + self.obj.notifyConnected(c) self.__thread = None def _wait(self, t): @@ -419,7 +469,7 @@ def closed(self, conn): self.connected = 0 - # perhaps call ClientStorage(XXX) and tell it we closed + self.obj.notifyDisconnected(None) self.connect() class ManagedServerConnection(ServerConnection): @@ -436,13 +486,35 @@ self.__mgr.closed(self) class ManagedConnection(Connection): - """A connection that notifies its ConnectionManager of closing""" + """A connection that notifies its ConnectionManager of closing. + + A managed connection also defers the ThreadedAsync work to its + manager. + """ __super_init = Connection.__init__ __super_close = Connection.close def __init__(self, sock, addr, obj, mgr, pickle=None): self.__mgr = mgr + if self.__mgr.async: + self.__async = 1 + self.trigger = self.__mgr.trigger + else: + self.__async = None self.__super_init(sock, addr, obj, pickle) + + def _prepare_async(self): + # Don't do the register_loop_callback that the superclass does + pass + + def is_async(self): + if self.__async: + return 1 + async = self.__mgr.async + if async: + self.__async = 1 + self.trigger = self.__mgr.trigger + return async def close(self): self.__super_close() --- Updated File zrpc2.py in package Packages/ZEO -- --- zrpc2.py 2001/03/30 04:47:30 1.1.2.4 +++ zrpc2.py 2001/03/30 21:19:36 1.1.2.5 @@ -34,15 +34,18 @@ REPLY = ".reply" # message name used for replies ASYNC = 1 -def log(message, level=zLOG.INFO, label="zrpc:%s" % os.getpid()): +def log(message, level=zLOG.BLATHER, label="zrpc:%s" % os.getpid()): zLOG.LOG(label, level, message) -class ZRPCError(Exception): +class ZRPCError(POSException.StorageError): pass class DecodingError(ZRPCError): """A ZRPC message could not be decoded.""" +class DisconnectedError(ZRPCError): + """The database storage is disconnected from the storage server.""" + # Export the mainloop function from asycnore to zrpc clients loop = asyncore.loop @@ -84,7 +87,8 @@ try: msgid, flags, name, args = unpickler.load() - except (cPickle.UnpicklingError, IndexError), msg: + except (cPickle.UnpicklingError, IndexError), err_msg: + log("can't decode %s" % repr(msg), level=zLOG.ERROR) raise DecodingError(msg) return msgid, flags, name, args @@ -104,18 +108,26 @@ self.send_reply(self.msgid, obj) class DebugLock: + + __locks = 0 + def __init__(self): self.lock = thread.allocate_lock() + # XXX this actually needs to be locked too + self.__lock_id = self.__locks + self.__locks += 1 def _debug(self): method = sys._getframe().f_back caller = method.f_back filename = os.path.split(caller.f_code.co_filename)[1] - log("LOCK %s: %s called by %s, %s, line %s" % (id(self.lock), - method.f_code.co_name, - caller.f_code.co_name, - filename, - caller.f_lineno)) + log("LOCK %s (tid %s): " \ + "%s called by %s, %s, line %s" % (self.__lock_id, + thread.get_ident(), + method.f_code.co_name, + caller.f_code.co_name, + filename, + caller.f_lineno)) def acquire(self, wait=None): self._debug() @@ -127,6 +139,10 @@ def release(self): self._debug() return self.lock.release() + + def locked(self): + self._debug() + return self.lock.locked() class Connection(smac.SizedMessageAsyncConnection): """Dispatcher for RPC on object @@ -153,12 +169,11 @@ self.async = 0 # The reply lock is used to block when a synchronous call is # waiting for a response - self.__reply_lock = thread.allocate_lock() - self.__reply_lock.acquire() - # The async mode lock is used to prevent a race between the - # write of self.async by set_async() and its use in _do_io() - self.__async_mode_lock = thread.allocate_lock() self.__super_init(sock, addr) + self._prepare_async() + self.__reply_lock = DebugLock() +## self.__reply_lock = thread.allocate_lock() + self.__reply_lock.acquire() if isinstance(obj, Handler): self.set_caller = 1 else: @@ -195,7 +210,7 @@ self.__reply_lock.release() # will fail if lock is unlocked def handle_request(self, msgid, flags, name, args): - log("%s%s" % (name, repr(args)[:40]), zLOG.BLATHER) + log("%s%s" % (name, repr(args)[:40]), zLOG.TRACE) if not self.check_method(name): raise ZRPCError("Invalid method name: %s" % name) @@ -221,7 +236,7 @@ if ret is not None: raise ZRPCError("async method returned value") else: - log("%s reply %s" % (name, repr(ret)[:40]), zLOG.BLATHER) + log("%s reply %s" % (name, repr(ret)[:40]), zLOG.TRACE) if isinstance(ret, Delay): ret.set_sender(msgid, self.send_reply) else: @@ -229,6 +244,7 @@ def handle_error(self): t, v, tb = sys.exc_info() + print t, v traceback.print_tb(tb) def check_method(self, name): @@ -287,19 +303,24 @@ # The next two methods are used by clients to invoke methods on # remote objects - # XXX these two methods should raise an nice exception if there - # are called after the connection is closed + # XXX Should revise design to allow multiple outstanding + # synchronous calls def call(self, method, *args): + if self.closed: + raise DisconnectedError("This action is temporarily unavailable") msgid = self.msgid self.msgid += 1 log("call %s %s" % (msgid, method)) self.message_output(self.marshal.encode(msgid, 0, method, args)) self.__reply = None + # lock is currently held self._do_io(wait=1) + # lock is held again... r_msgid, r_flags, r_args = self.__reply self.__reply_lock.acquire() + log("call acquired lock") assert r_msgid == msgid, "%s != %s: %s" % (r_msgid, msgid, r_args) if type(r_args) == types.TupleType \ @@ -311,6 +332,8 @@ return r_args def callAsync(self, method, *args): + if self.closed: + raise DisconnectedError("This action is temporarily unavailable") msgid = self.msgid self.msgid += 1 log("async %s %s" % (msgid, method)) @@ -319,16 +342,33 @@ # handle IO, possibly in async mode + def _prepare_async(self): + self._async = 0 + ThreadedAsync.register_loop_callback(self.set_async) + # XXX If we are not in async mode, this will cause dead + # Connections to be leaked. + + def set_async(self, map): + # XXX do we need a lock around this? I'm not sure there is + # any harm to a race with _do_io(). + self._async = 1 + self.trigger = trigger.trigger() + + def is_async(self): + return self._async + def _do_io(self, wait=0): # XXX need better name # XXX invariant? lock must be held when calling with wait==1 # otherwise, in non-async mode, there will be no poll - log("_do_io(wait=%d), async=%d" % (wait, self.async), - level=zLOG.BLATHER) - if self.async: + log("_do_io(wait=%d), async=%d" % (wait, self.is_async()), + level=zLOG.TRACE) + if self.is_async(): self.trigger.pull_trigger() if wait: self.__reply_lock.acquire() + # wait until reply... + self.__reply_lock.release() else: if wait: # do loop only if lock is already acquired @@ -338,7 +378,12 @@ else: asyncore.poll(0.0, self) + # XXX it seems that we need to release before returning if + # called with wait==1. perhaps the caller need not acquire + # upon return... + class ServerConnection(Connection): + # XXX this is a hack def _do_io(self, wait=0): """If this is a server, there is no explicit IO to do""" pass @@ -346,6 +391,9 @@ class ConnectionManager: """Keeps a connection up over time""" + # XXX requires that obj implement notifyConnected and + # notifyDisconnected. make this optional? + def __init__(self, addr, obj=None, debug=1, tmin=None, tmax=None): self.addr = addr self.obj = obj @@ -354,29 +402,34 @@ self.debug = debug self.connected = 0 self._thread = None - self._callback = None + self._connect_lock = threading.Lock() + self.trigger = None + self.async = 0 ThreadedAsync.register_loop_callback(self.set_async) def register_object(self, obj): self.obj = obj - def set_async(self): + def set_async(self, map): + # XXX need each connection started with async==0 to have a callback self.async = 1 # XXX needs to be set on the Connection self.trigger = trigger.trigger() - def connect(self, sync=0, callback=None): + def connect(self, sync=0): if self.connected == 1: return - self._callback = callback - # XXX probably need lock around self._thread - if self._thread is None: - self._thread = threading.Thread(target=self.__connect, args=(1,)) - self._thread.start() + self._connect_lock.acquire() + try: + if self._thread is None: + self._thread = threading.Thread(target=self.__connect, + args=(1,)) + self._thread.start() + finally: + self._connect_lock.release() if sync: self._thread.join() - def attempt_connect(self, callback=None): - self._callback = callback + def attempt_connect(self): self.__connect(repeat=0) return self.connected @@ -402,12 +455,9 @@ log("Connected to server", level=zLOG.DEBUG) self.connected = 1 if self.connected: - # XXX how do we get here with s being defined? c = ManagedConnection(s, self.addr, self.obj, self) log("Connection created: %s" % c) - log("callback = %s" % self._callback) - if self._callback: - self._callback(c) + self.obj.notifyConnected(c) self.__thread = None def _wait(self, t): @@ -419,7 +469,7 @@ def closed(self, conn): self.connected = 0 - # perhaps call ClientStorage(XXX) and tell it we closed + self.obj.notifyDisconnected(None) self.connect() class ManagedServerConnection(ServerConnection): @@ -436,13 +486,35 @@ self.__mgr.closed(self) class ManagedConnection(Connection): - """A connection that notifies its ConnectionManager of closing""" + """A connection that notifies its ConnectionManager of closing. + + A managed connection also defers the ThreadedAsync work to its + manager. + """ __super_init = Connection.__init__ __super_close = Connection.close def __init__(self, sock, addr, obj, mgr, pickle=None): self.__mgr = mgr + if self.__mgr.async: + self.__async = 1 + self.trigger = self.__mgr.trigger + else: + self.__async = None self.__super_init(sock, addr, obj, pickle) + + def _prepare_async(self): + # Don't do the register_loop_callback that the superclass does + pass + + def is_async(self): + if self.__async: + return 1 + async = self.__mgr.async + if async: + self.__async = 1 + self.trigger = self.__mgr.trigger + return async def close(self): self.__super_close() From jeremy at digicool.com Fri Mar 30 17:21:33 2001 From: jeremy at digicool.com (jeremy@digicool.com) Date: Sun Aug 10 16:31:14 2008 Subject: [ZEO-Checkins] CVS: Packages/ZEO - ServerStub.py:1.1.2.2 Message-ID: <20010330222133.0EF3C510E2@korak.digicool.com> Update of /cvs-repository/Packages/ZEO In directory korak:/tmp/cvs-serv472 Modified Files: Tag: ZEO-ZRPC-Dev ServerStub.py Log Message: endZeoVerify! --- Updated File ServerStub.py in package Packages/ZEO -- --- ServerStub.py 2001/03/17 00:14:51 1.1.2.1 +++ ServerStub.py 2001/03/30 22:21:31 1.1.2.2 @@ -21,7 +21,7 @@ self.rpc.callAsync('zeoVerify', oid, s, sv) def endZeoVerify(self): - self.rpc.callAsync('beginZeoVerify') + self.rpc.callAsync('endZeoVerify') def new_oids(self, n=None): if n is None: @@ -51,8 +51,8 @@ return self.rpc.call('vote', trans_id) def tpc_finish(self, id, user, descr, ext): -# self.rpc.callAsync('tpc_finish', id, user, descr, ext) - self.rpc.call('tpc_finish', id, user, descr, ext) +## self.rpc.callAsync('tpc_finish', id, user, descr, ext) + return self.rpc.call('tpc_finish', id, user, descr, ext) def tpc_abort(self, id): self.rpc.callAsync('tpc_abort', id) --- Updated File ServerStub.py in package Packages/ZEO -- --- ServerStub.py 2001/03/17 00:14:51 1.1.2.1 +++ ServerStub.py 2001/03/30 22:21:31 1.1.2.2 @@ -21,7 +21,7 @@ self.rpc.callAsync('zeoVerify', oid, s, sv) def endZeoVerify(self): - self.rpc.callAsync('beginZeoVerify') + self.rpc.callAsync('endZeoVerify') def new_oids(self, n=None): if n is None: @@ -51,8 +51,8 @@ return self.rpc.call('vote', trans_id) def tpc_finish(self, id, user, descr, ext): -# self.rpc.callAsync('tpc_finish', id, user, descr, ext) - self.rpc.call('tpc_finish', id, user, descr, ext) +## self.rpc.callAsync('tpc_finish', id, user, descr, ext) + return self.rpc.call('tpc_finish', id, user, descr, ext) def tpc_abort(self, id): self.rpc.callAsync('tpc_abort', id) From jeremy at digicool.com Fri Mar 30 17:55:02 2001 From: jeremy at digicool.com (jeremy@digicool.com) Date: Sun Aug 10 16:31:14 2008 Subject: [ZEO-Checkins] CVS: Packages/ZEO - ServerStub.py:1.1.2.3 Message-ID: <20010330225502.84453510E2@korak.digicool.com> Update of /cvs-repository/Packages/ZEO In directory korak:/tmp/cvs-serv5776 Modified Files: Tag: ZEO-ZRPC-Dev ServerStub.py Log Message: undo! --- Updated File ServerStub.py in package Packages/ZEO -- --- ServerStub.py 2001/03/30 22:21:31 1.1.2.2 +++ ServerStub.py 2001/03/30 22:55:01 1.1.2.3 @@ -88,7 +88,7 @@ return self.rpc.call('store', oid, serial, data, version, trans) def undo(self, trans_id): - return self.rpc.call('trans_id') + return self.rpc.call('undo', trans_id) def undoLog(self, first, last): # XXX filter not allowed across RPC --- Updated File ServerStub.py in package Packages/ZEO -- --- ServerStub.py 2001/03/30 22:21:31 1.1.2.2 +++ ServerStub.py 2001/03/30 22:55:01 1.1.2.3 @@ -88,7 +88,7 @@ return self.rpc.call('store', oid, serial, data, version, trans) def undo(self, trans_id): - return self.rpc.call('trans_id') + return self.rpc.call('undo', trans_id) def undoLog(self, first, last): # XXX filter not allowed across RPC From jeremy at digicool.com Fri Mar 30 17:55:24 2001 From: jeremy at digicool.com (jeremy@digicool.com) Date: Sun Aug 10 16:31:14 2008 Subject: [ZEO-Checkins] CVS: Packages/ZEO - StorageServer.py:1.21.4.3 Message-ID: <20010330225524.7A826510E2@korak.digicool.com> Update of /cvs-repository/Packages/ZEO In directory korak:/tmp/cvs-serv5839 Modified Files: Tag: ZEO-ZRPC-Dev StorageServer.py Log Message: Delegate several calls to the storage --- Updated File StorageServer.py in package Packages/ZEO -- --- StorageServer.py 2001/03/29 13:35:17 1.21.4.2 +++ StorageServer.py 2001/03/30 22:55:22 1.21.4.3 @@ -87,9 +87,20 @@ self.__storage_id = storage_id self.__storage = storage + self.setup_delegation() self.server.register(storage_id, self) log("registered storage %s: %s" % (storage_id, storage)) + def setup_delegation(self): + """Delegate several methods to the storage""" + self.undoInfo = self.__storage.undoInfo + self.undoLog = self.__storage.undoLog + self.versionEmpty = self.__storage.versionEmpty + self.versions = self.__storage.versions + self.history = self.__storage.history + self.load = self.__storage.load + self.loadSerial = self.__storage.loadSerial + def get_info(self): return { 'length': len(self.__storage), @@ -319,6 +330,7 @@ if n < 0: n = 1 return map(lambda x, self=self: self.__storage.new_oid(), range(n)) + def fixup_storage(storage): # backwards compatibility hack --- Updated File StorageServer.py in package Packages/ZEO -- --- StorageServer.py 2001/03/29 13:35:17 1.21.4.2 +++ StorageServer.py 2001/03/30 22:55:22 1.21.4.3 @@ -87,9 +87,20 @@ self.__storage_id = storage_id self.__storage = storage + self.setup_delegation() self.server.register(storage_id, self) log("registered storage %s: %s" % (storage_id, storage)) + def setup_delegation(self): + """Delegate several methods to the storage""" + self.undoInfo = self.__storage.undoInfo + self.undoLog = self.__storage.undoLog + self.versionEmpty = self.__storage.versionEmpty + self.versions = self.__storage.versions + self.history = self.__storage.history + self.load = self.__storage.load + self.loadSerial = self.__storage.loadSerial + def get_info(self): return { 'length': len(self.__storage), @@ -319,6 +330,7 @@ if n < 0: n = 1 return map(lambda x, self=self: self.__storage.new_oid(), range(n)) + def fixup_storage(storage): # backwards compatibility hack